modules/dnn/src/layers/convolution_layer.cpp

   1 /*M///////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
   4 //
   5 //  By downloading, copying, installing or using the software you agree to this license.
   6 //  If you do not agree to this license, do not download, install,
   7 //  copy or use the software.
   8 //
   9 //
  10 //                           License Agreement
  11 //                For Open Source Computer Vision Library
  12 //
  13 // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14 // Copyright (C) 2017, Intel Corporation, all rights reserved.
  15 // Third party copyrights are property of their respective owners.
  16 //
  17 // Redistribution and use in source and binary forms, with or without modification,
  18 // are permitted provided that the following conditions are met:
  19 //
  20 //   * Redistribution's of source code must retain the above copyright notice,
  21 //     this list of conditions and the following disclaimer.
  22 //
  23 //   * Redistribution's in binary form must reproduce the above copyright notice,
  24 //     this list of conditions and the following disclaimer in the documentation
  25 //     and/or other materials provided with the distribution.
  26 //
  27 //   * The name of the copyright holders may not be used to endorse or promote products
  28 //     derived from this software without specific prior written permission.
  29 //
  30 // This software is provided by the copyright holders and contributors "as is" and
  31 // any express or implied warranties, including, but not limited to, the implied
  32 // warranties of merchantability and fitness for a particular purpose are disclaimed.
  33 // In no event shall the Intel Corporation or contributors be liable for any direct,
  34 // indirect, incidental, special, exemplary, or consequential damages
  35 // (including, but not limited to, procurement of substitute goods or services;
  36 // loss of use, data, or profits; or business interruption) however caused
  37 // and on any theory of liability, whether in contract, strict liability,
  38 // or tort (including negligence or otherwise) arising in any way out of
  39 // the use of this software, even if advised of the possibility of such damage.
  40 //
  41 //M*/
  42
  43 #include "../precomp.hpp"
  44 #include "layers_common.hpp"
  45 #include "../op_cuda.hpp"
  46 #include "../op_halide.hpp"
  47 #include "../op_inf_engine.hpp"
  48 #include "../op_vkcom.hpp"
  49 #include "opencv2/core/hal/hal.hpp"
  50 #include "opencv2/core/hal/intrin.hpp"
  51 #include <iostream>
  52 #include <numeric>
  53
  54 #ifdef HAVE_OPENCL
  55 #include "opencl_kernels_dnn.hpp"
  56 using namespace cv::dnn::ocl4dnn;
  57 #endif
  58
  59 #ifdef HAVE_CUDA
  60 #include "../cuda4dnn/primitives/convolution.hpp"
  61 #include "../cuda4dnn/primitives/transpose_convolution.hpp"
  62 using namespace cv::dnn::cuda4dnn;
  63 #endif
  64
  65 namespace cv
  66 {
  67 namespace dnn
  68 {
  69
  70 class BaseConvolutionLayerImpl : public ConvolutionLayer
  71 {
  72 public:
  73     bool fusedWeights, fusedBias;
  74     std::vector<double> weightsMultipliers;
  75     BaseConvolutionLayerImpl(const LayerParams &params)
  76     {
  77         setParamsFrom(params);
  78         getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads);
  79
  80         numOutput = params.get<int>("num_output");
  81         int ngroups = params.get<int>("group", 1);
  82         CV_Assert(numOutput % ngroups == 0);
  83
  84         if (kernel_size.size() == 2) {
  85             kernel = Size(kernel_size[1], kernel_size[0]);
  86             stride = Size(strides[1], strides[0]);
  87             for (int i = 0; i < pads_begin.size(); i++) {
  88                 if (pads_begin[i] != pads_end[i])
  89                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer");
  90             }
  91             pad = Size(pads_begin[1], pads_begin[0]);
  92             dilation = Size(dilations[1], dilations[0]);
  93
  94             adjustPad.height = adjust_pads[0];
  95             adjustPad.width = adjust_pads[1];
  96         }
  97
  98         for (int i = 0; i < adjust_pads.size(); i++) {
  99             CV_Assert(adjust_pads[i] < strides[i]);
 100         }
 101
 102         fusedWeights = false;
 103         fusedBias = false;
 104     }
 105
 106     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
 107     {
 108         std::vector<Mat> inputs, outputs;
 109         inputs_arr.getMatVector(inputs);
 110         outputs_arr.getMatVector(outputs);
 111
 112         CV_Assert(inputs.size() > 0);
 113
 114         CV_Assert(blobs.size() == 1 || blobs.size() == 2);
 115         CV_Assert(inputs[0].dims == outputs[0].dims);
 116         CV_Assert(blobs[0].dims == kernel_size.size() + 2);
 117         for (int i = 0; i < kernel_size.size(); i++) {
 118             CV_Assert(blobs[0].size[i + 2] == kernel_size[i]);
 119         }
 120
 121         const Mat &input = inputs[0];
 122         CV_Assert((input.dims == 4 || input.dims == 5) && (input.type() == CV_32F || input.type() == CV_16S));
 123         for (size_t i = 0; i < inputs.size(); i++)
 124         {
 125             CV_Assert(inputs[i].type() == input.type());
 126             CV_Assert((inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]);
 127             for (int j = 0; j < inputs[i].dims; j++) {
 128                 CV_Assert(inputs[i].size[j] == input.size[j]);
 129             }
 130         }
 131
 132         std::vector<int> inpShape;
 133         std::vector<int> outShape;
 134         for (int i = 2; i < inputs[0].dims; i++) {
 135             inpShape.push_back(inputs[0].size[i]);
 136             outShape.push_back(outputs[0].size[i]);
 137         }
 138         getConvPoolPaddings(inpShape, kernel_size, strides, padMode, pads_begin, pads_end);
 139         if (pads_begin.size() == 2) {
 140             for (int i = 0; i < pads_begin.size(); i++) {
 141                 if (pads_begin[i] != pads_end[i])
 142                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer");
 143             }
 144             pad = Size(pads_begin[1], pads_begin[0]);
 145         }
 146         fusedWeights = false;
 147         fusedBias = false;
 148     }
 149
 150     bool hasBias() const
 151     {
 152         return blobs.size() >= 2;
 153     }
 154
 155     virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0;
 156     bool is1x1() const
 157     {
 158         return (kernel.height == 1 && kernel.width == 1) &&
 159                (stride.height == 1 && stride.width == 1) &&
 160                (dilation.height == 1 && dilation.width == 1);
 161     }
 162
 163     virtual bool tryFuse(Ptr<Layer>& top) CV_OVERRIDE
 164     {
 165         Mat w, b;
 166         top->getScaleShift(w, b);
 167         if (!w.empty() || !b.empty())
 168         {
 169             fuseWeights(w, b);
 170             fusedWeights = fusedWeights || !w.empty();
 171             fusedBias = fusedBias || (hasBias() && !w.empty()) || !b.empty();
 172             return true;
 173         }
 174         return false;
 175     }
 176
 177     virtual void fuseWeights(const Mat& w_, const Mat& b_) = 0;
 178
 179     virtual void applyHalideScheduler(Ptr<BackendNode>& node,
 180                                       const std::vector<Mat*> &inputs,
 181                                       const std::vector<Mat> &outputs,
 182                                       int targetId) const CV_OVERRIDE
 183     {
 184 #ifdef HAVE_HALIDE
 185         if (targetId != DNN_TARGET_CPU)
 186         {
 187             Layer::applyHalideScheduler(node, inputs, outputs, targetId);
 188             return;
 189         }
 190         Halide::Var x("x"), y("y"), c("c"), n("n"), tile("tile"), yi("yi"), yo("yo"), co("co"), ci("ci");
 191         Halide::Func& top = node.dynamicCast<HalideBackendNode>()->funcs[1];
 192         Halide::Func& padded_input = node.dynamicCast<HalideBackendNode>()->funcs[0];
 193
 194         int outW, outH, outC, outN;
 195         getCanonicalSize(outputs[0].size, &outW, &outH, &outC, &outN);
 196
 197         if (outW == 1 || outH <= 2)
 198             return;
 199
 200         if (is1x1() || outC <= 16)
 201             top.reorder(x, c, y)
 202                .split(y, yo, yi, 2)
 203                .fuse(yo, n, tile)
 204                .parallel(tile)
 205                .unroll(yi)
 206                .vectorize(x, outW >= 16 ? 16 : outW);
 207         else
 208             top.reorder(x, c, y)
 209                .split(y, yo, yi, 2)
 210                .split(c, co, ci, 16)
 211                .fuse(yo, co, tile).fuse(n, tile, tile)
 212                .parallel(tile)
 213                .unroll(yi)
 214                .vectorize(x, outW >= 16 ? 16 : outW);
 215         padded_input.compute_at(top, yi);
 216 #endif  // HAVE_HALIDE
 217     }
 218 };
 219
 220
 221 #define IS_POWER_LAYER(layer) \
 222             (!layer.empty() && !layer->type.compare("Power"))
 223 //TODO: simultaneously convolution and bias addition for cache optimization
 224 class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
 225 {
 226 public:
 227     enum { VEC_ALIGN = 8, DFT_TYPE = CV_32F };
 228     Mat weightsMat;
 229     std::vector<float> biasvec;
 230     std::vector<float> reluslope;
 231     Ptr<ActivationLayer> activ;
 232
 233 #ifdef HAVE_OPENCL
 234     Ptr<OCL4DNNConvSpatial<float> > convolutionOp;
 235     std::vector<UMat> umat_blobs;
 236     bool newActiv;
 237     ocl4dnnFusedActiv_t activType;
 238     float power;
 239 #endif
 240     ConvolutionLayerImpl(const LayerParams &params) : BaseConvolutionLayerImpl(params)
 241     {
 242 #ifdef HAVE_OPENCL
 243         newActiv = false;
 244         activType = OCL4DNN_CONV_FUSED_ACTIV_NONE;
 245         power = 0.f;
 246 #endif
 247     }
 248
 249     MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE
 250     {
 251         int dims = inpShape.size();
 252         int inpD = dims == 5 ? inpShape[2] : 1;
 253         int inpH = inpShape[dims - 2];
 254         int inpW = inpShape.back();
 255         int inpGroupCn = blobs[0].size[1];
 256         int ksize = inpGroupCn * std::accumulate(kernel_size.begin(), kernel_size.end(),
 257                                                  1, std::multiplies<size_t>());
 258         return shape(inpD * inpH * inpW, ksize);
 259     }
 260
 261     virtual bool supportBackend(int backendId) CV_OVERRIDE
 262     {
 263         if (backendId == DNN_BACKEND_CUDA)
 264         {
 265             /* only convolution 2d and 3d supported */
 266             if(kernel_size.size() == 2 || kernel_size.size() == 3)
 267                 return true;
 268
 269             return false;
 270         }
 271
 272 #ifdef HAVE_INF_ENGINE
 273         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
 274         {
 275             if (kernel_size.size() == 3)
 276                 return preferableTarget == DNN_TARGET_CPU;
 277             return (preferableTarget != DNN_TARGET_MYRIAD || dilation.width == dilation.height);
 278         }
 279         else
 280 #endif
 281         {
 282             if (kernel_size.size() == 3)
 283                 return (preferableTarget == DNN_TARGET_CPU && backendId == DNN_BACKEND_OPENCV);
 284             else if (kernel_size.size() == 2)
 285                 return backendId == DNN_BACKEND_OPENCV ||
 286                        backendId == DNN_BACKEND_HALIDE ||
 287                        (backendId == DNN_BACKEND_VKCOM && haveVulkan());
 288             else
 289                 return false;
 290         }
 291     }
 292
 293     bool getMemoryShapes(const std::vector<MatShape> &inputs,
 294                          const int requiredOutputs,
 295                          std::vector<MatShape> &outputs,
 296                          std::vector<MatShape> &internals) const CV_OVERRIDE
 297     {
 298         CV_Assert(blobs.size() != 0);
 299         CV_Assert(!hasBias() || blobs[1].total() == (size_t)blobs[0].size[0]);
 300         CV_Assert(inputs.size() == (size_t)1);
 301
 302         internals.clear();
 303
 304         CV_Assert(inputs.size() != 0);
 305         std::vector<int> inpShape(inputs[0].begin() + 2, inputs[0].end());
 306
 307         int outCn = blobs[0].size[0];
 308         std::vector<int> outShape;
 309         outShape.push_back(inputs[0][0]);
 310         outShape.push_back(outCn);
 311
 312         int inpCn = inputs[0][1];
 313         if (padMode.empty())
 314         {
 315             for (int i = 0; i < inpShape.size(); i++)
 316                 outShape.push_back((inpShape[i] + pads_begin[i] + pads_end[i] - dilations[i] * (kernel_size[i] - 1) - 1) / strides[i] + 1);
 317         }
 318         else
 319         {
 320             getConvPoolOutParams(inpShape, kernel_size, strides, padMode, dilations, outShape);
 321         }
 322
 323         int ngroups = inpCn / blobs[0].size[1];
 324         if (ngroups == 0 || ngroups * blobs[0].size[1] != inpCn)
 325             CV_Error(Error::StsError, format("Number of input channels should "
 326                      "be multiple of %d but got %d", blobs[0].size[1], inpCn));
 327         CV_Assert(ngroups > 0 && inpCn % ngroups == 0 && outCn % ngroups == 0);
 328
 329         outputs.resize(1, outShape);
 330
 331         return false;
 332     }
 333
 334     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
 335     {
 336         BaseConvolutionLayerImpl::finalize(inputs_arr, outputs_arr);
 337
 338         CV_Assert(!blobs.empty());
 339         const int outCn = blobs[0].size[0];
 340         // prepare weightsMat where each row is aligned and has enough zero padding on the right to
 341         // use vectorized (i.e. with intrinsics) loops without tail processing
 342         Mat wm = blobs[0].reshape(1, outCn);
 343         if( wm.step1() % VEC_ALIGN != 0 )
 344         {
 345             int newcols = (int)alignSize(wm.step1(), VEC_ALIGN);
 346             Mat wm_buffer = Mat(outCn, newcols, wm.type());
 347             Mat wm_padding = wm_buffer.colRange(wm.cols, newcols);
 348             wm_padding.setTo(Scalar::all(0.));
 349             Mat wm_aligned = wm_buffer.colRange(0, wm.cols);
 350             wm.copyTo(wm_aligned);
 351             wm = wm_aligned;
 352         }
 353         weightsMat = wm;
 354         weightsMultipliers.assign(outCn, 1.0);
 355
 356         Mat biasMat = hasBias() ? blobs[1].reshape(1, outCn) : Mat();
 357         biasvec.resize(outCn+2);
 358         if( biasMat.empty() )
 359         {
 360             for(int i = 0; i < outCn; i++ )
 361                 biasvec[i] = 0.f;
 362         }
 363         else
 364         {
 365             for(int i = 0; i < outCn; i++ )
 366                 biasvec[i] = biasMat.at<float>(i);
 367         }
 368 #ifdef HAVE_OPENCL
 369         convolutionOp.release();
 370 #endif
 371     }
 372
 373     bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
 374     {
 375         if (!activ.empty() && !layer.empty())
 376             return false;
 377
 378         activ = layer;
 379         if (activ.empty())
 380             reluslope.clear();
 381 #ifdef HAVE_OPENCL
 382         newActiv = true;
 383         activType = OCL4DNN_CONV_FUSED_ACTIV_NONE;
 384
 385         if (IS_DNN_OPENCL_TARGET(preferableTarget))
 386         {
 387             Ptr<PowerLayer> activ_power = activ.dynamicCast<PowerLayer>();
 388             if (!activ_power.empty())
 389             {
 390                 if (activ_power->scale != 1.f || activ_power->shift != 0.f)
 391                 {
 392                     const int outCh = blobs[0].size[0];
 393                     fuseWeights(Mat(1, outCh, CV_32F, Scalar(activ_power->scale)),
 394                                 Mat(1, outCh, CV_32F, Scalar(activ_power->shift)));
 395                 }
 396
 397                 power = activ_power->power;
 398                 activType = OCL4DNN_CONV_FUSED_ACTIV_POWER;
 399             }
 400             Ptr<TanHLayer> activ_tanh = activ.dynamicCast<TanHLayer>();
 401             if (!activ_tanh.empty())
 402             {
 403                 activType = OCL4DNN_CONV_FUSED_ACTIV_TANH;
 404             }
 405         }
 406 #endif
 407         return !activ.empty();
 408     }
 409
 410     void fuseWeights(const Mat& w_, const Mat& b_) CV_OVERRIDE
 411     {
 412         // Convolution weights have OIHW data layout. Parameters fusion in case of
 413         // (conv(I) + b1 ) * w + b2
 414         // means to replace convolution's weights to [w*conv(I)] and bias to [b1 * w + b2]
 415         const int outCn = weightsMat.size[0];
 416         Mat w = w_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(w_.at<float>(0))) : w_;
 417         Mat b = b_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(b_.at<float>(0))) : b_;
 418         CV_Assert_N(!weightsMat.empty(), biasvec.size() == outCn + 2,
 419                     w.empty() || outCn == w.total(), b.empty() || outCn == b.total());
 420
 421         if (!w.empty())
 422         {
 423             // Keep origin weights unchanged.
 424             if (weightsMat.data == blobs[0].data)
 425                 weightsMat = weightsMat.clone();
 426
 427             Mat originWeights = blobs[0].reshape(1, outCn);
 428             for (int i = 0; i < outCn; ++i)
 429             {
 430                 double wi = w.at<float>(i);
 431                 weightsMultipliers[i] *= wi;
 432                 cv::multiply(originWeights.row(i), weightsMultipliers[i], weightsMat.row(i));
 433                 biasvec[i] *= wi;
 434             }
 435         }
 436
 437         if (!b.empty())
 438         {
 439             for (int i = 0; i < outCn; ++i)
 440                 biasvec[i] += b.at<float>(i);
 441         }
 442         biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1];
 443     }
 444
 445     virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 446     {
 447 #ifdef HAVE_VULKAN
 448         int out_channel = blobs[0].size[0];
 449         bool has_bias = hasBias() || fusedBias;
 450         int filter_size[2] = {kernel.height, kernel.width};
 451         int pad_size[2] = {pad.height, pad.width};
 452         int stride_size[2] = {stride.height, stride.width};
 453         int dilation_size[2] = {dilation.height, dilation.width};
 454         int activation = 0;
 455         vkcom::Tensor input_tensor = VkComTensor(inputs[0]);
 456         int in_channel = input_tensor.dimSize(1);
 457         int group = in_channel / blobs[0].size[1];
 458
 459         // TODO: support group > 1
 460         if (group != 1)
 461             return Ptr<BackendNode>();
 462
 463         int padding_mode;
 464         if (padMode.empty())
 465         {
 466             padding_mode = vkcom::kPaddingModeCaffe;
 467         }
 468         else if (padMode == "VALID")
 469         {
 470             padding_mode = vkcom::kPaddingModeValid;
 471         }
 472         else if (padMode == "SAME")
 473         {
 474             padding_mode = vkcom::kPaddingModeSame;
 475         }
 476         else
 477             CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
 478
 479         std::shared_ptr<vkcom::OpBase> op(new vkcom::OpConv(out_channel, has_bias,
 480                     filter_size, pad_size,
 481                     stride_size, dilation_size,
 482                     activation, group,
 483                     padding_mode));
 484
 485         std::vector<Ptr<BackendWrapper> > blobsWrapper;
 486
 487         if (fusedWeights)
 488         {
 489             Mat wm;
 490             weightsMat.copyTo(wm); // to handle the case of isContinuous() == false
 491             wm = wm.reshape(1, blobs[0].dims, blobs[0].size);
 492             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(wm)));
 493         }
 494         else
 495         {
 496             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(blobs[0])));
 497         }
 498
 499         if (has_bias)
 500         {
 501             Mat biasesMat({out_channel}, CV_32F, &biasvec[0]);
 502             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(biasesMat)));
 503         }
 504
 505         return Ptr<BackendNode>(new VkComBackendNode(inputs, op, blobsWrapper));
 506 #endif  // HAVE_VULKAN
 507         return Ptr<BackendNode>();
 508     }
 509
 510     virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 511     {
 512 #ifdef HAVE_HALIDE
 513         Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
 514
 515         const int inpCn = inputBuffer.channels();
 516         const int outCn = blobs[0].size[0];
 517         const int inpGroupCn = blobs[0].size[1];
 518         const int group = inpCn / inpGroupCn;
 519         const int outGroupCn = outCn / group;
 520
 521         Halide::Buffer<float> weights = wrapToHalideBuffer(blobs[0]);
 522
 523         Halide::Var x("x"), y("y"), c("c"), n("n");
 524         Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
 525         Halide::Func padded_input(name + "_constant_exterior");
 526         if (pad.width || pad.height)
 527         {
 528             Halide::Func bounded =
 529                 Halide::BoundaryConditions::constant_exterior(inputBuffer, 0);
 530             padded_input(x, y, c, n) = bounded(x, y, c, n);
 531         }
 532         else
 533         {
 534             padded_input(x, y, c, n) = inputBuffer(x, y, c, n);
 535         }
 536
 537         Halide::RDom r(0, kernel.width, 0, kernel.height, 0, inpGroupCn);
 538         Halide::Expr kx = x * stride.width - pad.width + r.x * dilation.width;
 539         Halide::Expr ky = y * stride.height - pad.height + r.y * dilation.height;
 540         Halide::Expr kc = r.z;
 541         for (int i = 1; i < group; ++i)
 542         {
 543             kc = select(c < outGroupCn * i, kc, inpGroupCn * i + r.z);
 544         }
 545         Halide::Expr topExpr = sum(padded_input(kx, ky, kc, n) *
 546                                    weights(r.x, r.y, r.z, c));
 547         if (hasBias())
 548         {
 549             Halide::Buffer<float> bias = wrapToHalideBuffer(blobs[1], {outCn});
 550             topExpr += bias(c);
 551         }
 552         top(x, y, c, n) = topExpr;
 553         return Ptr<BackendNode>(new HalideBackendNode({ padded_input, top }));
 554 #endif  // HAVE_HALIDE
 555         return Ptr<BackendNode>();
 556     }
 557
 558 #ifdef HAVE_INF_ENGINE
 559     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 560     {
 561         InferenceEngine::DataPtr input = infEngineDataNode(inputs[0]);
 562         std::vector<size_t> dims = input->getDims();
 563         CV_Assert(dims.size() == 4 || dims.size() == 5);
 564         const int inpCn = dims[1];
 565         const int outCn = blobs[0].size[0];
 566         const int inpGroupCn = blobs[0].size[1];
 567         const int group = inpCn / inpGroupCn;
 568         InferenceEngine::Layout layout = (dims.size() == 4) ? InferenceEngine::Layout::OIHW :
 569                                                               InferenceEngine::Layout::NCDHW;
 570
 571         auto ieWeights = wrapToInfEngineBlob(blobs[0], layout);
 572         if (fusedWeights)
 573         {
 574             if (weightsMat.isContinuous())
 575             {
 576                 Mat cvWeights = weightsMat.reshape(1, blobs[0].dims, blobs[0].size);
 577                 ieWeights = wrapToInfEngineBlob(cvWeights, layout);
 578             }
 579             else
 580             {
 581                 ieWeights = InferenceEngine::make_shared_blob<float>({
 582                                 InferenceEngine::Precision::FP32,
 583                                 ieWeights->getTensorDesc().getDims(), layout
 584                             });
 585                 ieWeights->allocate();
 586
 587                 Mat newWeights = infEngineBlobToMat(ieWeights).reshape(1, outCn);
 588                 Mat cvWeights = weightsMat.colRange(0, newWeights.cols);
 589                 cvWeights.copyTo(newWeights);
 590             }
 591         }
 592         InferenceEngine::Blob::Ptr ieBiases;
 593         if (hasBias() || fusedBias)
 594         {
 595             Mat biasesMat({outCn}, CV_32F, &biasvec[0]);
 596             ieBiases = wrapToInfEngineBlob(biasesMat, {(size_t)outCn}, InferenceEngine::Layout::C);
 597         }
 598
 599         InferenceEngine::Builder::ConvolutionLayer ieLayer(name);
 600
 601         ieLayer.setKernel(kernel_size);
 602         ieLayer.setStrides(strides);
 603         ieLayer.setDilation(dilations);
 604         ieLayer.setPaddingsBegin(pads_begin);
 605         ieLayer.setPaddingsEnd(pads_end);
 606         ieLayer.setGroup((size_t)group);
 607         ieLayer.setOutDepth((size_t)outCn);
 608
 609         InferenceEngine::Builder::Layer l = ieLayer;
 610         addConstantData("weights", ieWeights, l);
 611         if (ieBiases)
 612             addConstantData("biases", ieBiases, l);
 613
 614         if (!padMode.empty())
 615             l.getParameters()["auto_pad"] = padMode == "VALID" ? std::string("valid") : std::string("same_upper");
 616
 617         return Ptr<BackendNode>(new InfEngineBackendNode(l));
 618     }
 619 #endif  // HAVE_INF_ENGINE
 620
 621     class ParallelConv : public cv::ParallelLoopBody
 622     {
 623     public:
 624         enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 };
 625
 626         const Mat* input_;
 627         const Mat* weights_;
 628         Mat* output_;
 629         int outShape[4]; // used only for conv2d
 630         std::vector<size_t> kernel_size, pads_begin, pads_end, strides, dilations;
 631         int ngroups_, nstripes_;
 632         std::vector<int> ofstab_;
 633         const std::vector<float>* biasvec_;
 634         const std::vector<float>* reluslope_;
 635         const ActivationLayer* activ_;
 636         bool is1x1_;
 637         bool useAVX;
 638         bool useAVX2;
 639         bool useAVX512;
 640
 641         ParallelConv()
 642             : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
 643               biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false)
 644         {}
 645
 646         static void run( const Mat& input, Mat& output, const Mat& weights,
 647                          const std::vector<float>& biasvec,
 648                          const std::vector<float>& reluslope,
 649                          const std::vector<size_t>& kernel_size, const std::vector<size_t>& strides,
 650                          const std::vector<size_t>& pads_begin, const std::vector<size_t>& pads_end,
 651                          const std::vector<size_t>& dilations,
 652                          const ActivationLayer* activ, int ngroups, int nstripes )
 653         {
 654             size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(),
 655                                            1, std::multiplies<size_t>());
 656             CV_Assert_N(
 657                        (input.dims == 4 || input.dims == 5) && (input.dims == output.dims),
 658                        input.size[0] == output.size[0],
 659                        weights.rows == output.size[1],
 660                        weights.cols == (input.size[1]/ngroups)*karea,
 661                        input.type() == output.type(),
 662                        input.type() == weights.type(),
 663                        input.type() == CV_32FC1,
 664                        input.isContinuous(),
 665                        output.isContinuous(),
 666                        biasvec.size() == (size_t)output.size[1]+2);
 667             ParallelConv p;
 668
 669             p.input_ = &input;
 670             p.weights_ = &weights;
 671             p.output_ = &output;
 672             for( int i = 0; i < 4; i++ ) p.outShape[i] = output.size[i];
 673             p.outShape[1] /= ngroups;
 674
 675             p.kernel_size = kernel_size; p.strides = strides; p.dilations = dilations;
 676             p.pads_begin = pads_begin; p.pads_end = pads_end;
 677
 678             p.ngroups_ = ngroups;
 679             p.nstripes_ = nstripes;
 680
 681             int inpCnAll = input.size[1];
 682             int depth = (input.dims == 5) ? input.size[2] : 1;
 683             int width = input.size[input.dims - 1];
 684             int height = input.size[input.dims - 2];
 685             int inpCn = inpCnAll / ngroups;
 686
 687             bool isConv2D = kernel_size.size() == 2;
 688
 689             p.is1x1_ = isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 &&
 690                        pads_begin[0] == 0  && pads_begin[1] == 0;
 691
 692             p.useAVX    = checkHardwareSupport(CPU_AVX)  && isConv2D;
 693             p.useAVX2   = checkHardwareSupport(CPU_AVX2) && isConv2D;
 694             p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX  && isConv2D;
 695
 696             int ncn = std::min(inpCn, (int)BLK_SIZE_CN);
 697
 698             int kernel_d = !isConv2D? kernel_size[0] : 1;
 699             int kernel_h = kernel_size[kernel_size.size() - 2];
 700             int kernel_w = kernel_size.back();
 701
 702             int dil_d = !isConv2D? dilations[0] : 1;
 703             int dil_h = dilations[dilations.size() - 2];
 704             int dil_w = dilations.back();
 705
 706             p.ofstab_.resize(karea * ncn);
 707             int* ofstab = &p.ofstab_[0];
 708
 709             if (isConv2D)
 710             {
 711                 for( int k = 0; k < ncn; k++ )
 712                     for( int k_r = 0; k_r < kernel_h; k_r++ )
 713                         for( int k_c = 0; k_c < kernel_w; k_c++ )
 714                             ofstab[(k*kernel_h + k_r)*kernel_w + k_c] =
 715                                    (k*height + k_r*dil_h)*width + k_c*dil_w;
 716             }
 717             else
 718             {
 719                 for( int k = 0; k < ncn; k++ )
 720                     for (int k_d = 0; k_d < kernel_d; k_d++)
 721                         for( int k_r = 0; k_r < kernel_h; k_r++ )
 722                             for( int k_c = 0; k_c < kernel_w; k_c++ )
 723                                 ofstab[(k*kernel_d*kernel_h + k_d*kernel_h + k_r)*kernel_w + k_c] =
 724                                        (k*depth*height + k_d*dil_d*height + k_r*dil_h)*width + k_c*dil_w;
 725             }
 726
 727             p.biasvec_ = &biasvec;
 728             p.reluslope_ = &reluslope;
 729             p.activ_ = p.reluslope_->empty() ? activ : 0;
 730
 731             parallel_for_(Range(0, nstripes), p, nstripes);
 732         }
 733
 734         virtual void operator ()(const Range &r0) const CV_OVERRIDE
 735         {
 736             const int valign = ConvolutionLayerImpl::VEC_ALIGN;
 737             int ngroups = ngroups_, batchSize = input_->size[0]*ngroups;
 738             bool isConv2D = input_->dims == 4;
 739
 740             int outW = output_->size[output_->dims - 1];
 741             int outH = output_->size[output_->dims - 2];
 742             int outCn = output_->size[1]/ngroups;
 743
 744             int depth = !isConv2D? input_->size[2] : 1;
 745             int height = input_->size[input_->dims - 2];
 746             int width = input_->size[input_->dims - 1];
 747             int inpCn = input_->size[1]/ngroups;
 748
 749             const int nstripes = nstripes_;
 750
 751             int kernel_d = !isConv2D? kernel_size[0] : 1;
 752             int kernel_h = kernel_size[kernel_size.size() - 2];
 753             int kernel_w = kernel_size.back();
 754             int karea = kernel_w*kernel_h*kernel_d;
 755
 756             int pad_d = !isConv2D? pads_begin[0] : 0;
 757             int pad_t = pads_begin[pads_begin.size() - 2];
 758             int pad_l = pads_begin.back();
 759
 760             int stride_d = !isConv2D? strides[0] : 0;
 761             int stride_h = strides[strides.size() - 2];
 762             int stride_w = strides.back();
 763
 764             int dilation_d = !isConv2D? dilations[0] : 1;
 765             int dilation_h = dilations[dilations.size() - 2];
 766             int dilation_w = dilations.back();
 767
 768             int i, j, k, d;
 769             size_t inpPlaneSize = input_->total(2);
 770             size_t outPlaneSize = output_->total(2);
 771             bool is1x1 = is1x1_;
 772
 773             int stripesPerSample;
 774             size_t stripeSize;
 775             Range r = r0;
 776
 777             if( nstripes >= batchSize*2 )
 778             {
 779                 stripesPerSample = nstripes/batchSize;
 780                 stripeSize = alignSize((outPlaneSize + stripesPerSample - 1)/stripesPerSample, valign);
 781                 stripeSize = std::min(stripeSize, outPlaneSize);
 782             }
 783             else
 784             {
 785                 stripesPerSample = 1;
 786                 int samplesPerStripe = std::max((batchSize + nstripes - 1)/nstripes, 1);
 787                 r.start *= samplesPerStripe;
 788                 r.end *= samplesPerStripe;
 789                 stripeSize = outPlaneSize;
 790             }
 791
 792             const float* data_inp0_ = input_->ptr<float>();
 793             const int* ofstab = &ofstab_[0];
 794             const float* wptr_orig_ = weights_->ptr<float>();
 795             size_t wstep = weights_->step1();
 796             const float* biasptr_ = &biasvec_->at(0);
 797             const float* reluptr_ = reluslope_->empty() ? 0 : &reluslope_->at(0);
 798             float* data_out0_ = output_->ptr<float>();
 799             size_t rowbufsz = (size_t)karea*BLK_SIZE_CN*BLK_SIZE;
 800             AutoBuffer<float> rowbuf0_(rowbufsz + valign);
 801             float* rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(float)));
 802
 803             // we clear the buffer once; ultimately, it lets us to avoid
 804             // tail processing after running the unrolled/vectorized loop.
 805             // the main idea is to make sure that the tail (a.k.a. padding) of each row
 806             // (i.e. the elements with indices between vsz=karea*ncn and vsz_a)
 807             // does not contain NaNs or Infs. Because the padding in the weights
 808             // matrix is explicitly initialized with 0's, we handle all other
 809             // cases nicely, i.e. we can skip expliciting re-initialization
 810             // of the padding - we just retain elements from the previous iteration
 811             // of the loop over channels (cn0).
 812             memset(rowbuf0, 0, rowbufsz*sizeof(rowbuf0[0]) );
 813
 814             for( int stripe = r.start; stripe < r.end; stripe++ )
 815             {
 816                 int subsampleIdx = stripe/stripesPerSample;
 817                 if( subsampleIdx >= batchSize )
 818                     break;
 819                 int stripeStart = (int)((stripe - subsampleIdx*stripesPerSample)*stripeSize);
 820                 int stripeEnd = (int)std::min(stripeStart + stripeSize, outPlaneSize);
 821                 const float* data_inp0 = data_inp0_ + subsampleIdx*inpPlaneSize*inpCn;
 822                 float* data_out0 = data_out0_ + subsampleIdx*outPlaneSize*outCn;
 823                 int startOutCn = (subsampleIdx % ngroups)*outCn;
 824                 const float* wptr_orig = wptr_orig_ + wstep*startOutCn;
 825                 const float* biasptr = biasptr_ + startOutCn;
 826
 827                 for( int cn0 = 0; cn0 < inpCn; cn0 += BLK_SIZE_CN )
 828                 {
 829                     int cn1 = std::min(cn0 + BLK_SIZE_CN, inpCn);
 830                     int ncn = cn1 - cn0, vsz = karea*ncn;
 831                     int vsz_a = (int)alignSize(vsz, valign);
 832                     const float* wptr = wptr_orig + cn0*karea;
 833                     // we apply [Channels][P]ReLU (if any) during the final pass only.
 834                     const float* relu = cn1 == inpCn && reluptr_ ? reluptr_ + startOutCn : 0;
 835
 836                     for( int ofs0 = stripeStart; ofs0 < stripeEnd; ofs0 += BLK_SIZE )
 837                     {
 838                         int ofs, ofs1 = std::min(ofs0 + BLK_SIZE, stripeEnd);
 839
 840                         int out_d = ofs0 / (outH * outW);
 841                         int out_i = (ofs0 - out_d * outH * outW) / outW;
 842                         int out_j = ofs0 % outW;
 843
 844                         // do im2row for a part of input tensor
 845                         float* rowbuf = rowbuf0;
 846
 847                         if (isConv2D)
 848                         {
 849                             for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i )
 850                             {
 851                                 int delta = std::min(ofs1 - ofs, outW - out_j);
 852                                 int out_j1 = out_j + delta;
 853
 854                                 int in_i = out_i * stride_h - pad_t;
 855                                 int in_j = out_j * stride_w - pad_l;
 856                                 const float* imgptr = data_inp0 + (cn0*height + in_i)*width + in_j;
 857                                 ofs += delta;
 858
 859                                 // do im2row for a part of input tensor
 860                                 if( is1x1 )
 861                                 {
 862                                     for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w )
 863                                     {
 864                                         for( k = 0; k < vsz; k++ )
 865                                             rowbuf[k] = imgptr[k*inpPlaneSize];
 866                                     }
 867                                 }
 868                                 else
 869                                 {
 870                                     bool ok_i = 0 <= in_i && in_i < height - (kernel_h-1)*dilation_h;
 871                                     int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h);
 872                                     int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h);
 873
 874                                     for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w )
 875                                     {
 876                                         // this condition should be true for most of the tensor elements, i.e.
 877                                         // most of the time the kernel aperture is inside the tensor X-Y plane.
 878                                         if( ok_i && out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w )
 879                                         {
 880                                             for( k = 0; k < vsz; k++ )
 881                                             {
 882                                                 int k1 = ofstab[k];
 883                                                 float v0 = imgptr[k1];
 884                                                 float v1 = imgptr[k1 + stride_w];
 885                                                 rowbuf[k] = v0;
 886                                                 rowbuf[k+vsz_a] = v1;
 887                                             }
 888                                             out_j++;
 889                                             rowbuf += vsz_a;
 890                                             imgptr += stride_w;
 891                                             in_j += stride_w;
 892                                         }
 893                                         else
 894                                         {
 895                                             int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w);
 896                                             int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w);
 897
 898                                             // here some non-continuous sub-row of the row will not be
 899                                             // filled from the tensor; we need to make sure that the uncovered
 900                                             // elements are explicitly set to 0's. the easiest way is to
 901                                             // set all the elements to 0's before the loop.
 902                                             memset(rowbuf, 0, vsz*sizeof(rowbuf[0]));
 903                                             for( k = 0; k < ncn; k++ )
 904                                             {
 905                                                 for( i = i0; i < i1; i++ )
 906                                                 {
 907                                                     for( j = j0; j < j1; j++ )
 908                                                     {
 909                                                         int imgofs = k*(width*height) + i*(dilation_h*width) + j*dilation_w;
 910                                                         rowbuf[(k*kernel_h + i)*kernel_w + j] = imgptr[imgofs];
 911                                                     }
 912                                                 }
 913                                             }
 914                                         }
 915                                     }
 916                                 }
 917                             }
 918                         }
 919                         else
 920                         {
 921                             for( ofs = ofs0; ofs < ofs1; out_d += (out_i + 1) / outH, out_i = (out_i + 1) % outH, out_j = 0 )
 922                             {
 923                                 int delta = std::min(ofs1 - ofs, outW - out_j);
 924                                 int out_j1 = out_j + delta;
 925
 926                                 int in_d = out_d * stride_d - pad_d;
 927                                 int in_i = out_i * stride_h - pad_t;
 928                                 int in_j = out_j * stride_w - pad_l;
 929                                 const float* imgptr = data_inp0 + (cn0*depth*height + in_d*height + in_i)*width + in_j;
 930                                 ofs += delta;
 931
 932                                 int d0 = std::max(0, (-in_d + dilation_d - 1) / dilation_d);
 933                                 int d1 = std::min(kernel_d, (depth - in_d + dilation_d - 1) / dilation_d);
 934
 935                                 int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h);
 936                                 int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h);
 937
 938                                 for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w )
 939                                 {
 940                                     int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w);
 941                                     int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w);
 942
 943                                     // here some non-continuous sub-row of the row will not be
 944                                     // filled from the tensor; we need to make sure that the uncovered
 945                                     // elements are explicitly set to 0's. the easiest way is to
 946                                     // set all the elements to 0's before the loop.
 947                                     memset(rowbuf, 0, vsz*sizeof(rowbuf[0]));
 948                                     for( k = 0; k < ncn; k++ )
 949                                     {
 950                                         for ( d = d0; d < d1; d++)
 951                                         {
 952                                             for( i = i0; i < i1; i++ )
 953                                             {
 954                                                 for( j = j0; j < j1; j++ )
 955                                                 {
 956                                                     int imgofs = k*(depth*width*height) + d*dilation_d*width*height + i*(dilation_h*width) + j*dilation_w;
 957                                                     rowbuf[(k*kernel_d*kernel_h + d*kernel_h + i)*kernel_w + j] = imgptr[imgofs];
 958                                                 }
 959                                             }
 960                                         }
 961                                     }
 962                                 }
 963                             }
 964                         }
 965
 966                         // now compute dot product of the weights
 967                         // and im2row-transformed part of the tensor
 968                         int bsz = ofs1 - ofs0;
 969                     #if CV_TRY_AVX512_SKX
 970                         /* AVX512 convolution requires an alignment of 16, and ROI is only there for larger vector sizes */
 971                         if(useAVX512)
 972                             opt_AVX512_SKX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 973                                           outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 974                         else
 975                     #endif
 976                     #if CV_TRY_AVX2
 977                         if(useAVX2)
 978                             opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 979                                           outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 980                         else
 981                     #endif
 982                     #if CV_TRY_AVX
 983                         if(useAVX)
 984                             opt_AVX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 985                                          outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 986                         else
 987                     #endif
 988                         for( int i = 0; i < outCn; i += 2 )
 989                         {
 990                             const float* wptr0 = wptr + i*wstep;
 991                             const float* wptr1 = wptr0 + wstep;
 992                             float* outptr0 = data_out0 + ofs0 + i*outPlaneSize;
 993                             float* outptr1 = outptr0 + outPlaneSize;
 994                             float bias0 = biasptr[i], bias1 = biasptr[i+1];
 995                             float r0 = 1.f, r1 = 1.f;
 996
 997                             if( i+1 >= outCn )
 998                             {
 999                                 wptr1 = wptr0;
1000                                 outptr1 = outptr0;
1001                                 bias1 = bias0;
1002                             }
1003
1004                             if( relu )
1005                             {
1006                                 r0 = relu[i]; r1 = relu[i+1];
1007                                 if( i+1 >= outCn )
1008                                     r1 = r0;
1009                             }
1010
1011                             int j = 0;
1012                         #if CV_SIMD128
1013                             v_float32x4 vr0 = v_setall_f32(r0), vr1 = v_setall_f32(r1), z = v_setzero_f32();
1014
1015                             for( ; j <= bsz - 4; j += 4 )
1016                             {
1017                                 const float* rptr = rowbuf0 + j*vsz_a;
1018                                 v_float32x4 s0, s1;
1019
1020                                 if( cn0 == 0 )
1021                                 {
1022                                     s0 = v_setall_f32(bias0);
1023                                     s1 = v_setall_f32(bias1);
1024                                 }
1025                                 else
1026                                 {
1027                                     s0 = v_load(outptr0 + j);
1028                                     s1 = v_load(outptr1 + j);
1029                                 }
1030
1031                                 v_float32x4 vs00 = v_setzero_f32(), vs01 = v_setzero_f32(),
1032                                             vs02 = v_setzero_f32(), vs03 = v_setzero_f32(),
1033                                             vs10 = v_setzero_f32(), vs11 = v_setzero_f32(),
1034                                             vs12 = v_setzero_f32(), vs13 = v_setzero_f32();
1035                                 for( k = 0; k < vsz; k += 4, rptr += 4 )
1036                                 {
1037                                     v_float32x4 w0 = v_load_aligned(wptr0 + k), w1 = v_load_aligned(wptr1 + k);
1038                                     v_float32x4 r0 = v_load_aligned(rptr), r1 = v_load_aligned(rptr + vsz_a),
1039                                                 r2 = v_load_aligned(rptr + vsz_a*2), r3 = v_load_aligned(rptr + vsz_a*3);
1040
1041                                     vs00 += w0*r0;
1042                                     vs01 += w0*r1;
1043                                     vs02 += w0*r2;
1044                                     vs03 += w0*r3;
1045
1046                                     vs10 += w1*r0;
1047                                     vs11 += w1*r1;
1048                                     vs12 += w1*r2;
1049                                     vs13 += w1*r3;
1050                                 }
1051                                 s0 += v_reduce_sum4(vs00, vs01, vs02, vs03);
1052                                 s1 += v_reduce_sum4(vs10, vs11, vs12, vs13);
1053                                 if( relu )
1054                                 {
1055                                     s0 = v_select(s0 > z, s0, s0*vr0);
1056                                     s1 = v_select(s1 > z, s1, s1*vr1);
1057                                 }
1058
1059                                 v_store(outptr0 + j, s0);
1060                                 v_store(outptr1 + j, s1);
1061                             }
1062                         #endif
1063                             for( ; j < bsz; j++ )
1064                             {
1065                                 const float* rptr = rowbuf0 + j*vsz_a;
1066                                 float s00, s10;
1067
1068                                 if( cn0 == 0 )
1069                                 {
1070                                     s00 = bias0;
1071                                     s10 = bias1;
1072                                 }
1073                                 else
1074                                 {
1075                                     s00 = outptr0[j];
1076                                     s10 = outptr1[j];
1077                                 }
1078
1079                                 for( k = 0; k < vsz; k++ )
1080                                 {
1081                                     float r0 = rptr[k];
1082                                     s00 += wptr0[k]*r0;
1083                                     s10 += wptr1[k]*r0;
1084                                 }
1085                                 if( relu )
1086                                 {
1087                                     s00 = s00 > 0.f ? s00 : s00*r0;
1088                                     s10 = s10 > 0.f ? s10 : s10*r1;
1089                                 }
1090
1091                                 outptr0[j] = s00;
1092                                 outptr1[j] = s10;
1093                             }
1094                         }
1095                     }
1096                 }
1097
1098                 if( activ_ )
1099                     activ_->forwardSlice(data_out0 + stripeStart, data_out0 + stripeStart,
1100                                          (int)(stripeEnd - stripeStart),
1101                                          outPlaneSize, startOutCn, startOutCn + outCn);
1102             }
1103         }
1104     };
1105
1106 #ifdef HAVE_OPENCL
1107     bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
1108     {
1109         std::vector<UMat> inputs;
1110         std::vector<UMat> outputs;
1111
1112         bool use_half = (inps.depth() == CV_16S);
1113         inps.getUMatVector(inputs);
1114         outs.getUMatVector(outputs);
1115
1116         CV_Assert(outputs.size() == 1);
1117         for (int i = 0; i < inputs.size(); ++i)
1118             CV_Assert(inputs[i].u != outputs[0].u);
1119
1120         if (umat_blobs.empty())
1121         {
1122             size_t n = blobs.size();
1123             umat_blobs.resize(n);
1124             for (size_t i = 0; i < n; i++)
1125             {
1126                 blobs[i].copyTo(umat_blobs[i]);
1127             }
1128         }
1129
1130         if (convolutionOp.empty())
1131         {
1132             OCL4DNNConvConfig config;
1133             config.in_shape = shape(inputs[0]);
1134             config.out_shape = shape(outputs[0]);
1135             config.kernel = kernel;
1136             config.pad = pad;
1137             config.stride = stride;
1138             config.dilation = dilation;
1139             config.group = inputs[0].size[1] / umat_blobs[0].size[1];
1140             config.bias_term = (hasBias()) ? true : false;
1141             config.use_half = use_half;
1142
1143             convolutionOp = Ptr<OCL4DNNConvSpatial<float> >(new OCL4DNNConvSpatial<float>(config));
1144         }
1145
1146         int outCn = umat_blobs[0].size[0];
1147
1148         reluslope.clear();
1149         if( activ )
1150         {
1151             Ptr<ReLULayer> activ_relu = activ.dynamicCast<ReLULayer>();
1152             if( !activ_relu.empty() )
1153             {
1154                 reluslope.assign(outCn+2, activ_relu->negativeSlope);
1155                 activType = OCL4DNN_CONV_FUSED_ACTIV_RELU;
1156             }
1157
1158             Ptr<ReLU6Layer> activ_relu6 = activ.dynamicCast<ReLU6Layer>();
1159             if( !activ_relu6.empty() )
1160             {
1161                 reluslope.resize(2);
1162                 reluslope[0] = activ_relu6->minValue;
1163                 reluslope[1] = activ_relu6->maxValue;
1164                 activType = OCL4DNN_CONV_FUSED_ACTIV_RELU6;
1165             }
1166
1167             Ptr<ChannelsPReLULayer> activ_chprelu = activ.dynamicCast<ChannelsPReLULayer>();
1168             if( !activ_chprelu.empty() )
1169             {
1170                 const Mat& m = activ_chprelu->blobs[0];
1171                 CV_Assert(m.isContinuous() && m.type() == CV_32F && (int)m.total() == outCn);
1172                 const float* mdata = m.ptr<float>();
1173                 reluslope.resize(outCn+2);
1174                 std::copy(mdata, mdata + outCn, reluslope.begin());
1175                 reluslope[outCn] = reluslope[outCn+1] = reluslope[outCn-1];
1176                 activType = OCL4DNN_CONV_FUSED_ACTIV_PRELU;
1177             }
1178         }
1179
1180         if (fusedWeights)
1181         {
1182             weightsMat.copyTo(umat_blobs[0]);
1183             fusedWeights = false;
1184         }
1185         if (fusedBias)
1186         {
1187             if ( umat_blobs.size() < 2 )
1188                 umat_blobs.resize(2);
1189             umat_blobs[1] = UMat(biasvec, true);
1190             convolutionOp->setBias(true);
1191             fusedBias = false;
1192         }
1193
1194         if ( newActiv )
1195         {
1196             if ( activType == OCL4DNN_CONV_FUSED_ACTIV_RELU )
1197             {
1198                 CV_Assert(!reluslope.empty());
1199                 convolutionOp->setActivReLU(true, reluslope[0]);
1200             }
1201             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_PRELU)
1202             {
1203                 CV_Assert(!reluslope.empty());
1204                 convolutionOp->setActivPReLU(true, reluslope);
1205             }
1206             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_POWER)
1207             {
1208                 convolutionOp->setActivPower(true, power);
1209             }
1210             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_TANH)
1211             {
1212                 convolutionOp->setActivTanh(true);
1213             }
1214             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_RELU6)
1215             {
1216                 convolutionOp->setActivReLU6(true, reluslope[0], reluslope[1]);
1217             }
1218             else
1219             {
1220                 convolutionOp->setActivReLU(false, 0);
1221                 convolutionOp->setActivPReLU(false, reluslope);
1222                 convolutionOp->setActivPower(false, 1.f);
1223                 convolutionOp->setActivTanh(false);
1224                 convolutionOp->setActivReLU6(false, 0, 0);
1225             }
1226             newActiv = false;
1227         }
1228
1229         UMat& inpMat = inputs[0];
1230         UMat& outMat = outputs[0];
1231         int batch_size = inpMat.size[0];
1232
1233         return convolutionOp->Forward(inpMat,
1234                                       inputs.size() == 2 ? inputs[1] : UMat(),
1235                                       umat_blobs[0],
1236                                       umat_blobs.size() > 1 ? umat_blobs[1] : UMat(),
1237                                       outMat,
1238                                       batch_size);
1239     }
1240 #endif
1241
1242     void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
1243     {
1244         CV_TRACE_FUNCTION();
1245         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
1246
1247         CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
1248                    forward_ocl(inputs_arr, outputs_arr, internals_arr))
1249
1250         if (inputs_arr.depth() == CV_16S)
1251         {
1252             forward_fallback(inputs_arr, outputs_arr, internals_arr);
1253             return;
1254         }
1255
1256         std::vector<Mat> inputs, outputs;
1257         inputs_arr.getMatVector(inputs);
1258         outputs_arr.getMatVector(outputs);
1259
1260         /*printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n",
1261                name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3],
1262                kernel.width, kernel.height, pad.width, pad.height,
1263                stride.width, stride.height, dilation.width, dilation.height);*/
1264         CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % blobs[0].size[1] == 0,
1265                     outputs.size() == 1, inputs[0].data != outputs[0].data);
1266
1267         int ngroups = inputs[0].size[1]/blobs[0].size[1];
1268         CV_Assert(outputs[0].size[1] % ngroups == 0);
1269         int outCn = blobs[0].size[0];
1270
1271         reluslope.clear();
1272         if( activ )
1273         {
1274             Ptr<ReLULayer> activ_relu = activ.dynamicCast<ReLULayer>();
1275             if( !activ_relu.empty() )
1276             {
1277                 reluslope.assign(outCn+2, activ_relu->negativeSlope);
1278             }
1279
1280             Ptr<ChannelsPReLULayer> activ_chprelu = activ.dynamicCast<ChannelsPReLULayer>();
1281             if( !activ_chprelu.empty() )
1282             {
1283                 const Mat& m = activ_chprelu->blobs[0];
1284                 CV_Assert(m.isContinuous() && m.type() == CV_32F && (int)m.total() == outCn);
1285                 const float* mdata = m.ptr<float>();
1286                 reluslope.resize(outCn+2);
1287                 std::copy(mdata, mdata + outCn, reluslope.begin());
1288                 reluslope[outCn] = reluslope[outCn+1] = reluslope[outCn-1];
1289             }
1290         }
1291
1292         int nstripes = std::max(getNumThreads(), 1);
1293
1294         ParallelConv::run(inputs[0], outputs[0], weightsMat, biasvec, reluslope,
1295                           kernel_size, strides, pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes);
1296     }
1297
1298 #ifdef HAVE_CUDA
1299     Ptr<BackendNode> initCUDA(
1300         void *context_,
1301         const std::vector<Ptr<BackendWrapper>>& inputs,
1302         const std::vector<Ptr<BackendWrapper>>& outputs
1303     ) override
1304     {
1305         auto context = reinterpret_cast<csl::CSLContext*>(context_);
1306
1307         CV_Assert(inputs.size() == 1);
1308         auto input_wrapper = inputs[0].dynamicCast<CUDABackendWrapper>();
1309         auto input_shape = input_wrapper->getShape();
1310
1311         CV_Assert(outputs.size() == 1);
1312         auto output_wrapper = outputs[0].dynamicCast<CUDABackendWrapper>();
1313         auto output_shape = output_wrapper->getShape();
1314
1315         const auto output_feature_maps = blobs[0].size[0];
1316         const auto input_feature_maps = input_shape[1];
1317         const auto input_feature_maps_per_group = blobs[0].size[1];
1318         const auto groups = input_feature_maps / input_feature_maps_per_group;
1319
1320         ConvolutionConfiguration config;
1321         config.kernel_size.assign(std::begin(kernel_size), std::end(kernel_size));
1322         config.dilations.assign(std::begin(dilations), std::end(dilations));
1323         config.strides.assign(std::begin(strides), std::end(strides));
1324
1325         if (padMode.empty())
1326         {
1327             config.padMode = ConvolutionConfiguration::PaddingMode::MANUAL;
1328             config.pads_begin.assign(std::begin(pads_begin), std::end(pads_begin));
1329             config.pads_end.assign(std::begin(pads_end), std::end(pads_end));
1330         }
1331         else if (padMode == "VALID")
1332         {
1333             config.padMode = ConvolutionConfiguration::PaddingMode::VALID;
1334         }
1335         else if (padMode == "SAME")
1336         {
1337             config.padMode = ConvolutionConfiguration::PaddingMode::SAME;
1338         }
1339         else
1340         {
1341             CV_Error(Error::StsNotImplemented, padMode + " padding mode not supported by ConvolutionLayer");
1342         }
1343
1344         config.input_shape.assign(std::begin(input_shape), std::end(input_shape));
1345         config.output_shape.assign(std::begin(output_shape), std::end(output_shape));
1346         config.groups = groups;
1347
1348         Mat filtersMat = fusedWeights ? weightsMat : blobs[0];
1349         Mat biasMat = (hasBias() || fusedBias) ? Mat(output_feature_maps, 1, CV_32F, biasvec.data()) : Mat();
1350         if (countNonZero(biasMat) == 0)
1351             biasMat = Mat();
1352
1353         return make_cuda_node<cuda4dnn::ConvolutionOp>(
1354             preferableTarget, std::move(context->stream), std::move(context->cudnn_handle), config, filtersMat, biasMat);
1355     }
1356 #endif
1357
1358     virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
1359                            const std::vector<MatShape> &outputs) const CV_OVERRIDE
1360     {
1361         CV_Assert(inputs.size() == outputs.size());
1362
1363         int64 flops = 0;
1364         int karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies<size_t>());
1365         for (int i = 0; i < inputs.size(); i++)
1366         {
1367             flops += total(outputs[i])*(CV_BIG_INT(2)*karea*inputs[i][1] + 1);
1368         }
1369
1370         return flops;
1371     }
1372 };
1373
1374 class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
1375 {
1376 public:
1377     Mat weightsMat, biasesMat;
1378     UMat umat_weights;
1379     UMat umat_biases;
1380
1381     DeConvolutionLayerImpl(const LayerParams& params) : BaseConvolutionLayerImpl(params) {}
1382
1383     MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE
1384     {
1385         int dims = inpShape.size();
1386         int inpCn = inpShape[1];
1387         int inpD = dims == 5 ? inpShape[2] : 1;
1388         int inpH = inpShape[dims - 2];
1389         int inpW = inpShape.back();
1390         int outCn = outShape[1];
1391         int ngroups = inpCn / blobs[0].size[0];
1392         int outGroupCn = outCn / ngroups;
1393         int ksize = outGroupCn * std::accumulate(kernel_size.begin(), kernel_size.end(),
1394                                                  1, std::multiplies<size_t>());
1395         return shape(ksize, inpD * inpH * inpW);
1396     }
1397
1398     virtual bool supportBackend(int backendId) CV_OVERRIDE
1399     {
1400         if (backendId == DNN_BACKEND_CUDA)
1401         {
1402             /* only deconvolution 2d and 3d supported */
1403             if (kernel_size.size() == 2 || kernel_size.size() == 3)
1404                 return true;
1405
1406             return false;
1407         }
1408
1409 #ifdef HAVE_INF_ENGINE
1410         const int outGroupCn = blobs[0].size[1];  // Weights are in IOHW or IODHW layout
1411         const int group = numOutput / outGroupCn;
1412
1413         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
1414         {
1415             if (kernel_size.size() == 3 && preferableTarget != DNN_TARGET_CPU) {
1416                 return false;
1417             }
1418
1419             if (std::accumulate(adjust_pads.begin(), adjust_pads.end(), 0, std::plus<size_t>()) > 0)
1420             {
1421                 if (padMode.empty())
1422                 {
1423                     if (preferableTarget != DNN_TARGET_CPU && group != 1)
1424                     {
1425                         for (int i = 0; i < adjust_pads.size(); i++) {
1426                             if (adjust_pads[i] && pads_begin[i])
1427                                 return false;
1428                         }
1429                     }
1430                     for (int i = 0; i < adjust_pads.size(); i++) {
1431                         if (pads_end[i] < adjust_pads[i])
1432                             return false;
1433                     }
1434                     return true;
1435                 }
1436                 else if (padMode == "SAME")
1437                 {
1438                     for (int i = 0; i < adjust_pads.size(); i++) {
1439                         if (kernel_size[i] < pads_begin[i] + 1 + adjust_pads[i])
1440                             return false;
1441                     }
1442                     return true;
1443                 }
1444                 else if (padMode == "VALID")
1445                     return false;
1446             }
1447
1448             if (group != 1)
1449             {
1450                 return preferableTarget == DNN_TARGET_CPU;
1451             }
1452             if (preferableTarget == DNN_TARGET_OPENCL || preferableTarget == DNN_TARGET_OPENCL_FP16)
1453                 return std::accumulate(dilations.begin(), dilations.end(), 1, std::multiplies<size_t>()) == 1;
1454             return true;
1455         }
1456         else
1457 #endif  // HAVE_INF_ENGINE
1458             return backendId == DNN_BACKEND_CUDA ||
1459             (kernel_size.size() == 2 && (backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE));
1460     }
1461
1462     bool getMemoryShapes(const std::vector<MatShape> &inputs,
1463                          const int requiredOutputs,
1464                          std::vector<MatShape> &outputs,
1465                          std::vector<MatShape> &internals) const CV_OVERRIDE
1466     {
1467         CV_Assert(!hasBias() || blobs[1].total() == (size_t)numOutput);
1468         CV_Assert(inputs.size() != 0);
1469
1470         int outCn = numOutput;
1471         std::vector<int> outShape;
1472         outShape.push_back(inputs[0][0]);  // batch
1473         outShape.push_back(outCn);
1474         if (padMode.empty())
1475         {
1476             for (int i = 0; i < kernel_size.size(); i++)
1477                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + kernel_size[i] - pads_begin[i] - pads_end[i] + adjust_pads[i]);
1478         }
1479         else if (padMode == "VALID")
1480         {
1481             for (int i = 0; i < kernel_size.size(); i++)
1482                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + kernel_size[i] + adjust_pads[i]);
1483         }
1484         else if (padMode == "SAME")
1485         {
1486             for (int i = 0; i < kernel_size.size(); i++)
1487                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + 1 + adjust_pads[i]);
1488         }
1489         else
1490             CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
1491
1492         CV_Assert(outCn % blobs[0].size[1] == 0);
1493         int ngroups = outCn / blobs[0].size[1];
1494
1495         int inpCn = inputs[0][1];
1496         CV_Assert(inpCn % ngroups == 0 && outCn % ngroups == 0);
1497         CV_Assert(blobs[0].size[0] == inpCn);
1498
1499         outputs.resize(1, outShape);
1500
1501         if (!is1x1())
1502             internals.push_back(computeColRowShape(inputs[0], outputs[0]));
1503
1504         return false;
1505     }
1506
1507     void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
1508     {
1509         BaseConvolutionLayerImpl::finalize(inputs_arr, outputs_arr);
1510
1511         std::vector<Mat> inputs, outputs;
1512         inputs_arr.getMatVector(inputs);
1513         outputs_arr.getMatVector(outputs);
1514
1515         std::vector<int> inpShape;
1516         std::vector<int> outShape;
1517         for (int i = 2; i < inputs[0].dims; i++) {
1518             inpShape.push_back(inputs[0].size[i]);
1519             outShape.push_back(outputs[0].size[i]);
1520         }
1521         getConvPoolPaddings(outShape, kernel_size, strides, padMode, pads_begin, pads_end);
1522         if (pads_begin.size() == 2) {
1523             for (int i = 0; i < pads_begin.size(); i++) {
1524                 if (pads_begin[i] != pads_end[i])
1525                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in deconvolution layer");
1526             }
1527             pad = Size(pads_begin[1], pads_begin[0]);
1528         }
1529
1530         weightsMultipliers.assign(numOutput, 1.0);
1531         if (weightsMat.empty())
1532         {
1533             transpose(blobs[0].reshape(1, blobs[0].size[0]), weightsMat);
1534             biasesMat = hasBias() ? blobs[1].reshape(1, numOutput)
1535                                   : Mat::zeros(numOutput, 1, CV_32F);
1536         }
1537     }
1538
1539     void fuseWeights(const Mat& w_, const Mat& b_) CV_OVERRIDE
1540     {
1541         Mat w = w_.total() == 1 ? Mat(1, numOutput, CV_32F, Scalar(w_.at<float>(0))) : w_;
1542         Mat b = b_.total() == 1 ? Mat(1, numOutput, CV_32F, Scalar(b_.at<float>(0))) : b_;
1543
1544         CV_Assert_N(!weightsMat.empty(),
1545                      w.empty() || numOutput == w.total(),
1546                      b.empty() || numOutput == b.total());
1547
1548         if (!w.empty())
1549         {
1550             transpose(blobs[0].reshape(1, blobs[0].size[0]), weightsMat);
1551             weightsMat = weightsMat.reshape(1, numOutput);
1552             for (int i = 0; i < numOutput; ++i)
1553             {
1554                 double wi = w.at<float>(i);
1555                 weightsMultipliers[i] *= wi;
1556                 cv::multiply(weightsMat.row(i), weightsMultipliers[i], weightsMat.row(i));
1557                 biasesMat.at<float>(i) *= wi;
1558             }
1559             weightsMat = weightsMat.reshape(1, weightsMat.total() / blobs[0].size[0]);
1560         }
1561
1562         if (!b.empty())
1563         {
1564             cv::add(biasesMat, b.reshape(1, numOutput), biasesMat);
1565         }
1566     }
1567
1568     class MatMulInvoker : public ParallelLoopBody
1569     {
1570     public:
1571         MatMulInvoker(const Mat& a, const Mat& b, Mat& c, int nstripes)
1572         {
1573             a_ = &a;
1574             b_ = &b;
1575             c_ = &c;
1576             nstripes_ = nstripes;
1577             useAVX = checkHardwareSupport(CPU_AVX);
1578             useAVX2 = checkHardwareSupport(CPU_AVX2);
1579             useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
1580         }
1581
1582         void operator()(const Range& range_) const CV_OVERRIDE
1583         {
1584             int stripeSize = (int)alignSize((b_->cols + nstripes_ - 1)/nstripes_, 16);
1585             Range range(range_.start*stripeSize, std::min(range_.end*stripeSize, b_->cols));
1586             int mmax = a_->rows;
1587             int nmax = range.end - range.start;
1588             int kmax = a_->cols;
1589             int m, n, k;
1590             const float* aptr = a_->ptr<float>();
1591             const float* bptr = b_->ptr<float>() + range.start;
1592             float* cptr = c_->ptr<float>() + range.start;
1593             size_t astep = a_->step1();
1594             size_t bstep = b_->step1();
1595             size_t cstep = c_->step1();
1596
1597         #if CV_TRY_AVX512_SKX
1598             if( useAVX512 )
1599                 opt_AVX512_SKX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1600             else
1601         #endif
1602         #if CV_TRY_AVX2
1603             if( useAVX2 )
1604                 opt_AVX2::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1605             else
1606         #endif
1607         #if CV_TRY_AVX
1608             if( useAVX )
1609                 opt_AVX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1610             else
1611         #endif
1612             for( m = 0; m < mmax; m += 2 )
1613             {
1614                 float* dst0 = cptr + cstep*m;
1615                 float* dst1 = cptr + cstep*std::min(m+1, mmax-1);
1616                 const float* aptr0 = aptr + astep*m;
1617                 const float* aptr1 = aptr + astep*std::min(m+1, mmax-1);
1618
1619                 for( n = 0; n < nmax; n++ )
1620                 {
1621                     dst0[n] = 0.f;
1622                     dst1[n] = 0.f;
1623                 }
1624
1625                 for( k = 0; k < kmax; k += 4 )
1626                 {
1627                     float alpha00 = aptr0[k];
1628                     float alpha01 = aptr1[k];
1629                     float alpha10 = 0.f, alpha11 = 0.f;
1630                     float alpha20 = 0.f, alpha21 = 0.f;
1631                     float alpha30 = 0.f, alpha31 = 0.f;
1632                     const float* bptr0 = bptr + k*bstep;
1633                     const float* bptr1 = bptr0;
1634                     const float* bptr2 = bptr0;
1635                     const float* bptr3 = bptr0;
1636
1637                     if( k+1 < kmax )
1638                     {
1639                         alpha10 = aptr0[k+1];
1640                         alpha11 = aptr1[k+1];
1641                         bptr1 = bptr0 + bstep;
1642                         if( k+2 < kmax )
1643                         {
1644                             alpha20 = aptr0[k+2];
1645                             alpha21 = aptr1[k+2];
1646                             bptr2 = bptr1 + bstep;
1647                             if( k+3 < kmax )
1648                             {
1649                                 alpha30 = aptr0[k+3];
1650                                 alpha31 = aptr1[k+3];
1651                                 bptr3 = bptr2 + bstep;
1652                             }
1653                         }
1654                     }
1655                     n = 0;
1656
1657                 #if CV_SIMD128
1658                     v_float32x4 a00 = v_setall_f32(alpha00);
1659                     v_float32x4 a01 = v_setall_f32(alpha01);
1660                     v_float32x4 a10 = v_setall_f32(alpha10);
1661                     v_float32x4 a11 = v_setall_f32(alpha11);
1662                     v_float32x4 a20 = v_setall_f32(alpha20);
1663                     v_float32x4 a21 = v_setall_f32(alpha21);
1664                     v_float32x4 a30 = v_setall_f32(alpha30);
1665                     v_float32x4 a31 = v_setall_f32(alpha31);
1666
1667                     for( ; n <= nmax - 4; n += 4 )
1668                     {
1669                         v_float32x4 b0 = v_load(bptr0 + n);
1670                         v_float32x4 b1 = v_load(bptr1 + n);
1671                         v_float32x4 b2 = v_load(bptr2 + n);
1672                         v_float32x4 b3 = v_load(bptr3 + n);
1673                         v_float32x4 d0 = v_load(dst0 + n);
1674                         v_float32x4 d1 = v_load(dst1 + n);
1675                         d0 += b0*a00;
1676                         d1 += b0*a01;
1677                         d0 += b1*a10;
1678                         d1 += b1*a11;
1679                         d0 += b2*a20;
1680                         d1 += b2*a21;
1681                         d0 += b3*a30;
1682                         d1 += b3*a31;
1683                         v_store(dst0 + n, d0);
1684                         v_store(dst1 + n, d1);
1685                     }
1686                 #endif
1687
1688                     for( ; n < nmax; n++ )
1689                     {
1690                         float b0 = bptr0[n], b1 = bptr1[n];
1691                         float b2 = bptr2[n], b3 = bptr3[n];
1692                         float d0 = dst0[n] + alpha00*b0 + alpha10*b1 + alpha20*b2 + alpha30*b3;
1693                         float d1 = dst1[n] + alpha01*b0 + alpha11*b1 + alpha21*b2 + alpha31*b3;
1694                         dst0[n] = d0;
1695                         dst1[n] = d1;
1696                     }
1697                 }
1698             }
1699         }
1700
1701         const Mat *a_, *b_;
1702         Mat* c_;
1703         int nstripes_;
1704         bool useAVX;
1705         bool useAVX2;
1706         bool useAVX512;
1707     };
1708
1709     class Col2ImInvoker : public cv::ParallelLoopBody
1710     {
1711     public:
1712         const float* data_col;
1713         const float* biasvec;
1714         int channels, height, width;
1715         int kernel_h, kernel_w;
1716         int pad_h, pad_w;
1717         int stride_h, stride_w;
1718         float* data_im;
1719         int height_col, width_col;
1720         int nstripes;
1721         bool is1x1;
1722
1723         Col2ImInvoker()
1724             : data_col(0), biasvec(0), channels(0), height(0), width(0),
1725               kernel_h(0), kernel_w(0), pad_h(0), pad_w(0), stride_h(0), stride_w(0), data_im(0),
1726               height_col(0), width_col(0), nstripes(0), is1x1(0)
1727         {}
1728
1729         static void run(const float* data_col,
1730                         int channels, int height, int width,
1731                         int kernel_h, int kernel_w,
1732                         int pad_h, int pad_w,
1733                         int stride_h, int stride_w,
1734                         int height_col, int width_col,
1735                         float* data_im,
1736                         const float* biasvec,
1737                         bool is1x1)
1738         {
1739             const int nstripes = getNumThreads();
1740
1741             Col2ImInvoker t;
1742             t.data_col = data_col;
1743             t.data_im = data_im;
1744             t.channels = channels; t.height = height; t.width = width;
1745             t.kernel_h = kernel_h; t.kernel_w = kernel_w;
1746             t.pad_h = pad_h; t.pad_w = pad_w;
1747             t.stride_h = stride_h; t.stride_w = stride_w;
1748             t.height_col = height_col;
1749             t.width_col = width_col;
1750             t.nstripes = nstripes;
1751             t.is1x1 = is1x1;
1752             t.biasvec = biasvec;
1753
1754             parallel_for_(Range(0, nstripes), t, nstripes);
1755         }
1756
1757         virtual void operator ()(const Range &r) const CV_OVERRIDE
1758         {
1759             const float* data_col_ = data_col;
1760             float* data_im_ = data_im;
1761             int coeff_h = (1 - stride_h * kernel_w * height_col) * width_col;
1762             int coeff_w = (1 - stride_w * height_col * width_col);
1763             size_t total = (size_t)channels * height * width;
1764             size_t stripeSize = (total + nstripes - 1)/nstripes;
1765             size_t startIndex = r.start*stripeSize;
1766             size_t endIndex = std::min(r.end*stripeSize, total);
1767             int w = (int)(startIndex % width + pad_w);
1768             int h = (int)((startIndex / width) % height + pad_h);
1769             int c = (int)(startIndex / (width * height));
1770             int h_col_start = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1;
1771             int h_col_end = std::min(h / stride_h + 1, height_col);
1772             int plane_size_col = height_col * width_col;
1773             int offset = (c * kernel_h * kernel_w + h * kernel_w + w) * plane_size_col;
1774             bool is1x1_ = is1x1;
1775             const float* biasvec_ = biasvec;
1776
1777             for (size_t index = startIndex; index < endIndex; index++)
1778             {
1779                 // compute the start and end of the output
1780                 int w_col_start = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1;
1781                 int w_col_end = std::min(w / stride_w + 1, width_col);
1782                 float val;
1783
1784                 if( is1x1_ )
1785                     val = data_im_[index];
1786                 else
1787                 {
1788                     val = 0.f;
1789                     for (int h_col = h_col_start; h_col < h_col_end; ++h_col) {
1790                         for (int w_col = w_col_start; w_col < w_col_end; ++w_col) {
1791                             val += data_col_[offset + h_col * coeff_h + w_col * coeff_w];
1792                         }
1793                     }
1794                 }
1795                 data_im_[index] = val + biasvec_[c];
1796
1797                 offset += plane_size_col;
1798                 if( ++w >= width + pad_w )
1799                 {
1800                     w = (int)((index + 1)% width + pad_w);
1801                     h = (int)(((index + 1) / width) % height + pad_h);
1802                     c = (int)((index + 1) / (width * height));
1803                     h_col_start = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1;
1804                     h_col_end = std::min(h / stride_h + 1, height_col);
1805                     offset = (c * kernel_h * kernel_w + h * kernel_w + w) * plane_size_col;
1806                 }
1807             }
1808         }
1809     };
1810
1811 #ifdef HAVE_OPENCL
1812     bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_)
1813     {
1814         std::vector<UMat> inputs;
1815         std::vector<UMat> outputs;
1816         std::vector<UMat> internals;
1817
1818         if (inputs_.depth() == CV_16S)
1819             return false;
1820
1821         inputs_.getUMatVector(inputs);
1822         outputs_.getUMatVector(outputs);
1823         internals_.getUMatVector(internals);
1824
1825         int outCn = numOutput;
1826         int inpCn = inputs[0].size[1];
1827
1828         if (is1x1())
1829             return false;
1830
1831         if (umat_weights.empty())
1832         {
1833             if (fusedWeights)
1834                 weightsMat.copyTo(umat_weights);
1835             else
1836                 transpose(blobs[0].reshape(1, inpCn), umat_weights);
1837
1838             if (fusedBias)
1839                 biasesMat.copyTo(umat_biases);
1840             else
1841             {
1842                 if (hasBias())
1843                     blobs[1].reshape(1, outCn).copyTo(umat_biases);
1844                 else
1845                     umat_biases = UMat::zeros(outCn, 1, CV_32F);
1846             }
1847         }
1848
1849         String buildopt = format("-DT=%s ", ocl::typeToStr(inputs[0].type()));
1850         buildopt += format("-DPAD_H=%d -DPAD_W=%d -DKERNEL_H=%d -DKERNEL_W=%d -DSTRIDE_H=%d -DSTRIDE_W=%d ",
1851                            pad.height, pad.width, kernel.height, kernel.width, stride.height, stride.width);
1852
1853         for (size_t ii = 0; ii < outputs.size(); ii++)
1854         {
1855             int ngroups = outCn / blobs[0].size[1];
1856             int inpGroupCn = inpCn / ngroups;
1857             int outGroupCn = blobs[0].size[1];
1858             const UMat& inp = inputs[ii];
1859             UMat& out = outputs[ii];
1860             int numImg = inp.size[0];
1861             int inpH = inp.size[2], inpW = inp.size[3];
1862             int outH = out.size[2], outW = out.size[3];
1863
1864             MatShape inpshape = shape(numImg*inpCn, inpH*inpW);
1865             MatShape outshape = shape(numImg*outCn, outH*outW);
1866             UMat convBlob = inputs[ii].reshape(1, inpshape.size(), &inpshape[0]);
1867             UMat decnBlob = out.reshape(1, outshape.size(), &outshape[0]);
1868             int rows = internals[0].rows / ngroups;
1869
1870             for (int n = 0; n < numImg; n++)
1871             {
1872                 for (int g = 0; g < ngroups; g++)
1873                 {
1874                     UMat colMat = internals[0].rowRange(_Range(g * rows, rows));
1875                     UMat convMat = convBlob.rowRange(_Range((g + n * ngroups) * inpGroupCn, inpGroupCn));
1876                     UMat wghtMat = umat_weights.colRange(_Range(g * inpGroupCn, inpGroupCn));
1877                     gemm(wghtMat, convMat, 1, noArray(), 0, colMat, 0);
1878                 }
1879
1880                 for (int g = 0; g < ngroups; g++)
1881                 {
1882                     int total = outGroupCn * decnBlob.cols;
1883                     int index = 0;
1884                     int height_col = inpH;
1885                     int width_col = inpW;
1886                     int coeff_h = (1 - stride.height * kernel.width * height_col) * width_col;
1887                     int coeff_w = (1 - stride.width * height_col * width_col);
1888
1889                     ocl::Kernel k("col2im", ocl::dnn::col2im_oclsrc, buildopt);
1890                     k.set(index++, total);
1891                     k.set(index++, ocl::KernelArg::PtrReadOnly(internals[0]));
1892                     k.set(index++, (int)(g * rows * internals[0].cols));
1893                     k.set(index++, outGroupCn);
1894                     k.set(index++, outH);
1895                     k.set(index++, outW);
1896                     k.set(index++, height_col);
1897                     k.set(index++, width_col);
1898                     k.set(index++, coeff_h);
1899                     k.set(index++, coeff_w);
1900                     k.set(index++, ocl::KernelArg::PtrReadOnly(umat_biases));
1901                     k.set(index++, (int)(g * outGroupCn * umat_biases.cols));
1902                     k.set(index++, ocl::KernelArg::PtrWriteOnly(decnBlob));
1903                     k.set(index++, (int)((g + n * ngroups) * outGroupCn * decnBlob.cols));
1904
1905                     size_t global[] = { (size_t)total };
1906                     bool ret = k.run(1, global, NULL, false);
1907                     if (!ret)
1908                         return false;
1909                 }
1910             }
1911         }
1912
1913         return true;
1914     }
1915 #endif
1916
1917     void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
1918     {
1919         CV_TRACE_FUNCTION();
1920         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
1921
1922         CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
1923                    forward_ocl(inputs_arr, outputs_arr, internals_arr));
1924
1925         if (inputs_arr.depth() == CV_16S)
1926         {
1927             forward_fallback(inputs_arr, outputs_arr, internals_arr);
1928             return;
1929         }
1930
1931         std::vector<Mat> inputs, outputs, internals;
1932         inputs_arr.getMatVector(inputs);
1933         outputs_arr.getMatVector(outputs);
1934         internals_arr.getMatVector(internals);
1935
1936         int outCn = numOutput;
1937         int inpCn = inputs[0].size[1];
1938         bool is1x1flag = is1x1();
1939         int nstripes = getNumThreads();
1940
1941         if( weightsMat.empty() )
1942         {
1943             transpose(blobs[0].reshape(1, inpCn), weightsMat);
1944             biasesMat = hasBias() ? blobs[1].reshape(1, outCn) : Mat::zeros(outCn, 1, CV_32F);
1945         }
1946
1947         for (size_t ii = 0; ii < outputs.size(); ii++)
1948         {
1949             int ngroups = outCn / blobs[0].size[1];
1950             int inpGroupCn = inpCn / ngroups;
1951             int outGroupCn = blobs[0].size[1];
1952             const Mat& inp = inputs[ii];
1953             Mat& out = outputs[ii];
1954             int numImg = inp.size[0];
1955             int inpH = inp.size[2], inpW = inp.size[3];
1956             int outH = out.size[2], outW = out.size[3];
1957
1958             Mat convBlob = inputs[ii].reshape(1, numImg*inpCn);
1959             Mat decnBlob = out.reshape(1, numImg*outCn);
1960
1961             for (int n = 0; n < numImg; n++)
1962             {
1963                 for (int g = 0; g < ngroups; g++)
1964                 {
1965                     Mat dstMat = decnBlob.rowRange(_Range((g + n * ngroups) * outGroupCn, outGroupCn));
1966                     Mat &colMat = is1x1flag ? dstMat : internals[0];
1967
1968                     Mat convMat = convBlob.rowRange(_Range((g + n * ngroups) * inpGroupCn, inpGroupCn));
1969                     Mat wghtMat = weightsMat.colRange(_Range(g * inpGroupCn, inpGroupCn));
1970                     Mat curBiasMat = biasesMat.rowRange(_Range(g * outGroupCn, outGroupCn));
1971
1972                     //gemm(wghtMat, convMat, 1, colMat, 0, colMat, 0);
1973                     MatMulInvoker mminvoker(wghtMat, convMat, colMat, nstripes);
1974                     parallel_for_(Range(0, nstripes), mminvoker, nstripes);
1975
1976                     Col2ImInvoker::run(colMat.ptr<float>(), outGroupCn, outH, outW,
1977                                        kernel.height, kernel.width, pad.height, pad.width,
1978                                        stride.height, stride.width, inpH, inpW, dstMat.ptr<float>(),
1979                                        curBiasMat.ptr<float>(), is1x1flag);
1980                 }
1981             }
1982         }
1983     }
1984
1985 #ifdef HAVE_CUDA
1986     Ptr<BackendNode> initCUDA(
1987         void *context_,
1988         const std::vector<Ptr<BackendWrapper>>& inputs,
1989         const std::vector<Ptr<BackendWrapper>>& outputs
1990     ) override
1991     {
1992         auto context = reinterpret_cast<csl::CSLContext*>(context_);
1993
1994         CV_Assert(inputs.size() == 1);
1995         auto input_wrapper = inputs[0].dynamicCast<CUDABackendWrapper>();
1996         auto input_shape = input_wrapper->getShape();
1997
1998         CV_Assert(outputs.size() == 1);
1999         auto output_wrapper = outputs[0].dynamicCast<CUDABackendWrapper>();
2000         auto output_shape = output_wrapper->getShape();
2001
2002         const auto output_feature_maps = numOutput;
2003         const auto output_feature_maps_per_group = blobs[0].size[1];
2004         const auto groups = output_feature_maps / output_feature_maps_per_group;
2005
2006         TransposeConvolutionConfiguration config;
2007         config.kernel_size.assign(std::begin(kernel_size), std::end(kernel_size));
2008         config.dilations.assign(std::begin(dilations), std::end(dilations));
2009         config.strides.assign(std::begin(strides), std::end(strides));
2010
2011         if (padMode.empty())
2012         {
2013             config.padMode = TransposeConvolutionConfiguration::PaddingMode::MANUAL;
2014             config.pads_begin.assign(std::begin(pads_begin), std::end(pads_begin));
2015             config.pads_end.assign(std::begin(pads_end), std::end(pads_end));
2016         }
2017         else if (padMode == "VALID")
2018         {
2019             config.padMode = TransposeConvolutionConfiguration::PaddingMode::VALID;
2020         }
2021         else if (padMode == "SAME")
2022         {
2023             config.padMode = TransposeConvolutionConfiguration::PaddingMode::SAME;
2024         }
2025         else
2026         {
2027             CV_Error(Error::StsNotImplemented, padMode + " padding mode not supported by DeconvolutionLayer");
2028         }
2029
2030         config.input_shape.assign(std::begin(input_shape), std::end(input_shape));
2031         config.output_shape.assign(std::begin(output_shape), std::end(output_shape));
2032         config.groups = groups;
2033
2034         CV_Assert(blobs.size() >= 1);
2035         Mat filtersMat = fusedWeights ? weightsMat.t() : blobs[0];
2036
2037         Mat biasMat = (hasBias() || fusedBias) ? biasesMat : Mat();
2038         if (countNonZero(biasMat) == 0)
2039             biasMat = Mat();
2040
2041         return make_cuda_node<cuda4dnn::TransposeConvolutionOp>(
2042             preferableTarget, std::move(context->stream), std::move(context->cudnn_handle), config, filtersMat, biasMat);
2043     }
2044 #endif
2045
2046     virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
2047     {
2048 #ifdef HAVE_HALIDE
2049         Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
2050
2051         int inW, inH, inC, inN;
2052         getCanonicalSize(inputBuffer, &inW, &inH, &inC, &inN);
2053         const int outGroupCn = blobs[0].size[1];
2054         const int group = numOutput / outGroupCn;
2055         const int inpGroupCn = blobs[0].size[0] / group;
2056
2057         Halide::Var x("x"), y("y"), c("c"), n("n");
2058         Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
2059         Halide::Func padded_input(name + "_constant_exterior");
2060         auto weights = wrapToHalideBuffer(blobs[0]);
2061
2062         Halide::Func dilated_input("dilated_input");
2063         dilated_input(x, y, c, n) = 0.0f;
2064         Halide::RDom r1(0, inW, 0, inH);
2065         dilated_input(r1.x * stride.width, r1.y * stride.height, c, n) =
2066               inputBuffer(r1.x, r1.y, c, n);
2067         dilated_input.compute_root();
2068
2069         Halide::Func bounded =
2070             Halide::BoundaryConditions::constant_exterior(dilated_input, 0,
2071                                                           0, (inW - 1) * stride.width + 1,
2072                                                           0, (inH - 1) * stride.height + 1,
2073                                                           0, inC, 0, inN);
2074         padded_input(x, y, c, n) = bounded(x, y, c, n);
2075
2076         Halide::RDom r(0, kernel.width, 0, kernel.height, 0, inpGroupCn);
2077         Halide::Expr kx = x + pad.width - r.x;
2078         Halide::Expr ky = y + pad.height - r.y;
2079         Halide::Expr kInC = r.z;
2080         Halide::Expr kOutC = c;
2081         for (int i = 1; i < group; ++i)
2082         {
2083             kInC = select(c < outGroupCn * i, kInC, inpGroupCn * i + r.z);
2084             kOutC = select(c < outGroupCn * i, kOutC, c - outGroupCn * i);
2085         }
2086         Halide::Expr topExpr = sum(padded_input(kx, ky, kInC, n) *
2087                                    weights(r.x, r.y, kOutC, kInC));
2088         if (hasBias())
2089         {
2090             auto bias = wrapToHalideBuffer(blobs[1], {numOutput});
2091             topExpr += bias(c);
2092         }
2093         top(x, y, c, n) = topExpr;
2094         return Ptr<BackendNode>(new HalideBackendNode({ padded_input, top }));
2095 #endif  // HAVE_HALIDE
2096         return Ptr<BackendNode>();
2097     }
2098
2099 #ifdef HAVE_INF_ENGINE
2100     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &) CV_OVERRIDE
2101     {
2102         InferenceEngine::Layout layout = blobs[0].dims == 5? InferenceEngine::Layout::NCDHW :
2103                                                              InferenceEngine::Layout::OIHW;
2104
2105         auto ieWeights = wrapToInfEngineBlob(blobs[0], layout);
2106         if (fusedWeights)
2107         {
2108             ieWeights = InferenceEngine::make_shared_blob<float>({
2109                             InferenceEngine::Precision::FP32,
2110                             ieWeights->getTensorDesc().getDims(), layout
2111                         });
2112             ieWeights->allocate();
2113
2114             int inpCn = blobs[0].size[0];
2115             Mat newWeights = infEngineBlobToMat(ieWeights).reshape(1, inpCn);
2116             transpose(weightsMat, newWeights);
2117         }
2118
2119         const int outGroupCn = blobs[0].size[1];  // Weights are in IOHW or OIDHW layout
2120         const int group = numOutput / outGroupCn;
2121
2122         InferenceEngine::Builder::DeconvolutionLayer ieLayer(name);
2123
2124         ieLayer.setKernel(kernel_size);
2125         ieLayer.setStrides(strides);
2126         ieLayer.setDilation(dilations);
2127         ieLayer.setPaddingsBegin(pads_begin);
2128
2129         if (padMode.empty())
2130         {
2131             std::vector<size_t> paddings_end;
2132             for (int i = 0; i < pads_end.size(); i++) {
2133                 paddings_end.push_back(pads_end[i] - adjust_pads[i]);
2134             }
2135             ieLayer.setPaddingsEnd(paddings_end);
2136         }
2137         else if (padMode == "SAME")
2138         {
2139             std::vector<size_t> paddings_end;
2140             for (int i = 0; i < pads_begin.size(); i++) {
2141                 paddings_end.push_back(kernel_size[i] - pads_begin[i] - 1 - adjust_pads[i]);
2142             }
2143             ieLayer.setPaddingsEnd(paddings_end);
2144         }
2145         ieLayer.setGroup((size_t)group);
2146         ieLayer.setOutDepth((size_t)numOutput);
2147
2148         InferenceEngine::Builder::Layer l = ieLayer;
2149         addConstantData("weights", ieWeights, l);
2150         if (hasBias())
2151             addConstantData("biases", wrapToInfEngineBlob(biasesMat, {(size_t)numOutput}, InferenceEngine::Layout::C), l);
2152         return Ptr<BackendNode>(new InfEngineBackendNode(l));
2153     }
2154 #endif  // HAVE_INF_ENGINE
2155
2156     virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
2157                            const std::vector<MatShape> &outputs) const CV_OVERRIDE
2158     {
2159         CV_Assert(inputs.size() == outputs.size());
2160
2161         float flops = 0;
2162         int outChannels = blobs[0].size[0];
2163         size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(),
2164                                        1, std::multiplies<size_t>());
2165
2166         for (int i = 0; i < inputs.size(); i++)
2167         {
2168             flops += CV_BIG_INT(2)*outChannels*karea*total(inputs[i]);
2169         }
2170
2171         return flops;
2172     }
2173 };
2174
2175 Ptr<BaseConvolutionLayer> ConvolutionLayer::create(const LayerParams &params)
2176 {
2177     Ptr<ConvolutionLayerImpl> l(new ConvolutionLayerImpl(params));
2178     return l;
2179 }
2180
2181 Ptr<BaseConvolutionLayer> DeconvolutionLayer::create(const LayerParams &params)
2182 {
2183     return Ptr<BaseConvolutionLayer>(new DeConvolutionLayerImpl(params));
2184 }
2185
2186 }
2187 }