modules/dnn/src/layers/convolution_layer.cpp

   1 /*M///////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
   4 //
   5 //  By downloading, copying, installing or using the software you agree to this license.
   6 //  If you do not agree to this license, do not download, install,
   7 //  copy or use the software.
   8 //
   9 //
  10 //                           License Agreement
  11 //                For Open Source Computer Vision Library
  12 //
  13 // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14 // Copyright (C) 2017, Intel Corporation, all rights reserved.
  15 // Third party copyrights are property of their respective owners.
  16 //
  17 // Redistribution and use in source and binary forms, with or without modification,
  18 // are permitted provided that the following conditions are met:
  19 //
  20 //   * Redistribution's of source code must retain the above copyright notice,
  21 //     this list of conditions and the following disclaimer.
  22 //
  23 //   * Redistribution's in binary form must reproduce the above copyright notice,
  24 //     this list of conditions and the following disclaimer in the documentation
  25 //     and/or other materials provided with the distribution.
  26 //
  27 //   * The name of the copyright holders may not be used to endorse or promote products
  28 //     derived from this software without specific prior written permission.
  29 //
  30 // This software is provided by the copyright holders and contributors "as is" and
  31 // any express or implied warranties, including, but not limited to, the implied
  32 // warranties of merchantability and fitness for a particular purpose are disclaimed.
  33 // In no event shall the Intel Corporation or contributors be liable for any direct,
  34 // indirect, incidental, special, exemplary, or consequential damages
  35 // (including, but not limited to, procurement of substitute goods or services;
  36 // loss of use, data, or profits; or business interruption) however caused
  37 // and on any theory of liability, whether in contract, strict liability,
  38 // or tort (including negligence or otherwise) arising in any way out of
  39 // the use of this software, even if advised of the possibility of such damage.
  40 //
  41 //M*/
  42
  43 #include "../precomp.hpp"
  44 #include "layers_common.hpp"
  45 #include "../op_halide.hpp"
  46 #include "../op_inf_engine.hpp"
  47 #include "../op_vkcom.hpp"
  48 #include "opencv2/core/hal/hal.hpp"
  49 #include "opencv2/core/hal/intrin.hpp"
  50 #include <iostream>
  51 #include <numeric>
  52
  53 #ifdef HAVE_OPENCL
  54 #include "opencl_kernels_dnn.hpp"
  55 using namespace cv::dnn::ocl4dnn;
  56 #endif
  57
  58 namespace cv
  59 {
  60 namespace dnn
  61 {
  62
  63 class BaseConvolutionLayerImpl : public ConvolutionLayer
  64 {
  65 public:
  66     bool fusedWeights, fusedBias;
  67     std::vector<double> weightsMultipliers;
  68     BaseConvolutionLayerImpl(const LayerParams &params)
  69     {
  70         setParamsFrom(params);
  71         getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads);
  72
  73         numOutput = params.get<int>("num_output");
  74         int ngroups = params.get<int>("group", 1);
  75         CV_Assert(numOutput % ngroups == 0);
  76
  77         if (kernel_size.size() == 2) {
  78             kernel = Size(kernel_size[1], kernel_size[0]);
  79             stride = Size(strides[1], strides[0]);
  80             for (int i = 0; i < pads_begin.size(); i++) {
  81                 if (pads_begin[i] != pads_end[i])
  82                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer");
  83             }
  84             pad = Size(pads_begin[1], pads_begin[0]);
  85             dilation = Size(dilations[1], dilations[0]);
  86
  87             adjustPad.height = adjust_pads[0];
  88             adjustPad.width = adjust_pads[1];
  89         }
  90
  91         for (int i = 0; i < adjust_pads.size(); i++) {
  92             CV_Assert(adjust_pads[i] < strides[i]);
  93         }
  94
  95         fusedWeights = false;
  96         fusedBias = false;
  97     }
  98
  99     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
 100     {
 101         std::vector<Mat> inputs, outputs;
 102         inputs_arr.getMatVector(inputs);
 103         outputs_arr.getMatVector(outputs);
 104
 105         CV_Assert(inputs.size() > 0);
 106
 107         CV_Assert(blobs.size() == 1 || blobs.size() == 2);
 108         CV_Assert(inputs[0].dims == outputs[0].dims);
 109         CV_Assert(blobs[0].dims == kernel_size.size() + 2);
 110         for (int i = 0; i < kernel_size.size(); i++) {
 111             CV_Assert(blobs[0].size[i + 2] == kernel_size[i]);
 112         }
 113
 114         const Mat &input = inputs[0];
 115         CV_Assert((input.dims == 4 || input.dims == 5) && (input.type() == CV_32F || input.type() == CV_16S));
 116         for (size_t i = 0; i < inputs.size(); i++)
 117         {
 118             CV_Assert(inputs[i].type() == input.type());
 119             CV_Assert((inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]);
 120             for (int j = 0; j < inputs[i].dims; j++) {
 121                 CV_Assert(inputs[i].size[j] == input.size[j]);
 122             }
 123         }
 124
 125         std::vector<int> inpShape;
 126         std::vector<int> outShape;
 127         for (int i = 2; i < inputs[0].dims; i++) {
 128             inpShape.push_back(inputs[0].size[i]);
 129             outShape.push_back(outputs[0].size[i]);
 130         }
 131         getConvPoolPaddings(inpShape, kernel_size, strides, padMode, pads_begin, pads_end);
 132         if (pads_begin.size() == 2) {
 133             for (int i = 0; i < pads_begin.size(); i++) {
 134                 if (pads_begin[i] != pads_end[i])
 135                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer");
 136             }
 137             pad = Size(pads_begin[1], pads_begin[0]);
 138         }
 139         fusedWeights = false;
 140         fusedBias = false;
 141     }
 142
 143     bool hasBias() const
 144     {
 145         return blobs.size() >= 2;
 146     }
 147
 148     virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0;
 149     bool is1x1() const
 150     {
 151         return (kernel.height == 1 && kernel.width == 1) &&
 152                (stride.height == 1 && stride.width == 1) &&
 153                (dilation.height == 1 && dilation.width == 1);
 154     }
 155
 156     virtual bool tryFuse(Ptr<Layer>& top) CV_OVERRIDE
 157     {
 158         Mat w, b;
 159         top->getScaleShift(w, b);
 160         if (!w.empty() || !b.empty())
 161         {
 162             fuseWeights(w, b);
 163             fusedWeights = fusedWeights || !w.empty();
 164             fusedBias = fusedBias || (hasBias() && !w.empty()) || !b.empty();
 165             return true;
 166         }
 167         return false;
 168     }
 169
 170     virtual void fuseWeights(const Mat& w_, const Mat& b_) = 0;
 171
 172     virtual void applyHalideScheduler(Ptr<BackendNode>& node,
 173                                       const std::vector<Mat*> &inputs,
 174                                       const std::vector<Mat> &outputs,
 175                                       int targetId) const CV_OVERRIDE
 176     {
 177 #ifdef HAVE_HALIDE
 178         if (targetId != DNN_TARGET_CPU)
 179         {
 180             Layer::applyHalideScheduler(node, inputs, outputs, targetId);
 181             return;
 182         }
 183         Halide::Var x("x"), y("y"), c("c"), n("n"), tile("tile"), yi("yi"), yo("yo"), co("co"), ci("ci");
 184         Halide::Func& top = node.dynamicCast<HalideBackendNode>()->funcs[1];
 185         Halide::Func& padded_input = node.dynamicCast<HalideBackendNode>()->funcs[0];
 186
 187         int outW, outH, outC, outN;
 188         getCanonicalSize(outputs[0].size, &outW, &outH, &outC, &outN);
 189
 190         if (outW == 1 || outH <= 2)
 191             return;
 192
 193         if (is1x1() || outC <= 16)
 194             top.reorder(x, c, y)
 195                .split(y, yo, yi, 2)
 196                .fuse(yo, n, tile)
 197                .parallel(tile)
 198                .unroll(yi)
 199                .vectorize(x, outW >= 16 ? 16 : outW);
 200         else
 201             top.reorder(x, c, y)
 202                .split(y, yo, yi, 2)
 203                .split(c, co, ci, 16)
 204                .fuse(yo, co, tile).fuse(n, tile, tile)
 205                .parallel(tile)
 206                .unroll(yi)
 207                .vectorize(x, outW >= 16 ? 16 : outW);
 208         padded_input.compute_at(top, yi);
 209 #endif  // HAVE_HALIDE
 210     }
 211 };
 212
 213
 214 #define IS_POWER_LAYER(layer) \
 215             (!layer.empty() && !layer->type.compare("Power"))
 216 //TODO: simultaneously convolution and bias addition for cache optimization
 217 class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
 218 {
 219 public:
 220     enum { VEC_ALIGN = 8, DFT_TYPE = CV_32F };
 221     Mat weightsMat;
 222     std::vector<float> biasvec;
 223     std::vector<float> reluslope;
 224     Ptr<ActivationLayer> activ;
 225
 226 #ifdef HAVE_OPENCL
 227     Ptr<OCL4DNNConvSpatial<float> > convolutionOp;
 228     std::vector<UMat> umat_blobs;
 229     bool newActiv;
 230     ocl4dnnFusedActiv_t activType;
 231     float power;
 232 #endif
 233     ConvolutionLayerImpl(const LayerParams &params) : BaseConvolutionLayerImpl(params)
 234     {
 235 #ifdef HAVE_OPENCL
 236         newActiv = false;
 237         activType = OCL4DNN_CONV_FUSED_ACTIV_NONE;
 238         power = 0.f;
 239 #endif
 240     }
 241
 242     MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE
 243     {
 244         Size out(outShape[3], outShape[2]);
 245         int inpGroupCn = blobs[0].size[1];
 246         int ksize = inpGroupCn * kernel.height * kernel.width;
 247         return shape(out.area(), ksize);
 248     }
 249
 250     virtual bool supportBackend(int backendId) CV_OVERRIDE
 251     {
 252 #ifdef HAVE_INF_ENGINE
 253         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
 254         {
 255             if (kernel_size.size() == 3)
 256                 return preferableTarget == DNN_TARGET_CPU;
 257             return (preferableTarget != DNN_TARGET_MYRIAD || dilation.width == dilation.height);
 258         }
 259         else
 260 #endif
 261         {
 262             if (kernel_size.size() == 3)
 263                 return (preferableTarget == DNN_TARGET_CPU && backendId == DNN_BACKEND_OPENCV);
 264             else if (kernel_size.size() == 2)
 265                 return backendId == DNN_BACKEND_OPENCV ||
 266                        backendId == DNN_BACKEND_HALIDE ||
 267                        (backendId == DNN_BACKEND_VKCOM && haveVulkan());
 268             else
 269                 return false;
 270         }
 271     }
 272
 273     bool getMemoryShapes(const std::vector<MatShape> &inputs,
 274                          const int requiredOutputs,
 275                          std::vector<MatShape> &outputs,
 276                          std::vector<MatShape> &internals) const CV_OVERRIDE
 277     {
 278         CV_Assert(blobs.size() != 0);
 279         CV_Assert(!hasBias() || blobs[1].total() == (size_t)blobs[0].size[0]);
 280         CV_Assert(inputs.size() == (size_t)1);
 281
 282         internals.clear();
 283
 284         CV_Assert(inputs.size() != 0);
 285         std::vector<int> inpShape(inputs[0].begin() + 2, inputs[0].end());
 286
 287         int outCn = blobs[0].size[0];
 288         std::vector<int> outShape;
 289         outShape.push_back(inputs[0][0]);
 290         outShape.push_back(outCn);
 291
 292         int inpCn = inputs[0][1];
 293         if (padMode.empty())
 294         {
 295             for (int i = 0; i < inpShape.size(); i++)
 296                 outShape.push_back((inpShape[i] + pads_begin[i] + pads_end[i] - dilations[i] * (kernel_size[i] - 1) - 1) / strides[i] + 1);
 297         }
 298         else
 299         {
 300             getConvPoolOutParams(inpShape, kernel_size, strides, padMode, dilations, outShape);
 301         }
 302
 303         int ngroups = inpCn / blobs[0].size[1];
 304         if (ngroups == 0 || ngroups * blobs[0].size[1] != inpCn)
 305             CV_Error(Error::StsError, format("Number of input channels should "
 306                      "be multiple of %d but got %d", blobs[0].size[1], inpCn));
 307         CV_Assert(ngroups > 0 && inpCn % ngroups == 0 && outCn % ngroups == 0);
 308
 309         outputs.resize(1, outShape);
 310
 311         return false;
 312     }
 313
 314     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
 315     {
 316         BaseConvolutionLayerImpl::finalize(inputs_arr, outputs_arr);
 317
 318         CV_Assert(!blobs.empty());
 319         const int outCn = blobs[0].size[0];
 320         // prepare weightsMat where each row is aligned and has enough zero padding on the right to
 321         // use vectorized (i.e. with intrinsics) loops without tail processing
 322         Mat wm = blobs[0].reshape(1, outCn);
 323         if( wm.step1() % VEC_ALIGN != 0 )
 324         {
 325             int newcols = (int)alignSize(wm.step1(), VEC_ALIGN);
 326             Mat wm_buffer = Mat(outCn, newcols, wm.type());
 327             Mat wm_padding = wm_buffer.colRange(wm.cols, newcols);
 328             wm_padding.setTo(Scalar::all(0.));
 329             Mat wm_aligned = wm_buffer.colRange(0, wm.cols);
 330             wm.copyTo(wm_aligned);
 331             wm = wm_aligned;
 332         }
 333         weightsMat = wm;
 334         weightsMultipliers.assign(outCn, 1.0);
 335
 336         Mat biasMat = hasBias() ? blobs[1].reshape(1, outCn) : Mat();
 337         biasvec.resize(outCn+2);
 338         if( biasMat.empty() )
 339         {
 340             for(int i = 0; i < outCn; i++ )
 341                 biasvec[i] = 0.f;
 342         }
 343         else
 344         {
 345             for(int i = 0; i < outCn; i++ )
 346                 biasvec[i] = biasMat.at<float>(i);
 347         }
 348 #ifdef HAVE_OPENCL
 349         convolutionOp.release();
 350 #endif
 351     }
 352
 353     bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
 354     {
 355         if (!activ.empty() && !layer.empty())
 356             return false;
 357
 358         activ = layer;
 359         if (activ.empty())
 360             reluslope.clear();
 361 #ifdef HAVE_OPENCL
 362         newActiv = true;
 363         activType = OCL4DNN_CONV_FUSED_ACTIV_NONE;
 364
 365         if (IS_DNN_OPENCL_TARGET(preferableTarget))
 366         {
 367             Ptr<PowerLayer> activ_power = activ.dynamicCast<PowerLayer>();
 368             if (!activ_power.empty())
 369             {
 370                 if (activ_power->scale != 1.f || activ_power->shift != 0.f)
 371                 {
 372                     const int outCh = blobs[0].size[0];
 373                     fuseWeights(Mat(1, outCh, CV_32F, Scalar(activ_power->scale)),
 374                                 Mat(1, outCh, CV_32F, Scalar(activ_power->shift)));
 375                 }
 376
 377                 power = activ_power->power;
 378                 activType = OCL4DNN_CONV_FUSED_ACTIV_POWER;
 379             }
 380             Ptr<TanHLayer> activ_tanh = activ.dynamicCast<TanHLayer>();
 381             if (!activ_tanh.empty())
 382             {
 383                 activType = OCL4DNN_CONV_FUSED_ACTIV_TANH;
 384             }
 385         }
 386 #endif
 387         return !activ.empty();
 388     }
 389
 390     void fuseWeights(const Mat& w_, const Mat& b_) CV_OVERRIDE
 391     {
 392         // Convolution weights have OIHW data layout. Parameters fusion in case of
 393         // (conv(I) + b1 ) * w + b2
 394         // means to replace convolution's weights to [w*conv(I)] and bias to [b1 * w + b2]
 395         const int outCn = weightsMat.size[0];
 396         Mat w = w_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(w_.at<float>(0))) : w_;
 397         Mat b = b_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(b_.at<float>(0))) : b_;
 398         CV_Assert_N(!weightsMat.empty(), biasvec.size() == outCn + 2,
 399                     w.empty() || outCn == w.total(), b.empty() || outCn == b.total());
 400
 401         if (!w.empty())
 402         {
 403             // Keep origin weights unchanged.
 404             if (weightsMat.data == blobs[0].data)
 405                 weightsMat = weightsMat.clone();
 406
 407             Mat originWeights = blobs[0].reshape(1, outCn);
 408             for (int i = 0; i < outCn; ++i)
 409             {
 410                 double wi = w.at<float>(i);
 411                 weightsMultipliers[i] *= wi;
 412                 cv::multiply(originWeights.row(i), weightsMultipliers[i], weightsMat.row(i));
 413                 biasvec[i] *= wi;
 414             }
 415         }
 416
 417         if (!b.empty())
 418         {
 419             for (int i = 0; i < outCn; ++i)
 420                 biasvec[i] += b.at<float>(i);
 421         }
 422         biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1];
 423     }
 424
 425     virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 426     {
 427 #ifdef HAVE_VULKAN
 428         int out_channel = blobs[0].size[0];
 429         bool has_bias = hasBias() || fusedBias;
 430         int filter_size[2] = {kernel.height, kernel.width};
 431         int pad_size[2] = {pad.height, pad.width};
 432         int stride_size[2] = {stride.height, stride.width};
 433         int dilation_size[2] = {dilation.height, dilation.width};
 434         int activation = 0;
 435         vkcom::Tensor input_tensor = VkComTensor(inputs[0]);
 436         int in_channel = input_tensor.dimSize(1);
 437         int group = in_channel / blobs[0].size[1];
 438
 439         // TODO: support group > 1
 440         if (group != 1)
 441             return Ptr<BackendNode>();
 442
 443         int padding_mode;
 444         if (padMode.empty())
 445         {
 446             padding_mode = vkcom::kPaddingModeCaffe;
 447         }
 448         else if (padMode == "VALID")
 449         {
 450             padding_mode = vkcom::kPaddingModeValid;
 451         }
 452         else if (padMode == "SAME")
 453         {
 454             padding_mode = vkcom::kPaddingModeSame;
 455         }
 456         else
 457             CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
 458
 459         std::shared_ptr<vkcom::OpBase> op(new vkcom::OpConv(out_channel, has_bias,
 460                     filter_size, pad_size,
 461                     stride_size, dilation_size,
 462                     activation, group,
 463                     padding_mode));
 464
 465         std::vector<Ptr<BackendWrapper> > blobsWrapper;
 466
 467         if (fusedWeights)
 468         {
 469             Mat wm;
 470             weightsMat.copyTo(wm); // to handle the case of isContinuous() == false
 471             wm = wm.reshape(1, blobs[0].dims, blobs[0].size);
 472             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(wm)));
 473         }
 474         else
 475         {
 476             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(blobs[0])));
 477         }
 478
 479         if (has_bias)
 480         {
 481             Mat biasesMat({out_channel}, CV_32F, &biasvec[0]);
 482             blobsWrapper.push_back(Ptr<BackendWrapper>(new VkComBackendWrapper(biasesMat)));
 483         }
 484
 485         return Ptr<BackendNode>(new VkComBackendNode(inputs, op, blobsWrapper));
 486 #endif  // HAVE_VULKAN
 487         return Ptr<BackendNode>();
 488     }
 489
 490
 491
 492     virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 493     {
 494 #ifdef HAVE_HALIDE
 495         Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
 496
 497         const int inpCn = inputBuffer.channels();
 498         const int outCn = blobs[0].size[0];
 499         const int inpGroupCn = blobs[0].size[1];
 500         const int group = inpCn / inpGroupCn;
 501         const int outGroupCn = outCn / group;
 502
 503         Halide::Buffer<float> weights = wrapToHalideBuffer(blobs[0]);
 504
 505         Halide::Var x("x"), y("y"), c("c"), n("n");
 506         Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
 507         Halide::Func padded_input(name + "_constant_exterior");
 508         if (pad.width || pad.height)
 509         {
 510             Halide::Func bounded =
 511                 Halide::BoundaryConditions::constant_exterior(inputBuffer, 0);
 512             padded_input(x, y, c, n) = bounded(x, y, c, n);
 513         }
 514         else
 515         {
 516             padded_input(x, y, c, n) = inputBuffer(x, y, c, n);
 517         }
 518
 519         Halide::RDom r(0, kernel.width, 0, kernel.height, 0, inpGroupCn);
 520         Halide::Expr kx = x * stride.width - pad.width + r.x * dilation.width;
 521         Halide::Expr ky = y * stride.height - pad.height + r.y * dilation.height;
 522         Halide::Expr kc = r.z;
 523         for (int i = 1; i < group; ++i)
 524         {
 525             kc = select(c < outGroupCn * i, kc, inpGroupCn * i + r.z);
 526         }
 527         Halide::Expr topExpr = sum(padded_input(kx, ky, kc, n) *
 528                                    weights(r.x, r.y, r.z, c));
 529         if (hasBias())
 530         {
 531             Halide::Buffer<float> bias = wrapToHalideBuffer(blobs[1], {outCn});
 532             topExpr += bias(c);
 533         }
 534         top(x, y, c, n) = topExpr;
 535         return Ptr<BackendNode>(new HalideBackendNode({ padded_input, top }));
 536 #endif  // HAVE_HALIDE
 537         return Ptr<BackendNode>();
 538     }
 539
 540 #ifdef HAVE_INF_ENGINE
 541     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
 542     {
 543         InferenceEngine::DataPtr input = infEngineDataNode(inputs[0]);
 544         CV_Assert(input->dims.size() == 4 || input->dims.size() == 5);
 545
 546         const int inpCn = input->dims[input->dims.size() - 2];  // NOTE: input->dims are reversed (WHIO or WHDIO)
 547         const int outCn = blobs[0].size[0];
 548         const int inpGroupCn = blobs[0].size[1];
 549         const int group = inpCn / inpGroupCn;
 550
 551         InferenceEngine::Layout layout = (input->dims.size() == 4) ? InferenceEngine::Layout::OIHW :
 552                                                                      InferenceEngine::Layout::NCDHW;
 553
 554         auto ieWeights = wrapToInfEngineBlob(blobs[0], layout);
 555         if (fusedWeights)
 556         {
 557             if (weightsMat.isContinuous())
 558             {
 559                 Mat cvWeights = weightsMat.reshape(1, blobs[0].dims, blobs[0].size);
 560                 ieWeights = wrapToInfEngineBlob(cvWeights, layout);
 561             }
 562             else
 563             {
 564                 ieWeights = InferenceEngine::make_shared_blob<float>(
 565                                     InferenceEngine::Precision::FP32, layout,
 566                                     ieWeights->dims());
 567                 ieWeights->allocate();
 568
 569                 Mat newWeights = infEngineBlobToMat(ieWeights).reshape(1, outCn);
 570                 Mat cvWeights = weightsMat.colRange(0, newWeights.cols);
 571                 cvWeights.copyTo(newWeights);
 572             }
 573         }
 574         InferenceEngine::Blob::Ptr ieBiases;
 575         if (hasBias() || fusedBias)
 576         {
 577             Mat biasesMat({outCn}, CV_32F, &biasvec[0]);
 578             ieBiases = wrapToInfEngineBlob(biasesMat, {(size_t)outCn}, InferenceEngine::Layout::C);
 579         }
 580
 581         InferenceEngine::Builder::ConvolutionLayer ieLayer(name);
 582
 583         ieLayer.setKernel(kernel_size);
 584         ieLayer.setStrides(strides);
 585         ieLayer.setDilation(dilations);
 586         ieLayer.setPaddingsBegin(pads_begin);
 587         ieLayer.setPaddingsEnd(pads_end);
 588         ieLayer.setGroup((size_t)group);
 589         ieLayer.setOutDepth((size_t)outCn);
 590
 591         InferenceEngine::Builder::Layer l = ieLayer;
 592         addConstantData("weights", ieWeights, l);
 593         if (ieBiases)
 594             addConstantData("biases", ieBiases, l);
 595
 596         if (!padMode.empty())
 597             l.getParameters()["auto_pad"] = padMode == "VALID" ? std::string("valid") : std::string("same_upper");
 598
 599         return Ptr<BackendNode>(new InfEngineBackendNode(l));
 600     }
 601 #endif  // HAVE_INF_ENGINE
 602
 603     class ParallelConv : public cv::ParallelLoopBody
 604     {
 605     public:
 606         enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 };
 607
 608         const Mat* input_;
 609         const Mat* weights_;
 610         Mat* output_;
 611         int outShape[4]; // used only for conv2d
 612         std::vector<size_t> kernel_size, pads_begin, pads_end, strides, dilations;
 613         int ngroups_, nstripes_;
 614         std::vector<int> ofstab_;
 615         const std::vector<float>* biasvec_;
 616         const std::vector<float>* reluslope_;
 617         const ActivationLayer* activ_;
 618         bool is1x1_;
 619         bool useAVX;
 620         bool useAVX2;
 621         bool useAVX512;
 622
 623         ParallelConv()
 624             : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
 625               biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false)
 626         {}
 627
 628         static void run( const Mat& input, Mat& output, const Mat& weights,
 629                          const std::vector<float>& biasvec,
 630                          const std::vector<float>& reluslope,
 631                          const std::vector<size_t>& kernel_size, const std::vector<size_t>& strides,
 632                          const std::vector<size_t>& pads_begin, const std::vector<size_t>& pads_end,
 633                          const std::vector<size_t>& dilations,
 634                          const ActivationLayer* activ, int ngroups, int nstripes )
 635         {
 636             size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(),
 637                                            1, std::multiplies<size_t>());
 638             CV_Assert_N(
 639                        (input.dims == 4 || input.dims == 5) && (input.dims == output.dims),
 640                        input.size[0] == output.size[0],
 641                        weights.rows == output.size[1],
 642                        weights.cols == (input.size[1]/ngroups)*karea,
 643                        input.type() == output.type(),
 644                        input.type() == weights.type(),
 645                        input.type() == CV_32FC1,
 646                        input.isContinuous(),
 647                        output.isContinuous(),
 648                        biasvec.size() == (size_t)output.size[1]+2);
 649             ParallelConv p;
 650
 651             p.input_ = &input;
 652             p.weights_ = &weights;
 653             p.output_ = &output;
 654             for( int i = 0; i < 4; i++ ) p.outShape[i] = output.size[i];
 655             p.outShape[1] /= ngroups;
 656
 657             p.kernel_size = kernel_size; p.strides = strides; p.dilations = dilations;
 658             p.pads_begin = pads_begin; p.pads_end = pads_end;
 659
 660             p.ngroups_ = ngroups;
 661             p.nstripes_ = nstripes;
 662
 663             int inpCnAll = input.size[1];
 664             int depth = (input.dims == 5) ? input.size[2] : 1;
 665             int width = input.size[input.dims - 1];
 666             int height = input.size[input.dims - 2];
 667             int inpCn = inpCnAll / ngroups;
 668
 669             bool isConv2D = kernel_size.size() == 2;
 670
 671             p.is1x1_ = isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 &&
 672                        pads_begin[0] == 0  && pads_begin[1] == 0;
 673
 674             p.useAVX    = checkHardwareSupport(CPU_AVX)  && isConv2D;
 675             p.useAVX2   = checkHardwareSupport(CPU_AVX2) && isConv2D;
 676             p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX  && isConv2D;
 677
 678             int ncn = std::min(inpCn, (int)BLK_SIZE_CN);
 679
 680             int kernel_d = !isConv2D? kernel_size[0] : 1;
 681             int kernel_h = kernel_size[kernel_size.size() - 2];
 682             int kernel_w = kernel_size.back();
 683
 684             int dil_d = !isConv2D? dilations[0] : 1;
 685             int dil_h = dilations[dilations.size() - 2];
 686             int dil_w = dilations.back();
 687
 688             p.ofstab_.resize(karea * ncn);
 689             int* ofstab = &p.ofstab_[0];
 690
 691             if (isConv2D)
 692             {
 693                 for( int k = 0; k < ncn; k++ )
 694                     for( int k_r = 0; k_r < kernel_h; k_r++ )
 695                         for( int k_c = 0; k_c < kernel_w; k_c++ )
 696                             ofstab[(k*kernel_h + k_r)*kernel_w + k_c] =
 697                                    (k*height + k_r*dil_h)*width + k_c*dil_w;
 698             }
 699             else
 700             {
 701                 for( int k = 0; k < ncn; k++ )
 702                     for (int k_d = 0; k_d < kernel_d; k_d++)
 703                         for( int k_r = 0; k_r < kernel_h; k_r++ )
 704                             for( int k_c = 0; k_c < kernel_w; k_c++ )
 705                                 ofstab[(k*kernel_d*kernel_h + k_d*kernel_h + k_r)*kernel_w + k_c] =
 706                                        (k*depth*height + k_d*dil_d*height + k_r*dil_h)*width + k_c*dil_w;
 707             }
 708
 709             p.biasvec_ = &biasvec;
 710             p.reluslope_ = &reluslope;
 711             p.activ_ = p.reluslope_->empty() ? activ : 0;
 712
 713             parallel_for_(Range(0, nstripes), p, nstripes);
 714         }
 715
 716         virtual void operator ()(const Range &r0) const CV_OVERRIDE
 717         {
 718             const int valign = ConvolutionLayerImpl::VEC_ALIGN;
 719             int ngroups = ngroups_, batchSize = input_->size[0]*ngroups;
 720             bool isConv2D = input_->dims == 4;
 721
 722             int outW = output_->size[output_->dims - 1];
 723             int outH = output_->size[output_->dims - 2];
 724             int outCn = output_->size[1]/ngroups;
 725
 726             int depth = !isConv2D? input_->size[2] : 1;
 727             int height = input_->size[input_->dims - 2];
 728             int width = input_->size[input_->dims - 1];
 729             int inpCn = input_->size[1]/ngroups;
 730
 731             const int nstripes = nstripes_;
 732
 733             int kernel_d = !isConv2D? kernel_size[0] : 1;
 734             int kernel_h = kernel_size[kernel_size.size() - 2];
 735             int kernel_w = kernel_size.back();
 736             int karea = kernel_w*kernel_h*kernel_d;
 737
 738             int pad_d = !isConv2D? pads_begin[0] : 0;
 739             int pad_t = pads_begin[pads_begin.size() - 2];
 740             int pad_l = pads_begin.back();
 741
 742             int stride_d = !isConv2D? strides[0] : 0;
 743             int stride_h = strides[strides.size() - 2];
 744             int stride_w = strides.back();
 745
 746             int dilation_d = !isConv2D? dilations[0] : 1;
 747             int dilation_h = dilations[dilations.size() - 2];
 748             int dilation_w = dilations.back();
 749
 750             int i, j, k, d;
 751             size_t inpPlaneSize = input_->total(2);
 752             size_t outPlaneSize = output_->total(2);
 753             bool is1x1 = is1x1_;
 754
 755             int stripesPerSample;
 756             size_t stripeSize;
 757             Range r = r0;
 758
 759             if( nstripes >= batchSize*2 )
 760             {
 761                 stripesPerSample = nstripes/batchSize;
 762                 stripeSize = alignSize((outPlaneSize + stripesPerSample - 1)/stripesPerSample, valign);
 763                 stripeSize = std::min(stripeSize, outPlaneSize);
 764             }
 765             else
 766             {
 767                 stripesPerSample = 1;
 768                 int samplesPerStripe = std::max((batchSize + nstripes - 1)/nstripes, 1);
 769                 r.start *= samplesPerStripe;
 770                 r.end *= samplesPerStripe;
 771                 stripeSize = outPlaneSize;
 772             }
 773
 774             const float* data_inp0_ = input_->ptr<float>();
 775             const int* ofstab = &ofstab_[0];
 776             const float* wptr_orig_ = weights_->ptr<float>();
 777             size_t wstep = weights_->step1();
 778             const float* biasptr_ = &biasvec_->at(0);
 779             const float* reluptr_ = reluslope_->empty() ? 0 : &reluslope_->at(0);
 780             float* data_out0_ = output_->ptr<float>();
 781             size_t rowbufsz = (size_t)karea*BLK_SIZE_CN*BLK_SIZE;
 782             AutoBuffer<float> rowbuf0_(rowbufsz + valign);
 783             float* rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(float)));
 784
 785             // we clear the buffer once; ultimately, it lets us to avoid
 786             // tail processing after running the unrolled/vectorized loop.
 787             // the main idea is to make sure that the tail (a.k.a. padding) of each row
 788             // (i.e. the elements with indices between vsz=karea*ncn and vsz_a)
 789             // does not contain NaNs or Infs. Because the padding in the weights
 790             // matrix is explicitly initialized with 0's, we handle all other
 791             // cases nicely, i.e. we can skip expliciting re-initialization
 792             // of the padding - we just retain elements from the previous iteration
 793             // of the loop over channels (cn0).
 794             memset(rowbuf0, 0, rowbufsz*sizeof(rowbuf0[0]) );
 795
 796             for( int stripe = r.start; stripe < r.end; stripe++ )
 797             {
 798                 int subsampleIdx = stripe/stripesPerSample;
 799                 if( subsampleIdx >= batchSize )
 800                     break;
 801                 int stripeStart = (int)((stripe - subsampleIdx*stripesPerSample)*stripeSize);
 802                 int stripeEnd = (int)std::min(stripeStart + stripeSize, outPlaneSize);
 803                 const float* data_inp0 = data_inp0_ + subsampleIdx*inpPlaneSize*inpCn;
 804                 float* data_out0 = data_out0_ + subsampleIdx*outPlaneSize*outCn;
 805                 int startOutCn = (subsampleIdx % ngroups)*outCn;
 806                 const float* wptr_orig = wptr_orig_ + wstep*startOutCn;
 807                 const float* biasptr = biasptr_ + startOutCn;
 808
 809                 for( int cn0 = 0; cn0 < inpCn; cn0 += BLK_SIZE_CN )
 810                 {
 811                     int cn1 = std::min(cn0 + BLK_SIZE_CN, inpCn);
 812                     int ncn = cn1 - cn0, vsz = karea*ncn;
 813                     int vsz_a = (int)alignSize(vsz, valign);
 814                     const float* wptr = wptr_orig + cn0*karea;
 815                     // we apply [Channels][P]ReLU (if any) during the final pass only.
 816                     const float* relu = cn1 == inpCn && reluptr_ ? reluptr_ + startOutCn : 0;
 817
 818                     for( int ofs0 = stripeStart; ofs0 < stripeEnd; ofs0 += BLK_SIZE )
 819                     {
 820                         int ofs, ofs1 = std::min(ofs0 + BLK_SIZE, stripeEnd);
 821
 822                         int out_d = ofs0 / (outH * outW);
 823                         int out_i = (ofs0 - out_d * outH * outW) / outW;
 824                         int out_j = ofs0 % outW;
 825
 826                         // do im2row for a part of input tensor
 827                         float* rowbuf = rowbuf0;
 828
 829                         if (isConv2D)
 830                         {
 831                             for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i )
 832                             {
 833                                 int delta = std::min(ofs1 - ofs, outW - out_j);
 834                                 int out_j1 = out_j + delta;
 835
 836                                 int in_i = out_i * stride_h - pad_t;
 837                                 int in_j = out_j * stride_w - pad_l;
 838                                 const float* imgptr = data_inp0 + (cn0*height + in_i)*width + in_j;
 839                                 ofs += delta;
 840
 841                                 // do im2row for a part of input tensor
 842                                 if( is1x1 )
 843                                 {
 844                                     for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w )
 845                                     {
 846                                         for( k = 0; k < vsz; k++ )
 847                                             rowbuf[k] = imgptr[k*inpPlaneSize];
 848                                     }
 849                                 }
 850                                 else
 851                                 {
 852                                     bool ok_i = 0 <= in_i && in_i < height - (kernel_h-1)*dilation_h;
 853                                     int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h);
 854                                     int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h);
 855
 856                                     for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w )
 857                                     {
 858                                         // this condition should be true for most of the tensor elements, i.e.
 859                                         // most of the time the kernel aperture is inside the tensor X-Y plane.
 860                                         if( ok_i && out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w )
 861                                         {
 862                                             for( k = 0; k < vsz; k++ )
 863                                             {
 864                                                 int k1 = ofstab[k];
 865                                                 float v0 = imgptr[k1];
 866                                                 float v1 = imgptr[k1 + stride_w];
 867                                                 rowbuf[k] = v0;
 868                                                 rowbuf[k+vsz_a] = v1;
 869                                             }
 870                                             out_j++;
 871                                             rowbuf += vsz_a;
 872                                             imgptr += stride_w;
 873                                             in_j += stride_w;
 874                                         }
 875                                         else
 876                                         {
 877                                             int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w);
 878                                             int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w);
 879
 880                                             // here some non-continuous sub-row of the row will not be
 881                                             // filled from the tensor; we need to make sure that the uncovered
 882                                             // elements are explicitly set to 0's. the easiest way is to
 883                                             // set all the elements to 0's before the loop.
 884                                             memset(rowbuf, 0, vsz*sizeof(rowbuf[0]));
 885                                             for( k = 0; k < ncn; k++ )
 886                                             {
 887                                                 for( i = i0; i < i1; i++ )
 888                                                 {
 889                                                     for( j = j0; j < j1; j++ )
 890                                                     {
 891                                                         int imgofs = k*(width*height) + i*(dilation_h*width) + j*dilation_w;
 892                                                         rowbuf[(k*kernel_h + i)*kernel_w + j] = imgptr[imgofs];
 893                                                     }
 894                                                 }
 895                                             }
 896                                         }
 897                                     }
 898                                 }
 899                             }
 900                         }
 901                         else
 902                         {
 903                             for( ofs = ofs0; ofs < ofs1; out_d += (out_i + 1) / outH, out_i = (out_i + 1) % outH, out_j = 0 )
 904                             {
 905                                 int delta = std::min(ofs1 - ofs, outW - out_j);
 906                                 int out_j1 = out_j + delta;
 907
 908                                 int in_d = out_d * stride_d - pad_d;
 909                                 int in_i = out_i * stride_h - pad_t;
 910                                 int in_j = out_j * stride_w - pad_l;
 911                                 const float* imgptr = data_inp0 + (cn0*depth*height + in_d*height + in_i)*width + in_j;
 912                                 ofs += delta;
 913
 914                                 int d0 = std::max(0, (-in_d + dilation_d - 1) / dilation_d);
 915                                 int d1 = std::min(kernel_d, (depth - in_d + dilation_d - 1) / dilation_d);
 916
 917                                 int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h);
 918                                 int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h);
 919
 920                                 for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w )
 921                                 {
 922                                     int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w);
 923                                     int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w);
 924
 925                                     // here some non-continuous sub-row of the row will not be
 926                                     // filled from the tensor; we need to make sure that the uncovered
 927                                     // elements are explicitly set to 0's. the easiest way is to
 928                                     // set all the elements to 0's before the loop.
 929                                     memset(rowbuf, 0, vsz*sizeof(rowbuf[0]));
 930                                     for( k = 0; k < ncn; k++ )
 931                                     {
 932                                         for ( d = d0; d < d1; d++)
 933                                         {
 934                                             for( i = i0; i < i1; i++ )
 935                                             {
 936                                                 for( j = j0; j < j1; j++ )
 937                                                 {
 938                                                     int imgofs = k*(depth*width*height) + d*dilation_d*width*height + i*(dilation_h*width) + j*dilation_w;
 939                                                     rowbuf[(k*kernel_d*kernel_h + d*kernel_h + i)*kernel_w + j] = imgptr[imgofs];
 940                                                 }
 941                                             }
 942                                         }
 943                                     }
 944                                 }
 945                             }
 946                         }
 947
 948                         // now compute dot product of the weights
 949                         // and im2row-transformed part of the tensor
 950                         int bsz = ofs1 - ofs0;
 951                     #if CV_TRY_AVX512_SKX
 952                         /* AVX512 convolution requires an alignment of 16, and ROI is only there for larger vector sizes */
 953                         if(useAVX512)
 954                             opt_AVX512_SKX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 955                                           outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 956                         else
 957                     #endif
 958                     #if CV_TRY_AVX2
 959                         if(useAVX2)
 960                             opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 961                                           outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 962                         else
 963                     #endif
 964                     #if CV_TRY_AVX
 965                         if(useAVX)
 966                             opt_AVX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
 967                                          outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
 968                         else
 969                     #endif
 970                         for( int i = 0; i < outCn; i += 2 )
 971                         {
 972                             const float* wptr0 = wptr + i*wstep;
 973                             const float* wptr1 = wptr0 + wstep;
 974                             float* outptr0 = data_out0 + ofs0 + i*outPlaneSize;
 975                             float* outptr1 = outptr0 + outPlaneSize;
 976                             float bias0 = biasptr[i], bias1 = biasptr[i+1];
 977                             float r0 = 1.f, r1 = 1.f;
 978
 979                             if( i+1 >= outCn )
 980                             {
 981                                 wptr1 = wptr0;
 982                                 outptr1 = outptr0;
 983                                 bias1 = bias0;
 984                             }
 985
 986                             if( relu )
 987                             {
 988                                 r0 = relu[i]; r1 = relu[i+1];
 989                                 if( i+1 >= outCn )
 990                                     r1 = r0;
 991                             }
 992
 993                             int j = 0;
 994                         #if CV_SIMD128
 995                             v_float32x4 vr0 = v_setall_f32(r0), vr1 = v_setall_f32(r1), z = v_setzero_f32();
 996
 997                             for( ; j <= bsz - 4; j += 4 )
 998                             {
 999                                 const float* rptr = rowbuf0 + j*vsz_a;
1000                                 v_float32x4 s0, s1;
1001
1002                                 if( cn0 == 0 )
1003                                 {
1004                                     s0 = v_setall_f32(bias0);
1005                                     s1 = v_setall_f32(bias1);
1006                                 }
1007                                 else
1008                                 {
1009                                     s0 = v_load(outptr0 + j);
1010                                     s1 = v_load(outptr1 + j);
1011                                 }
1012
1013                                 v_float32x4 vs00 = v_setzero_f32(), vs01 = v_setzero_f32(),
1014                                             vs02 = v_setzero_f32(), vs03 = v_setzero_f32(),
1015                                             vs10 = v_setzero_f32(), vs11 = v_setzero_f32(),
1016                                             vs12 = v_setzero_f32(), vs13 = v_setzero_f32();
1017                                 for( k = 0; k < vsz; k += 4, rptr += 4 )
1018                                 {
1019                                     v_float32x4 w0 = v_load_aligned(wptr0 + k), w1 = v_load_aligned(wptr1 + k);
1020                                     v_float32x4 r0 = v_load_aligned(rptr), r1 = v_load_aligned(rptr + vsz_a),
1021                                                 r2 = v_load_aligned(rptr + vsz_a*2), r3 = v_load_aligned(rptr + vsz_a*3);
1022
1023                                     vs00 += w0*r0;
1024                                     vs01 += w0*r1;
1025                                     vs02 += w0*r2;
1026                                     vs03 += w0*r3;
1027
1028                                     vs10 += w1*r0;
1029                                     vs11 += w1*r1;
1030                                     vs12 += w1*r2;
1031                                     vs13 += w1*r3;
1032                                 }
1033                                 s0 += v_reduce_sum4(vs00, vs01, vs02, vs03);
1034                                 s1 += v_reduce_sum4(vs10, vs11, vs12, vs13);
1035                                 if( relu )
1036                                 {
1037                                     s0 = v_select(s0 > z, s0, s0*vr0);
1038                                     s1 = v_select(s1 > z, s1, s1*vr1);
1039                                 }
1040
1041                                 v_store(outptr0 + j, s0);
1042                                 v_store(outptr1 + j, s1);
1043                             }
1044                         #endif
1045                             for( ; j < bsz; j++ )
1046                             {
1047                                 const float* rptr = rowbuf0 + j*vsz_a;
1048                                 float s00, s10;
1049
1050                                 if( cn0 == 0 )
1051                                 {
1052                                     s00 = bias0;
1053                                     s10 = bias1;
1054                                 }
1055                                 else
1056                                 {
1057                                     s00 = outptr0[j];
1058                                     s10 = outptr1[j];
1059                                 }
1060
1061                                 for( k = 0; k < vsz; k++ )
1062                                 {
1063                                     float r0 = rptr[k];
1064                                     s00 += wptr0[k]*r0;
1065                                     s10 += wptr1[k]*r0;
1066                                 }
1067                                 if( relu )
1068                                 {
1069                                     s00 = s00 > 0.f ? s00 : s00*r0;
1070                                     s10 = s10 > 0.f ? s10 : s10*r1;
1071                                 }
1072
1073                                 outptr0[j] = s00;
1074                                 outptr1[j] = s10;
1075                             }
1076                         }
1077                     }
1078                 }
1079
1080                 if( activ_ )
1081                     activ_->forwardSlice(data_out0 + stripeStart, data_out0 + stripeStart,
1082                                          (int)(stripeEnd - stripeStart),
1083                                          outPlaneSize, startOutCn, startOutCn + outCn);
1084             }
1085         }
1086     };
1087
1088 #ifdef HAVE_OPENCL
1089     bool forward_ocl(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals)
1090     {
1091         std::vector<UMat> inputs;
1092         std::vector<UMat> outputs;
1093
1094         bool use_half = (inps.depth() == CV_16S);
1095         inps.getUMatVector(inputs);
1096         outs.getUMatVector(outputs);
1097
1098         CV_Assert(outputs.size() == 1);
1099         for (int i = 0; i < inputs.size(); ++i)
1100             CV_Assert(inputs[i].u != outputs[0].u);
1101
1102         if (umat_blobs.empty())
1103         {
1104             size_t n = blobs.size();
1105             umat_blobs.resize(n);
1106             for (size_t i = 0; i < n; i++)
1107             {
1108                 blobs[i].copyTo(umat_blobs[i]);
1109             }
1110         }
1111
1112         if (convolutionOp.empty())
1113         {
1114             OCL4DNNConvConfig config;
1115             config.in_shape = shape(inputs[0]);
1116             config.out_shape = shape(outputs[0]);
1117             config.kernel = kernel;
1118             config.pad = pad;
1119             config.stride = stride;
1120             config.dilation = dilation;
1121             config.group = inputs[0].size[1] / umat_blobs[0].size[1];
1122             config.bias_term = (hasBias()) ? true : false;
1123             config.use_half = use_half;
1124
1125             convolutionOp = Ptr<OCL4DNNConvSpatial<float> >(new OCL4DNNConvSpatial<float>(config));
1126         }
1127
1128         int outCn = umat_blobs[0].size[0];
1129
1130         reluslope.clear();
1131         if( activ )
1132         {
1133             Ptr<ReLULayer> activ_relu = activ.dynamicCast<ReLULayer>();
1134             if( !activ_relu.empty() )
1135             {
1136                 reluslope.assign(outCn+2, activ_relu->negativeSlope);
1137                 activType = OCL4DNN_CONV_FUSED_ACTIV_RELU;
1138             }
1139
1140             Ptr<ReLU6Layer> activ_relu6 = activ.dynamicCast<ReLU6Layer>();
1141             if( !activ_relu6.empty() )
1142             {
1143                 reluslope.resize(2);
1144                 reluslope[0] = activ_relu6->minValue;
1145                 reluslope[1] = activ_relu6->maxValue;
1146                 activType = OCL4DNN_CONV_FUSED_ACTIV_RELU6;
1147             }
1148
1149             Ptr<ChannelsPReLULayer> activ_chprelu = activ.dynamicCast<ChannelsPReLULayer>();
1150             if( !activ_chprelu.empty() )
1151             {
1152                 const Mat& m = activ_chprelu->blobs[0];
1153                 CV_Assert(m.isContinuous() && m.type() == CV_32F && (int)m.total() == outCn);
1154                 const float* mdata = m.ptr<float>();
1155                 reluslope.resize(outCn+2);
1156                 std::copy(mdata, mdata + outCn, reluslope.begin());
1157                 reluslope[outCn] = reluslope[outCn+1] = reluslope[outCn-1];
1158                 activType = OCL4DNN_CONV_FUSED_ACTIV_PRELU;
1159             }
1160         }
1161
1162         if (fusedWeights)
1163         {
1164             weightsMat.copyTo(umat_blobs[0]);
1165             fusedWeights = false;
1166         }
1167         if (fusedBias)
1168         {
1169             if ( umat_blobs.size() < 2 )
1170                 umat_blobs.resize(2);
1171             umat_blobs[1] = UMat(biasvec, true);
1172             convolutionOp->setBias(true);
1173             fusedBias = false;
1174         }
1175
1176         if ( newActiv )
1177         {
1178             if ( activType == OCL4DNN_CONV_FUSED_ACTIV_RELU )
1179             {
1180                 CV_Assert(!reluslope.empty());
1181                 convolutionOp->setActivReLU(true, reluslope[0]);
1182             }
1183             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_PRELU)
1184             {
1185                 CV_Assert(!reluslope.empty());
1186                 convolutionOp->setActivPReLU(true, reluslope);
1187             }
1188             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_POWER)
1189             {
1190                 convolutionOp->setActivPower(true, power);
1191             }
1192             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_TANH)
1193             {
1194                 convolutionOp->setActivTanh(true);
1195             }
1196             else if ( activType == OCL4DNN_CONV_FUSED_ACTIV_RELU6)
1197             {
1198                 convolutionOp->setActivReLU6(true, reluslope[0], reluslope[1]);
1199             }
1200             else
1201             {
1202                 convolutionOp->setActivReLU(false, 0);
1203                 convolutionOp->setActivPReLU(false, reluslope);
1204                 convolutionOp->setActivPower(false, 1.f);
1205                 convolutionOp->setActivTanh(false);
1206                 convolutionOp->setActivReLU6(false, 0, 0);
1207             }
1208             newActiv = false;
1209         }
1210
1211         UMat& inpMat = inputs[0];
1212         UMat& outMat = outputs[0];
1213         int batch_size = inpMat.size[0];
1214
1215         return convolutionOp->Forward(inpMat,
1216                                       inputs.size() == 2 ? inputs[1] : UMat(),
1217                                       umat_blobs[0],
1218                                       umat_blobs.size() > 1 ? umat_blobs[1] : UMat(),
1219                                       outMat,
1220                                       batch_size);
1221     }
1222 #endif
1223
1224     void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
1225     {
1226         CV_TRACE_FUNCTION();
1227         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
1228
1229         CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
1230                    forward_ocl(inputs_arr, outputs_arr, internals_arr))
1231
1232         if (inputs_arr.depth() == CV_16S)
1233         {
1234             forward_fallback(inputs_arr, outputs_arr, internals_arr);
1235             return;
1236         }
1237
1238         std::vector<Mat> inputs, outputs;
1239         inputs_arr.getMatVector(inputs);
1240         outputs_arr.getMatVector(outputs);
1241
1242         /*printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n",
1243                name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3],
1244                kernel.width, kernel.height, pad.width, pad.height,
1245                stride.width, stride.height, dilation.width, dilation.height);*/
1246         CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % blobs[0].size[1] == 0,
1247                     outputs.size() == 1, inputs[0].data != outputs[0].data);
1248
1249         int ngroups = inputs[0].size[1]/blobs[0].size[1];
1250         CV_Assert(outputs[0].size[1] % ngroups == 0);
1251         int outCn = blobs[0].size[0];
1252
1253         reluslope.clear();
1254         if( activ )
1255         {
1256             Ptr<ReLULayer> activ_relu = activ.dynamicCast<ReLULayer>();
1257             if( !activ_relu.empty() )
1258             {
1259                 reluslope.assign(outCn+2, activ_relu->negativeSlope);
1260             }
1261
1262             Ptr<ChannelsPReLULayer> activ_chprelu = activ.dynamicCast<ChannelsPReLULayer>();
1263             if( !activ_chprelu.empty() )
1264             {
1265                 const Mat& m = activ_chprelu->blobs[0];
1266                 CV_Assert(m.isContinuous() && m.type() == CV_32F && (int)m.total() == outCn);
1267                 const float* mdata = m.ptr<float>();
1268                 reluslope.resize(outCn+2);
1269                 std::copy(mdata, mdata + outCn, reluslope.begin());
1270                 reluslope[outCn] = reluslope[outCn+1] = reluslope[outCn-1];
1271             }
1272         }
1273
1274         int nstripes = std::max(getNumThreads(), 1);
1275
1276         ParallelConv::run(inputs[0], outputs[0], weightsMat, biasvec, reluslope,
1277                           kernel_size, strides, pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes);
1278     }
1279
1280     virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
1281                            const std::vector<MatShape> &outputs) const CV_OVERRIDE
1282     {
1283         CV_Assert(inputs.size() == outputs.size());
1284
1285         int64 flops = 0;
1286         int karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies<size_t>());
1287         for (int i = 0; i < inputs.size(); i++)
1288         {
1289             flops += total(outputs[i])*(CV_BIG_INT(2)*karea*inputs[i][1] + 1);
1290         }
1291
1292         return flops;
1293     }
1294 };
1295
1296 class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl
1297 {
1298 public:
1299     Mat weightsMat, biasesMat;
1300     UMat umat_weights;
1301     UMat umat_biases;
1302
1303     DeConvolutionLayerImpl(const LayerParams& params) : BaseConvolutionLayerImpl(params) {}
1304
1305     MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE
1306     {
1307         int inpCn = inpShape[1];
1308         int inpH = inpShape[2];
1309         int inpW = inpShape[3];
1310         int outCn = outShape[1];
1311         int ngroups = inpCn / blobs[0].size[0];
1312         int outGroupCn = outCn / ngroups;
1313         int ksize = outGroupCn * kernel.height * kernel.width;
1314         return shape(ksize, inpH * inpW);
1315     }
1316
1317     virtual bool supportBackend(int backendId) CV_OVERRIDE
1318     {
1319 #ifdef HAVE_INF_ENGINE
1320         const int outGroupCn = blobs[0].size[1];  // Weights are in IOHW or IODHW layout
1321         const int group = numOutput / outGroupCn;
1322
1323         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
1324         {
1325             if (kernel_size.size() == 3 && preferableTarget != DNN_TARGET_CPU) {
1326                 return false;
1327             }
1328
1329             if (std::accumulate(adjust_pads.begin(), adjust_pads.end(), 0, std::plus<size_t>()) > 0)
1330             {
1331                 if (padMode.empty())
1332                 {
1333                     if (preferableTarget != DNN_TARGET_CPU && group != 1)
1334                     {
1335                         for (int i = 0; i < adjust_pads.size(); i++) {
1336                             if (adjust_pads[i] && pads_begin[i])
1337                                 return false;
1338                         }
1339                     }
1340                     for (int i = 0; i < adjust_pads.size(); i++) {
1341                         if (pads_end[i] < adjust_pads[i])
1342                             return false;
1343                     }
1344                     return true;
1345                 }
1346                 else if (padMode == "SAME")
1347                 {
1348                     for (int i = 0; i < adjust_pads.size(); i++) {
1349                         if (kernel_size[i] < pads_begin[i] + 1 + adjust_pads[i])
1350                             return false;
1351                     }
1352                     return true;
1353                 }
1354                 else if (padMode == "VALID")
1355                     return false;
1356             }
1357
1358             if (group != 1)
1359             {
1360                 return preferableTarget == DNN_TARGET_CPU;
1361             }
1362             if (preferableTarget == DNN_TARGET_OPENCL || preferableTarget == DNN_TARGET_OPENCL_FP16)
1363                 return std::accumulate(dilations.begin(), dilations.end(), 1, std::multiplies<size_t>()) == 1;
1364             return true;
1365         }
1366         else
1367 #endif  // HAVE_INF_ENGINE
1368             return kernel_size.size() == 2 && (backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE);
1369     }
1370
1371     bool getMemoryShapes(const std::vector<MatShape> &inputs,
1372                          const int requiredOutputs,
1373                          std::vector<MatShape> &outputs,
1374                          std::vector<MatShape> &internals) const CV_OVERRIDE
1375     {
1376         CV_Assert(!hasBias() || blobs[1].total() == (size_t)numOutput);
1377         CV_Assert(inputs.size() != 0);
1378
1379         int outCn = numOutput;
1380         std::vector<int> outShape;
1381         outShape.push_back(inputs[0][0]);  // batch
1382         outShape.push_back(outCn);
1383         if (padMode.empty())
1384         {
1385             for (int i = 0; i < kernel_size.size(); i++)
1386                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + kernel_size[i] - pads_begin[i] - pads_end[i] + adjust_pads[i]);
1387         }
1388         else if (padMode == "VALID")
1389         {
1390             for (int i = 0; i < kernel_size.size(); i++)
1391                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + kernel_size[i] + adjust_pads[i]);
1392         }
1393         else if (padMode == "SAME")
1394         {
1395             for (int i = 0; i < kernel_size.size(); i++)
1396                 outShape.push_back(strides[i] * (inputs[0][2 + i] - 1) + 1 + adjust_pads[i]);
1397         }
1398         else
1399             CV_Error(Error::StsError, "Unsupported padding mode " + padMode);
1400
1401         CV_Assert(outCn % blobs[0].size[1] == 0);
1402         int ngroups = outCn / blobs[0].size[1];
1403
1404         int inpCn = inputs[0][1];
1405         CV_Assert(inpCn % ngroups == 0 && outCn % ngroups == 0);
1406         CV_Assert(blobs[0].size[0] == inpCn);
1407
1408         outputs.resize(1, outShape);
1409
1410         if (!is1x1())
1411             internals.push_back(computeColRowShape(inputs[0], outputs[0]));
1412
1413         return false;
1414     }
1415
1416     void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
1417     {
1418         BaseConvolutionLayerImpl::finalize(inputs_arr, outputs_arr);
1419
1420         std::vector<Mat> inputs, outputs;
1421         inputs_arr.getMatVector(inputs);
1422         outputs_arr.getMatVector(outputs);
1423
1424         std::vector<int> inpShape;
1425         std::vector<int> outShape;
1426         for (int i = 2; i < inputs[0].dims; i++) {
1427             inpShape.push_back(inputs[0].size[i]);
1428             outShape.push_back(outputs[0].size[i]);
1429         }
1430         getConvPoolPaddings(outShape, kernel_size, strides, padMode, pads_begin, pads_end);
1431         if (pads_begin.size() == 2) {
1432             for (int i = 0; i < pads_begin.size(); i++) {
1433                 if (pads_begin[i] != pads_end[i])
1434                     CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in deconvolution layer");
1435             }
1436             pad = Size(pads_begin[1], pads_begin[0]);
1437         }
1438
1439         weightsMultipliers.assign(numOutput, 1.0);
1440         if (weightsMat.empty())
1441         {
1442             transpose(blobs[0].reshape(1, blobs[0].size[0]), weightsMat);
1443             biasesMat = hasBias() ? blobs[1].reshape(1, numOutput)
1444                                   : Mat::zeros(numOutput, 1, CV_32F);
1445         }
1446     }
1447
1448     void fuseWeights(const Mat& w_, const Mat& b_) CV_OVERRIDE
1449     {
1450         Mat w = w_.total() == 1 ? Mat(1, numOutput, CV_32F, Scalar(w_.at<float>(0))) : w_;
1451         Mat b = b_.total() == 1 ? Mat(1, numOutput, CV_32F, Scalar(b_.at<float>(0))) : b_;
1452
1453         CV_Assert_N(!weightsMat.empty(),
1454                      w.empty() || numOutput == w.total(),
1455                      b.empty() || numOutput == b.total());
1456
1457         if (!w.empty())
1458         {
1459             transpose(blobs[0].reshape(1, blobs[0].size[0]), weightsMat);
1460             weightsMat = weightsMat.reshape(1, numOutput);
1461             for (int i = 0; i < numOutput; ++i)
1462             {
1463                 double wi = w.at<float>(i);
1464                 weightsMultipliers[i] *= wi;
1465                 cv::multiply(weightsMat.row(i), weightsMultipliers[i], weightsMat.row(i));
1466                 biasesMat.at<float>(i) *= wi;
1467             }
1468             weightsMat = weightsMat.reshape(1, weightsMat.total() / blobs[0].size[0]);
1469         }
1470
1471         if (!b.empty())
1472         {
1473             cv::add(biasesMat, b.reshape(1, numOutput), biasesMat);
1474         }
1475     }
1476
1477     class MatMulInvoker : public ParallelLoopBody
1478     {
1479     public:
1480         MatMulInvoker(const Mat& a, const Mat& b, Mat& c, int nstripes)
1481         {
1482             a_ = &a;
1483             b_ = &b;
1484             c_ = &c;
1485             nstripes_ = nstripes;
1486             useAVX = checkHardwareSupport(CPU_AVX);
1487             useAVX2 = checkHardwareSupport(CPU_AVX2);
1488             useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
1489         }
1490
1491         void operator()(const Range& range_) const CV_OVERRIDE
1492         {
1493             int stripeSize = (int)alignSize((b_->cols + nstripes_ - 1)/nstripes_, 16);
1494             Range range(range_.start*stripeSize, std::min(range_.end*stripeSize, b_->cols));
1495             int mmax = a_->rows;
1496             int nmax = range.end - range.start;
1497             int kmax = a_->cols;
1498             int m, n, k;
1499             const float* aptr = a_->ptr<float>();
1500             const float* bptr = b_->ptr<float>() + range.start;
1501             float* cptr = c_->ptr<float>() + range.start;
1502             size_t astep = a_->step1();
1503             size_t bstep = b_->step1();
1504             size_t cstep = c_->step1();
1505
1506         #if CV_TRY_AVX512_SKX
1507             if( useAVX512 )
1508                 opt_AVX512_SKX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1509             else
1510         #endif
1511         #if CV_TRY_AVX2
1512             if( useAVX2 )
1513                 opt_AVX2::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1514             else
1515         #endif
1516         #if CV_TRY_AVX
1517             if( useAVX )
1518                 opt_AVX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1519             else
1520         #endif
1521             for( m = 0; m < mmax; m += 2 )
1522             {
1523                 float* dst0 = cptr + cstep*m;
1524                 float* dst1 = cptr + cstep*std::min(m+1, mmax-1);
1525                 const float* aptr0 = aptr + astep*m;
1526                 const float* aptr1 = aptr + astep*std::min(m+1, mmax-1);
1527
1528                 for( n = 0; n < nmax; n++ )
1529                 {
1530                     dst0[n] = 0.f;
1531                     dst1[n] = 0.f;
1532                 }
1533
1534                 for( k = 0; k < kmax; k += 4 )
1535                 {
1536                     float alpha00 = aptr0[k];
1537                     float alpha01 = aptr1[k];
1538                     float alpha10 = 0.f, alpha11 = 0.f;
1539                     float alpha20 = 0.f, alpha21 = 0.f;
1540                     float alpha30 = 0.f, alpha31 = 0.f;
1541                     const float* bptr0 = bptr + k*bstep;
1542                     const float* bptr1 = bptr0;
1543                     const float* bptr2 = bptr0;
1544                     const float* bptr3 = bptr0;
1545
1546                     if( k+1 < kmax )
1547                     {
1548                         alpha10 = aptr0[k+1];
1549                         alpha11 = aptr1[k+1];
1550                         bptr1 = bptr0 + bstep;
1551                         if( k+2 < kmax )
1552                         {
1553                             alpha20 = aptr0[k+2];
1554                             alpha21 = aptr1[k+2];
1555                             bptr2 = bptr1 + bstep;
1556                             if( k+3 < kmax )
1557                             {
1558                                 alpha30 = aptr0[k+3];
1559                                 alpha31 = aptr1[k+3];
1560                                 bptr3 = bptr2 + bstep;
1561                             }
1562                         }
1563                     }
1564                     n = 0;
1565
1566                 #if CV_SIMD128
1567                     v_float32x4 a00 = v_setall_f32(alpha00);
1568                     v_float32x4 a01 = v_setall_f32(alpha01);
1569                     v_float32x4 a10 = v_setall_f32(alpha10);
1570                     v_float32x4 a11 = v_setall_f32(alpha11);
1571                     v_float32x4 a20 = v_setall_f32(alpha20);
1572                     v_float32x4 a21 = v_setall_f32(alpha21);
1573                     v_float32x4 a30 = v_setall_f32(alpha30);
1574                     v_float32x4 a31 = v_setall_f32(alpha31);
1575
1576                     for( ; n <= nmax - 4; n += 4 )
1577                     {
1578                         v_float32x4 b0 = v_load(bptr0 + n);
1579                         v_float32x4 b1 = v_load(bptr1 + n);
1580                         v_float32x4 b2 = v_load(bptr2 + n);
1581                         v_float32x4 b3 = v_load(bptr3 + n);
1582                         v_float32x4 d0 = v_load(dst0 + n);
1583                         v_float32x4 d1 = v_load(dst1 + n);
1584                         d0 += b0*a00;
1585                         d1 += b0*a01;
1586                         d0 += b1*a10;
1587                         d1 += b1*a11;
1588                         d0 += b2*a20;
1589                         d1 += b2*a21;
1590                         d0 += b3*a30;
1591                         d1 += b3*a31;
1592                         v_store(dst0 + n, d0);
1593                         v_store(dst1 + n, d1);
1594                     }
1595                 #endif
1596
1597                     for( ; n < nmax; n++ )
1598                     {
1599                         float b0 = bptr0[n], b1 = bptr1[n];
1600                         float b2 = bptr2[n], b3 = bptr3[n];
1601                         float d0 = dst0[n] + alpha00*b0 + alpha10*b1 + alpha20*b2 + alpha30*b3;
1602                         float d1 = dst1[n] + alpha01*b0 + alpha11*b1 + alpha21*b2 + alpha31*b3;
1603                         dst0[n] = d0;
1604                         dst1[n] = d1;
1605                     }
1606                 }
1607             }
1608         }
1609
1610         const Mat *a_, *b_;
1611         Mat* c_;
1612         int nstripes_;
1613         bool useAVX;
1614         bool useAVX2;
1615         bool useAVX512;
1616     };
1617
1618     class Col2ImInvoker : public cv::ParallelLoopBody
1619     {
1620     public:
1621         const float* data_col;
1622         const float* biasvec;
1623         int channels, height, width;
1624         int kernel_h, kernel_w;
1625         int pad_h, pad_w;
1626         int stride_h, stride_w;
1627         float* data_im;
1628         int height_col, width_col;
1629         int nstripes;
1630         bool is1x1;
1631
1632         Col2ImInvoker()
1633             : data_col(0), biasvec(0), channels(0), height(0), width(0),
1634               kernel_h(0), kernel_w(0), pad_h(0), pad_w(0), stride_h(0), stride_w(0), data_im(0),
1635               height_col(0), width_col(0), nstripes(0), is1x1(0)
1636         {}
1637
1638         static void run(const float* data_col,
1639                         int channels, int height, int width,
1640                         int kernel_h, int kernel_w,
1641                         int pad_h, int pad_w,
1642                         int stride_h, int stride_w,
1643                         int height_col, int width_col,
1644                         float* data_im,
1645                         const float* biasvec,
1646                         bool is1x1)
1647         {
1648             const int nstripes = getNumThreads();
1649
1650             Col2ImInvoker t;
1651             t.data_col = data_col;
1652             t.data_im = data_im;
1653             t.channels = channels; t.height = height; t.width = width;
1654             t.kernel_h = kernel_h; t.kernel_w = kernel_w;
1655             t.pad_h = pad_h; t.pad_w = pad_w;
1656             t.stride_h = stride_h; t.stride_w = stride_w;
1657             t.height_col = height_col;
1658             t.width_col = width_col;
1659             t.nstripes = nstripes;
1660             t.is1x1 = is1x1;
1661             t.biasvec = biasvec;
1662
1663             parallel_for_(Range(0, nstripes), t, nstripes);
1664         }
1665
1666         virtual void operator ()(const Range &r) const CV_OVERRIDE
1667         {
1668             const float* data_col_ = data_col;
1669             float* data_im_ = data_im;
1670             int coeff_h = (1 - stride_h * kernel_w * height_col) * width_col;
1671             int coeff_w = (1 - stride_w * height_col * width_col);
1672             size_t total = (size_t)channels * height * width;
1673             size_t stripeSize = (total + nstripes - 1)/nstripes;
1674             size_t startIndex = r.start*stripeSize;
1675             size_t endIndex = std::min(r.end*stripeSize, total);
1676             int w = (int)(startIndex % width + pad_w);
1677             int h = (int)((startIndex / width) % height + pad_h);
1678             int c = (int)(startIndex / (width * height));
1679             int h_col_start = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1;
1680             int h_col_end = std::min(h / stride_h + 1, height_col);
1681             int plane_size_col = height_col * width_col;
1682             int offset = (c * kernel_h * kernel_w + h * kernel_w + w) * plane_size_col;
1683             bool is1x1_ = is1x1;
1684             const float* biasvec_ = biasvec;
1685
1686             for (size_t index = startIndex; index < endIndex; index++)
1687             {
1688                 // compute the start and end of the output
1689                 int w_col_start = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1;
1690                 int w_col_end = std::min(w / stride_w + 1, width_col);
1691                 float val;
1692
1693                 if( is1x1_ )
1694                     val = data_im_[index];
1695                 else
1696                 {
1697                     val = 0.f;
1698                     for (int h_col = h_col_start; h_col < h_col_end; ++h_col) {
1699                         for (int w_col = w_col_start; w_col < w_col_end; ++w_col) {
1700                             val += data_col_[offset + h_col * coeff_h + w_col * coeff_w];
1701                         }
1702                     }
1703                 }
1704                 data_im_[index] = val + biasvec_[c];
1705
1706                 offset += plane_size_col;
1707                 if( ++w >= width + pad_w )
1708                 {
1709                     w = (int)((index + 1)% width + pad_w);
1710                     h = (int)(((index + 1) / width) % height + pad_h);
1711                     c = (int)((index + 1) / (width * height));
1712                     h_col_start = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1;
1713                     h_col_end = std::min(h / stride_h + 1, height_col);
1714                     offset = (c * kernel_h * kernel_w + h * kernel_w + w) * plane_size_col;
1715                 }
1716             }
1717         }
1718     };
1719
1720 #ifdef HAVE_OPENCL
1721     bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_)
1722     {
1723         std::vector<UMat> inputs;
1724         std::vector<UMat> outputs;
1725         std::vector<UMat> internals;
1726
1727         if (inputs_.depth() == CV_16S)
1728             return false;
1729
1730         inputs_.getUMatVector(inputs);
1731         outputs_.getUMatVector(outputs);
1732         internals_.getUMatVector(internals);
1733
1734         int outCn = numOutput;
1735         int inpCn = inputs[0].size[1];
1736
1737         if (is1x1())
1738             return false;
1739
1740         if (umat_weights.empty())
1741         {
1742             if (fusedWeights)
1743                 weightsMat.copyTo(umat_weights);
1744             else
1745                 transpose(blobs[0].reshape(1, inpCn), umat_weights);
1746
1747             if (fusedBias)
1748                 biasesMat.copyTo(umat_biases);
1749             else
1750             {
1751                 if (hasBias())
1752                     blobs[1].reshape(1, outCn).copyTo(umat_biases);
1753                 else
1754                     umat_biases = UMat::zeros(outCn, 1, CV_32F);
1755             }
1756         }
1757
1758         String buildopt = format("-DT=%s ", ocl::typeToStr(inputs[0].type()));
1759         buildopt += format("-DPAD_H=%d -DPAD_W=%d -DKERNEL_H=%d -DKERNEL_W=%d -DSTRIDE_H=%d -DSTRIDE_W=%d ",
1760                            pad.height, pad.width, kernel.height, kernel.width, stride.height, stride.width);
1761
1762         for (size_t ii = 0; ii < outputs.size(); ii++)
1763         {
1764             int ngroups = outCn / blobs[0].size[1];
1765             int inpGroupCn = inpCn / ngroups;
1766             int outGroupCn = blobs[0].size[1];
1767             const UMat& inp = inputs[ii];
1768             UMat& out = outputs[ii];
1769             int numImg = inp.size[0];
1770             int inpH = inp.size[2], inpW = inp.size[3];
1771             int outH = out.size[2], outW = out.size[3];
1772
1773             MatShape inpshape = shape(numImg*inpCn, inpH*inpW);
1774             MatShape outshape = shape(numImg*outCn, outH*outW);
1775             UMat convBlob = inputs[ii].reshape(1, inpshape.size(), &inpshape[0]);
1776             UMat decnBlob = out.reshape(1, outshape.size(), &outshape[0]);
1777             int rows = internals[0].rows / ngroups;
1778
1779             for (int n = 0; n < numImg; n++)
1780             {
1781                 for (int g = 0; g < ngroups; g++)
1782                 {
1783                     UMat colMat = internals[0].rowRange(_Range(g * rows, rows));
1784                     UMat convMat = convBlob.rowRange(_Range((g + n * ngroups) * inpGroupCn, inpGroupCn));
1785                     UMat wghtMat = umat_weights.colRange(_Range(g * inpGroupCn, inpGroupCn));
1786                     gemm(wghtMat, convMat, 1, noArray(), 0, colMat, 0);
1787                 }
1788
1789                 for (int g = 0; g < ngroups; g++)
1790                 {
1791                     int total = outGroupCn * decnBlob.cols;
1792                     int index = 0;
1793                     int height_col = inpH;
1794                     int width_col = inpW;
1795                     int coeff_h = (1 - stride.height * kernel.width * height_col) * width_col;
1796                     int coeff_w = (1 - stride.width * height_col * width_col);
1797
1798                     ocl::Kernel k("col2im", ocl::dnn::col2im_oclsrc, buildopt);
1799                     k.set(index++, total);
1800                     k.set(index++, ocl::KernelArg::PtrReadOnly(internals[0]));
1801                     k.set(index++, (int)(g * rows * internals[0].cols));
1802                     k.set(index++, outGroupCn);
1803                     k.set(index++, outH);
1804                     k.set(index++, outW);
1805                     k.set(index++, height_col);
1806                     k.set(index++, width_col);
1807                     k.set(index++, coeff_h);
1808                     k.set(index++, coeff_w);
1809                     k.set(index++, ocl::KernelArg::PtrReadOnly(umat_biases));
1810                     k.set(index++, (int)(g * outGroupCn * umat_biases.cols));
1811                     k.set(index++, ocl::KernelArg::PtrWriteOnly(decnBlob));
1812                     k.set(index++, (int)((g + n * ngroups) * outGroupCn * decnBlob.cols));
1813
1814                     size_t global[] = { (size_t)total };
1815                     bool ret = k.run(1, global, NULL, false);
1816                     if (!ret)
1817                         return false;
1818                 }
1819             }
1820         }
1821
1822         return true;
1823     }
1824 #endif
1825
1826     void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
1827     {
1828         CV_TRACE_FUNCTION();
1829         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
1830
1831         CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
1832                    forward_ocl(inputs_arr, outputs_arr, internals_arr));
1833
1834         if (inputs_arr.depth() == CV_16S)
1835         {
1836             forward_fallback(inputs_arr, outputs_arr, internals_arr);
1837             return;
1838         }
1839
1840         std::vector<Mat> inputs, outputs, internals;
1841         inputs_arr.getMatVector(inputs);
1842         outputs_arr.getMatVector(outputs);
1843         internals_arr.getMatVector(internals);
1844
1845         int outCn = numOutput;
1846         int inpCn = inputs[0].size[1];
1847         bool is1x1flag = is1x1();
1848         int nstripes = getNumThreads();
1849
1850         if( weightsMat.empty() )
1851         {
1852             transpose(blobs[0].reshape(1, inpCn), weightsMat);
1853             biasesMat = hasBias() ? blobs[1].reshape(1, outCn) : Mat::zeros(outCn, 1, CV_32F);
1854         }
1855
1856         for (size_t ii = 0; ii < outputs.size(); ii++)
1857         {
1858             int ngroups = outCn / blobs[0].size[1];
1859             int inpGroupCn = inpCn / ngroups;
1860             int outGroupCn = blobs[0].size[1];
1861             const Mat& inp = inputs[ii];
1862             Mat& out = outputs[ii];
1863             int numImg = inp.size[0];
1864             int inpH = inp.size[2], inpW = inp.size[3];
1865             int outH = out.size[2], outW = out.size[3];
1866
1867             Mat convBlob = inputs[ii].reshape(1, numImg*inpCn);
1868             Mat decnBlob = out.reshape(1, numImg*outCn);
1869
1870             for (int n = 0; n < numImg; n++)
1871             {
1872                 for (int g = 0; g < ngroups; g++)
1873                 {
1874                     Mat dstMat = decnBlob.rowRange(_Range((g + n * ngroups) * outGroupCn, outGroupCn));
1875                     Mat &colMat = is1x1flag ? dstMat : internals[0];
1876
1877                     Mat convMat = convBlob.rowRange(_Range((g + n * ngroups) * inpGroupCn, inpGroupCn));
1878                     Mat wghtMat = weightsMat.colRange(_Range(g * inpGroupCn, inpGroupCn));
1879                     Mat curBiasMat = biasesMat.rowRange(_Range(g * outGroupCn, outGroupCn));
1880
1881                     //gemm(wghtMat, convMat, 1, colMat, 0, colMat, 0);
1882                     MatMulInvoker mminvoker(wghtMat, convMat, colMat, nstripes);
1883                     parallel_for_(Range(0, nstripes), mminvoker, nstripes);
1884
1885                     Col2ImInvoker::run(colMat.ptr<float>(), outGroupCn, outH, outW,
1886                                        kernel.height, kernel.width, pad.height, pad.width,
1887                                        stride.height, stride.width, inpH, inpW, dstMat.ptr<float>(),
1888                                        curBiasMat.ptr<float>(), is1x1flag);
1889                 }
1890             }
1891         }
1892     }
1893
1894     virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
1895     {
1896 #ifdef HAVE_HALIDE
1897         Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
1898
1899         int inW, inH, inC, inN;
1900         getCanonicalSize(inputBuffer, &inW, &inH, &inC, &inN);
1901         const int outGroupCn = blobs[0].size[1];
1902         const int group = numOutput / outGroupCn;
1903         const int inpGroupCn = blobs[0].size[0] / group;
1904
1905         Halide::Var x("x"), y("y"), c("c"), n("n");
1906         Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
1907         Halide::Func padded_input(name + "_constant_exterior");
1908         auto weights = wrapToHalideBuffer(blobs[0]);
1909
1910         Halide::Func dilated_input("dilated_input");
1911         dilated_input(x, y, c, n) = 0.0f;
1912         Halide::RDom r1(0, inW, 0, inH);
1913         dilated_input(r1.x * stride.width, r1.y * stride.height, c, n) =
1914               inputBuffer(r1.x, r1.y, c, n);
1915         dilated_input.compute_root();
1916
1917         Halide::Func bounded =
1918             Halide::BoundaryConditions::constant_exterior(dilated_input, 0,
1919                                                           0, (inW - 1) * stride.width + 1,
1920                                                           0, (inH - 1) * stride.height + 1,
1921                                                           0, inC, 0, inN);
1922         padded_input(x, y, c, n) = bounded(x, y, c, n);
1923
1924         Halide::RDom r(0, kernel.width, 0, kernel.height, 0, inpGroupCn);
1925         Halide::Expr kx = x + pad.width - r.x;
1926         Halide::Expr ky = y + pad.height - r.y;
1927         Halide::Expr kInC = r.z;
1928         Halide::Expr kOutC = c;
1929         for (int i = 1; i < group; ++i)
1930         {
1931             kInC = select(c < outGroupCn * i, kInC, inpGroupCn * i + r.z);
1932             kOutC = select(c < outGroupCn * i, kOutC, c - outGroupCn * i);
1933         }
1934         Halide::Expr topExpr = sum(padded_input(kx, ky, kInC, n) *
1935                                    weights(r.x, r.y, kOutC, kInC));
1936         if (hasBias())
1937         {
1938             auto bias = wrapToHalideBuffer(blobs[1], {numOutput});
1939             topExpr += bias(c);
1940         }
1941         top(x, y, c, n) = topExpr;
1942         return Ptr<BackendNode>(new HalideBackendNode({ padded_input, top }));
1943 #endif  // HAVE_HALIDE
1944         return Ptr<BackendNode>();
1945     }
1946
1947 #ifdef HAVE_INF_ENGINE
1948     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &) CV_OVERRIDE
1949     {
1950         InferenceEngine::Layout layout = blobs[0].dims == 5? InferenceEngine::Layout::NCDHW :
1951                                                              InferenceEngine::Layout::OIHW;
1952
1953         auto ieWeights = wrapToInfEngineBlob(blobs[0], layout);
1954         if (fusedWeights)
1955         {
1956             ieWeights = InferenceEngine::make_shared_blob<float>(
1957                                 InferenceEngine::Precision::FP32, layout,
1958                                 ieWeights->dims());
1959             ieWeights->allocate();
1960
1961             int inpCn = blobs[0].size[0];
1962             Mat newWeights = infEngineBlobToMat(ieWeights).reshape(1, inpCn);
1963             transpose(weightsMat, newWeights);
1964         }
1965
1966         const int outGroupCn = blobs[0].size[1];  // Weights are in IOHW or OIDHW layout
1967         const int group = numOutput / outGroupCn;
1968
1969         InferenceEngine::Builder::DeconvolutionLayer ieLayer(name);
1970
1971         ieLayer.setKernel(kernel_size);
1972         ieLayer.setStrides(strides);
1973         ieLayer.setDilation(dilations);
1974         ieLayer.setPaddingsBegin(pads_begin);
1975
1976         if (padMode.empty())
1977         {
1978             std::vector<size_t> paddings_end;
1979             for (int i = 0; i < pads_end.size(); i++) {
1980                 paddings_end.push_back(pads_end[i] - adjust_pads[i]);
1981             }
1982             ieLayer.setPaddingsEnd(paddings_end);
1983         }
1984         else if (padMode == "SAME")
1985         {
1986             std::vector<size_t> paddings_end;
1987             for (int i = 0; i < pads_begin.size(); i++) {
1988                 paddings_end.push_back(kernel_size[i] - pads_begin[i] - 1 - adjust_pads[i]);
1989             }
1990             ieLayer.setPaddingsEnd(paddings_end);
1991         }
1992         ieLayer.setGroup((size_t)group);
1993         ieLayer.setOutDepth((size_t)numOutput);
1994
1995         InferenceEngine::Builder::Layer l = ieLayer;
1996         addConstantData("weights", ieWeights, l);
1997         if (hasBias())
1998             addConstantData("biases", wrapToInfEngineBlob(biasesMat, {(size_t)numOutput}, InferenceEngine::Layout::C), l);
1999         return Ptr<BackendNode>(new InfEngineBackendNode(l));
2000     }
2001 #endif  // HAVE_INF_ENGINE
2002
2003     virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
2004                            const std::vector<MatShape> &outputs) const CV_OVERRIDE
2005     {
2006         CV_Assert(inputs.size() == outputs.size());
2007
2008         float flops = 0;
2009         int outChannels = blobs[0].size[0];
2010         size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(),
2011                                        1, std::multiplies<size_t>());
2012
2013         for (int i = 0; i < inputs.size(); i++)
2014         {
2015             flops += CV_BIG_INT(2)*outChannels*karea*total(inputs[i]);
2016         }
2017
2018         return flops;
2019     }
2020 };
2021
2022 Ptr<BaseConvolutionLayer> ConvolutionLayer::create(const LayerParams &params)
2023 {
2024     Ptr<ConvolutionLayerImpl> l(new ConvolutionLayerImpl(params));
2025     return l;
2026 }
2027
2028 Ptr<BaseConvolutionLayer> DeconvolutionLayer::create(const LayerParams &params)
2029 {
2030     return Ptr<BaseConvolutionLayer>(new DeConvolutionLayerImpl(params));
2031 }
2032
2033 }
2034 }