modules/dnn/src/layers/resize_layer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7 #include "../precomp.hpp"
   8 #include "layers_common.hpp"
   9 #include "../op_cuda.hpp"
  10 #include "../op_inf_engine.hpp"
  11 #include <opencv2/imgproc.hpp>
  12
  13 #ifdef HAVE_CUDA
  14 #include "../cuda4dnn/primitives/resize.hpp"
  15 using namespace cv::dnn::cuda4dnn;
  16 #endif
  17
  18 namespace cv { namespace dnn {
  19
  20 class ResizeLayerImpl : public ResizeLayer
  21 {
  22 public:
  23     ResizeLayerImpl(const LayerParams& params) : zoomFactorWidth(0), zoomFactorHeight(0), scaleWidth(0), scaleHeight(0)
  24     {
  25         setParamsFrom(params);
  26         outWidth = params.get<float>("width", 0);
  27         outHeight = params.get<float>("height", 0);
  28         if (params.has("zoom_factor"))
  29         {
  30             CV_Assert(!params.has("zoom_factor_x") && !params.has("zoom_factor_y"));
  31             zoomFactorWidth = zoomFactorHeight = params.get<int>("zoom_factor");
  32         }
  33         else if (params.has("zoom_factor_x") || params.has("zoom_factor_y"))
  34         {
  35             CV_Assert(params.has("zoom_factor_x") && params.has("zoom_factor_y"));
  36             zoomFactorWidth = params.get<int>("zoom_factor_x");
  37             zoomFactorHeight = params.get<int>("zoom_factor_y");
  38         }
  39         interpolation = params.get<String>("interpolation");
  40         CV_Assert(interpolation == "nearest" || interpolation == "bilinear");
  41
  42         alignCorners = params.get<bool>("align_corners", false);
  43     }
  44
  45     bool getMemoryShapes(const std::vector<MatShape> &inputs,
  46                          const int requiredOutputs,
  47                          std::vector<MatShape> &outputs,
  48                          std::vector<MatShape> &internals) const CV_OVERRIDE
  49     {
  50         CV_Assert_N(inputs.size() == 1, inputs[0].size() == 4);
  51         outputs.resize(1, inputs[0]);
  52         outputs[0][2] = outHeight > 0 ? outHeight : (outputs[0][2] * zoomFactorHeight);
  53         outputs[0][3] = outWidth > 0 ? outWidth : (outputs[0][3] * zoomFactorWidth);
  54         // We can work in-place (do nothing) if input shape == output shape.
  55         return (outputs[0][2] == inputs[0][2]) && (outputs[0][3] == inputs[0][3]);
  56     }
  57
  58     virtual bool supportBackend(int backendId) CV_OVERRIDE
  59     {
  60         if (backendId == DNN_BACKEND_CUDA)
  61             return interpolation == "nearest" || interpolation == "bilinear";
  62
  63 #ifdef HAVE_INF_ENGINE
  64         if (backendId == DNN_BACKEND_INFERENCE_ENGINE)
  65         {
  66             return (interpolation == "nearest" && scaleWidth == scaleHeight) ||
  67                    (interpolation == "bilinear");
  68         }
  69 #endif
  70         return backendId == DNN_BACKEND_OPENCV;
  71     }
  72
  73     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
  74     {
  75         std::vector<Mat> inputs, outputs;
  76         inputs_arr.getMatVector(inputs);
  77         outputs_arr.getMatVector(outputs);
  78
  79         if (!outWidth && !outHeight)
  80         {
  81             outHeight = outputs[0].size[2];
  82             outWidth = outputs[0].size[3];
  83         }
  84         if (alignCorners && outHeight > 1)
  85             scaleHeight = static_cast<float>(inputs[0].size[2] - 1) / (outHeight - 1);
  86         else
  87             scaleHeight = static_cast<float>(inputs[0].size[2]) / outHeight;
  88
  89         if (alignCorners && outWidth > 1)
  90             scaleWidth = static_cast<float>(inputs[0].size[3] - 1) / (outWidth - 1);
  91         else
  92             scaleWidth = static_cast<float>(inputs[0].size[3]) / outWidth;
  93     }
  94
  95     void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
  96     {
  97         CV_TRACE_FUNCTION();
  98         CV_TRACE_ARG_VALUE(name, "name", name.c_str());
  99
 100         if (inputs_arr.depth() == CV_16S)
 101         {
 102             forward_fallback(inputs_arr, outputs_arr, internals_arr);
 103             return;
 104         }
 105
 106         std::vector<Mat> inputs, outputs, internals;
 107         inputs_arr.getMatVector(inputs);
 108         outputs_arr.getMatVector(outputs);
 109         internals_arr.getMatVector(internals);
 110
 111         if (outHeight == inputs[0].size[2] && outWidth == inputs[0].size[3])
 112             return;
 113
 114         Mat& inp = inputs[0];
 115         Mat& out = outputs[0];
 116         if (interpolation == "nearest")
 117         {
 118             for (size_t n = 0; n < inputs[0].size[0]; ++n)
 119             {
 120                 for (size_t ch = 0; ch < inputs[0].size[1]; ++ch)
 121                 {
 122                     resize(getPlane(inp, n, ch), getPlane(out, n, ch),
 123                            Size(outWidth, outHeight), 0, 0, INTER_NEAREST);
 124                 }
 125             }
 126         }
 127         else if (interpolation == "bilinear")
 128         {
 129             const int inpHeight = inp.size[2];
 130             const int inpWidth = inp.size[3];
 131             const int inpSpatialSize = inpHeight * inpWidth;
 132             const int outSpatialSize = outHeight * outWidth;
 133             const int numPlanes = inp.size[0] * inp.size[1];
 134             CV_Assert_N(inp.isContinuous(), out.isContinuous());
 135
 136             Mat inpPlanes = inp.reshape(1, numPlanes * inpHeight);
 137             Mat outPlanes = out.reshape(1, numPlanes * outHeight);
 138             for (int y = 0; y < outHeight; ++y)
 139             {
 140                 float input_y = y * scaleHeight;
 141                 int y0 = static_cast<int>(input_y);
 142                 const float* inpData_row0 = inpPlanes.ptr<float>(y0);
 143                 const float* inpData_row1 = inpPlanes.ptr<float>(std::min(y0 + 1, inpHeight - 1));
 144                 for (int x = 0; x < outWidth; ++x)
 145                 {
 146                     float input_x = x * scaleWidth;
 147                     int x0 = static_cast<int>(input_x);
 148                     int x1 = std::min(x0 + 1, inpWidth - 1);
 149
 150                     float* outData = outPlanes.ptr<float>(y, x);
 151                     const float* inpData_row0_c = inpData_row0;
 152                     const float* inpData_row1_c = inpData_row1;
 153                     for (int c = 0; c < numPlanes; ++c)
 154                     {
 155                         *outData = inpData_row0_c[x0] +
 156                             (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) +
 157                             (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] +
 158                             (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0]));
 159
 160                         inpData_row0_c += inpSpatialSize;
 161                         inpData_row1_c += inpSpatialSize;
 162                         outData += outSpatialSize;
 163                     }
 164                 }
 165             }
 166         }
 167         else
 168             CV_Error(Error::StsNotImplemented, "Unknown interpolation: " + interpolation);
 169     }
 170
 171 #ifdef HAVE_CUDA
 172     Ptr<BackendNode> initCUDA(
 173         void *context_,
 174         const std::vector<Ptr<BackendWrapper>>& inputs,
 175         const std::vector<Ptr<BackendWrapper>>& outputs
 176     ) override
 177     {
 178         auto context = reinterpret_cast<csl::CSLContext*>(context_);
 179
 180         cuda4dnn::InterpolationType itype;
 181         if (interpolation == "nearest")
 182             itype = InterpolationType::NEAREST_NEIGHBOUR;
 183         else if (interpolation == "bilinear")
 184             itype = InterpolationType::BILINEAR;
 185         else
 186             CV_Error(Error::StsNotImplemented, "Requested interpolation mode is not available in resize layer.");
 187
 188         return make_cuda_node<cuda4dnn::ResizeOp>(preferableTarget, std::move(context->stream), itype, scaleHeight, scaleWidth);
 189     }
 190 #endif
 191
 192     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&) CV_OVERRIDE
 193     {
 194 #ifdef HAVE_INF_ENGINE
 195         InferenceEngine::Builder::Layer ieLayer(name);
 196         ieLayer.setName(name);
 197         if (interpolation == "nearest")
 198         {
 199             ieLayer.setType("Resample");
 200             ieLayer.getParameters()["type"] = std::string("caffe.ResampleParameter.NEAREST");
 201             ieLayer.getParameters()["antialias"] = false;
 202             if (scaleWidth != scaleHeight)
 203                 CV_Error(Error::StsNotImplemented, "resample with sw != sh");
 204             ieLayer.getParameters()["factor"] = 1.0f / scaleWidth;
 205         }
 206         else if (interpolation == "bilinear")
 207         {
 208             ieLayer.setType("Interp");
 209             ieLayer.getParameters()["pad_beg"] = 0;
 210             ieLayer.getParameters()["pad_end"] = 0;
 211             ieLayer.getParameters()["align_corners"] = false;
 212         }
 213         else
 214             CV_Error(Error::StsNotImplemented, "Unsupported interpolation: " + interpolation);
 215         ieLayer.getParameters()["width"] = outWidth;
 216         ieLayer.getParameters()["height"] = outHeight;
 217         ieLayer.setInputPorts(std::vector<InferenceEngine::Port>(1));
 218         ieLayer.setOutputPorts(std::vector<InferenceEngine::Port>(1));
 219         return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
 220 #endif  // HAVE_INF_ENGINE
 221         return Ptr<BackendNode>();
 222     }
 223
 224 protected:
 225     int outWidth, outHeight, zoomFactorWidth, zoomFactorHeight;
 226     String interpolation;
 227     float scaleWidth, scaleHeight;
 228     bool alignCorners;
 229 };
 230
 231
 232 Ptr<ResizeLayer> ResizeLayer::create(const LayerParams& params)
 233 {
 234     return Ptr<ResizeLayer>(new ResizeLayerImpl(params));
 235 }
 236
 237 class InterpLayerImpl CV_FINAL : public ResizeLayerImpl
 238 {
 239 public:
 240     InterpLayerImpl(const LayerParams& params) : ResizeLayerImpl(params) {}
 241
 242     bool getMemoryShapes(const std::vector<MatShape> &inputs,
 243                          const int requiredOutputs,
 244                          std::vector<MatShape> &outputs,
 245                          std::vector<MatShape> &internals) const CV_OVERRIDE
 246     {
 247         CV_Assert_N(inputs.size() == 1, inputs[0].size() == 4);
 248         outputs.resize(1, inputs[0]);
 249         outputs[0][2] = outHeight > 0 ? outHeight : (1 + zoomFactorHeight * (outputs[0][2] - 1));
 250         outputs[0][3] = outWidth > 0 ? outWidth : (1 + zoomFactorWidth * (outputs[0][3] - 1));
 251         // We can work in-place (do nothing) if input shape == output shape.
 252         return (outputs[0][2] == inputs[0][2]) && (outputs[0][3] == inputs[0][3]);
 253     }
 254
 255     virtual bool supportBackend(int backendId) CV_OVERRIDE
 256     {
 257         return backendId == DNN_BACKEND_OPENCV ||
 258                backendId == DNN_BACKEND_INFERENCE_ENGINE ||
 259                backendId == DNN_BACKEND_CUDA;
 260     }
 261
 262     virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
 263     {
 264         std::vector<Mat> inputs, outputs;
 265         inputs_arr.getMatVector(inputs);
 266         outputs_arr.getMatVector(outputs);
 267
 268         if (!outWidth && !outHeight)
 269         {
 270             outHeight = outputs[0].size[2];
 271             outWidth = outputs[0].size[3];
 272         }
 273         int inpHeight = inputs[0].size[2];
 274         int inpWidth = inputs[0].size[3];
 275         scaleHeight = (outHeight > 1) ? (static_cast<float>(inpHeight - 1) / (outHeight - 1)) : 0.f;
 276         scaleWidth = (outWidth > 1) ? (static_cast<float>(inpWidth - 1) / (outWidth - 1)) : 0.f;
 277     }
 278
 279 #ifdef HAVE_INF_ENGINE
 280     virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> >&) CV_OVERRIDE
 281     {
 282         InferenceEngine::Builder::Layer ieLayer(name);
 283         ieLayer.setName(name);
 284         ieLayer.setType("Interp");
 285         ieLayer.getParameters()["pad_beg"] = 0;
 286         ieLayer.getParameters()["pad_end"] = 0;
 287         ieLayer.getParameters()["width"] = outWidth;
 288         ieLayer.getParameters()["height"] = outHeight;
 289         ieLayer.setInputPorts(std::vector<InferenceEngine::Port>(1));
 290         ieLayer.setOutputPorts(std::vector<InferenceEngine::Port>(1));
 291         return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
 292     }
 293 #endif  // HAVE_INF_ENGINE
 294
 295 };
 296
 297 Ptr<Layer> InterpLayer::create(const LayerParams& params)
 298 {
 299     LayerParams lp(params);
 300     lp.set("interpolation", "bilinear");
 301     return Ptr<Layer>(new InterpLayerImpl(lp));
 302 }
 303
 304 }  // namespace dnn
 305 }  // namespace cv