compute/ARMComputeEx/src/runtime/NEON/functions/NETransposeConvLayer.cpp

   1 /*
   2  * Copyright (c) 2017-2020 ARM Limited.
   3  *
   4  * SPDX-License-Identifier: MIT
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to
   8  * deal in the Software without restriction, including without limitation the
   9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  10  * sell copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in all
  14  * copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  */
  24 #include "arm_compute/runtime/NEON/functions/NETransposeConvLayer.h"
  25
  26 #include "arm_compute/core/Helpers.h"
  27 #include "arm_compute/core/UtilsEx.h"
  28 #include "arm_compute/core/Validate.h"
  29 #include "arm_compute/core/utils/misc/ShapeCalculatorEx.h"
  30 #include "arm_compute/runtime/NEON/NEScheduler.h"
  31
  32 using namespace arm_compute::misc::shape_calculator;
  33
  34 namespace arm_compute
  35 {
  36
  37 NETransposeConvLayer::NETransposeConvLayer(std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
  38     : _memory_group(std::move(memory_manager)),
  39       _conv_f(),
  40       _upsample_f(),
  41       _flip_weights(),
  42       _scaled_output(),
  43       _weights_flipped(),
  44       _flip_axis(),
  45       _original_weights(nullptr),
  46       _input(nullptr),
  47       _info(),
  48       _is_prepared(false)
  49 {
  50 }
  51
  52 Status NETransposeConvLayer::validate(const ITensorInfo *input, const ITensorInfo *weights,
  53                                       const ITensorInfo *bias, const ITensorInfo *output,
  54                                       const PadStrideInfo &info, unsigned int invalid_right,
  55                                       unsigned int invalid_bottom)
  56 {
  57   ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
  58   ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32, DataType::F16,
  59                                                        DataType::QASYMM8, DataType::QASYMM8_SIGNED);
  60   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, input);
  61   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(weights, input);
  62   const unsigned int width_idx =
  63       get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::WIDTH);
  64   const unsigned int height_idx =
  65       get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::HEIGHT);
  66   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) != weights->dimension(height_idx));
  67   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) < 1);
  68
  69   auto out_dims = transposeconv_output_dimensions(
  70       input->dimension(width_idx), input->dimension(height_idx), weights->dimension(width_idx),
  71       weights->dimension(height_idx), info, invalid_right, invalid_bottom);
  72
  73   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
  74   if (bias != nullptr)
  75   {
  76     if (is_data_type_quantized_asymmetric(input->data_type()))
  77     {
  78       ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::S32);
  79     }
  80     else
  81     {
  82       ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias);
  83     }
  84   }
  85
  86   if (output->tensor_shape().total_size() > 0)
  87   {
  88     ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
  89
  90     const TensorShape output_shape = compute_transposeconv_output_shape(out_dims, *input, *weights);
  91
  92     ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimX) != output_shape.x(),
  93                                     "Output's width is invalid.");
  94     ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimY) != output_shape.y(),
  95                                     "Output's height is invalid.");
  96     ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimZ) != output_shape.z(),
  97                                     "Output's depth is invalid.");
  98   }
  99
 100   unsigned int pad_left = 0;
 101   unsigned int pad_right = 0;
 102   unsigned int pad_top = 0;
 103   unsigned int pad_bottom = 0;
 104   const TensorShape scale_out_shape = compute_transposeconv_upsampled_shape(
 105       *input, *weights, info, out_dims, invalid_right, invalid_bottom, pad_left, pad_right, pad_top,
 106       pad_bottom);
 107   TensorInfo scale_out_info(
 108       input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(scale_out_shape));
 109   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 110
 111   const unsigned int batches_idx =
 112       get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::BATCHES);
 113   const unsigned int channel_idx =
 114       get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::CHANNEL);
 115   ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(batches_idx) !=
 116                               scale_out_info.dimension(batches_idx));
 117   ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(channel_idx) !=
 118                               scale_out_info.dimension(channel_idx));
 119
 120   ARM_COMPUTE_RETURN_ON_ERROR(NEConvolutionLayer::validate(&scale_out_info, weights, bias, output,
 121                                                            conv_info, WeightsInfo()));
 122
 123   return Status{};
 124 }
 125
 126 void NETransposeConvLayer::configure(ITensor *input, const ITensor *weights, const ITensor *bias,
 127                                      ITensor *output, const PadStrideInfo &info,
 128                                      unsigned int invalid_right, unsigned int invalid_bottom)
 129 {
 130   // Perform validation step
 131   ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
 132   ARM_COMPUTE_ERROR_THROW_ON(NETransposeConvLayer::validate(
 133       input->info(), weights->info(), (bias == nullptr) ? nullptr : bias->info(), output->info(),
 134       info, invalid_right, invalid_bottom));
 135
 136   const DataLayout data_layout = input->info()->data_layout();
 137   const unsigned int width_idx =
 138       get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
 139   const unsigned int height_idx =
 140       get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
 141   auto out_dims = transposeconv_output_dimensions(
 142       input->info()->dimension(width_idx), input->info()->dimension(height_idx),
 143       weights->info()->dimension(width_idx), weights->info()->dimension(height_idx), info,
 144       invalid_right, invalid_bottom);
 145
 146   const TensorShape output_shape =
 147       compute_transposeconv_output_shape(out_dims, *input->info(), *weights->info());
 148
 149   _input = input;
 150   _original_weights = weights;
 151   _info = info;
 152   _is_prepared = false;
 153
 154   unsigned int pad_left = 0;
 155   unsigned int pad_right = 0;
 156   unsigned int pad_top = 0;
 157   unsigned int pad_bottom = 0;
 158   const unsigned int stride_x = info.stride().first;
 159   const unsigned int stride_y = info.stride().second;
 160
 161   // Output auto initialization if not yet initialized
 162   auto_init_if_empty(*output->info(), output_shape, 1, input->info()->data_type(),
 163                      input->info()->quantization_info());
 164
 165   _flip_axis.allocator()->init(TensorInfo(TensorShape(2U), 1, DataType::U32));
 166   _memory_group.manage(&_scaled_output);
 167
 168   _weights_flipped.allocator()->init(weights->info()->clone()->set_data_layout(data_layout));
 169   _flip_weights.configure(weights, &_weights_flipped, &_flip_axis);
 170
 171   // setup the function to convolve the upscaled output
 172   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 173
 174   const TensorShape scale_out_shape = compute_transposeconv_upsampled_shape(
 175       *input->info(), *weights->info(), info, out_dims, invalid_right, invalid_bottom, pad_left,
 176       pad_right, pad_top, pad_bottom);
 177
 178   const PadStrideInfo upsample_info(stride_x, stride_y, pad_left, pad_right, pad_top, pad_bottom,
 179                                     DimensionRoundingType::FLOOR);
 180
 181   TensorInfo scale_out_info(scale_out_shape, 1, input->info()->data_type(),
 182                             input->info()->quantization_info());
 183   scale_out_info.set_data_layout(data_layout);
 184   _scaled_output.allocator()->init(scale_out_info);
 185
 186   _upsample_f.configure(input, &_scaled_output, upsample_info);
 187
 188   _conv_f.configure(&_scaled_output, &_weights_flipped, bias, output, conv_info);
 189
 190   // Setup flip axis data
 191   _flip_axis.allocator()->allocate();
 192   auto axis_data = reinterpret_cast<uint32_t *>(_flip_axis.buffer());
 193   axis_data[0] = static_cast<uint32_t>(width_idx);
 194   axis_data[1] = static_cast<uint32_t>(height_idx);
 195
 196   _scaled_output.allocator()->allocate();
 197 }
 198
 199 void NETransposeConvLayer::run()
 200 {
 201   prepare();
 202
 203   MemoryGroupResourceScope scope_mg(_memory_group);
 204
 205   _upsample_f.run();
 206   _conv_f.run();
 207 }
 208
 209 void NETransposeConvLayer::prepare()
 210 {
 211   if (!_is_prepared)
 212   {
 213     ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
 214
 215     // Run weights flipping and mark original weights tensor as unused
 216     _weights_flipped.allocator()->allocate();
 217     _flip_weights.run();
 218     _original_weights->mark_as_unused();
 219
 220     // Prepare convolution
 221     _conv_f.prepare();
 222
 223     _is_prepared = true;
 224   }
 225 }
 226 } // namespace arm_compute