compute/ARMComputeEx/src/runtime/NEON/functions/NETransposeConvLayer.cpp

   1 /*
   2  * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * Copyright (c) 2017-2020 ARM Limited.
  19  *
  20  * SPDX-License-Identifier: MIT
  21  *
  22  * Permission is hereby granted, free of charge, to any person obtaining a copy
  23  * of this software and associated documentation files (the "Software"), to
  24  * deal in the Software without restriction, including without limitation the
  25  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  26  * sell copies of the Software, and to permit persons to whom the Software is
  27  * furnished to do so, subject to the following conditions:
  28  *
  29  * The above copyright notice and this permission notice shall be included in all
  30  * copies or substantial portions of the Software.
  31  *
  32  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  33  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  34  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  35  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  36  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  37  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  38  * SOFTWARE.
  39  */
  40 #include "arm_compute/runtime/NEON/functions/NETransposeConvLayer.h"
  41
  42 #include "arm_compute/core/Helpers.h"
  43 #include "arm_compute/core/UtilsEx.h"
  44 #include "arm_compute/core/Validate.h"
  45 #include "arm_compute/core/utils/misc/ShapeCalculatorEx.h"
  46 #include "arm_compute/runtime/NEON/NEScheduler.h"
  47
  48 using namespace arm_compute::misc::shape_calculator;
  49
  50 namespace arm_compute
  51 {
  52
  53 NETransposeConvLayer::NETransposeConvLayer(std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
  54   : _memory_group(std::move(memory_manager)), _conv_f(), _upsample_f(), _flip_weights(),
  55     _scaled_output(), _weights_flipped(), _flip_axis(), _original_weights(nullptr), _input(nullptr),
  56     _info(), _is_prepared(false)
  57 {
  58 }
  59
  60 Status NETransposeConvLayer::validate(const ITensorInfo *input, const ITensorInfo *weights,
  61                                       const ITensorInfo *bias, const ITensorInfo *output,
  62                                       const PadStrideInfo &info, unsigned int invalid_right,
  63                                       unsigned int invalid_bottom)
  64 {
  65   ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
  66   ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F32, DataType::F16,
  67                                                        DataType::QASYMM8, DataType::QASYMM8_SIGNED);
  68   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(weights, input);
  69   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(weights, input);
  70   const unsigned int width_idx =
  71     get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::WIDTH);
  72   const unsigned int height_idx =
  73     get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::HEIGHT);
  74   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) != weights->dimension(height_idx));
  75   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) < 1);
  76
  77   auto out_dims = transposeconv_output_dimensions(
  78     input->dimension(width_idx), input->dimension(height_idx), weights->dimension(width_idx),
  79     weights->dimension(height_idx), info, invalid_right, invalid_bottom);
  80
  81   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights);
  82   if (bias != nullptr)
  83   {
  84     if (is_data_type_quantized_asymmetric(input->data_type()))
  85     {
  86       ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::S32);
  87     }
  88     else
  89     {
  90       ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias);
  91     }
  92   }
  93
  94   if (output->tensor_shape().total_size() > 0)
  95   {
  96     ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
  97
  98     const TensorShape output_shape = compute_transposeconv_output_shape(out_dims, *input, *weights);
  99
 100     ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimX) != output_shape.x(),
 101                                     "Output's width is invalid.");
 102     ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimY) != output_shape.y(),
 103                                     "Output's height is invalid.");
 104     ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(Window::DimZ) != output_shape.z(),
 105                                     "Output's depth is invalid.");
 106   }
 107
 108   unsigned int pad_left = 0;
 109   unsigned int pad_right = 0;
 110   unsigned int pad_top = 0;
 111   unsigned int pad_bottom = 0;
 112   const TensorShape scale_out_shape =
 113     compute_transposeconv_upsampled_shape(*input, *weights, info, out_dims, invalid_right,
 114                                           invalid_bottom, pad_left, pad_right, pad_top, pad_bottom);
 115   TensorInfo scale_out_info(
 116     input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(scale_out_shape));
 117   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 118
 119   const unsigned int batches_idx =
 120     get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::BATCHES);
 121   const unsigned int channel_idx =
 122     get_data_layout_dimension_index(weights->data_layout(), DataLayoutDimension::CHANNEL);
 123   ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(batches_idx) !=
 124                               scale_out_info.dimension(batches_idx));
 125   ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(channel_idx) !=
 126                               scale_out_info.dimension(channel_idx));
 127
 128   ARM_COMPUTE_RETURN_ON_ERROR(
 129     NEConvolutionLayer::validate(&scale_out_info, weights, bias, output, conv_info, WeightsInfo()));
 130
 131   return Status{};
 132 }
 133
 134 void NETransposeConvLayer::configure(ITensor *input, const ITensor *weights, const ITensor *bias,
 135                                      ITensor *output, const PadStrideInfo &info,
 136                                      unsigned int invalid_right, unsigned int invalid_bottom)
 137 {
 138   // Perform validation step
 139   ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
 140   ARM_COMPUTE_ERROR_THROW_ON(NETransposeConvLayer::validate(
 141     input->info(), weights->info(), (bias == nullptr) ? nullptr : bias->info(), output->info(),
 142     info, invalid_right, invalid_bottom));
 143
 144   const DataLayout data_layout = input->info()->data_layout();
 145   const unsigned int width_idx =
 146     get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
 147   const unsigned int height_idx =
 148     get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
 149   auto out_dims = transposeconv_output_dimensions(
 150     input->info()->dimension(width_idx), input->info()->dimension(height_idx),
 151     weights->info()->dimension(width_idx), weights->info()->dimension(height_idx), info,
 152     invalid_right, invalid_bottom);
 153
 154   const TensorShape output_shape =
 155     compute_transposeconv_output_shape(out_dims, *input->info(), *weights->info());
 156
 157   _input = input;
 158   _original_weights = weights;
 159   _info = info;
 160   _is_prepared = false;
 161
 162   unsigned int pad_left = 0;
 163   unsigned int pad_right = 0;
 164   unsigned int pad_top = 0;
 165   unsigned int pad_bottom = 0;
 166   const unsigned int stride_x = info.stride().first;
 167   const unsigned int stride_y = info.stride().second;
 168
 169   // Output auto initialization if not yet initialized
 170   auto_init_if_empty(*output->info(), output_shape, 1, input->info()->data_type(),
 171                      input->info()->quantization_info());
 172
 173   _flip_axis.allocator()->init(TensorInfo(TensorShape(2U), 1, DataType::U32));
 174   _memory_group.manage(&_scaled_output);
 175
 176   _weights_flipped.allocator()->init(weights->info()->clone()->set_data_layout(data_layout));
 177   _flip_weights.configure(weights, &_weights_flipped, &_flip_axis);
 178
 179   // setup the function to convolve the upscaled output
 180   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 181
 182   const TensorShape scale_out_shape = compute_transposeconv_upsampled_shape(
 183     *input->info(), *weights->info(), info, out_dims, invalid_right, invalid_bottom, pad_left,
 184     pad_right, pad_top, pad_bottom);
 185
 186   const PadStrideInfo upsample_info(stride_x, stride_y, pad_left, pad_right, pad_top, pad_bottom,
 187                                     DimensionRoundingType::FLOOR);
 188
 189   TensorInfo scale_out_info(scale_out_shape, 1, input->info()->data_type(),
 190                             input->info()->quantization_info());
 191   scale_out_info.set_data_layout(data_layout);
 192   _scaled_output.allocator()->init(scale_out_info);
 193
 194   _upsample_f.configure(input, &_scaled_output, upsample_info);
 195
 196   _conv_f.configure(&_scaled_output, &_weights_flipped, bias, output, conv_info);
 197
 198   // Setup flip axis data
 199   _flip_axis.allocator()->allocate();
 200   auto axis_data = reinterpret_cast<uint32_t *>(_flip_axis.buffer());
 201   axis_data[0] = static_cast<uint32_t>(width_idx);
 202   axis_data[1] = static_cast<uint32_t>(height_idx);
 203
 204   _scaled_output.allocator()->allocate();
 205 }
 206
 207 void NETransposeConvLayer::run()
 208 {
 209   prepare();
 210
 211   MemoryGroupResourceScope scope_mg(_memory_group);
 212
 213   _upsample_f.run();
 214   _conv_f.run();
 215 }
 216
 217 void NETransposeConvLayer::prepare()
 218 {
 219   if (!_is_prepared)
 220   {
 221     ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
 222
 223     // Run weights flipping and mark original weights tensor as unused
 224     _weights_flipped.allocator()->allocate();
 225     _flip_weights.run();
 226     _original_weights->mark_as_unused();
 227
 228     // Prepare convolution
 229     _conv_f.prepare();
 230
 231     _is_prepared = true;
 232   }
 233 }
 234 } // namespace arm_compute