compute/ARMComputeEx/src/runtime/CL/functions/CLDirectTransposeConvLayer.cpp

   1 /*
   2  * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * Copyright (c) 2019-2020 ARM Limited.
  19  *
  20  * SPDX-License-Identifier: MIT
  21  *
  22  * Permission is hereby granted, free of charge, to any person obtaining a copy
  23  * of this software and associated documentation files (the "Software"), to
  24  * deal in the Software without restriction, including without limitation the
  25  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  26  * sell copies of the Software, and to permit persons to whom the Software is
  27  * furnished to do so, subject to the following conditions:
  28  *
  29  * The above copyright notice and this permission notice shall be included in all
  30  * copies or substantial portions of the Software.
  31  *
  32  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  33  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  34  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  35  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  36  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  37  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  38  * SOFTWARE.
  39  */
  40 #include "arm_compute/runtime/CL/functions/CLDirectTransposeConvLayer.h"
  41
  42 #include "arm_compute/core/Helpers.h"
  43 #include "arm_compute/core/UtilsEx.h"
  44 #include "arm_compute/core/Validate.h"
  45 #include "arm_compute/core/utils/misc/ShapeCalculatorEx.h"
  46 #include "arm_compute/runtime/CL/CLScheduler.h"
  47
  48 #include <memory>
  49 #include <tuple>
  50
  51 namespace arm_compute
  52 {
  53 using namespace arm_compute::misc::shape_calculator;
  54
  55 CLDirectTransposeConvLayer::CLDirectTransposeConvLayer(
  56     std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
  57     : _memory_group(std::move(memory_manager)),
  58       _scale_f(),
  59       _conv_f(),
  60       _flip_weights(),
  61       _scaled_output(),
  62       _original_weights(nullptr),
  63       _weights_flipped(),
  64       _flip_axis(),
  65       _is_prepared(false)
  66 {
  67 }
  68
  69 Status CLDirectTransposeConvLayer::validate(const ITensorInfo *input, const ITensorInfo *weights,
  70                                             const ITensorInfo *bias, ITensorInfo *output,
  71                                             const PadStrideInfo &info, unsigned int invalid_right,
  72                                             unsigned int invalid_bottom,
  73                                             const WeightsInfo &weights_info)
  74 {
  75   ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
  76   ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(
  77       input, 1, DataType::QASYMM8_SIGNED, DataType::QASYMM8, DataType::F16, DataType::F32);
  78   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(input, weights);
  79   const DataLayout data_layout = input->data_layout();
  80
  81   const size_t idx_w = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
  82   const size_t idx_h = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
  83   const size_t idx_c = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL);
  84
  85   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_w) != weights->dimension(idx_h));
  86   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_w) < 1);
  87
  88   auto out_dims = transposeconv_output_dimensions(
  89       input->dimension(idx_w), input->dimension(idx_h), weights->dimension(idx_w),
  90       weights->dimension(idx_h), info, invalid_right, invalid_bottom);
  91
  92   const TensorShape output_shape = compute_transposeconv_output_shape(out_dims, *input, *weights);
  93
  94   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output, weights);
  95
  96   if (bias != nullptr)
  97   {
  98     if (is_data_type_quantized_asymmetric(input->data_type()))
  99     {
 100       ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::S32);
 101     }
 102     else
 103     {
 104       ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias);
 105     }
 106     ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(input, bias);
 107   }
 108
 109   ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(idx_w) != output_shape[idx_w],
 110                                   "Output's width is invalid.");
 111   ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(idx_h) != output_shape[idx_h],
 112                                   "Output's height is invalid.");
 113   ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(idx_c) != output_shape[idx_c],
 114                                   "Output's depth is invalid.");
 115
 116   unsigned int pad_left = 0;
 117   unsigned int pad_right = 0;
 118   unsigned int pad_top = 0;
 119   unsigned int pad_bottom = 0;
 120   const TensorShape scale_out_shape = compute_transposeconv_upsampled_shape(
 121       *input, *weights, info, out_dims, invalid_right, invalid_bottom, pad_left, pad_right, pad_top,
 122       pad_bottom);
 123   TensorInfo scale_out_info(input->clone()
 124                                 ->set_is_resizable(true)
 125                                 .reset_padding()
 126                                 .set_tensor_shape(scale_out_shape)
 127                                 .set_data_layout(data_layout));
 128   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 129
 130   ARM_COMPUTE_RETURN_ON_ERROR(CLDeconvolutionLayerUpsample::validate(input, &scale_out_info, info));
 131   ARM_COMPUTE_RETURN_ON_ERROR(CLConvolutionLayer::validate(&scale_out_info, weights, bias, output,
 132                                                            conv_info, weights_info));
 133
 134   return Status{};
 135 }
 136
 137 void CLDirectTransposeConvLayer::configure(ICLTensor *input, ICLTensor *weights,
 138                                            const ICLTensor *bias, ICLTensor *output,
 139                                            const PadStrideInfo &info, unsigned int invalid_right,
 140                                            unsigned int invalid_bottom,
 141                                            const WeightsInfo &weights_info)
 142 {
 143   configure(CLKernelLibrary::get().get_compile_context(), input, weights, bias, output, info,
 144             invalid_right, invalid_bottom, weights_info);
 145 }
 146
 147 void CLDirectTransposeConvLayer::configure(const CLCompileContext &compile_context,
 148                                            ICLTensor *input, ICLTensor *weights,
 149                                            const ICLTensor *bias, ICLTensor *output,
 150                                            const PadStrideInfo &info, unsigned int invalid_right,
 151                                            unsigned int invalid_bottom,
 152                                            const WeightsInfo &weights_info)
 153 {
 154   ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
 155
 156   unsigned int pad_left = 0;
 157   unsigned int pad_right = 0;
 158   unsigned int pad_top = 0;
 159   unsigned int pad_bottom = 0;
 160   const unsigned int stride_x = info.stride().first;
 161   const unsigned int stride_y = info.stride().second;
 162
 163   const DataLayout data_layout = input->info()->data_layout();
 164
 165   const size_t idx_w = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
 166   const size_t idx_h = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
 167
 168   _original_weights = weights;
 169   _flip_axis.allocator()->init(TensorInfo(TensorShape(2U), 1, DataType::U32));
 170   _weights_flipped.allocator()->init(weights->info()->clone()->set_data_layout(data_layout));
 171   _flip_weights.configure(compile_context, weights, &_weights_flipped, &_flip_axis);
 172
 173   auto out_dims = transposeconv_output_dimensions(
 174       input->info()->dimension(idx_w), input->info()->dimension(idx_h),
 175       weights->info()->dimension(idx_w), weights->info()->dimension(idx_h), info, invalid_right,
 176       invalid_bottom);
 177
 178   const TensorShape output_shape =
 179       compute_transposeconv_output_shape(out_dims, *input->info(), *weights->info());
 180
 181   // Output auto initialization if not yet initialized
 182   auto_init_if_empty(
 183       *output->info(),
 184       input->info()->clone()->set_tensor_shape(output_shape).set_data_layout(data_layout));
 185
 186   // Perform validation step
 187   ARM_COMPUTE_ERROR_THROW_ON(CLDirectTransposeConvLayer::validate(
 188       input->info(), weights->info(), bias == nullptr ? nullptr : bias->info(), output->info(),
 189       info, invalid_right, invalid_bottom));
 190
 191   _is_prepared = weights_info.retain_internal_weights();
 192
 193   _memory_group.manage(&_scaled_output);
 194
 195   // Find the upsampled dimensions and the padding needed for the convolution with stride 1 in order
 196   // to match output shape
 197   const TensorShape scale_out_shape = compute_transposeconv_upsampled_shape(
 198       *input->info(), *weights->info(), info, out_dims, invalid_right, invalid_bottom, pad_left,
 199       pad_right, pad_top, pad_bottom);
 200
 201   TensorInfo scale_out_info(scale_out_shape, 1, input->info()->data_type(),
 202                             input->info()->quantization_info());
 203   scale_out_info.set_data_layout(data_layout);
 204   _scaled_output.allocator()->init(scale_out_info);
 205
 206   // configure scale function
 207   const PadStrideInfo upsample_info(stride_x, stride_y, pad_left, pad_right, pad_top, pad_bottom,
 208                                     DimensionRoundingType::FLOOR);
 209   _scale_f.configure(input, &_scaled_output, upsample_info);
 210
 211   // Setup the function to convolve the upscaled output
 212   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 213   _conv_f.configure(compile_context, &_scaled_output, &_weights_flipped, bias, output, conv_info,
 214                     weights_info);
 215   _scaled_output.allocator()->allocate();
 216
 217   // Setup flip axis data
 218   _flip_axis.allocator()->allocate();
 219   _flip_axis.map(true);
 220   auto axis_data = reinterpret_cast<uint32_t *>(_flip_axis.buffer());
 221   if (weights->info()->data_layout() == DataLayout::NHWC)
 222   {
 223     axis_data[0] = 1;
 224     axis_data[1] = 2;
 225   }
 226   else
 227   {
 228     axis_data[0] = 0;
 229     axis_data[1] = 1;
 230   }
 231   _flip_axis.unmap();
 232 }
 233
 234 void CLDirectTransposeConvLayer::run()
 235 {
 236   prepare();
 237
 238   MemoryGroupResourceScope scope_mg(_memory_group);
 239
 240   _scale_f.run();
 241   _conv_f.run();
 242 }
 243
 244 void CLDirectTransposeConvLayer::prepare()
 245 {
 246   if (!_is_prepared)
 247   {
 248     ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
 249
 250     // Run weights flipping and mark original weights tensor as unused
 251     _weights_flipped.allocator()->allocate();
 252     _flip_weights.run();
 253     _original_weights->mark_as_unused();
 254
 255     // Prepare convolution
 256     _conv_f.prepare();
 257
 258     // Free flipped weights
 259     if (!_weights_flipped.is_used())
 260     {
 261       _weights_flipped.allocator()->free();
 262     }
 263
 264     _is_prepared = true;
 265   }
 266 }
 267 } // namespace arm_compute