compute/ARMComputeEx/src/runtime/CL/functions/CLDirectTransposeConvLayer.cpp

   1 /*
   2  * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * Copyright (c) 2019-2020 ARM Limited.
  19  *
  20  * SPDX-License-Identifier: MIT
  21  *
  22  * Permission is hereby granted, free of charge, to any person obtaining a copy
  23  * of this software and associated documentation files (the "Software"), to
  24  * deal in the Software without restriction, including without limitation the
  25  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  26  * sell copies of the Software, and to permit persons to whom the Software is
  27  * furnished to do so, subject to the following conditions:
  28  *
  29  * The above copyright notice and this permission notice shall be included in all
  30  * copies or substantial portions of the Software.
  31  *
  32  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  33  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  34  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  35  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  36  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  37  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  38  * SOFTWARE.
  39  */
  40 #include "arm_compute/runtime/CL/functions/CLDirectTransposeConvLayer.h"
  41
  42 #include "arm_compute/core/Helpers.h"
  43 #include "arm_compute/core/UtilsEx.h"
  44 #include "arm_compute/core/Validate.h"
  45 #include "arm_compute/core/utils/misc/ShapeCalculatorEx.h"
  46 #include "arm_compute/runtime/CL/CLScheduler.h"
  47
  48 #include <memory>
  49 #include <tuple>
  50
  51 namespace arm_compute
  52 {
  53 using namespace arm_compute::misc::shape_calculator;
  54
  55 CLDirectTransposeConvLayer::CLDirectTransposeConvLayer(
  56   std::shared_ptr<IMemoryManager> memory_manager) // NOLINT
  57   : _memory_group(std::move(memory_manager)), _scale_f(), _conv_f(), _flip_weights(),
  58     _scaled_output(), _original_weights(nullptr), _weights_flipped(), _flip_axis(),
  59     _is_prepared(false)
  60 {
  61 }
  62
  63 Status CLDirectTransposeConvLayer::validate(const ITensorInfo *input, const ITensorInfo *weights,
  64                                             const ITensorInfo *bias, ITensorInfo *output,
  65                                             const PadStrideInfo &info, unsigned int invalid_right,
  66                                             unsigned int invalid_bottom,
  67                                             const WeightsInfo &weights_info)
  68 {
  69   ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
  70   ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(
  71     input, 1, DataType::QASYMM8_SIGNED, DataType::QASYMM8, DataType::F16, DataType::F32);
  72   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(input, weights);
  73   const DataLayout data_layout = input->data_layout();
  74
  75   const size_t idx_w = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
  76   const size_t idx_h = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
  77   const size_t idx_c = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL);
  78
  79   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_w) != weights->dimension(idx_h));
  80   ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_w) < 1);
  81
  82   auto out_dims = transposeconv_output_dimensions(
  83     input->dimension(idx_w), input->dimension(idx_h), weights->dimension(idx_w),
  84     weights->dimension(idx_h), info, invalid_right, invalid_bottom);
  85
  86   const TensorShape output_shape = compute_transposeconv_output_shape(out_dims, *input, *weights);
  87
  88   ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output, weights);
  89
  90   if (bias != nullptr)
  91   {
  92     if (is_data_type_quantized_asymmetric(input->data_type()))
  93     {
  94       ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(bias, 1, DataType::S32);
  95     }
  96     else
  97     {
  98       ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, bias);
  99     }
 100     ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(input, bias);
 101   }
 102
 103   ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(idx_w) != output_shape[idx_w],
 104                                   "Output's width is invalid.");
 105   ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(idx_h) != output_shape[idx_h],
 106                                   "Output's height is invalid.");
 107   ARM_COMPUTE_RETURN_ERROR_ON_MSG(output->dimension(idx_c) != output_shape[idx_c],
 108                                   "Output's depth is invalid.");
 109
 110   unsigned int pad_left = 0;
 111   unsigned int pad_right = 0;
 112   unsigned int pad_top = 0;
 113   unsigned int pad_bottom = 0;
 114   const TensorShape scale_out_shape =
 115     compute_transposeconv_upsampled_shape(*input, *weights, info, out_dims, invalid_right,
 116                                           invalid_bottom, pad_left, pad_right, pad_top, pad_bottom);
 117   TensorInfo scale_out_info(input->clone()
 118                               ->set_is_resizable(true)
 119                               .reset_padding()
 120                               .set_tensor_shape(scale_out_shape)
 121                               .set_data_layout(data_layout));
 122   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 123
 124   ARM_COMPUTE_RETURN_ON_ERROR(CLDeconvolutionLayerUpsample::validate(input, &scale_out_info, info));
 125   ARM_COMPUTE_RETURN_ON_ERROR(
 126     CLConvolutionLayer::validate(&scale_out_info, weights, bias, output, conv_info, weights_info));
 127
 128   return Status{};
 129 }
 130
 131 void CLDirectTransposeConvLayer::configure(ICLTensor *input, ICLTensor *weights,
 132                                            const ICLTensor *bias, ICLTensor *output,
 133                                            const PadStrideInfo &info, unsigned int invalid_right,
 134                                            unsigned int invalid_bottom,
 135                                            const WeightsInfo &weights_info)
 136 {
 137   configure(CLKernelLibrary::get().get_compile_context(), input, weights, bias, output, info,
 138             invalid_right, invalid_bottom, weights_info);
 139 }
 140
 141 void CLDirectTransposeConvLayer::configure(const CLCompileContext &compile_context,
 142                                            ICLTensor *input, ICLTensor *weights,
 143                                            const ICLTensor *bias, ICLTensor *output,
 144                                            const PadStrideInfo &info, unsigned int invalid_right,
 145                                            unsigned int invalid_bottom,
 146                                            const WeightsInfo &weights_info)
 147 {
 148   ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
 149
 150   unsigned int pad_left = 0;
 151   unsigned int pad_right = 0;
 152   unsigned int pad_top = 0;
 153   unsigned int pad_bottom = 0;
 154   const unsigned int stride_x = info.stride().first;
 155   const unsigned int stride_y = info.stride().second;
 156
 157   const DataLayout data_layout = input->info()->data_layout();
 158
 159   const size_t idx_w = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH);
 160   const size_t idx_h = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT);
 161
 162   _original_weights = weights;
 163   _flip_axis.allocator()->init(TensorInfo(TensorShape(2U), 1, DataType::U32));
 164   _weights_flipped.allocator()->init(weights->info()->clone()->set_data_layout(data_layout));
 165   _flip_weights.configure(compile_context, weights, &_weights_flipped, &_flip_axis);
 166
 167   auto out_dims = transposeconv_output_dimensions(
 168     input->info()->dimension(idx_w), input->info()->dimension(idx_h),
 169     weights->info()->dimension(idx_w), weights->info()->dimension(idx_h), info, invalid_right,
 170     invalid_bottom);
 171
 172   const TensorShape output_shape =
 173     compute_transposeconv_output_shape(out_dims, *input->info(), *weights->info());
 174
 175   // Output auto initialization if not yet initialized
 176   auto_init_if_empty(
 177     *output->info(),
 178     input->info()->clone()->set_tensor_shape(output_shape).set_data_layout(data_layout));
 179
 180   // Perform validation step
 181   ARM_COMPUTE_ERROR_THROW_ON(CLDirectTransposeConvLayer::validate(
 182     input->info(), weights->info(), bias == nullptr ? nullptr : bias->info(), output->info(), info,
 183     invalid_right, invalid_bottom));
 184
 185   _is_prepared = weights_info.retain_internal_weights();
 186
 187   _memory_group.manage(&_scaled_output);
 188
 189   // Find the upsampled dimensions and the padding needed for the convolution with stride 1 in order
 190   // to match output shape
 191   const TensorShape scale_out_shape = compute_transposeconv_upsampled_shape(
 192     *input->info(), *weights->info(), info, out_dims, invalid_right, invalid_bottom, pad_left,
 193     pad_right, pad_top, pad_bottom);
 194
 195   TensorInfo scale_out_info(scale_out_shape, 1, input->info()->data_type(),
 196                             input->info()->quantization_info());
 197   scale_out_info.set_data_layout(data_layout);
 198   _scaled_output.allocator()->init(scale_out_info);
 199
 200   // configure scale function
 201   const PadStrideInfo upsample_info(stride_x, stride_y, pad_left, pad_right, pad_top, pad_bottom,
 202                                     DimensionRoundingType::FLOOR);
 203   _scale_f.configure(input, &_scaled_output, upsample_info);
 204
 205   // Setup the function to convolve the upscaled output
 206   const PadStrideInfo conv_info(1, 1, 0, 0, 0, 0, DimensionRoundingType::CEIL);
 207   _conv_f.configure(compile_context, &_scaled_output, &_weights_flipped, bias, output, conv_info,
 208                     weights_info);
 209   _scaled_output.allocator()->allocate();
 210
 211   // Setup flip axis data
 212   _flip_axis.allocator()->allocate();
 213   _flip_axis.map(true);
 214   auto axis_data = reinterpret_cast<uint32_t *>(_flip_axis.buffer());
 215   if (weights->info()->data_layout() == DataLayout::NHWC)
 216   {
 217     axis_data[0] = 1;
 218     axis_data[1] = 2;
 219   }
 220   else
 221   {
 222     axis_data[0] = 0;
 223     axis_data[1] = 1;
 224   }
 225   _flip_axis.unmap();
 226 }
 227
 228 void CLDirectTransposeConvLayer::run()
 229 {
 230   prepare();
 231
 232   MemoryGroupResourceScope scope_mg(_memory_group);
 233
 234   _scale_f.run();
 235   _conv_f.run();
 236 }
 237
 238 void CLDirectTransposeConvLayer::prepare()
 239 {
 240   if (!_is_prepared)
 241   {
 242     ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
 243
 244     // Run weights flipping and mark original weights tensor as unused
 245     _weights_flipped.allocator()->allocate();
 246     _flip_weights.run();
 247     _original_weights->mark_as_unused();
 248
 249     // Prepare convolution
 250     _conv_f.prepare();
 251
 252     // Free flipped weights
 253     if (!_weights_flipped.is_used())
 254     {
 255       _weights_flipped.allocator()->free();
 256     }
 257
 258     _is_prepared = true;
 259   }
 260 }
 261 } // namespace arm_compute