2 * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * Copyright (c) 2017-2019 ARM Limited.
20 * SPDX-License-Identifier: MIT
22 * Permission is hereby granted, free of charge, to any person obtaining a copy
23 * of this software and associated documentation files (the "Software"), to
24 * deal in the Software without restriction, including without limitation the
25 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
26 * sell copies of the Software, and to permit persons to whom the Software is
27 * furnished to do so, subject to the following conditions:
29 * The above copyright notice and this permission notice shall be included in all
30 * copies or substantial portions of the Software.
32 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41 #include "arm_compute/runtime/NEON/functions/NEFullyConnectedLayerEx.h"
43 #include "arm_compute/core/Helpers.h"
44 #include "arm_compute/core/Size2D.h"
45 #include "arm_compute/core/Validate.h"
46 #include "arm_compute/core/utils/misc/ShapeCalculator.h"
47 #include "arm_compute/core/utils/quantization/AsymmHelpers.h"
48 #include "arm_compute/runtime/NEON/NEScheduler.h"
53 using namespace arm_compute;
54 using namespace arm_compute::misc::shape_calculator;
58 Status validate_mm(const ITensorInfo &input, const ITensorInfo &weights, const ITensorInfo &output)
60 if (is_data_type_quantized_asymmetric(input.data_type()))
62 // Since we need negative offsets for computing convolution, we need to change
64 // Extract and negate input and weights offset
65 const QuantizationInfo input_quantization_info(input.quantization_info().uniform().scale,
66 -input.quantization_info().uniform().offset);
67 const QuantizationInfo weights_quantization_info(weights.quantization_info().uniform().scale,
68 -weights.quantization_info().uniform().offset);
70 // Validate gemmlowp function
71 ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpMatrixMultiplyCore::validate(
72 &input.clone()->set_quantization_info(input_quantization_info),
73 &weights.clone()->set_quantization_info(weights_quantization_info), nullptr, &output));
77 ARM_COMPUTE_RETURN_ON_ERROR(
78 NEGEMM::validate(&input, &weights, nullptr, &output, 1.f, 0.0f,
79 GEMMInfo(false, false, false /* Reshape weights only for the first run */)));
86 NEFullyConnectedLayerEx::NEFullyConnectedLayerEx(std::shared_ptr<IMemoryManager> memory_manager)
87 : _memory_group(std::move(memory_manager)), _flatten_kernel(), _convert_weights(),
88 _reshape_weights_function(), _mm_gemm(), _mm_gemmlowp(), _gemmlowp_output_stage(),
89 _accumulate_biases_kernel(), _flatten_output(), _gemmlowp_output(), _converted_weights_output(),
90 _reshape_weights_output(), _original_weights(nullptr), _are_weights_converted(true),
91 _are_weights_reshaped(false), _is_fc_after_conv(false), _accumulate_biases(false),
92 _is_quantized(false), _is_prepared(false)
96 void NEFullyConnectedLayerEx::configure_mm(const ITensor *input, const ITensor *weights,
101 // Since we need negative offsets for computing convolution, we need to change
102 // QuantizationInfo()
103 // Extract and negate input and weights offset
104 const QuantizationInfo input_quantization_info = input->info()->quantization_info();
105 const QuantizationInfo weights_quantization_info = weights->info()->quantization_info();
107 input->info()->set_quantization_info(QuantizationInfo(
108 input_quantization_info.uniform().scale, -input_quantization_info.uniform().offset));
109 weights->info()->set_quantization_info(QuantizationInfo(
110 weights_quantization_info.uniform().scale, -weights_quantization_info.uniform().offset));
112 // Configure gemmlowp function
113 _mm_gemmlowp.configure(input, weights, nullptr, output);
115 // Revert back QuantizatioInfo as input and weights could be used in other fully connected
117 input->info()->set_quantization_info(input_quantization_info);
118 weights->info()->set_quantization_info(weights_quantization_info);
122 // Configure matrix multiply kernel
123 _mm_gemm.configure(input, weights, nullptr, output, 1.f, 0.0f,
124 GEMMInfo(false, false, false /* Reshape weights only for the first run */));
128 void NEFullyConnectedLayerEx::configure_conv_fc(const ITensor *input, const ITensor *weights,
131 ARM_COMPUTE_ERROR_ON(
132 (weights->info()->dimension(1) !=
133 (input->info()->dimension(0) * input->info()->dimension(1) * input->info()->dimension(2))));
135 // If the fully connected layer is called after a convolution layer, the input tensor must be
138 // Initialize output tensor for flatten
139 TensorShape shape_flatten = compute_flatten_shape(input->info());
140 _flatten_output.allocator()->init(
141 input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(shape_flatten));
143 // Configure flatten kernel
144 _memory_group.manage(&_flatten_output);
145 _flatten_kernel.configure(input, &_flatten_output);
147 // Configure matrix multiply kernel
148 configure_mm(&_flatten_output, weights, output);
150 // Allocate the output tensor for flatten once all the configure methods have been called
151 _flatten_output.allocator()->allocate();
154 void NEFullyConnectedLayerEx::configure_fc_fc(const ITensor *input, const ITensor *weights,
157 ARM_COMPUTE_ERROR_ON(input->info()->dimension(0) != weights->info()->dimension(1));
159 // Configure matrix multiply kernel
160 configure_mm(input, weights, output);
163 void NEFullyConnectedLayerEx::configure(const ITensor *input, const ITensor *weights,
164 const ITensor *biases, ITensor *output,
165 FullyConnectedLayerInfo fc_info)
167 ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
169 // Perform validate step
170 ARM_COMPUTE_ERROR_THROW_ON(NEFullyConnectedLayerEx::validate(
171 input->info(), weights->info(), biases != nullptr ? biases->info() : nullptr, output->info(),
174 _are_weights_converted = true;
175 _are_weights_reshaped = fc_info.transpose_weights ? fc_info.are_weights_reshaped : true;
176 _is_fc_after_conv = true;
177 _accumulate_biases = false;
178 _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type());
179 _original_weights = weights;
181 // Configure gemmlowp output
184 _gemmlowp_output.allocator()->init(
185 output->info()->clone()->set_is_resizable(true).reset_padding().set_data_type(DataType::S32));
188 // Configure accumulate biases kernel for non quantized asymmetric types
189 if (biases != nullptr && !_is_quantized)
191 _accumulate_biases = true;
193 // Configure accumulate biases kernel
194 _accumulate_biases_kernel.configure(output, biases);
197 // With the Fully Connected layer we can have 4 different cases:
198 // 1) Convolution layer -> Fully Connected layer without batches
199 // 2) Fully Connected layer -> Fully Connected layer without batches
200 // 3) Convolution layer -> Fully Connected layer with batches
201 // 4) Fully Connected layer -> Fully Connected layer with batches
203 const ITensor *weights_to_use = weights;
205 // Check if we have a fully connected layer with batches
206 const bool is_batched_fc_layer = output->info()->dimension(1) > 1;
207 if (is_batched_fc_layer)
210 (TensorShape::num_max_dimensions >= 4) &&
211 (std::equal(input->info()->tensor_shape().cbegin() + 3, input->info()->tensor_shape().cend(),
212 output->info()->tensor_shape().cbegin() + 1));
216 _is_fc_after_conv = input->info()->num_dimensions() > 1;
219 // Reshape weights if needed
220 if (!_are_weights_reshaped)
222 // Reshape the weights
223 _reshape_weights_function.configure(weights, &_reshape_weights_output);
224 weights_to_use = &_reshape_weights_output;
227 // Convert weights if needed
228 if (_is_fc_after_conv && (input->info()->data_layout() != fc_info.weights_trained_layout))
231 _convert_weights.configure(weights_to_use, &_converted_weights_output,
232 input->info()->tensor_shape(), fc_info.weights_trained_layout);
234 weights_to_use = &_converted_weights_output;
235 _are_weights_converted = false;
238 ITensor *tmp_output = (_is_quantized) ? &_gemmlowp_output : output;
239 if (_is_fc_after_conv)
241 // Fully Connected layer after a Convolution Layer without batches
242 configure_conv_fc(input, weights_to_use, tmp_output);
246 // Fully Connected layer after a Fully Connected Layer without batches
247 configure_fc_fc(input, weights_to_use, tmp_output);
250 // Configure output stage for asymmetric quantized types
253 float multiplier = input->info()->quantization_info().uniform().scale *
254 weights->info()->quantization_info().uniform().scale /
255 output->info()->quantization_info().uniform().scale;
256 int output_multiplier;
258 quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier,
260 _gemmlowp_output_stage.configure(&_gemmlowp_output, biases, output, output_multiplier,
262 output->info()->quantization_info().uniform().offset);
263 _gemmlowp_output.allocator()->allocate();
266 _are_weights_reshaped = _are_weights_reshaped || fc_info.retain_internal_weights;
269 Status NEFullyConnectedLayerEx::validate(const ITensorInfo *input, const ITensorInfo *weights,
270 const ITensorInfo *biases, const ITensorInfo *output,
271 FullyConnectedLayerInfo fc_info)
273 ARM_COMPUTE_UNUSED(fc_info.retain_internal_weights);
274 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
275 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F16,
277 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights, output);
278 ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 2);
280 bool weights_reshaped = fc_info.transpose_weights ? fc_info.are_weights_reshaped : true;
281 bool is_fc_after_conv = true;
282 bool is_quantized = is_data_type_quantized_asymmetric(input->data_type());
284 const ITensorInfo &flatten_input =
285 TensorInfo(input->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(
286 compute_flatten_shape(input)));
287 const ITensorInfo &reshaped_weights =
288 TensorInfo(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(
289 compute_transposed_shape(*weights)));
290 const ITensorInfo &converted_weights =
291 weights_reshaped ? TensorInfo(weights->clone()->set_is_resizable(true).reset_padding())
292 : TensorInfo(*reshaped_weights.clone());
293 const ITensorInfo &gemmlowp_output = TensorInfo(
294 output->clone()->set_is_resizable(true).reset_padding().set_data_type(DataType::S32));
296 // Configure accumulate biases kernel for non quantized asymmetric types
297 if (biases != nullptr && !is_quantized)
299 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
300 ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMMatrixAccumulateBiasesKernel::validate(output, biases));
303 // With the Fully Connected layer we can have 4 different cases:
304 // 1) Convolution layer -> Fully Connected layer without batches
305 // 2) Fully Connected layer -> Fully Connected layer without batches
306 // 3) Convolution layer -> Fully Connected layer with batches
307 // 4) Fully Connected layer -> Fully Connected layer with batches
309 const ITensorInfo *input_to_use = input;
310 const ITensorInfo *weights_to_use = weights;
311 const ITensorInfo *tmp_output = (is_quantized) ? &gemmlowp_output : output;
313 // Check if we have a fully connected layer with batches
314 const bool is_batched_fc_layer = output->dimension(1) > 1;
316 if (is_batched_fc_layer)
318 is_fc_after_conv = (TensorShape::num_max_dimensions >= 4) &&
319 (std::equal(input->tensor_shape().cbegin() + 3, input->tensor_shape().cend(),
320 output->tensor_shape().cbegin() + 1));
324 is_fc_after_conv = input->num_dimensions() > 1;
327 if (!weights_reshaped)
329 // Validate reshape weights kernel
330 ARM_COMPUTE_RETURN_ON_ERROR(
331 NEFullyConnectedLayerReshapeWeights::validate(weights, &reshaped_weights));
332 weights_to_use = &reshaped_weights;
335 if (is_fc_after_conv && (input->data_layout() != fc_info.weights_trained_layout))
337 // Validate convert weights kernel
338 ARM_COMPUTE_RETURN_ON_ERROR(NEConvertFullyConnectedWeights::validate(
339 weights_to_use, &converted_weights, input->tensor_shape(), fc_info.weights_trained_layout));
340 weights_to_use = &converted_weights;
343 if (is_fc_after_conv)
345 // Fully Connected layer after a Convolution Layer without batches
346 ARM_COMPUTE_RETURN_ERROR_ON(
347 (weights_to_use->dimension(1) !=
348 (input->dimension(0) * input->dimension(1) * input->dimension(2))));
350 // Validate flatten kernel
351 ARM_COMPUTE_RETURN_ON_ERROR(NEFlattenLayerKernel::validate(input, &flatten_input));
352 input_to_use = &flatten_input;
356 // Fully Connected layer after a Fully Connected Layer without batches
357 ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(0) != weights_to_use->dimension(1));
359 // Validate matrix multiply kernel
360 ARM_COMPUTE_RETURN_ON_ERROR(validate_mm(*input_to_use, *weights_to_use, *tmp_output));
362 // Validate output stage for asymmetric quantized types
365 ARM_COMPUTE_RETURN_ON_ERROR(NEGEMMLowpQuantizeDownInt32ToUint8ScaleByFixedPoint::validate(
366 &gemmlowp_output, biases, output));
372 void NEFullyConnectedLayerEx::run()
376 if (!_are_weights_reshaped)
377 _reshape_weights_output.allocator()->allocate();
378 if (!_are_weights_converted)
379 _converted_weights_output.allocator()->allocate();
384 ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
386 // Reshape of the weights
387 if (!_are_weights_reshaped)
389 _reshape_weights_function.run();
392 // Convert weights if needed
393 if (!_are_weights_converted)
395 _convert_weights.run();
398 // Prepare GEMM prepare
405 MemoryGroupResourceScope scope_mg(_memory_group);
407 // Linearize input if it comes from a convolutional layer
408 if (_is_fc_after_conv)
410 NEScheduler::get().schedule(&_flatten_kernel, Window::DimY);
413 // Run matrix multiply
423 // Accumulate biases if provided
426 _gemmlowp_output_stage.run();
430 if (_accumulate_biases)
432 NEScheduler::get().schedule(&_accumulate_biases_kernel, Window::DimY);
437 void NEFullyConnectedLayerEx::prepare()
439 #if 0 // TODO Remove this block
442 ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
444 auto release_unused = [](Tensor *w) {
447 w->allocator()->free();
451 // Pointer to current weights
452 const ITensor *cur_weights = _original_weights;
454 // Reshape of the weights (happens only once)
455 if (!_are_weights_reshaped)
457 // Run reshape weights kernel and mark weights as unused
458 _reshape_weights_output.allocator()->allocate();
459 _reshape_weights_function.run();
461 cur_weights->mark_as_unused();
462 cur_weights = &_reshape_weights_output;
463 _are_weights_reshaped = true;
466 // Convert weights if needed (happens only once)
467 if (!_are_weights_converted)
469 _converted_weights_output.allocator()->allocate();
470 _convert_weights.run();
472 cur_weights->mark_as_unused();
473 _are_weights_converted = true;
476 // Release reshaped weights if unused
477 release_unused(&_reshape_weights_output);
479 // Prepare GEMM prepare and release unused weights
485 // Release converted weights if unused
486 release_unused(&_reshape_weights_output);
487 release_unused(&_converted_weights_output);