2 * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * Copyright (c) 2017-2019 ARM Limited.
20 * SPDX-License-Identifier: MIT
22 * Permission is hereby granted, free of charge, to any person obtaining a copy
23 * of this software and associated documentation files (the "Software"), to
24 * deal in the Software without restriction, including without limitation the
25 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
26 * sell copies of the Software, and to permit persons to whom the Software is
27 * furnished to do so, subject to the following conditions:
29 * The above copyright notice and this permission notice shall be included in all
30 * copies or substantial portions of the Software.
32 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41 #include "arm_compute/runtime/CL/functions/CLFullyConnectedHybridLayer.h"
43 #include "arm_compute/core/Size2D.h"
44 #include "arm_compute/core/Validate.h"
45 #include "arm_compute/core/utils/misc/ShapeCalculator.h"
46 #include "arm_compute/core/utils/quantization/AsymmHelpers.h"
47 #include "arm_compute/runtime/CL/CLScheduler.h"
48 #include "support/MemorySupport.h"
52 using namespace arm_compute;
53 using namespace arm_compute::misc::shape_calculator;
57 Status validate_mm(const ITensorInfo &input, const ITensorInfo &weights, const ITensorInfo &output)
59 ARM_COMPUTE_UNUSED(input);
60 ARM_COMPUTE_UNUSED(weights);
61 ARM_COMPUTE_UNUSED(output);
62 ARM_COMPUTE_RETURN_ON_ERROR(
63 CLGEMMLowpMatrixMultiplyCore::validate(&input, &weights, nullptr, &output));
69 void CLFullyConnectedHybridLayerReshapeWeights::configure(const ICLTensor *input, ICLTensor *output)
71 auto k = support::cpp14::make_unique<CLTransposeKernel>();
72 k->configure(input, output);
73 _kernel = std::move(k);
76 Status CLFullyConnectedHybridLayerReshapeWeights::validate(const ITensorInfo *input,
77 const ITensorInfo *output)
79 return CLTransposeKernel::validate(input, output);
82 CLFullyConnectedHybridLayer::CLFullyConnectedHybridLayer(
83 std::shared_ptr<IMemoryManager> memory_manager)
84 : _memory_group(memory_manager), _reshape_weights_kernel(), _quant_input_kernel(),
85 _mm_gemmlowp(memory_manager), _multiply_scale_kernel(), _accumulate_biases_kernel(),
86 _reshape_weights_output(), _quantized_input(), _scale_factor(), _gemmlowp_output(),
87 _are_weights_reshaped(true), _accumulate_biases(false), _is_prepared(false),
88 _original_weights(nullptr)
91 void CLFullyConnectedHybridLayer::configure_mm(const ICLTensor *input, const ICLTensor *weights,
92 ICLTensor *output, bool retain_internal_weights)
94 ARM_COMPUTE_ERROR_ON(input->info()->dimension(0) != weights->info()->dimension(1));
96 ARM_COMPUTE_UNUSED(output);
97 ARM_COMPUTE_UNUSED(retain_internal_weights);
98 // Configure gemmlowp function
99 _mm_gemmlowp.configure(input, weights, nullptr, output);
102 void CLFullyConnectedHybridLayer::configure(const ICLTensor *input, const ICLTensor *weights,
103 const ICLTensor *biases, ICLTensor *output,
104 FullyConnectedLayerInfo fc_info)
106 ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
108 // Perform validate step
109 ARM_COMPUTE_ERROR_THROW_ON(CLFullyConnectedHybridLayer::validate(
110 input->info(), weights->info(), biases != nullptr ? biases->info() : nullptr, output->info(),
113 _are_weights_reshaped = fc_info.transpose_weights ? fc_info.are_weights_reshaped : true;
114 _accumulate_biases = false;
115 _is_prepared = fc_info.retain_internal_weights;
116 _original_weights = weights;
118 // Configure accumulate biases kernel for non quantized asymmetric types
119 if (biases != nullptr)
121 ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
123 _accumulate_biases = true;
125 // Configure accumulate biases kernel
126 _accumulate_biases_kernel.set_target(CLScheduler::get().target());
127 _accumulate_biases_kernel.configure(output, biases);
130 const ICLTensor *weights_to_use = weights;
132 // With the Fully Connected layer we can have 4 different cases:
133 // 1) Convolution layer -> Fully Connected layer without batches
134 // 2) Fully Connected layer -> Fully Connected layer without batches
135 // 3) Convolution layer -> Fully Connected layer with batches
136 // 4) Fully Connected layer -> Fully Connected layer with batches
138 // Check if we have a fully connected layer with batches
139 const bool is_batched_fc_layer = output->info()->dimension(1) > 1;
140 bool is_fc_after_conv = false;
141 if (is_batched_fc_layer)
143 is_fc_after_conv = (TensorShape::num_max_dimensions >= 4) &&
144 (std::equal(input->info()->tensor_shape().cbegin() + 3,
145 input->info()->tensor_shape().cend(),
146 output->info()->tensor_shape().cbegin() + 1));
150 is_fc_after_conv = input->info()->num_dimensions() > 1 && input->info()->dimension(1) > 1;
152 ARM_COMPUTE_ERROR_ON_MSG(is_fc_after_conv,
153 "CLFullyConnectedHybridLayer does not support after conv");
154 ARM_COMPUTE_UNUSED(is_fc_after_conv);
156 // Reshape weights if needed
157 if (!_are_weights_reshaped)
159 // Reshape the weights
160 _reshape_weights_output.allocator()->init(
161 weights->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(
162 compute_transposed_shape(*weights->info())));
163 _reshape_weights_kernel.configure(weights_to_use, &_reshape_weights_output);
164 weights_to_use = &_reshape_weights_output;
167 // Extract scale factor
168 _scale_factor.allocator()->init(
169 TensorInfo(TensorShape{output->info()->dimension(1)}, 1, input->info()->data_type()));
170 _memory_group.manage(&_scale_factor);
171 _scale_factor_kernel.configure(input, &_scale_factor);
174 _quantized_input.allocator()->init(
175 input->info()->clone()->set_is_resizable(true).reset_padding().set_data_type(
176 DataType::QASYMM8_SIGNED));
177 _memory_group.manage(&_quantized_input);
178 _quant_input_kernel.configure(input, &_scale_factor, &_quantized_input);
181 _gemmlowp_output.allocator()->init(
182 output->info()->clone()->set_is_resizable(true).reset_padding().set_data_type(DataType::S32));
183 _memory_group.manage(&_gemmlowp_output);
184 configure_mm(&_quantized_input, weights_to_use, &_gemmlowp_output,
185 fc_info.retain_internal_weights);
186 _quantized_input.allocator()->allocate();
189 _multiply_scale_kernel.configure(&_gemmlowp_output, &_scale_factor, output,
190 weights->info()->quantization_info().uniform().scale);
191 _gemmlowp_output.allocator()->allocate();
192 _scale_factor.allocator()->allocate();
194 _are_weights_reshaped = _are_weights_reshaped || fc_info.retain_internal_weights;
197 Status CLFullyConnectedHybridLayer::validate(const ITensorInfo *input, const ITensorInfo *weights,
198 const ITensorInfo *biases, const ITensorInfo *output,
199 FullyConnectedLayerInfo fc_info)
201 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output);
202 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::F16, DataType::F32);
203 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(weights, 1, DataType::QASYMM8_SIGNED);
204 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
205 ARM_COMPUTE_RETURN_ERROR_ON(weights->num_dimensions() > 2);
207 bool weights_reshaped = fc_info.transpose_weights ? fc_info.are_weights_reshaped : true;
208 bool is_fc_after_conv = true;
209 const GPUTarget gpu_target = CLScheduler::get().target();
211 const ITensorInfo &reshaped_weights =
212 TensorInfo(weights->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(
213 compute_transposed_shape(*weights)));
215 // Configure accumulate biases kernel for non quantized asymmetric types
216 if (biases != nullptr)
218 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, biases);
219 ARM_COMPUTE_RETURN_ON_ERROR(
220 CLGEMMMatrixAccumulateBiasesKernel::validate(output, biases, gpu_target));
223 // With the Fully Connected layer we can have 4 different cases:
224 // 1) Convolution layer -> Fully Connected layer without batches
225 // 2) Fully Connected layer -> Fully Connected layer without batches
226 // 3) Convolution layer -> Fully Connected layer with batches
227 // 4) Fully Connected layer -> Fully Connected layer with batches
229 const ITensorInfo *weights_to_use = weights;
231 // Check if we have a fully connected layer with batches
232 const bool is_batched_fc_layer = output->dimension(1) > 1;
233 if (is_batched_fc_layer)
235 is_fc_after_conv = (TensorShape::num_max_dimensions >= 4) &&
236 (std::equal(input->tensor_shape().cbegin() + 3, input->tensor_shape().cend(),
237 output->tensor_shape().cbegin() + 1));
241 is_fc_after_conv = input->num_dimensions() > 1 && input->dimension(1) > 1;
243 ARM_COMPUTE_RETURN_ERROR_ON_MSG(is_fc_after_conv,
244 "CLFullyConnectedHybridLayer does not support after conv");
246 if (!weights_reshaped)
248 // Validate reshape weights kernel
249 ARM_COMPUTE_RETURN_ON_ERROR(
250 CLFullyConnectedHybridLayerReshapeWeights::validate(weights_to_use, &reshaped_weights));
251 weights_to_use = &reshaped_weights;
254 // Validate Scale factor kernel
255 const ITensorInfo &scale_factor =
256 TensorInfo(TensorShape{output->dimension(1)}, 1, input->data_type());
257 ARM_COMPUTE_RETURN_ON_ERROR(CLScaleFactorSymm8Kernel::validate(input, &scale_factor));
259 // Validate quantization symm8 kernel
260 const ITensorInfo &quantized_input =
261 TensorInfo(input->clone()->set_is_resizable(true).reset_padding().set_data_type(
262 DataType::QASYMM8_SIGNED));
263 ARM_COMPUTE_RETURN_ON_ERROR(
264 CLQuantizationSymmetricKernel::validate(input, &scale_factor, &quantized_input));
266 // Fully Connected layer after a Fully Connected Layer without batches
267 ARM_COMPUTE_RETURN_ERROR_ON(input->dimension(0) != weights_to_use->dimension(1));
269 // Validate matrix multiply kernel
270 const ITensorInfo &gemmlowp_output = TensorInfo(
271 output->clone()->set_is_resizable(true).reset_padding().set_data_type(DataType::S32));
272 ARM_COMPUTE_RETURN_ON_ERROR(validate_mm(quantized_input, *weights_to_use, gemmlowp_output));
275 ARM_COMPUTE_RETURN_ON_ERROR(
276 CLMultiplyScaleFactorKernel::validate(&gemmlowp_output, &scale_factor, output));
281 void CLFullyConnectedHybridLayer::run()
285 MemoryGroupResourceScope scope_mg(_memory_group);
287 // Extract scale_factor
288 CLScheduler::get().enqueue(_scale_factor_kernel);
291 CLScheduler::get().enqueue(_quant_input_kernel);
293 // Run matrix multiply
296 // Multiply scale factor
297 CLScheduler::get().enqueue(_multiply_scale_kernel);
299 // Accumulate biases if provided
300 if (_accumulate_biases)
302 CLScheduler::get().enqueue(_accumulate_biases_kernel);
306 void CLFullyConnectedHybridLayer::prepare()
310 ARM_COMPUTE_ERROR_ON(!_original_weights->is_used());
312 auto release_unused = [](CLTensor *w) {
315 CLScheduler::get().queue().finish();
316 w->allocator()->free();
320 // Reshape of the weights if needed (happens only once)
321 if (!_are_weights_reshaped)
323 // Run reshape weights kernel and mark weights as unused
324 _reshape_weights_output.allocator()->allocate();
325 _reshape_weights_kernel.run();
327 _are_weights_reshaped = true;
328 // We can not release _original_weights because it can be used in other nodes
331 // Prepare GEMM prepare and release unused weights
332 _mm_gemmlowp.prepare();
334 // Release reshaped weights if unused
335 release_unused(&_reshape_weights_output);