2 * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
3 * Copyright 2019 The TensorFlow Authors. All Rights Reserved.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 #include "kernels/Conv2D.h"
20 #include "kernels/Utils.h"
22 #include <tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h>
27 namespace luci_interpreter
32 Conv2D::Conv2D(const Tensor *input, const Tensor *filter, const Tensor *bias, Tensor *output,
33 const Conv2DParams ¶ms)
34 : KernelWithParams<Conv2DParams>({input, filter, bias}, {output}, params)
38 void Conv2D::configure()
40 // TensorFlow Lite (as of v2.2.0) supports the following combinations of types:
41 // | input filter bias output |
42 // ----+---------------------------+
43 // (1) | float float float float |
44 // (2) | float int8 float float | hybrid
45 // (3) | uint8 uint8 int32 uint8 | quantized
46 // (4) | int8 int8 int32 int8 | quantized per channel
48 // We only support (1) and (3) for now, and additionally the following:
49 // | input filter bias output |
50 // ----+---------------------------+
51 // (5) | int16 int16 int64 int16 |
53 if (input()->element_type() == DataType::FLOAT32 && filter()->element_type() == DataType::FLOAT32)
55 LUCI_INTERPRETER_CHECK(bias() == nullptr || bias()->element_type() == DataType::FLOAT32);
57 else if (input()->element_type() == DataType::U8 && filter()->element_type() == DataType::U8)
59 LUCI_INTERPRETER_CHECK(bias() == nullptr || bias()->element_type() == DataType::S32);
61 else if (input()->element_type() == DataType::S16 && filter()->element_type() == DataType::S16)
63 LUCI_INTERPRETER_CHECK(bias() == nullptr || bias()->element_type() == DataType::S64);
67 throw std::runtime_error("Unsupported type.");
69 LUCI_INTERPRETER_CHECK(output()->element_type() == input()->element_type());
71 const Shape &input_shape = input()->shape();
72 const Shape &filter_shape = filter()->shape();
73 LUCI_INTERPRETER_CHECK(input_shape.num_dims() == 4 && filter_shape.num_dims() == 4);
75 const int32_t batches = input_shape.dim(0);
76 const int32_t input_height = input_shape.dim(1);
77 const int32_t input_width = input_shape.dim(2);
78 const int32_t output_depth = filter_shape.dim(0);
79 const int32_t filter_height = filter_shape.dim(1);
80 const int32_t filter_width = filter_shape.dim(2);
81 LUCI_INTERPRETER_CHECK(filter_shape.dim(3) == input_shape.dim(3));
83 LUCI_INTERPRETER_CHECK(bias() == nullptr || (bias()->shape().num_dims() == 1 &&
84 bias()->shape().dim(0) == output_depth));
86 const int32_t output_height =
87 computeOutputSize(_params.padding, input_height, filter_height, _params.stride_height,
88 _params.dilation_height_factor);
89 const int32_t output_width =
90 computeOutputSize(_params.padding, input_width, filter_width, _params.stride_width,
91 _params.dilation_width_factor);
93 _padding_height = computePadding(_params.stride_height, _params.dilation_height_factor,
94 input_height, filter_height, output_height);
95 _padding_width = computePadding(_params.stride_width, _params.dilation_width_factor, input_width,
96 filter_width, output_width);
98 output()->resize({batches, output_height, output_width, output_depth});
100 // Allocate tensor for Im2Col, if needed.
101 // The checks here should be aligned with the actual implementation.
102 const bool need_dilated_im2col =
103 _params.dilation_height_factor != 1 || _params.dilation_width_factor != 1;
104 const bool need_non_dilated_im2col = _params.stride_height != 1 || _params.stride_width != 1 ||
105 filter_height != 1 || filter_width != 1;
106 const bool need_im2col =
107 input()->element_type() != DataType::S16 && (need_dilated_im2col || need_non_dilated_im2col);
110 const int input_depth = input_shape.dim(3);
111 Shape im2col_shape{batches, output_height, output_width,
112 input_depth * filter_height * filter_width};
116 std::make_unique<Tensor>(input()->element_type(), im2col_shape, AffineQuantization{}, "");
118 catch (std::bad_alloc &ba)
120 // Failed memory allocation
126 void Conv2D::execute() const
128 switch (input()->element_type())
130 case DataType::FLOAT32:
131 if (filter()->element_type() == DataType::FLOAT32)
136 throw std::runtime_error("Unsupported type.");
138 if (filter()->scales().size() == 1)
142 else if (filter()->scales().size() > 1)
144 LUCI_INTERPRETER_CHECK(filter()->shape().num_dims() == 4);
145 LUCI_INTERPRETER_CHECK(filter()->scales().size() ==
146 static_cast<size_t>(filter()->shape().dim(0)));
147 evalQuantizedPerChannel();
154 throw std::runtime_error("Unsupported type.");
157 _im2col->deallocate();
160 void Conv2D::evalFloat() const
162 float activation_min{};
163 float activation_max{};
164 calculateActivationRange(_params.activation, &activation_min, &activation_max);
166 tflite::ConvParams params{};
167 params.padding_values.height = _padding_height;
168 params.padding_values.width = _padding_width;
169 params.stride_height = _params.stride_height;
170 params.stride_width = _params.stride_width;
171 params.dilation_height_factor = _params.dilation_height_factor;
172 params.dilation_width_factor = _params.dilation_width_factor;
173 params.float_activation_min = activation_min;
174 params.float_activation_max = activation_max;
180 tflite::optimized_ops::Conv(
181 params, getTensorShape(input()), getTensorData<float>(input()), getTensorShape(filter()),
182 getTensorData<float>(filter()), getTensorShape(bias()), getTensorData<float>(bias()),
183 getTensorShape(output()), getTensorData<float>(output()), getTensorShape(_im2col.get()),
184 getTensorData<float>(_im2col.get()));
186 catch (std::bad_alloc &ba)
188 // Failed memory allocation
189 _im2col->deallocate();
191 tflite::reference_ops::Conv(
192 params, getTensorShape(input()), getTensorData<float>(input()), getTensorShape(filter()),
193 getTensorData<float>(filter()), getTensorShape(bias()), getTensorData<float>(bias()),
194 getTensorShape(output()), getTensorData<float>(output()), tflite::RuntimeShape(), nullptr);
198 tflite::reference_ops::Conv(
199 params, getTensorShape(input()), getTensorData<float>(input()), getTensorShape(filter()),
200 getTensorData<float>(filter()), getTensorShape(bias()), getTensorData<float>(bias()),
201 getTensorShape(output()), getTensorData<float>(output()), tflite::RuntimeShape(), nullptr);
204 void Conv2D::evalQuantized() const
206 const auto input_scale = static_cast<double>(input()->scale());
207 const auto filter_scale = static_cast<double>(filter()->scale());
208 const auto output_scale = static_cast<double>(output()->scale());
210 const double real_multiplier = input_scale * filter_scale / output_scale;
211 int32_t output_multiplier{};
213 quantizeMultiplier(real_multiplier, &output_multiplier, &output_shift);
215 int32_t activation_min{};
216 int32_t activation_max{};
217 calculateActivationRangeQuantized(_params.activation, output(), &activation_min, &activation_max);
219 tflite::ConvParams params{};
220 params.padding_values.height = _padding_height;
221 params.padding_values.width = _padding_width;
222 params.stride_height = _params.stride_height;
223 params.stride_width = _params.stride_width;
224 params.dilation_height_factor = _params.dilation_height_factor;
225 params.dilation_width_factor = _params.dilation_width_factor;
226 // The kernel expects input and filter zero points to be negated.
227 params.input_offset = -input()->zero_point(); // Note the '-'.
228 params.weights_offset = -filter()->zero_point(); // Note the '-'.
229 params.output_offset = output()->zero_point();
230 params.output_multiplier = output_multiplier;
231 params.output_shift = output_shift;
232 params.quantized_activation_min = activation_min;
233 params.quantized_activation_max = activation_max;
235 // TODO This should only be done once (although it takes only a few microseconds).
236 // Also, the user should be able to adjust the number of threads.
237 auto gemmlowp_context = std::make_unique<gemmlowp::GemmContext>();
238 gemmlowp_context->set_max_num_threads(static_cast<int>(std::thread::hardware_concurrency()));
240 tflite::optimized_ops::Conv(
241 params, getTensorShape(input()), getTensorData<uint8_t>(input()), getTensorShape(filter()),
242 getTensorData<uint8_t>(filter()), getTensorShape(bias()), getTensorData<int32_t>(bias()),
243 getTensorShape(output()), getTensorData<uint8_t>(output()), getTensorShape(_im2col.get()),
244 getTensorData<uint8_t>(_im2col.get()), gemmlowp_context.get());
247 void Conv2D::evalQuantizedPerChannel() const
249 const auto *input_data = getTensorData<uint8_t>(input());
250 const auto *filter_data = getTensorData<uint8_t>(filter());
251 const auto *bias_data = getTensorData<int32_t>(bias());
252 auto *output_data = getTensorData<uint8_t>(output());
254 const Shape &input_shape = input()->shape();
255 const Shape &filter_shape = filter()->shape();
256 const Shape &output_shape = output()->shape();
258 const int32_t batches = input_shape.dim(0);
259 const int32_t input_height = input_shape.dim(1);
260 const int32_t input_width = input_shape.dim(2);
261 const int32_t input_depth = input_shape.dim(3);
262 const int32_t output_depth = filter_shape.dim(0);
263 const int32_t filter_height = filter_shape.dim(1);
264 const int32_t filter_width = filter_shape.dim(2);
265 const int32_t output_height = output_shape.dim(1);
266 const int32_t output_width = output_shape.dim(2);
268 const int32_t stride_height = _params.stride_height;
269 const int32_t stride_width = _params.stride_width;
270 const int32_t dilation_height_factor = _params.dilation_height_factor;
271 const int32_t dilation_width_factor = _params.dilation_width_factor;
273 int32_t activation_min{};
274 int32_t activation_max{};
275 calculateActivationRangeQuantized(_params.activation, output(), &activation_min, &activation_max);
277 const std::vector<double> effective_output_scale =
278 getQuantizedConvolutionMultiplers(input()->scale(), filter()->scales(), output()->scale());
280 const std::vector<ChannelQuantMultipliers> multipliers_raw =
281 quantizeMultipliers(effective_output_scale);
282 BroadcastableWrapper<ChannelQuantMultipliers> quant_multipliers(multipliers_raw);
284 for (int32_t batch = 0; batch < batches; ++batch)
286 for (int32_t out_y = 0; out_y < output_height; ++out_y)
288 for (int32_t out_x = 0; out_x < output_width; ++out_x)
290 for (int32_t out_c = 0; out_c < output_depth; ++out_c)
292 const int32_t in_y_origin = out_y * stride_height - _padding_height;
293 const int32_t in_x_origin = out_x * stride_width - _padding_width;
295 for (int32_t filter_y = 0; filter_y < filter_height; ++filter_y)
297 for (int32_t filter_x = 0; filter_x < filter_width; ++filter_x)
299 const int32_t in_y = in_y_origin + dilation_height_factor * filter_y;
300 const int32_t in_x = in_x_origin + dilation_width_factor * filter_x;
301 if ((in_y >= 0 && in_y < input_height) && (in_x >= 0 && in_x < input_width))
303 for (int32_t in_c = 0; in_c < input_depth; ++in_c)
305 const uint8_t input_val =
306 input_data[calcOffset(input_shape, batch, in_y, in_x, in_c)];
307 const uint8_t filter_val =
308 filter_data[calcOffset(filter_shape, out_c, filter_y, filter_x, in_c)];
309 acc += static_cast<int32_t>(input_val - input()->zero_point()) *
310 static_cast<int32_t>(filter_val - filter()->zero_points()[out_c]);
317 acc += bias_data[out_c];
320 int32_t scaled_acc = tflite::MultiplyByQuantizedMultiplier(
321 acc, quant_multipliers[out_c].multiplier, quant_multipliers[out_c].shift);
323 scaled_acc += output()->zero_point();
324 scaled_acc = std::max(scaled_acc, activation_min);
325 scaled_acc = std::min(scaled_acc, activation_max);
326 output_data[calcOffset(output_shape, batch, out_y, out_x, out_c)] = scaled_acc;
333 void Conv2D::evalQuantizedS16() const
335 const auto *input_data = getTensorData<int16_t>(input());
336 const auto *filter_data = getTensorData<int16_t>(filter());
337 const auto *bias_data = getTensorData<int64_t>(bias());
338 auto *output_data = getTensorData<int16_t>(output());
340 const Shape &input_shape = input()->shape();
341 const Shape &filter_shape = filter()->shape();
342 const Shape &output_shape = output()->shape();
344 const int32_t batches = input_shape.dim(0);
345 const int32_t input_height = input_shape.dim(1);
346 const int32_t input_width = input_shape.dim(2);
347 const int32_t input_depth = input_shape.dim(3);
348 const int32_t output_depth = filter_shape.dim(0);
349 const int32_t filter_height = filter_shape.dim(1);
350 const int32_t filter_width = filter_shape.dim(2);
351 const int32_t output_height = output_shape.dim(1);
352 const int32_t output_width = output_shape.dim(2);
354 const int32_t stride_height = _params.stride_height;
355 const int32_t stride_width = _params.stride_width;
356 const int32_t dilation_height_factor = _params.dilation_height_factor;
357 const int32_t dilation_width_factor = _params.dilation_width_factor;
359 int32_t activation_min{};
360 int32_t activation_max{};
361 calculateActivationRangeQuantized(_params.activation, output(), &activation_min, &activation_max);
363 const std::vector<double> effective_output_scale =
364 getQuantizedConvolutionMultiplers(input()->scale(), filter()->scales(), output()->scale());
366 const std::vector<ChannelQuantMultipliers> multipliers_raw =
367 quantizeMultipliers(effective_output_scale);
368 BroadcastableWrapper<ChannelQuantMultipliers> multipliers(multipliers_raw);
370 for (int32_t batch = 0; batch < batches; ++batch)
372 for (int32_t out_y = 0; out_y < output_height; ++out_y)
374 for (int32_t out_x = 0; out_x < output_width; ++out_x)
376 for (int32_t out_c = 0; out_c < output_depth; ++out_c)
378 const int32_t in_y_origin = out_y * stride_height - _padding_height;
379 const int32_t in_x_origin = out_x * stride_width - _padding_width;
381 for (int32_t filter_y = 0; filter_y < filter_height; ++filter_y)
383 for (int32_t filter_x = 0; filter_x < filter_width; ++filter_x)
385 const int32_t in_y = in_y_origin + dilation_height_factor * filter_y;
386 const int32_t in_x = in_x_origin + dilation_width_factor * filter_x;
387 if ((in_y >= 0 && in_y < input_height) && (in_x >= 0 && in_x < input_width))
389 for (int32_t in_c = 0; in_c < input_depth; ++in_c)
391 const int16_t input_val =
392 input_data[calcOffset(input_shape, batch, in_y, in_x, in_c)];
393 const int16_t filter_val =
394 filter_data[calcOffset(filter_shape, out_c, filter_y, filter_x, in_c)];
395 acc += static_cast<int64_t>(input_val) * static_cast<int64_t>(filter_val);
402 acc += bias_data[out_c];
405 int32_t scaled_acc = tflite::MultiplyByQuantizedMultiplier(
406 acc, multipliers[out_c].multiplier, multipliers[out_c].shift);
408 scaled_acc = std::max(scaled_acc, activation_min);
409 scaled_acc = std::min(scaled_acc, activation_max);
411 output_data[calcOffset(output_shape, batch, out_y, out_x, out_c)] = scaled_acc;
418 } // namespace kernels
419 } // namespace luci_interpreter