2 * Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "QuantizeWeights.h"
18 #include "QuantizationUtils.h"
20 #include <luci/Service/Nodes/CircleConst.h>
33 using IterFunc = std::function<void(uint32_t *, loco::TensorShape &, int32_t)>;
35 void iterate_per_channel(CircleConst *node, int32_t &channel_dim_index, IterFunc func)
37 loco::TensorShape dimension;
39 uint32_t indices[4] = {
43 if (!get_channel_dim_index(node, dimension, channel_dim_index))
49 for (indices[0] = 0; indices[0] < dimension.dim(0).value(); indices[0]++)
51 for (indices[1] = 0; indices[1] < dimension.dim(1).value(); indices[1]++)
53 for (indices[2] = 0; indices[2] < dimension.dim(2).value(); indices[2]++)
55 for (indices[3] = 0; indices[3] < dimension.dim(3).value(); indices[3]++)
57 func(indices, dimension, channel_dim_index);
64 void asym_wquant_per_channel(CircleConst *node, std::vector<float> &min,
65 std::vector<float> &scaling_factor, int32_t &channel_dim_index)
67 assert(node->dtype() == loco::DataType::FLOAT32);
69 const int32_t kMinScale = 0;
70 const int32_t kMaxScale = 255;
72 uint32_t size = node->size<loco::DataType::FLOAT32>();
73 std::vector<int32_t> quantized_values(size);
75 auto quantize = [&](uint32_t *indices, loco::TensorShape &dimension, int32_t channel_dim_index) {
76 int channel_idx = indices[channel_dim_index];
77 const float scaling_factor_inv = 1.0 / scaling_factor[channel_idx];
78 auto data = node->at<loco::DataType::FLOAT32>(cal_offset(dimension, indices));
79 quantized_values[cal_offset(dimension, indices)] =
80 static_cast<int32_t>(std::round((data - min[channel_idx]) * scaling_factor_inv));
83 iterate_per_channel(node, channel_dim_index, quantize);
85 node->dtype(loco::DataType::U8); // change the type of tensor
86 node->size<loco::DataType::U8>(size); // resize tensor
87 for (uint32_t i = 0; i < size; ++i)
89 node->at<loco::DataType::U8>(i) = std::min(kMaxScale, std::max(kMinScale, quantized_values[i]));
93 // TODO Reduce duplicate code with QuantizeDequantizeWeights
94 void sym_wquant_per_channel(CircleConst *node, std::vector<float> &min, std::vector<float> &max,
95 std::vector<float> &scaling_factor, std::vector<float> &nudged_min,
96 std::vector<float> &nudged_max, int32_t &channel_dim_index)
98 assert(node->dtype() == loco::DataType::FLOAT32);
99 const int32_t kMaxScale = std::numeric_limits<int16_t>::max();
100 const int32_t kMinScale = -kMaxScale;
102 uint32_t size = node->size<loco::DataType::FLOAT32>();
103 std::vector<int32_t> quantized_values(size);
105 for (size_t i = 0; i < min.size(); ++i)
107 compute_sym_scale(min[i], max[i], scaling_factor[i], nudged_min[i], nudged_max[i]);
110 auto quantize = [&](uint32_t *indices, loco::TensorShape &dimension, int channel_dim_index) {
111 int channel_idx = indices[channel_dim_index];
112 const float scaling_factor_inv = 1.0 / scaling_factor[channel_idx];
113 auto data = node->at<loco::DataType::FLOAT32>(cal_offset(dimension, indices));
114 data = data < nudged_min[channel_idx] ? nudged_min[channel_idx] : data;
115 data = data > nudged_max[channel_idx] ? nudged_max[channel_idx] : data;
116 quantized_values[cal_offset(dimension, indices)] =
117 static_cast<int32_t>(std::round(data * scaling_factor_inv));
120 iterate_per_channel(node, channel_dim_index, quantize);
122 node->dtype(loco::DataType::S16); // change the type of tensor
123 node->size<loco::DataType::S16>(size); // resize tensor
124 for (uint32_t i = 0; i < size; ++i)
126 node->at<loco::DataType::S16>(i) =
127 std::min(kMaxScale, std::max(kMinScale, quantized_values[i]));
131 void cal_minmax_per_channel(CircleConst *node, std::vector<float> &min, std::vector<float> &max,
132 int32_t &channel_dim_index)
134 loco::TensorShape dimension;
137 if (!get_channel_dim_index(node, dimension, channel_dim_index))
139 throw std::runtime_error("Failed to find channel index in " + node->name());
141 auto size = dimension.dim(channel_dim_index).value();
143 std::vector<bool> has_min_max_value(size, false);
147 auto cal_minmax = [&](uint32_t *indices, loco::TensorShape &dimension, int channel_dim_index) {
148 int channel_idx = indices[channel_dim_index];
149 auto data = node->at<loco::DataType::FLOAT32>(cal_offset(dimension, indices));
150 if (has_min_max_value[channel_idx])
152 min[channel_idx] = data < min[channel_idx] ? data : min[channel_idx];
153 max[channel_idx] = data > max[channel_idx] ? data : max[channel_idx];
157 min[channel_idx] = data;
158 max[channel_idx] = data;
159 has_min_max_value[channel_idx] = true;
163 iterate_per_channel(node, channel_dim_index, cal_minmax);
166 void asymmetric_wquant_per_channel(CircleConst *node, std::vector<float> &min,
167 std::vector<float> &max, std::vector<float> &scaling_factor,
168 std::vector<int64_t> &zp, std::vector<float> &nudged_min,
169 std::vector<float> &nudged_max, int32_t &channel_dim_index)
171 assert(node->dtype() == loco::DataType::FLOAT32);
173 const int32_t kMinScale = 0;
174 const int32_t kMaxScale = 255;
176 uint32_t size = node->size<loco::DataType::FLOAT32>();
177 std::vector<int32_t> quantized_values(size);
179 for (size_t i = 0; i < min.size(); ++i)
181 compute_asym_scale_zp(min[i], max[i], scaling_factor[i], zp[i], nudged_min[i], nudged_max[i]);
184 auto quantize = [&](uint32_t *indices, loco::TensorShape &dimension, int channel_dim_index) {
185 int channel_idx = indices[channel_dim_index];
186 const float scaling_factor_inv = 1.0 / scaling_factor[channel_idx];
187 auto data = node->at<loco::DataType::FLOAT32>(cal_offset(dimension, indices));
188 data = data < nudged_min[channel_idx] ? nudged_min[channel_idx] : data;
189 data = data > nudged_max[channel_idx] ? nudged_max[channel_idx] : data;
190 quantized_values[cal_offset(dimension, indices)] =
191 static_cast<int32_t>(std::round((data - nudged_min[channel_idx]) * scaling_factor_inv));
194 iterate_per_channel(node, channel_dim_index, quantize);
196 node->dtype(loco::DataType::U8); // change the type of tensor
197 node->size<loco::DataType::U8>(size); // resize tensor
198 for (uint32_t i = 0; i < size; ++i)
200 node->at<loco::DataType::U8>(i) = std::min(kMaxScale, std::max(kMinScale, quantized_values[i]));
204 void sym_wquant_per_channel(CircleConst *node, std::vector<float> &scaling_factor,
205 int32_t &channel_dim_index)
207 assert(node->dtype() == loco::DataType::FLOAT32);
209 const int32_t kMaxScale = std::numeric_limits<int16_t>::max();
210 const int32_t kMinScale = -kMaxScale;
212 uint32_t size = node->size<loco::DataType::FLOAT32>();
213 std::vector<int32_t> quantized_values(size);
215 auto quantize = [&](uint32_t *indices, loco::TensorShape &dimension, int32_t channel_dim_index) {
216 int channel_idx = indices[channel_dim_index];
217 const float scaling_factor_inv = 1.0 / scaling_factor[channel_idx];
218 auto data = node->at<loco::DataType::FLOAT32>(cal_offset(dimension, indices));
219 quantized_values[cal_offset(dimension, indices)] =
220 static_cast<int32_t>(std::round(data * scaling_factor_inv));
223 iterate_per_channel(node, channel_dim_index, quantize);
225 node->dtype(loco::DataType::S16); // change the type of tensor
226 node->size<loco::DataType::S16>(size); // resize tensor
227 for (uint32_t i = 0; i < size; ++i)
229 node->at<loco::DataType::S16>(i) =
230 std::min(kMaxScale, std::max(kMinScale, quantized_values[i]));
234 void asym_wquant_per_layer(CircleConst *node, float min, float scaling_factor)
236 const int32_t kMinScale = 0;
237 const int32_t kMaxScale = 255;
239 uint32_t size = node->size<loco::DataType::FLOAT32>();
241 const float scaling_factor_inv = 1.0 / scaling_factor;
242 std::vector<int32_t> quantized_values(size);
243 for (uint32_t i = 0; i < size; ++i)
245 auto data = node->at<loco::DataType::FLOAT32>(i);
246 quantized_values[i] = static_cast<int32_t>(std::round((data - min) * scaling_factor_inv));
249 node->dtype(loco::DataType::U8); // change the type of tensor
250 node->size<loco::DataType::U8>(size); // resize tensor
251 for (uint32_t i = 0; i < size; ++i)
253 node->at<loco::DataType::U8>(i) = std::min(kMaxScale, std::max(kMinScale, quantized_values[i]));
257 // Quantize const per channel
259 // The last dimension of const is the same as the dimension of channel
260 // And the rest of the const dimensions should be 1
261 // So, a 'single value' is quantized per channel
263 // Quantization spec (f: fp value, q: quantized value)
266 // Positive f: f = f * (q - 0) [q = 1, scale = f, zp = 0]
267 // Negative f: f = (-f) * (q - 1) [q = 0, scale = -f, zp = 1]
270 // Positive f: f = f * (q - 0) [q = 1, scale = f, zp = 0]
271 // Negative f: f = (-f) * (q - 0) [q = -1, scale = -f, zp = 0]
272 void quant_const_per_channel(CircleConst *node, loco::DataType quant_type)
274 assert(node->dtype() == loco::DataType::FLOAT32);
275 assert(node->rank() > 0);
277 for (uint32_t i = 0; i < node->rank() - 1; i++)
279 // Caller should call this function when the below condition is satisfied
280 if (node->dim(i).value() != 1)
281 throw std::runtime_error("Non-channel dimension of const node must be 1");
284 uint32_t size = node->size<loco::DataType::FLOAT32>();
285 assert(size == node->dim(node->rank() - 1).value());
287 auto quantparam = std::make_unique<CircleQuantParam>();
288 quantparam->quantized_dimension = node->rank() - 1;
289 std::vector<int32_t> quantized_data(size);
291 for (uint32_t i = 0; i < size; ++i)
293 auto data = node->at<loco::DataType::FLOAT32>(i);
294 if (quant_type == loco::DataType::U8)
298 quantparam->scale.push_back(data);
299 quantparam->zerop.push_back(0);
300 quantized_data[i] = 1;
304 quantparam->scale.push_back(-data);
305 quantparam->zerop.push_back(1);
306 quantized_data[i] = 0;
309 else if (quant_type == loco::DataType::S16)
313 quantparam->scale.push_back(data);
314 quantized_data[i] = 1;
318 quantparam->scale.push_back(-data);
319 quantized_data[i] = -1;
321 quantparam->zerop.push_back(0);
324 node->quantparam(std::move(quantparam));
328 case loco::DataType::U8:
329 node->dtype(loco::DataType::U8);
330 node->size<loco::DataType::U8>(size);
331 for (uint32_t i = 0; i < size; ++i)
333 assert(quantized_data[i] == 0 || quantized_data[i] == 1);
334 node->at<loco::DataType::U8>(i) = quantized_data[i];
337 case loco::DataType::S16:
338 node->dtype(loco::DataType::S16);
339 node->size<loco::DataType::S16>(size);
340 for (uint32_t i = 0; i < size; ++i)
342 assert(quantized_data[i] == -1 || quantized_data[i] == 1);
343 node->at<loco::DataType::S16>(i) = quantized_data[i];
347 throw std::runtime_error("Unsupported data type");
356 void QuantizeWeights::quantize_weights(luci::CircleConst *weights)
358 // Find min/max per channel-wise
359 if (granularity == QuantizationGranularity::ChannelWise)
361 auto quantparam = weights->quantparam();
362 if (quantparam == nullptr)
364 // Find min/max on the fly
365 // NOTE This is for the case when QuantizeDequantizeWeights is skipped
366 // TODO Reduce duplicate codes
367 std::vector<float> min;
368 std::vector<float> max;
369 int32_t channel_dim_index = 0;
371 cal_minmax_per_channel(weights, min, max, channel_dim_index);
373 std::vector<float> nudged_min(min.size());
374 std::vector<float> nudged_max(min.size());
375 std::vector<float> scaling_factor(min.size());
376 std::vector<int64_t> zp(min.size());
378 if (output_type == loco::DataType::U8)
380 asymmetric_wquant_per_channel(weights, min, max, scaling_factor, zp, nudged_min, nudged_max,
385 sym_wquant_per_channel(weights, min, max, scaling_factor, nudged_min, nudged_max,
389 auto quantparam = std::make_unique<CircleQuantParam>();
390 quantparam->scale = scaling_factor;
391 quantparam->zerop = zp;
392 quantparam->quantized_dimension = channel_dim_index;
393 weights->quantparam(std::move(quantparam));
398 auto min = quantparam->min;
399 auto scaling_factor = quantparam->scale;
400 int32_t channel_dim_index = 0;
402 if (output_type == loco::DataType::U8)
404 asym_wquant_per_channel(weights, min, scaling_factor, channel_dim_index);
408 sym_wquant_per_channel(weights, scaling_factor, channel_dim_index);
410 quantparam->min.clear();
411 quantparam->max.clear();
412 quantparam->quantized_dimension = channel_dim_index;
414 // Find min/max per layer-wise
417 auto quantparam = weights->quantparam();
418 if (quantparam == nullptr)
420 // Find min/max on the fly
421 // NOTE This is for the case when QuantizeDequantizeWeights is skipped
422 // TODO Reduce duplicate codes
423 float min = std::numeric_limits<float>::max();
424 float max = std::numeric_limits<float>::lowest();
425 for (uint32_t i = 0; i < weights->size<loco::DataType::FLOAT32>(); i++)
427 auto data = weights->at<loco::DataType::FLOAT32>(i);
428 min = data < min ? data : min;
429 max = data > max ? data : max;
431 float scaling_factor{0};
436 asymmetric_wquant_with_minmax_per_layer(weights, min, max, scaling_factor, zp, nudged_min,
438 auto quantparam = std::make_unique<CircleQuantParam>();
439 quantparam->scale.push_back(scaling_factor);
440 quantparam->zerop.push_back(zp);
441 weights->quantparam(std::move(quantparam));
445 // Quantize using recorded quantparam
446 assert(quantparam != nullptr);
447 assert(quantparam->min.size() == 1); // only support layer-wise quant
448 assert(quantparam->scale.size() == 1); // only support layer-wise quant
449 auto min = quantparam->min[0];
450 auto scaling_factor = quantparam->scale[0];
451 asym_wquant_per_layer(weights, min, scaling_factor);
452 quantparam->min.clear();
453 quantparam->max.clear();
456 void QuantizeWeights::visit(luci::CircleConv2D *node)
459 INFO(l) << "QuantizeWeights QuantizeWeights::visit node: " << node->name() << std::endl;
461 auto weights = loco::must_cast<luci::CircleConst *>(node->filter());
462 if (!is_quantized(weights))
464 auto new_weights = luci::clone(weights);
465 node->filter(new_weights);
466 quantize_weights(new_weights);
470 void QuantizeWeights::visit(luci::CircleDepthwiseConv2D *node)
473 INFO(l) << "QuantizeWeights QuantizeWeights::visit node: " << node->name() << std::endl;
475 auto weights = loco::must_cast<luci::CircleConst *>(node->filter());
476 if (!is_quantized(weights))
478 auto new_weights = luci::clone(weights);
479 node->filter(new_weights);
480 quantize_weights(new_weights);
484 void QuantizeWeights::visit(luci::CircleInstanceNorm *node)
487 INFO(l) << "QuantizeWeights QuantizeWeights::visit node: " << node->name() << std::endl;
489 auto gamma = loco::must_cast<luci::CircleConst *>(node->gamma());
490 auto beta = loco::must_cast<luci::CircleConst *>(node->beta());
492 if (!is_quantized(gamma))
494 assert(gamma->dtype() == loco::DataType::FLOAT32);
495 auto new_gamma = luci::clone(gamma);
496 if (granularity == QuantizationGranularity::LayerWise)
497 quant_const(new_gamma, output_type);
498 else if (granularity == QuantizationGranularity::ChannelWise)
499 quant_const_per_channel(new_gamma, output_type);
500 node->gamma(new_gamma);
502 if (!is_quantized(beta))
504 assert(beta->dtype() == loco::DataType::FLOAT32);
505 auto new_beta = luci::clone(beta);
506 if (granularity == QuantizationGranularity::LayerWise)
507 quant_const(new_beta, output_type);
508 else if (granularity == QuantizationGranularity::ChannelWise)
509 quant_const_per_channel(new_beta, output_type);
510 node->beta(new_beta);
514 void QuantizeWeights::visit(luci::CirclePRelu *node)
517 INFO(l) << "QuantizeWeights QuantizeWeights::visit node: " << node->name() << std::endl;
519 auto alpha = loco::must_cast<luci::CircleConst *>(node->alpha());
521 if (!is_quantized(alpha))
523 assert(alpha->dtype() == loco::DataType::FLOAT32);
524 auto new_alpha = luci::clone(alpha);
525 if (granularity == QuantizationGranularity::LayerWise)
526 quant_const(new_alpha, output_type);
527 else if (granularity == QuantizationGranularity::ChannelWise)
528 quant_const_per_channel(new_alpha, output_type);
529 node->alpha(new_alpha);
533 void QuantizeWeights::visit(luci::CircleTransposeConv *node)
536 INFO(l) << "QuantizeWeights QuantizeWeights::visit node: " << node->name() << std::endl;
538 auto weights = loco::must_cast<luci::CircleConst *>(node->filter());
539 if (!is_quantized(weights))
541 auto new_weights = luci::clone(weights);
542 node->filter(new_weights);
543 quantize_weights(new_weights);
547 void QuantizeWeights::visit(luci::CircleFullyConnected *node)
550 INFO(l) << "QuantizeWeights QuantizeWeights::visit node: " << node->name() << std::endl;
552 auto weights = loco::must_cast<luci::CircleConst *>(node->weights());
553 if (!is_quantized(weights))
555 auto new_weights = luci::clone(weights);
556 node->weights(new_weights);
557 quantize_weights(new_weights);
561 void QuantizeWeights::visit(luci::CircleNode *) {}