2 * Copyright (c) 2022 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "luci/CircleQuantizer.h"
19 #include "luci/Pass/CopyQuantParamPass.h"
20 #include "luci/Pass/ForceQuantParamPass.h"
21 #include "luci/Pass/PropagateQParamForwardPass.h"
22 #include "luci/Pass/RequantizePass.h"
23 #include "luci/Pass/ConvertToFakeQuantizedModelPass.h"
24 #include "luci/Pass/FoldDequantizePass.h"
25 #include "luci/Pass/RemoveRedundantDequantizePass.h"
26 #include "luci/Pass/QuantizePreCheckerPass.h"
27 #include "luci/Pass/QuantizeWithMinMaxPass.h"
28 #include "luci/Pass/QuantizeDequantizeWeightsPass.h"
30 #include "luci/Pass/CircleShapeInferencePass.h"
31 #include "luci/Pass/CircleTypeInferencePass.h"
34 #include <logo/RemoveDeadNodeWithQueryPass.h>
36 #include "ProgressReporter.h"
37 #include "helpers/Strings.h"
39 #include "QuantizedModelVerifier.h"
41 #include <luci/IR/CircleNode.h>
42 #include <logo/Phase.h>
43 #include <pepper/csv2vec.h>
51 using LayerParam = luci::CircleQuantizer::Options::LayerParam;
53 // This function updates user-given input_type to match with the input signature of graph
54 // If user gives only one input_type, it will be expanded to the number of graph inputs
55 void canonicalize_input_type(loco::Graph *g, std::vector<loco::DataType> &input_type)
60 const auto inputs = g->inputs();
62 assert(inputs); // FIX_CALLER_UNLESS
64 // Check validity of the number of input dtype given by a user
65 if (input_type.size() != 1 and input_type.size() != inputs->size())
67 throw std::runtime_error(
68 "Invalid number of input dtype. The number of input dtype should be 1 or "
69 "the same as the number of graph inputs.");
72 // Handle the case when a user gives only one input dtype
73 if (input_type.size() == 1)
75 const auto user_given_dtype = input_type[0];
78 // Expand input dtype to the number of graph inputs
79 // Since quantizer can only quantize float32, user_given_dtype is set only for float32 inputs
80 auto input_nodes = loco::input_nodes(g);
81 for (uint32_t i = 0; i < input_nodes.size(); i++)
83 auto input = loco::must_cast<luci::CircleInput *>(input_nodes[i]);
85 if (input->dtype() == loco::DataType::FLOAT32)
86 input_type.push_back(user_given_dtype);
88 input_type.push_back(input->dtype());
92 // Finally, check validity of input_type
93 // input_type is valid if
94 // C1. for non-float32 model input, input_type == model's input dtype
96 // C2. for float32 model input, input_type == uint8, int16, or float32
97 auto input_nodes = loco::input_nodes(g);
98 for (uint32_t i = 0; i < input_nodes.size(); i++)
100 auto input = loco::must_cast<luci::CircleInput *>(input_nodes[i]);
101 assert(i == input->index()); // FIX_ME_UNLESS
103 if (input->dtype() != loco::DataType::FLOAT32)
106 if (input->dtype() != input_type[i])
107 throw std::runtime_error(
108 "Input dtype of " + input->name() +
109 " is invalid. It has to be the same with the model's input dtype.");
114 if (input_type[i] != loco::DataType::FLOAT32 and input_type[i] != loco::DataType::U8 and
115 input_type[i] != loco::DataType::S16)
117 throw std::runtime_error("Input dtype of " + input->name() +
118 " is invalid. For float32 input, the input dtype after "
119 "quantization must be one of uint8, int16, or float32.");
125 // This function updates user-given output_type to match with the output signature of graph
126 // If user gives only one output_type, it will be expanded to the number of graph outputs
127 // NOTE This function is almost same with canonicalize_input_type, but it is written as a
128 // separate function for more precise error messaging.
129 // TODO Find a way to reduce duplicate codes
130 void canonicalize_output_type(loco::Graph *g, std::vector<loco::DataType> &output_type)
135 const auto outputs = g->outputs();
137 assert(outputs); // FIX_CALLER_UNLESS
139 // Check validity of the number of output dtype given by a user
140 if (output_type.size() != 1 and output_type.size() != outputs->size())
142 throw std::runtime_error(
143 "Invalid number of output dtype. The number of output dtype should be 1 or "
144 "the same as the number of graph outputs.");
147 // Handle the case when a user gives only one output dtype
148 if (output_type.size() == 1)
150 const auto user_given_dtype = output_type[0];
153 // Expand output dtype to the number of graph outputs
154 // If dtype of graph output is float32, it will be replaced with user_given_dtype
155 // Otherwise, it will not change
156 auto output_nodes = loco::output_nodes(g);
157 for (uint32_t i = 0; i < output_nodes.size(); i++)
159 auto output = loco::must_cast<luci::CircleOutput *>(output_nodes[i]);
161 if (output->dtype() == loco::DataType::FLOAT32)
162 output_type.push_back(user_given_dtype);
164 output_type.push_back(output->dtype());
168 // Finally, check validity of output_type
169 // output_type is valid if
170 // C1. for non-float32 model output, output_type == model's output dtype
172 // C2. for float32 model output, output_type == uint8, int16, or float32
173 auto output_nodes = loco::output_nodes(g);
174 for (uint32_t i = 0; i < output_nodes.size(); i++)
176 auto output = loco::must_cast<luci::CircleOutput *>(output_nodes[i]);
177 assert(i == output->index()); // FIX_ME_UNLESS
179 if (output->dtype() != loco::DataType::FLOAT32)
182 if (output->dtype() != output_type[i])
183 throw std::runtime_error(
184 "Output dtype of " + output->name() +
185 " is invalid. It has to be the same with the model's output dtype.");
190 if (output_type[i] != loco::DataType::FLOAT32 and output_type[i] != loco::DataType::U8 and
191 output_type[i] != loco::DataType::S16)
193 throw std::runtime_error("Output dtype of " + output->name() +
194 " is invalid. For float32 output, the output dtype after "
195 "quantization must be one of uint8, int16, or float32.");
201 template <typename T> T lexical_cast(const std::string &str)
203 std::istringstream ss;
210 template <typename T> std::vector<T> lexical_cast(std::vector<std::string> &sv)
212 std::vector<T> result;
213 std::transform(sv.begin(), sv.end(), std::back_inserter(result),
214 [](std::string str) -> T { return lexical_cast<T>(str); });
218 class QuantizeOptionsImpl final : public luci::CircleQuantizer::Options
221 void enable(Algorithm) final;
222 void param(AlgorithmParameters, const std::string &) final;
223 const std::string param(AlgorithmParameters) const final;
224 void params(AlgorithmParameters, std::vector<std::string> &) final;
225 std::vector<std::string> params(AlgorithmParameters) const final;
226 void layer_params(AlgorithmParameters, std::vector<std::shared_ptr<LayerParam>> &) final;
227 std::vector<std::shared_ptr<LayerParam>> layer_params(AlgorithmParameters) const final;
228 bool query(Algorithm) final;
231 std::vector<Algorithm> _algorithms;
232 std::map<AlgorithmParameters, const std::string> _algorithm_params;
233 std::map<AlgorithmParameters, std::vector<std::string>> _multiple_params;
234 std::map<AlgorithmParameters, std::vector<std::shared_ptr<LayerParam>>> _layer_params;
237 void QuantizeOptionsImpl::enable(Algorithm algo) { _algorithms.push_back(algo); }
239 void QuantizeOptionsImpl::param(AlgorithmParameters param, const std::string &str)
241 _algorithm_params.insert(std::pair<AlgorithmParameters, const std::string>(param, str));
244 const std::string QuantizeOptionsImpl::param(AlgorithmParameters param) const
246 auto param_str = _algorithm_params.find(param);
247 if (param_str != _algorithm_params.end())
249 return param_str->second;
253 return std::string();
257 void QuantizeOptionsImpl::params(AlgorithmParameters param, std::vector<std::string> &vec)
259 _multiple_params[param] = vec;
262 std::vector<std::string> QuantizeOptionsImpl::params(AlgorithmParameters param) const
264 auto param_vec = _multiple_params.find(param);
265 if (param_vec != _multiple_params.end())
267 return param_vec->second;
271 return std::vector<std::string>();
275 void QuantizeOptionsImpl::layer_params(AlgorithmParameters param,
276 std::vector<std::shared_ptr<LayerParam>> &vec)
278 _layer_params[param] = vec;
281 std::vector<std::shared_ptr<LayerParam>>
282 QuantizeOptionsImpl::layer_params(AlgorithmParameters param) const
284 auto param_vec = _layer_params.find(param);
285 if (param_vec != _layer_params.end())
287 return param_vec->second;
291 return std::vector<std::shared_ptr<LayerParam>>();
295 bool QuantizeOptionsImpl::query(Algorithm algo)
297 std::vector<Algorithm>::iterator it = std::find(_algorithms.begin(), _algorithms.end(), algo);
298 if (it == _algorithms.end())
309 CircleQuantizer::Options *CircleQuantizer::options(void)
311 if (_options == nullptr)
313 _options = std::make_unique<QuantizeOptionsImpl>();
316 return _options.get();
319 void CircleQuantizer::quantize(loco::Graph *g) const
321 // Fake quantization of weights
322 if (_options->query(Options::Algorithm::QuantizeDequantizeWeights))
324 static const std::vector<std::string> fakeq_supported_input_model_dtype{"float32"};
325 static const std::vector<std::string> fakeq_supported_output_model_dtype{"uint8", "int16"};
326 static const std::vector<std::string> fakeq_supported_granularity{"layer", "channel"};
328 auto input_model_dtype =
329 _options->param(Options::AlgorithmParameters::Quantize_input_model_dtype);
330 auto output_model_dtype =
331 _options->param(Options::AlgorithmParameters::Quantize_output_model_dtype);
332 auto granularity = _options->param(Options::AlgorithmParameters::Quantize_granularity);
333 auto layer_params = _options->layer_params(Options::AlgorithmParameters::Quantize_layer_params);
335 if (!in_array(to_lower_case(input_model_dtype), fakeq_supported_input_model_dtype))
336 throw std::runtime_error("Unsupported input type. List of supported input type: " +
337 to_string(fakeq_supported_input_model_dtype));
339 if (!in_array(to_lower_case(output_model_dtype), fakeq_supported_output_model_dtype))
340 throw std::runtime_error("Unsupported output type. List of supported output type: " +
341 to_string(fakeq_supported_output_model_dtype));
343 if (!in_array(to_lower_case(granularity), fakeq_supported_granularity))
344 throw std::runtime_error("Unsupported granularity. List of supported granularity: " +
345 to_string(fakeq_supported_granularity));
347 if (str_to_granularity(granularity) == QuantizationGranularity::LayerWise &&
348 str_to_dtype(output_model_dtype) != loco::DataType::U8)
349 throw std::runtime_error("Layer-wise quantization only supports uint8 dtype.");
351 // Check dtype/granularity of layer params
352 for (auto layer_param : layer_params)
354 auto name = layer_param->name;
355 if (!in_array(to_lower_case(layer_param->dtype), fakeq_supported_output_model_dtype))
357 throw std::runtime_error("Unsupported dtype in " + name + ". List of supported dtype: " +
358 to_string(fakeq_supported_output_model_dtype));
360 if (!in_array(to_lower_case(layer_param->granularity), fakeq_supported_granularity))
362 throw std::runtime_error(
363 "Unsupported granularity in " + name +
364 ". List of supported granularity: " + to_string(fakeq_supported_granularity));
368 // Clear existing quantparams before doing fake quantization
369 for (auto node : loco::active_nodes(loco::output_nodes(g)))
371 auto circle_node = loco::must_cast<luci::CircleNode *>(node);
372 if (circle_node->quantparam() != nullptr)
373 circle_node->quantparam(nullptr);
376 auto ctx = std::make_unique<luci::QuantizeDequantizeWeightsPass::Context>();
378 ctx->input_model_dtype = str_to_dtype(input_model_dtype);
379 ctx->output_model_dtype = str_to_dtype(output_model_dtype);
380 ctx->granularity = str_to_granularity(granularity);
382 for (auto layer_param : layer_params)
386 info.name = layer_param->name;
387 info.dtype = str_to_dtype(layer_param->dtype);
388 info.granularity = str_to_granularity(layer_param->granularity);
390 ctx->layers_info.emplace_back(info);
394 luci::QuantizeDequantizeWeightsPass fake_quantizer(std::move(ctx));
396 fake_quantizer.run(g);
399 // Actual quantization of weights, bias, and activation
400 if (_options->query(Options::Algorithm::QuantizeWithMinMax))
402 static const std::vector<std::string> qwmm_supported_input_model_dtype{"float32"};
403 static const std::vector<std::string> qwmm_supported_output_model_dtype{"uint8", "int16"};
404 static const std::vector<std::string> qwmm_supported_granularity{"layer", "channel"};
405 static const std::vector<std::string> qwmm_supported_input_type{"uint8", "int16", "int32",
406 "int64", "float32", "bool"};
407 static const std::vector<std::string> qwmm_supported_output_type{"uint8", "int16", "int32",
408 "int64", "float32", "bool"};
410 auto input_model_dtype =
411 _options->param(Options::AlgorithmParameters::Quantize_input_model_dtype);
412 auto output_model_dtype =
413 _options->param(Options::AlgorithmParameters::Quantize_output_model_dtype);
414 auto granularity = _options->param(Options::AlgorithmParameters::Quantize_granularity);
415 auto input_type = _options->param(Options::AlgorithmParameters::Quantize_input_type);
416 if (input_type.empty())
417 input_type = output_model_dtype;
418 auto output_type = _options->param(Options::AlgorithmParameters::Quantize_output_type);
419 if (output_type.empty())
420 output_type = output_model_dtype;
422 auto input_type_vec = pepper::csv_to_vector<std::string>(input_type);
423 auto output_type_vec = pepper::csv_to_vector<std::string>(output_type);
425 bool TF_style_maxpool =
426 _options->param(Options::AlgorithmParameters::Quantize_TF_style_maxpool) == "True";
428 auto layer_params = _options->layer_params(Options::AlgorithmParameters::Quantize_layer_params);
430 if (!in_array(to_lower_case(input_model_dtype), qwmm_supported_input_model_dtype))
431 throw std::runtime_error("Unsupported input type. List of supported input types: " +
432 to_string(qwmm_supported_input_model_dtype));
434 if (!in_array(to_lower_case(output_model_dtype), qwmm_supported_output_model_dtype))
435 throw std::runtime_error("Unsupported output type. List of supported output types: " +
436 to_string(qwmm_supported_output_model_dtype));
438 if (!in_array(to_lower_case(granularity), qwmm_supported_granularity))
439 throw std::runtime_error("Unsupported granularity. List of supported granularity: " +
440 to_string(qwmm_supported_granularity));
442 for (auto dtype : input_type_vec)
444 if (!in_array(to_lower_case(dtype), qwmm_supported_input_type))
445 throw std::runtime_error("Unsupported input type. List of supported input types: " +
446 to_string(qwmm_supported_input_type));
449 for (auto dtype : output_type_vec)
451 if (!in_array(to_lower_case(dtype), qwmm_supported_output_type))
452 throw std::runtime_error("Unsupported output type. List of supported output types: " +
453 to_string(qwmm_supported_output_type));
456 if (str_to_granularity(granularity) == QuantizationGranularity::LayerWise &&
457 str_to_dtype(output_model_dtype) != loco::DataType::U8)
458 throw std::runtime_error("Layer-wise quantization only supports uint8 dtype.");
460 // Check dtype/granularity of layer params
461 for (auto layer_param : layer_params)
463 auto name = layer_param->name;
464 if (!in_array(to_lower_case(layer_param->dtype), qwmm_supported_output_model_dtype))
466 throw std::runtime_error("Unsupported dtype in " + name + ". List of supported dtype: " +
467 to_string(qwmm_supported_output_model_dtype));
469 if (!in_array(to_lower_case(layer_param->granularity), qwmm_supported_granularity))
471 throw std::runtime_error(
472 "Unsupported granularity in " + name +
473 ". List of supported granularity: " + to_string(qwmm_supported_granularity));
477 auto input_types = str_vec_to_dtype_vec(input_type_vec);
478 auto output_types = str_vec_to_dtype_vec(output_type_vec);
480 // Canonicalize user-given input/output_type (match with # of inputs/outputs)
481 canonicalize_input_type(g, input_types);
482 canonicalize_output_type(g, output_types);
484 // Input model checker for quantization
485 luci::QuantizePreCheckerPass input_model_checker{};
486 input_model_checker.run(g);
488 auto ctx = std::make_unique<luci::QuantizeWithMinMaxPass::Context>();
490 ctx->input_model_dtype = str_to_dtype(input_model_dtype);
491 ctx->output_model_dtype = str_to_dtype(output_model_dtype);
492 ctx->granularity = str_to_granularity(granularity);
493 ctx->input_types = input_types;
494 ctx->output_types = output_types;
495 ctx->TF_style_maxpool = TF_style_maxpool;
497 for (auto layer_param : layer_params)
501 info.name = layer_param->name;
502 info.dtype = str_to_dtype(layer_param->dtype);
503 info.granularity = str_to_granularity(layer_param->granularity);
505 ctx->layers_info.emplace_back(info);
509 luci::QuantizeWithMinMaxPass quantizer(std::move(ctx));
513 auto verify_ctx = std::make_unique<luci::QuantizedModelVerifier::Context>();
515 verify_ctx->output_model_dtype = str_to_dtype(output_model_dtype);
516 verify_ctx->granularity = str_to_granularity(granularity);
517 verify_ctx->input_types = input_types;
518 verify_ctx->output_types = output_types;
519 verify_ctx->TF_style_maxpool = TF_style_maxpool;
521 for (auto layer_param : layer_params)
525 info.name = layer_param->name;
526 info.dtype = str_to_dtype(layer_param->dtype);
527 info.granularity = str_to_granularity(layer_param->granularity);
529 verify_ctx->layers_info.emplace_back(info);
533 // Verify the type/granularity of the quantized model
534 luci::QuantizedModelVerifier verifier(std::move(verify_ctx));
540 if (_options->query(Options::Algorithm::Requantize))
542 static const std::vector<std::string> rq_supported_input_model_dtype{"int8"};
543 static const std::vector<std::string> rq_supported_output_model_dtype{"uint8"};
545 auto input_model_dtype =
546 _options->param(Options::AlgorithmParameters::Quantize_input_model_dtype);
547 auto output_model_dtype =
548 _options->param(Options::AlgorithmParameters::Quantize_output_model_dtype);
550 if (!in_array(to_lower_case(input_model_dtype), rq_supported_input_model_dtype))
551 throw std::runtime_error("Unsupported input type. List of supported input types: " +
552 to_string(rq_supported_input_model_dtype));
554 if (!in_array(to_lower_case(output_model_dtype), rq_supported_output_model_dtype))
555 throw std::runtime_error("Unsupported output type. List of supported output types: " +
556 to_string(rq_supported_output_model_dtype));
558 luci::RequantizePass requantizer(str_to_dtype(input_model_dtype),
559 str_to_dtype(output_model_dtype));
563 // Force to write quantparam to specified tensors
564 // NOTE Only per-tensor (not per-channel) qparam can be written
565 if (_options->query(Options::Algorithm::ForceQuantParam))
567 ForceQuantParamPass::TensorVector tensors =
568 _options->params(Options::AlgorithmParameters::Quantize_tensor_names);
569 auto str_scales = _options->params(Options::AlgorithmParameters::Quantize_scales);
570 auto str_zero_points = _options->params(Options::AlgorithmParameters::Quantize_zero_points);
572 // Cast scales/zero_points to proper types
573 ForceQuantParamPass::ScaleVector scales = lexical_cast<float>(str_scales);
574 ForceQuantParamPass::ZPVector zero_points = lexical_cast<int64_t>(str_zero_points);
576 ForceQuantParamPass fq(tensors, scales, zero_points);
580 // Copy quantparam of a tensor to another tensor
581 if (_options->query(Options::Algorithm::CopyQuantParam))
583 CopyQuantParamPass::TensorVector src_tensors =
584 _options->params(Options::AlgorithmParameters::Quantize_src_tensor_names);
585 CopyQuantParamPass::TensorVector dst_tensors =
586 _options->params(Options::AlgorithmParameters::Quantize_dst_tensor_names);
588 CopyQuantParamPass cq(src_tensors, dst_tensors);
592 // Convert quantized model to fake-quantized model
593 if (_options->query(Options::Algorithm::ConvertToFakeQuantizedModel))
595 luci::ConvertToFakeQuantizedModelPass fake_quantizer;
596 fake_quantizer.run(g);
601 phase.emplace_back(std::make_unique<logo::RemoveDeadNodeWithQueryPass>());
602 phase.emplace_back(std::make_unique<luci::CircleShapeInferencePass>());
603 phase.emplace_back(std::make_unique<luci::CircleTypeInferencePass>());
605 // Remove redundant Dequantize Ops generated during fake quantization
606 phase.emplace_back(std::make_unique<luci::RemoveRedundantDequantizePass>());
607 // Fold Dequantize Ops generated during fake quantization
608 phase.emplace_back(std::make_unique<luci::FoldDequantizePass>());
610 ProgressReporter prog(g, logo::PhaseStrategy::Restart);
611 logo::PhaseRunner<logo::PhaseStrategy::Restart> phase_runner{g};
612 phase_runner.attach(&prog);
613 phase_runner.run(phase);
618 // Do Shape/Type inference
619 phase.emplace_back(std::make_unique<luci::CircleShapeInferencePass>());
620 phase.emplace_back(std::make_unique<luci::CircleTypeInferencePass>());
622 ProgressReporter prog(g, logo::PhaseStrategy::Saturate);
623 logo::PhaseRunner<logo::PhaseStrategy::Saturate> phase_runner{g};
624 phase_runner.attach(&prog);
625 phase_runner.run(phase);