/*! \brief Attribute for requantize operator */
struct RequantizeAttrs : public tvm::AttrsNode<RequantizeAttrs> {
- double input_scale;
- int32_t input_zero_point;
- double output_scale;
- int32_t output_zero_point;
std::string rounding;
DataType out_dtype;
TVM_DECLARE_ATTRS(RequantizeAttrs, "relay.attrs.RequantizeAttrs") {
- TVM_ATTR_FIELD(input_scale)
- .describe("The scale of the input tensor.");
- TVM_ATTR_FIELD(input_zero_point)
- .describe("The zero point of the input tensor.");
- TVM_ATTR_FIELD(output_scale)
- .describe("The scale of the output tensor.");
- TVM_ATTR_FIELD(output_zero_point)
- .describe("The zero point of the output tensor.");
TVM_ATTR_FIELD(rounding).set_default("UPWARD")
.describe("Defines the rounding direction when the value is midway between"
"two representable values. There are two supported modes - UPWARD"
/*! \brief Attribute for quantize operator */
struct QuantizeAttrs : public tvm::AttrsNode<QuantizeAttrs> {
- int32_t output_zero_point;
- double output_scale;
DataType out_dtype;
TVM_DECLARE_ATTRS(QuantizeAttrs, "relay.attrs.QuantizeAttrs") {
TVM_ATTR_FIELD(out_dtype)
.describe("Output data type, can be one of [int8 or uint8].");
- TVM_ATTR_FIELD(output_zero_point)
- .describe("The zero_point for the activation of this op.");
- TVM_ATTR_FIELD(output_scale)
- .describe("The scale for the activation of this op.");
- }
-};
-
-/*! \brief Attribute for dequantize operator */
-struct DequantizeAttrs : public tvm::AttrsNode<DequantizeAttrs> {
- int32_t input_zero_point;
- double input_scale;
-
- TVM_DECLARE_ATTRS(DequantizeAttrs, "relay.attrs.DequantizeAttrs") {
- TVM_ATTR_FIELD(input_zero_point)
- .describe("The zero_point for the input tensor of this op.");
- TVM_ATTR_FIELD(input_scale)
- .describe("The scale for the input tensor of this op.");
- }
-};
-
-/*! \brief Attributes used in QNN concatenate operator */
-struct QnnConcatenateAttrs : public tvm::AttrsNode<QnnConcatenateAttrs> {
- Array<tvm::Expr> input_scales;
- Array<tvm::Expr> input_zero_points;
- double output_scale;
- int32_t output_zero_point;
- int axis;
-
- TVM_DECLARE_ATTRS(QnnConcatenateAttrs, "relay.attrs.QnnConcatenateAttrs") {
- TVM_ATTR_FIELD(input_scales)
- .describe("The list of scales of input quantized tensors.");
- TVM_ATTR_FIELD(input_zero_points)
- .describe("The list of zero points of input quantized tensors.");
- TVM_ATTR_FIELD(output_zero_point)
- .describe("The zero_point for the output tensor.");
- TVM_ATTR_FIELD(output_scale)
- .describe("The scale for the output tensor.");
- TVM_ATTR_FIELD(axis)
- .describe("The axis at which the input arrays are concatenated."
- "Should lie in range `[-ndim, ndim)`.")
- .set_default(0);
- }
-}; // struct QnnConcatenateAttrs
-
-/*! \brief Attribute for QNN Conv2d operator */
-struct QnnConv2DAttrs : public tvm::AttrsNode<QnnConv2DAttrs> {
- // Traditional conv2d attributes.
- Array<IndexExpr> strides;
- Array<IndexExpr> padding;
- Array<IndexExpr> dilation;
- int groups;
- IndexExpr channels;
- Array<IndexExpr> kernel_size;
- std::string data_layout;
- std::string kernel_layout;
- std::string out_layout;
- DataType out_dtype;
-
- // Quantization related attributes.
- int32_t input_zero_point;
- int32_t kernel_zero_point;
- // The input tensor scale and kernel tensor scales are stored
- // for easy access to this information.
- double input_scale;
- double kernel_scale;
-
- TVM_DECLARE_ATTRS(QnnConv2DAttrs, "relay.attrs.QnnConv2DAttrs") {
- TVM_ATTR_FIELD(strides).set_default(Array<IndexExpr>({1, 1}))
- .describe("Specifies the strides of the convolution.");
- TVM_ATTR_FIELD(padding).set_default(Array<IndexExpr>({0, 0}))
- .describe("If padding is non-zero, then the input is implicitly zero-padded"
- "on both sides for padding number of points");
- TVM_ATTR_FIELD(dilation).set_default(Array<IndexExpr>({1, 1}))
- .describe("Specifies the dilation rate to use for dilated convolution.");
- TVM_ATTR_FIELD(groups).set_default(1)
- .describe("Controls the connections between inputs and outputs."
- "At groups=1, all inputs are convolved to all outputs."
- "At groups=2, the operation becomes equivalent to having two convolution"
- "layers side by side, each seeing half the input channels, and producing"
- "half the output channels, and both subsequently concatenated.");
- TVM_ATTR_FIELD(channels)
- .describe("The number of output channels in the convolution."
- " If it is not set, inferred by shape of the weight.")
- .set_default(NullValue<IndexExpr>());
- TVM_ATTR_FIELD(kernel_size)
- .describe("Specifies the dimensions of the convolution window.")
- .set_default(NullValue<Array<IndexExpr> >());
- TVM_ATTR_FIELD(data_layout).set_default("NCHW")
- .describe("Dimension ordering of input data. Can be 'NCHW', 'NHWC', etc."
- "'N', 'C', 'H', 'W' stands for batch, channel, height, and width"
- "dimensions respectively. Convolution is applied on the 'H' and"
- "'W' dimensions.");
- TVM_ATTR_FIELD(kernel_layout).set_default("OIHW")
- .describe("Dimension ordering of weight. Can be 'OIHW', 'OIHW16o16i', etc."
- "'O', 'I', 'H', 'W' stands for num_filter, input_channel, height, and width"
- "dimensions respectively.");
- TVM_ATTR_FIELD(out_layout).set_default("")
- .describe("Dimension ordering of output. Can be 'NCHW', 'NHWC', etc."
- "'N', 'C', 'H', 'W' stands for batch, channel, height, and width"
- "dimensions respectively. Default to be same as input layout.");
- TVM_ATTR_FIELD(out_dtype)
- .set_default(NullValue<DataType>())
- .describe("Output data type, set to explicit type under mixed precision setting");
- TVM_ATTR_FIELD(input_zero_point)
- .describe("The zero point of the input tensor.");
- TVM_ATTR_FIELD(kernel_zero_point)
- .describe("The zero point of the kernel tensor.");
- TVM_ATTR_FIELD(input_scale)
- .describe("The quantization scale for the input tensor.");
- TVM_ATTR_FIELD(kernel_scale)
- .describe("The quantization scale for the weight tensor.");
- }
-};
-
-/*! \brief Attribute for QNN binary operator */
-struct QnnBinaryOpAttrs : public tvm::AttrsNode<QnnBinaryOpAttrs> {
- int32_t lhs_zero_point;
- double lhs_scale;
- int32_t rhs_zero_point;
- double rhs_scale;
- int32_t output_zero_point;
- double output_scale;
-
- TVM_DECLARE_ATTRS(QnnBinaryOpAttrs, "relay.attrs.QnnBinaryOpAttrs") {
- TVM_ATTR_FIELD(lhs_zero_point)
- .describe("The zero_point for the lhs input tensor of this op.");
- TVM_ATTR_FIELD(lhs_scale)
- .describe("The scale for the lhs input tensor of this op.");
- TVM_ATTR_FIELD(rhs_zero_point)
- .describe("The zero_point for the rhs input tensor of this op.");
- TVM_ATTR_FIELD(rhs_scale)
- .describe("The scale for the rhs input tensor of this op.");
- TVM_ATTR_FIELD(output_zero_point)
- .describe("The zero_point for the activation of this op.");
- TVM_ATTR_FIELD(output_scale)
- .describe("The scale for the activation of this op.");
- }
-};
-
-/*! \brief Attributes for qnn dense operator */
-struct QnnDenseAttrs : public tvm::AttrsNode<QnnDenseAttrs> {
- IndexExpr units;
- DataType out_dtype;
- // Quantization related attributes.
- int32_t input_zero_point;
- int32_t kernel_zero_point;
- double input_scale;
- double kernel_scale;
-
- TVM_DECLARE_ATTRS(QnnDenseAttrs, "relay.attrs.QnnDenseAttrs") {
- TVM_ATTR_FIELD(units)
- .describe("Number of hidden units of the dense transformation.");
- TVM_ATTR_FIELD(out_dtype)
- .describe("Output data type, set to explicit type under mixed precision setting");
- TVM_ATTR_FIELD(input_zero_point)
- .describe("The zero point of the input tensor.");
- TVM_ATTR_FIELD(kernel_zero_point)
- .describe("The zero point of the kernel tensor.");
- TVM_ATTR_FIELD(input_scale)
- .describe("The input tensor scale.");
- TVM_ATTR_FIELD(kernel_scale)
- .describe("The kernel tensor scale.");
}
};
"""
import numpy as np
+from tvm import relay
from tvm.relay.qnn.op.qnn import dequantize
zero_centered_uint8_quantized_range = np.float32(255)
real_range = np.max([np.abs(np.float32(data_min)),
np.abs(np.float32(data_max))])
- scale = np.divide(real_range, quantized_range)
- zero_point = 0
+ scale = relay.const(np.divide(real_range, quantized_range), 'float32')
+ zero_point = relay.const(0, 'int32')
return dequantize(data, scale, zero_point)
max_limit = np.float64(iinfo.max)
imin_range = np.float64(imin_range)
imax_range = np.float64(imax_range)
- scale = np.divide((imax_range - imin_range),
- (max_limit - min_limit))
- zero_point = np.int(-1 * np.divide(imin_range, scale))
+ scale_val = np.divide((imax_range - imin_range),
+ (max_limit - min_limit))
+ zero_point_val = np.int(-1 * np.divide(imin_range, scale_val))
+ scale = relay.const(scale_val, 'float32')
+ zero_point = relay.const(zero_point_val, 'int32')
return dequantize(data, scale, zero_point)
import math
import numpy as np
import tvm
+from tvm import relay
from .. import analysis
from .. import expr as _expr
from .. import module as _module
from .. import op as _op
from .. import qnn as _qnn
+from ..util import get_scalar_from_constant
from ... import nd as _nd
from .common import ExprTable
from .common import infer_shape as _infer_shape
# Check that the scale and zero points are valid.
if scale != 0 or zero_point != 0:
qnn_params = dict()
- qnn_params['scale'] = scale
- qnn_params['zero_point'] = zero_point
+ qnn_params['scale'] = relay.const(scale, 'float32')
+ qnn_params['zero_point'] = relay.const(zero_point, 'int32')
return_list.append(TensorWrapper(tensor_idx, tensor, buffer, qnn_params))
return return_list
.format(str(tensor_type)))
def has_same_qnn_params(self, lhs_tensor, rhs_tensor):
- return lhs_tensor.qnn_params['scale'] == rhs_tensor.qnn_params['scale'] and \
- lhs_tensor.qnn_params['zero_point'] == rhs_tensor.qnn_params['zero_point']
+ lhs_scale = lhs_tensor.qnn_params['scale']
+ rhs_scale = rhs_tensor.qnn_params['scale']
+ lhs_zero_point = lhs_tensor.qnn_params['zero_point']
+ rhs_zero_point = rhs_tensor.qnn_params['zero_point']
+ lhs_scale_value = get_scalar_from_constant(lhs_scale)
+ rhs_scale_value = get_scalar_from_constant(rhs_scale)
+ lhs_zero_point_value = get_scalar_from_constant(lhs_zero_point)
+ rhs_zero_point_value = get_scalar_from_constant(rhs_zero_point)
+ return lhs_scale_value == rhs_scale_value and \
+ lhs_zero_point_value == rhs_zero_point_value
def is_quantized(self, op):
"""Check if an input tensor is quantized."""
weight_expr = self.exp_tab.new_const(weight_value, dtype=weight_tensor_type_str)
if input_tensor.qnn_params:
- input_scale = input_tensor.qnn_params['scale']
- kernel_scale = weight_tensor.qnn_params['scale']
out = _qnn.op.dense(in_expr, weight_expr,
input_zero_point=input_tensor.qnn_params['zero_point'],
kernel_zero_point=weight_tensor.qnn_params['zero_point'],
- input_scale=input_scale,
- kernel_scale=kernel_scale,
+ input_scale=input_tensor.qnn_params['scale'],
+ kernel_scale=weight_tensor.qnn_params['scale'],
out_dtype='int32')
else:
out = _op.nn.dense(in_expr, weight_expr)
# Finally if the dense is quantized. Add a requantize at the end.
if output_tensor.qnn_params:
- input_scale = input_tensor.qnn_params['scale'] * weight_tensor.qnn_params['scale']
- input_zero_point = 0
+ data_scale = input_tensor.qnn_params['scale']
+ weight_scale = weight_tensor.qnn_params['scale']
+ data_scale_val = get_scalar_from_constant(data_scale)
+ weight_scale_val = get_scalar_from_constant(weight_scale)
+ new_input_scale_val = data_scale_val * weight_scale_val
+ new_input_scale = relay.const(new_input_scale_val, 'float32')
+ new_input_zero_point = relay.const(0, 'int32')
out = _qnn.op.requantize(out,
- input_scale=input_scale,
- input_zero_point=input_zero_point,
+ input_scale=new_input_scale,
+ input_zero_point=new_input_zero_point,
output_scale=output_tensor.qnn_params['scale'],
output_zero_point=output_tensor.qnn_params['zero_point'],
out_dtype=output_tensor_type_str)
# Finally if the conv is quantized. Add a requantize at the end.
if output_tensor.qnn_params:
- input_scale = input_tensor.qnn_params['scale'] * weight_tensor.qnn_params['scale']
- input_zero_point = 0
+ data_scale = input_tensor.qnn_params['scale']
+ weight_scale = weight_tensor.qnn_params['scale']
+ data_scale_val = get_scalar_from_constant(data_scale)
+ weight_scale_val = get_scalar_from_constant(weight_scale)
+ new_input_scale_val = data_scale_val * weight_scale_val
+ new_input_scale = relay.const(new_input_scale_val, 'float32')
+ new_input_zero_point = relay.const(0, 'int32')
out = _qnn.op.requantize(out,
- input_scale=input_scale,
- input_zero_point=input_zero_point,
+ input_scale=new_input_scale,
+ input_zero_point=new_input_zero_point,
output_scale=output_tensor.qnn_params['scale'],
output_zero_point=output_tensor.qnn_params['zero_point'],
out_dtype=output_tensor_type_str)
from .qnn import *
from .op import register_qnn_legalize
from . import legalizations
-from . import op_attrs
import tvm
from tvm import relay
from .. import op as reg
+from ...util import get_scalar_from_constant
#################################################
# Register the functions for different operators.
"""
# Collect the input exprs.
- data, kernel = inputs
-
- input_zp = attrs['input_zero_point']
- kernel_zp = attrs['kernel_zero_point']
+ data, kernel, input_zero_point, kernel_zero_point, _, _ = inputs
shift_data = relay.subtract(relay.cast(data, dtype='int16'),
- relay.const(input_zp, 'int16'))
+ relay.cast(input_zero_point, 'int16'))
shift_kernel = relay.subtract(relay.cast(kernel, dtype='int16'),
- relay.const(kernel_zp, 'int16'))
+ relay.cast(kernel_zero_point, 'int16'))
new_attrs = {k : attrs[k] for k in attrs.keys()}
- del new_attrs['kernel_zero_point']
- del new_attrs['input_zero_point']
- del new_attrs['input_scale']
- del new_attrs['kernel_scale']
return relay_op(shift_data, shift_kernel, **new_attrs)
# Helper function to change dtypes to uint8 x int8. Intel VNNI instructions prefer this setting.
data_modified = relay.cast(data, 'int32')
data_modified = relay.add(data_modified, relay.const(shift, 'int32'))
data_modified = relay.cast(data_modified, out_dtype)
- return (data_modified, zero_point + shift)
+ zero_point_val = get_scalar_from_constant(zero_point)
+ zero_point_modified = relay.const(zero_point_val + shift, 'int32')
+ return (data_modified, zero_point_modified)
# Collect the dtypes.
data_dtype = types[0].dtype
kernel_dtype = types[1].dtype
# Collect the input exprs.
- data, kernel = inputs
+ data, kernel, input_zero_point, kernel_zero_point, input_scale, kernel_scale = inputs
# VNNI supports u8 x i8 fast conv/MM. Don't do anything if it is already satisfied.
if data_dtype == 'uint8' and kernel_dtype == 'int8':
return None
# Shift input if necessary.
- input_zp = attrs['input_zero_point']
if data_dtype == 'int8':
# Compute (QA + 128) and (zp_a + 128)
- data, input_zp = _shift(data, input_zp, 'uint8')
+ data, input_zero_point = _shift(data, input_zero_point, 'uint8')
# Shift kernel if necessary.
- kernel_zp = attrs['kernel_zero_point']
if kernel_dtype == 'uint8':
# Compute (QA - 128) and (zp_a - 128)
- kernel, kernel_zp = _shift(kernel, kernel_zp, 'int8')
+ kernel, kernel_zero_point = _shift(kernel, kernel_zero_point, 'int8')
# Call qnn.conv2d with modified inputs and zero points.
new_attrs = {k : attrs[k] for k in attrs.keys()}
- new_attrs['input_zero_point'] = input_zp
- new_attrs['kernel_zero_point'] = kernel_zp
- return relay_op(data, kernel, **new_attrs)
+ return relay_op(data, kernel,
+ input_zero_point, kernel_zero_point,
+ input_scale, kernel_scale, **new_attrs)
# Helper function to change dtypes to be same. ARM dotprod instructions prefer this setting.
def helper_change_dtypes_to_be_same(attrs, inputs, types, relay_op):
data_modified = relay.cast(data, 'int32')
data_modified = relay.add(data_modified, relay.const(shift, 'int32'))
data_modified = relay.cast(data_modified, out_dtype)
- return (data_modified, zero_point + shift)
+ zero_point_val = get_scalar_from_constant(zero_point)
+ zero_point_modified = relay.const(zero_point_val + shift, 'int32')
+ return (data_modified, zero_point_modified)
# Collect the dtypes.
data_dtype = types[0].dtype
return None
# Collect the input exprs.
- data, kernel = inputs
+ data, kernel, input_zero_point, kernel_zero_point, input_scale, kernel_scale = inputs
assert 'int8' in data_dtype and 'int8' in kernel_dtype, \
"Qnn Conv2D/Dense only accepts uint8 or int8 inputs"
# Shift input if necessary.
- input_zp = attrs['input_zero_point']
- data, input_zp = _shift(data, input_zp, kernel_dtype)
+ data, input_zero_point = _shift(data, input_zero_point, kernel_dtype)
new_attrs = {k : attrs[k] for k in attrs.keys()}
- new_attrs['input_zero_point'] = input_zp
- return relay_op(data, kernel, **new_attrs)
+ return relay_op(data, kernel,
+ input_zero_point, kernel_zero_point,
+ input_scale, kernel_scale, **new_attrs)
def is_fast_int8_on_intel():
""" Checks whether the hardware has support for fast Int8 arithmetic operations. """
"""QNN dialect operators."""
from __future__ import absolute_import as _abs
-from tvm.expr import FloatImm, IntImm
from tvm.relay.expr import Tuple
from . import _make
data : tvm.relay.Expr
The input data to the operator.
- input_scale: float
+ input_scale: tvm.relay.Expr
The quantization scale for the input tensor.
- input_zero_point: int
+ input_zero_point: tvm.relay.Expr
The zero point of the input tensor.
- output_scale: float
+ output_scale: tvm.relay.Expr
The quantization scale for the output tensor.
- output_zero_point: int
+ output_zero_point: tvm.relay.Expr
The zero point of the output tensor.
rounding : string, optional
----------
data : tvm.relay.Expr
The input tensor to be quantized. Can be of type float32.
- output_zero_point : int
+ output_zero_point : tvm.relay.Expr
The output zero_point.
- output_scale : float
+ output_scale : tvm.relay.Expr
The output scale.
out_dtype : str, optional
The data type of the input tensor. Can be [int8, uint8]
----------
data : tvm.relay.Expr
The input tensor to be dequantized. Can be of type [int8, uint8].
- input_zero_point : int
+ input_zero_point : tvm.relay.Expr
The output zero_point.
- input_scale : float
+ input_scale : tvm.relay.Expr
The output scale.
Returns
-------
data : Union(List[relay.Expr], Tuple[relay.Expr])
The list of quantized tensors.
- input_scales : List[float32]
+ input_scales : List[relay.Expr]
The list of scales of input quantized tensors.
- input_zero_points : List[int32]
+ input_zero_points : List[relay.Expr]
The list of zero points of input quantized tensors.
- output_scale : float32
+ output_scale : relay.Expr
The scale of the output quantized tensor.
- output_zero_point : int32
+ output_zero_point : relay.Expr
The zero point of the output quantized tensor.
axis : int
raise ValueError("relay.concatenate requires data to be non-empty.")
if not isinstance(axis, int):
raise ValueError("For now, we only support integer axis")
+ input_scales = list(input_scales)
+ input_zero_points = list(input_zero_points)
return _make.concatenate(Tuple(data),
- [FloatImm("float64", x) for x in input_scales],
- [IntImm("int32", x) for x in input_zero_points],
+ Tuple(input_scales),
+ Tuple(input_zero_points),
output_scale,
output_zero_point,
axis)
kernel : tvm.relay.Expr
The kernel expressions.
- input_zero_point: int
+ input_zero_point: tvm.relay.Expr
The zero point of the data distribution.
- input_scale: float
+ kernel_zero_point: tvm.relay.Expr
+ The zero point of the quantized_kernel distribution.
+
+ input_scale: tvm.relay.Expr
The scale for the input tensor. The scale for the input tensor is
stored purely for convenience here. See more commentary below.
- kernel_scale: float
+ kernel_scale: tvm.relay.Expr
The scale for the weight tensor. The scale for the weight tensor is
stored for access to this during relay. This information is not
needed in the pass pipeline after qnn.conv2d is lowered to the
sequence of steps as in nn.conv2d. See also input_scale in Requantize.
- kernel_zero_point: int
- The zero point of the quantized_kernel distribution.
-
strides : tuple of int, optional
The strides of convolution.
lhs_scale: float
The scale of the lhs quantized expr.
- lhs_zero_point: int
+ lhs_scale: relay.Expr
+ The scale of the lhs quantized expr.
+
+ lhs_zero_point: relay.Expr
The zero point of lhs quantized expr.
- rhs_scale: float
+ rhs_scale: relay.Expr
The scale of the rhs quantized expr.
- rhs_zero_point: int
+ rhs_zero_point: relay.Expr
The zero point of rhs quantized expr.
- output_scale: float
+ output_scale: relay.Expr
The scale of the output quantized expr.
- output_zero_point: int
+ output_zero_point: relay.Expr
The zero point of output quantized expr.
Returns
The quantized input data to the operator.
weight : tvm.relay.Expr
The quantized weight expressions.
- input_zero_point: int
+ input_zero_point: tvm.relay.Expr
The input zero point.
- kernel_zero_point: int
+ kernel_zero_point: tvm.relay.Expr
The kernel zero point.
- input_scale: float
+ input_scale: tvm.relay.Expr
The scale for the input tensor.
- kernel_scale: float
+ kernel_scale: tvm.relay.Expr
The scale for the weight tensor. The scale for the weight tensor is
stored for access to this during relay. This information is not
needed in the pass pipeline after qnn.conv2d is lowered to the
rhs : relay.Expr
The right hand side quantized input data.
- lhs_scale: float
+ lhs_scale: relay.Expr
The scale of the lhs quantized expr.
- lhs_zero_point: int
+ lhs_zero_point: relay.Expr
The zero point of lhs quantized expr.
- rhs_scale: float
+ rhs_scale: relay.Expr
The scale of the rhs quantized expr.
- rhs_zero_point: int
+ rhs_zero_point: relay.Expr
The zero point of rhs quantized expr.
- output_scale: float
+ output_scale: relay.Expr
The scale of the output quantized expr.
- output_zero_point: int
+ output_zero_point: relay.Expr
The zero point of output quantized expr.
Returns
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""The attributes node used for QNN operators"""
+# pylint: disable=wildcard-import, redefined-builtin, invalid-name
+""" Utility functions that are used across many directories. """
+from __future__ import absolute_import
+import numpy as np
+from . import expr as _expr
-from ....attrs import Attrs
-from ...base import register_relay_attr_node
-
-@register_relay_attr_node
-class QnnConv2DAttrs(Attrs):
- """Attributes for qnn.conv2d"""
-
-@register_relay_attr_node
-class QnnDenseAttrs(Attrs):
- """Attributes for qnn.dense"""
+def get_scalar_from_constant(expr):
+ """ Returns scalar value from Relay constant scalar. """
+ assert isinstance(expr, _expr.Constant) and not expr.data.shape, \
+ "Expr is not a constant scalar."
+ value = expr.data.asnumpy()
+ if value.dtype == np.dtype(np.int32):
+ return int(value)
+ if value.dtype == np.dtype(np.float32):
+ return float(value)
+ assert False, "Constant expr must be float32/int32"
+ return None # To suppress pylint
template <typename T>
T GetScalarFromConstant(Expr expr) {
const auto* n = expr.as<ConstantNode>();
+ CHECK(n) << "Expr must be a constant expr - " << AsText(expr, false);
CHECK(n->is_scalar());
return static_cast<T*>(n->data->data)[0];
}
Expr QnnAddCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& arg_types) {
// Get the attrs.
- CHECK_EQ(new_args.size(), 2);
+ CHECK_EQ(new_args.size(), 8);
auto& lhs = new_args[0];
auto& rhs = new_args[1];
- const auto* binary_op_attrs = attrs.as<QnnBinaryOpAttrs>();
- CHECK(binary_op_attrs != nullptr);
- auto lhs_scale = binary_op_attrs->lhs_scale;
- auto lhs_zero_point = binary_op_attrs->lhs_zero_point;
- auto rhs_scale = binary_op_attrs->rhs_scale;
- auto rhs_zero_point = binary_op_attrs->rhs_zero_point;
- auto output_scale = binary_op_attrs->output_scale;
- auto output_zero_point = binary_op_attrs->output_zero_point;
+ auto& lhs_scale = new_args[2];
+ auto& lhs_zero_point = new_args[3];
+ auto& rhs_scale = new_args[4];
+ auto& rhs_zero_point = new_args[5];
+ auto& output_scale = new_args[6];
+ auto& output_zero_point = new_args[7];
// Get the input dtype and shape.
- CHECK_EQ(arg_types.size(), 3);
+ CHECK_EQ(arg_types.size(), 9);
auto tensor_type = arg_types[0].as<TensorTypeNode>();
auto input_dtype = tensor_type->dtype;
auto input_shape = tensor_type->shape;
// Requantize LHS if necessary.
auto requantized_lhs = lhs;
- if (lhs_scale != output_scale || lhs_zero_point != output_zero_point) {
+ if (!IsEqualScalar(lhs_scale, output_scale) ||
+ !IsEqualScalar(lhs_zero_point, output_zero_point)) {
requantized_lhs = Requantize(lhs, input_shape, lhs_scale, lhs_zero_point, output_scale,
output_zero_point, DataType::Int(32));
} else {
// Requantize RHS if necessary.
auto requantized_rhs = rhs;
- if (rhs_scale != output_scale || rhs_zero_point != output_zero_point) {
+ if (!IsEqualScalar(rhs_scale, output_scale) ||
+ !IsEqualScalar(rhs_zero_point, output_zero_point)) {
requantized_rhs = Requantize(rhs, input_shape, rhs_scale, rhs_zero_point, output_scale,
output_zero_point, DataType::Int(32));
} else {
auto output = Add(requantized_lhs, requantized_rhs);
// Subtract zero point.
- if (output_zero_point != 0) {
- auto output_zp = MakeConstantScalar(DataType::Int(32), output_zero_point);
- output = Subtract(output, output_zp);
+ auto zero_scalar = MakeConstantScalar(DataType::Int(32), 0);
+ if (!IsEqualScalar(output_zero_point, zero_scalar)) {
+ output = Subtract(output, output_zero_point);
}
// Go back to lower precision.
namespace relay {
namespace qnn {
-TVM_REGISTER_NODE_TYPE(QnnConcatenateAttrs);
-
-Expr MakeQnnConcatenate(Expr data, Array<tvm::Expr> input_scales,
- Array<tvm::Expr> input_zero_points, double output_scale,
- int32_t output_zero_point, int axis) {
- auto attrs = make_object<QnnConcatenateAttrs>();
- attrs->input_scales = std::move(input_scales);
- attrs->input_zero_points = std::move(input_zero_points);
- attrs->output_scale = output_scale;
- attrs->output_zero_point = output_zero_point;
+bool QnnConcatenateRel(const Array<Type>& types, int num_inputs, const Attrs& attrs,
+ const TypeReporter& reporter) {
+ CHECK_EQ(types.size(), 6);
+
+ // Check the scale and zero point types
+ const auto* input_scales_tuple = types[1].as<TupleTypeNode>();
+ if (input_scales_tuple == nullptr) {
+ throw relay::Error(
+ RELAY_ERROR("qnn concatenate requires a tuple of scales as the second argument, found "
+ << PrettyPrint(types[1])));
+ }
+ for (const auto& input_scale : input_scales_tuple->fields) {
+ CHECK(IsScalarType(input_scale, DataType::Float(32))); // input_scales[idx]
+ }
+
+ const auto* input_zero_points_tuple = types[2].as<TupleTypeNode>();
+ if (input_zero_points_tuple == nullptr) {
+ throw relay::Error(
+ RELAY_ERROR("qnn concatenate requires a tuple of zero_points as the third argument, found "
+ << PrettyPrint(types[2])));
+ }
+ for (const auto& input_zero_point : input_zero_points_tuple->fields) {
+ CHECK(IsScalarType(input_zero_point, DataType::Int(32))); // input_zero_points[idx]
+ }
+
+ CHECK(IsScalarType(types[3], DataType::Float(32))); // output_scale
+ CHECK(IsScalarType(types[4], DataType::Int(32))); // output_zero_point
+
+ // Collect the input tensor and output tensor devoid of scale and zero points to reuse Relay
+ // Concatenate infer type function.
+ Array<Type> tensor_types = {types[0], types[5]};
+ return ConcatenateRel<ConcatenateAttrs>(tensor_types, 2, attrs, reporter);
+}
+
+Expr MakeQnnConcatenate(Expr data, Expr input_scales, Expr input_zero_points, Expr output_scale,
+ Expr output_zero_point, int axis) {
+ auto attrs = make_object<ConcatenateAttrs>();
attrs->axis = axis;
static const Op& op = Op::Get("qnn.concatenate");
- return CallNode::make(op, {data}, Attrs(attrs), {});
+ return CallNode::make(op,
+ {data, input_scales, input_zero_points, output_scale, output_zero_point},
+ Attrs(attrs), {});
}
/*
Expr ConcatenateQnnCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& arg_types) {
// Get the attrs.
- CHECK_EQ(new_args.size(), 1);
+ CHECK_EQ(new_args.size(), 5);
auto& data = new_args[0];
- const auto* concatenate_attrs = attrs.as<QnnConcatenateAttrs>();
+ auto& input_scales = new_args[1];
+ auto& input_zero_points = new_args[2];
+ auto& output_scale = new_args[3];
+ auto& output_zero_point = new_args[4];
+ const auto* concatenate_attrs = attrs.as<ConcatenateAttrs>();
CHECK(concatenate_attrs != nullptr);
- auto input_scales = concatenate_attrs->input_scales;
- auto input_zero_points = concatenate_attrs->input_zero_points;
- auto output_scale = concatenate_attrs->output_scale;
- auto output_zero_point = concatenate_attrs->output_zero_point;
// Get the input dtype and shape.
CHECK_GE(arg_types.size(), 1);
auto tuple_data = data.as<TupleNode>();
CHECK(tuple_data != nullptr);
+ auto tuple_input_scales = input_scales.as<TupleNode>();
+ CHECK(tuple_input_scales != nullptr);
+
+ auto tuple_input_zero_points = input_zero_points.as<TupleNode>();
+ CHECK(tuple_input_zero_points != nullptr);
+
int idx = 0;
Array<Expr> requantized_exprs;
for (auto quantized_expr : tuple_data->fields) {
// Get the input scale for the idx quantized input tensor.
- auto input_scale_expr = input_scales[idx].as<tvm::ir::FloatImm>();
- CHECK(input_scale_expr != nullptr);
- auto input_scale = input_scale_expr->value;
+ auto input_scale = tuple_input_scales->fields[idx];
// Get the zero point for the idx quantized input tensor.
- auto input_zero_point_expr = input_zero_points[idx].as<tvm::ir::IntImm>();
- CHECK(input_zero_point_expr != nullptr);
- auto input_zero_point = input_zero_point_expr->value;
+ auto input_zero_point = tuple_input_zero_points->fields[idx];
// Check if output and input qnn params are same. If not, requantize.
- if (input_scale != output_scale || input_zero_point != output_zero_point) {
+ if (!IsEqualScalar(input_scale, output_scale) ||
+ !IsEqualScalar(input_zero_point, output_zero_point)) {
// Get the input shape and dtype.
auto tensor_type = tuple_type->fields[idx].as<TensorTypeNode>();
auto input_dtype = tensor_type->dtype;
RELAY_REGISTER_OP("qnn.concatenate")
.describe(R"code(Concatenate the quantized input tensors along the given axis.
)code" TVM_ADD_FILELINE)
-.set_attrs_type<QnnConcatenateAttrs>()
-.set_num_inputs(1)
+.set_attrs_type<ConcatenateAttrs>()
+.set_num_inputs(5)
.add_argument("data", "Tensor", "The tensor to concatenate.")
+.add_argument("input_scales", "Tensor", "The quantization scales of the input tensors.")
+.add_argument("input_zero_points", "Tensor", "The quantization zero_points of the input tensors.")
+.add_argument("output_scale", "Tensor", "The quantization scale of the output tensor.")
+.add_argument("output_zero_point", "Tensor", "The quantization zero_point of the output tensor.")
.set_support_level(11)
-.add_type_rel("QnnConcatenate", ConcatenateRel<QnnConcatenateAttrs>)
+.add_type_rel("QnnConcatenate", QnnConcatenateRel)
.set_attr<FTVMLegalize>("FTVMQnnCanonicalize", ConcatenateQnnCanonicalize);
TVM_REGISTER_API("relay.qnn.op._make.concatenate")
namespace qnn {
// relay.op.qnn.conv2d
-TVM_REGISTER_NODE_TYPE(QnnConv2DAttrs);
bool QnnConv2DRel(const Array<Type>& types, int num_inputs, const Attrs& attrs,
const TypeReporter& reporter) {
- CHECK_EQ(types.size(), 3);
+ CHECK_EQ(types.size(), 7);
const auto* data = types[0].as<TensorTypeNode>();
const auto* weight = types[1].as<TensorTypeNode>();
if (data == nullptr || weight == nullptr) return false;
- const auto* param = attrs.as<QnnConv2DAttrs>();
- CHECK(param != nullptr) << "QnnConv2DAttrs cannot be nullptr.";
+ const auto* param = attrs.as<Conv2DAttrs>();
+ CHECK(param != nullptr) << "Conv2DAttrs cannot be nullptr.";
CHECK(data->dtype == DataType::Int(8) || data->dtype == DataType::UInt(8))
<< "Expected qnn conv2d type(int8, uint8) for input but was " << data->dtype;
CHECK(weight->dtype == DataType::Int(8) || weight->dtype == DataType::UInt(8))
CHECK(param->out_dtype == DataType::Int(16) || param->out_dtype == DataType::Int(32))
<< "Expected qnn conv2d type(int32, int16) for output but was " << param->out_dtype;
CHECK(param->out_dtype.bits() > 0) << "Output dtype bits should be greater than 0.";
- return Conv2DRel<QnnConv2DAttrs>(types, num_inputs, attrs, reporter);
+
+ // Check the types of scale and zero points.
+ CHECK(IsScalarType(types[2], DataType::Int(32))); // input_zero_point
+ CHECK(IsScalarType(types[3], DataType::Int(32))); // kernel_zero_point
+ CHECK(IsScalarType(types[4], DataType::Float(32))); // input_scale
+ CHECK(IsScalarType(types[5], DataType::Float(32))); // kernel_scale
+
+ // Collect the input tensor and output tensor devoid of scale and zero points to reuse Relay
+ // Conv2D infer type function.
+ Array<Type> tensor_types = {types[0], types[1], types[6]};
+ return Conv2DRel<Conv2DAttrs>(tensor_types, 3, attrs, reporter);
}
-bool is_depthwise(const QnnConv2DAttrs* param) {
+bool is_depthwise(const Conv2DAttrs* param) {
return param->channels.defined() && tvm::ir::Equal(param->channels, param->groups) &&
param->groups != 1;
}
* \param param The qnn conv2d attributes.
* \return A tuple of workload.
*/
-WorkloadType GetWorkload(const Array<tvm::relay::Type>& arg_types, const QnnConv2DAttrs* param) {
+WorkloadType GetWorkload(const Array<tvm::relay::Type>& arg_types, const Conv2DAttrs* param) {
// Get conv parameters.
const auto in_shape = get_shape(arg_types[0]);
int batch_size, in_channels;
* \brief Fallback to simpler lowering for dilation or grouped conv.
* \param data The input expr.
* \param weight The weight expr.
+ * \param input_zero_point The input zero point expr.
+ * \param kernel_zero_point The kernel zero point expr.
* \param param The qnn conv2d attributes.
* \return The fallback lowered sequence of Relay expr.
* \note In case of dilation, normal lowering would require a dilated pool.
* Relay operations. This will potentially lead to performance degradation
* as the convolution is called on int32 tensors instead of int8 tensors.
*/
-Expr Conv2DFallBack(const Expr& data, const Expr& weight, const QnnConv2DAttrs* param) {
+Expr Conv2DFallBack(const Expr& data, const Expr& weight, const Expr& input_zero_point,
+ const Expr& kernel_zero_point, const Conv2DAttrs* param) {
// Upcast the zero point to Int16.
- auto zp_data = MakeConstantScalar(DataType::Int(16), param->input_zero_point);
- auto zp_kernel = MakeConstantScalar(DataType::Int(16), param->kernel_zero_point);
+ auto zp_data = Cast(input_zero_point, DataType::Int(16));
+ auto zp_kernel = Cast(kernel_zero_point, DataType::Int(16));
auto shifted_data = Cast(data, DataType::Int(16));
- if (param->input_zero_point != 0) {
+ auto zero_scalar = MakeConstantScalar(DataType::Int(32), 0);
+ if (!IsEqualScalar(input_zero_point, zero_scalar)) {
shifted_data = Subtract(Cast(data, DataType::Int(16)), zp_data);
}
auto shifted_kernel = Cast(weight, DataType::Int(16));
- if (param->kernel_zero_point != 0) {
+ if (!IsEqualScalar(kernel_zero_point, zero_scalar)) {
shifted_kernel = Subtract(Cast(weight, DataType::Int(16)), zp_kernel);
}
/*
* \brief Pad the input data.
* \param data The input expr.
+ * \param input_zero_point The input zero point expr.
* \return The padded input expr.
* \note For quantized convolution, the input has to be padded with zero point
* instead of zero. This might lead to performance degradation as pad
* cannot be fused with conv in Relay. In case we see performance
* degradation, we can change the conv2D API to accept a pad_const value.
*/
-Expr Conv2DPadInput(const Expr& data, const QnnConv2DAttrs* param) {
+Expr Conv2DPadInput(const Expr& data, const Expr& input_zero_point, const Conv2DAttrs* param) {
// 1) Pad the input data
auto padded_data = data;
auto pad_h_value = get_const_int(param->padding[0]);
} else {
LOG(FATAL) << "qnn.conv2d does not support " << param->data_layout << " layout";
}
- padded_data = Pad(data, pad_width, param->input_zero_point, "constant");
+ auto pad_value = GetScalarFromConstant<int>(input_zero_point);
+ padded_data = Pad(data, pad_width, pad_value, "constant");
}
return padded_data;
}
/*
* \brief Calculates the second term in the qnn.conv2d depthwise lowering sequence.
* \param padded_data The padded data expr.
+ * \param kernel_zero_point The kernel zero point expr.
* \param param The qnn conv2d attributes.
* \param kernel_h The height of kernel.
* \param kernel_w The width of kernel.
* However, deeper analysis shows that we can reduce r,s using avg_pool2d,
* followed by repeat on the C axis by cm times.
*/
-Expr DepthwiseConv2DSecondTerm(const Expr& padded_data, const QnnConv2DAttrs* param, int kernel_h,
- int kernel_w, int channel_multiplier) {
- // Constant Expr for the kernel zero point.
- auto zp_kernel = MakeConstantScalar(DataType::Int(32), param->kernel_zero_point);
-
+Expr DepthwiseConv2DSecondTerm(const Expr& padded_data, const Expr& kernel_zero_point,
+ const Conv2DAttrs* param, int kernel_h, int kernel_w,
+ int channel_multiplier) {
auto casted_t2 = Cast(padded_data, DataType::Int(32));
// We can reduce the H and W axis by using avg_pool2d. However, avg_pool2d averages the sum.
// pool_size is 1x1, we don't need avg_pool2d.
auto reduced_t2 = casted_t2;
if (kernel_h * kernel_w != 1) {
- auto scaled_hw_t2 = Multiply(
- casted_t2, MakeConstantScalar(DataType::Int(32), kernel_h * kernel_w));
+ auto scaled_hw_t2 =
+ Multiply(casted_t2, MakeConstantScalar(DataType::Int(32), kernel_h * kernel_w));
Array<IndexExpr> padding({0, 0});
reduced_t2 =
AvgPool2D(scaled_hw_t2, param->kernel_size, param->strides, padding, param->data_layout,
}
auto multiplied_t2 = reduced_t2;
- if (param->kernel_zero_point != 1) {
- multiplied_t2 = Multiply(zp_kernel, reduced_t2);
+ auto one_scalar = MakeConstantScalar(DataType::Int(32), 1);
+ if (!IsEqualScalar(kernel_zero_point, one_scalar)) {
+ multiplied_t2 = Multiply(kernel_zero_point, reduced_t2);
}
// Reduce the C dimension. Find the dimension.
/*
* \brief Calculates the third term in the qnn.conv2d depthwise lowering sequence.
* \param weight The weight expr.
+ * \param input_zero_point The input zero point expr.
* \param param The qnn conv2d attributes.
* \param out_channels The number of output channels.
* \param channel_multiplier The channel/depth multiplier.
* This can be achieved by calling reduce on r and s axis. The tensor can be then reshaped to
* (1, oc, 1, 1) as (oc/m, oc%m) are just contiguous memory locations.
*/
-Expr DepthwiseConv2DThirdTerm(const Expr& weight, const QnnConv2DAttrs* param, int out_channels,
- int channel_multiplier) {
- // Constant expr for input zero point.
- auto zp_data = MakeConstantScalar(DataType::Int(32), param->input_zero_point);
-
+Expr DepthwiseConv2DThirdTerm(const Expr& weight, const Expr& input_zero_point,
+ const Conv2DAttrs* param, int out_channels, int channel_multiplier) {
// Find which dimensions are R, S.
Array<Integer> axes_t3;
if (param->kernel_layout == "OIHW") {
}
auto reshaped_t3 = Reshape(reduced_t3, newshape);
- if (param->input_zero_point == 1) {
+ auto one_scalar = MakeConstantScalar(DataType::Int(32), 1);
+ if (IsEqualScalar(input_zero_point, one_scalar)) {
return reshaped_t3;
}
- return Multiply(zp_data, reshaped_t3);
+ return Multiply(input_zero_point, reshaped_t3);
}
/*
* \brief Calculates the fourth term in the qnn.conv2d depthwise lowering sequence.
- * \param param The qnn conv2d attributes.
+ * \param input_zero_point_int The int value of input zero point.
+ * \param kernel_zero_point_int The int value of kernel zero point.
* \param kernel_h The height of kernel.
* \param kernel_w The width of kernel.
* \return The sequence of Relay operators for term4.
*
* Sigma(r, s) zp_a * zp_w
*/
-Expr DepthwiseConv2DFourthTerm(const QnnConv2DAttrs* param, int kernel_h, int kernel_w) {
- int scalar_term4 = param->input_zero_point * param->kernel_zero_point * kernel_h * kernel_w;
+Expr DepthwiseConv2DFourthTerm(int input_zero_point_int, int kernel_zero_point_int, int kernel_h,
+ int kernel_w) {
+ int scalar_term4 = input_zero_point_int * kernel_zero_point_int * kernel_h * kernel_w;
return MakeConstantScalar(DataType::Int(32), scalar_term4);
}
* Sigma(c,r,s) QW(k, c, r, s) * QA(n, c, h + r, w + s)
* This is just conv2d on int tensors.
*/
-Expr Conv2DFirstTerm(const Expr& padded_data, const Expr& weight, const QnnConv2DAttrs* param) {
+Expr Conv2DFirstTerm(const Expr& padded_data, const Expr& weight, const Conv2DAttrs* param) {
// Lowering for Term 1
Array<IndexExpr> padding({0, 0});
return Conv2D(padded_data, weight, param->strides, padding, param->dilation, param->groups,
/*
* \brief Calculates the second term in the qnn.conv2d lowering sequence.
* \param padded_data The padded data expr.
+ * \param kernel_zero_point The kernel zero point expr.
* \param param The qnn conv2d attributes.
* \param kernel_h The height of kernel.
* \param kernel_w The width of kernel.
* followed by a reduce on the C axis. Using avg_pool2d also gives an
* opportunity to reuse alter_op_layout infrastructure.
*/
-Expr Conv2DSecondTerm(const Expr& padded_data, const QnnConv2DAttrs* param, int kernel_h,
- int kernel_w, int out_channels) {
- // Constant Expr for the kernel zero point.
- auto zp_kernel = MakeConstantScalar(DataType::Int(32), param->kernel_zero_point);
-
+Expr Conv2DSecondTerm(const Expr& padded_data, const Expr& kernel_zero_point,
+ const Conv2DAttrs* param, int kernel_h, int kernel_w, int out_channels) {
auto casted_t2 = Cast(padded_data, DataType::Int(32));
// We can reduce the H and W axis by using avg_pool2d. However, avg_pool2d averages the sum.
// If the pool_size is 1x1, we don't need avg_pool2d.
auto reduced_t2 = reduced_c_t2;
if (kernel_h * kernel_w != 1) {
- reduced_c_t2 = Multiply(
- reduced_c_t2, MakeConstantScalar(DataType::Int(32), kernel_h * kernel_w));
+ reduced_c_t2 =
+ Multiply(reduced_c_t2, MakeConstantScalar(DataType::Int(32), kernel_h * kernel_w));
reduced_t2 =
- AvgPool2D(reduced_c_t2,
- param->kernel_size,
- param->strides,
- padding, param->data_layout,
+ AvgPool2D(reduced_c_t2, param->kernel_size, param->strides, padding, param->data_layout,
false, // ceil_mode
false); // count_include_pad
}
auto multiplied_t2 = reduced_t2;
- if (param->kernel_zero_point != 1) {
- multiplied_t2 = Multiply(zp_kernel, reduced_t2);
+ auto one_scalar = MakeConstantScalar(DataType::Int(32), 1);
+ if (!IsEqualScalar(kernel_zero_point, one_scalar)) {
+ multiplied_t2 = Multiply(kernel_zero_point, reduced_t2);
}
return multiplied_t2;
}
/*
* \brief Calculates the third term in the qnn.conv2d lowering sequence.
* \param weight The weight expr.
+ * \param input_zero_point The input zero point expr.
* \param param The qnn conv2d attributes.
* \param out_channels The number of output channels.
* \return The sequence of Relay operatos for term3.
* a 1D tensor. The tensor is then reshaped to conform to NHWC/NCHW
* format.
*/
-Expr Conv2DThirdTerm(const Expr& weight, const QnnConv2DAttrs* param, int out_channels) {
- // Constant expr for input zero point.
- auto zp_data = MakeConstantScalar(DataType::Int(32), param->input_zero_point);
-
+Expr Conv2DThirdTerm(const Expr& weight, const Expr& input_zero_point, const Conv2DAttrs* param,
+ int out_channels) {
// Find which dimensions are C, R, S.
Array<Integer> axes_t3;
if (param->kernel_layout == "OIHW") {
}
auto reshaped_t3 = Reshape(reduced_t3, newshape);
- if (param->input_zero_point == 1) {
+ auto one_scalar = MakeConstantScalar(DataType::Int(32), 1);
+ if (IsEqualScalar(input_zero_point, one_scalar)) {
return reshaped_t3;
}
- return Multiply(zp_data, reshaped_t3);
+ return Multiply(input_zero_point, reshaped_t3);
}
/*
* \brief Calculates the fourth term in the qnn.conv2d lowering sequence.
- * \param param The qnn conv2d attributes.
+ * \param input_zero_point_int The int value of input zero point.
+ * \param kernel_zero_point_int The int value of kernel zero point.
* \param in_channels The number of input channels.
* \param kernel_h The height of kernel.
* \param kernel_w The width of kernel.
* Sigma(c,r,s) zp_a * zp_w
*
*/
-Expr Conv2DFourthTerm(const QnnConv2DAttrs* param, int in_channels, int kernel_h, int kernel_w) {
+Expr Conv2DFourthTerm(int input_zero_point_int, int kernel_zero_point_int, int in_channels,
+ int kernel_h, int kernel_w) {
int scalar_term4 =
- param->input_zero_point * param->kernel_zero_point * in_channels * kernel_h * kernel_w;
+ input_zero_point_int * kernel_zero_point_int * in_channels * kernel_h * kernel_w;
return MakeConstantScalar(DataType::Int(32), scalar_term4);
}
* \param term2 The term2 of qnn conv2d lowering.
* \param term3 The term3 of qnn conv2d lowering.
* \param term4 The term4 of qnn conv2d lowering.
+ * \param input_zero_point_int The int value of input zero point.
+ * \param kernel_zero_point_int The int value of kernel zero point.
* \param param The qnn conv2d attributes.
* \return The combined sequence of relay operations.
* \note The combined operation looks like this
*
*/
Expr Conv2DCombineTerms(const Expr& term1, const Expr& term2, const Expr& term3, const Expr& term4,
- const QnnConv2DAttrs* param) {
- if (param->input_zero_point == 0 && param->kernel_zero_point == 0) {
+ int input_zero_point_int, int kernel_zero_point_int) {
+ if (input_zero_point_int == 0 && kernel_zero_point_int == 0) {
// term 2, 3 and 4 become zero.
return term1;
- } else if (param->input_zero_point == 0 && param->kernel_zero_point != 0) {
+ } else if (input_zero_point_int == 0 && kernel_zero_point_int != 0) {
// term 3 and term 4 become zero.
return Subtract(term1, term2);
- } else if (param->input_zero_point != 0 && param->kernel_zero_point == 0) {
+ } else if (input_zero_point_int != 0 && kernel_zero_point_int == 0) {
// term 2 and term 4 become zero.
return Subtract(term1, term3);
} else {
*/
Expr QnnConv2DCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& arg_types) {
- CHECK_EQ(new_args.size(), 2);
+ CHECK_EQ(new_args.size(), 6);
Expr data = new_args[0];
Expr weight = new_args[1];
- const auto* param = attrs.as<QnnConv2DAttrs>();
+ Expr input_zero_point = new_args[2];
+ Expr kernel_zero_point = new_args[3];
+ const auto* param = attrs.as<Conv2DAttrs>();
CHECK(param != nullptr);
// Assertion checks for exisiing support.
CHECK_EQ(param->padding.size(), 2) << "qnn.conv2d only supports 2D padding";
std::tie(batch_size, in_channels, out_channels, kernel_h, kernel_w, channel_multiplier) =
GetWorkload(arg_types, param);
+ // Extract the integer zero points.
+ auto input_zero_point_int = GetScalarFromConstant<int>(input_zero_point);
+ auto kernel_zero_point_int = GetScalarFromConstant<int>(kernel_zero_point);
+
// Fallback to int32 conv if there is dilation or grouped conv2d
CHECK_EQ(param->dilation.size(), 2) << "qnn.conv2d only supports 2D dilation";
auto dilation_h = get_const_int(param->dilation[0]);
auto dilation_w = get_const_int(param->dilation[1]);
if (dilation_h != 1 || dilation_w != 1 || (param->groups != 1 && !is_depthwise(param))) {
- return Conv2DFallBack(data, weight, param);
+ return Conv2DFallBack(data, weight, input_zero_point, kernel_zero_point, param);
} else if (is_depthwise(param)) {
CHECK_NE(channel_multiplier, -1);
- auto padded_data = Conv2DPadInput(data, param);
+ auto padded_data = Conv2DPadInput(data, input_zero_point, param);
auto term1 = Conv2DFirstTerm(padded_data, weight, param);
- auto term2 =
- DepthwiseConv2DSecondTerm(padded_data, param, kernel_h, kernel_w, channel_multiplier);
- auto term3 = DepthwiseConv2DThirdTerm(weight, param, out_channels, channel_multiplier);
- auto term4 = DepthwiseConv2DFourthTerm(param, kernel_h, kernel_w);
- return Conv2DCombineTerms(term1, term2, term3, term4, param);
+ auto term2 = DepthwiseConv2DSecondTerm(padded_data, kernel_zero_point, param, kernel_h,
+ kernel_w, channel_multiplier);
+ auto term3 =
+ DepthwiseConv2DThirdTerm(weight, input_zero_point, param, out_channels, channel_multiplier);
+ auto term4 =
+ DepthwiseConv2DFourthTerm(input_zero_point_int, kernel_zero_point_int, kernel_h, kernel_w);
+ return Conv2DCombineTerms(term1, term2, term3, term4, input_zero_point_int,
+ kernel_zero_point_int);
}
- auto padded_data = Conv2DPadInput(data, param);
+ auto padded_data = Conv2DPadInput(data, input_zero_point, param);
auto term1 = Conv2DFirstTerm(padded_data, weight, param);
- auto term2 = Conv2DSecondTerm(padded_data, param, kernel_h, kernel_w, out_channels);
- auto term3 = Conv2DThirdTerm(weight, param, out_channels);
- auto term4 = Conv2DFourthTerm(param, in_channels, kernel_h, kernel_w);
- return Conv2DCombineTerms(term1, term2, term3, term4, param);
+ auto term2 =
+ Conv2DSecondTerm(padded_data, kernel_zero_point, param, kernel_h, kernel_w, out_channels);
+ auto term3 = Conv2DThirdTerm(weight, input_zero_point, param, out_channels);
+ auto term4 = Conv2DFourthTerm(input_zero_point_int, kernel_zero_point_int, in_channels, kernel_h,
+ kernel_w);
+ return Conv2DCombineTerms(term1, term2, term3, term4, input_zero_point_int,
+ kernel_zero_point_int);
}
// Positional relay function to create quantized conv2d operator
// used by frontend FFI.
-Expr MakeQnnConv2D(Expr data, Expr weight, int32_t input_zero_point, int32_t kernel_zero_point,
- double input_scale, double kernel_scale, Array<IndexExpr> strides,
- Array<IndexExpr> padding, Array<IndexExpr> dilation,
- int groups, IndexExpr channels, Array<IndexExpr> kernel_size,
- std::string data_layout, std::string kernel_layout, std::string out_layout,
- DataType out_dtype) {
- auto attrs = make_object<QnnConv2DAttrs>();
+Expr MakeQnnConv2D(Expr data, Expr weight, Expr input_zero_point, Expr kernel_zero_point,
+ Expr input_scale, Expr kernel_scale, Array<IndexExpr> strides,
+ Array<IndexExpr> padding, Array<IndexExpr> dilation, int groups,
+ IndexExpr channels, Array<IndexExpr> kernel_size, std::string data_layout,
+ std::string kernel_layout, std::string out_layout, DataType out_dtype) {
+ auto attrs = make_object<Conv2DAttrs>();
attrs->strides = std::move(strides);
attrs->padding = std::move(padding);
attrs->dilation = std::move(dilation);
attrs->kernel_layout = std::move(kernel_layout);
attrs->out_layout = std::move(out_layout);
attrs->out_dtype = std::move(out_dtype);
- attrs->input_zero_point = std::move(input_zero_point);
- attrs->kernel_zero_point = std::move(kernel_zero_point);
- attrs->input_scale = std::move(input_scale);
- attrs->kernel_scale = std::move(kernel_scale);
static const Op& op = Op::Get("qnn.conv2d");
- return CallNode::make(op, {data, weight}, Attrs(attrs), {});
+ return CallNode::make(
+ op, {data, weight, input_zero_point, kernel_zero_point, input_scale, kernel_scale},
+ Attrs(attrs), {});
}
RELAY_REGISTER_OP("qnn.conv2d")
- **out**: This depends on the `layout` parameter. Output is 4D array of shape
(batch_size, channels, out_height, out_width) if `layout` is `NCHW`.
)code" TVM_ADD_FILELINE)
-.set_attrs_type<QnnConv2DAttrs>()
-.set_num_inputs(2)
+.set_attrs_type<Conv2DAttrs>()
+.set_num_inputs(6)
.add_argument("data", "Tensor", "The quantized input data tensor.")
.add_argument("weight", "Tensor", "The quantized weight tensor.")
+.add_argument("input_scale", "Tensor", "The quantization scale of the input tensor.")
+.add_argument("input_zero_point", "Tensor", "The quantization zero_point of the input tensor.")
+.add_argument("weight_scale", "Tensor", "The quantization scale of the weight tensor.")
+.add_argument("weight_zero_point", "Tensor", "The quantization zero_point of the weight tensor.")
.set_support_level(11)
.add_type_rel("QnnConv2D", QnnConv2DRel)
.set_attr<FTVMLegalize>("FTVMQnnCanonicalize", QnnConv2DCanonicalize);
namespace qnn {
// relay.op.qnn.dense
-TVM_REGISTER_NODE_TYPE(QnnDenseAttrs);
bool QnnDenseRel(const Array<Type>& types, int num_inputs, const Attrs& attrs,
const TypeReporter& reporter) {
- CHECK_EQ(types.size(), 3);
+ CHECK_EQ(types.size(), 7);
const auto* data = types[0].as<TensorTypeNode>();
const auto* weight = types[1].as<TensorTypeNode>();
if (data == nullptr || weight == nullptr) return false;
- const auto* param = attrs.as<QnnDenseAttrs>();
- CHECK(param != nullptr) << "QnnDenseAttrs cannot be nullptr.";
+ const auto* param = attrs.as<DenseAttrs>();
+ CHECK(param != nullptr) << "DenseAttrs cannot be nullptr.";
CHECK(data->dtype == DataType::Int(8) || data->dtype == DataType::UInt(8))
<< "Expected quantized dense type(int8, uint8) for input but was " << data->dtype;
CHECK(weight->dtype == DataType::Int(8) || weight->dtype == DataType::UInt(8))
<< "Expected quantized dense type(int8, uint8) for weight but was " << weight->dtype;
CHECK(param->out_dtype == DataType::Int(32))
<< "Expected quantized dense type(int32) for output but was " << param->out_dtype;
+
+ // Check the types of scale and zero points.
+ CHECK(IsScalarType(types[2], DataType::Int(32))); // input_zero_point
+ CHECK(IsScalarType(types[3], DataType::Int(32))); // kernel_zero_point
+ CHECK(IsScalarType(types[4], DataType::Float(32))); // input_scale
+ CHECK(IsScalarType(types[5], DataType::Float(32))); // kernel_scale
+
CHECK(param->out_dtype.bits() > 0) << "Output dtype bits should be greater than 0.";
- return DenseRel<QnnDenseAttrs>(types, num_inputs, attrs, reporter);
+
+ // Collect the input tensor and output tensor devoid of scale and zero points to reuse Relay
+ // Dense infer type function.
+ Array<Type> tensor_types = {types[0], types[1], types[6]};
+ return DenseRel<DenseAttrs>(tensor_types, 3, attrs, reporter);
}
// Positional relay function to create quantized dense operator used by frontend FFI.
-Expr MakeQuantizedDense(Expr data, Expr weight, int32_t input_zero_point,
- int32_t kernel_zero_point, double input_scale,
- double kernel_scale, IndexExpr units,
- DataType out_dtype) {
- auto attrs = make_object<QnnDenseAttrs>();
+Expr MakeQuantizedDense(Expr data, Expr weight, Expr input_zero_point, Expr kernel_zero_point,
+ Expr input_scale, Expr kernel_scale, IndexExpr units, DataType out_dtype) {
+ auto attrs = make_object<DenseAttrs>();
attrs->units = std::move(units);
attrs->out_dtype = out_dtype;
- attrs->input_zero_point = input_zero_point;
- attrs->kernel_zero_point = kernel_zero_point;
- attrs->input_scale = input_scale;
- attrs->kernel_scale = kernel_scale;
static const Op& op = Op::Get("qnn.dense");
- return CallNode::make(op, {data, weight}, Attrs(attrs), {});
+ return CallNode::make(
+ op, {data, weight, input_zero_point, kernel_zero_point, input_scale, kernel_scale},
+ Attrs(attrs), {});
}
Expr DenseFirstTerm(const Expr& quantized_data, const Expr& quantized_kernel,
- const QnnDenseAttrs* attrs) {
+ const DenseAttrs* attrs) {
return Dense(quantized_data, quantized_kernel, attrs->units, attrs->out_dtype);
}
-Expr DenseSecondTerm(const Expr& quantized_data, const Expr& zp_kernel) {
+Expr DenseSecondTerm(const Expr& quantized_data, const Expr& kernel_zero_point) {
Array<Integer> axes = {1};
- return Multiply(zp_kernel, Sum(Cast(quantized_data, DataType::Int(32)), axes, true, false));
+ return Multiply(kernel_zero_point,
+ Sum(Cast(quantized_data, DataType::Int(32)), axes, true, false));
}
-Expr DenseThirdTerm(const Expr& quantized_kernel, const Expr& zp_data) {
+Expr DenseThirdTerm(const Expr& quantized_kernel, const Expr& input_zero_point) {
Array<Integer> axes = {1};
- return Multiply(zp_data, Sum(Cast(quantized_kernel, DataType::Int(32)), axes, false, false));
+ return Multiply(input_zero_point,
+ Sum(Cast(quantized_kernel, DataType::Int(32)), axes, false, false));
}
-Expr DenseFourthTerm(const QnnDenseAttrs* attrs, int reduction_dim_size) {
- int32_t scalar_term = attrs->input_zero_point * attrs->kernel_zero_point * reduction_dim_size;
+Expr DenseFourthTerm(int input_zero_point_int, int kernel_zero_point_int, int reduction_dim_size) {
+ int32_t scalar_term = input_zero_point_int * kernel_zero_point_int * reduction_dim_size;
return MakeConstantScalar(DataType::Int(32), scalar_term);
}
*/
Expr QnnDenseCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& arg_types) {
- CHECK_EQ(new_args.size(), 2);
+ CHECK_EQ(new_args.size(), 6);
Expr quantized_data = new_args[0];
Expr quantized_kernel = new_args[1];
+ Expr input_zero_point = new_args[2];
+ Expr kernel_zero_point = new_args[3];
const auto in_shape = get_shape(arg_types[0]);
const int reduction_dim_size = get_const_int(in_shape[1]);
- const auto* qnn_dense_attrs = attrs.as<QnnDenseAttrs>();
- auto zp_kernel = MakeConstantScalar(DataType::Int(32), qnn_dense_attrs->kernel_zero_point);
- auto zp_data = MakeConstantScalar(DataType::Int(32), qnn_dense_attrs->input_zero_point);
+ const auto* qnn_dense_attrs = attrs.as<DenseAttrs>();
+
+ // Extract the integer zero points.
+ auto input_zero_point_int = GetScalarFromConstant<int>(input_zero_point);
+ auto kernel_zero_point_int = GetScalarFromConstant<int>(kernel_zero_point);
// Get all the terms as described in the comments.
auto term1 = DenseFirstTerm(quantized_data, quantized_kernel, qnn_dense_attrs);
- auto term2 = DenseSecondTerm(quantized_data, zp_kernel);
- auto term3 = DenseThirdTerm(quantized_kernel, zp_data);
- auto term4 = DenseFourthTerm(qnn_dense_attrs, reduction_dim_size);
+ auto term2 = DenseSecondTerm(quantized_data, kernel_zero_point);
+ auto term3 = DenseThirdTerm(quantized_kernel, input_zero_point);
+ auto term4 = DenseFourthTerm(input_zero_point_int, kernel_zero_point_int, reduction_dim_size);
// Combine those 4 terms depending on the zero points to get the best lowering.
- if (qnn_dense_attrs->input_zero_point == 0 && qnn_dense_attrs->kernel_zero_point == 0) {
+ if (input_zero_point_int == 0 && kernel_zero_point_int == 0) {
// term 2, 3 and 4 become zero.
return term1;
- } else if (qnn_dense_attrs->input_zero_point == 0 && qnn_dense_attrs->kernel_zero_point != 0) {
+ } else if (input_zero_point_int == 0 && kernel_zero_point_int != 0) {
// term 3 and term 4 become zero.
return Subtract(term1, term2);
- } else if (qnn_dense_attrs->input_zero_point != 0 && qnn_dense_attrs->kernel_zero_point == 0) {
+ } else if (input_zero_point_int != 0 && kernel_zero_point_int == 0) {
// term 2 and term 4 become zero.
return Subtract(term1, term3);
} else {
- **weight**: quantized(int8, unit8) `(units, input_dim)`
- **out**: quantized(int32) `(x1, x2, ..., xn, units)`.
)code" TVM_ADD_FILELINE)
-.set_attrs_type<QnnDenseAttrs>()
-.set_num_inputs(2)
+.set_attrs_type<DenseAttrs>()
+.set_num_inputs(6)
.add_argument("data", "quantized nD Tensor", "Input data.")
.add_argument("weight", "quantized 2D Tensor", "Weight matrix.")
+.add_argument("input_scale", "Tensor", "The quantization scale of the input tensor.")
+.add_argument("input_zero_point", "Tensor", "The quantization zero_point of the input tensor.")
+.add_argument("weight_scale", "Tensor", "The quantization scale of the weight tensor.")
+.add_argument("weight_zero_point", "Tensor", "The quantization zero_point of the weight tensor.")
.set_support_level(11)
-.add_type_rel("QDense", DenseRel<QnnDenseAttrs>)
+.add_type_rel("QDense", QnnDenseRel)
.set_attr<FTVMLegalize>("FTVMQnnCanonicalize", QnnDenseCanonicalize);
TVM_REGISTER_API("relay.qnn.op._make.dense")
namespace relay {
namespace qnn {
-TVM_REGISTER_NODE_TYPE(DequantizeAttrs);
-
bool DequantizeRel(const Array<Type>& types,
int num_inputs,
const Attrs& attrs,
const TypeReporter& reporter) {
- CHECK_EQ(types.size(), 2);
+ CHECK_EQ(types.size(), 4);
const auto* data = types[0].as<TensorTypeNode>();
const auto input_dtype = data->dtype;
CHECK(input_dtype == DataType::Int(8) ||
input_dtype == DataType::Int(32))
<< "Input type should be one of the quantized types [unit8, int8, int32] but was "
<< input_dtype;
+
+ // Check the types of scale and zero points.
+ CHECK(IsScalarType(types[1], DataType::Float(32))); // input_scale
+ CHECK(IsScalarType(types[2], DataType::Int(32))); // input_zero_point
+
const Array<tvm::Expr> oshape = data->shape;
// assign output type, output will always be float 32.
- reporter->Assign(types[1], TensorTypeNode::make(oshape, DataType::Float(32)));
+ reporter->Assign(types[3], TensorTypeNode::make(oshape, DataType::Float(32)));
return true;
}
-Expr MakeDequantize(Expr data,
- double input_scale,
- int32_t input_zero_point) {
- auto attrs = make_object<DequantizeAttrs>();
- attrs->input_scale = input_scale;
- attrs->input_zero_point = input_zero_point;
+Expr MakeDequantize(Expr data, Expr input_scale, Expr input_zero_point) {
// real_value = scale * (quantized_value - zero_point)
- // A more detailed explanation can be found here - https://github.com/google/gemmlowp/blob/master/doc/quantization.md
+ // A more detailed explanation can be found here -
+ // https://github.com/google/gemmlowp/blob/master/doc/quantization.md
static const Op& op = Op::Get("qnn.dequantize");
- return CallNode::make(op, {data}, Attrs(attrs), {});
+ return CallNode::make(op, {data, input_scale, input_zero_point}, Attrs(), {});
}
-Expr DequantizeLower(const Expr& input_tensor,
- const DequantizeAttrs* attrs) {
- const auto input_zero_point = MakeConstantScalar(DataType::Int(32), attrs->input_zero_point);
- const auto input_scale = MakeConstantScalar(DataType::Float(32), attrs->input_scale);
+Expr DequantizeLower(const Expr& input_tensor, const Expr& input_scale,
+ const Expr& input_zero_point) {
auto shift = Subtract(Cast(input_tensor, DataType::Int(32)), input_zero_point);
auto scaled_output = Multiply(Cast(shift, DataType::Float(32)), input_scale);
return scaled_output;
}
-Expr DequantizeQnnCanonicalize(const Attrs& attrs,
- const Array<Expr>& new_args,
+Expr DequantizeQnnCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& types) {
- CHECK_EQ(new_args.size(), 1);
+ CHECK_EQ(new_args.size(), 3);
auto& data = new_args[0];
- const auto* dequantize_attrs = attrs.as<DequantizeAttrs>();
- CHECK(dequantize_attrs != nullptr);
- CHECK_EQ(types.size(), 2);
- return DequantizeLower(data, dequantize_attrs);
+ auto& input_scale = new_args[1];
+ auto& input_zero_point = new_args[2];
+ CHECK_EQ(types.size(), 4);
+ return DequantizeLower(data, input_scale, input_zero_point);
}
RELAY_REGISTER_OP("qnn.dequantize")
The input is always quantized (int8, uint8) and will be converted to float32 given input scale and zero_point.
- **data**: Quantized tensor of any shape to dequantize. The input data can be of floating point
)code" TVM_ADD_FILELINE)
-.set_attrs_type<DequantizeAttrs>()
-.set_num_inputs(1)
+.set_num_inputs(3)
.add_argument("data", "Tensor", "The tensor to dequantize.")
+.add_argument("input_scale", "Tensor", "The quantization scale of the input tensor.")
+.add_argument("input_zero_point", "Tensor", "The quantization zero_point of the input tensor.")
.set_support_level(11)
.add_type_rel("Dequantize", DequantizeRel)
.set_attr<FTVMLegalize>("FTVMQnnCanonicalize", DequantizeQnnCanonicalize);
Expr QnnMulCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& arg_types) {
// Get the attrs.
- CHECK_EQ(new_args.size(), 2);
+ CHECK_EQ(new_args.size(), 8);
auto& lhs = new_args[0];
auto& rhs = new_args[1];
- const auto* binary_op_attrs = attrs.as<QnnBinaryOpAttrs>();
- CHECK(binary_op_attrs != nullptr);
- auto lhs_scale = binary_op_attrs->lhs_scale;
- auto lhs_zero_point = binary_op_attrs->lhs_zero_point;
- auto rhs_scale = binary_op_attrs->rhs_scale;
- auto rhs_zero_point = binary_op_attrs->rhs_zero_point;
- auto output_scale = binary_op_attrs->output_scale;
- auto output_zero_point = binary_op_attrs->output_zero_point;
+ auto& lhs_scale = new_args[2];
+ auto& lhs_zero_point = new_args[3];
+ auto& rhs_scale = new_args[4];
+ auto& rhs_zero_point = new_args[5];
+ auto& output_scale = new_args[6];
+ auto& output_zero_point = new_args[7];
// Get the input dtype and shape.
- CHECK_EQ(arg_types.size(), 3);
+ CHECK_EQ(arg_types.size(), 9);
auto tensor_type = arg_types[0].as<TensorTypeNode>();
auto input_dtype = tensor_type->dtype;
auto input_shape = tensor_type->shape;
auto lhs_shifted = Cast(lhs, DataType::Int(32));
auto rhs_shifted = Cast(rhs, DataType::Int(32));
- if (lhs_zero_point != 0) {
- auto lhs_zp = MakeConstantScalar(DataType::Int(32), lhs_zero_point);
- lhs_shifted = Subtract(lhs_shifted, lhs_zp);
+ auto zero_scalar = MakeConstantScalar(DataType::Int(32), 0);
+ if (!IsEqualScalar(lhs_zero_point, zero_scalar)) {
+ lhs_shifted = Subtract(lhs_shifted, lhs_zero_point);
}
- if (rhs_zero_point != 0) {
- auto rhs_zp = MakeConstantScalar(DataType::Int(32), rhs_zero_point);
- rhs_shifted = Subtract(rhs_shifted, rhs_zp);
+ if (!IsEqualScalar(rhs_zero_point, zero_scalar)) {
+ rhs_shifted = Subtract(rhs_shifted, rhs_zero_point);
}
// Create a new tensor Q'
auto output = Multiply(lhs_shifted, rhs_shifted);
- auto scale_new = rhs_scale * lhs_scale;
+ // Get the adjusted new scale and zero points.
+ float lhs_scale_float = GetScalarFromConstant<float>(lhs_scale);
+ float rhs_scale_float = GetScalarFromConstant<float>(rhs_scale);
+ float new_scale_float = lhs_scale_float * rhs_scale_float;
+ auto new_input_scale = MakeConstantScalar(DataType::Float(32), new_scale_float);
+ auto new_input_zero_point = zero_scalar;
// Requantize to get Q_c
- output = Requantize(output, input_shape, scale_new, 0, output_scale,
- output_zero_point, input_dtype);
+ output = Requantize(output, input_shape, new_input_scale, new_input_zero_point, output_scale,
+ output_zero_point, input_dtype);
return output;
}
namespace relay {
namespace qnn {
+static inline bool QnnBroadcastRel(const Array<Type>& types, int num_inputs, const Attrs& attrs,
+ const TypeReporter& reporter) {
+ CHECK_EQ(types.size(), 9);
+
+ // Check the scale and zero point types
+ CHECK(IsScalarType(types[2], DataType::Float(32))); // lhs_scale
+ CHECK(IsScalarType(types[3], DataType::Int(32))); // lhs_zero_point
+ CHECK(IsScalarType(types[4], DataType::Float(32))); // rhs_scale
+ CHECK(IsScalarType(types[5], DataType::Int(32))); // rhs_zero_point
+ CHECK(IsScalarType(types[6], DataType::Float(32))); // output_scale
+ CHECK(IsScalarType(types[7], DataType::Int(32))); // output_zero_point
+
+ // Collect the input tensor and output tensor devoid of scale and zero points to reuse Relay
+ // BroadcastRel infer type function.
+ Array<Type> tensor_types = {types[0], types[1], types[8]};
+ return BroadcastRel(tensor_types, 3, attrs, reporter);
+}
+
/*! Quick helper macro
* - Expose a positional make function to construct the node.
* - Register op to the registry.
*/
#define QNN_REGISTER_BINARY_OP(OpName) \
TVM_REGISTER_API("relay.qnn.op._make." OpName) \
- .set_body_typed<Expr(Expr, Expr, double, int32_t, double, int32_t, double, int32_t)>( \
- [](Expr lhs, Expr rhs, double lhs_scale, int32_t lhs_zero_point, double rhs_scale, \
- int32_t rhs_zero_point, double output_scale, int32_t output_zero_point) { \
- auto attrs = make_object<QnnBinaryOpAttrs>(); \
- attrs->lhs_scale = lhs_scale; \
- attrs->lhs_zero_point = lhs_zero_point; \
- attrs->rhs_scale = rhs_scale; \
- attrs->rhs_zero_point = rhs_zero_point; \
- attrs->output_scale = output_scale; \
- attrs->output_zero_point = output_zero_point; \
+ .set_body_typed<Expr(Expr, Expr, Expr, Expr, Expr, Expr, Expr, Expr)>( \
+ [](Expr lhs, Expr rhs, Expr lhs_scale, Expr lhs_zero_point, Expr rhs_scale, \
+ Expr rhs_zero_point, Expr output_scale, Expr output_zero_point) { \
static const Op& op = Op::Get("qnn." OpName); \
- return CallNode::make(op, {lhs, rhs}, Attrs(attrs), {}); \
+ return CallNode::make(op, {lhs, rhs, \
+ lhs_scale, lhs_zero_point, \
+ rhs_scale, rhs_zero_point, \
+ output_scale, output_zero_point}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP("qnn." OpName) \
- .set_num_inputs(2) \
+ .set_num_inputs(8) \
.add_argument("lhs", "Tensor", "The left hand side quantized tensor.") \
.add_argument("rhs", "Tensor", "The right hand side quantized tensor.") \
- .add_type_rel("Broadcast", BroadcastRel)
+ .add_argument("lhs_scale", "Tensor", "The scale of the lhs tensor.") \
+ .add_argument("lhs_zero_point", "Tensor", "The zero_point of the lhs tensor.") \
+ .add_argument("rhs_scale", "Tensor", "The scale of the rhs tensor.") \
+ .add_argument("rhs_zero_point", "Tensor", "The zero_point of the rhs tensor.") \
+ .add_argument("output_scale", "Tensor", "The scale of the output tensor.") \
+ .add_argument("output_zero_point", "Tensor", "The zero_point of the output tensor.") \
+ .add_type_rel("QnnBroadcast", QnnBroadcastRel)
} // namespace qnn
} // namespace relay
int num_inputs,
const Attrs& attrs,
const TypeReporter& reporter) {
- CHECK_EQ(types.size(), 2);
+ CHECK_EQ(types.size(), 4);
const auto* data = types[0].as<TensorTypeNode>();
const auto input_dtype = data->dtype;
CHECK(input_dtype == DataType::Float(32))
<< "Input type should be one of float32 but was " << input_dtype;
+
+ // Check the types of scale and zero points.
+ CHECK(IsScalarType(types[1], DataType::Float(32))); // output_scale
+ CHECK(IsScalarType(types[2], DataType::Int(32))); // output_zero_point
+
const auto* quantize_attrs = attrs.as<QuantizeAttrs>();
const Array<tvm::Expr> oshape = data->shape;
const DataType out_dtype = quantize_attrs->out_dtype;
- CHECK(out_dtype == DataType::Int(8) ||
- out_dtype == DataType::UInt(8) ||
+ CHECK(out_dtype == DataType::Int(8) || out_dtype == DataType::UInt(8) ||
out_dtype == DataType::Int(32))
- << "Output type should be one of [int8, unit8, int32] but was " << out_dtype;
+ << "Output type should be one of [int8, unit8, int32] but was " << out_dtype;
// assign output type
- reporter->Assign(types[1], TensorTypeNode::make(oshape, out_dtype));
+ reporter->Assign(types[3], TensorTypeNode::make(oshape, out_dtype));
return true;
}
-Expr MakeQuantize(Expr data,
- double output_scale,
- int32_t output_zero_point,
- DataType out_dtype) {
+Expr MakeQuantize(Expr data, Expr output_scale, Expr output_zero_point, DataType out_dtype) {
auto attrs = make_object<QuantizeAttrs>();
- attrs->output_scale = output_scale;
- attrs->output_zero_point = output_zero_point;
attrs->out_dtype = std::move(out_dtype);
// result_quantized_value = result_zero_point + result_real_value / result_scale.
- // A more detailed explanation can be found here - https://github.com/google/gemmlowp/blob/master/doc/quantization.md
+ // A more detailed explanation can be found here -
+ // https://github.com/google/gemmlowp/blob/master/doc/quantization.md
static const Op& op = Op::Get("qnn.quantize");
- return CallNode::make(op, {data}, Attrs(attrs), {});
+ return CallNode::make(op, {data, output_scale, output_zero_point}, Attrs(attrs), {});
}
-Expr QuantizeLower(const Expr& input_tensor,
- const QuantizeAttrs* attrs) {
+Expr QuantizeLower(const Expr& input_tensor, const Expr& output_scale,
+ const Expr& output_zero_point, const QuantizeAttrs* attrs) {
const auto out_dtype = attrs->out_dtype;
- const auto output_zero_point = MakeConstantScalar(DataType::Float(32), attrs->output_zero_point);
- const auto scale = MakeConstantScalar(DataType::Float(32), attrs->output_scale);
const int32_t min_val = GetQmin(out_dtype);
const int32_t max_val = GetQmax(out_dtype);
- auto scale_data = Divide(input_tensor, scale);
- auto add_zero_point = Cast(Round(Add(scale_data, output_zero_point)), DataType::Int(32));
+ auto scale_data = Divide(input_tensor, output_scale);
+ auto add_zero_point =
+ Cast(Round(Add(scale_data, Cast(output_zero_point, DataType::Float(32)))), DataType::Int(32));
auto clamped_output = Clip(add_zero_point, min_val, max_val);
auto clamp_out_dtype = Cast(clamped_output, out_dtype);
return clamp_out_dtype;
}
-Expr QuantizeQnnCanonicalize(const Attrs& attrs,
- const Array<Expr>& new_args,
+Expr QuantizeQnnCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& types) {
- CHECK_EQ(new_args.size(), 1);
+ CHECK_EQ(new_args.size(), 3);
auto& data = new_args[0];
+ auto& output_scale = new_args[1];
+ auto& output_zero_point = new_args[2];
const auto* quantize_attrs = attrs.as<QuantizeAttrs>();
CHECK(quantize_attrs != nullptr);
- CHECK_EQ(types.size(), 2);
- return QuantizeLower(data, quantize_attrs);
+ CHECK_EQ(types.size(), 4);
+ return QuantizeLower(data, output_scale, output_zero_point, quantize_attrs);
}
RELAY_REGISTER_OP("qnn.quantize")
or quantized.
)code" TVM_ADD_FILELINE)
.set_attrs_type<QuantizeAttrs>()
-.set_num_inputs(1)
+.set_num_inputs(3)
.add_argument("data", "Tensor", "The tensor to quantize.")
+.add_argument("output_scale", "Tensor", "The quantization scale of the output tensor.")
+.add_argument("output_zero_point", "Tensor", "The quantization zero_point of the output tensor.")
.set_support_level(11)
.add_type_rel("Quantize", QuantizeRel)
.set_attr<FTVMLegalize>("FTVMQnnCanonicalize", QuantizeQnnCanonicalize);
* 4) Add the output zero point.
* 5) Cast to the out_dtype.
*/
-Expr RequantizeLower(const Expr& input_tensor, const RequantizeAttrs* param,
+Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale,
+ const Expr& input_zero_point, const Expr& output_scale,
+ const Expr& output_zero_point, const RequantizeAttrs* param,
const Array<IndexExpr>& input_shape, const DataType& out_dtype) {
- double double_multiplier = param->input_scale / param->output_scale;
+ float input_scale_float = GetScalarFromConstant<float>(input_scale);
+ float output_scale_float = GetScalarFromConstant<float>(output_scale);
+ double double_multiplier =
+ static_cast<double>(input_scale_float) / static_cast<double>(output_scale_float);
DataType hp_dtype = DataType::Int(64);
auto tensor = Cast(input_tensor, hp_dtype);
// 1) Subtract the input_zero_point
- if (param->input_zero_point != 0) {
- auto input_zp = MakeConstantScalar(hp_dtype, param->input_zero_point);
- tensor = Subtract(tensor, input_zp);
+ auto zero_scalar = MakeConstantScalar(DataType::Int(32), 0);
+ if (!IsEqualScalar(input_zero_point, zero_scalar)) {
+ tensor = Subtract(tensor, Cast(input_zero_point, hp_dtype));
}
// 2) If the input and output scales are same, we can skip the fixed point multiplication.
auto scaled_int64_t = tensor;
- if (param->input_scale != param->output_scale) {
+ if (!IsEqualScalar(input_scale, output_scale)) {
scaled_int64_t =
FixedPointMultiply(scaled_int64_t, double_multiplier, input_shape, param->rounding);
}
// 3) Add the output zero point.
auto shifted_int64_t = scaled_int64_t;
- if (param->output_zero_point != 0) {
- auto output_zp = MakeConstantScalar(hp_dtype, param->output_zero_point);
- shifted_int64_t = Add(output_zp, scaled_int64_t);
+ if (!IsEqualScalar(output_zero_point, zero_scalar)) {
+ shifted_int64_t = Add(Cast(output_zero_point, hp_dtype), scaled_int64_t);
}
// 4) Clip to the out_dtype min/max.
*/
Expr RequantizeQnnCanonicalize(const Attrs& attrs, const Array<Expr>& new_args,
const Array<tvm::relay::Type>& types) {
- CHECK_EQ(new_args.size(), 1);
+ CHECK_EQ(new_args.size(), 5);
auto& quantized_data = new_args[0];
+ auto& input_scale = new_args[1];
+ auto& input_zero_point = new_args[2];
+ auto& output_scale = new_args[3];
+ auto& output_zero_point = new_args[4];
const auto* param = attrs.as<RequantizeAttrs>();
CHECK(param != nullptr);
// Find input shape.
- CHECK_EQ(types.size(), 2);
+ CHECK_EQ(types.size(), 6);
auto in_type = types[0];
auto in_tensor_type = in_type.as<TensorTypeNode>();
CHECK(in_tensor_type != nullptr) << "Type information missing."
Array<IndexExpr> input_shape = in_tensor_type->shape;
// Find the output dtype.
- auto out_type = types[1];
+ auto out_type = types[5];
auto out_tensor_type = out_type.as<TensorTypeNode>();
CHECK(out_tensor_type != nullptr) << "Type information missing."
<< " Please run infer_type pass.";
CHECK(param->rounding == "UPWARD" || param->rounding == "TONEAREST")
<< "QNN requantize supports two rounding modes - UPWARD and "
<< "TONEAREST";
- return RequantizeLower(quantized_data, param, input_shape, out_dtype);
+ return RequantizeLower(quantized_data, input_scale, input_zero_point, output_scale,
+ output_zero_point, param, input_shape, out_dtype);
}
/*
*/
bool RequantizeRel(const Array<Type>& types, int num_inputs, const Attrs& attrs,
const TypeReporter& reporter) {
- CHECK_EQ(types.size(), 2);
+ CHECK_EQ(types.size(), 6);
const auto* data = types[0].as<TensorTypeNode>();
const auto in_dtype = data->dtype;
CHECK(in_dtype == DataType::Int(8) ||
in_dtype == DataType::Int(32))
<< "Input type should be one of [int8, uint8, int32] but was " << in_dtype;
+ // Check the types of scale and zero points.
+ CHECK(IsScalarType(types[1], DataType::Float(32))); // input_scale
+ CHECK(IsScalarType(types[2], DataType::Int(32))); // input_zero_point
+ CHECK(IsScalarType(types[3], DataType::Float(32))); // output_scale
+ CHECK(IsScalarType(types[4], DataType::Int(32))); // output_zero_point
+
const Array<tvm::Expr> oshape = data->shape;
// assign output type
const RequantizeAttrs* param = attrs.as<RequantizeAttrs>();
out_dtype == DataType::UInt(8) ||
out_dtype == DataType::Int(32))
<< "Output type should be one of [int8, uint8, int32] but was " << out_dtype;
- reporter->Assign(types[1], TensorTypeNode::make(oshape, out_dtype));
+ reporter->Assign(types[5], TensorTypeNode::make(oshape, out_dtype));
return true;
}
// Positional relay function to create qnn requantize operator
// used by frontend FFI.
-Expr MakeRequantize(Expr data, double input_scale, int32_t input_zero_point, double output_scale,
- int32_t output_zero_point, std::string rounding, DataType out_dtype) {
+Expr MakeRequantize(Expr data, Expr input_scale, Expr input_zero_point, Expr output_scale,
+ Expr output_zero_point, std::string rounding, DataType out_dtype) {
auto attrs = make_object<RequantizeAttrs>();
- attrs->input_scale = std::move(input_scale);
- attrs->input_zero_point = std::move(input_zero_point);
- attrs->output_scale = std::move(output_scale);
- attrs->output_zero_point = std::move(output_zero_point);
attrs->rounding = std::move(rounding);
attrs->out_dtype = std::move(out_dtype);
static const Op& op = Op::Get("qnn.requantize");
- return CallNode::make(op, {data}, Attrs(attrs), {});
+ return CallNode::make(op, {data, input_scale, input_zero_point, output_scale, output_zero_point},
+ Attrs(attrs), {});
}
RELAY_REGISTER_OP("qnn.requantize")
)code" TVM_ADD_FILELINE)
.set_attrs_type<RequantizeAttrs>()
-.set_num_inputs(1)
+.set_num_inputs(5)
.add_argument("data", "Tensor", "The quantized input tensor.")
+.add_argument("input_scale", "Tensor", "The quantization scale of the input tensor.")
+.add_argument("input_zero_point", "Tensor", "The quantization zero_point of the input tensor.")
+.add_argument("output_scale", "Tensor", "The quantization scale of the output tensor.")
+.add_argument("output_zero_point", "Tensor", "The quantization zero_point of the output tensor.")
.set_support_level(11)
.add_type_rel("Requantize", RequantizeRel)
.set_attr<FTVMLegalize>("FTVMQnnCanonicalize", RequantizeQnnCanonicalize);
}
}
-Expr RequantizeLower(const Expr& input_tensor, const RequantizeAttrs* param,
+Expr RequantizeLower(const Expr& input_tensor, const Expr& input_scale,
+ const Expr& input_zero_point, const Expr& output_scale,
+ const Expr& output_zero_point, const RequantizeAttrs* param,
const Array<IndexExpr>& input_shape, const DataType& out_dtype);
static inline Expr Requantize(const Expr& data, const Array<IndexExpr>& input_shape,
- double input_scale, int32_t input_zero_point, double output_scale,
- int32_t output_zero_point, const DataType& out_dtype,
- const std::string& rounding = "UPWARD") {
+ const Expr& input_scale, const Expr& input_zero_point,
+ const Expr& output_scale, const Expr& output_zero_point,
+ const DataType& out_dtype, const std::string& rounding = "UPWARD") {
auto attrs = make_object<RequantizeAttrs>();
- attrs->input_scale = std::move(input_scale);
- attrs->input_zero_point = std::move(input_zero_point);
- attrs->output_scale = std::move(output_scale);
- attrs->output_zero_point = std::move(output_zero_point);
attrs->rounding = std::move(rounding);
attrs->out_dtype = std::move(out_dtype);
- return RequantizeLower(data, attrs.operator->(), input_shape, out_dtype);
+ return RequantizeLower(data, input_scale, input_zero_point, output_scale, output_zero_point,
+ attrs.operator->(), input_shape, out_dtype);
}
static inline int64_t get_const_int(const tvm::Expr& x) {
* 2) Round the result.
* 3) Right shift the result
*/
-Expr FixedPointMultiply(Expr tensor, double multiplier,
- const Array<IndexExpr>& input_shape,
+Expr FixedPointMultiply(Expr tensor, double multiplier, const Array<IndexExpr>& input_shape,
const std::string& rounding);
+/*
+ * \brief Checks whether an expr type is scalar of a given data type.
+ * \param expr_type The type of expr to be checked.
+ * \param dtype The expected dtype.
+ * \return True if the type is a scalar of given dtype
+ */
+static inline bool IsScalarType(const Type& expr_type, const DataType& dtype) {
+ const auto* scale = expr_type.as<TensorTypeNode>();
+ CHECK_EQ(scale->shape.size(), 0);
+ CHECK(scale->dtype == dtype) << "Expected " << dtype << " but got " << scale->dtype;
+ return true;
+}
+
} // namespace qnn
} // namespace relay
} // namespace tvm
x = relay.var("x", shape=(1, 4), dtype=data_dtype)
y = relay.var("y", shape=(1, 4), dtype=data_dtype)
z = relay.qnn.op.add(lhs=x, rhs=y,
- lhs_scale=0.00784314,
- lhs_zero_point=127,
- rhs_scale=0.00784314,
- rhs_zero_point=127,
- output_scale=0.00784314,
- output_zero_point=127)
+ lhs_scale=relay.const(0.00784314, 'float32'),
+ lhs_zero_point=relay.const(127, 'int32'),
+ rhs_scale=relay.const(0.00784314, 'float32'),
+ rhs_zero_point=relay.const(127, 'int32'),
+ output_scale=relay.const(0.00784314, 'float32'),
+ output_zero_point=relay.const(127, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
x = relay.var("x", shape=(1, 4), dtype=data_dtype)
y = relay.var("y", shape=(1, 4), dtype=data_dtype)
z = relay.qnn.op.add(lhs=x, rhs=y,
- lhs_scale=0.0156863,
- lhs_zero_point=127,
- rhs_scale=0.0117647,
- rhs_zero_point=85,
- output_scale=0.0235294,
- output_zero_point=128)
+ lhs_scale=relay.const(0.0156863, 'float32'),
+ lhs_zero_point=relay.const(127, 'int32'),
+ rhs_scale=relay.const(0.0117647, 'float32'),
+ rhs_zero_point=relay.const(85, 'int32'),
+ output_scale=relay.const(0.0235294, 'float32'),
+ output_zero_point=relay.const(128, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
x = relay.var("x", shape=(1, 4), dtype=data_dtype)
y = relay.var("y", shape=(1, 4), dtype=data_dtype)
z = relay.qnn.op.add(lhs=x, rhs=y,
- lhs_scale=0.125,
- lhs_zero_point=0,
- rhs_scale=0.125,
- rhs_zero_point=0,
- output_scale=0.125,
- output_zero_point=0)
+ lhs_scale=relay.const(0.125, 'float32'),
+ lhs_zero_point=relay.const(0, 'int32'),
+ rhs_scale=relay.const(0.125, 'float32'),
+ rhs_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(0.125, 'float32'),
+ output_zero_point=relay.const(0, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
# Same params, different scale
z = relay.qnn.op.add(lhs=x, rhs=y,
- lhs_scale=0.125,
- lhs_zero_point=0,
- rhs_scale=0.125,
- rhs_zero_point=0,
- output_scale=0.25,
- output_zero_point=0)
+ lhs_scale=relay.const(0.125, 'float32'),
+ lhs_zero_point=relay.const(0, 'int32'),
+ rhs_scale=relay.const(0.125, 'float32'),
+ rhs_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(0.25, 'float32'),
+ output_zero_point=relay.const(0, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
# Same io params, different output scale
z = relay.qnn.op.add(lhs=x, rhs=y,
- lhs_scale=0.125,
- lhs_zero_point=0,
- rhs_scale=0.125,
- rhs_zero_point=0,
- output_scale=0.25,
- output_zero_point=0)
+ lhs_scale=relay.const(0.125, 'float32'),
+ lhs_zero_point=relay.const(0, 'int32'),
+ rhs_scale=relay.const(0.125, 'float32'),
+ rhs_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(0.25, 'float32'),
+ output_zero_point=relay.const(0, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
# All params different
z = relay.qnn.op.add(lhs=x, rhs=y,
- lhs_scale=0.5,
- lhs_zero_point=0,
- rhs_scale=0.25,
- rhs_zero_point=0,
- output_scale=0.125,
- output_zero_point=0)
+ lhs_scale=relay.const(0.5, 'float32'),
+ lhs_zero_point=relay.const(0, 'int32'),
+ rhs_scale=relay.const(0.25, 'float32'),
+ rhs_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(0.125, 'float32'),
+ output_zero_point=relay.const(0, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
axis = 0
x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype)
y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype)
- x_scale = (62 + 64) / (np.power(2, 32) - 1.0)
- y_scale = (62 + 64) / (np.power(2, 32) - 1.0)
+ x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ zero = relay.const(0, 'int32')
x = relay.var("x", shape=(1, 64), dtype=data_dtype)
y = relay.var("y", shape=(1, 64), dtype=data_dtype)
z = relay.qnn.op.concatenate((x, y),
- input_scales=[x_scale, y_scale],
- input_zero_points=[0, 0],
+ input_scales=(x_scale, y_scale),
+ input_zero_points=(zero, zero),
output_scale=y_scale,
- output_zero_point=0,
+ output_zero_point=zero,
axis=axis)
func = relay.Function([x, y], z)
axis = 0
x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype)
y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype)
- x_scale = (62 + 64) / (np.power(2, 32) - 1.0)
- y_scale = (62 + 64) / (np.power(2, 32) - 1.0)
+
+ x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ x_zero_point = relay.const(3, 'int32')
+ y_zero_point = relay.const(4, 'int32')
x = relay.var("x", shape=(1, 64), dtype=data_dtype)
y = relay.var("y", shape=(1, 64), dtype=data_dtype)
z = relay.qnn.op.concatenate((x, y),
- input_scales=[x_scale, y_scale],
- input_zero_points=[3, 4],
+ input_scales=(x_scale, y_scale),
+ input_zero_points=(x_zero_point, y_zero_point),
output_scale=y_scale,
- output_zero_point=1,
+ output_zero_point=relay.const(1, 'int32'),
axis=axis)
func = relay.Function([x, y], z)
axis = 0
x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype)
y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype)
- x_scale = (62 + 64) / (np.power(2, 32) - 1.0)
- y_scale = (62 + 64) / (np.power(2, 32) - 1.0)
+
+ x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ x_zero_point = relay.const(0, 'int32')
+ y_zero_point = relay.const(1, 'int32')
x = relay.var("x", shape=(1, 64), dtype=data_dtype)
y = relay.var("y", shape=(1, 64), dtype=data_dtype)
z = relay.qnn.op.concatenate((x, y),
- input_scales=[x_scale, y_scale],
- input_zero_points=[0, 1],
+ input_scales=(x_scale, y_scale),
+ input_zero_points=(x_zero_point, y_zero_point),
output_scale=y_scale,
- output_zero_point=1,
+ output_zero_point=relay.const(1, 'int32'),
axis=axis)
func = relay.Function([x, y], z)
axis = 0
x_data = np.arange(-32, 32, 1).reshape(1, 64).astype(data_dtype)
y_data = np.arange(-64, 64, 2).reshape(1, 64).astype(data_dtype)
- x_scale = (62 + 64) / (np.power(2, 32) - 1.0)
- y_scale = (62 + 64) / (np.power(2, 32) - 1.0)
+
+ x_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ y_scale = relay.const((62 + 64) / (np.power(2, 32) - 1.0), 'float32')
+ x_zero_point = relay.const(0, 'int32')
+ y_zero_point = relay.const(0, 'int32')
x = relay.var("x", shape=(1, 64), dtype=data_dtype)
y = relay.var("y", shape=(1, 64), dtype=data_dtype)
z = relay.qnn.op.concatenate((x, y),
- input_scales=[x_scale, y_scale],
- input_zero_points=[0, 0],
+ input_scales=(x_scale, y_scale),
+ input_zero_points=(x_zero_point, y_zero_point),
output_scale=y_scale,
- output_zero_point=1,
+ output_zero_point=relay.const(1, 'int32'),
axis=axis)
func = relay.Function([x, y], z)
shifted_kernel = relay.op.subtract(casted_kernel,
relay.const(kernel_zero_point, "int32"))
func = relay.op.nn.conv2d(shifted_data,
- shifted_kernel,
- padding=padding,
- strides=strides,
- dilation=dilation,
- groups=groups,
- channels=channels,
- kernel_size=kernel_size,
- out_dtype=out_dtype,
- data_layout=data_layout,
- kernel_layout=kernel_layout)
+ shifted_kernel,
+ padding=padding,
+ strides=strides,
+ dilation=dilation,
+ groups=groups,
+ channels=channels,
+ kernel_size=kernel_size,
+ out_dtype=out_dtype,
+ data_layout=data_layout,
+ kernel_layout=kernel_layout)
func = relay.Function(relay.analysis.free_vars(func), func)
return func
channels=None):
func = relay.qnn.op.conv2d(
data, kernel,
- input_zero_point=input_zero_point,
- kernel_zero_point=kernel_zero_point,
- input_scale=input_scale,
- kernel_scale=kernel_scale,
+ input_zero_point=relay.const(input_zero_point, 'int32'),
+ kernel_zero_point=relay.const(kernel_zero_point, 'int32'),
+ input_scale=relay.const(input_scale, 'float32'),
+ kernel_scale=relay.const(kernel_scale, 'float32'),
kernel_size=kernel_size,
strides=strides,
dilation=dilation,
mod = relay.qnn.op.dense(
quantized_data,
quantized_kernel,
- test_configuration['input_zero_point'],
- test_configuration['kernel_zero_point'],
- test_configuration['input_scale'],
- test_configuration['kernel_scale'],
+ relay.const(test_configuration['input_zero_point'], 'int32'),
+ relay.const(test_configuration['kernel_zero_point'], 'int32'),
+ relay.const(test_configuration['input_scale'], 'float32'),
+ relay.const(test_configuration['kernel_scale'], 'float32'),
test_configuration['units'])
if test_configuration[bias_name] is not None:
bias = relay.var(bias_name,
requantize_config = test_configuration['requantize']
mod = relay.qnn.op.requantize(
mod,
- input_scale=requantize_config['input_scale'],
- input_zero_point=0,
- output_scale=requantize_config['output_scale'],
- output_zero_point=requantize_config['output_zero_point'],
+ input_scale=relay.const(requantize_config['input_scale'], 'float32'),
+ input_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(requantize_config['output_scale'], 'float32'),
+ output_zero_point=relay.const(requantize_config['output_zero_point'], 'int32'),
out_dtype=requantize_config['out_dtype'])
expected_out_dtype = requantize_config['out_dtype']
from tvm import relay
from tvm.contrib import graph_runtime
-def test_dequantize_op():
+def quantize_test_driver(in_dtype, quant_args, in_data, verify_output_data):
+ shape = in_data.shape
+ input_data = relay.var("input_data", shape=shape, dtype=in_dtype)
+ input_zero_point = relay.const(quant_args['in_zero_point'], 'int32')
+ input_scale = relay.const(quant_args['in_scale'], 'float32')
+ quantized_output = relay.qnn.op.dequantize(input_data, input_scale=input_scale,
+ input_zero_point=input_zero_point)
+ mod = relay.Function(relay.analysis.free_vars(quantized_output), quantized_output)
+ mod = relay.Module.from_expr(mod)
+ with relay.build_config(opt_level=3):
+ graph, lib, params = relay.build(mod, "llvm", params=None)
+ rt_mod = graph_runtime.create(graph, lib, ctx=tvm.cpu(0))
+ rt_mod.set_input(input_data=in_data)
+ rt_mod.set_input(**params)
+ rt_mod.run()
+ res = rt_mod.get_output(0).asnumpy()
+ np.testing.assert_equal(res, verify_output_data)
+ assert res.dtype == np.float32
- def quantize_test_driver(in_dtype, quant_args, in_data, verify_output_data):
- shape = in_data.shape
- input_data = relay.var("input_data", shape=shape, dtype=in_dtype)
- input_zero_point = quant_args['in_zero_point']
- input_scale = quant_args['in_scale']
- quantized_output = relay.qnn.op.dequantize(input_data, input_scale=input_scale,
- input_zero_point=input_zero_point)
- mod = relay.Function(relay.analysis.free_vars(quantized_output), quantized_output)
- mod = relay.Module.from_expr(mod)
- with relay.build_config(opt_level=3):
- graph, lib, params = relay.build(mod, "llvm", params=None)
- rt_mod = graph_runtime.create(graph, lib, ctx=tvm.cpu(0))
- rt_mod.set_input(input_data=in_data)
- rt_mod.set_input(**params)
- rt_mod.run()
- res = rt_mod.get_output(0).asnumpy()
- np.testing.assert_equal(res, verify_output_data)
- assert res.dtype == np.float32
+def test_uint8_to_float32():
+ data = np.array([0, 1, 2, 3, 4, 251, 252, 253, 254, 255]) \
+ .astype('uint8') \
+ .reshape((2, 5))
+ output = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
+ .astype('float32') \
+ .reshape((2, 5))
+ quant_args = {"in_zero_point":127, "in_scale":0.5}
+ quantize_test_driver(in_dtype='uint8', quant_args=quant_args, in_data=data,
+ verify_output_data=output)
- def test_uint8_to_float32():
- data = np.array([0, 1, 2, 3, 4, 251, 252, 253, 254, 255]) \
- .astype('uint8') \
- .reshape((2, 5))
- output = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
- .astype('float32') \
- .reshape((2, 5))
- quant_args = {"in_zero_point":127, "in_scale":0.5}
- quantize_test_driver(in_dtype='uint8', quant_args=quant_args, in_data=data,
- verify_output_data=output)
+def test_int8_to_float32():
+ data = np.array([-128, -127, -126, -125, -124, 123, 124, 125, 126, 127]) \
+ .astype('int8') \
+ .reshape((2, 5))
+ output = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
+ .astype('float32') \
+ .reshape((2, 5))
+ quant_args = {"in_zero_point": -1, "in_scale": 0.5}
+ quantize_test_driver(in_dtype='int8', quant_args=quant_args, in_data=data,
+ verify_output_data=output)
- def test_int8_to_float32():
- data = np.array([-128, -127, -126, -125, -124, 123, 124, 125, 126, 127]) \
- .astype('int8') \
- .reshape((2, 5))
- output = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
- .astype('float32') \
- .reshape((2, 5))
- quant_args = {"in_zero_point": -1, "in_scale": 0.5}
- quantize_test_driver(in_dtype='int8', quant_args=quant_args, in_data=data,
- verify_output_data=output)
+def test_int32_to_float32():
+ data = np.array([113, 29, -1052]).astype('int32')
+ output = np.array([0.6550452, 0.16810896, -6.098297]).astype('float32')
+ quant_args = {"in_zero_point": 0, "in_scale": 0.0057968604}
+ quantize_test_driver(in_dtype='int32', quant_args=quant_args, in_data=data,
+ verify_output_data=output)
- def test_int32_to_float32():
- data = np.array([113, 29, -1052]).astype('int32')
- output = np.array([0.6550452, 0.16810896, -6.098297]).astype('float32')
- quant_args = {"in_zero_point": 0, "in_scale": 0.0057968604}
- quantize_test_driver(in_dtype='int32', quant_args=quant_args, in_data=data,
- verify_output_data=output)
+if __name__ == "__main__":
test_uint8_to_float32()
test_int8_to_float32()
test_int32_to_float32()
-
-
-if __name__ == "__main__":
- test_dequantize_op()
-
x = relay.var("x", shape=(1, 4), dtype=data_dtype)
y = relay.var("y", shape=(1, 4), dtype=data_dtype)
z = relay.qnn.op.mul(lhs=x, rhs=y,
- lhs_scale=lhs_scale,
- lhs_zero_point=lhs_zero_point,
- rhs_scale=rhs_scale,
- rhs_zero_point=rhs_zero_point,
- output_scale=output_scale,
- output_zero_point=output_zero_point)
+ lhs_scale=relay.const(lhs_scale, 'float32'),
+ lhs_zero_point=relay.const(lhs_zero_point, 'int32'),
+ rhs_scale=relay.const(rhs_scale, 'float32'),
+ rhs_zero_point=relay.const(rhs_zero_point, 'int32'),
+ output_scale=relay.const(output_scale, 'float32'),
+ output_zero_point=relay.const(output_zero_point, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
x = relay.var("x", shape=(1, 4), dtype=data_dtype)
y = relay.var("y", shape=(1, 4), dtype=data_dtype)
z = relay.qnn.op.mul(lhs=x, rhs=y,
- lhs_scale=lhs_scale,
- lhs_zero_point=lhs_zero_point,
- rhs_scale=rhs_scale,
- rhs_zero_point=rhs_zero_point,
- output_scale=output_scale,
- output_zero_point=output_zero_point)
+ lhs_scale=relay.const(lhs_scale, 'float32'),
+ lhs_zero_point=relay.const(lhs_zero_point, 'int32'),
+ rhs_scale=relay.const(rhs_scale, 'float32'),
+ rhs_zero_point=relay.const(rhs_zero_point, 'int32'),
+ output_scale=relay.const(output_scale, 'float32'),
+ output_zero_point=relay.const(output_zero_point, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
x = relay.var("x", shape=(1, 4), dtype=data_dtype)
y = relay.var("y", shape=(1, 4), dtype=data_dtype)
z = relay.qnn.op.mul(lhs=x, rhs=y,
- lhs_scale=lhs_scale,
- lhs_zero_point=lhs_zero_point,
- rhs_scale=rhs_scale,
- rhs_zero_point=rhs_zero_point,
- output_scale=output_scale,
- output_zero_point=output_zero_point)
+ lhs_scale=relay.const(lhs_scale, 'float32'),
+ lhs_zero_point=relay.const(lhs_zero_point, 'int32'),
+ rhs_scale=relay.const(rhs_scale, 'float32'),
+ rhs_zero_point=relay.const(rhs_zero_point, 'int32'),
+ output_scale=relay.const(output_scale, 'float32'),
+ output_zero_point=relay.const(output_zero_point, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
output_scale = 0.25
z = relay.qnn.op.mul(lhs=x, rhs=y,
- lhs_scale=lhs_scale,
- lhs_zero_point=lhs_zero_point,
- rhs_scale=rhs_scale,
- rhs_zero_point=rhs_zero_point,
- output_scale=output_scale,
- output_zero_point=output_zero_point)
+ lhs_scale=relay.const(lhs_scale, 'float32'),
+ lhs_zero_point=relay.const(lhs_zero_point, 'int32'),
+ rhs_scale=relay.const(rhs_scale, 'float32'),
+ rhs_zero_point=relay.const(rhs_zero_point, 'int32'),
+ output_scale=relay.const(output_scale, 'float32'),
+ output_zero_point=relay.const(output_zero_point, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
output_scale = 0.125
z = relay.qnn.op.mul(lhs=x, rhs=y,
- lhs_scale=lhs_scale,
- lhs_zero_point=lhs_zero_point,
- rhs_scale=rhs_scale,
- rhs_zero_point=rhs_zero_point,
- output_scale=output_scale,
- output_zero_point=output_zero_point)
+ lhs_scale=relay.const(lhs_scale, 'float32'),
+ lhs_zero_point=relay.const(lhs_zero_point, 'int32'),
+ rhs_scale=relay.const(rhs_scale, 'float32'),
+ rhs_zero_point=relay.const(rhs_zero_point, 'int32'),
+ output_scale=relay.const(output_scale, 'float32'),
+ output_zero_point=relay.const(output_zero_point, 'int32'))
func = relay.Function([x, y], z)
mod = relay.Module.from_expr(func)
from tvm import relay
from tvm.contrib import graph_runtime
-def test_quantize_op():
+def quantize_test_driver(in_dtype, quant_args, out_dtype, in_data, verify_output_data):
+ shape = in_data.shape
+ input_data = relay.var("input_data", shape=shape, dtype=in_dtype)
+ output_zero_point = relay.const(quant_args['out_zero_point'], 'int32')
+ output_scale = relay.const(quant_args['out_scale'], 'float32')
+ quantized_output = relay.qnn.op.quantize(input_data, output_scale=output_scale,
+ output_zero_point=output_zero_point,out_dtype=out_dtype)
+ mod = relay.Function(relay.analysis.free_vars(quantized_output), quantized_output)
+ mod = relay.Module.from_expr(mod)
+ with relay.build_config(opt_level=3):
+ graph, lib, params = relay.build(mod, "llvm", params=None)
+ rt_mod = graph_runtime.create(graph, lib, ctx=tvm.cpu(0))
+ rt_mod.set_input(input_data=in_data)
+ rt_mod.set_input(**params)
+ rt_mod.run()
+ res = rt_mod.get_output(0).asnumpy()
+ np.testing.assert_equal(res, verify_output_data)
+ assert res.dtype == out_dtype
- def quantize_test_driver(in_dtype, quant_args, out_dtype, in_data, verify_output_data):
- shape = in_data.shape
- input_data = relay.var("input_data", shape=shape, dtype=in_dtype)
- output_zero_point = quant_args['out_zero_point']
- output_scale = quant_args['out_scale']
- quantized_output = relay.qnn.op.quantize(input_data, output_scale=output_scale,
- output_zero_point=output_zero_point,out_dtype=out_dtype)
- mod = relay.Function(relay.analysis.free_vars(quantized_output), quantized_output)
- mod = relay.Module.from_expr(mod)
- with relay.build_config(opt_level=3):
- graph, lib, params = relay.build(mod, "llvm", params=None)
- rt_mod = graph_runtime.create(graph, lib, ctx=tvm.cpu(0))
- rt_mod.set_input(input_data=in_data)
- rt_mod.set_input(**params)
- rt_mod.run()
- res = rt_mod.get_output(0).asnumpy()
- np.testing.assert_equal(res, verify_output_data)
- assert res.dtype == out_dtype
+def test_float32_to_uint8():
+ data = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
+ .astype('float32') \
+ .reshape((2,5))
+ output = np.array([0, 1, 2, 3, 4, 251, 252, 253, 254, 255]) \
+ .astype('uint8') \
+ .reshape((2,5))
+ quant_args = {"out_zero_point":127, "out_scale":0.5}
+ quantize_test_driver(in_dtype='float32', quant_args=quant_args, out_dtype='uint8', in_data=data,
+ verify_output_data=output)
- def test_float32_to_uint8():
- data = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
- .astype('float32') \
- .reshape((2,5))
- output = np.array([0, 1, 2, 3, 4, 251, 252, 253, 254, 255]) \
- .astype('uint8') \
- .reshape((2,5))
- quant_args = {"out_zero_point":127, "out_scale":0.5}
- quantize_test_driver(in_dtype='float32', quant_args=quant_args, out_dtype='uint8', in_data=data,
- verify_output_data=output)
-
- def test_float32_to_int8():
- data = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
- .astype('float32') \
- .reshape((2,5))
- output = np.array([-128, -127, -126, -125, -124, 123, 124, 125, 126, 127]) \
- .astype('int8') \
- .reshape((2,5))
- quant_args = {"out_zero_point":-1, "out_scale":0.5}
- quantize_test_driver(in_dtype='float32', quant_args=quant_args, out_dtype='int8', in_data=data,
- verify_output_data=output)
+def test_float32_to_int8():
+ data = np.array([-63.5, -63, -62.5, -62, -61.5, 62, 62.5, 63, 63.5, 64]) \
+ .astype('float32') \
+ .reshape((2,5))
+ output = np.array([-128, -127, -126, -125, -124, 123, 124, 125, 126, 127]) \
+ .astype('int8') \
+ .reshape((2,5))
+ quant_args = {"out_zero_point":-1, "out_scale":0.5}
+ quantize_test_driver(in_dtype='float32', quant_args=quant_args, out_dtype='int8', in_data=data,
+ verify_output_data=output)
+if __name__ == "__main__":
test_float32_to_uint8()
test_float32_to_int8()
-
-if __name__ == "__main__":
- test_quantize_op()
dtype=data_dtype)
mod = relay.qnn.op.requantize(
quantized_data,
- input_scale=input_scale,
- input_zero_point=input_zero_point,
- output_scale=output_scale,
- output_zero_point=output_zero_point,
+ input_scale=relay.const(input_scale, 'float32'),
+ input_zero_point=relay.const(input_zero_point, 'int32'),
+ output_scale=relay.const(output_scale, 'float32'),
+ output_zero_point=relay.const(output_zero_point, 'int32'),
rounding=rounding,
out_dtype=out_dtype)
def before():
x = relay.var("x", shape=(1, 64, 56, 56), dtype='int8')
y = relay.qnn.op.requantize(x,
- input_scale=1,
- input_zero_point=0,
- output_scale=1,
- output_zero_point=0,
+ input_scale=relay.const(1, 'float32'),
+ input_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(1, 'float32'),
+ output_zero_point=relay.const(0, 'int32'),
out_dtype='int8')
y = relay.Function([x], y)
return y
data = inputs[0]
data = relay.add(relay.const(0, 'int8'), data)
y = relay.qnn.op.requantize(data,
- input_scale=1,
- input_zero_point=0,
- output_scale=1,
- output_zero_point=0,
+ input_scale=relay.const(1, 'float32'),
+ input_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(1, 'float32'),
+ output_zero_point=relay.const(0, 'int32'),
out_dtype='int8')
return y
x = relay.var("x", shape=(1, 64, 56, 56), dtype='int8')
y = relay.add(relay.const(0, 'int8'), x)
z = relay.qnn.op.requantize(y,
- input_scale=1,
- input_zero_point=0,
- output_scale=1,
- output_zero_point=0,
+ input_scale=relay.const(1, 'float32'),
+ input_zero_point=relay.const(0, 'int32'),
+ output_scale=relay.const(1, 'float32'),
+ output_zero_point=relay.const(0, 'int32'),
out_dtype='int8')
z = relay.Function([x], z)
return z
dtype=kernel_dtype)
func = relay.qnn.op.conv2d(
data, kernel,
- input_zero_point=1,
- kernel_zero_point=1,
- input_scale=1.0,
- kernel_scale=1.0,
+ input_zero_point=relay.const(1, 'int32'),
+ kernel_zero_point=relay.const(1, 'int32'),
+ input_scale=relay.const(1.0, 'float32'),
+ kernel_scale=relay.const(1.0, 'float32'),
kernel_size=(3, 3),
strides=(1, 1),
dilation=(1, 1),
dtype=kernel_dtype)
func = relay.qnn.op.dense(
data, kernel,
- input_zero_point=1,
- kernel_zero_point=1,
- input_scale=1,
- kernel_scale=1,
+ input_zero_point=relay.const(1, 'int32'),
+ kernel_zero_point=relay.const(1, 'int32'),
+ input_scale=relay.const(1, 'float32'),
+ kernel_scale=relay.const(1, 'float32'),
out_dtype='int32')
mod = relay.Function(relay.analysis.free_vars(func), func)
projects / platform / upstream / tvm.git / commitdiff