/// Returns if a `Tensor` has sparse backend.
bool is_sparse() const;
+ /// Returns if a `Tensor` has quantized backend.
+ bool is_quantized() const;
+
/// Returns the `TensorOptions` corresponding to this `Tensor`. Defined in
/// TensorOptions.h.
TensorOptions options() const;
return self.is_sparse();
}
+inline bool Tensor::is_quantized() const {
+ // NB: this is not a native function to avoid dispatching overhead.
+ return impl_->is_quantized();
+}
+
+inline bool is_quantized(Tensor self) {
+ return self.is_quantized();
+}
+
#define DEFINE_CAST(T, name, _) \
template <> \
inline T* Tensor::data() const { \
AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_EXCEPT_COMPLEX_HALF_AND_QINT(DEFINE_ITEM)
#undef DEFINE_ITEM
-// TODO: after is_quantized() is implemented,
-// implement item() (returnning a float) for quantized Tensor
-
} //namespace at
SparseCUDALong,
SparseCUDAShort,
SparseCUDAQInt8,
+ QuantizedCPUQInt8,
MSNPUBool,
MSNPUByte,
MSNPUChar,
virtual bool is_cuda() const = 0;
virtual bool is_hip() const = 0;
virtual bool is_sparse() const = 0;
+ virtual bool is_quantized() const = 0;
virtual bool is_distributed() const = 0;
bool is_variable() const noexcept { return is_variable_; }
bool is_undefined() const noexcept { return is_undefined_; }
# type: (Environment, List[FunctionOption]) -> Tuple[List[str], List[str]]
type_object_declarations = []
type_object_definitions = []
-
is_cuda = 'CUDA' in backend_type_env['Backend']
def replace_with_null(argument):
},
}
-backends = ['CPU', 'CUDA']
+backends = ['CPU', 'CUDA', 'QuantizedCPU']
densities = ['Dense', 'Sparse', 'Mkldnn'] # TODO: layout instead of densities?
+def backend_to_devicetype(backend):
+ if backend == 'QuantizedCPU':
+ return 'CPU'
+ return backend
+
+quantized_backends = ['QuantizedCPU']
+
extension_backends = ['MSNPU', 'XLA']
# scalar_name, c_type, accreal, is_floating_type
+quantized_scalar_types = [
+ ('QInt8', 'qint8', 'QInt8AccrealNotDefined', 'Qint8IsFloatingTypeNotDefined'),
+]
+
scalar_types = [
('Bool', 'bool', 'BoolAccrealNotDefined', False),
('Byte', 'uint8_t', 'Long', False),
('Long', 'int64_t', 'Long', False),
('Short', 'int16_t', 'Long', False),
('Half', 'Half', 'Double', True),
- ('QInt8', 'qint8', 'Long', False),
-]
+] + quantized_scalar_types
+
+
# shared environment for non-derived base classes Type.h Tensor.h Storage.h
top_env = {
env['isFloatingType'] = is_floating_type
env['isIntegralType'] = not is_floating_type
env['Type'] = "{}{}{}Type".format(density_tag, backend, scalar_name)
- env['DenseTensor'] = "{}{}Tensor".format(backend, scalar_name)
+ env['DeviceType'] = backend_to_devicetype(backend)
env['Backend'] = density_tag + backend
- env['DenseBackend'] = backend
env['storage_tensor_headers'] = []
if density != 'Sparse':
env['storage_tensor_headers'] = ['#include <c10/core/TensorImpl.h>']
env['type_derived_method_definitions'] = definitions
fm = file_manager
- if env['DenseBackend'] == 'CUDA':
+ if env['DeviceType'] == 'CUDA':
fm = cuda_file_manager
if density != 'Sparse':
fm.write(env['Type'] + ".h", TYPE_DERIVED_H, env)
type_register = TYPE_REGISTER.substitute(backend=env['Backend'], scalar_type=scalar_name, type_name=env['Type'])
- if env['DenseBackend'] == 'CPU':
+ if env['DeviceType'] == 'CPU':
top_env['cpu_type_registrations'].append(type_register)
top_env['cpu_type_headers'].append(
'#include "ATen/{}.h"'.format(env['Type']))
else:
- assert env['DenseBackend'] == 'CUDA'
+ assert env['DeviceType'] == 'CUDA'
top_env['cuda_type_registrations'].append(type_register)
top_env['cuda_type_headers'].append(
'#include "ATen/{}.h"'.format(env['Type']))
env = {}
env['Type'] = "{}Type".format(backend)
env['Backend'] = backend
- env['DeviceType'] = backend
+ env['DeviceType'] = backend_to_devicetype(backend)
declarations, definitions = function_wrapper.create_extension_backend(
env, declarations)
if density == 'Sparse' and scalar_type[0] == 'Half':
# THS does not do half type yet.
continue
- yield (backend, density, scalar_type)
+ if backend in quantized_backends:
+ if density == 'Dense' and scalar_type in quantized_scalar_types:
+ yield (backend, density, scalar_type)
+ else:
+ continue
+ else:
+ yield (backend, density, scalar_type)
###################
if (self._nnz() == 0) return Scalar(0);
if (self.is_coalesced()) return at::_local_scalar_dense(self._values());
return at::_local_scalar_dense(self._values().sum());
+ } else if (self.is_quantized()) {
+ return self.dequantize().item();
} else {
return _local_scalar_dense(self);
}
return self.is_sparse();
}
+bool is_quantized(const Tensor& self) {
+ return self.is_quantized();
+}
+
Tensor type_as(const Tensor& self, const Tensor& other) {
return self.toType(other.type());
}
- func: quantize_linear(Tensor self, float scale, int zero_point) -> Tensor
variants: function, method
- requires_tensor: True
+ dispatch:
+ CPU: quantize_linear_cpu
- func: dequantize(Tensor self) -> Tensor
variants: function, method
- requires_tensor: True
+ dispatch:
+ QuantizedCPU: dequantize_quant
- func: q_scale(Tensor self) -> Scalar
variants: function, method
- requires_tensor: True
+ dispatch:
+ QuantizedCPU: q_scale_quant
- func: q_zero_point(Tensor self) -> Scalar
variants: function, method
- requires_tensor: True
+ dispatch:
+ QuantizedCPU: q_zero_point_quant
# to(Device) must not exist because all constructors of Device also works for
# TensorOptions. Otherwise, an ambiguity error is thrown.
namespace at {
namespace native {
-QTensor quantize_linear(const RealTensor& self, double scale, int64_t zero_point) {
+QTensor quantize_linear_cpu(const RealTensor& self, double scale, int64_t zero_point) {
auto quantizer = make_per_tensor_affine_quantizer(scale, zero_point);
return quantizer->quantize(self);
}
-RealTensor dequantize(const QTensor& self) {
+RealTensor dequantize_quant(const QTensor& self) {
return get_qtensorimpl(self)->quantizer()->dequantize(self);
}
-Scalar q_scale(const QTensor& self) {
+Scalar q_scale_quant(const QTensor& self) {
auto quantizer = get_qtensorimpl(self)->quantizer();
AT_ASSERT(quantizer->qscheme() == kPerTensorAffine);
return Scalar(static_cast<PerTensorAffineQuantizer*>(quantizer.get())->scale());
}
-Scalar q_zero_point(const QTensor& self) {
+Scalar q_zero_point_quant(const QTensor& self) {
auto quantizer = get_qtensorimpl(self)->quantizer();
AT_ASSERT(quantizer->qscheme() == kPerTensorAffine);
return Scalar(static_cast<PerTensorAffineQuantizer*>(quantizer.get())->zero_point());
all_types = type_map['floating_point'] + type_map['integral']
type_map['all'] = all_types
-all_backends = ['CPU', 'CUDA', 'SparseCPU', 'SparseCUDA', 'MkldnnCPU']
+all_backends = ['CPU', 'CUDA', 'SparseCPU', 'SparseCUDA', 'MkldnnCPU', 'QuantizedCPU']
default_backends = ['CPU', 'CUDA']
allocator->allocate(nelements * dtype.itemsize()),
allocator,
/*resizable=*/true);
+ // TODO: get TensorTypeId from quantizer
auto tensor = detail::make_tensor<QTensorImpl>(
- storage_impl, at::CPUTensorId(), quantizer);
+ storage_impl, at::QuantizedCPUTensorId(), quantizer);
get_qtensorimpl(tensor)->set_sizes_contiguous(sizes);
return tensor;
}
namespace at {
${Type}::${Type}()
- : ${DenseBackend}TypeDefault(${Backend}TensorId(), /*is_variable=*/false, /*is_undefined=*/false) {}
+ : ${DeviceType}TypeDefault(${Backend}TensorId(), /*is_variable=*/false, /*is_undefined=*/false) {}
ScalarType ${Type}::scalarType() const {
return ScalarType::${ScalarName};
}
/// Returns if a `Tensor` has sparse backend.
bool is_sparse() const;
+ /// Returns if a `Tensor` has quantized backend.
+ bool is_quantized() const;
+
/// Returns the `TensorOptions` corresponding to this `Tensor`. Defined in
/// TensorOptions.h.
TensorOptions options() const;
return self.is_sparse();
}
+inline bool Tensor::is_quantized() const {
+ // NB: this is not a native function to avoid dispatching overhead.
+ return impl_->is_quantized();
+}
+
+inline bool is_quantized(Tensor self) {
+ return self.is_quantized();
+}
+
#define DEFINE_CAST(T, name, _) \
template <> \
inline T* Tensor::data() const { \
AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_EXCEPT_COMPLEX_HALF_AND_QINT(DEFINE_ITEM)
#undef DEFINE_ITEM
-// TODO: after is_quantized() is implemented,
-// implement item() (returnning a float) for quantized Tensor
-
} //namespace at
virtual bool is_cuda() const = 0;
virtual bool is_hip() const = 0;
virtual bool is_sparse() const = 0;
+ virtual bool is_quantized() const = 0;
virtual bool is_distributed() const = 0;
bool is_variable() const noexcept { return is_variable_; }
bool is_undefined() const noexcept { return is_undefined_; }
bool is_sparse() const override {
return backend() == Backend::SparseCPU || backend() == Backend::SparseCUDA || backend() == Backend::SparseHIP;
}
+ bool is_quantized() const override {
+ return backend() == Backend::QuantizedCPU;
+ }
bool is_distributed() const override {
return false;
}
namespace at {
${Type}::${Type}()
- : ${DenseBackend}TypeDefault(${Backend}TensorId(), /*is_variable=*/false, /*is_undefined=*/false) {}
+ : ${DeviceType}TypeDefault(${Backend}TensorId(), /*is_variable=*/false, /*is_undefined=*/false) {}
ScalarType ${Type}::scalarType() const {
return ScalarType::${ScalarName};
namespace at {
-struct ${Type} final : public ${DenseBackend}TypeDefault {
+struct ${Type} final : public ${DeviceType}TypeDefault {
explicit ${Type}();
virtual ScalarType scalarType() const override;
virtual caffe2::TypeMeta typeMeta() const override;
Tensor qr = r.quantize_linear(scale, zero_point);
ASSERT_EQ(qr.q_scale().to<float>(), scale);
ASSERT_EQ(qr.q_zero_point().to<int32_t>(), zero_point);
-
- // TODO: Uncomment when quantizer is ready
- // auto* quantizer = static_cast<PerTensorAffineQuantizer*>(qr.quantizer());
- // ASSERT_EQ(quantizer->scale(), scale);
- // ASSERT_EQ(quantizer->zero_point(), zero_point);
+ ASSERT_TRUE(qr.is_quantized());
// Check for correct quantization
auto r_data = r.data<float>();
ASSERT_EQ(r_data[i], rqr_data[i]);
}
}
+
+TEST(TestQTensor, Item) {
+ Tensor r = at::ones({1});
+ const float scale = 1;
+ const int32_t zero_point = 2;
+ Tensor qr = r.quantize_linear(scale, zero_point);
+ ASSERT_EQ(r.item().to<float>(), qr.item().to<float>());
+}
* or "SparseCUDA"; backend in torch.backends is something like "MKL" or
* "CUDNN".
*/
-enum class Backend { CPU, CUDA, HIP, SparseCPU, SparseCUDA, SparseHIP, MSNPU, XLA, Undefined, MkldnnCPU, NumOptions };
+enum class Backend { CPU, CUDA, HIP, SparseCPU, SparseCUDA, SparseHIP, MSNPU, XLA, QuantizedCPU, Undefined, MkldnnCPU, NumOptions };
static inline Backend toSparse(Backend b) {
switch (b) {
return Backend::CUDA;
case Backend::SparseHIP:
return Backend::HIP;
+ case Backend::QuantizedCPU:
+ return Backend::QuantizedCPU;
default:
throw std::runtime_error("Unknown backend");
}
return Backend::SparseHIP;
} else if (t == MkldnnCPUTensorId()) {
return Backend::MkldnnCPU;
+ } else if (t == QuantizedCPUTensorId()) {
+ return Backend::QuantizedCPU;
} else if (t == UndefinedTensorId()) {
return Backend::Undefined;
} else {
return SparseHIPTensorId();
case Backend::MkldnnCPU:
return MkldnnCPUTensorId();
+ case Backend::QuantizedCPU:
+ return QuantizedCPUTensorId();
case Backend::Undefined:
return UndefinedTensorId();
default:
case Backend::SparseHIP:
return DeviceType::HIP;
case Backend::MkldnnCPU:
+ case Backend::QuantizedCPU:
return DeviceType::CPU;
case Backend::Undefined:
AT_ERROR("Undefined backend is not a valid device type");
return Backend::CPU;
case Backend::MkldnnCPU:
return Backend::MkldnnCPU;
+ case Backend::QuantizedCPU:
+ return Backend::QuantizedCPU;
case Backend::Undefined:
return Backend::Undefined;
default:
return "SparseHIP";
case Backend::MkldnnCPU:
return "MkldnnCPU";
+ case Backend::QuantizedCPU:
+ return "QuantizedCPU";
default:
return "UNKNOWN_BACKEND";
}
return tid == SparseCPUTensorId() || tid == SparseCUDATensorId() || tid == SparseHIPTensorId();
}
+ bool is_quantized() const {
+ // NB: This method is not virtual and avoid dispatches for performance reasons.
+ auto tid = type_id();
+ // NB: At the moment, variables have the same TensorTypeId as their
+ // corresponding tensor, but if this ever changes, we need to modify this.
+ return tid == QuantizedCPUTensorId();
+ }
+
bool is_cuda() const {
// NB: This method is not virtual and avoid dispatches for performance reasons.
auto tid = type_id();
C10_DEFINE_TENSOR_TYPE(MSNPUTensorId);
C10_DEFINE_TENSOR_TYPE(XLATensorId);
C10_DEFINE_TENSOR_TYPE(MkldnnCPUTensorId);
+C10_DEFINE_TENSOR_TYPE(QuantizedCPUTensorId);
} // namespace c10
C10_DECLARE_TENSOR_TYPE(MSNPUTensorId); // PyTorch only
C10_DECLARE_TENSOR_TYPE(XLATensorId); // PyTorch only
C10_DECLARE_TENSOR_TYPE(MkldnnCPUTensorId);
+C10_DECLARE_TENSOR_TYPE(QuantizedCPUTensorId); // PyTorch only
} // namespace c10
qr = r.quantize_linear(scale, zero_point)
self.assertEqual(qr.q_scale(), scale)
self.assertEqual(qr.q_zero_point(), zero_point)
+ self.assertTrue(qr.is_quantized)
+ self.assertFalse(r.is_quantized)
rqr = qr.dequantize()
for i in range(num_elements):
self.assertEqual(r[i], rqr[i])
+ # Testing item
+ r = torch.ones(1, dtype=torch.float)
+ qr = r.quantize_linear(scale, zero_point)
+ self.assertEqual(qr.item(), 1)
@unittest.skipIf(torch.cuda.device_count() < 2, 'fewer than 2 GPUs detected')
def test_device_guard(self):
END_HANDLE_TH_ERRORS
}
+PyObject *THPVariable_is_quantized(THPVariable *self)
+{
+ HANDLE_TH_ERRORS
+ auto& self_ = self->cdata;
+ return torch::autograd::utils::wrap(self_.is_quantized());
+ END_HANDLE_TH_ERRORS
+}
+
static PyObject *THPVariable_dtype(THPVariable *self)
{
HANDLE_TH_ERRORS
{"shape", (getter)THPVariable_get_shape, nullptr, nullptr, nullptr},
{"is_cuda", (getter)THPVariable_is_cuda, nullptr, nullptr, nullptr},
{"is_sparse", (getter)THPVariable_is_sparse, nullptr, nullptr, nullptr},
+ {"is_quantized", (getter)THPVariable_is_quantized, nullptr, nullptr, nullptr},
{"dtype", (getter)THPVariable_dtype, nullptr, nullptr, nullptr},
{"layout", (getter)THPVariable_layout, nullptr, nullptr, nullptr},
{"device", (getter)THPVariable_device, nullptr, nullptr, nullptr},
projects / platform / upstream / pytorch.git / commitdiff