Fix type conversion in dequant and add a test (#19226)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/19226

Type conversoin was wrong previously. Thanks zafartahirov for finding it!

Differential Revision: D14926610

fbshipit-source-id: 6824f9813137a3d171694d743fbb437a663b1f88

diff --git a/aten/src/ATen/quantized/Quantizer.cpp b/aten/src/ATen/quantized/Quantizer.cpp
index b4c6936..82ee8ed 100644 (file)
--- a/aten/src/ATen/quantized/Quantizer.cpp
+++ b/aten/src/ATen/quantized/Quantizer.cpp
@@ -95,7 +95,9 @@ RealTensor PerTensorAffineQuantizer::dequantize(QTensor tensor) {
   const auto* qvd = tensor.data<qint8>();
   float* rvd = rv.data<float>();
   for (auto i = 0; i < tensor.numel(); ++i) {
-    rvd[i] = (static_cast<uint32_t>(qvd[i].val_) - zero_point_) * scale_;
+    // We need to convert the qint8 value to float to ensure the subtraction
+    // subexpression returns a float
+    rvd[i] = (static_cast<float>(qvd[i].val_) - zero_point_) * scale_;
   }
   return rv;
 }

diff --git a/test/test_torch.py b/test/test_torch.py
index d7e1ddb..4c44cae 100644 (file)
--- a/test/test_torch.py
+++ b/test/test_torch.py
@@ -2671,6 +2671,17 @@ class _TestTorchMixin(object):
         qr = r.quantize_linear(scale, zero_point)
         self.assertEqual(qr.item(), 1)
 
+    def test_qtensor_quant_dequant(self):
+        r = np.random.rand(3, 2) * 2 - 4
+        r = torch.from_numpy(r).float()
+        scale = 2
+        zero_point = 2
+        qr = r.quantize_linear(scale, zero_point)
+        rqr = qr.dequantize()
+        print(r.numpy())
+        print(rqr.numpy())
+        self.assertTrue(np.allclose(r.numpy(), rqr.numpy(), atol=2/scale))
+
     @unittest.skipIf(torch.cuda.device_count() < 2, 'fewer than 2 GPUs detected')
     def test_device_guard(self):
         # verify that all operators with `device_guard: False` behave properly with multiple devices.