From: Yaowu Xu <yaowu@google.com>
Date: Wed, 18 May 2016 17:05:38 +0000 (-0700)
Subject: Clarify integer value ranges
X-Git-Tag: v1.6.0~120^2
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a564b18d7fe6d348068619d2d8099cc6bf438a83;p=platform%2Fupstream%2Flibvpx.git

Clarify integer value ranges

This commit clarifies integer value range for vairables used in
several variance functions, also change to use proper type
conversion to reflect the value ranges.

Change-Id: Ic3234b83a912ce1ad12d1b254f3378763e15cc5c
---

diff --git a/vpx_dsp/x86/halfpix_variance_sse2.c b/vpx_dsp/x86/halfpix_variance_sse2.c
index 5782155..4a8fb6d 100644
--- a/vpx_dsp/x86/halfpix_variance_sse2.c
+++ b/vpx_dsp/x86/halfpix_variance_sse2.c
@@ -8,6 +8,8 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <assert.h>
+
 #include "./vpx_config.h"
 #include "./vpx_dsp_rtcd.h"
 #include "vpx/vpx_integer.h"
@@ -40,7 +42,9 @@ uint32_t vpx_variance_halfpixvar16x16_h_sse2(const unsigned char *src,
                                     &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
+  assert(xsum0 <= 255 * 16 * 16);
+  assert(xsum0 >= -255 * 16 * 16);
+  return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
 }
 
 uint32_t vpx_variance_halfpixvar16x16_v_sse2(const unsigned char *src,
@@ -54,7 +58,9 @@ uint32_t vpx_variance_halfpixvar16x16_v_sse2(const unsigned char *src,
                                    &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
+  assert(xsum0 <= 255 * 16 * 16);
+  assert(xsum0 >= -255 * 16 * 16);
+  return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
 }
 
 
@@ -70,5 +76,7 @@ uint32_t vpx_variance_halfpixvar16x16_hv_sse2(const unsigned char *src,
                                          &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
+  assert(xsum0 <= 255 * 16 * 16);
+  assert(xsum0 >= -255 * 16 * 16);
+  return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
 }