From: Ronald S. Bultje <rsbultje@gmail.com>
Date: Thu, 15 Sep 2011 14:30:34 +0000 (-0700)
Subject: swscale: fix byte overreads in SSE-optimized hscale().
X-Git-Tag: v0.8b1~1511
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=ea540401d6082474df8364169e2041e29e4dc407;p=platform%2Fupstream%2Flibav.git

swscale: fix byte overreads in SSE-optimized hscale().

SSE-optimized hScale() scales up to 4 pixels at once, so we need to
allocate up to 3 padding pixels to prevent overreads. This fixes
valgrind errors in various swscale-tests on fate.
---

diff --git a/libswscale/utils.c b/libswscale/utils.c
index 2b52199..4096574 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -184,8 +184,8 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
 
     emms_c(); //FIXME this should not be required but it IS (even for non-MMX versions)
 
-    // NOTE: the +1 is for the MMX scaler which reads over the end
-    FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+1)*sizeof(int16_t), fail);
+    // NOTE: the +3 is for the MMX(+1)/SSE(+3) scaler which reads over the end
+    FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+3)*sizeof(int16_t), fail);
 
     if (FFABS(xInc - 0x10000) <10) { // unscaled
         int i;
@@ -471,7 +471,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
 
     // Note the +1 is for the MMX scaler which reads over the end
     /* align at 16 for AltiVec (needed by hScale_altivec_real) */
-    FF_ALLOCZ_OR_GOTO(NULL, *outFilter, *outFilterSize*(dstW+1)*sizeof(int16_t), fail);
+    FF_ALLOCZ_OR_GOTO(NULL, *outFilter, *outFilterSize*(dstW+3)*sizeof(int16_t), fail);
 
     /* normalize & store in outFilter */
     for (i=0; i<dstW; i++) {
@@ -491,10 +491,14 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
         }
     }
 
-    (*filterPos)[dstW]= (*filterPos)[dstW-1]; // the MMX scaler will read over the end
+    (*filterPos)[dstW+0] =
+    (*filterPos)[dstW+1] =
+    (*filterPos)[dstW+2] = (*filterPos)[dstW-1]; // the MMX/SSE scaler will read over the end
     for (i=0; i<*outFilterSize; i++) {
-        int j= dstW*(*outFilterSize);
-        (*outFilter)[j + i]= (*outFilter)[j + i - (*outFilterSize)];
+        int k= (dstW - 1) * (*outFilterSize) + i;
+        (*outFilter)[k + 1 * (*outFilterSize)] =
+        (*outFilter)[k + 2 * (*outFilterSize)] =
+        (*outFilter)[k + 3 * (*outFilterSize)] = (*outFilter)[k];
     }
 
     ret=0;