Fix uninitialized read in postprocessing

This patch fixed WebRTC Issue 3020: "Uninit error at
vp8_mbpost_proc_down_xmm". The first 8 values in d were not initialized,
but was accessed. This patch fixed c code as well as mmx and sse2 code.

Change-Id: Iaa5b41a4ed3bea971b15fb826ce34b7ab4e36fb1

diff --git a/vp8/common/postproc.c b/vp8/common/postproc.c
index e3bee32..7d0fbf6 100644 (file)
--- a/vp8/common/postproc.c
+++ b/vp8/common/postproc.c
@@ -303,8 +303,8 @@ void vp8_mbpost_proc_down_c(unsigned char *dst, int pitch, int rows, int cols, i
             {
                 d[r&15] = (rv2[r&127] + sum + s[0]) >> 4;
             }
-
-            s[-8*pitch] = d[(r-8)&15];
+            if (r >= 8)
+              s[-8*pitch] = d[(r-8)&15];
             s += pitch;
         }
     }

diff --git a/vp8/common/x86/postproc_mmx.asm b/vp8/common/x86/postproc_mmx.asm
index 5cf110b..8be3431 100644 (file)
--- a/vp8/common/x86/postproc_mmx.asm
+++ b/vp8/common/x86/postproc_mmx.asm
@@ -204,13 +204,16 @@ sym(vp8_mbpost_proc_down_mmx):
             and         rcx,        15
             movd        DWORD PTR   [rsp+rcx*4], mm1 ;d[rcx*4]
 
+            cmp         edx,        8
+            jl          .skip_assignment
+
             mov         rcx,        rdx
             sub         rcx,        8
-
             and         rcx,        15
             movd        mm1,        DWORD PTR [rsp+rcx*4] ;d[rcx*4]
-
             movd        [rsi],      mm1
+
+.skip_assignment
             lea         rsi,        [rsi+rax]
 
             lea         rdi,        [rdi+rax]

diff --git a/vp8/common/x86/postproc_sse2.asm b/vp8/common/x86/postproc_sse2.asm
index 00f84a3..f53daa7 100644 (file)
--- a/vp8/common/x86/postproc_sse2.asm
+++ b/vp8/common/x86/postproc_sse2.asm
@@ -425,13 +425,16 @@ sym(vp8_mbpost_proc_down_xmm):
             and         rcx,        15
             movq        QWORD PTR   [rsp + rcx*8], xmm1 ;d[rcx*8]
 
+            cmp         edx,        8
+            jl          .skip_assignment
+
             mov         rcx,        rdx
             sub         rcx,        8
-
             and         rcx,        15
             movq        mm0,        [rsp + rcx*8] ;d[rcx*8]
-
             movq        [rsi],      mm0
+
+.skip_assignment
             lea         rsi,        [rsi+rax]
 
             lea         rdi,        [rdi+rax]