fix some potential arithmetic overflows in pred_direct_motion() and

ff_h264_weight_WxH_mmx2().

Originally committed as revision 4795 to svn://svn.ffmpeg.org/ffmpeg/trunk

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 71b3909a6f2e0edf6e9c68b70bb62b4b3c8c0e32..746cf01847f0e5961c9527b1783114f85a12d7f0 100644 (file)
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1286,8 +1286,8 @@ static inline void pred_direct_motion(H264Context * const h, int *mb_type){
         }
 
         if(IS_16X16(*mb_type)){
-            fill_rectangle(&h->ref_cache[0][scan8[0]], 4, 4, 8, ref[0], 1);
-            fill_rectangle(&h->ref_cache[1][scan8[0]], 4, 4, 8, ref[1], 1);
+            fill_rectangle(&h->ref_cache[0][scan8[0]], 4, 4, 8, (uint8_t)ref[0], 1);
+            fill_rectangle(&h->ref_cache[1][scan8[0]], 4, 4, 8, (uint8_t)ref[1], 1);
             if(!IS_INTRA(mb_type_col)
                && (   (l1ref0[0] == 0 && ABS(l1mv0[0][0]) <= 1 && ABS(l1mv0[0][1]) <= 1)
                    || (l1ref0[0]  < 0 && l1ref1[0] == 0 && ABS(l1mv1[0][0]) <= 1 && ABS(l1mv1[0][1]) <= 1
@@ -1315,8 +1315,8 @@ static inline void pred_direct_motion(H264Context * const h, int *mb_type){
 
                 fill_rectangle(&h->mv_cache[0][scan8[i8*4]], 2, 2, 8, pack16to32(mv[0][0],mv[0][1]), 4);
                 fill_rectangle(&h->mv_cache[1][scan8[i8*4]], 2, 2, 8, pack16to32(mv[1][0],mv[1][1]), 4);
-                fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, ref[0], 1);
-                fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, ref[1], 1);
+                fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[0], 1);
+                fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[1], 1);
 
                 /* col_zero_flag */
                 if(!IS_INTRA(mb_type_col) && (   l1ref0[x8 + y8*h->b8_stride] == 0

diff --git a/libavcodec/i386/h264dsp_mmx.c b/libavcodec/i386/h264dsp_mmx.c
index 5356b9c23fc44d9a54fdc0a781ac610d5b58d6bd..d51b21ad3c6501dd7c4af9bbd48637ee01614e2d 100644 (file)
--- a/libavcodec/i386/h264dsp_mmx.c
+++ b/libavcodec/i386/h264dsp_mmx.c
@@ -518,9 +518,9 @@ static void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp,
         "psraw $2, %%mm0            \n\t"/*(a-b)/4 */\
         "psubw %%mm1, %%mm0         \n\t"/*(a-b)/4-b */\
         "paddsw %%mm2, %%mm0        \n\t"\
-        "psraw $2, %%mm0            \n\t"/*((a-b)/4-b)/4 */\
+        "psraw $2, %%mm0            \n\t"/*((a-b)/4-b+c)/4 */\
         "paddw %%mm6, %%mm2         \n\t"\
-        "paddw %%mm2, %%mm0         \n\t"\
+        "paddw %%mm2, %%mm0         \n\t"/*(a-5*b+20*c)/16 +32 */\
         "psraw $6, %%mm0            \n\t"\
         "packuswb %%mm0, %%mm0      \n\t"\
         OP(%%mm0, (%1),%%mm7, d)\
@@ -955,8 +955,8 @@ static inline void ff_h264_weight_WxH_mmx2(uint8_t *dst, int stride, int log2_de
                 "punpcklbw %%mm7, %%mm1 \n\t"
                 "pmullw    %%mm4, %%mm0 \n\t"
                 "pmullw    %%mm4, %%mm1 \n\t"
-                "paddw     %%mm5, %%mm0 \n\t"
-                "paddw     %%mm5, %%mm1 \n\t"
+                "paddsw    %%mm5, %%mm0 \n\t"
+                "paddsw    %%mm5, %%mm1 \n\t"
                 "psraw     %%mm6, %%mm0 \n\t"
                 "psraw     %%mm6, %%mm1 \n\t"
                 "packuswb  %%mm7, %%mm0 \n\t"
@@ -995,8 +995,8 @@ static inline void ff_h264_biweight_WxH_mmx2(uint8_t *dst, uint8_t *src, int str
                 "punpcklbw %%mm7, %%mm1 \n\t"
                 "pmullw    %%mm3, %%mm0 \n\t"
                 "pmullw    %%mm4, %%mm1 \n\t"
-                "paddw     %%mm5, %%mm0 \n\t"
-                "paddw     %%mm1, %%mm0 \n\t"
+                "paddsw    %%mm1, %%mm0 \n\t"
+                "paddsw    %%mm5, %%mm0 \n\t"
                 "psraw     %%mm6, %%mm0 \n\t"
                 "packuswb  %%mm0, %%mm0 \n\t"
                 "movd      %%mm0, %0    \n\t"