mpeg2 QP clamping fix

Originally committed as revision 2663 to svn://svn.ffmpeg.org/ffmpeg/trunk

diff --git a/libavcodec/libpostproc/postprocess.c b/libavcodec/libpostproc/postprocess.c
index 6acf58d..093d94a 100644 (file)
--- a/libavcodec/libpostproc/postprocess.c
+++ b/libavcodec/libpostproc/postprocess.c
@@ -943,10 +943,10 @@ for(y=0; y<mbHeight; y++){
                int i;
                const int count= mbHeight * QPStride;
                for(i=0; i<(count>>2); i++){
-                       ((uint32_t*)c->nonBQPTable)[i] = ((uint32_t*)QP_store)[i] & 0x1F1F1F1F;
+                       ((uint32_t*)c->nonBQPTable)[i] = ((uint32_t*)QP_store)[i] & 0x3F3F3F3F;
                }
                for(i<<=2; i<count; i++){
-                       c->nonBQPTable[i] = QP_store[i] & 0x1F;
+                       c->nonBQPTable[i] = QP_store[i] & 0x3F;
                }
        }
 

diff --git a/libavcodec/libpostproc/postprocess_internal.h b/libavcodec/libpostproc/postprocess_internal.h
index 13b3e38..db50fa3 100644 (file)
--- a/libavcodec/libpostproc/postprocess_internal.h
+++ b/libavcodec/libpostproc/postprocess_internal.h
@@ -134,8 +134,8 @@ typedef struct PPContext{
        uint64_t __attribute__((aligned(8))) pQPb;
        uint64_t __attribute__((aligned(8))) pQPb2;
 
-       uint64_t __attribute__((aligned(8))) mmxDcOffset[32];
-       uint64_t __attribute__((aligned(8))) mmxDcThreshold[32];
+       uint64_t __attribute__((aligned(8))) mmxDcOffset[64];
+       uint64_t __attribute__((aligned(8))) mmxDcThreshold[64];
 
        QP_STORE_T *stdQPTable;       ///< used to fix MPEG2 style qscale
        QP_STORE_T *nonBQPTable;

diff --git a/libavcodec/libpostproc/postprocess_template.c b/libavcodec/libpostproc/postprocess_template.c
index e3dbfbf..7ebc08b 100644 (file)
--- a/libavcodec/libpostproc/postprocess_template.c
+++ b/libavcodec/libpostproc/postprocess_template.c
@@ -2792,7 +2792,7 @@ static void RENAME(postProcess)(uint8_t src[], int srcStride, uint8_t dst[], int
        const int mbWidth= isColor ? (width+7)>>3 : (width+15)>>4;
 
 #ifdef HAVE_MMX
-       for(i=0; i<32; i++){
+       for(i=0; i<57; i++){
                int offset= ((i*c.ppMode.baseDcDiff)>>8) + 1;
                int threshold= offset*2 + 1;
                c.mmxDcOffset[i]= 0x7F - offset;