fix asymmetric CQMs with pre-transposed DCT.

Originally committed as revision 4799 to svn://svn.ffmpeg.org/ffmpeg/trunk

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 8d05229f556b9702513596af342d0ac30c6e153d..42e7a5bc8c085a0c99816f9ab8f5cb82d377c1da 100644 (file)
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -2973,6 +2973,7 @@ static void init_dequant8_coeff_table(H264Context *h){
 
 static void init_dequant4_coeff_table(H264Context *h){
     int i,j,q,x;
+    const int transpose = (h->s.dsp.h264_idct_add != ff_h264_idct_add_c); //FIXME ugly
     for(i=0; i<6; i++ ){
         h->dequant4_coeff[i] = h->dequant4_buffer[i];
         for(j=0; j<i; j++){
@@ -2988,7 +2989,8 @@ static void init_dequant4_coeff_table(H264Context *h){
             int shift = div6[q] + 2;
             int idx = rem6[q];
             for(x=0; x<16; x++)
-                h->dequant4_coeff[i][q][x] = ((uint32_t)dequant4_coeff_init[idx][(x&1) + ((x>>2)&1)] *
+                h->dequant4_coeff[i][q][transpose ? (x>>2)|((x<<2)&0xF) : x] =
+                    ((uint32_t)dequant4_coeff_init[idx][(x&1) + ((x>>2)&1)] *
                     h->pps.scaling_matrix4[i][x]) << shift;
         }
     }