From 105746c8fec14874babf97aee0d3660130e2ae08 Mon Sep 17 00:00:00 2001 From: Deb Mukherjee Date: Fri, 5 Oct 2012 05:08:20 -0700 Subject: [PATCH] Mismatch bug-fix when transform expts are off A variety of issues related to tangling of various hybrid transform experiments. Change-Id: I84dac65ff513f8b024407760df55fffa95e198de --- vp8/common/default_coef_probs.h | 2 -- vp8/decoder/decodframe.c | 63 ++++++++++++++++------------------------- vp8/decoder/detokenize.c | 16 ++--------- vp8/encoder/tokenize.c | 44 +++++++++++++++++----------- 4 files changed, 55 insertions(+), 70 deletions(-) diff --git a/vp8/common/default_coef_probs.h b/vp8/common/default_coef_probs.h index 717cef7..848104b 100644 --- a/vp8/common/default_coef_probs.h +++ b/vp8/common/default_coef_probs.h @@ -679,7 +679,6 @@ default_coef_probs_8x8[BLOCK_TYPES_8X8] { 6, 117, 180, 254, 199, 216, 255, 251, 128, 128, 128} } }, -#if CONFIG_HYBRIDTRANSFORM8X8 { /* block Type 3 */ { /* Coeff Band 0 */ { 192, 18, 155, 172, 145, 164, 192, 135, 246, 223, 255}, @@ -730,7 +729,6 @@ default_coef_probs_8x8[BLOCK_TYPES_8X8] { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128} } } -#endif }; #if CONFIG_HYBRIDTRANSFORM8X8 diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c index b4d4006..34b43ce 100644 --- a/vp8/decoder/decodframe.c +++ b/vp8/decoder/decodframe.c @@ -407,7 +407,6 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, int i8x8mode; BLOCKD *b; -#if CONFIG_HYBRIDTRANSFORM8X8 int idx = (ib & 0x02) ? (ib + 2) : ib; short *q = xd->block[idx].qcoeff; @@ -415,7 +414,6 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, unsigned char *pre = xd->block[ib].predictor; unsigned char *dst = *(xd->block[ib].base_dst) + xd->block[ib].dst; int stride = xd->dst.y_stride; -#endif b = &xd->block[ib]; i8x8mode = b->bmi.as_mode.first; @@ -427,19 +425,8 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, q, dq, pre, dst, 16, stride); q += 64; #else - for (j = 0; j < 4; j++) { - b = &xd->block[ib + iblock[j]]; - if (xd->eobs[ib + iblock[j]] > 1) { - DEQUANT_INVOKE(&pbi->dequant, idct_add) - (b->qcoeff, b->dequant, b->predictor, - *(b->base_dst) + b->dst, 16, b->dst_stride); - } else { - IDCT_INVOKE(RTCD_VTABLE(idct), idct1_scalar_add) - (b->qcoeff[0] * b->dequant[0], b->predictor, - *(b->base_dst) + b->dst, 16, b->dst_stride); - ((int *)b->qcoeff)[0] = 0; - } - } + vp8_dequant_idct_add_8x8_c(q, dq, pre, dst, 16, stride); + q += 64; #endif b = &xd->block[16 + i]; @@ -484,16 +471,14 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, #else if (xd->eobs[i] > 1) { - DEQUANT_INVOKE(&pbi->dequant, idct_add) - (b->qcoeff, b->dequant, b->predictor, - *(b->base_dst) + b->dst, 16, b->dst_stride); - } - else - { - IDCT_INVOKE(RTCD_VTABLE(idct), idct1_scalar_add) - (b->qcoeff[0] * b->dequant[0], b->predictor, - *(b->base_dst) + b->dst, 16, b->dst_stride); - ((int *)b->qcoeff)[0] = 0; + DEQUANT_INVOKE(&pbi->dequant, idct_add) + (b->qcoeff, b->dequant, b->predictor, + *(b->base_dst) + b->dst, 16, b->dst_stride); + } else { + IDCT_INVOKE(RTCD_VTABLE(idct), idct1_scalar_add) + (b->qcoeff[0] * b->dequant[0], b->predictor, + *(b->base_dst) + b->dst, 16, b->dst_stride); + ((int *)b->qcoeff)[0] = 0; } #endif } @@ -613,23 +598,23 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, } #if CONFIG_SUPERBLOCKS - if (!xd->mode_info_context->mbmi.encoded_as_sb) { + if (!xd->mode_info_context->mbmi.encoded_as_sb) { #endif - if ((tx_type == TX_8X8 && - xd->mode_info_context->mbmi.mode != I8X8_PRED) + if ((tx_type == TX_8X8 && + xd->mode_info_context->mbmi.mode != I8X8_PRED) #if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16 - || tx_type == TX_16X16 + || tx_type == TX_16X16 #endif - ) - DEQUANT_INVOKE(&pbi->dequant, idct_add_uv_block_8x8) // - (xd->qcoeff + 16 * 16, xd->block[16].dequant, - xd->predictor + 16 * 16, xd->dst.u_buffer, xd->dst.v_buffer, - xd->dst.uv_stride, xd->eobs + 16, xd); // - else if (xd->mode_info_context->mbmi.mode != I8X8_PRED) - DEQUANT_INVOKE(&pbi->dequant, idct_add_uv_block) - (xd->qcoeff + 16 * 16, xd->block[16].dequant, - xd->predictor + 16 * 16, xd->dst.u_buffer, xd->dst.v_buffer, - xd->dst.uv_stride, xd->eobs + 16); + ) + DEQUANT_INVOKE(&pbi->dequant, idct_add_uv_block_8x8) // + (xd->qcoeff + 16 * 16, xd->block[16].dequant, + xd->predictor + 16 * 16, xd->dst.u_buffer, xd->dst.v_buffer, + xd->dst.uv_stride, xd->eobs + 16, xd); // + else if (xd->mode_info_context->mbmi.mode != I8X8_PRED) + DEQUANT_INVOKE(&pbi->dequant, idct_add_uv_block) + (xd->qcoeff + 16 * 16, xd->block[16].dequant, + xd->predictor + 16 * 16, xd->dst.u_buffer, xd->dst.v_buffer, + xd->dst.uv_stride, xd->eobs + 16); #if CONFIG_SUPERBLOCKS } #endif diff --git a/vp8/decoder/detokenize.c b/vp8/decoder/detokenize.c index b1d5a52..cfb0768 100644 --- a/vp8/decoder/detokenize.c +++ b/vp8/decoder/detokenize.c @@ -548,15 +548,10 @@ int vp8_decode_mb_tokens_8x8(VP8D_COMP *pbi, MACROBLOCKD *xd) { TX_TYPE tx_type = DCT_DCT; #endif -#if CONFIG_HYBRIDTRANSFORM8X8 int bufthred = (xd->mode_info_context->mbmi.mode == I8X8_PRED) ? 16 : 24; if (xd->mode_info_context->mbmi.mode != B_PRED && xd->mode_info_context->mbmi.mode != SPLITMV && xd->mode_info_context->mbmi.mode != I8X8_PRED) { -#else - if (xd->mode_info_context->mbmi.mode != B_PRED && - xd->mode_info_context->mbmi.mode != SPLITMV) { -#endif ENTROPY_CONTEXT *const a = A + vp8_block2above_8x8[24]; ENTROPY_CONTEXT *const l = L + vp8_block2left_8x8[24]; const int *const scan = vp8_default_zig_zag1d; @@ -585,11 +580,7 @@ int vp8_decode_mb_tokens_8x8(VP8D_COMP *pbi, MACROBLOCKD *xd) { else seg_eob = 64; -#if CONFIG_HYBRIDTRANSFORM8X8 for (i = 0; i < bufthred ; i += 4) { -#else - for (i = 0; i < 24; i += 4) { -#endif ENTROPY_CONTEXT *const a = A + vp8_block2above_8x8[i]; ENTROPY_CONTEXT *const l = L + vp8_block2left_8x8[i]; const int *const scan = vp8_default_zig_zag1d_8x8; @@ -606,7 +597,6 @@ int vp8_decode_mb_tokens_8x8(VP8D_COMP *pbi, MACROBLOCKD *xd) { } #endif - //printf("8: %d\n", tx_type); c = vp8_decode_coefs(pbi, xd, a, l, type, #if CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM16X16 tx_type, @@ -621,10 +611,11 @@ int vp8_decode_mb_tokens_8x8(VP8D_COMP *pbi, MACROBLOCKD *xd) { qcoeff_ptr += 64; } -#if CONFIG_HYBRIDTRANSFORM8X8 - if (xd->mode_info_context->mbmi.mode == I8X8_PRED) { + if (bufthred == 16) { type = PLANE_TYPE_UV; +#if CONFIG_HYBRIDTRANSFORM8X8 || CONFIG_HYBRIDTRANSFORM || CONFIG_HYBRIDTRANSFORM16X16 tx_type = DCT_DCT; +#endif seg_eob = 16; // use 4x4 transform for U, V components in I8X8 prediction mode @@ -645,7 +636,6 @@ int vp8_decode_mb_tokens_8x8(VP8D_COMP *pbi, MACROBLOCKD *xd) { qcoeff_ptr += 16; } } -#endif return eobtotal; } diff --git a/vp8/encoder/tokenize.c b/vp8/encoder/tokenize.c index 91fafd0..4f4bd50 100644 --- a/vp8/encoder/tokenize.c +++ b/vp8/encoder/tokenize.c @@ -252,6 +252,7 @@ static void tokenize2nd_order_b_8x8 t->context_tree = cpi->common.fc.coef_probs_8x8 [type] [band] [pt]; t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts_8x8 [type] [band] [pt] [x]; @@ -302,6 +303,7 @@ static void tokenize2nd_order_b(MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [1] [band] [pt] [token]; @@ -316,6 +318,7 @@ static void tokenize2nd_order_b(MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [1] [band] [pt] [DCT_EOB_TOKEN]; @@ -346,7 +349,8 @@ static void tokenize1st_order_b_8x8 TOKENEXTRA *t = *tp; /* store tokens starting here */ const short *qcoeff_ptr = b->qcoeff; #if CONFIG_HYBRIDTRANSFORM8X8 - TX_TYPE tx_type = type == 3 ? get_tx_type(xd, b) : DCT_DCT; + TX_TYPE tx_type = xd->mode_info_context->mbmi.mode == I8X8_PRED ? + get_tx_type(xd, b) : DCT_DCT; #endif int seg_eob = 64; @@ -377,6 +381,7 @@ static void tokenize1st_order_b_8x8 t->context_tree = cpi->common.fc.coef_probs_8x8[type][band][pt]; t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { #if CONFIG_HYBRIDTRANSFORM8X8 @@ -403,6 +408,11 @@ static void tokenize1st_order_b_8x8 t->context_tree = cpi->common.fc.coef_probs_8x8 [type] [band] [pt]; t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + if (vp8_coef_encodings[t->Token].Len - t->skip_eob_node <= 0) { + printf("type %d, seg-eob %d, eob %d, pt %d, c %d band %d\n", type, seg_eob, b->eob, pt, c, band); + fflush(stdout); + } + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { #if CONFIG_HYBRIDTRANSFORM8X8 @@ -458,7 +468,6 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, b_mode = b->bmi.as_mode.first; tx_type = get_tx_type(xd, b); } - //if (!dry_run) printf("4: %d\n", tx_type); // assign scanning order for luma components coded in intra4x4 mode if( (xd->mode_info_context->mbmi.mode == B_PRED) && @@ -508,6 +517,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { if (tx_type != DCT_DCT) @@ -530,6 +540,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { if (tx_type != DCT_DCT) ++cpi->hybrid_coef_counts[type] [band] [pt] [DCT_EOB_TOKEN]; @@ -570,6 +581,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [token]; @@ -584,6 +596,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [DCT_EOB_TOKEN]; t++; @@ -597,7 +610,6 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, #endif -#if CONFIG_HYBRIDTRANSFORM8X8 static void tokenize1st_order_chroma ( MACROBLOCKD *xd, @@ -649,6 +661,7 @@ static void tokenize1st_order_chroma t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [token]; @@ -663,6 +676,7 @@ static void tokenize1st_order_chroma t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [DCT_EOB_TOKEN]; @@ -674,7 +688,6 @@ static void tokenize1st_order_chroma *a = *l = pt; } } -#endif static void tokenize1st_order_b ( @@ -708,6 +721,8 @@ static void tokenize1st_order_b c = type ? 0 : 1; + assert(b->eob <= 16); + for (; c < b->eob; c++) { rc = vp8_default_zig_zag1d[c]; band = vp8_coef_bands[c]; @@ -721,6 +736,7 @@ static void tokenize1st_order_b t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [type] [band] [pt] [token]; @@ -735,6 +751,7 @@ static void tokenize1st_order_b t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [type] [band] [pt] [DCT_EOB_TOKEN]; @@ -753,6 +770,7 @@ static void tokenize1st_order_b VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l); + assert(b->eob <= 16); for (c = 0; c < b->eob; c++) { rc = vp8_default_zig_zag1d[c]; band = vp8_coef_bands[c]; @@ -765,6 +783,7 @@ static void tokenize1st_order_b t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [token]; @@ -779,6 +798,7 @@ static void tokenize1st_order_b t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [DCT_EOB_TOKEN]; @@ -908,12 +928,10 @@ void vp8_tokenize_mb(VP8_COMP *cpi, break; #endif case TX_8X8: -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8_4x4uv(xd, 0); else -#endif - xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(xd, has_y2_block); + xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(xd, has_y2_block); break; default: @@ -931,11 +949,9 @@ void vp8_tokenize_mb(VP8_COMP *cpi, else #endif if (tx_size == TX_8X8) { -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) vp8_stuff_mb_8x8_4x4uv(cpi, xd, t, dry_run); else -#endif vp8_stuff_mb_8x8(cpi, xd, t, dry_run); } else vp8_stuff_mb(cpi, xd, t, dry_run); @@ -992,11 +1008,9 @@ void vp8_tokenize_mb(VP8_COMP *cpi, if (tx_size == TX_8X8) { ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)xd->above_context; ENTROPY_CONTEXT *L = (ENTROPY_CONTEXT *)xd->left_context; -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) { plane_type = PLANE_TYPE_Y_WITH_DC; } -#endif for (b = 0; b < 16; b += 4) { tokenize1st_order_b_8x8(xd, xd->block + b, @@ -1007,12 +1021,9 @@ void vp8_tokenize_mb(VP8_COMP *cpi, *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]); *(L + vp8_block2left_8x8[b] + 1) = *(L + vp8_block2left_8x8[b]); } -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) { tokenize1st_order_chroma(xd, t, PLANE_TYPE_UV, cpi, dry_run); - } else -#endif - { + } else { for (b = 16; b < 24; b += 4) { tokenize1st_order_b_8x8(xd, xd->block + b, t, 2, xd->frame_type, @@ -1343,7 +1354,8 @@ static __inline void stuff1st_order_b_8x8 int pt; /* near block/prev token context index */ TOKENEXTRA *t = *tp; /* store tokens starting here */ #if CONFIG_HYBRIDTRANSFORM8X8 - TX_TYPE tx_type = type == 3 ? get_tx_type(xd, b) : DCT_DCT; + TX_TYPE tx_type = xd->mode_info_context->mbmi.mode == I8X8_PRED ? + get_tx_type(xd, b) : DCT_DCT; #endif VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l); -- 2.7.4