Fix the transform type selection in 4x4 partition

This commit allows proper transform type (DCT/ADST) selection in
the settings of partition 4x4 level.

Change-Id: Iec6f922a46480d777e7ca9142a99e8c131f0077b

diff --git a/vp9/common/vp9_blockd.h b/vp9/common/vp9_blockd.h
index 07607d8..ad38730 100644 (file)
--- a/vp9/common/vp9_blockd.h
+++ b/vp9/common/vp9_blockd.h
@@ -625,13 +625,11 @@ static TX_TYPE get_tx_type_4x4(const MACROBLOCKD *xd, int ib) {
   // is smaller than the prediction size
   TX_TYPE tx_type = DCT_DCT;
   const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
-  const int wb = b_width_log2(sb_type), hb = b_height_log2(sb_type);
+  const int wb = b_width_log2(sb_type);
 #if !USE_ADST_FOR_SB
   if (sb_type > BLOCK_SIZE_MB16X16)
     return tx_type;
 #endif
-  if (ib >= (1 << (wb + hb)))  // no chroma adst
-    return tx_type;
   if (xd->lossless)
     return DCT_DCT;
   if (xd->mode_info_context->mbmi.mode == I4X4_PRED &&

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 271a63f..5097664 100644 (file)
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2331,8 +2331,7 @@ void vp9_rd_pick_intra_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     *returnrate = rate_y + rate_uv - rate_y_tokenonly - rate_uv_tokenonly +
                   vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 1);
     *returndist = dist_y + (dist_uv >> 2);
-    memset(ctx->txfm_rd_diff, 0,
-           sizeof(x->sb32_context[xd->sb_index].txfm_rd_diff));
+    memset(ctx->txfm_rd_diff, 0, sizeof(ctx->txfm_rd_diff));
     xd->mode_info_context->mbmi.mode = mode;
     xd->mode_info_context->mbmi.txfm_size = txfm_size;
 #if CONFIG_AB4X4