From 75cfadbd1fa2dbefa5452fdc33fb4441ad53a1a4 Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Mon, 16 Dec 2013 14:38:45 -0800 Subject: [PATCH] Fix rd_filter_cache update in sub8x8 RD loop Properly handle the rd_filter_cache update, when early termination or skip prediction filter type check is triggered. Change-Id: Ie7b9a75fed3358f45ffd15817f2b36670c14eb2d --- vp9/encoder/vp9_rdopt.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 5c754c2..6578462 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -3876,6 +3876,7 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, int this_skip2 = 0; int64_t total_sse = INT_MAX; int early_term = 0; + int64_t mask_rd = 0; for (i = 0; i < TX_MODES; ++i) tx_cache[i] = INT64_MAX; @@ -4069,21 +4070,20 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, cpi->rd_thresh_sub8x8[segment_id][bsize][THR_GOLD] : this_rd_thresh; xd->mi_8x8[0]->mbmi.tx_size = TX_4X4; - cpi->rd_filter_cache[SWITCHABLE_FILTERS] = INT64_MAX; + for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i) + cpi->rd_filter_cache[i] = INT64_MAX; + if (cm->mcomp_filter_type != BILINEAR) { tmp_best_filter = EIGHTTAP; if (x->source_variance < cpi->sf.disable_filter_search_var_thresh) { tmp_best_filter = EIGHTTAP; - vp9_zero(cpi->rd_filter_cache); } else if (cpi->sf.adaptive_pred_filter_type == 1 && ctx->pred_filter_type < SWITCHABLE) { tmp_best_filter = ctx->pred_filter_type; - vp9_zero(cpi->rd_filter_cache); } else if (cpi->sf.adaptive_pred_filter_type == 2) { tmp_best_filter = ctx->pred_filter_type < SWITCHABLE ? ctx->pred_filter_type : 0; - vp9_zero(cpi->rd_filter_cache); } else { for (switchable_filter_index = 0; switchable_filter_index < SWITCHABLE_FILTERS; @@ -4105,7 +4105,6 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, if (tmp_rd == INT64_MAX) continue; - cpi->rd_filter_cache[switchable_filter_index] = tmp_rd; rs = get_switchable_rate(x); rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0); cpi->rd_filter_cache[SWITCHABLE_FILTERS] = @@ -4114,6 +4113,9 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, if (cm->mcomp_filter_type == SWITCHABLE) tmp_rd += rs_rd; + cpi->rd_filter_cache[switchable_filter_index] = tmp_rd; + mask_rd = MAX(tmp_rd, mask_rd); + newbest = (tmp_rd < tmp_best_rd); if (newbest) { tmp_best_filter = mbmi->interp_filter; @@ -4361,16 +4363,17 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, cm->mcomp_filter_type != BILINEAR) { int64_t ref = cpi->rd_filter_cache[cm->mcomp_filter_type == SWITCHABLE ? SWITCHABLE_FILTERS : cm->mcomp_filter_type]; + int64_t adj_rd; + for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) { - int64_t adj_rd; - // In cases of poor prediction, filter_cache[] can contain really big - // values, which actually are bigger than this_rd itself. This can - // cause negative best_filter_rd[] values, which is obviously silly. - // Therefore, if filter_cache < ref, we do an adjusted calculation. - if (cpi->rd_filter_cache[i] >= ref) - adj_rd = this_rd + cpi->rd_filter_cache[i] - ref; - else // FIXME(rbultje) do this for comppred also - adj_rd = this_rd - (ref - cpi->rd_filter_cache[i]) * this_rd / ref; + if (ref == INT64_MAX) + adj_rd = 0; + else if (cpi->rd_filter_cache[i] == INT64_MAX) + adj_rd = mask_rd - ref + 10; + else + adj_rd = cpi->rd_filter_cache[i] - ref; + + adj_rd += this_rd; best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd); } } -- 2.7.4