From cb9698951c8c57b6cdb185c73db1dec42f4574ec Mon Sep 17 00:00:00 2001 From: John Koleszar Date: Thu, 9 Dec 2010 15:59:22 -0500 Subject: [PATCH] fix uninitialized read in encode breakout Change I3430820 performed an uninitialized read when encode_breakout == 0, since the sum and sse wouldn't be set: if(x->encode_breakout) VARIANCE_INVOKE(..., get16x16var)(..., &sum, &sse); if (cpi->active_map_enabled && x->active_ptr[0] == 0) { ... } else if (sse < x->encode_breakout) Change-Id: I915eb76d1227b4b6d1137a0dedf2c143860098a2 --- vp8/encoder/rdopt.c | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c index bc5fd40..9b8b3b3 100644 --- a/vp8/encoder/rdopt.c +++ b/vp8/encoder/rdopt.c @@ -1642,8 +1642,6 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int //int intermodecost[MAX_MODES]; MB_PREDICTION_MODE uv_intra_mode; - int sse; - int sum; int uvintra_eob = 0; int tteob = 0; int force_no_skip = 0; @@ -2238,36 +2236,44 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int vp8_set_mbmode_and_mvs(x, this_mode, &mode_mv[this_mode]); vp8_build_inter_predictors_mby(&x->e_mbd); - if(x->encode_breakout) - VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var)(x->src.y_buffer, x->src.y_stride, x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum); if (cpi->active_map_enabled && x->active_ptr[0] == 0) { x->skip = 1; } - else if (sse < x->encode_breakout) + else if (x->encode_breakout) { - // Check u and v to make sure skip is ok - int sse2 = 0; + int sum, sse; - // add dc check - if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2)) + VARIANCE_INVOKE(&cpi->rtcd.variance, get16x16var) + (x->src.y_buffer, x->src.y_stride, + x->e_mbd.predictor, 16, (unsigned int *)(&sse), &sum); + + if (sse < x->encode_breakout) { - sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance)); + // Check u and v to make sure skip is ok + int sse2 = 0; - if (sse2 * 2 < x->encode_breakout) + // add dc check + if (abs(sum) < (cpi->common.Y2dequant[0][0] << 2)) { - x->skip = 1; - distortion2 = sse; - rate2 = 500; - disable_skip = 1; - this_rd = RDFUNC(x->rdmult, x->rddiv, rate2, - distortion2, cpi->target_bits_per_mb); + sse2 = VP8_UVSSE(x, IF_RTCD(&cpi->rtcd.variance)); - break; + if (sse2 * 2 < x->encode_breakout) + { + x->skip = 1; + distortion2 = sse; + rate2 = 500; + disable_skip = 1; + this_rd = RDFUNC(x->rdmult, x->rddiv, rate2, + distortion2, cpi->target_bits_per_mb); + + break; + } } } } + //intermodecost[mode_index] = vp8_cost_mv_ref(this_mode, mdcounts); // Experimental debug code // Add in the Mv/mode cost -- 2.7.4