From db5f2cb57b39545d01e623e04450ba21ae1371aa Mon Sep 17 00:00:00 2001 From: John Koleszar Date: Fri, 15 Mar 2013 17:26:24 -0700 Subject: [PATCH] Fix use of NaN in firstpass If the second reference is better than the first in the long term, it was possible to try to take the fractional exponent of a negative number, giving an undefined result. Change-Id: I1dd08286747ceae960eb03bb5d98a383cc9d253b --- vp9/encoder/vp9_firstpass.c | 56 ++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c index 5c2067b..d646fe2 100644 --- a/vp9/encoder/vp9_firstpass.c +++ b/vp9/encoder/vp9_firstpass.c @@ -859,6 +859,8 @@ static double calc_correction_factor(double err_per_mb, power_term = (power_term > pt_high) ? pt_high : power_term; // Calculate correction factor + if (power_term < 1.0) + assert(error_term >= 0.0); correction_factor = pow(error_term, power_term); // Clip range @@ -920,15 +922,19 @@ static int estimate_max_q(VP9_COMP *cpi, // Look at the drop in prediction quality between the last frame // and the GF buffer (which contained an older frame). - sr_err_diff = - (fpstats->sr_coded_error - fpstats->coded_error) / - (fpstats->count * cpi->common.MBs); - sr_correction = (sr_err_diff / 32.0); - sr_correction = pow(sr_correction, 0.25); - if (sr_correction < 0.75) + if (fpstats->sr_coded_error > fpstats->coded_error) { + sr_err_diff = + (fpstats->sr_coded_error - fpstats->coded_error) / + (fpstats->count * cpi->common.MBs); + sr_correction = (sr_err_diff / 32.0); + sr_correction = pow(sr_correction, 0.25); + if (sr_correction < 0.75) + sr_correction = 0.75; + else if (sr_correction > 1.25) + sr_correction = 1.25; + } else { sr_correction = 0.75; - else if (sr_correction > 1.25) - sr_correction = 1.25; + } // Calculate a corrective factor based on a rolling ratio of bits spent // vs target bits @@ -1031,15 +1037,19 @@ static int estimate_cq(VP9_COMP *cpi, // Look at the drop in prediction quality between the last frame // and the GF buffer (which contained an older frame). - sr_err_diff = - (fpstats->sr_coded_error - fpstats->coded_error) / - (fpstats->count * cpi->common.MBs); - sr_correction = (sr_err_diff / 32.0); - sr_correction = pow(sr_correction, 0.25); - if (sr_correction < 0.75) + if (fpstats->sr_coded_error > fpstats->coded_error) { + sr_err_diff = + (fpstats->sr_coded_error - fpstats->coded_error) / + (fpstats->count * cpi->common.MBs); + sr_correction = (sr_err_diff / 32.0); + sr_correction = pow(sr_correction, 0.25); + if (sr_correction < 0.75) + sr_correction = 0.75; + else if (sr_correction > 1.25) + sr_correction = 1.25; + } else { sr_correction = 0.75; - else if (sr_correction > 1.25) - sr_correction = 1.25; + } // II ratio correction factor for clip as a whole clip_iiratio = cpi->twopass.total_stats->intra_error / @@ -1178,12 +1188,16 @@ static double get_prediction_decay_rate(VP9_COMP *cpi, mb_sr_err_diff = (next_frame->sr_coded_error - next_frame->coded_error) / (cpi->common.MBs); - second_ref_decay = 1.0 - (mb_sr_err_diff / 512.0); - second_ref_decay = pow(second_ref_decay, 0.5); - if (second_ref_decay < 0.85) + if (mb_sr_err_diff <= 512.0) { + second_ref_decay = 1.0 - (mb_sr_err_diff / 512.0); + second_ref_decay = pow(second_ref_decay, 0.5); + if (second_ref_decay < 0.85) + second_ref_decay = 0.85; + else if (second_ref_decay > 1.0) + second_ref_decay = 1.0; + } else { second_ref_decay = 0.85; - else if (second_ref_decay > 1.0) - second_ref_decay = 1.0; + } if (second_ref_decay < prediction_decay_rate) prediction_decay_rate = second_ref_decay; -- 2.7.4