static void rd_pick_sb_modes(VP9_COMP *cpi, TileDataEnc *tile_data,
MACROBLOCK *const x, int mi_row, int mi_col,
RD_COST *rd_cost, BLOCK_SIZE bsize,
- PICK_MODE_CONTEXT *ctx, int64_t best_rd) {
+ PICK_MODE_CONTEXT *ctx, int rate_in_best_rd,
+ int64_t dist_in_best_rd) {
VP9_COMMON *const cm = &cpi->common;
TileInfo *const tile_info = &tile_data->tile_info;
MACROBLOCKD *const xd = &x->e_mbd;
struct macroblockd_plane *const pd = xd->plane;
const AQ_MODE aq_mode = cpi->oxcf.aq_mode;
int i, orig_rdmult;
+ int64_t best_rd = INT64_MAX;
vpx_clear_system_state();
set_segment_index(cpi, x, mi_row, mi_col, bsize, 0);
set_segment_rdmult(cpi, x, mi_row, mi_col, bsize, aq_mode);
+ if (rate_in_best_rd < INT_MAX && dist_in_best_rd < INT64_MAX) {
+ best_rd = vp9_calculate_rd_cost(x->rdmult, x->rddiv, rate_in_best_rd,
+ dist_in_best_rd);
+ }
// Find best coding mode & reconstruct the MB so it is available
// as a predictor for MBs that follow in the SB
// TODO(jingning) The rate-distortion optimization flow needs to be
// refactored to provide proper exit/return handle.
- if (rd_cost->rate == INT_MAX)
+ if (rd_cost->rate == INT_MAX || rd_cost->dist == INT64_MAX)
rd_cost->rdcost = INT64_MAX;
else
rd_cost->rdcost = RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist);
mi_col + (mi_step >> 1) < cm->mi_cols) {
pc_tree->partitioning = PARTITION_NONE;
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &none_rdc, bsize, ctx,
- INT64_MAX);
+ INT_MAX, INT64_MAX);
pl = partition_plane_context(xd, mi_row, mi_col, bsize);
switch (partition) {
case PARTITION_NONE:
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc, bsize,
- ctx, INT64_MAX);
+ ctx, INT_MAX, INT64_MAX);
break;
case PARTITION_HORZ:
pc_tree->horizontal[0].skip_ref_frame_mask = 0;
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc,
- subsize, &pc_tree->horizontal[0], INT64_MAX);
+ subsize, &pc_tree->horizontal[0], INT_MAX, INT64_MAX);
if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 &&
mi_row + (mi_step >> 1) < cm->mi_rows) {
RD_COST tmp_rdc;
encode_superblock(cpi, td, tp, 0, mi_row, mi_col, subsize, ctx);
pc_tree->horizontal[1].skip_ref_frame_mask = 0;
rd_pick_sb_modes(cpi, tile_data, x, mi_row + (mi_step >> 1), mi_col,
- &tmp_rdc, subsize, &pc_tree->horizontal[1], INT64_MAX);
+ &tmp_rdc, subsize, &pc_tree->horizontal[1], INT_MAX,
+ INT64_MAX);
if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) {
vp9_rd_cost_reset(&last_part_rdc);
break;
case PARTITION_VERT:
pc_tree->vertical[0].skip_ref_frame_mask = 0;
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc,
- subsize, &pc_tree->vertical[0], INT64_MAX);
+ subsize, &pc_tree->vertical[0], INT_MAX, INT64_MAX);
if (last_part_rdc.rate != INT_MAX && bsize >= BLOCK_8X8 &&
mi_col + (mi_step >> 1) < cm->mi_cols) {
RD_COST tmp_rdc;
update_state(cpi, td, ctx, mi_row, mi_col, subsize, 0);
encode_superblock(cpi, td, tp, 0, mi_row, mi_col, subsize, ctx);
pc_tree->vertical[bsize > BLOCK_8X8].skip_ref_frame_mask = 0;
- rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + (mi_step >> 1),
- &tmp_rdc, subsize,
- &pc_tree->vertical[bsize > BLOCK_8X8], INT64_MAX);
+ rd_pick_sb_modes(
+ cpi, tile_data, x, mi_row, mi_col + (mi_step >> 1), &tmp_rdc,
+ subsize, &pc_tree->vertical[bsize > BLOCK_8X8], INT_MAX, INT64_MAX);
if (tmp_rdc.rate == INT_MAX || tmp_rdc.dist == INT64_MAX) {
vp9_rd_cost_reset(&last_part_rdc);
break;
assert(partition == PARTITION_SPLIT);
if (bsize == BLOCK_8X8) {
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &last_part_rdc,
- subsize, pc_tree->leaf_split[0], INT64_MAX);
+ subsize, pc_tree->leaf_split[0], INT_MAX, INT64_MAX);
break;
}
last_part_rdc.rate = 0;
pc_tree->split[i]->partitioning = PARTITION_NONE;
rd_pick_sb_modes(cpi, tile_data, x, mi_row + y_idx, mi_col + x_idx,
&tmp_rdc, split_subsize, &pc_tree->split[i]->none,
- INT64_MAX);
+ INT_MAX, INT64_MAX);
restore_context(x, mi_row, mi_col, a, l, sa, sl, bsize);
// PARTITION_NONE
if (partition_none_allowed) {
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &this_rdc, bsize, ctx,
- best_rdc.rdcost);
+ best_rdc.rate, best_rdc.dist);
ctx->rdcost = this_rdc.rdcost;
if (this_rdc.rate != INT_MAX) {
if (cpi->sf.prune_ref_frame_for_rect_partitions) {
if (cpi->sf.adaptive_pred_interp_filter && partition_none_allowed)
pc_tree->leaf_split[0]->pred_interp_filter = pred_interp_filter;
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &sum_rdc, subsize,
- pc_tree->leaf_split[0], best_rdc.rdcost);
+ pc_tree->leaf_split[0], best_rdc.rate, best_rdc.dist);
if (sum_rdc.rate == INT_MAX) {
sum_rdc.rdcost = INT64_MAX;
} else {
partition_none_allowed)
pc_tree->horizontal[0].pred_interp_filter = pred_interp_filter;
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &sum_rdc, subsize,
- &pc_tree->horizontal[0],
- best_rdc.rdcost - part_mode_rdcost);
+ &pc_tree->horizontal[0], best_rdc.rate - part_mode_rate,
+ best_rdc.dist);
if (sum_rdc.rdcost < INT64_MAX) {
sum_rdc.rdcost += part_mode_rdcost;
sum_rdc.rate += part_mode_rate;
pc_tree->horizontal[1].pred_interp_filter = pred_interp_filter;
rd_pick_sb_modes(cpi, tile_data, x, mi_row + mi_step, mi_col, &this_rdc,
subsize, &pc_tree->horizontal[1],
- best_rdc.rdcost - sum_rdc.rdcost);
+ best_rdc.rate - sum_rdc.rate,
+ best_rdc.dist - sum_rdc.dist);
if (this_rdc.rate == INT_MAX) {
sum_rdc.rdcost = INT64_MAX;
} else {
partition_none_allowed)
pc_tree->vertical[0].pred_interp_filter = pred_interp_filter;
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col, &sum_rdc, subsize,
- &pc_tree->vertical[0], best_rdc.rdcost - part_mode_rdcost);
+ &pc_tree->vertical[0], best_rdc.rate - part_mode_rate,
+ best_rdc.dist);
if (sum_rdc.rdcost < INT64_MAX) {
sum_rdc.rdcost += part_mode_rdcost;
sum_rdc.rate += part_mode_rate;
pc_tree->vertical[1].pred_interp_filter = pred_interp_filter;
rd_pick_sb_modes(cpi, tile_data, x, mi_row, mi_col + mi_step, &this_rdc,
subsize, &pc_tree->vertical[1],
- best_rdc.rdcost - sum_rdc.rdcost);
+ best_rdc.rate - sum_rdc.rate,
+ best_rdc.dist - sum_rdc.dist);
if (this_rdc.rate == INT_MAX) {
sum_rdc.rdcost = INT64_MAX;
} else {