2 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
14 #include "./vp9_rtcd.h"
15 #include "./vpx_dsp_rtcd.h"
17 #include "vpx_dsp/vpx_dsp_common.h"
18 #include "vpx_mem/vpx_mem.h"
19 #include "vpx_ports/mem.h"
20 #include "vpx_ports/system_state.h"
22 #include "vp9/common/vp9_common.h"
23 #include "vp9/common/vp9_entropy.h"
24 #include "vp9/common/vp9_entropymode.h"
25 #include "vp9/common/vp9_idct.h"
26 #include "vp9/common/vp9_mvref_common.h"
27 #include "vp9/common/vp9_pred_common.h"
28 #include "vp9/common/vp9_quant_common.h"
29 #include "vp9/common/vp9_reconinter.h"
30 #include "vp9/common/vp9_reconintra.h"
31 #include "vp9/common/vp9_scan.h"
32 #include "vp9/common/vp9_seg_common.h"
34 #include "vp9/encoder/vp9_cost.h"
35 #include "vp9/encoder/vp9_encodemb.h"
36 #include "vp9/encoder/vp9_encodemv.h"
37 #include "vp9/encoder/vp9_encoder.h"
38 #include "vp9/encoder/vp9_mcomp.h"
39 #include "vp9/encoder/vp9_quantize.h"
40 #include "vp9/encoder/vp9_ratectrl.h"
41 #include "vp9/encoder/vp9_rd.h"
42 #include "vp9/encoder/vp9_rdopt.h"
43 #include "vp9/encoder/vp9_aq_variance.h"
45 #define LAST_FRAME_MODE_MASK \
46 ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
47 #define GOLDEN_FRAME_MODE_MASK \
48 ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
49 #define ALT_REF_MODE_MASK \
50 ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | (1 << INTRA_FRAME))
52 #define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | 0x01)
54 #define MIN_EARLY_TERM_INDEX 3
55 #define NEW_MV_DISCOUNT_FACTOR 8
59 MV_REFERENCE_FRAME ref_frame[2];
63 MV_REFERENCE_FRAME ref_frame[2];
66 struct rdcost_block_args {
69 ENTROPY_CONTEXT t_above[16];
70 ENTROPY_CONTEXT t_left[16];
77 int use_fast_coef_costing;
80 struct buf_2d *this_recon;
83 #define LAST_NEW_MV_INDEX 6
84 static const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
85 { NEARESTMV, { LAST_FRAME, NONE } },
86 { NEARESTMV, { ALTREF_FRAME, NONE } },
87 { NEARESTMV, { GOLDEN_FRAME, NONE } },
89 { DC_PRED, { INTRA_FRAME, NONE } },
91 { NEWMV, { LAST_FRAME, NONE } },
92 { NEWMV, { ALTREF_FRAME, NONE } },
93 { NEWMV, { GOLDEN_FRAME, NONE } },
95 { NEARMV, { LAST_FRAME, NONE } },
96 { NEARMV, { ALTREF_FRAME, NONE } },
97 { NEARMV, { GOLDEN_FRAME, NONE } },
99 { ZEROMV, { LAST_FRAME, NONE } },
100 { ZEROMV, { GOLDEN_FRAME, NONE } },
101 { ZEROMV, { ALTREF_FRAME, NONE } },
103 { NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
104 { NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
106 { TM_PRED, { INTRA_FRAME, NONE } },
108 { NEARMV, { LAST_FRAME, ALTREF_FRAME } },
109 { NEWMV, { LAST_FRAME, ALTREF_FRAME } },
110 { NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
111 { NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
113 { ZEROMV, { LAST_FRAME, ALTREF_FRAME } },
114 { ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } },
116 { H_PRED, { INTRA_FRAME, NONE } },
117 { V_PRED, { INTRA_FRAME, NONE } },
118 { D135_PRED, { INTRA_FRAME, NONE } },
119 { D207_PRED, { INTRA_FRAME, NONE } },
120 { D153_PRED, { INTRA_FRAME, NONE } },
121 { D63_PRED, { INTRA_FRAME, NONE } },
122 { D117_PRED, { INTRA_FRAME, NONE } },
123 { D45_PRED, { INTRA_FRAME, NONE } },
126 static const REF_DEFINITION vp9_ref_order[MAX_REFS] = {
127 { { LAST_FRAME, NONE } }, { { GOLDEN_FRAME, NONE } },
128 { { ALTREF_FRAME, NONE } }, { { LAST_FRAME, ALTREF_FRAME } },
129 { { GOLDEN_FRAME, ALTREF_FRAME } }, { { INTRA_FRAME, NONE } },
132 static void swap_block_ptr(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, int m, int n,
133 int min_plane, int max_plane) {
136 for (i = min_plane; i < max_plane; ++i) {
137 struct macroblock_plane *const p = &x->plane[i];
138 struct macroblockd_plane *const pd = &x->e_mbd.plane[i];
140 p->coeff = ctx->coeff_pbuf[i][m];
141 p->qcoeff = ctx->qcoeff_pbuf[i][m];
142 pd->dqcoeff = ctx->dqcoeff_pbuf[i][m];
143 p->eobs = ctx->eobs_pbuf[i][m];
145 ctx->coeff_pbuf[i][m] = ctx->coeff_pbuf[i][n];
146 ctx->qcoeff_pbuf[i][m] = ctx->qcoeff_pbuf[i][n];
147 ctx->dqcoeff_pbuf[i][m] = ctx->dqcoeff_pbuf[i][n];
148 ctx->eobs_pbuf[i][m] = ctx->eobs_pbuf[i][n];
150 ctx->coeff_pbuf[i][n] = p->coeff;
151 ctx->qcoeff_pbuf[i][n] = p->qcoeff;
152 ctx->dqcoeff_pbuf[i][n] = pd->dqcoeff;
153 ctx->eobs_pbuf[i][n] = p->eobs;
157 static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, MACROBLOCK *x,
158 MACROBLOCKD *xd, int *out_rate_sum,
159 int64_t *out_dist_sum, int *skip_txfm_sb,
160 int64_t *skip_sse_sb) {
161 // Note our transform coeffs are 8 times an orthogonal transform.
162 // Hence quantizer step is also 8 times. To get effective quantizer
163 // we need to divide by 8 before sending to modeling function.
165 int64_t rate_sum = 0;
166 int64_t dist_sum = 0;
167 const int ref = xd->mi[0]->ref_frame[0];
169 unsigned int var = 0;
170 int64_t total_sse = 0;
174 const int dequant_shift =
175 #if CONFIG_VP9_HIGHBITDEPTH
176 (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 :
177 #endif // CONFIG_VP9_HIGHBITDEPTH
179 unsigned int qstep_vec[MAX_MB_PLANE];
180 unsigned int nlog2_vec[MAX_MB_PLANE];
181 unsigned int sum_sse_vec[MAX_MB_PLANE];
182 int any_zero_sum_sse = 0;
184 x->pred_sse[ref] = 0;
186 for (i = 0; i < MAX_MB_PLANE; ++i) {
187 struct macroblock_plane *const p = &x->plane[i];
188 struct macroblockd_plane *const pd = &xd->plane[i];
189 const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
190 const TX_SIZE max_tx_size = max_txsize_lookup[bs];
191 const BLOCK_SIZE unit_size = txsize_to_bsize[max_tx_size];
192 const int64_t dc_thr = p->quant_thred[0] >> shift;
193 const int64_t ac_thr = p->quant_thred[1] >> shift;
194 unsigned int sum_sse = 0;
195 // The low thresholds are used to measure if the prediction errors are
196 // low enough so that we can skip the mode search.
197 const int64_t low_dc_thr = VPXMIN(50, dc_thr >> 2);
198 const int64_t low_ac_thr = VPXMIN(80, ac_thr >> 2);
199 int bw = 1 << (b_width_log2_lookup[bs] - b_width_log2_lookup[unit_size]);
200 int bh = 1 << (b_height_log2_lookup[bs] - b_width_log2_lookup[unit_size]);
202 int lw = b_width_log2_lookup[unit_size] + 2;
203 int lh = b_height_log2_lookup[unit_size] + 2;
205 for (idy = 0; idy < bh; ++idy) {
206 for (idx = 0; idx < bw; ++idx) {
207 uint8_t *src = p->src.buf + (idy * p->src.stride << lh) + (idx << lw);
208 uint8_t *dst = pd->dst.buf + (idy * pd->dst.stride << lh) + (idx << lh);
209 int block_idx = (idy << 1) + idx;
210 int low_err_skip = 0;
212 var = cpi->fn_ptr[unit_size].vf(src, p->src.stride, dst, pd->dst.stride,
214 x->bsse[(i << 2) + block_idx] = sse;
217 x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_NONE;
218 if (!x->select_tx_size) {
219 // Check if all ac coefficients can be quantized to zero.
220 if (var < ac_thr || var == 0) {
221 x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_AC_ONLY;
223 // Check if dc coefficient can be quantized to zero.
224 if (sse - var < dc_thr || sse == var) {
225 x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_AC_DC;
227 if (!sse || (var < low_ac_thr && sse - var < low_dc_thr))
233 if (skip_flag && !low_err_skip) skip_flag = 0;
235 if (i == 0) x->pred_sse[ref] += sse;
239 total_sse += sum_sse;
240 sum_sse_vec[i] = sum_sse;
241 any_zero_sum_sse = any_zero_sum_sse || (sum_sse == 0);
242 qstep_vec[i] = pd->dequant[1] >> dequant_shift;
243 nlog2_vec[i] = num_pels_log2_lookup[bs];
246 // Fast approximate the modelling function.
247 if (cpi->sf.simple_model_rd_from_var) {
248 for (i = 0; i < MAX_MB_PLANE; ++i) {
250 const int64_t square_error = sum_sse_vec[i];
251 int quantizer = qstep_vec[i];
254 rate = (square_error * (280 - quantizer)) >> (16 - VP9_PROB_COST_SHIFT);
257 dist = (square_error * quantizer) >> 8;
262 if (any_zero_sum_sse) {
263 for (i = 0; i < MAX_MB_PLANE; ++i) {
265 vp9_model_rd_from_var_lapndz(sum_sse_vec[i], nlog2_vec[i], qstep_vec[i],
271 vp9_model_rd_from_var_lapndz_vec(sum_sse_vec, nlog2_vec, qstep_vec,
272 &rate_sum, &dist_sum);
276 *skip_txfm_sb = skip_flag;
277 *skip_sse_sb = total_sse << VP9_DIST_SCALE_LOG2;
278 *out_rate_sum = (int)rate_sum;
279 *out_dist_sum = dist_sum << VP9_DIST_SCALE_LOG2;
282 #if CONFIG_VP9_HIGHBITDEPTH
283 int64_t vp9_highbd_block_error_c(const tran_low_t *coeff,
284 const tran_low_t *dqcoeff, intptr_t block_size,
285 int64_t *ssz, int bd) {
287 int64_t error = 0, sqcoeff = 0;
288 int shift = 2 * (bd - 8);
289 int rounding = shift > 0 ? 1 << (shift - 1) : 0;
291 for (i = 0; i < block_size; i++) {
292 const int64_t diff = coeff[i] - dqcoeff[i];
293 error += diff * diff;
294 sqcoeff += (int64_t)coeff[i] * (int64_t)coeff[i];
296 assert(error >= 0 && sqcoeff >= 0);
297 error = (error + rounding) >> shift;
298 sqcoeff = (sqcoeff + rounding) >> shift;
304 static int64_t vp9_highbd_block_error_dispatch(const tran_low_t *coeff,
305 const tran_low_t *dqcoeff,
307 int64_t *ssz, int bd) {
309 return vp9_block_error(coeff, dqcoeff, block_size, ssz);
311 return vp9_highbd_block_error(coeff, dqcoeff, block_size, ssz, bd);
314 #endif // CONFIG_VP9_HIGHBITDEPTH
316 int64_t vp9_block_error_c(const tran_low_t *coeff, const tran_low_t *dqcoeff,
317 intptr_t block_size, int64_t *ssz) {
319 int64_t error = 0, sqcoeff = 0;
321 for (i = 0; i < block_size; i++) {
322 const int diff = coeff[i] - dqcoeff[i];
323 error += diff * diff;
324 sqcoeff += coeff[i] * coeff[i];
331 int64_t vp9_block_error_fp_c(const tran_low_t *coeff, const tran_low_t *dqcoeff,
336 for (i = 0; i < block_size; i++) {
337 const int diff = coeff[i] - dqcoeff[i];
338 error += diff * diff;
344 /* The trailing '0' is a terminator which is used inside cost_coeffs() to
345 * decide whether to include cost of a trailing EOB node or not (i.e. we
346 * can skip this if the last coefficient in this transform block, e.g. the
347 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
349 static const int16_t band_counts[TX_SIZES][8] = {
350 { 1, 2, 3, 4, 3, 16 - 13, 0 },
351 { 1, 2, 3, 4, 11, 64 - 21, 0 },
352 { 1, 2, 3, 4, 11, 256 - 21, 0 },
353 { 1, 2, 3, 4, 11, 1024 - 21, 0 },
355 static int cost_coeffs(MACROBLOCK *x, int plane, int block, TX_SIZE tx_size,
356 int pt, const int16_t *scan, const int16_t *nb,
357 int use_fast_coef_costing) {
358 MACROBLOCKD *const xd = &x->e_mbd;
359 MODE_INFO *mi = xd->mi[0];
360 const struct macroblock_plane *p = &x->plane[plane];
361 const PLANE_TYPE type = get_plane_type(plane);
362 const int16_t *band_count = &band_counts[tx_size][1];
363 const int eob = p->eobs[block];
364 const tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
365 unsigned int(*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] =
366 x->token_costs[tx_size][type][is_inter_block(mi)];
367 uint8_t token_cache[32 * 32];
369 #if CONFIG_VP9_HIGHBITDEPTH
370 const uint16_t *cat6_high_cost = vp9_get_high_cost_table(xd->bd);
372 const uint16_t *cat6_high_cost = vp9_get_high_cost_table(8);
375 // Check for consistency of tx_size with mode info
376 assert(type == PLANE_TYPE_Y
377 ? mi->tx_size == tx_size
378 : get_uv_tx_size(mi, &xd->plane[plane]) == tx_size);
382 cost = token_costs[0][0][pt][EOB_TOKEN];
384 if (use_fast_coef_costing) {
385 int band_left = *band_count++;
391 cost = vp9_get_token_cost(v, &prev_t, cat6_high_cost);
392 cost += (*token_costs)[0][pt][prev_t];
394 token_cache[0] = vp9_pt_energy_class[prev_t];
398 for (c = 1; c < eob; c++) {
399 const int rc = scan[c];
403 cost += vp9_get_token_cost(v, &t, cat6_high_cost);
404 cost += (*token_costs)[!prev_t][!prev_t][t];
407 band_left = *band_count++;
413 if (band_left) cost += (*token_costs)[0][!prev_t][EOB_TOKEN];
415 } else { // !use_fast_coef_costing
416 int band_left = *band_count++;
422 unsigned int(*tok_cost_ptr)[COEFF_CONTEXTS][ENTROPY_TOKENS];
423 cost = vp9_get_token_cost(v, &tok, cat6_high_cost);
424 cost += (*token_costs)[0][pt][tok];
426 token_cache[0] = vp9_pt_energy_class[tok];
429 tok_cost_ptr = &((*token_costs)[!tok]);
432 for (c = 1; c < eob; c++) {
433 const int rc = scan[c];
436 cost += vp9_get_token_cost(v, &tok, cat6_high_cost);
437 pt = get_coef_context(nb, token_cache, c);
438 cost += (*tok_cost_ptr)[pt][tok];
439 token_cache[rc] = vp9_pt_energy_class[tok];
441 band_left = *band_count++;
444 tok_cost_ptr = &((*token_costs)[!tok]);
449 pt = get_coef_context(nb, token_cache, c);
450 cost += (*token_costs)[0][pt][EOB_TOKEN];
458 static INLINE int num_4x4_to_edge(int plane_4x4_dim, int mb_to_edge_dim,
459 int subsampling_dim, int blk_dim) {
460 return plane_4x4_dim + (mb_to_edge_dim >> (5 + subsampling_dim)) - blk_dim;
463 // Copy all visible 4x4s in the transform block.
464 static void copy_block_visible(const MACROBLOCKD *xd,
465 const struct macroblockd_plane *const pd,
466 const uint8_t *src, const int src_stride,
467 uint8_t *dst, const int dst_stride, int blk_row,
468 int blk_col, const BLOCK_SIZE plane_bsize,
469 const BLOCK_SIZE tx_bsize) {
470 const int plane_4x4_w = num_4x4_blocks_wide_lookup[plane_bsize];
471 const int plane_4x4_h = num_4x4_blocks_high_lookup[plane_bsize];
472 const int tx_4x4_w = num_4x4_blocks_wide_lookup[tx_bsize];
473 const int tx_4x4_h = num_4x4_blocks_high_lookup[tx_bsize];
474 int b4x4s_to_right_edge = num_4x4_to_edge(plane_4x4_w, xd->mb_to_right_edge,
475 pd->subsampling_x, blk_col);
476 int b4x4s_to_bottom_edge = num_4x4_to_edge(plane_4x4_h, xd->mb_to_bottom_edge,
477 pd->subsampling_y, blk_row);
478 const int is_highbd = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH;
479 if (tx_bsize == BLOCK_4X4 ||
480 (b4x4s_to_right_edge >= tx_4x4_w && b4x4s_to_bottom_edge >= tx_4x4_h)) {
481 const int w = tx_4x4_w << 2;
482 const int h = tx_4x4_h << 2;
483 #if CONFIG_VP9_HIGHBITDEPTH
485 vpx_highbd_convolve_copy(CONVERT_TO_SHORTPTR(src), src_stride,
486 CONVERT_TO_SHORTPTR(dst), dst_stride, NULL, 0, 0,
490 vpx_convolve_copy(src, src_stride, dst, dst_stride, NULL, 0, 0, 0, 0, w,
492 #if CONFIG_VP9_HIGHBITDEPTH
497 int max_r = VPXMIN(b4x4s_to_bottom_edge, tx_4x4_h);
498 int max_c = VPXMIN(b4x4s_to_right_edge, tx_4x4_w);
499 // if we are in the unrestricted motion border.
500 for (r = 0; r < max_r; ++r) {
501 // Skip visiting the sub blocks that are wholly within the UMV.
502 for (c = 0; c < max_c; ++c) {
503 const uint8_t *src_ptr = src + r * src_stride * 4 + c * 4;
504 uint8_t *dst_ptr = dst + r * dst_stride * 4 + c * 4;
505 #if CONFIG_VP9_HIGHBITDEPTH
507 vpx_highbd_convolve_copy(CONVERT_TO_SHORTPTR(src_ptr), src_stride,
508 CONVERT_TO_SHORTPTR(dst_ptr), dst_stride,
509 NULL, 0, 0, 0, 0, 4, 4, xd->bd);
512 vpx_convolve_copy(src_ptr, src_stride, dst_ptr, dst_stride, NULL, 0,
514 #if CONFIG_VP9_HIGHBITDEPTH
523 // Compute the pixel domain sum square error on all visible 4x4s in the
525 static unsigned pixel_sse(const VP9_COMP *const cpi, const MACROBLOCKD *xd,
526 const struct macroblockd_plane *const pd,
527 const uint8_t *src, const int src_stride,
528 const uint8_t *dst, const int dst_stride, int blk_row,
529 int blk_col, const BLOCK_SIZE plane_bsize,
530 const BLOCK_SIZE tx_bsize) {
531 unsigned int sse = 0;
532 const int plane_4x4_w = num_4x4_blocks_wide_lookup[plane_bsize];
533 const int plane_4x4_h = num_4x4_blocks_high_lookup[plane_bsize];
534 const int tx_4x4_w = num_4x4_blocks_wide_lookup[tx_bsize];
535 const int tx_4x4_h = num_4x4_blocks_high_lookup[tx_bsize];
536 int b4x4s_to_right_edge = num_4x4_to_edge(plane_4x4_w, xd->mb_to_right_edge,
537 pd->subsampling_x, blk_col);
538 int b4x4s_to_bottom_edge = num_4x4_to_edge(plane_4x4_h, xd->mb_to_bottom_edge,
539 pd->subsampling_y, blk_row);
540 if (tx_bsize == BLOCK_4X4 ||
541 (b4x4s_to_right_edge >= tx_4x4_w && b4x4s_to_bottom_edge >= tx_4x4_h)) {
542 cpi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
544 const vpx_variance_fn_t vf_4x4 = cpi->fn_ptr[BLOCK_4X4].vf;
546 unsigned this_sse = 0;
547 int max_r = VPXMIN(b4x4s_to_bottom_edge, tx_4x4_h);
548 int max_c = VPXMIN(b4x4s_to_right_edge, tx_4x4_w);
550 // if we are in the unrestricted motion border.
551 for (r = 0; r < max_r; ++r) {
552 // Skip visiting the sub blocks that are wholly within the UMV.
553 for (c = 0; c < max_c; ++c) {
554 vf_4x4(src + r * src_stride * 4 + c * 4, src_stride,
555 dst + r * dst_stride * 4 + c * 4, dst_stride, &this_sse);
563 // Compute the squares sum squares on all visible 4x4s in the transform block.
564 static int64_t sum_squares_visible(const MACROBLOCKD *xd,
565 const struct macroblockd_plane *const pd,
566 const int16_t *diff, const int diff_stride,
567 int blk_row, int blk_col,
568 const BLOCK_SIZE plane_bsize,
569 const BLOCK_SIZE tx_bsize) {
571 const int plane_4x4_w = num_4x4_blocks_wide_lookup[plane_bsize];
572 const int plane_4x4_h = num_4x4_blocks_high_lookup[plane_bsize];
573 const int tx_4x4_w = num_4x4_blocks_wide_lookup[tx_bsize];
574 const int tx_4x4_h = num_4x4_blocks_high_lookup[tx_bsize];
575 int b4x4s_to_right_edge = num_4x4_to_edge(plane_4x4_w, xd->mb_to_right_edge,
576 pd->subsampling_x, blk_col);
577 int b4x4s_to_bottom_edge = num_4x4_to_edge(plane_4x4_h, xd->mb_to_bottom_edge,
578 pd->subsampling_y, blk_row);
579 if (tx_bsize == BLOCK_4X4 ||
580 (b4x4s_to_right_edge >= tx_4x4_w && b4x4s_to_bottom_edge >= tx_4x4_h)) {
581 assert(tx_4x4_w == tx_4x4_h);
582 sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_4x4_w << 2);
585 int max_r = VPXMIN(b4x4s_to_bottom_edge, tx_4x4_h);
586 int max_c = VPXMIN(b4x4s_to_right_edge, tx_4x4_w);
588 // if we are in the unrestricted motion border.
589 for (r = 0; r < max_r; ++r) {
590 // Skip visiting the sub blocks that are wholly within the UMV.
591 for (c = 0; c < max_c; ++c) {
592 sse += (int64_t)vpx_sum_squares_2d_i16(
593 diff + r * diff_stride * 4 + c * 4, diff_stride, 4);
600 static void dist_block(const VP9_COMP *cpi, MACROBLOCK *x, int plane,
601 BLOCK_SIZE plane_bsize, int block, int blk_row,
602 int blk_col, TX_SIZE tx_size, int64_t *out_dist,
603 int64_t *out_sse, struct buf_2d *out_recon) {
604 MACROBLOCKD *const xd = &x->e_mbd;
605 const struct macroblock_plane *const p = &x->plane[plane];
606 const struct macroblockd_plane *const pd = &xd->plane[plane];
607 const int eob = p->eobs[block];
609 if (!out_recon && x->block_tx_domain && eob) {
610 const int ss_txfrm_size = tx_size << 1;
612 const int shift = tx_size == TX_32X32 ? 0 : 2;
613 const tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
614 const tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
615 #if CONFIG_VP9_HIGHBITDEPTH
616 const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8;
617 *out_dist = vp9_highbd_block_error_dispatch(
618 coeff, dqcoeff, 16 << ss_txfrm_size, &this_sse, bd) >>
622 vp9_block_error(coeff, dqcoeff, 16 << ss_txfrm_size, &this_sse) >>
624 #endif // CONFIG_VP9_HIGHBITDEPTH
625 *out_sse = this_sse >> shift;
627 if (x->skip_encode && !is_inter_block(xd->mi[0])) {
628 // TODO(jingning): tune the model to better capture the distortion.
630 (pd->dequant[1] * pd->dequant[1] * (1 << ss_txfrm_size)) >>
631 #if CONFIG_VP9_HIGHBITDEPTH
632 (shift + 2 + (bd - 8) * 2);
635 #endif // CONFIG_VP9_HIGHBITDEPTH
636 *out_dist += (p >> 4);
640 const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size];
641 const int bs = 4 * num_4x4_blocks_wide_lookup[tx_bsize];
642 const int src_stride = p->src.stride;
643 const int dst_stride = pd->dst.stride;
644 const int src_idx = 4 * (blk_row * src_stride + blk_col);
645 const int dst_idx = 4 * (blk_row * dst_stride + blk_col);
646 const uint8_t *src = &p->src.buf[src_idx];
647 const uint8_t *dst = &pd->dst.buf[dst_idx];
648 uint8_t *out_recon_ptr = 0;
650 const tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
653 tmp = pixel_sse(cpi, xd, pd, src, src_stride, dst, dst_stride, blk_row,
654 blk_col, plane_bsize, tx_bsize);
655 *out_sse = (int64_t)tmp * 16;
657 const int out_recon_idx = 4 * (blk_row * out_recon->stride + blk_col);
658 out_recon_ptr = &out_recon->buf[out_recon_idx];
659 copy_block_visible(xd, pd, dst, dst_stride, out_recon_ptr,
660 out_recon->stride, blk_row, blk_col, plane_bsize,
665 #if CONFIG_VP9_HIGHBITDEPTH
666 DECLARE_ALIGNED(16, uint16_t, recon16[1024]);
667 uint8_t *recon = (uint8_t *)recon16;
669 DECLARE_ALIGNED(16, uint8_t, recon[1024]);
670 #endif // CONFIG_VP9_HIGHBITDEPTH
672 #if CONFIG_VP9_HIGHBITDEPTH
673 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
674 vpx_highbd_convolve_copy(CONVERT_TO_SHORTPTR(dst), dst_stride, recon16,
675 32, NULL, 0, 0, 0, 0, bs, bs, xd->bd);
677 vp9_highbd_iwht4x4_add(dqcoeff, recon16, 32, eob, xd->bd);
681 vp9_highbd_idct4x4_add(dqcoeff, recon16, 32, eob, xd->bd);
684 vp9_highbd_idct8x8_add(dqcoeff, recon16, 32, eob, xd->bd);
687 vp9_highbd_idct16x16_add(dqcoeff, recon16, 32, eob, xd->bd);
690 assert(tx_size == TX_32X32);
691 vp9_highbd_idct32x32_add(dqcoeff, recon16, 32, eob, xd->bd);
695 recon = CONVERT_TO_BYTEPTR(recon16);
697 #endif // CONFIG_VP9_HIGHBITDEPTH
698 vpx_convolve_copy(dst, dst_stride, recon, 32, NULL, 0, 0, 0, 0, bs, bs);
700 case TX_32X32: vp9_idct32x32_add(dqcoeff, recon, 32, eob); break;
701 case TX_16X16: vp9_idct16x16_add(dqcoeff, recon, 32, eob); break;
702 case TX_8X8: vp9_idct8x8_add(dqcoeff, recon, 32, eob); break;
704 assert(tx_size == TX_4X4);
705 // this is like vp9_short_idct4x4 but has a special case around
706 // eob<=1, which is significant (not just an optimization) for
707 // the lossless case.
708 x->inv_txfm_add(dqcoeff, recon, 32, eob);
711 #if CONFIG_VP9_HIGHBITDEPTH
713 #endif // CONFIG_VP9_HIGHBITDEPTH
715 tmp = pixel_sse(cpi, xd, pd, src, src_stride, recon, 32, blk_row, blk_col,
716 plane_bsize, tx_bsize);
718 copy_block_visible(xd, pd, recon, 32, out_recon_ptr, out_recon->stride,
719 blk_row, blk_col, plane_bsize, tx_bsize);
723 *out_dist = (int64_t)tmp * 16;
727 static int rate_block(int plane, int block, TX_SIZE tx_size, int coeff_ctx,
728 struct rdcost_block_args *args) {
729 return cost_coeffs(args->x, plane, block, tx_size, coeff_ctx, args->so->scan,
730 args->so->neighbors, args->use_fast_coef_costing);
733 static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
734 BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) {
735 struct rdcost_block_args *args = arg;
736 MACROBLOCK *const x = args->x;
737 MACROBLOCKD *const xd = &x->e_mbd;
738 MODE_INFO *const mi = xd->mi[0];
739 int64_t rd1, rd2, rd;
743 const int coeff_ctx =
744 combine_entropy_contexts(args->t_left[blk_row], args->t_above[blk_col]);
745 struct buf_2d *recon = args->this_recon;
746 const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size];
747 const struct macroblockd_plane *const pd = &xd->plane[plane];
748 const int dst_stride = pd->dst.stride;
749 const uint8_t *dst = &pd->dst.buf[4 * (blk_row * dst_stride + blk_col)];
751 if (args->exit_early) return;
753 if (!is_inter_block(mi)) {
754 struct encode_b_args intra_arg = { x, x->block_qcoeff_opt, args->t_above,
755 args->t_left, &mi->skip };
756 vp9_encode_block_intra(plane, block, blk_row, blk_col, plane_bsize, tx_size,
759 uint8_t *rec_ptr = &recon->buf[4 * (blk_row * recon->stride + blk_col)];
760 copy_block_visible(xd, pd, dst, dst_stride, rec_ptr, recon->stride,
761 blk_row, blk_col, plane_bsize, tx_bsize);
763 if (x->block_tx_domain) {
764 dist_block(args->cpi, x, plane, plane_bsize, block, blk_row, blk_col,
765 tx_size, &dist, &sse, /*recon =*/0);
767 const struct macroblock_plane *const p = &x->plane[plane];
768 const int src_stride = p->src.stride;
769 const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
770 const uint8_t *src = &p->src.buf[4 * (blk_row * src_stride + blk_col)];
771 const int16_t *diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];
773 sse = sum_squares_visible(xd, pd, diff, diff_stride, blk_row, blk_col,
774 plane_bsize, tx_bsize);
775 #if CONFIG_VP9_HIGHBITDEPTH
776 if ((xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) && (xd->bd > 8))
777 sse = ROUND64_POWER_OF_TWO(sse, (xd->bd - 8) * 2);
778 #endif // CONFIG_VP9_HIGHBITDEPTH
780 tmp = pixel_sse(args->cpi, xd, pd, src, src_stride, dst, dst_stride,
781 blk_row, blk_col, plane_bsize, tx_bsize);
782 dist = (int64_t)tmp * 16;
785 int skip_txfm_flag = SKIP_TXFM_NONE;
786 if (max_txsize_lookup[plane_bsize] == tx_size)
787 skip_txfm_flag = x->skip_txfm[(plane << 2) + (block >> (tx_size << 1))];
789 if (skip_txfm_flag == SKIP_TXFM_NONE ||
790 (recon && skip_txfm_flag == SKIP_TXFM_AC_ONLY)) {
791 // full forward transform and quantization
792 vp9_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size);
793 if (x->block_qcoeff_opt)
794 vp9_optimize_b(x, plane, block, tx_size, coeff_ctx);
795 dist_block(args->cpi, x, plane, plane_bsize, block, blk_row, blk_col,
796 tx_size, &dist, &sse, recon);
797 } else if (skip_txfm_flag == SKIP_TXFM_AC_ONLY) {
798 // compute DC coefficient
799 tran_low_t *const coeff = BLOCK_OFFSET(x->plane[plane].coeff, block);
800 tran_low_t *const dqcoeff = BLOCK_OFFSET(xd->plane[plane].dqcoeff, block);
801 vp9_xform_quant_dc(x, plane, block, blk_row, blk_col, plane_bsize,
803 sse = x->bsse[(plane << 2) + (block >> (tx_size << 1))] << 4;
805 if (x->plane[plane].eobs[block]) {
806 const int64_t orig_sse = (int64_t)coeff[0] * coeff[0];
807 const int64_t resd_sse = coeff[0] - dqcoeff[0];
808 int64_t dc_correct = orig_sse - resd_sse * resd_sse;
809 #if CONFIG_VP9_HIGHBITDEPTH
810 dc_correct >>= ((xd->bd - 8) * 2);
812 if (tx_size != TX_32X32) dc_correct >>= 2;
814 dist = VPXMAX(0, sse - dc_correct);
818 // skip forward transform. Because this is handled here, the quantization
819 // does not need to do it.
820 x->plane[plane].eobs[block] = 0;
821 sse = x->bsse[(plane << 2) + (block >> (tx_size << 1))] << 4;
824 uint8_t *rec_ptr = &recon->buf[4 * (blk_row * recon->stride + blk_col)];
825 copy_block_visible(xd, pd, dst, dst_stride, rec_ptr, recon->stride,
826 blk_row, blk_col, plane_bsize, tx_bsize);
831 rd = RDCOST(x->rdmult, x->rddiv, 0, dist);
832 if (args->this_rd + rd > args->best_rd) {
833 args->exit_early = 1;
837 rate = rate_block(plane, block, tx_size, coeff_ctx, args);
838 args->t_above[blk_col] = (x->plane[plane].eobs[block] > 0) ? 1 : 0;
839 args->t_left[blk_row] = (x->plane[plane].eobs[block] > 0) ? 1 : 0;
840 rd1 = RDCOST(x->rdmult, x->rddiv, rate, dist);
841 rd2 = RDCOST(x->rdmult, x->rddiv, 0, sse);
843 // TODO(jingning): temporarily enabled only for luma component
844 rd = VPXMIN(rd1, rd2);
846 x->zcoeff_blk[tx_size][block] =
847 !x->plane[plane].eobs[block] ||
848 (x->sharpness == 0 && rd1 > rd2 && !xd->lossless);
849 x->sum_y_eobs[tx_size] += x->plane[plane].eobs[block];
852 args->this_rate += rate;
853 args->this_dist += dist;
854 args->this_sse += sse;
857 if (args->this_rd > args->best_rd) {
858 args->exit_early = 1;
862 args->skippable &= !x->plane[plane].eobs[block];
865 static void txfm_rd_in_plane(const VP9_COMP *cpi, MACROBLOCK *x, int *rate,
866 int64_t *distortion, int *skippable, int64_t *sse,
867 int64_t ref_best_rd, int plane, BLOCK_SIZE bsize,
868 TX_SIZE tx_size, int use_fast_coef_costing,
869 struct buf_2d *recon) {
870 MACROBLOCKD *const xd = &x->e_mbd;
871 const struct macroblockd_plane *const pd = &xd->plane[plane];
872 struct rdcost_block_args args;
876 args.best_rd = ref_best_rd;
877 args.use_fast_coef_costing = use_fast_coef_costing;
879 args.this_recon = recon;
881 if (plane == 0) xd->mi[0]->tx_size = tx_size;
883 vp9_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);
885 args.so = get_scan(xd, tx_size, get_plane_type(plane), 0);
887 vp9_foreach_transformed_block_in_plane(xd, bsize, plane, block_rd_txfm,
889 if (args.exit_early) {
891 *distortion = INT64_MAX;
895 *distortion = args.this_dist;
896 *rate = args.this_rate;
897 *sse = args.this_sse;
898 *skippable = args.skippable;
902 static void choose_largest_tx_size(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
903 int64_t *distortion, int *skip, int64_t *sse,
904 int64_t ref_best_rd, BLOCK_SIZE bs,
905 struct buf_2d *recon) {
906 const TX_SIZE max_tx_size = max_txsize_lookup[bs];
907 VP9_COMMON *const cm = &cpi->common;
908 const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
909 MACROBLOCKD *const xd = &x->e_mbd;
910 MODE_INFO *const mi = xd->mi[0];
912 mi->tx_size = VPXMIN(max_tx_size, largest_tx_size);
914 txfm_rd_in_plane(cpi, x, rate, distortion, skip, sse, ref_best_rd, 0, bs,
915 mi->tx_size, cpi->sf.use_fast_coef_costing, recon);
918 static void choose_tx_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
919 int64_t *distortion, int *skip,
920 int64_t *psse, int64_t ref_best_rd,
921 BLOCK_SIZE bs, struct buf_2d *recon) {
922 const TX_SIZE max_tx_size = max_txsize_lookup[bs];
923 VP9_COMMON *const cm = &cpi->common;
924 MACROBLOCKD *const xd = &x->e_mbd;
925 MODE_INFO *const mi = xd->mi[0];
926 vpx_prob skip_prob = vp9_get_skip_prob(cm, xd);
927 int r[TX_SIZES][2], s[TX_SIZES];
928 int64_t d[TX_SIZES], sse[TX_SIZES];
929 int64_t rd[TX_SIZES][2] = { { INT64_MAX, INT64_MAX },
930 { INT64_MAX, INT64_MAX },
931 { INT64_MAX, INT64_MAX },
932 { INT64_MAX, INT64_MAX } };
935 int64_t best_rd = ref_best_rd;
936 TX_SIZE best_tx = max_tx_size;
937 int start_tx, end_tx;
938 const int tx_size_ctx = get_tx_size_context(xd);
939 #if CONFIG_VP9_HIGHBITDEPTH
940 DECLARE_ALIGNED(16, uint16_t, recon_buf16[TX_SIZES][64 * 64]);
941 uint8_t *recon_buf[TX_SIZES];
942 for (n = 0; n < TX_SIZES; ++n) {
943 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
944 recon_buf[n] = CONVERT_TO_BYTEPTR(recon_buf16[n]);
946 recon_buf[n] = (uint8_t *)recon_buf16[n];
950 DECLARE_ALIGNED(16, uint8_t, recon_buf[TX_SIZES][64 * 64]);
951 #endif // CONFIG_VP9_HIGHBITDEPTH
953 assert(skip_prob > 0);
954 s0 = vp9_cost_bit(skip_prob, 0);
955 s1 = vp9_cost_bit(skip_prob, 1);
957 if (cm->tx_mode == TX_MODE_SELECT) {
958 start_tx = max_tx_size;
959 end_tx = VPXMAX(start_tx - cpi->sf.tx_size_search_depth, 0);
960 if (bs > BLOCK_32X32) end_tx = VPXMIN(end_tx + 1, start_tx);
962 TX_SIZE chosen_tx_size =
963 VPXMIN(max_tx_size, tx_mode_to_biggest_tx_size[cm->tx_mode]);
964 start_tx = chosen_tx_size;
965 end_tx = chosen_tx_size;
968 for (n = start_tx; n >= end_tx; n--) {
969 const int r_tx_size = cpi->tx_size_cost[max_tx_size - 1][tx_size_ctx][n];
971 struct buf_2d this_recon;
972 this_recon.buf = recon_buf[n];
973 this_recon.stride = recon->stride;
974 txfm_rd_in_plane(cpi, x, &r[n][0], &d[n], &s[n], &sse[n], best_rd, 0, bs,
975 n, cpi->sf.use_fast_coef_costing, &this_recon);
977 txfm_rd_in_plane(cpi, x, &r[n][0], &d[n], &s[n], &sse[n], best_rd, 0, bs,
978 n, cpi->sf.use_fast_coef_costing, 0);
981 if (r[n][0] < INT_MAX) {
982 r[n][1] += r_tx_size;
984 if (d[n] == INT64_MAX || r[n][0] == INT_MAX) {
985 rd[n][0] = rd[n][1] = INT64_MAX;
987 if (is_inter_block(mi)) {
988 rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, sse[n]);
989 r[n][1] -= r_tx_size;
991 rd[n][0] = RDCOST(x->rdmult, x->rddiv, s1, sse[n]);
992 rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1 + r_tx_size, sse[n]);
995 rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
996 rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
999 if (is_inter_block(mi) && !xd->lossless && !s[n] && sse[n] != INT64_MAX) {
1000 rd[n][0] = VPXMIN(rd[n][0], RDCOST(x->rdmult, x->rddiv, s1, sse[n]));
1001 rd[n][1] = VPXMIN(rd[n][1], RDCOST(x->rdmult, x->rddiv, s1, sse[n]));
1004 // Early termination in transform size search.
1005 if (cpi->sf.tx_size_search_breakout &&
1006 (rd[n][1] == INT64_MAX ||
1007 (n < (int)max_tx_size && rd[n][1] > rd[n + 1][1]) || s[n] == 1))
1010 if (rd[n][1] < best_rd) {
1015 mi->tx_size = best_tx;
1017 *distortion = d[mi->tx_size];
1018 *rate = r[mi->tx_size][cm->tx_mode == TX_MODE_SELECT];
1019 *skip = s[mi->tx_size];
1020 *psse = sse[mi->tx_size];
1022 #if CONFIG_VP9_HIGHBITDEPTH
1023 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
1024 memcpy(CONVERT_TO_SHORTPTR(recon->buf),
1025 CONVERT_TO_SHORTPTR(recon_buf[mi->tx_size]),
1026 64 * 64 * sizeof(uint16_t));
1029 memcpy(recon->buf, recon_buf[mi->tx_size], 64 * 64);
1030 #if CONFIG_VP9_HIGHBITDEPTH
1036 static void super_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1037 int64_t *distortion, int *skip, int64_t *psse,
1038 BLOCK_SIZE bs, int64_t ref_best_rd,
1039 struct buf_2d *recon) {
1040 MACROBLOCKD *xd = &x->e_mbd;
1042 int64_t *ret_sse = psse ? psse : &sse;
1044 assert(bs == xd->mi[0]->sb_type);
1046 if (cpi->sf.tx_size_search_method == USE_LARGESTALL || xd->lossless) {
1047 choose_largest_tx_size(cpi, x, rate, distortion, skip, ret_sse, ref_best_rd,
1050 choose_tx_size_from_rd(cpi, x, rate, distortion, skip, ret_sse, ref_best_rd,
1055 static int conditional_skipintra(PREDICTION_MODE mode,
1056 PREDICTION_MODE best_intra_mode) {
1057 if (mode == D117_PRED && best_intra_mode != V_PRED &&
1058 best_intra_mode != D135_PRED)
1060 if (mode == D63_PRED && best_intra_mode != V_PRED &&
1061 best_intra_mode != D45_PRED)
1063 if (mode == D207_PRED && best_intra_mode != H_PRED &&
1064 best_intra_mode != D45_PRED)
1066 if (mode == D153_PRED && best_intra_mode != H_PRED &&
1067 best_intra_mode != D135_PRED)
1072 static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int row,
1073 int col, PREDICTION_MODE *best_mode,
1074 const int *bmode_costs, ENTROPY_CONTEXT *a,
1075 ENTROPY_CONTEXT *l, int *bestrate,
1076 int *bestratey, int64_t *bestdistortion,
1077 BLOCK_SIZE bsize, int64_t rd_thresh) {
1078 PREDICTION_MODE mode;
1079 MACROBLOCKD *const xd = &x->e_mbd;
1080 int64_t best_rd = rd_thresh;
1081 struct macroblock_plane *p = &x->plane[0];
1082 struct macroblockd_plane *pd = &xd->plane[0];
1083 const int src_stride = p->src.stride;
1084 const int dst_stride = pd->dst.stride;
1085 const uint8_t *src_init = &p->src.buf[row * 4 * src_stride + col * 4];
1086 uint8_t *dst_init = &pd->dst.buf[row * 4 * src_stride + col * 4];
1087 ENTROPY_CONTEXT ta[2], tempa[2];
1088 ENTROPY_CONTEXT tl[2], templ[2];
1089 const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bsize];
1090 const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize];
1092 uint8_t best_dst[8 * 8];
1093 #if CONFIG_VP9_HIGHBITDEPTH
1094 uint16_t best_dst16[8 * 8];
1096 memcpy(ta, a, num_4x4_blocks_wide * sizeof(a[0]));
1097 memcpy(tl, l, num_4x4_blocks_high * sizeof(l[0]));
1099 xd->mi[0]->tx_size = TX_4X4;
1101 #if CONFIG_VP9_HIGHBITDEPTH
1102 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
1103 for (mode = DC_PRED; mode <= TM_PRED; ++mode) {
1106 int64_t distortion = 0;
1107 int rate = bmode_costs[mode];
1109 if (!(cpi->sf.intra_y_mode_mask[TX_4X4] & (1 << mode))) continue;
1111 // Only do the oblique modes if the best so far is
1112 // one of the neighboring directional modes
1113 if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
1114 if (conditional_skipintra(mode, *best_mode)) continue;
1117 memcpy(tempa, ta, num_4x4_blocks_wide * sizeof(ta[0]));
1118 memcpy(templ, tl, num_4x4_blocks_high * sizeof(tl[0]));
1120 for (idy = 0; idy < num_4x4_blocks_high; ++idy) {
1121 for (idx = 0; idx < num_4x4_blocks_wide; ++idx) {
1122 const int block = (row + idy) * 2 + (col + idx);
1123 const uint8_t *const src = &src_init[idx * 4 + idy * 4 * src_stride];
1124 uint8_t *const dst = &dst_init[idx * 4 + idy * 4 * dst_stride];
1125 uint16_t *const dst16 = CONVERT_TO_SHORTPTR(dst);
1126 int16_t *const src_diff =
1127 vp9_raster_block_offset_int16(BLOCK_8X8, block, p->src_diff);
1128 tran_low_t *const coeff = BLOCK_OFFSET(x->plane[0].coeff, block);
1129 xd->mi[0]->bmi[block].as_mode = mode;
1130 vp9_predict_intra_block(xd, 1, TX_4X4, mode,
1131 x->skip_encode ? src : dst,
1132 x->skip_encode ? src_stride : dst_stride, dst,
1133 dst_stride, col + idx, row + idy, 0);
1134 vpx_highbd_subtract_block(4, 4, src_diff, 8, src, src_stride, dst,
1135 dst_stride, xd->bd);
1137 const scan_order *so = &vp9_default_scan_orders[TX_4X4];
1138 const int coeff_ctx =
1139 combine_entropy_contexts(tempa[idx], templ[idy]);
1140 vp9_highbd_fwht4x4(src_diff, coeff, 8);
1141 vp9_regular_quantize_b_4x4(x, 0, block, so->scan, so->iscan);
1142 ratey += cost_coeffs(x, 0, block, TX_4X4, coeff_ctx, so->scan,
1143 so->neighbors, cpi->sf.use_fast_coef_costing);
1144 tempa[idx] = templ[idy] = (x->plane[0].eobs[block] > 0 ? 1 : 0);
1145 if (RDCOST(x->rdmult, x->rddiv, ratey, distortion) >= best_rd)
1147 vp9_highbd_iwht4x4_add(BLOCK_OFFSET(pd->dqcoeff, block), dst16,
1148 dst_stride, p->eobs[block], xd->bd);
1151 const TX_TYPE tx_type = get_tx_type_4x4(PLANE_TYPE_Y, xd, block);
1152 const scan_order *so = &vp9_scan_orders[TX_4X4][tx_type];
1153 const int coeff_ctx =
1154 combine_entropy_contexts(tempa[idx], templ[idy]);
1155 if (tx_type == DCT_DCT)
1156 vpx_highbd_fdct4x4(src_diff, coeff, 8);
1158 vp9_highbd_fht4x4(src_diff, coeff, 8, tx_type);
1159 vp9_regular_quantize_b_4x4(x, 0, block, so->scan, so->iscan);
1160 ratey += cost_coeffs(x, 0, block, TX_4X4, coeff_ctx, so->scan,
1161 so->neighbors, cpi->sf.use_fast_coef_costing);
1162 distortion += vp9_highbd_block_error_dispatch(
1163 coeff, BLOCK_OFFSET(pd->dqcoeff, block), 16,
1166 tempa[idx] = templ[idy] = (x->plane[0].eobs[block] > 0 ? 1 : 0);
1167 if (RDCOST(x->rdmult, x->rddiv, ratey, distortion) >= best_rd)
1169 vp9_highbd_iht4x4_add(tx_type, BLOCK_OFFSET(pd->dqcoeff, block),
1170 dst16, dst_stride, p->eobs[block], xd->bd);
1176 this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
1178 if (this_rd < best_rd) {
1181 *bestdistortion = distortion;
1184 memcpy(a, tempa, num_4x4_blocks_wide * sizeof(tempa[0]));
1185 memcpy(l, templ, num_4x4_blocks_high * sizeof(templ[0]));
1186 for (idy = 0; idy < num_4x4_blocks_high * 4; ++idy) {
1187 memcpy(best_dst16 + idy * 8,
1188 CONVERT_TO_SHORTPTR(dst_init + idy * dst_stride),
1189 num_4x4_blocks_wide * 4 * sizeof(uint16_t));
1194 if (best_rd >= rd_thresh || x->skip_encode) return best_rd;
1196 for (idy = 0; idy < num_4x4_blocks_high * 4; ++idy) {
1197 memcpy(CONVERT_TO_SHORTPTR(dst_init + idy * dst_stride),
1198 best_dst16 + idy * 8, num_4x4_blocks_wide * 4 * sizeof(uint16_t));
1203 #endif // CONFIG_VP9_HIGHBITDEPTH
1205 for (mode = DC_PRED; mode <= TM_PRED; ++mode) {
1208 int64_t distortion = 0;
1209 int rate = bmode_costs[mode];
1211 if (!(cpi->sf.intra_y_mode_mask[TX_4X4] & (1 << mode))) continue;
1213 // Only do the oblique modes if the best so far is
1214 // one of the neighboring directional modes
1215 if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
1216 if (conditional_skipintra(mode, *best_mode)) continue;
1219 memcpy(tempa, ta, num_4x4_blocks_wide * sizeof(ta[0]));
1220 memcpy(templ, tl, num_4x4_blocks_high * sizeof(tl[0]));
1222 for (idy = 0; idy < num_4x4_blocks_high; ++idy) {
1223 for (idx = 0; idx < num_4x4_blocks_wide; ++idx) {
1224 const int block = (row + idy) * 2 + (col + idx);
1225 const uint8_t *const src = &src_init[idx * 4 + idy * 4 * src_stride];
1226 uint8_t *const dst = &dst_init[idx * 4 + idy * 4 * dst_stride];
1227 int16_t *const src_diff =
1228 vp9_raster_block_offset_int16(BLOCK_8X8, block, p->src_diff);
1229 tran_low_t *const coeff = BLOCK_OFFSET(x->plane[0].coeff, block);
1230 xd->mi[0]->bmi[block].as_mode = mode;
1231 vp9_predict_intra_block(xd, 1, TX_4X4, mode, x->skip_encode ? src : dst,
1232 x->skip_encode ? src_stride : dst_stride, dst,
1233 dst_stride, col + idx, row + idy, 0);
1234 vpx_subtract_block(4, 4, src_diff, 8, src, src_stride, dst, dst_stride);
1237 const scan_order *so = &vp9_default_scan_orders[TX_4X4];
1238 const int coeff_ctx =
1239 combine_entropy_contexts(tempa[idx], templ[idy]);
1240 vp9_fwht4x4(src_diff, coeff, 8);
1241 vp9_regular_quantize_b_4x4(x, 0, block, so->scan, so->iscan);
1242 ratey += cost_coeffs(x, 0, block, TX_4X4, coeff_ctx, so->scan,
1243 so->neighbors, cpi->sf.use_fast_coef_costing);
1244 tempa[idx] = templ[idy] = (x->plane[0].eobs[block] > 0) ? 1 : 0;
1245 if (RDCOST(x->rdmult, x->rddiv, ratey, distortion) >= best_rd)
1247 vp9_iwht4x4_add(BLOCK_OFFSET(pd->dqcoeff, block), dst, dst_stride,
1251 const TX_TYPE tx_type = get_tx_type_4x4(PLANE_TYPE_Y, xd, block);
1252 const scan_order *so = &vp9_scan_orders[TX_4X4][tx_type];
1253 const int coeff_ctx =
1254 combine_entropy_contexts(tempa[idx], templ[idy]);
1255 vp9_fht4x4(src_diff, coeff, 8, tx_type);
1256 vp9_regular_quantize_b_4x4(x, 0, block, so->scan, so->iscan);
1257 ratey += cost_coeffs(x, 0, block, TX_4X4, coeff_ctx, so->scan,
1258 so->neighbors, cpi->sf.use_fast_coef_costing);
1259 tempa[idx] = templ[idy] = (x->plane[0].eobs[block] > 0) ? 1 : 0;
1260 distortion += vp9_block_error(coeff, BLOCK_OFFSET(pd->dqcoeff, block),
1263 if (RDCOST(x->rdmult, x->rddiv, ratey, distortion) >= best_rd)
1265 vp9_iht4x4_add(tx_type, BLOCK_OFFSET(pd->dqcoeff, block), dst,
1266 dst_stride, p->eobs[block]);
1272 this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
1274 if (this_rd < best_rd) {
1277 *bestdistortion = distortion;
1280 memcpy(a, tempa, num_4x4_blocks_wide * sizeof(tempa[0]));
1281 memcpy(l, templ, num_4x4_blocks_high * sizeof(templ[0]));
1282 for (idy = 0; idy < num_4x4_blocks_high * 4; ++idy)
1283 memcpy(best_dst + idy * 8, dst_init + idy * dst_stride,
1284 num_4x4_blocks_wide * 4);
1289 if (best_rd >= rd_thresh || x->skip_encode) return best_rd;
1291 for (idy = 0; idy < num_4x4_blocks_high * 4; ++idy)
1292 memcpy(dst_init + idy * dst_stride, best_dst + idy * 8,
1293 num_4x4_blocks_wide * 4);
1298 static int64_t rd_pick_intra_sub_8x8_y_mode(VP9_COMP *cpi, MACROBLOCK *mb,
1299 int *rate, int *rate_y,
1300 int64_t *distortion,
1303 const MACROBLOCKD *const xd = &mb->e_mbd;
1304 MODE_INFO *const mic = xd->mi[0];
1305 const MODE_INFO *above_mi = xd->above_mi;
1306 const MODE_INFO *left_mi = xd->left_mi;
1307 const BLOCK_SIZE bsize = xd->mi[0]->sb_type;
1308 const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bsize];
1309 const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize];
1312 int64_t total_distortion = 0;
1314 int64_t total_rd = 0;
1315 const int *bmode_costs = cpi->mbmode_cost;
1317 // Pick modes for each sub-block (of size 4x4, 4x8, or 8x4) in an 8x8 block.
1318 for (idy = 0; idy < 2; idy += num_4x4_blocks_high) {
1319 for (idx = 0; idx < 2; idx += num_4x4_blocks_wide) {
1320 PREDICTION_MODE best_mode = DC_PRED;
1321 int r = INT_MAX, ry = INT_MAX;
1322 int64_t d = INT64_MAX, this_rd = INT64_MAX;
1324 if (cpi->common.frame_type == KEY_FRAME) {
1325 const PREDICTION_MODE A = vp9_above_block_mode(mic, above_mi, i);
1326 const PREDICTION_MODE L = vp9_left_block_mode(mic, left_mi, i);
1328 bmode_costs = cpi->y_mode_costs[A][L];
1331 this_rd = rd_pick_intra4x4block(
1332 cpi, mb, idy, idx, &best_mode, bmode_costs,
1333 xd->plane[0].above_context + idx, xd->plane[0].left_context + idy, &r,
1334 &ry, &d, bsize, best_rd - total_rd);
1336 if (this_rd >= best_rd - total_rd) return INT64_MAX;
1338 total_rd += this_rd;
1340 total_distortion += d;
1343 mic->bmi[i].as_mode = best_mode;
1344 for (j = 1; j < num_4x4_blocks_high; ++j)
1345 mic->bmi[i + j * 2].as_mode = best_mode;
1346 for (j = 1; j < num_4x4_blocks_wide; ++j)
1347 mic->bmi[i + j].as_mode = best_mode;
1349 if (total_rd >= best_rd) return INT64_MAX;
1354 *rate_y = tot_rate_y;
1355 *distortion = total_distortion;
1356 mic->mode = mic->bmi[3].as_mode;
1358 return RDCOST(mb->rdmult, mb->rddiv, cost, total_distortion);
1361 // This function is used only for intra_only frames
1362 static int64_t rd_pick_intra_sby_mode(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1363 int *rate_tokenonly, int64_t *distortion,
1364 int *skippable, BLOCK_SIZE bsize,
1366 PREDICTION_MODE mode;
1367 PREDICTION_MODE mode_selected = DC_PRED;
1368 MACROBLOCKD *const xd = &x->e_mbd;
1369 MODE_INFO *const mic = xd->mi[0];
1370 int this_rate, this_rate_tokenonly, s;
1371 int64_t this_distortion, this_rd;
1372 TX_SIZE best_tx = TX_4X4;
1374 const MODE_INFO *above_mi = xd->above_mi;
1375 const MODE_INFO *left_mi = xd->left_mi;
1376 const PREDICTION_MODE A = vp9_above_block_mode(mic, above_mi, 0);
1377 const PREDICTION_MODE L = vp9_left_block_mode(mic, left_mi, 0);
1378 bmode_costs = cpi->y_mode_costs[A][L];
1380 memset(x->skip_txfm, SKIP_TXFM_NONE, sizeof(x->skip_txfm));
1381 /* Y Search for intra prediction mode */
1382 for (mode = DC_PRED; mode <= TM_PRED; mode++) {
1383 if (cpi->sf.use_nonrd_pick_mode) {
1384 // These speed features are turned on in hybrid non-RD and RD mode
1385 // for key frame coding in the context of real-time setting.
1386 if (conditional_skipintra(mode, mode_selected)) continue;
1387 if (*skippable) break;
1392 super_block_yrd(cpi, x, &this_rate_tokenonly, &this_distortion, &s, NULL,
1393 bsize, best_rd, /*recon = */ 0);
1395 if (this_rate_tokenonly == INT_MAX) continue;
1397 this_rate = this_rate_tokenonly + bmode_costs[mode];
1398 this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);
1400 if (this_rd < best_rd) {
1401 mode_selected = mode;
1403 best_tx = mic->tx_size;
1405 *rate_tokenonly = this_rate_tokenonly;
1406 *distortion = this_distortion;
1411 mic->mode = mode_selected;
1412 mic->tx_size = best_tx;
1417 // Return value 0: early termination triggered, no valid rd cost available;
1418 // 1: rd cost values are valid.
1419 static int super_block_uvrd(const VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1420 int64_t *distortion, int *skippable, int64_t *sse,
1421 BLOCK_SIZE bsize, int64_t ref_best_rd) {
1422 MACROBLOCKD *const xd = &x->e_mbd;
1423 MODE_INFO *const mi = xd->mi[0];
1424 const TX_SIZE uv_tx_size = get_uv_tx_size(mi, &xd->plane[1]);
1426 int pnrate = 0, pnskip = 1;
1427 int64_t pndist = 0, pnsse = 0;
1428 int is_cost_valid = 1;
1430 if (ref_best_rd < 0) is_cost_valid = 0;
1432 if (is_inter_block(mi) && is_cost_valid) {
1434 for (plane = 1; plane < MAX_MB_PLANE; ++plane)
1435 vp9_subtract_plane(x, bsize, plane);
1443 for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
1444 txfm_rd_in_plane(cpi, x, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
1445 plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing,
1447 if (pnrate == INT_MAX) {
1452 *distortion += pndist;
1454 *skippable &= pnskip;
1457 if (!is_cost_valid) {
1460 *distortion = INT64_MAX;
1465 return is_cost_valid;
1468 static int64_t rd_pick_intra_sbuv_mode(VP9_COMP *cpi, MACROBLOCK *x,
1469 PICK_MODE_CONTEXT *ctx, int *rate,
1470 int *rate_tokenonly, int64_t *distortion,
1471 int *skippable, BLOCK_SIZE bsize,
1472 TX_SIZE max_tx_size) {
1473 MACROBLOCKD *xd = &x->e_mbd;
1474 PREDICTION_MODE mode;
1475 PREDICTION_MODE mode_selected = DC_PRED;
1476 int64_t best_rd = INT64_MAX, this_rd;
1477 int this_rate_tokenonly, this_rate, s;
1478 int64_t this_distortion, this_sse;
1480 memset(x->skip_txfm, SKIP_TXFM_NONE, sizeof(x->skip_txfm));
1481 for (mode = DC_PRED; mode <= TM_PRED; ++mode) {
1482 if (!(cpi->sf.intra_uv_mode_mask[max_tx_size] & (1 << mode))) continue;
1483 #if CONFIG_BETTER_HW_COMPATIBILITY && CONFIG_VP9_HIGHBITDEPTH
1484 if ((xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) &&
1485 (xd->above_mi == NULL || xd->left_mi == NULL) && need_top_left[mode])
1487 #endif // CONFIG_BETTER_HW_COMPATIBILITY && CONFIG_VP9_HIGHBITDEPTH
1489 xd->mi[0]->uv_mode = mode;
1491 if (!super_block_uvrd(cpi, x, &this_rate_tokenonly, &this_distortion, &s,
1492 &this_sse, bsize, best_rd))
1495 this_rate_tokenonly +
1496 cpi->intra_uv_mode_cost[cpi->common.frame_type][xd->mi[0]->mode][mode];
1497 this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);
1499 if (this_rd < best_rd) {
1500 mode_selected = mode;
1503 *rate_tokenonly = this_rate_tokenonly;
1504 *distortion = this_distortion;
1506 if (!x->select_tx_size) swap_block_ptr(x, ctx, 2, 0, 1, MAX_MB_PLANE);
1510 xd->mi[0]->uv_mode = mode_selected;
1514 static int64_t rd_sbuv_dcpred(const VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1515 int *rate_tokenonly, int64_t *distortion,
1516 int *skippable, BLOCK_SIZE bsize) {
1517 const VP9_COMMON *cm = &cpi->common;
1520 x->e_mbd.mi[0]->uv_mode = DC_PRED;
1521 memset(x->skip_txfm, SKIP_TXFM_NONE, sizeof(x->skip_txfm));
1522 super_block_uvrd(cpi, x, rate_tokenonly, distortion, skippable, &unused,
1526 cpi->intra_uv_mode_cost[cm->frame_type][x->e_mbd.mi[0]->mode][DC_PRED];
1527 return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
1530 static void choose_intra_uv_mode(VP9_COMP *cpi, MACROBLOCK *const x,
1531 PICK_MODE_CONTEXT *ctx, BLOCK_SIZE bsize,
1532 TX_SIZE max_tx_size, int *rate_uv,
1533 int *rate_uv_tokenonly, int64_t *dist_uv,
1534 int *skip_uv, PREDICTION_MODE *mode_uv) {
1535 // Use an estimated rd for uv_intra based on DC_PRED if the
1536 // appropriate speed flag is set.
1537 if (cpi->sf.use_uv_intra_rd_estimate) {
1538 rd_sbuv_dcpred(cpi, x, rate_uv, rate_uv_tokenonly, dist_uv, skip_uv,
1539 bsize < BLOCK_8X8 ? BLOCK_8X8 : bsize);
1540 // Else do a proper rd search for each possible transform size that may
1541 // be considered in the main rd loop.
1543 rd_pick_intra_sbuv_mode(cpi, x, ctx, rate_uv, rate_uv_tokenonly, dist_uv,
1544 skip_uv, bsize < BLOCK_8X8 ? BLOCK_8X8 : bsize,
1547 *mode_uv = x->e_mbd.mi[0]->uv_mode;
1550 static int cost_mv_ref(const VP9_COMP *cpi, PREDICTION_MODE mode,
1552 assert(is_inter_mode(mode));
1553 return cpi->inter_mode_cost[mode_context][INTER_OFFSET(mode)];
1556 static int set_and_cost_bmi_mvs(VP9_COMP *cpi, MACROBLOCK *x, MACROBLOCKD *xd,
1557 int i, PREDICTION_MODE mode, int_mv this_mv[2],
1558 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES],
1559 int_mv seg_mvs[MAX_REF_FRAMES],
1560 int_mv *best_ref_mv[2], const int *mvjcost,
1562 MODE_INFO *const mi = xd->mi[0];
1563 const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
1566 const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[mi->sb_type];
1567 const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[mi->sb_type];
1568 const int is_compound = has_second_ref(mi);
1572 this_mv[0].as_int = seg_mvs[mi->ref_frame[0]].as_int;
1573 thismvcost += vp9_mv_bit_cost(&this_mv[0].as_mv, &best_ref_mv[0]->as_mv,
1574 mvjcost, mvcost, MV_COST_WEIGHT_SUB);
1576 this_mv[1].as_int = seg_mvs[mi->ref_frame[1]].as_int;
1577 thismvcost += vp9_mv_bit_cost(&this_mv[1].as_mv, &best_ref_mv[1]->as_mv,
1578 mvjcost, mvcost, MV_COST_WEIGHT_SUB);
1583 this_mv[0].as_int = frame_mv[mode][mi->ref_frame[0]].as_int;
1585 this_mv[1].as_int = frame_mv[mode][mi->ref_frame[1]].as_int;
1588 assert(mode == ZEROMV);
1589 this_mv[0].as_int = 0;
1590 if (is_compound) this_mv[1].as_int = 0;
1594 mi->bmi[i].as_mv[0].as_int = this_mv[0].as_int;
1595 if (is_compound) mi->bmi[i].as_mv[1].as_int = this_mv[1].as_int;
1597 mi->bmi[i].as_mode = mode;
1599 for (idy = 0; idy < num_4x4_blocks_high; ++idy)
1600 for (idx = 0; idx < num_4x4_blocks_wide; ++idx)
1601 memmove(&mi->bmi[i + idy * 2 + idx], &mi->bmi[i], sizeof(mi->bmi[i]));
1603 return cost_mv_ref(cpi, mode, mbmi_ext->mode_context[mi->ref_frame[0]]) +
1607 static int64_t encode_inter_mb_segment(VP9_COMP *cpi, MACROBLOCK *x,
1608 int64_t best_yrd, int i, int *labelyrate,
1609 int64_t *distortion, int64_t *sse,
1610 ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl,
1611 int mi_row, int mi_col) {
1613 MACROBLOCKD *xd = &x->e_mbd;
1614 struct macroblockd_plane *const pd = &xd->plane[0];
1615 struct macroblock_plane *const p = &x->plane[0];
1616 MODE_INFO *const mi = xd->mi[0];
1617 const BLOCK_SIZE plane_bsize = get_plane_block_size(mi->sb_type, pd);
1618 const int width = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
1619 const int height = 4 * num_4x4_blocks_high_lookup[plane_bsize];
1622 const uint8_t *const src =
1623 &p->src.buf[vp9_raster_block_offset(BLOCK_8X8, i, p->src.stride)];
1624 uint8_t *const dst =
1625 &pd->dst.buf[vp9_raster_block_offset(BLOCK_8X8, i, pd->dst.stride)];
1626 int64_t thisdistortion = 0, thissse = 0;
1627 int thisrate = 0, ref;
1628 const scan_order *so = &vp9_default_scan_orders[TX_4X4];
1629 const int is_compound = has_second_ref(mi);
1630 const InterpKernel *kernel = vp9_filter_kernels[mi->interp_filter];
1632 for (ref = 0; ref < 1 + is_compound; ++ref) {
1633 const int bw = b_width_log2_lookup[BLOCK_8X8];
1634 const int h = 4 * (i >> bw);
1635 const int w = 4 * (i & ((1 << bw) - 1));
1636 const struct scale_factors *sf = &xd->block_refs[ref]->sf;
1637 int y_stride = pd->pre[ref].stride;
1638 uint8_t *pre = pd->pre[ref].buf + (h * pd->pre[ref].stride + w);
1640 if (vp9_is_scaled(sf)) {
1641 const int x_start = (-xd->mb_to_left_edge >> (3 + pd->subsampling_x));
1642 const int y_start = (-xd->mb_to_top_edge >> (3 + pd->subsampling_y));
1644 y_stride = xd->block_refs[ref]->buf->y_stride;
1645 pre = xd->block_refs[ref]->buf->y_buffer;
1646 pre += scaled_buffer_offset(x_start + w, y_start + h, y_stride, sf);
1648 #if CONFIG_VP9_HIGHBITDEPTH
1649 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
1650 vp9_highbd_build_inter_predictor(
1651 CONVERT_TO_SHORTPTR(pre), y_stride, CONVERT_TO_SHORTPTR(dst),
1652 pd->dst.stride, &mi->bmi[i].as_mv[ref].as_mv,
1653 &xd->block_refs[ref]->sf, width, height, ref, kernel, MV_PRECISION_Q3,
1654 mi_col * MI_SIZE + 4 * (i % 2), mi_row * MI_SIZE + 4 * (i / 2),
1657 vp9_build_inter_predictor(
1658 pre, y_stride, dst, pd->dst.stride, &mi->bmi[i].as_mv[ref].as_mv,
1659 &xd->block_refs[ref]->sf, width, height, ref, kernel, MV_PRECISION_Q3,
1660 mi_col * MI_SIZE + 4 * (i % 2), mi_row * MI_SIZE + 4 * (i / 2));
1663 vp9_build_inter_predictor(
1664 pre, y_stride, dst, pd->dst.stride, &mi->bmi[i].as_mv[ref].as_mv,
1665 &xd->block_refs[ref]->sf, width, height, ref, kernel, MV_PRECISION_Q3,
1666 mi_col * MI_SIZE + 4 * (i % 2), mi_row * MI_SIZE + 4 * (i / 2));
1667 #endif // CONFIG_VP9_HIGHBITDEPTH
1670 #if CONFIG_VP9_HIGHBITDEPTH
1671 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
1672 vpx_highbd_subtract_block(
1673 height, width, vp9_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff),
1674 8, src, p->src.stride, dst, pd->dst.stride, xd->bd);
1676 vpx_subtract_block(height, width,
1677 vp9_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff),
1678 8, src, p->src.stride, dst, pd->dst.stride);
1681 vpx_subtract_block(height, width,
1682 vp9_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff),
1683 8, src, p->src.stride, dst, pd->dst.stride);
1684 #endif // CONFIG_VP9_HIGHBITDEPTH
1687 for (idy = 0; idy < height / 4; ++idy) {
1688 for (idx = 0; idx < width / 4; ++idx) {
1689 #if CONFIG_VP9_HIGHBITDEPTH
1690 const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8;
1692 int64_t ssz, rd, rd1, rd2;
1695 k += (idy * 2 + idx);
1696 coeff_ctx = combine_entropy_contexts(ta[k & 1], tl[k >> 1]);
1697 coeff = BLOCK_OFFSET(p->coeff, k);
1698 x->fwd_txfm4x4(vp9_raster_block_offset_int16(BLOCK_8X8, k, p->src_diff),
1700 vp9_regular_quantize_b_4x4(x, 0, k, so->scan, so->iscan);
1701 #if CONFIG_VP9_HIGHBITDEPTH
1702 thisdistortion += vp9_highbd_block_error_dispatch(
1703 coeff, BLOCK_OFFSET(pd->dqcoeff, k), 16, &ssz, bd);
1706 vp9_block_error(coeff, BLOCK_OFFSET(pd->dqcoeff, k), 16, &ssz);
1707 #endif // CONFIG_VP9_HIGHBITDEPTH
1709 thisrate += cost_coeffs(x, 0, k, TX_4X4, coeff_ctx, so->scan,
1710 so->neighbors, cpi->sf.use_fast_coef_costing);
1711 ta[k & 1] = tl[k >> 1] = (x->plane[0].eobs[k] > 0) ? 1 : 0;
1712 rd1 = RDCOST(x->rdmult, x->rddiv, thisrate, thisdistortion >> 2);
1713 rd2 = RDCOST(x->rdmult, x->rddiv, 0, thissse >> 2);
1714 rd = VPXMIN(rd1, rd2);
1715 if (rd >= best_yrd) return INT64_MAX;
1719 *distortion = thisdistortion >> 2;
1720 *labelyrate = thisrate;
1721 *sse = thissse >> 2;
1723 return RDCOST(x->rdmult, x->rddiv, *labelyrate, *distortion);
1734 ENTROPY_CONTEXT ta[2];
1735 ENTROPY_CONTEXT tl[2];
1747 PREDICTION_MODE modes[4];
1748 SEG_RDSTAT rdstat[4][INTER_MODES];
1752 static INLINE int mv_check_bounds(const MvLimits *mv_limits, const MV *mv) {
1753 return (mv->row >> 3) < mv_limits->row_min ||
1754 (mv->row >> 3) > mv_limits->row_max ||
1755 (mv->col >> 3) < mv_limits->col_min ||
1756 (mv->col >> 3) > mv_limits->col_max;
1759 static INLINE void mi_buf_shift(MACROBLOCK *x, int i) {
1760 MODE_INFO *const mi = x->e_mbd.mi[0];
1761 struct macroblock_plane *const p = &x->plane[0];
1762 struct macroblockd_plane *const pd = &x->e_mbd.plane[0];
1765 &p->src.buf[vp9_raster_block_offset(BLOCK_8X8, i, p->src.stride)];
1766 assert(((intptr_t)pd->pre[0].buf & 0x7) == 0);
1768 &pd->pre[0].buf[vp9_raster_block_offset(BLOCK_8X8, i, pd->pre[0].stride)];
1769 if (has_second_ref(mi))
1772 .buf[vp9_raster_block_offset(BLOCK_8X8, i, pd->pre[1].stride)];
1775 static INLINE void mi_buf_restore(MACROBLOCK *x, struct buf_2d orig_src,
1776 struct buf_2d orig_pre[2]) {
1777 MODE_INFO *mi = x->e_mbd.mi[0];
1778 x->plane[0].src = orig_src;
1779 x->e_mbd.plane[0].pre[0] = orig_pre[0];
1780 if (has_second_ref(mi)) x->e_mbd.plane[0].pre[1] = orig_pre[1];
1783 static INLINE int mv_has_subpel(const MV *mv) {
1784 return (mv->row & 0x0F) || (mv->col & 0x0F);
1787 // Check if NEARESTMV/NEARMV/ZEROMV is the cheapest way encode zero motion.
1788 // TODO(aconverse): Find out if this is still productive then clean up or remove
1789 static int check_best_zero_mv(const VP9_COMP *cpi,
1790 const uint8_t mode_context[MAX_REF_FRAMES],
1791 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES],
1793 const MV_REFERENCE_FRAME ref_frames[2]) {
1794 if ((this_mode == NEARMV || this_mode == NEARESTMV || this_mode == ZEROMV) &&
1795 frame_mv[this_mode][ref_frames[0]].as_int == 0 &&
1796 (ref_frames[1] == NONE ||
1797 frame_mv[this_mode][ref_frames[1]].as_int == 0)) {
1798 int rfc = mode_context[ref_frames[0]];
1799 int c1 = cost_mv_ref(cpi, NEARMV, rfc);
1800 int c2 = cost_mv_ref(cpi, NEARESTMV, rfc);
1801 int c3 = cost_mv_ref(cpi, ZEROMV, rfc);
1803 if (this_mode == NEARMV) {
1804 if (c1 > c3) return 0;
1805 } else if (this_mode == NEARESTMV) {
1806 if (c2 > c3) return 0;
1808 assert(this_mode == ZEROMV);
1809 if (ref_frames[1] == NONE) {
1810 if ((c3 >= c2 && frame_mv[NEARESTMV][ref_frames[0]].as_int == 0) ||
1811 (c3 >= c1 && frame_mv[NEARMV][ref_frames[0]].as_int == 0))
1814 if ((c3 >= c2 && frame_mv[NEARESTMV][ref_frames[0]].as_int == 0 &&
1815 frame_mv[NEARESTMV][ref_frames[1]].as_int == 0) ||
1816 (c3 >= c1 && frame_mv[NEARMV][ref_frames[0]].as_int == 0 &&
1817 frame_mv[NEARMV][ref_frames[1]].as_int == 0))
1825 static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
1826 int_mv *frame_mv, int mi_row, int mi_col,
1827 int_mv single_newmv[MAX_REF_FRAMES],
1829 const VP9_COMMON *const cm = &cpi->common;
1830 const int pw = 4 * num_4x4_blocks_wide_lookup[bsize];
1831 const int ph = 4 * num_4x4_blocks_high_lookup[bsize];
1832 MACROBLOCKD *xd = &x->e_mbd;
1833 MODE_INFO *mi = xd->mi[0];
1834 const int refs[2] = { mi->ref_frame[0],
1835 mi->ref_frame[1] < 0 ? 0 : mi->ref_frame[1] };
1838 const InterpKernel *kernel = vp9_filter_kernels[mi->interp_filter];
1839 struct scale_factors sf;
1841 // Do joint motion search in compound mode to get more accurate mv.
1842 struct buf_2d backup_yv12[2][MAX_MB_PLANE];
1843 uint32_t last_besterr[2] = { UINT_MAX, UINT_MAX };
1844 const YV12_BUFFER_CONFIG *const scaled_ref_frame[2] = {
1845 vp9_get_scaled_ref_frame(cpi, mi->ref_frame[0]),
1846 vp9_get_scaled_ref_frame(cpi, mi->ref_frame[1])
1849 // Prediction buffer from second frame.
1850 #if CONFIG_VP9_HIGHBITDEPTH
1851 DECLARE_ALIGNED(16, uint16_t, second_pred_alloc_16[64 * 64]);
1852 uint8_t *second_pred;
1854 DECLARE_ALIGNED(16, uint8_t, second_pred[64 * 64]);
1855 #endif // CONFIG_VP9_HIGHBITDEPTH
1857 for (ref = 0; ref < 2; ++ref) {
1858 ref_mv[ref] = x->mbmi_ext->ref_mvs[refs[ref]][0];
1860 if (scaled_ref_frame[ref]) {
1862 // Swap out the reference frame for a version that's been scaled to
1863 // match the resolution of the current frame, allowing the existing
1864 // motion search code to be used without additional modifications.
1865 for (i = 0; i < MAX_MB_PLANE; i++)
1866 backup_yv12[ref][i] = xd->plane[i].pre[ref];
1867 vp9_setup_pre_planes(xd, ref, scaled_ref_frame[ref], mi_row, mi_col,
1871 frame_mv[refs[ref]].as_int = single_newmv[refs[ref]].as_int;
1874 // Since we have scaled the reference frames to match the size of the current
1875 // frame we must use a unit scaling factor during mode selection.
1876 #if CONFIG_VP9_HIGHBITDEPTH
1877 vp9_setup_scale_factors_for_frame(&sf, cm->width, cm->height, cm->width,
1878 cm->height, cm->use_highbitdepth);
1880 vp9_setup_scale_factors_for_frame(&sf, cm->width, cm->height, cm->width,
1882 #endif // CONFIG_VP9_HIGHBITDEPTH
1884 // Allow joint search multiple times iteratively for each reference frame
1885 // and break out of the search loop if it couldn't find a better mv.
1886 for (ite = 0; ite < 4; ite++) {
1887 struct buf_2d ref_yv12[2];
1888 uint32_t bestsme = UINT_MAX;
1889 int sadpb = x->sadperbit16;
1891 int search_range = 3;
1893 const MvLimits tmp_mv_limits = x->mv_limits;
1894 int id = ite % 2; // Even iterations search in the first reference frame,
1895 // odd iterations search in the second. The predictor
1896 // found for the 'other' reference frame is factored in.
1898 // Initialized here because of compiler problem in Visual Studio.
1899 ref_yv12[0] = xd->plane[0].pre[0];
1900 ref_yv12[1] = xd->plane[0].pre[1];
1902 // Get the prediction block from the 'other' reference frame.
1903 #if CONFIG_VP9_HIGHBITDEPTH
1904 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
1905 second_pred = CONVERT_TO_BYTEPTR(second_pred_alloc_16);
1906 vp9_highbd_build_inter_predictor(
1907 CONVERT_TO_SHORTPTR(ref_yv12[!id].buf), ref_yv12[!id].stride,
1908 second_pred_alloc_16, pw, &frame_mv[refs[!id]].as_mv, &sf, pw, ph, 0,
1909 kernel, MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd->bd);
1911 second_pred = (uint8_t *)second_pred_alloc_16;
1912 vp9_build_inter_predictor(ref_yv12[!id].buf, ref_yv12[!id].stride,
1913 second_pred, pw, &frame_mv[refs[!id]].as_mv,
1914 &sf, pw, ph, 0, kernel, MV_PRECISION_Q3,
1915 mi_col * MI_SIZE, mi_row * MI_SIZE);
1918 vp9_build_inter_predictor(ref_yv12[!id].buf, ref_yv12[!id].stride,
1919 second_pred, pw, &frame_mv[refs[!id]].as_mv, &sf,
1920 pw, ph, 0, kernel, MV_PRECISION_Q3,
1921 mi_col * MI_SIZE, mi_row * MI_SIZE);
1922 #endif // CONFIG_VP9_HIGHBITDEPTH
1924 // Do compound motion search on the current reference frame.
1925 if (id) xd->plane[0].pre[0] = ref_yv12[id];
1926 vp9_set_mv_search_range(&x->mv_limits, &ref_mv[id].as_mv);
1928 // Use the mv result from the single mode as mv predictor.
1929 tmp_mv = frame_mv[refs[id]].as_mv;
1934 // Small-range full-pixel motion search.
1935 bestsme = vp9_refining_search_8p_c(x, &tmp_mv, sadpb, search_range,
1936 &cpi->fn_ptr[bsize], &ref_mv[id].as_mv,
1938 if (bestsme < UINT_MAX)
1939 bestsme = vp9_get_mvpred_av_var(x, &tmp_mv, &ref_mv[id].as_mv,
1940 second_pred, &cpi->fn_ptr[bsize], 1);
1942 x->mv_limits = tmp_mv_limits;
1944 if (bestsme < UINT_MAX) {
1945 uint32_t dis; /* TODO: use dis in distortion calculation later. */
1947 bestsme = cpi->find_fractional_mv_step(
1948 x, &tmp_mv, &ref_mv[id].as_mv, cpi->common.allow_high_precision_mv,
1949 x->errorperbit, &cpi->fn_ptr[bsize], 0,
1950 cpi->sf.mv.subpel_search_level, NULL, x->nmvjointcost, x->mvcost,
1951 &dis, &sse, second_pred, pw, ph, cpi->sf.use_accurate_subpel_search);
1954 // Restore the pointer to the first (possibly scaled) prediction buffer.
1955 if (id) xd->plane[0].pre[0] = ref_yv12[0];
1957 if (bestsme < last_besterr[id]) {
1958 frame_mv[refs[id]].as_mv = tmp_mv;
1959 last_besterr[id] = bestsme;
1967 for (ref = 0; ref < 2; ++ref) {
1968 if (scaled_ref_frame[ref]) {
1969 // Restore the prediction frame pointers to their unscaled versions.
1971 for (i = 0; i < MAX_MB_PLANE; i++)
1972 xd->plane[i].pre[ref] = backup_yv12[ref][i];
1975 *rate_mv += vp9_mv_bit_cost(&frame_mv[refs[ref]].as_mv,
1976 &x->mbmi_ext->ref_mvs[refs[ref]][0].as_mv,
1977 x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
1981 static int64_t rd_pick_best_sub8x8_mode(
1982 VP9_COMP *cpi, MACROBLOCK *x, int_mv *best_ref_mv,
1983 int_mv *second_best_ref_mv, int64_t best_rd, int *returntotrate,
1984 int *returnyrate, int64_t *returndistortion, int *skippable, int64_t *psse,
1985 int mvthresh, int_mv seg_mvs[4][MAX_REF_FRAMES], BEST_SEG_INFO *bsi_buf,
1986 int filter_idx, int mi_row, int mi_col) {
1988 BEST_SEG_INFO *bsi = bsi_buf + filter_idx;
1989 MACROBLOCKD *xd = &x->e_mbd;
1990 MODE_INFO *mi = xd->mi[0];
1992 int k, br = 0, idx, idy;
1993 int64_t bd = 0, block_sse = 0;
1994 PREDICTION_MODE this_mode;
1995 VP9_COMMON *cm = &cpi->common;
1996 struct macroblock_plane *const p = &x->plane[0];
1997 struct macroblockd_plane *const pd = &xd->plane[0];
1998 const int label_count = 4;
1999 int64_t this_segment_rd = 0;
2000 int label_mv_thresh;
2001 int segmentyrate = 0;
2002 const BLOCK_SIZE bsize = mi->sb_type;
2003 const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bsize];
2004 const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize];
2005 const int pw = num_4x4_blocks_wide << 2;
2006 const int ph = num_4x4_blocks_high << 2;
2007 ENTROPY_CONTEXT t_above[2], t_left[2];
2008 int subpelmv = 1, have_ref = 0;
2009 SPEED_FEATURES *const sf = &cpi->sf;
2010 const int has_second_rf = has_second_ref(mi);
2011 const int inter_mode_mask = sf->inter_mode_mask[bsize];
2012 MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
2016 bsi->segment_rd = best_rd;
2017 bsi->ref_mv[0] = best_ref_mv;
2018 bsi->ref_mv[1] = second_best_ref_mv;
2019 bsi->mvp.as_int = best_ref_mv->as_int;
2020 bsi->mvthresh = mvthresh;
2022 for (i = 0; i < 4; i++) bsi->modes[i] = ZEROMV;
2024 memcpy(t_above, pd->above_context, sizeof(t_above));
2025 memcpy(t_left, pd->left_context, sizeof(t_left));
2027 // 64 makes this threshold really big effectively
2028 // making it so that we very rarely check mvs on
2029 // segments. setting this to 1 would make mv thresh
2030 // roughly equal to what it is for macroblocks
2031 label_mv_thresh = 1 * bsi->mvthresh / label_count;
2033 // Segmentation method overheads
2034 for (idy = 0; idy < 2; idy += num_4x4_blocks_high) {
2035 for (idx = 0; idx < 2; idx += num_4x4_blocks_wide) {
2036 // TODO(jingning,rbultje): rewrite the rate-distortion optimization
2037 // loop for 4x4/4x8/8x4 block coding. to be replaced with new rd loop
2038 int_mv mode_mv[MB_MODE_COUNT][2];
2039 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
2040 PREDICTION_MODE mode_selected = ZEROMV;
2041 int64_t best_rd = INT64_MAX;
2042 const int i = idy * 2 + idx;
2045 for (ref = 0; ref < 1 + has_second_rf; ++ref) {
2046 const MV_REFERENCE_FRAME frame = mi->ref_frame[ref];
2047 frame_mv[ZEROMV][frame].as_int = 0;
2048 vp9_append_sub8x8_mvs_for_idx(
2049 cm, xd, i, ref, mi_row, mi_col, &frame_mv[NEARESTMV][frame],
2050 &frame_mv[NEARMV][frame], mbmi_ext->mode_context);
2053 // search for the best motion vector on this segment
2054 for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode) {
2055 const struct buf_2d orig_src = x->plane[0].src;
2056 struct buf_2d orig_pre[2];
2058 mode_idx = INTER_OFFSET(this_mode);
2059 bsi->rdstat[i][mode_idx].brdcost = INT64_MAX;
2060 if (!(inter_mode_mask & (1 << this_mode))) continue;
2062 if (!check_best_zero_mv(cpi, mbmi_ext->mode_context, frame_mv,
2063 this_mode, mi->ref_frame))
2066 memcpy(orig_pre, pd->pre, sizeof(orig_pre));
2067 memcpy(bsi->rdstat[i][mode_idx].ta, t_above,
2068 sizeof(bsi->rdstat[i][mode_idx].ta));
2069 memcpy(bsi->rdstat[i][mode_idx].tl, t_left,
2070 sizeof(bsi->rdstat[i][mode_idx].tl));
2072 // motion search for newmv (single predictor case only)
2073 if (!has_second_rf && this_mode == NEWMV &&
2074 seg_mvs[i][mi->ref_frame[0]].as_int == INVALID_MV) {
2075 MV *const new_mv = &mode_mv[NEWMV][0].as_mv;
2077 uint32_t bestsme = UINT_MAX;
2078 int sadpb = x->sadperbit4;
2082 const MvLimits tmp_mv_limits = x->mv_limits;
2084 /* Is the best so far sufficiently good that we cant justify doing
2085 * and new motion search. */
2086 if (best_rd < label_mv_thresh) break;
2088 if (cpi->oxcf.mode != BEST) {
2089 // use previous block's result as next block's MV predictor.
2091 bsi->mvp.as_int = mi->bmi[i - 1].as_mv[0].as_int;
2092 if (i == 2) bsi->mvp.as_int = mi->bmi[i - 2].as_mv[0].as_int;
2096 max_mv = x->max_mv_context[mi->ref_frame[0]];
2099 VPXMAX(abs(bsi->mvp.as_mv.row), abs(bsi->mvp.as_mv.col)) >> 3;
2101 if (sf->mv.auto_mv_step_size && cm->show_frame) {
2102 // Take wtd average of the step_params based on the last frame's
2103 // max mv magnitude and the best ref mvs of the current block for
2104 // the given reference.
2106 (vp9_init_search_range(max_mv) + cpi->mv_step_param) / 2;
2108 step_param = cpi->mv_step_param;
2111 mvp_full.row = bsi->mvp.as_mv.row >> 3;
2112 mvp_full.col = bsi->mvp.as_mv.col >> 3;
2114 if (sf->adaptive_motion_search) {
2115 if (x->pred_mv[mi->ref_frame[0]].row != INT16_MAX &&
2116 x->pred_mv[mi->ref_frame[0]].col != INT16_MAX) {
2117 mvp_full.row = x->pred_mv[mi->ref_frame[0]].row >> 3;
2118 mvp_full.col = x->pred_mv[mi->ref_frame[0]].col >> 3;
2120 step_param = VPXMAX(step_param, 8);
2123 // adjust src pointer for this block
2126 vp9_set_mv_search_range(&x->mv_limits, &bsi->ref_mv[0]->as_mv);
2128 bestsme = vp9_full_pixel_search(
2129 cpi, x, bsize, &mvp_full, step_param, cpi->sf.mv.search_method,
2131 sf->mv.subpel_search_method != SUBPEL_TREE ? cost_list : NULL,
2132 &bsi->ref_mv[0]->as_mv, new_mv, INT_MAX, 1);
2134 x->mv_limits = tmp_mv_limits;
2136 if (bestsme < UINT_MAX) {
2137 uint32_t distortion;
2138 cpi->find_fractional_mv_step(
2139 x, new_mv, &bsi->ref_mv[0]->as_mv, cm->allow_high_precision_mv,
2140 x->errorperbit, &cpi->fn_ptr[bsize], sf->mv.subpel_force_stop,
2141 sf->mv.subpel_search_level, cond_cost_list(cpi, cost_list),
2142 x->nmvjointcost, x->mvcost, &distortion,
2143 &x->pred_sse[mi->ref_frame[0]], NULL, pw, ph,
2144 cpi->sf.use_accurate_subpel_search);
2146 // save motion search result for use in compound prediction
2147 seg_mvs[i][mi->ref_frame[0]].as_mv = *new_mv;
2150 x->pred_mv[mi->ref_frame[0]] = *new_mv;
2152 // restore src pointers
2153 mi_buf_restore(x, orig_src, orig_pre);
2156 if (has_second_rf) {
2157 if (seg_mvs[i][mi->ref_frame[1]].as_int == INVALID_MV ||
2158 seg_mvs[i][mi->ref_frame[0]].as_int == INVALID_MV)
2162 if (has_second_rf && this_mode == NEWMV &&
2163 mi->interp_filter == EIGHTTAP) {
2164 // adjust src pointers
2166 if (sf->comp_inter_joint_search_thresh <= bsize) {
2168 joint_motion_search(cpi, x, bsize, frame_mv[this_mode], mi_row,
2169 mi_col, seg_mvs[i], &rate_mv);
2170 seg_mvs[i][mi->ref_frame[0]].as_int =
2171 frame_mv[this_mode][mi->ref_frame[0]].as_int;
2172 seg_mvs[i][mi->ref_frame[1]].as_int =
2173 frame_mv[this_mode][mi->ref_frame[1]].as_int;
2175 // restore src pointers
2176 mi_buf_restore(x, orig_src, orig_pre);
2179 bsi->rdstat[i][mode_idx].brate = set_and_cost_bmi_mvs(
2180 cpi, x, xd, i, this_mode, mode_mv[this_mode], frame_mv, seg_mvs[i],
2181 bsi->ref_mv, x->nmvjointcost, x->mvcost);
2183 for (ref = 0; ref < 1 + has_second_rf; ++ref) {
2184 bsi->rdstat[i][mode_idx].mvs[ref].as_int =
2185 mode_mv[this_mode][ref].as_int;
2186 if (num_4x4_blocks_wide > 1)
2187 bsi->rdstat[i + 1][mode_idx].mvs[ref].as_int =
2188 mode_mv[this_mode][ref].as_int;
2189 if (num_4x4_blocks_high > 1)
2190 bsi->rdstat[i + 2][mode_idx].mvs[ref].as_int =
2191 mode_mv[this_mode][ref].as_int;
2194 // Trap vectors that reach beyond the UMV borders
2195 if (mv_check_bounds(&x->mv_limits, &mode_mv[this_mode][0].as_mv) ||
2197 mv_check_bounds(&x->mv_limits, &mode_mv[this_mode][1].as_mv)))
2200 if (filter_idx > 0) {
2201 BEST_SEG_INFO *ref_bsi = bsi_buf;
2205 for (ref = 0; ref < 1 + has_second_rf; ++ref) {
2206 subpelmv |= mv_has_subpel(&mode_mv[this_mode][ref].as_mv);
2207 have_ref &= mode_mv[this_mode][ref].as_int ==
2208 ref_bsi->rdstat[i][mode_idx].mvs[ref].as_int;
2211 if (filter_idx > 1 && !subpelmv && !have_ref) {
2212 ref_bsi = bsi_buf + 1;
2214 for (ref = 0; ref < 1 + has_second_rf; ++ref)
2215 have_ref &= mode_mv[this_mode][ref].as_int ==
2216 ref_bsi->rdstat[i][mode_idx].mvs[ref].as_int;
2219 if (!subpelmv && have_ref &&
2220 ref_bsi->rdstat[i][mode_idx].brdcost < INT64_MAX) {
2221 memcpy(&bsi->rdstat[i][mode_idx], &ref_bsi->rdstat[i][mode_idx],
2222 sizeof(SEG_RDSTAT));
2223 if (num_4x4_blocks_wide > 1)
2224 bsi->rdstat[i + 1][mode_idx].eobs =
2225 ref_bsi->rdstat[i + 1][mode_idx].eobs;
2226 if (num_4x4_blocks_high > 1)
2227 bsi->rdstat[i + 2][mode_idx].eobs =
2228 ref_bsi->rdstat[i + 2][mode_idx].eobs;
2230 if (bsi->rdstat[i][mode_idx].brdcost < best_rd) {
2231 mode_selected = this_mode;
2232 best_rd = bsi->rdstat[i][mode_idx].brdcost;
2238 bsi->rdstat[i][mode_idx].brdcost = encode_inter_mb_segment(
2239 cpi, x, bsi->segment_rd - this_segment_rd, i,
2240 &bsi->rdstat[i][mode_idx].byrate, &bsi->rdstat[i][mode_idx].bdist,
2241 &bsi->rdstat[i][mode_idx].bsse, bsi->rdstat[i][mode_idx].ta,
2242 bsi->rdstat[i][mode_idx].tl, mi_row, mi_col);
2243 if (bsi->rdstat[i][mode_idx].brdcost < INT64_MAX) {
2244 bsi->rdstat[i][mode_idx].brdcost +=
2245 RDCOST(x->rdmult, x->rddiv, bsi->rdstat[i][mode_idx].brate, 0);
2246 bsi->rdstat[i][mode_idx].brate += bsi->rdstat[i][mode_idx].byrate;
2247 bsi->rdstat[i][mode_idx].eobs = p->eobs[i];
2248 if (num_4x4_blocks_wide > 1)
2249 bsi->rdstat[i + 1][mode_idx].eobs = p->eobs[i + 1];
2250 if (num_4x4_blocks_high > 1)
2251 bsi->rdstat[i + 2][mode_idx].eobs = p->eobs[i + 2];
2254 if (bsi->rdstat[i][mode_idx].brdcost < best_rd) {
2255 mode_selected = this_mode;
2256 best_rd = bsi->rdstat[i][mode_idx].brdcost;
2258 } /*for each 4x4 mode*/
2260 if (best_rd == INT64_MAX) {
2262 for (iy = i + 1; iy < 4; ++iy)
2263 for (midx = 0; midx < INTER_MODES; ++midx)
2264 bsi->rdstat[iy][midx].brdcost = INT64_MAX;
2265 bsi->segment_rd = INT64_MAX;
2269 mode_idx = INTER_OFFSET(mode_selected);
2270 memcpy(t_above, bsi->rdstat[i][mode_idx].ta, sizeof(t_above));
2271 memcpy(t_left, bsi->rdstat[i][mode_idx].tl, sizeof(t_left));
2273 set_and_cost_bmi_mvs(cpi, x, xd, i, mode_selected, mode_mv[mode_selected],
2274 frame_mv, seg_mvs[i], bsi->ref_mv, x->nmvjointcost,
2277 br += bsi->rdstat[i][mode_idx].brate;
2278 bd += bsi->rdstat[i][mode_idx].bdist;
2279 block_sse += bsi->rdstat[i][mode_idx].bsse;
2280 segmentyrate += bsi->rdstat[i][mode_idx].byrate;
2281 this_segment_rd += bsi->rdstat[i][mode_idx].brdcost;
2283 if (this_segment_rd > bsi->segment_rd) {
2285 for (iy = i + 1; iy < 4; ++iy)
2286 for (midx = 0; midx < INTER_MODES; ++midx)
2287 bsi->rdstat[iy][midx].brdcost = INT64_MAX;
2288 bsi->segment_rd = INT64_MAX;
2292 } /* for each label */
2296 bsi->segment_yrate = segmentyrate;
2297 bsi->segment_rd = this_segment_rd;
2298 bsi->sse = block_sse;
2300 // update the coding decisions
2301 for (k = 0; k < 4; ++k) bsi->modes[k] = mi->bmi[k].as_mode;
2303 if (bsi->segment_rd > best_rd) return INT64_MAX;
2304 /* set it to the best */
2305 for (i = 0; i < 4; i++) {
2306 mode_idx = INTER_OFFSET(bsi->modes[i]);
2307 mi->bmi[i].as_mv[0].as_int = bsi->rdstat[i][mode_idx].mvs[0].as_int;
2308 if (has_second_ref(mi))
2309 mi->bmi[i].as_mv[1].as_int = bsi->rdstat[i][mode_idx].mvs[1].as_int;
2310 x->plane[0].eobs[i] = bsi->rdstat[i][mode_idx].eobs;
2311 mi->bmi[i].as_mode = bsi->modes[i];
2315 * used to set mbmi->mv.as_int
2317 *returntotrate = bsi->r;
2318 *returndistortion = bsi->d;
2319 *returnyrate = bsi->segment_yrate;
2320 *skippable = vp9_is_skippable_in_plane(x, BLOCK_8X8, 0);
2322 mi->mode = bsi->modes[3];
2324 return bsi->segment_rd;
2327 static void estimate_ref_frame_costs(const VP9_COMMON *cm,
2328 const MACROBLOCKD *xd, int segment_id,
2329 unsigned int *ref_costs_single,
2330 unsigned int *ref_costs_comp,
2331 vpx_prob *comp_mode_p) {
2332 int seg_ref_active =
2333 segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME);
2334 if (seg_ref_active) {
2335 memset(ref_costs_single, 0, MAX_REF_FRAMES * sizeof(*ref_costs_single));
2336 memset(ref_costs_comp, 0, MAX_REF_FRAMES * sizeof(*ref_costs_comp));
2339 vpx_prob intra_inter_p = vp9_get_intra_inter_prob(cm, xd);
2340 vpx_prob comp_inter_p = 128;
2342 if (cm->reference_mode == REFERENCE_MODE_SELECT) {
2343 comp_inter_p = vp9_get_reference_mode_prob(cm, xd);
2344 *comp_mode_p = comp_inter_p;
2349 ref_costs_single[INTRA_FRAME] = vp9_cost_bit(intra_inter_p, 0);
2351 if (cm->reference_mode != COMPOUND_REFERENCE) {
2352 vpx_prob ref_single_p1 = vp9_get_pred_prob_single_ref_p1(cm, xd);
2353 vpx_prob ref_single_p2 = vp9_get_pred_prob_single_ref_p2(cm, xd);
2354 unsigned int base_cost = vp9_cost_bit(intra_inter_p, 1);
2356 if (cm->reference_mode == REFERENCE_MODE_SELECT)
2357 base_cost += vp9_cost_bit(comp_inter_p, 0);
2359 ref_costs_single[LAST_FRAME] = ref_costs_single[GOLDEN_FRAME] =
2360 ref_costs_single[ALTREF_FRAME] = base_cost;
2361 ref_costs_single[LAST_FRAME] += vp9_cost_bit(ref_single_p1, 0);
2362 ref_costs_single[GOLDEN_FRAME] += vp9_cost_bit(ref_single_p1, 1);
2363 ref_costs_single[ALTREF_FRAME] += vp9_cost_bit(ref_single_p1, 1);
2364 ref_costs_single[GOLDEN_FRAME] += vp9_cost_bit(ref_single_p2, 0);
2365 ref_costs_single[ALTREF_FRAME] += vp9_cost_bit(ref_single_p2, 1);
2367 ref_costs_single[LAST_FRAME] = 512;
2368 ref_costs_single[GOLDEN_FRAME] = 512;
2369 ref_costs_single[ALTREF_FRAME] = 512;
2371 if (cm->reference_mode != SINGLE_REFERENCE) {
2372 vpx_prob ref_comp_p = vp9_get_pred_prob_comp_ref_p(cm, xd);
2373 unsigned int base_cost = vp9_cost_bit(intra_inter_p, 1);
2375 if (cm->reference_mode == REFERENCE_MODE_SELECT)
2376 base_cost += vp9_cost_bit(comp_inter_p, 1);
2378 ref_costs_comp[LAST_FRAME] = base_cost + vp9_cost_bit(ref_comp_p, 0);
2379 ref_costs_comp[GOLDEN_FRAME] = base_cost + vp9_cost_bit(ref_comp_p, 1);
2381 ref_costs_comp[LAST_FRAME] = 512;
2382 ref_costs_comp[GOLDEN_FRAME] = 512;
2387 static void store_coding_context(
2388 MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, int mode_index,
2389 int64_t comp_pred_diff[REFERENCE_MODES],
2390 int64_t best_filter_diff[SWITCHABLE_FILTER_CONTEXTS], int skippable) {
2391 MACROBLOCKD *const xd = &x->e_mbd;
2393 // Take a snapshot of the coding context so it can be
2394 // restored if we decide to encode this way
2395 ctx->skip = x->skip;
2396 ctx->skippable = skippable;
2397 ctx->best_mode_index = mode_index;
2398 ctx->mic = *xd->mi[0];
2399 ctx->mbmi_ext = *x->mbmi_ext;
2400 ctx->single_pred_diff = (int)comp_pred_diff[SINGLE_REFERENCE];
2401 ctx->comp_pred_diff = (int)comp_pred_diff[COMPOUND_REFERENCE];
2402 ctx->hybrid_pred_diff = (int)comp_pred_diff[REFERENCE_MODE_SELECT];
2404 memcpy(ctx->best_filter_diff, best_filter_diff,
2405 sizeof(*best_filter_diff) * SWITCHABLE_FILTER_CONTEXTS);
2408 static void setup_buffer_inter(VP9_COMP *cpi, MACROBLOCK *x,
2409 MV_REFERENCE_FRAME ref_frame,
2410 BLOCK_SIZE block_size, int mi_row, int mi_col,
2411 int_mv frame_nearest_mv[MAX_REF_FRAMES],
2412 int_mv frame_near_mv[MAX_REF_FRAMES],
2413 struct buf_2d yv12_mb[4][MAX_MB_PLANE]) {
2414 const VP9_COMMON *cm = &cpi->common;
2415 const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame);
2416 MACROBLOCKD *const xd = &x->e_mbd;
2417 MODE_INFO *const mi = xd->mi[0];
2418 int_mv *const candidates = x->mbmi_ext->ref_mvs[ref_frame];
2419 const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf;
2420 MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
2422 assert(yv12 != NULL);
2424 // TODO(jkoleszar): Is the UV buffer ever used here? If so, need to make this
2425 // use the UV scaling factors.
2426 vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf);
2428 // Gets an initial list of candidate vectors from neighbours and orders them
2429 vp9_find_mv_refs(cm, xd, mi, ref_frame, candidates, mi_row, mi_col,
2430 mbmi_ext->mode_context);
2432 // Candidate refinement carried out at encoder and decoder
2433 vp9_find_best_ref_mvs(xd, cm->allow_high_precision_mv, candidates,
2434 &frame_nearest_mv[ref_frame],
2435 &frame_near_mv[ref_frame]);
2437 // Further refinement that is encode side only to test the top few candidates
2438 // in full and choose the best as the centre point for subsequent searches.
2439 // The current implementation doesn't support scaling.
2440 if (!vp9_is_scaled(sf) && block_size >= BLOCK_8X8)
2441 vp9_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame,
2445 #if CONFIG_NON_GREEDY_MV
2446 static int ref_frame_to_gf_rf_idx(int ref_frame) {
2447 if (ref_frame == GOLDEN_FRAME) {
2450 if (ref_frame == LAST_FRAME) {
2453 if (ref_frame == ALTREF_FRAME) {
2461 static void single_motion_search(VP9_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
2462 int mi_row, int mi_col, int_mv *tmp_mv,
2464 MACROBLOCKD *xd = &x->e_mbd;
2465 const VP9_COMMON *cm = &cpi->common;
2466 MODE_INFO *mi = xd->mi[0];
2467 struct buf_2d backup_yv12[MAX_MB_PLANE] = { { 0, 0 } };
2470 int ref = mi->ref_frame[0];
2471 MV ref_mv = x->mbmi_ext->ref_mvs[ref][0].as_mv;
2472 const MvLimits tmp_mv_limits = x->mv_limits;
2474 const int best_predmv_idx = x->mv_best_ref_index[ref];
2475 const YV12_BUFFER_CONFIG *scaled_ref_frame =
2476 vp9_get_scaled_ref_frame(cpi, ref);
2477 const int pw = num_4x4_blocks_wide_lookup[bsize] << 2;
2478 const int ph = num_4x4_blocks_high_lookup[bsize] << 2;
2481 #if CONFIG_NON_GREEDY_MV
2484 double lambda = (pw * ph) / 4;
2486 int_mv nb_full_mvs[NB_MVS_NUM];
2487 const int nb_full_mv_num = NB_MVS_NUM;
2488 int gf_group_idx = cpi->twopass.gf_group.index;
2489 int gf_rf_idx = ref_frame_to_gf_rf_idx(ref);
2490 BLOCK_SIZE square_bsize = get_square_block_size(bsize);
2491 vp9_prepare_nb_full_mvs(&cpi->tpl_stats[gf_group_idx], mi_row, mi_col,
2492 gf_rf_idx, square_bsize, nb_full_mvs);
2493 #else // CONFIG_NON_GREEDY_MV
2494 int bestsme = INT_MAX;
2495 int sadpb = x->sadperbit16;
2496 #endif // CONFIG_NON_GREEDY_MV
2498 pred_mv[0] = x->mbmi_ext->ref_mvs[ref][0].as_mv;
2499 pred_mv[1] = x->mbmi_ext->ref_mvs[ref][1].as_mv;
2500 pred_mv[2] = x->pred_mv[ref];
2502 if (scaled_ref_frame) {
2504 // Swap out the reference frame for a version that's been scaled to
2505 // match the resolution of the current frame, allowing the existing
2506 // motion search code to be used without additional modifications.
2507 for (i = 0; i < MAX_MB_PLANE; i++) backup_yv12[i] = xd->plane[i].pre[0];
2509 vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
2512 // Work out the size of the first step in the mv step search.
2513 // 0 here is maximum length first step. 1 is VPXMAX >> 1 etc.
2514 if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) {
2515 // Take wtd average of the step_params based on the last frame's
2516 // max mv magnitude and that based on the best ref mvs of the current
2517 // block for the given reference.
2519 (vp9_init_search_range(x->max_mv_context[ref]) + cpi->mv_step_param) /
2522 step_param = cpi->mv_step_param;
2525 if (cpi->sf.adaptive_motion_search && bsize < BLOCK_64X64) {
2527 2 * (b_width_log2_lookup[BLOCK_64X64] -
2528 VPXMIN(b_height_log2_lookup[bsize], b_width_log2_lookup[bsize]));
2529 step_param = VPXMAX(step_param, boffset);
2532 if (cpi->sf.adaptive_motion_search) {
2533 int bwl = b_width_log2_lookup[bsize];
2534 int bhl = b_height_log2_lookup[bsize];
2535 int tlevel = x->pred_mv_sad[ref] >> (bwl + bhl + 4);
2537 if (tlevel < 5) step_param += 2;
2539 // prev_mv_sad is not setup for dynamically scaled frames.
2540 if (cpi->oxcf.resize_mode != RESIZE_DYNAMIC) {
2542 for (i = LAST_FRAME; i <= ALTREF_FRAME && cm->show_frame; ++i) {
2543 if ((x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[i]) {
2544 x->pred_mv[ref].row = INT16_MAX;
2545 x->pred_mv[ref].col = INT16_MAX;
2546 tmp_mv->as_int = INVALID_MV;
2548 if (scaled_ref_frame) {
2550 for (i = 0; i < MAX_MB_PLANE; ++i)
2551 xd->plane[i].pre[0] = backup_yv12[i];
2559 // Note: MV limits are modified here. Always restore the original values
2560 // after full-pixel motion search.
2561 vp9_set_mv_search_range(&x->mv_limits, &ref_mv);
2563 mvp_full = pred_mv[best_predmv_idx];
2567 #if CONFIG_NON_GREEDY_MV
2568 bestsme = vp9_full_pixel_diamond_new(
2569 cpi, x, &mvp_full, step_param, lambda, 1, &cpi->fn_ptr[bsize],
2570 nb_full_mvs, nb_full_mv_num, &tmp_mv->as_mv, &mv_dist, &mv_cost);
2571 #else // CONFIG_NON_GREEDY_MV
2572 bestsme = vp9_full_pixel_search(
2573 cpi, x, bsize, &mvp_full, step_param, cpi->sf.mv.search_method, sadpb,
2574 cond_cost_list(cpi, cost_list), &ref_mv, &tmp_mv->as_mv, INT_MAX, 1);
2575 #endif // CONFIG_NON_GREEDY_MV
2577 if (cpi->sf.enhanced_full_pixel_motion_search) {
2579 for (i = 0; i < 3; ++i) {
2580 #if CONFIG_NON_GREEDY_MV
2582 #else // CONFIG_NON_GREEDY_MV
2584 #endif // CONFIG_NON_GREEDY_MV
2590 if (pred_mv[i].row == INT16_MAX || pred_mv[i].col == INT16_MAX) continue;
2591 if (i == best_predmv_idx) continue;
2593 diff_row = ((int)pred_mv[i].row -
2594 pred_mv[i > 0 ? (i - 1) : best_predmv_idx].row) >>
2596 diff_col = ((int)pred_mv[i].col -
2597 pred_mv[i > 0 ? (i - 1) : best_predmv_idx].col) >>
2599 if (diff_row == 0 && diff_col == 0) continue;
2600 if (diff_row < 0) diff_row = -diff_row;
2601 if (diff_col < 0) diff_col = -diff_col;
2602 step = get_msb((diff_row + diff_col + 1) >> 1);
2603 if (step <= 0) continue;
2605 mvp_full = pred_mv[i];
2608 #if CONFIG_NON_GREEDY_MV
2609 this_me = vp9_full_pixel_diamond_new(
2610 cpi, x, &mvp_full, VPXMAX(step_param, MAX_MVSEARCH_STEPS - step),
2611 lambda, 1, &cpi->fn_ptr[bsize], nb_full_mvs, nb_full_mv_num, &this_mv,
2612 &mv_dist, &mv_cost);
2613 #else // CONFIG_NON_GREEDY_MV
2614 this_me = vp9_full_pixel_search(
2615 cpi, x, bsize, &mvp_full,
2616 VPXMAX(step_param, MAX_MVSEARCH_STEPS - step),
2617 cpi->sf.mv.search_method, sadpb, cond_cost_list(cpi, cost_list),
2618 &ref_mv, &this_mv, INT_MAX, 1);
2619 #endif // CONFIG_NON_GREEDY_MV
2620 if (this_me < bestsme) {
2621 tmp_mv->as_mv = this_mv;
2627 x->mv_limits = tmp_mv_limits;
2629 if (bestsme < INT_MAX) {
2630 uint32_t dis; /* TODO: use dis in distortion calculation later. */
2631 cpi->find_fractional_mv_step(
2632 x, &tmp_mv->as_mv, &ref_mv, cm->allow_high_precision_mv, x->errorperbit,
2633 &cpi->fn_ptr[bsize], cpi->sf.mv.subpel_force_stop,
2634 cpi->sf.mv.subpel_search_level, cond_cost_list(cpi, cost_list),
2635 x->nmvjointcost, x->mvcost, &dis, &x->pred_sse[ref], NULL, pw, ph,
2636 cpi->sf.use_accurate_subpel_search);
2638 *rate_mv = vp9_mv_bit_cost(&tmp_mv->as_mv, &ref_mv, x->nmvjointcost,
2639 x->mvcost, MV_COST_WEIGHT);
2641 x->pred_mv[ref] = tmp_mv->as_mv;
2643 if (scaled_ref_frame) {
2645 for (i = 0; i < MAX_MB_PLANE; i++) xd->plane[i].pre[0] = backup_yv12[i];
2649 static INLINE void restore_dst_buf(MACROBLOCKD *xd,
2650 uint8_t *orig_dst[MAX_MB_PLANE],
2651 int orig_dst_stride[MAX_MB_PLANE]) {
2653 for (i = 0; i < MAX_MB_PLANE; i++) {
2654 xd->plane[i].dst.buf = orig_dst[i];
2655 xd->plane[i].dst.stride = orig_dst_stride[i];
2659 // In some situations we want to discount tha pparent cost of a new motion
2660 // vector. Where there is a subtle motion field and especially where there is
2661 // low spatial complexity then it can be hard to cover the cost of a new motion
2662 // vector in a single block, even if that motion vector reduces distortion.
2663 // However, once established that vector may be usable through the nearest and
2664 // near mv modes to reduce distortion in subsequent blocks and also improve
2666 static int discount_newmv_test(const VP9_COMP *cpi, int this_mode,
2668 int_mv (*mode_mv)[MAX_REF_FRAMES], int ref_frame,
2669 int mi_row, int mi_col, BLOCK_SIZE bsize) {
2670 #if CONFIG_NON_GREEDY_MV
2673 if (this_mode == NEWMV && bsize >= BLOCK_8X8 && cpi->tpl_ready) {
2674 const int gf_group_idx = cpi->twopass.gf_group.index;
2675 const int gf_rf_idx = ref_frame_to_gf_rf_idx(ref_frame);
2676 const TplDepFrame tpl_frame = cpi->tpl_stats[gf_group_idx];
2677 const int tpl_block_mi_h = num_8x8_blocks_high_lookup[cpi->tpl_bsize];
2678 const int tpl_block_mi_w = num_8x8_blocks_wide_lookup[cpi->tpl_bsize];
2679 const int tpl_mi_row = mi_row - (mi_row % tpl_block_mi_h);
2680 const int tpl_mi_col = mi_col - (mi_col % tpl_block_mi_w);
2683 .mv_mode_arr[gf_rf_idx][tpl_mi_row * tpl_frame.stride + tpl_mi_col];
2684 if (mv_mode == NEW_MV_MODE) {
2685 int_mv tpl_new_mv = *get_pyramid_mv(&tpl_frame, gf_rf_idx, cpi->tpl_bsize,
2686 tpl_mi_row, tpl_mi_col);
2687 int row_diff = abs(tpl_new_mv.as_mv.row - this_mv.as_mv.row);
2688 int col_diff = abs(tpl_new_mv.as_mv.col - this_mv.as_mv.col);
2689 if (VPXMAX(row_diff, col_diff) <= 8) {
2704 return (!cpi->rc.is_src_frame_alt_ref && (this_mode == NEWMV) &&
2705 (this_mv.as_int != 0) &&
2706 ((mode_mv[NEARESTMV][ref_frame].as_int == 0) ||
2707 (mode_mv[NEARESTMV][ref_frame].as_int == INVALID_MV)) &&
2708 ((mode_mv[NEARMV][ref_frame].as_int == 0) ||
2709 (mode_mv[NEARMV][ref_frame].as_int == INVALID_MV)));
2713 static int64_t handle_inter_mode(
2714 VP9_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int *rate2,
2715 int64_t *distortion, int *skippable, int *rate_y, int *rate_uv,
2716 struct buf_2d *recon, int *disable_skip, int_mv (*mode_mv)[MAX_REF_FRAMES],
2717 int mi_row, int mi_col, int_mv single_newmv[MAX_REF_FRAMES],
2718 INTERP_FILTER (*single_filter)[MAX_REF_FRAMES],
2719 int (*single_skippable)[MAX_REF_FRAMES], int64_t *psse,
2720 const int64_t ref_best_rd, int64_t *mask_filter, int64_t filter_cache[]) {
2721 VP9_COMMON *cm = &cpi->common;
2722 MACROBLOCKD *xd = &x->e_mbd;
2723 MODE_INFO *mi = xd->mi[0];
2724 MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
2725 const int is_comp_pred = has_second_ref(mi);
2726 const int this_mode = mi->mode;
2727 int_mv *frame_mv = mode_mv[this_mode];
2729 int refs[2] = { mi->ref_frame[0],
2730 (mi->ref_frame[1] < 0 ? 0 : mi->ref_frame[1]) };
2732 #if CONFIG_VP9_HIGHBITDEPTH
2733 DECLARE_ALIGNED(16, uint16_t, tmp_buf16[MAX_MB_PLANE * 64 * 64]);
2736 DECLARE_ALIGNED(16, uint8_t, tmp_buf[MAX_MB_PLANE * 64 * 64]);
2737 #endif // CONFIG_VP9_HIGHBITDEPTH
2738 int pred_exists = 0;
2740 int64_t rd, tmp_rd, best_rd = INT64_MAX;
2741 int best_needs_copy = 0;
2742 uint8_t *orig_dst[MAX_MB_PLANE];
2743 int orig_dst_stride[MAX_MB_PLANE];
2745 INTERP_FILTER best_filter = SWITCHABLE;
2746 uint8_t skip_txfm[MAX_MB_PLANE << 2] = { 0 };
2747 int64_t bsse[MAX_MB_PLANE << 2] = { 0 };
2749 int bsl = mi_width_log2_lookup[bsize];
2750 int pred_filter_search =
2751 cpi->sf.cb_pred_filter_search
2752 ? (((mi_row + mi_col) >> bsl) +
2753 get_chessboard_index(cm->current_video_frame)) &
2757 int skip_txfm_sb = 0;
2758 int64_t skip_sse_sb = INT64_MAX;
2759 int64_t distortion_y = 0, distortion_uv = 0;
2761 #if CONFIG_VP9_HIGHBITDEPTH
2762 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
2763 tmp_buf = CONVERT_TO_BYTEPTR(tmp_buf16);
2765 tmp_buf = (uint8_t *)tmp_buf16;
2767 #endif // CONFIG_VP9_HIGHBITDEPTH
2769 if (pred_filter_search) {
2770 INTERP_FILTER af = SWITCHABLE, lf = SWITCHABLE;
2771 if (xd->above_mi && is_inter_block(xd->above_mi))
2772 af = xd->above_mi->interp_filter;
2773 if (xd->left_mi && is_inter_block(xd->left_mi))
2774 lf = xd->left_mi->interp_filter;
2776 if ((this_mode != NEWMV) || (af == lf)) best_filter = af;
2780 if (frame_mv[refs[0]].as_int == INVALID_MV ||
2781 frame_mv[refs[1]].as_int == INVALID_MV)
2784 if (cpi->sf.adaptive_mode_search) {
2785 if (single_filter[this_mode][refs[0]] ==
2786 single_filter[this_mode][refs[1]])
2787 best_filter = single_filter[this_mode][refs[0]];
2791 if (this_mode == NEWMV) {
2794 // Initialize mv using single prediction mode result.
2795 frame_mv[refs[0]].as_int = single_newmv[refs[0]].as_int;
2796 frame_mv[refs[1]].as_int = single_newmv[refs[1]].as_int;
2798 if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
2799 joint_motion_search(cpi, x, bsize, frame_mv, mi_row, mi_col,
2800 single_newmv, &rate_mv);
2802 rate_mv = vp9_mv_bit_cost(&frame_mv[refs[0]].as_mv,
2803 &x->mbmi_ext->ref_mvs[refs[0]][0].as_mv,
2804 x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
2805 rate_mv += vp9_mv_bit_cost(&frame_mv[refs[1]].as_mv,
2806 &x->mbmi_ext->ref_mvs[refs[1]][0].as_mv,
2807 x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
2812 single_motion_search(cpi, x, bsize, mi_row, mi_col, &tmp_mv, &rate_mv);
2813 if (tmp_mv.as_int == INVALID_MV) return INT64_MAX;
2815 frame_mv[refs[0]].as_int = xd->mi[0]->bmi[0].as_mv[0].as_int =
2817 single_newmv[refs[0]].as_int = tmp_mv.as_int;
2819 // Estimate the rate implications of a new mv but discount this
2820 // under certain circumstances where we want to help initiate a weak
2821 // motion field, where the distortion gain for a single block may not
2822 // be enough to overcome the cost of a new mv.
2823 if (discount_newmv_test(cpi, this_mode, tmp_mv, mode_mv, refs[0], mi_row,
2825 *rate2 += VPXMAX((rate_mv / NEW_MV_DISCOUNT_FACTOR), 1);
2832 for (i = 0; i < is_comp_pred + 1; ++i) {
2833 cur_mv[i] = frame_mv[refs[i]];
2834 // Clip "next_nearest" so that it does not extend to far out of image
2835 if (this_mode != NEWMV) clamp_mv2(&cur_mv[i].as_mv, xd);
2837 if (mv_check_bounds(&x->mv_limits, &cur_mv[i].as_mv)) return INT64_MAX;
2838 mi->mv[i].as_int = cur_mv[i].as_int;
2841 // do first prediction into the destination buffer. Do the next
2842 // prediction into a temporary buffer. Then keep track of which one
2843 // of these currently holds the best predictor, and use the other
2844 // one for future predictions. In the end, copy from tmp_buf to
2845 // dst if necessary.
2846 for (i = 0; i < MAX_MB_PLANE; i++) {
2847 orig_dst[i] = xd->plane[i].dst.buf;
2848 orig_dst_stride[i] = xd->plane[i].dst.stride;
2851 // We don't include the cost of the second reference here, because there
2852 // are only two options: Last/ARF or Golden/ARF; The second one is always
2853 // known, which is ARF.
2855 // Under some circumstances we discount the cost of new mv mode to encourage
2856 // initiation of a motion field.
2857 if (discount_newmv_test(cpi, this_mode, frame_mv[refs[0]], mode_mv, refs[0],
2858 mi_row, mi_col, bsize)) {
2860 VPXMIN(cost_mv_ref(cpi, this_mode, mbmi_ext->mode_context[refs[0]]),
2861 cost_mv_ref(cpi, NEARESTMV, mbmi_ext->mode_context[refs[0]]));
2863 *rate2 += cost_mv_ref(cpi, this_mode, mbmi_ext->mode_context[refs[0]]);
2866 if (RDCOST(x->rdmult, x->rddiv, *rate2, 0) > ref_best_rd &&
2867 mi->mode != NEARESTMV)
2871 // Are all MVs integer pel for Y and UV
2872 intpel_mv = !mv_has_subpel(&mi->mv[0].as_mv);
2873 if (is_comp_pred) intpel_mv &= !mv_has_subpel(&mi->mv[1].as_mv);
2875 // Search for best switchable filter by checking the variance of
2876 // pred error irrespective of whether the filter will be used
2877 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i) filter_cache[i] = INT64_MAX;
2879 if (cm->interp_filter != BILINEAR) {
2880 if (x->source_variance < cpi->sf.disable_filter_search_var_thresh) {
2881 best_filter = EIGHTTAP;
2882 } else if (best_filter == SWITCHABLE) {
2884 int tmp_rate_sum = 0;
2885 int64_t tmp_dist_sum = 0;
2887 for (i = 0; i < SWITCHABLE_FILTERS; ++i) {
2890 int tmp_skip_sb = 0;
2891 int64_t tmp_skip_sse = INT64_MAX;
2893 mi->interp_filter = i;
2894 rs = vp9_get_switchable_rate(cpi, xd);
2895 rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0);
2897 if (i > 0 && intpel_mv) {
2898 rd = RDCOST(x->rdmult, x->rddiv, tmp_rate_sum, tmp_dist_sum);
2899 filter_cache[i] = rd;
2900 filter_cache[SWITCHABLE_FILTERS] =
2901 VPXMIN(filter_cache[SWITCHABLE_FILTERS], rd + rs_rd);
2902 if (cm->interp_filter == SWITCHABLE) rd += rs_rd;
2903 *mask_filter = VPXMAX(*mask_filter, rd);
2906 int64_t dist_sum = 0;
2907 if (i > 0 && cpi->sf.adaptive_interp_filter_search &&
2908 (cpi->sf.interp_filter_search_mask & (1 << i))) {
2910 dist_sum = INT64_MAX;
2914 if ((cm->interp_filter == SWITCHABLE && (!i || best_needs_copy)) ||
2915 (cm->interp_filter != SWITCHABLE &&
2916 (cm->interp_filter == mi->interp_filter ||
2917 (i == 0 && intpel_mv)))) {
2918 restore_dst_buf(xd, orig_dst, orig_dst_stride);
2920 for (j = 0; j < MAX_MB_PLANE; j++) {
2921 xd->plane[j].dst.buf = tmp_buf + j * 64 * 64;
2922 xd->plane[j].dst.stride = 64;
2925 vp9_build_inter_predictors_sb(xd, mi_row, mi_col, bsize);
2926 model_rd_for_sb(cpi, bsize, x, xd, &rate_sum, &dist_sum, &tmp_skip_sb,
2929 rd = RDCOST(x->rdmult, x->rddiv, rate_sum, dist_sum);
2930 filter_cache[i] = rd;
2931 filter_cache[SWITCHABLE_FILTERS] =
2932 VPXMIN(filter_cache[SWITCHABLE_FILTERS], rd + rs_rd);
2933 if (cm->interp_filter == SWITCHABLE) rd += rs_rd;
2934 *mask_filter = VPXMAX(*mask_filter, rd);
2936 if (i == 0 && intpel_mv) {
2937 tmp_rate_sum = rate_sum;
2938 tmp_dist_sum = dist_sum;
2942 if (i == 0 && cpi->sf.use_rd_breakout && ref_best_rd < INT64_MAX) {
2943 if (rd / 2 > ref_best_rd) {
2944 restore_dst_buf(xd, orig_dst, orig_dst_stride);
2948 newbest = i == 0 || rd < best_rd;
2952 best_filter = mi->interp_filter;
2953 if (cm->interp_filter == SWITCHABLE && i && !intpel_mv)
2954 best_needs_copy = !best_needs_copy;
2957 if ((cm->interp_filter == SWITCHABLE && newbest) ||
2958 (cm->interp_filter != SWITCHABLE &&
2959 cm->interp_filter == mi->interp_filter)) {
2963 skip_txfm_sb = tmp_skip_sb;
2964 skip_sse_sb = tmp_skip_sse;
2965 memcpy(skip_txfm, x->skip_txfm, sizeof(skip_txfm));
2966 memcpy(bsse, x->bsse, sizeof(bsse));
2969 restore_dst_buf(xd, orig_dst, orig_dst_stride);
2972 // Set the appropriate filter
2974 cm->interp_filter != SWITCHABLE ? cm->interp_filter : best_filter;
2975 rs = cm->interp_filter == SWITCHABLE ? vp9_get_switchable_rate(cpi, xd) : 0;
2978 if (best_needs_copy) {
2979 // again temporarily set the buffers to local memory to prevent a memcpy
2980 for (i = 0; i < MAX_MB_PLANE; i++) {
2981 xd->plane[i].dst.buf = tmp_buf + i * 64 * 64;
2982 xd->plane[i].dst.stride = 64;
2985 rd = tmp_rd + RDCOST(x->rdmult, x->rddiv, rs, 0);
2989 // Handles the special case when a filter that is not in the
2990 // switchable list (ex. bilinear) is indicated at the frame level, or
2991 // skip condition holds.
2992 vp9_build_inter_predictors_sb(xd, mi_row, mi_col, bsize);
2993 model_rd_for_sb(cpi, bsize, x, xd, &tmp_rate, &tmp_dist, &skip_txfm_sb,
2995 rd = RDCOST(x->rdmult, x->rddiv, rs + tmp_rate, tmp_dist);
2996 memcpy(skip_txfm, x->skip_txfm, sizeof(skip_txfm));
2997 memcpy(bsse, x->bsse, sizeof(bsse));
3000 if (!is_comp_pred) single_filter[this_mode][refs[0]] = mi->interp_filter;
3002 if (cpi->sf.adaptive_mode_search)
3004 if (single_skippable[this_mode][refs[0]] &&
3005 single_skippable[this_mode][refs[1]])
3006 memset(skip_txfm, SKIP_TXFM_AC_DC, sizeof(skip_txfm));
3008 if (cpi->sf.use_rd_breakout && ref_best_rd < INT64_MAX) {
3009 // if current pred_error modeled rd is substantially more than the best
3010 // so far, do not bother doing full rd
3011 if (rd / 2 > ref_best_rd) {
3012 restore_dst_buf(xd, orig_dst, orig_dst_stride);
3017 if (cm->interp_filter == SWITCHABLE) *rate2 += rs;
3019 memcpy(x->skip_txfm, skip_txfm, sizeof(skip_txfm));
3020 memcpy(x->bsse, bsse, sizeof(bsse));
3022 if (!skip_txfm_sb || xd->lossless) {
3023 int skippable_y, skippable_uv;
3024 int64_t sseuv = INT64_MAX;
3025 int64_t rdcosty = INT64_MAX;
3027 // Y cost and distortion
3028 vp9_subtract_plane(x, bsize, 0);
3029 super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse, bsize,
3030 ref_best_rd, recon);
3032 if (*rate_y == INT_MAX) {
3034 *distortion = INT64_MAX;
3035 restore_dst_buf(xd, orig_dst, orig_dst_stride);
3040 *distortion += distortion_y;
3042 rdcosty = RDCOST(x->rdmult, x->rddiv, *rate2, *distortion);
3043 rdcosty = VPXMIN(rdcosty, RDCOST(x->rdmult, x->rddiv, 0, *psse));
3045 if (!super_block_uvrd(cpi, x, rate_uv, &distortion_uv, &skippable_uv,
3046 &sseuv, bsize, ref_best_rd - rdcosty)) {
3048 *distortion = INT64_MAX;
3049 restore_dst_buf(xd, orig_dst, orig_dst_stride);
3055 *distortion += distortion_uv;
3056 *skippable = skippable_y && skippable_uv;
3061 // The cost of skip bit needs to be added.
3062 *rate2 += vp9_cost_bit(vp9_get_skip_prob(cm, xd), 1);
3064 *distortion = skip_sse_sb;
3067 if (!is_comp_pred) single_skippable[this_mode][refs[0]] = *skippable;
3069 restore_dst_buf(xd, orig_dst, orig_dst_stride);
3070 return 0; // The rate-distortion cost will be re-calculated by caller.
3073 void vp9_rd_pick_intra_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, RD_COST *rd_cost,
3074 BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx,
3076 VP9_COMMON *const cm = &cpi->common;
3077 MACROBLOCKD *const xd = &x->e_mbd;
3078 struct macroblockd_plane *const pd = xd->plane;
3079 int rate_y = 0, rate_uv = 0, rate_y_tokenonly = 0, rate_uv_tokenonly = 0;
3080 int y_skip = 0, uv_skip = 0;
3081 int64_t dist_y = 0, dist_uv = 0;
3082 TX_SIZE max_uv_tx_size;
3085 xd->mi[0]->ref_frame[0] = INTRA_FRAME;
3086 xd->mi[0]->ref_frame[1] = NONE;
3087 // Initialize interp_filter here so we do not have to check for inter block
3088 // modes in get_pred_context_switchable_interp()
3089 xd->mi[0]->interp_filter = SWITCHABLE_FILTERS;
3091 if (bsize >= BLOCK_8X8) {
3092 if (rd_pick_intra_sby_mode(cpi, x, &rate_y, &rate_y_tokenonly, &dist_y,
3093 &y_skip, bsize, best_rd) >= best_rd) {
3094 rd_cost->rate = INT_MAX;
3099 if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate_y, &rate_y_tokenonly,
3100 &dist_y, best_rd) >= best_rd) {
3101 rd_cost->rate = INT_MAX;
3105 max_uv_tx_size = uv_txsize_lookup[bsize][xd->mi[0]->tx_size]
3106 [pd[1].subsampling_x][pd[1].subsampling_y];
3107 rd_pick_intra_sbuv_mode(cpi, x, ctx, &rate_uv, &rate_uv_tokenonly, &dist_uv,
3108 &uv_skip, VPXMAX(BLOCK_8X8, bsize), max_uv_tx_size);
3110 if (y_skip && uv_skip) {
3111 rd_cost->rate = rate_y + rate_uv - rate_y_tokenonly - rate_uv_tokenonly +
3112 vp9_cost_bit(vp9_get_skip_prob(cm, xd), 1);
3113 rd_cost->dist = dist_y + dist_uv;
3116 rate_y + rate_uv + vp9_cost_bit(vp9_get_skip_prob(cm, xd), 0);
3117 rd_cost->dist = dist_y + dist_uv;
3120 ctx->mic = *xd->mi[0];
3121 ctx->mbmi_ext = *x->mbmi_ext;
3122 rd_cost->rdcost = RDCOST(x->rdmult, x->rddiv, rd_cost->rate, rd_cost->dist);
3125 // This function is designed to apply a bias or adjustment to an rd value based
3126 // on the relative variance of the source and reconstruction.
3127 #define LOW_VAR_THRESH 250
3128 #define VAR_MULT 250
3129 static unsigned int max_var_adjust[VP9E_CONTENT_INVALID] = { 16, 16, 250 };
3131 static void rd_variance_adjustment(VP9_COMP *cpi, MACROBLOCK *x,
3132 BLOCK_SIZE bsize, int64_t *this_rd,
3133 struct buf_2d *recon,
3134 MV_REFERENCE_FRAME ref_frame,
3135 PREDICTION_MODE this_mode) {
3136 MACROBLOCKD *const xd = &x->e_mbd;
3137 unsigned int rec_variance;
3138 unsigned int src_variance;
3139 unsigned int src_rec_min;
3140 unsigned int absvar_diff = 0;
3141 unsigned int var_factor = 0;
3142 unsigned int adj_max;
3143 const int bw = num_8x8_blocks_wide_lookup[bsize];
3144 const int bh = num_8x8_blocks_high_lookup[bsize];
3145 vp9e_tune_content content_type = cpi->oxcf.content;
3147 if (*this_rd == INT64_MAX) return;
3149 #if CONFIG_VP9_HIGHBITDEPTH
3150 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
3151 rec_variance = vp9_high_get_sby_variance(cpi, recon, bsize, xd->bd);
3153 vp9_high_get_sby_variance(cpi, &x->plane[0].src, bsize, xd->bd);
3155 rec_variance = vp9_get_sby_variance(cpi, recon, bsize);
3156 src_variance = vp9_get_sby_variance(cpi, &x->plane[0].src, bsize);
3159 rec_variance = vp9_get_sby_variance(cpi, recon, bsize);
3160 src_variance = vp9_get_sby_variance(cpi, &x->plane[0].src, bsize);
3161 #endif // CONFIG_VP9_HIGHBITDEPTH
3163 // Scale based on area in 8x8 blocks
3164 rec_variance /= (bw * bh);
3165 src_variance /= (bw * bh);
3167 // Lower of source (raw per pixel value) and recon variance. Note that
3168 // if the source per pixel is 0 then the recon value here will not be per
3169 // pixel (see above) so will likely be much larger.
3170 src_rec_min = VPXMIN(src_variance, rec_variance);
3172 if (src_rec_min > LOW_VAR_THRESH) return;
3174 absvar_diff = (src_variance > rec_variance) ? (src_variance - rec_variance)
3175 : (rec_variance - src_variance);
3177 adj_max = max_var_adjust[content_type];
3180 (unsigned int)((int64_t)VAR_MULT * absvar_diff) / VPXMAX(1, src_variance);
3181 var_factor = VPXMIN(adj_max, var_factor);
3183 *this_rd += (*this_rd * var_factor) / 100;
3185 if (content_type == VP9E_CONTENT_FILM) {
3186 if (src_rec_min <= LOW_VAR_THRESH / 2) {
3187 if (ref_frame == INTRA_FRAME) {
3188 if (this_mode == DC_PRED)
3191 *this_rd += (*this_rd / 4);
3199 // Do we have an internal image edge (e.g. formatting bars).
3200 int vp9_internal_image_edge(VP9_COMP *cpi) {
3201 return (cpi->oxcf.pass == 2) &&
3202 ((cpi->twopass.this_frame_stats.inactive_zone_rows > 0) ||
3203 (cpi->twopass.this_frame_stats.inactive_zone_cols > 0));
3206 // Checks to see if a super block is on a horizontal image edge.
3207 // In most cases this is the "real" edge unless there are formatting
3208 // bars embedded in the stream.
3209 int vp9_active_h_edge(VP9_COMP *cpi, int mi_row, int mi_step) {
3211 int bottom_edge = cpi->common.mi_rows;
3212 int is_active_h_edge = 0;
3214 // For two pass account for any formatting bars detected.
3215 if (cpi->oxcf.pass == 2) {
3216 TWO_PASS *twopass = &cpi->twopass;
3218 // The inactive region is specified in MBs not mi units.
3219 // The image edge is in the following MB row.
3220 top_edge += (int)(twopass->this_frame_stats.inactive_zone_rows * 2);
3222 bottom_edge -= (int)(twopass->this_frame_stats.inactive_zone_rows * 2);
3223 bottom_edge = VPXMAX(top_edge, bottom_edge);
3226 if (((top_edge >= mi_row) && (top_edge < (mi_row + mi_step))) ||
3227 ((bottom_edge >= mi_row) && (bottom_edge < (mi_row + mi_step)))) {
3228 is_active_h_edge = 1;
3230 return is_active_h_edge;
3233 // Checks to see if a super block is on a vertical image edge.
3234 // In most cases this is the "real" edge unless there are formatting
3235 // bars embedded in the stream.
3236 int vp9_active_v_edge(VP9_COMP *cpi, int mi_col, int mi_step) {
3238 int right_edge = cpi->common.mi_cols;
3239 int is_active_v_edge = 0;
3241 // For two pass account for any formatting bars detected.
3242 if (cpi->oxcf.pass == 2) {
3243 TWO_PASS *twopass = &cpi->twopass;
3245 // The inactive region is specified in MBs not mi units.
3246 // The image edge is in the following MB row.
3247 left_edge += (int)(twopass->this_frame_stats.inactive_zone_cols * 2);
3249 right_edge -= (int)(twopass->this_frame_stats.inactive_zone_cols * 2);
3250 right_edge = VPXMAX(left_edge, right_edge);
3253 if (((left_edge >= mi_col) && (left_edge < (mi_col + mi_step))) ||
3254 ((right_edge >= mi_col) && (right_edge < (mi_col + mi_step)))) {
3255 is_active_v_edge = 1;
3257 return is_active_v_edge;
3260 // Checks to see if a super block is at the edge of the active image.
3261 // In most cases this is the "real" edge unless there are formatting
3262 // bars embedded in the stream.
3263 int vp9_active_edge_sb(VP9_COMP *cpi, int mi_row, int mi_col) {
3264 return vp9_active_h_edge(cpi, mi_row, MI_BLOCK_SIZE) ||
3265 vp9_active_v_edge(cpi, mi_col, MI_BLOCK_SIZE);
3268 void vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, TileDataEnc *tile_data,
3269 MACROBLOCK *x, int mi_row, int mi_col,
3270 RD_COST *rd_cost, BLOCK_SIZE bsize,
3271 PICK_MODE_CONTEXT *ctx, int64_t best_rd_so_far) {
3272 VP9_COMMON *const cm = &cpi->common;
3273 TileInfo *const tile_info = &tile_data->tile_info;
3274 RD_OPT *const rd_opt = &cpi->rd;
3275 SPEED_FEATURES *const sf = &cpi->sf;
3276 MACROBLOCKD *const xd = &x->e_mbd;
3277 MODE_INFO *const mi = xd->mi[0];
3278 MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
3279 const struct segmentation *const seg = &cm->seg;
3280 PREDICTION_MODE this_mode;
3281 MV_REFERENCE_FRAME ref_frame, second_ref_frame;
3282 unsigned char segment_id = mi->segment_id;
3283 int comp_pred, i, k;
3284 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
3285 struct buf_2d yv12_mb[4][MAX_MB_PLANE];
3286 int_mv single_newmv[MAX_REF_FRAMES] = { { 0 } };
3287 INTERP_FILTER single_inter_filter[MB_MODE_COUNT][MAX_REF_FRAMES];
3288 int single_skippable[MB_MODE_COUNT][MAX_REF_FRAMES];
3289 static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
3291 int64_t best_rd = best_rd_so_far;
3292 int64_t best_pred_diff[REFERENCE_MODES];
3293 int64_t best_pred_rd[REFERENCE_MODES];
3294 int64_t best_filter_rd[SWITCHABLE_FILTER_CONTEXTS];
3295 int64_t best_filter_diff[SWITCHABLE_FILTER_CONTEXTS];
3296 MODE_INFO best_mbmode;
3297 int best_mode_skippable = 0;
3298 int midx, best_mode_index = -1;
3299 unsigned int ref_costs_single[MAX_REF_FRAMES], ref_costs_comp[MAX_REF_FRAMES];
3300 vpx_prob comp_mode_p;
3301 int64_t best_intra_rd = INT64_MAX;
3302 unsigned int best_pred_sse = UINT_MAX;
3303 PREDICTION_MODE best_intra_mode = DC_PRED;
3304 int rate_uv_intra[TX_SIZES], rate_uv_tokenonly[TX_SIZES];
3305 int64_t dist_uv[TX_SIZES];
3306 int skip_uv[TX_SIZES];
3307 PREDICTION_MODE mode_uv[TX_SIZES];
3308 const int intra_cost_penalty =
3309 vp9_get_intra_cost_penalty(cpi, bsize, cm->base_qindex, cm->y_dc_delta_q);
3311 uint8_t ref_frame_skip_mask[2] = { 0, 1 };
3312 uint16_t mode_skip_mask[MAX_REF_FRAMES] = { 0 };
3313 int mode_skip_start = sf->mode_skip_start + 1;
3314 const int *const rd_threshes = rd_opt->threshes[segment_id][bsize];
3315 const int *const rd_thresh_freq_fact = tile_data->thresh_freq_fact[bsize];
3316 int64_t mode_threshold[MAX_MODES];
3317 int8_t *tile_mode_map = tile_data->mode_map[bsize];
3318 int8_t mode_map[MAX_MODES]; // Maintain mode_map information locally to avoid
3319 // lock mechanism involved with reads from
3321 const int mode_search_skip_flags = sf->mode_search_skip_flags;
3322 const int is_rect_partition =
3323 num_4x4_blocks_wide_lookup[bsize] != num_4x4_blocks_high_lookup[bsize];
3324 int64_t mask_filter = 0;
3325 int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
3327 struct buf_2d *recon;
3328 struct buf_2d recon_buf;
3329 #if CONFIG_VP9_HIGHBITDEPTH
3330 DECLARE_ALIGNED(16, uint16_t, recon16[64 * 64]);
3331 recon_buf.buf = xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH
3332 ? CONVERT_TO_BYTEPTR(recon16)
3333 : (uint8_t *)recon16;
3335 DECLARE_ALIGNED(16, uint8_t, recon8[64 * 64]);
3336 recon_buf.buf = recon8;
3337 #endif // CONFIG_VP9_HIGHBITDEPTH
3338 recon_buf.stride = 64;
3339 recon = cpi->oxcf.content == VP9E_CONTENT_FILM ? &recon_buf : 0;
3341 vp9_zero(best_mbmode);
3343 x->skip_encode = sf->skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
3345 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i) filter_cache[i] = INT64_MAX;
3347 estimate_ref_frame_costs(cm, xd, segment_id, ref_costs_single, ref_costs_comp,
3350 for (i = 0; i < REFERENCE_MODES; ++i) best_pred_rd[i] = INT64_MAX;
3351 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++)
3352 best_filter_rd[i] = INT64_MAX;
3353 for (i = 0; i < TX_SIZES; i++) rate_uv_intra[i] = INT_MAX;
3354 for (i = 0; i < MAX_REF_FRAMES; ++i) x->pred_sse[i] = INT_MAX;
3355 for (i = 0; i < MB_MODE_COUNT; ++i) {
3356 for (k = 0; k < MAX_REF_FRAMES; ++k) {
3357 single_inter_filter[i][k] = SWITCHABLE;
3358 single_skippable[i][k] = 0;
3362 rd_cost->rate = INT_MAX;
3364 for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
3365 x->pred_mv_sad[ref_frame] = INT_MAX;
3366 if ((cpi->ref_frame_flags & flag_list[ref_frame]) &&
3367 !(is_rect_partition && (ctx->skip_ref_frame_mask & (1 << ref_frame)))) {
3368 assert(get_ref_frame_buffer(cpi, ref_frame) != NULL);
3369 setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col,
3370 frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb);
3372 frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
3373 frame_mv[ZEROMV][ref_frame].as_int = 0;
3376 for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
3377 if (!(cpi->ref_frame_flags & flag_list[ref_frame])) {
3378 // Skip checking missing references in both single and compound reference
3379 // modes. Note that a mode will be skipped if both reference frames
3381 ref_frame_skip_mask[0] |= (1 << ref_frame);
3382 ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
3383 } else if (sf->reference_masking) {
3384 for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
3385 // Skip fixed mv modes for poor references
3386 if ((x->pred_mv_sad[ref_frame] >> 2) > x->pred_mv_sad[i]) {
3387 mode_skip_mask[ref_frame] |= INTER_NEAREST_NEAR_ZERO;
3392 // If the segment reference frame feature is enabled....
3393 // then do nothing if the current ref frame is not allowed..
3394 if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) &&
3395 get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) {
3396 ref_frame_skip_mask[0] |= (1 << ref_frame);
3397 ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
3401 // Disable this drop out case if the ref frame
3402 // segment level feature is enabled for this segment. This is to
3403 // prevent the possibility that we end up unable to pick any mode.
3404 if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) {
3405 // Only consider ZEROMV/ALTREF_FRAME for alt ref frame,
3406 // unless ARNR filtering is enabled in which case we want
3407 // an unfiltered alternative. We allow near/nearest as well
3408 // because they may result in zero-zero MVs but be cheaper.
3409 if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) {
3410 ref_frame_skip_mask[0] = (1 << LAST_FRAME) | (1 << GOLDEN_FRAME);
3411 ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK;
3412 mode_skip_mask[ALTREF_FRAME] = ~INTER_NEAREST_NEAR_ZERO;
3413 if (frame_mv[NEARMV][ALTREF_FRAME].as_int != 0)
3414 mode_skip_mask[ALTREF_FRAME] |= (1 << NEARMV);
3415 if (frame_mv[NEARESTMV][ALTREF_FRAME].as_int != 0)
3416 mode_skip_mask[ALTREF_FRAME] |= (1 << NEARESTMV);
3420 if (cpi->rc.is_src_frame_alt_ref) {
3421 if (sf->alt_ref_search_fp) {
3422 mode_skip_mask[ALTREF_FRAME] = 0;
3423 ref_frame_skip_mask[0] = ~(1 << ALTREF_FRAME);
3424 ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK;
3428 if (sf->alt_ref_search_fp)
3429 if (!cm->show_frame && x->pred_mv_sad[GOLDEN_FRAME] < INT_MAX)
3430 if (x->pred_mv_sad[ALTREF_FRAME] > (x->pred_mv_sad[GOLDEN_FRAME] << 1))
3431 mode_skip_mask[ALTREF_FRAME] |= INTER_ALL;
3433 if (sf->adaptive_mode_search) {
3434 if (cm->show_frame && !cpi->rc.is_src_frame_alt_ref &&
3435 cpi->rc.frames_since_golden >= 3)
3436 if (x->pred_mv_sad[GOLDEN_FRAME] > (x->pred_mv_sad[LAST_FRAME] << 1))
3437 mode_skip_mask[GOLDEN_FRAME] |= INTER_ALL;
3440 if (bsize > sf->max_intra_bsize) {
3441 ref_frame_skip_mask[0] |= (1 << INTRA_FRAME);
3442 ref_frame_skip_mask[1] |= (1 << INTRA_FRAME);
3445 mode_skip_mask[INTRA_FRAME] |=
3446 ~(sf->intra_y_mode_mask[max_txsize_lookup[bsize]]);
3448 for (i = 0; i <= LAST_NEW_MV_INDEX; ++i) mode_threshold[i] = 0;
3450 for (i = LAST_NEW_MV_INDEX + 1; i < MAX_MODES; ++i)
3451 mode_threshold[i] = ((int64_t)rd_threshes[i] * rd_thresh_freq_fact[i]) >> 5;
3453 midx = sf->schedule_mode_search ? mode_skip_start : 0;
3456 uint8_t end_pos = 0;
3457 for (i = 5; i < midx; ++i) {
3458 if (mode_threshold[tile_mode_map[i - 1]] >
3459 mode_threshold[tile_mode_map[i]]) {
3460 uint8_t tmp = tile_mode_map[i];
3461 tile_mode_map[i] = tile_mode_map[i - 1];
3462 tile_mode_map[i - 1] = tmp;
3469 memcpy(mode_map, tile_mode_map, sizeof(mode_map));
3471 for (midx = 0; midx < MAX_MODES; ++midx) {
3472 int mode_index = mode_map[midx];
3473 int mode_excluded = 0;
3474 int64_t this_rd = INT64_MAX;
3475 int disable_skip = 0;
3476 int compmode_cost = 0;
3477 int rate2 = 0, rate_y = 0, rate_uv = 0;
3478 int64_t distortion2 = 0, distortion_y = 0, distortion_uv = 0;
3481 int64_t total_sse = INT64_MAX;
3484 this_mode = vp9_mode_order[mode_index].mode;
3485 ref_frame = vp9_mode_order[mode_index].ref_frame[0];
3486 second_ref_frame = vp9_mode_order[mode_index].ref_frame[1];
3488 vp9_zero(x->sum_y_eobs);
3490 if (is_rect_partition) {
3491 if (ctx->skip_ref_frame_mask & (1 << ref_frame)) continue;
3492 if (second_ref_frame > 0 &&
3493 (ctx->skip_ref_frame_mask & (1 << second_ref_frame)))
3497 // Look at the reference frame of the best mode so far and set the
3498 // skip mask to look at a subset of the remaining modes.
3499 if (midx == mode_skip_start && best_mode_index >= 0) {
3500 switch (best_mbmode.ref_frame[0]) {
3501 case INTRA_FRAME: break;
3502 case LAST_FRAME: ref_frame_skip_mask[0] |= LAST_FRAME_MODE_MASK; break;
3504 ref_frame_skip_mask[0] |= GOLDEN_FRAME_MODE_MASK;
3506 case ALTREF_FRAME: ref_frame_skip_mask[0] |= ALT_REF_MODE_MASK; break;
3508 case MAX_REF_FRAMES: assert(0 && "Invalid Reference frame"); break;
3512 if ((ref_frame_skip_mask[0] & (1 << ref_frame)) &&
3513 (ref_frame_skip_mask[1] & (1 << VPXMAX(0, second_ref_frame))))
3516 if (mode_skip_mask[ref_frame] & (1 << this_mode)) continue;
3518 // Test best rd so far against threshold for trying this mode.
3519 if (best_mode_skippable && sf->schedule_mode_search)
3520 mode_threshold[mode_index] <<= 1;
3522 if (best_rd < mode_threshold[mode_index]) continue;
3524 // This is only used in motion vector unit test.
3525 if (cpi->oxcf.motion_vector_unit_test && ref_frame == INTRA_FRAME) continue;
3527 if (sf->motion_field_mode_search) {
3528 const int mi_width = VPXMIN(num_8x8_blocks_wide_lookup[bsize],
3529 tile_info->mi_col_end - mi_col);
3530 const int mi_height = VPXMIN(num_8x8_blocks_high_lookup[bsize],
3531 tile_info->mi_row_end - mi_row);
3532 const int bsl = mi_width_log2_lookup[bsize];
3533 int cb_partition_search_ctrl =
3534 (((mi_row + mi_col) >> bsl) +
3535 get_chessboard_index(cm->current_video_frame)) &
3538 int const_motion = 1;
3539 int skip_ref_frame = !cb_partition_search_ctrl;
3540 MV_REFERENCE_FRAME rf = NONE;
3542 ref_mv.as_int = INVALID_MV;
3544 if ((mi_row - 1) >= tile_info->mi_row_start) {
3545 ref_mv = xd->mi[-xd->mi_stride]->mv[0];
3546 rf = xd->mi[-xd->mi_stride]->ref_frame[0];
3547 for (i = 0; i < mi_width; ++i) {
3548 ref_mi = xd->mi[-xd->mi_stride + i];
3549 const_motion &= (ref_mv.as_int == ref_mi->mv[0].as_int) &&
3550 (ref_frame == ref_mi->ref_frame[0]);
3551 skip_ref_frame &= (rf == ref_mi->ref_frame[0]);
3555 if ((mi_col - 1) >= tile_info->mi_col_start) {
3556 if (ref_mv.as_int == INVALID_MV) ref_mv = xd->mi[-1]->mv[0];
3557 if (rf == NONE) rf = xd->mi[-1]->ref_frame[0];
3558 for (i = 0; i < mi_height; ++i) {
3559 ref_mi = xd->mi[i * xd->mi_stride - 1];
3560 const_motion &= (ref_mv.as_int == ref_mi->mv[0].as_int) &&
3561 (ref_frame == ref_mi->ref_frame[0]);
3562 skip_ref_frame &= (rf == ref_mi->ref_frame[0]);
3566 if (skip_ref_frame && this_mode != NEARESTMV && this_mode != NEWMV)
3567 if (rf > INTRA_FRAME)
3568 if (ref_frame != rf) continue;
3571 if (this_mode == NEARMV || this_mode == ZEROMV) continue;
3574 comp_pred = second_ref_frame > INTRA_FRAME;
3576 if (!cpi->allow_comp_inter_inter) continue;
3578 if (cm->ref_frame_sign_bias[ref_frame] ==
3579 cm->ref_frame_sign_bias[second_ref_frame])
3582 // Skip compound inter modes if ARF is not available.
3583 if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue;
3585 // Do not allow compound prediction if the segment level reference frame
3586 // feature is in use as in this case there can only be one reference.
3587 if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue;
3589 if ((mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) &&
3590 best_mode_index >= 0 && best_mbmode.ref_frame[0] == INTRA_FRAME)
3593 mode_excluded = cm->reference_mode == SINGLE_REFERENCE;
3595 if (ref_frame != INTRA_FRAME)
3596 mode_excluded = cm->reference_mode == COMPOUND_REFERENCE;
3599 if (ref_frame == INTRA_FRAME) {
3600 if (sf->adaptive_mode_search)
3601 if ((x->source_variance << num_pels_log2_lookup[bsize]) > best_pred_sse)
3604 if (this_mode != DC_PRED) {
3605 // Disable intra modes other than DC_PRED for blocks with low variance
3606 // Threshold for intra skipping based on source variance
3607 // TODO(debargha): Specialize the threshold for super block sizes
3608 const unsigned int skip_intra_var_thresh =
3609 (cpi->oxcf.content == VP9E_CONTENT_FILM) ? 0 : 64;
3610 if ((mode_search_skip_flags & FLAG_SKIP_INTRA_LOWVAR) &&
3611 x->source_variance < skip_intra_var_thresh)
3613 // Only search the oblique modes if the best so far is
3614 // one of the neighboring directional modes
3615 if ((mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) &&
3616 (this_mode >= D45_PRED && this_mode <= TM_PRED)) {
3617 if (best_mode_index >= 0 && best_mbmode.ref_frame[0] > INTRA_FRAME)
3620 if (mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
3621 if (conditional_skipintra(this_mode, best_intra_mode)) continue;
3625 const MV_REFERENCE_FRAME ref_frames[2] = { ref_frame, second_ref_frame };
3626 if (!check_best_zero_mv(cpi, mbmi_ext->mode_context, frame_mv, this_mode,
3631 mi->mode = this_mode;
3632 mi->uv_mode = DC_PRED;
3633 mi->ref_frame[0] = ref_frame;
3634 mi->ref_frame[1] = second_ref_frame;
3635 // Evaluate all sub-pel filters irrespective of whether we can use
3636 // them for this frame.
3638 cm->interp_filter == SWITCHABLE ? EIGHTTAP : cm->interp_filter;
3639 mi->mv[0].as_int = mi->mv[1].as_int = 0;
3642 set_ref_ptrs(cm, xd, ref_frame, second_ref_frame);
3644 // Select prediction reference frames.
3645 for (i = 0; i < MAX_MB_PLANE; i++) {
3646 xd->plane[i].pre[0] = yv12_mb[ref_frame][i];
3647 if (comp_pred) xd->plane[i].pre[1] = yv12_mb[second_ref_frame][i];
3650 if (ref_frame == INTRA_FRAME) {
3652 struct macroblockd_plane *const pd = &xd->plane[1];
3653 memset(x->skip_txfm, 0, sizeof(x->skip_txfm));
3654 super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable, NULL, bsize,
3656 if (rate_y == INT_MAX) continue;
3658 uv_tx = uv_txsize_lookup[bsize][mi->tx_size][pd->subsampling_x]
3659 [pd->subsampling_y];
3660 if (rate_uv_intra[uv_tx] == INT_MAX) {
3661 choose_intra_uv_mode(cpi, x, ctx, bsize, uv_tx, &rate_uv_intra[uv_tx],
3662 &rate_uv_tokenonly[uv_tx], &dist_uv[uv_tx],
3663 &skip_uv[uv_tx], &mode_uv[uv_tx]);
3666 rate_uv = rate_uv_tokenonly[uv_tx];
3667 distortion_uv = dist_uv[uv_tx];
3668 skippable = skippable && skip_uv[uv_tx];
3669 mi->uv_mode = mode_uv[uv_tx];
3671 rate2 = rate_y + cpi->mbmode_cost[mi->mode] + rate_uv_intra[uv_tx];
3672 if (this_mode != DC_PRED && this_mode != TM_PRED)
3673 rate2 += intra_cost_penalty;
3674 distortion2 = distortion_y + distortion_uv;
3676 this_rd = handle_inter_mode(
3677 cpi, x, bsize, &rate2, &distortion2, &skippable, &rate_y, &rate_uv,
3678 recon, &disable_skip, frame_mv, mi_row, mi_col, single_newmv,
3679 single_inter_filter, single_skippable, &total_sse, best_rd,
3680 &mask_filter, filter_cache);
3681 if (this_rd == INT64_MAX) continue;
3683 compmode_cost = vp9_cost_bit(comp_mode_p, comp_pred);
3685 if (cm->reference_mode == REFERENCE_MODE_SELECT) rate2 += compmode_cost;
3688 // Estimate the reference frame signaling cost and add it
3689 // to the rolling cost variable.
3691 rate2 += ref_costs_comp[ref_frame];
3693 rate2 += ref_costs_single[ref_frame];
3696 if (!disable_skip) {
3697 const vpx_prob skip_prob = vp9_get_skip_prob(cm, xd);
3698 const int skip_cost0 = vp9_cost_bit(skip_prob, 0);
3699 const int skip_cost1 = vp9_cost_bit(skip_prob, 1);
3702 // Back out the coefficient coding costs
3703 rate2 -= (rate_y + rate_uv);
3705 // Cost the skip mb case
3706 rate2 += skip_cost1;
3707 } else if (ref_frame != INTRA_FRAME && !xd->lossless) {
3708 if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv + skip_cost0,
3710 RDCOST(x->rdmult, x->rddiv, skip_cost1, total_sse)) {
3711 // Add in the cost of the no skip flag.
3712 rate2 += skip_cost0;
3714 // FIXME(rbultje) make this work for splitmv also
3715 assert(total_sse >= 0);
3717 rate2 += skip_cost1;
3718 distortion2 = total_sse;
3719 rate2 -= (rate_y + rate_uv);
3723 // Add in the cost of the no skip flag.
3724 rate2 += skip_cost0;
3727 // Calculate the final RD estimate for this mode.
3728 this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
3731 // Apply an adjustment to the rd value based on the similarity of the
3732 // source variance and reconstructed variance.
3734 rd_variance_adjustment(cpi, x, bsize, &this_rd, recon, ref_frame,
3738 if (ref_frame == INTRA_FRAME) {
3739 // Keep record of best intra rd
3740 if (this_rd < best_intra_rd) {
3741 best_intra_rd = this_rd;
3742 best_intra_mode = mi->mode;
3746 if (!disable_skip && ref_frame == INTRA_FRAME) {
3747 for (i = 0; i < REFERENCE_MODES; ++i)
3748 best_pred_rd[i] = VPXMIN(best_pred_rd[i], this_rd);
3749 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++)
3750 best_filter_rd[i] = VPXMIN(best_filter_rd[i], this_rd);
3753 // Did this mode help.. i.e. is it the new best mode
3754 if (this_rd < best_rd || x->skip) {
3755 int max_plane = MAX_MB_PLANE;
3756 if (!mode_excluded) {
3757 // Note index of best mode so far
3758 best_mode_index = mode_index;
3760 if (ref_frame == INTRA_FRAME) {
3761 /* required for left and above block mv */
3762 mi->mv[0].as_int = 0;
3764 // Initialize interp_filter here so we do not have to check for
3765 // inter block modes in get_pred_context_switchable_interp()
3766 mi->interp_filter = SWITCHABLE_FILTERS;
3768 best_pred_sse = x->pred_sse[ref_frame];
3771 rd_cost->rate = rate2;
3772 rd_cost->dist = distortion2;
3773 rd_cost->rdcost = this_rd;
3776 best_skip2 = this_skip2;
3777 best_mode_skippable = skippable;
3779 if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, max_plane);
3780 memcpy(ctx->zcoeff_blk, x->zcoeff_blk[mi->tx_size],
3781 sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
3782 ctx->sum_y_eobs = x->sum_y_eobs[mi->tx_size];
3784 // TODO(debargha): enhance this test with a better distortion prediction
3785 // based on qp, activity mask and history
3786 if ((mode_search_skip_flags & FLAG_EARLY_TERMINATE) &&
3787 (mode_index > MIN_EARLY_TERM_INDEX)) {
3788 int qstep = xd->plane[0].dequant[1];
3789 // TODO(debargha): Enhance this by specializing for each mode_index
3791 #if CONFIG_VP9_HIGHBITDEPTH
3792 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
3793 qstep >>= (xd->bd - 8);
3795 #endif // CONFIG_VP9_HIGHBITDEPTH
3796 if (x->source_variance < UINT_MAX) {
3797 const int var_adjust = (x->source_variance < 16);
3798 scale -= var_adjust;
3800 if (ref_frame > INTRA_FRAME && distortion2 * scale < qstep * qstep) {
3807 /* keep record of best compound/single-only prediction */
3808 if (!disable_skip && ref_frame != INTRA_FRAME) {
3809 int64_t single_rd, hybrid_rd, single_rate, hybrid_rate;
3811 if (cm->reference_mode == REFERENCE_MODE_SELECT) {
3812 single_rate = rate2 - compmode_cost;
3813 hybrid_rate = rate2;
3815 single_rate = rate2;
3816 hybrid_rate = rate2 + compmode_cost;
3819 single_rd = RDCOST(x->rdmult, x->rddiv, single_rate, distortion2);
3820 hybrid_rd = RDCOST(x->rdmult, x->rddiv, hybrid_rate, distortion2);
3823 if (single_rd < best_pred_rd[SINGLE_REFERENCE])
3824 best_pred_rd[SINGLE_REFERENCE] = single_rd;
3826 if (single_rd < best_pred_rd[COMPOUND_REFERENCE])
3827 best_pred_rd[COMPOUND_REFERENCE] = single_rd;
3829 if (hybrid_rd < best_pred_rd[REFERENCE_MODE_SELECT])
3830 best_pred_rd[REFERENCE_MODE_SELECT] = hybrid_rd;
3832 /* keep record of best filter type */
3833 if (!mode_excluded && cm->interp_filter != BILINEAR) {
3835 filter_cache[cm->interp_filter == SWITCHABLE ? SWITCHABLE_FILTERS
3836 : cm->interp_filter];
3838 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) {
3840 if (ref == INT64_MAX)
3842 else if (filter_cache[i] == INT64_MAX)
3843 // when early termination is triggered, the encoder does not have
3844 // access to the rate-distortion cost. it only knows that the cost
3845 // should be above the maximum valid value. hence it takes the known
3846 // maximum plus an arbitrary constant as the rate-distortion cost.
3847 adj_rd = mask_filter - ref + 10;
3849 adj_rd = filter_cache[i] - ref;
3852 best_filter_rd[i] = VPXMIN(best_filter_rd[i], adj_rd);
3857 if (early_term) break;
3859 if (x->skip && !comp_pred) break;
3862 // The inter modes' rate costs are not calculated precisely in some cases.
3863 // Therefore, sometimes, NEWMV is chosen instead of NEARESTMV, NEARMV, and
3864 // ZEROMV. Here, checks are added for those cases, and the mode decisions
3866 if (best_mbmode.mode == NEWMV) {
3867 const MV_REFERENCE_FRAME refs[2] = { best_mbmode.ref_frame[0],
3868 best_mbmode.ref_frame[1] };
3869 int comp_pred_mode = refs[1] > INTRA_FRAME;
3871 if (frame_mv[NEARESTMV][refs[0]].as_int == best_mbmode.mv[0].as_int &&
3873 frame_mv[NEARESTMV][refs[1]].as_int == best_mbmode.mv[1].as_int) ||
3875 best_mbmode.mode = NEARESTMV;
3876 else if (frame_mv[NEARMV][refs[0]].as_int == best_mbmode.mv[0].as_int &&
3878 frame_mv[NEARMV][refs[1]].as_int == best_mbmode.mv[1].as_int) ||
3880 best_mbmode.mode = NEARMV;
3881 else if (best_mbmode.mv[0].as_int == 0 &&
3882 ((comp_pred_mode && best_mbmode.mv[1].as_int == 0) ||
3884 best_mbmode.mode = ZEROMV;
3887 if (best_mode_index < 0 || best_rd >= best_rd_so_far) {
3888 // If adaptive interp filter is enabled, then the current leaf node of 8x8
3889 // data is needed for sub8x8. Hence preserve the context.
3890 #if CONFIG_CONSISTENT_RECODE
3891 if (bsize == BLOCK_8X8) ctx->mic = *xd->mi[0];
3893 if (cpi->row_mt && bsize == BLOCK_8X8) ctx->mic = *xd->mi[0];
3895 rd_cost->rate = INT_MAX;
3896 rd_cost->rdcost = INT64_MAX;
3900 // If we used an estimate for the uv intra rd in the loop above...
3901 if (sf->use_uv_intra_rd_estimate) {
3902 // Do Intra UV best rd mode selection if best mode choice above was intra.
3903 if (best_mbmode.ref_frame[0] == INTRA_FRAME) {
3906 uv_tx_size = get_uv_tx_size(mi, &xd->plane[1]);
3907 rd_pick_intra_sbuv_mode(cpi, x, ctx, &rate_uv_intra[uv_tx_size],
3908 &rate_uv_tokenonly[uv_tx_size],
3909 &dist_uv[uv_tx_size], &skip_uv[uv_tx_size],
3910 bsize < BLOCK_8X8 ? BLOCK_8X8 : bsize,
3915 assert((cm->interp_filter == SWITCHABLE) ||
3916 (cm->interp_filter == best_mbmode.interp_filter) ||
3917 !is_inter_block(&best_mbmode));
3919 if (!cpi->rc.is_src_frame_alt_ref)
3920 vp9_update_rd_thresh_fact(tile_data->thresh_freq_fact,
3921 sf->adaptive_rd_thresh, bsize, best_mode_index);
3925 x->skip |= best_skip2;
3927 for (i = 0; i < REFERENCE_MODES; ++i) {
3928 if (best_pred_rd[i] == INT64_MAX)
3929 best_pred_diff[i] = INT_MIN;
3931 best_pred_diff[i] = best_rd - best_pred_rd[i];
3935 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) {
3936 if (best_filter_rd[i] == INT64_MAX)
3937 best_filter_diff[i] = 0;
3939 best_filter_diff[i] = best_rd - best_filter_rd[i];
3941 if (cm->interp_filter == SWITCHABLE)
3942 assert(best_filter_diff[SWITCHABLE_FILTERS] == 0);
3944 vp9_zero(best_filter_diff);
3947 // TODO(yunqingwang): Moving this line in front of the above best_filter_diff
3948 // updating code causes PSNR loss. Need to figure out the confliction.
3949 x->skip |= best_mode_skippable;
3951 if (!x->skip && !x->select_tx_size) {
3952 int has_high_freq_coeff = 0;
3954 int max_plane = is_inter_block(xd->mi[0]) ? MAX_MB_PLANE : 1;
3955 for (plane = 0; plane < max_plane; ++plane) {
3956 x->plane[plane].eobs = ctx->eobs_pbuf[plane][1];
3957 has_high_freq_coeff |= vp9_has_high_freq_in_plane(x, bsize, plane);
3960 for (plane = max_plane; plane < MAX_MB_PLANE; ++plane) {
3961 x->plane[plane].eobs = ctx->eobs_pbuf[plane][2];
3962 has_high_freq_coeff |= vp9_has_high_freq_in_plane(x, bsize, plane);
3965 best_mode_skippable |= !has_high_freq_coeff;
3968 assert(best_mode_index >= 0);
3970 store_coding_context(x, ctx, best_mode_index, best_pred_diff,
3971 best_filter_diff, best_mode_skippable);
3974 void vp9_rd_pick_inter_mode_sb_seg_skip(VP9_COMP *cpi, TileDataEnc *tile_data,
3975 MACROBLOCK *x, RD_COST *rd_cost,
3977 PICK_MODE_CONTEXT *ctx,
3978 int64_t best_rd_so_far) {
3979 VP9_COMMON *const cm = &cpi->common;
3980 MACROBLOCKD *const xd = &x->e_mbd;
3981 MODE_INFO *const mi = xd->mi[0];
3982 unsigned char segment_id = mi->segment_id;
3983 const int comp_pred = 0;
3985 int64_t best_pred_diff[REFERENCE_MODES];
3986 int64_t best_filter_diff[SWITCHABLE_FILTER_CONTEXTS];
3987 unsigned int ref_costs_single[MAX_REF_FRAMES], ref_costs_comp[MAX_REF_FRAMES];
3988 vpx_prob comp_mode_p;
3989 INTERP_FILTER best_filter = SWITCHABLE;
3990 int64_t this_rd = INT64_MAX;
3992 const int64_t distortion2 = 0;
3994 x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
3996 estimate_ref_frame_costs(cm, xd, segment_id, ref_costs_single, ref_costs_comp,
3999 for (i = 0; i < MAX_REF_FRAMES; ++i) x->pred_sse[i] = INT_MAX;
4000 for (i = LAST_FRAME; i < MAX_REF_FRAMES; ++i) x->pred_mv_sad[i] = INT_MAX;
4002 rd_cost->rate = INT_MAX;
4004 assert(segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP));
4007 mi->uv_mode = DC_PRED;
4008 mi->ref_frame[0] = LAST_FRAME;
4009 mi->ref_frame[1] = NONE;
4010 mi->mv[0].as_int = 0;
4013 ctx->sum_y_eobs = 0;
4015 if (cm->interp_filter != BILINEAR) {
4016 best_filter = EIGHTTAP;
4017 if (cm->interp_filter == SWITCHABLE &&
4018 x->source_variance >= cpi->sf.disable_filter_search_var_thresh) {
4020 int best_rs = INT_MAX;
4021 for (i = 0; i < SWITCHABLE_FILTERS; ++i) {
4022 mi->interp_filter = i;
4023 rs = vp9_get_switchable_rate(cpi, xd);
4026 best_filter = mi->interp_filter;
4031 // Set the appropriate filter
4032 if (cm->interp_filter == SWITCHABLE) {
4033 mi->interp_filter = best_filter;
4034 rate2 += vp9_get_switchable_rate(cpi, xd);
4036 mi->interp_filter = cm->interp_filter;
4039 if (cm->reference_mode == REFERENCE_MODE_SELECT)
4040 rate2 += vp9_cost_bit(comp_mode_p, comp_pred);
4042 // Estimate the reference frame signaling cost and add it
4043 // to the rolling cost variable.
4044 rate2 += ref_costs_single[LAST_FRAME];
4045 this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
4047 rd_cost->rate = rate2;
4048 rd_cost->dist = distortion2;
4049 rd_cost->rdcost = this_rd;
4051 if (this_rd >= best_rd_so_far) {
4052 rd_cost->rate = INT_MAX;
4053 rd_cost->rdcost = INT64_MAX;
4057 assert((cm->interp_filter == SWITCHABLE) ||
4058 (cm->interp_filter == mi->interp_filter));
4060 vp9_update_rd_thresh_fact(tile_data->thresh_freq_fact,
4061 cpi->sf.adaptive_rd_thresh, bsize, THR_ZEROMV);
4063 vp9_zero(best_pred_diff);
4064 vp9_zero(best_filter_diff);
4066 if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, MAX_MB_PLANE);
4067 store_coding_context(x, ctx, THR_ZEROMV, best_pred_diff, best_filter_diff, 0);
4070 void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, TileDataEnc *tile_data,
4071 MACROBLOCK *x, int mi_row, int mi_col,
4072 RD_COST *rd_cost, BLOCK_SIZE bsize,
4073 PICK_MODE_CONTEXT *ctx,
4074 int64_t best_rd_so_far) {
4075 VP9_COMMON *const cm = &cpi->common;
4076 RD_OPT *const rd_opt = &cpi->rd;
4077 SPEED_FEATURES *const sf = &cpi->sf;
4078 MACROBLOCKD *const xd = &x->e_mbd;
4079 MODE_INFO *const mi = xd->mi[0];
4080 const struct segmentation *const seg = &cm->seg;
4081 MV_REFERENCE_FRAME ref_frame, second_ref_frame;
4082 unsigned char segment_id = mi->segment_id;
4084 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
4085 struct buf_2d yv12_mb[4][MAX_MB_PLANE];
4086 static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
4088 int64_t best_rd = best_rd_so_far;
4089 int64_t best_yrd = best_rd_so_far; // FIXME(rbultje) more precise
4090 int64_t best_pred_diff[REFERENCE_MODES];
4091 int64_t best_pred_rd[REFERENCE_MODES];
4092 int64_t best_filter_rd[SWITCHABLE_FILTER_CONTEXTS];
4093 int64_t best_filter_diff[SWITCHABLE_FILTER_CONTEXTS];
4094 MODE_INFO best_mbmode;
4095 int ref_index, best_ref_index = 0;
4096 unsigned int ref_costs_single[MAX_REF_FRAMES], ref_costs_comp[MAX_REF_FRAMES];
4097 vpx_prob comp_mode_p;
4098 INTERP_FILTER tmp_best_filter = SWITCHABLE;
4099 int rate_uv_intra, rate_uv_tokenonly;
4102 PREDICTION_MODE mode_uv = DC_PRED;
4103 const int intra_cost_penalty =
4104 vp9_get_intra_cost_penalty(cpi, bsize, cm->base_qindex, cm->y_dc_delta_q);
4105 int_mv seg_mvs[4][MAX_REF_FRAMES];
4106 b_mode_info best_bmodes[4];
4108 int ref_frame_skip_mask[2] = { 0 };
4109 int64_t mask_filter = 0;
4110 int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
4111 int internal_active_edge =
4112 vp9_active_edge_sb(cpi, mi_row, mi_col) && vp9_internal_image_edge(cpi);
4113 const int *const rd_thresh_freq_fact = tile_data->thresh_freq_fact[bsize];
4115 x->skip_encode = sf->skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
4116 memset(x->zcoeff_blk[TX_4X4], 0, 4);
4117 vp9_zero(best_mbmode);
4119 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i) filter_cache[i] = INT64_MAX;
4121 for (i = 0; i < 4; i++) {
4123 for (j = 0; j < MAX_REF_FRAMES; j++) seg_mvs[i][j].as_int = INVALID_MV;
4126 estimate_ref_frame_costs(cm, xd, segment_id, ref_costs_single, ref_costs_comp,
4129 for (i = 0; i < REFERENCE_MODES; ++i) best_pred_rd[i] = INT64_MAX;
4130 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++)
4131 best_filter_rd[i] = INT64_MAX;
4132 rate_uv_intra = INT_MAX;
4134 rd_cost->rate = INT_MAX;
4136 for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
4137 if (cpi->ref_frame_flags & flag_list[ref_frame]) {
4138 setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col,
4139 frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb);
4141 ref_frame_skip_mask[0] |= (1 << ref_frame);
4142 ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
4144 frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
4145 frame_mv[ZEROMV][ref_frame].as_int = 0;
4148 for (ref_index = 0; ref_index < MAX_REFS; ++ref_index) {
4149 int mode_excluded = 0;
4150 int64_t this_rd = INT64_MAX;
4151 int disable_skip = 0;
4152 int compmode_cost = 0;
4153 int rate2 = 0, rate_y = 0, rate_uv = 0;
4154 int64_t distortion2 = 0, distortion_y = 0, distortion_uv = 0;
4158 int64_t total_sse = INT_MAX;
4160 struct buf_2d backup_yv12[2][MAX_MB_PLANE];
4162 ref_frame = vp9_ref_order[ref_index].ref_frame[0];
4163 second_ref_frame = vp9_ref_order[ref_index].ref_frame[1];
4165 vp9_zero(x->sum_y_eobs);
4167 #if CONFIG_BETTER_HW_COMPATIBILITY
4168 // forbid 8X4 and 4X8 partitions if any reference frame is scaled.
4169 if (bsize == BLOCK_8X4 || bsize == BLOCK_4X8) {
4170 int ref_scaled = ref_frame > INTRA_FRAME &&
4171 vp9_is_scaled(&cm->frame_refs[ref_frame - 1].sf);
4172 if (second_ref_frame > INTRA_FRAME)
4173 ref_scaled += vp9_is_scaled(&cm->frame_refs[second_ref_frame - 1].sf);
4174 if (ref_scaled) continue;
4177 // Look at the reference frame of the best mode so far and set the
4178 // skip mask to look at a subset of the remaining modes.
4179 if (ref_index > 2 && sf->mode_skip_start < MAX_MODES) {
4180 if (ref_index == 3) {
4181 switch (best_mbmode.ref_frame[0]) {
4182 case INTRA_FRAME: break;
4184 ref_frame_skip_mask[0] |= (1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME);
4185 ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
4188 ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | (1 << ALTREF_FRAME);
4189 ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
4192 ref_frame_skip_mask[0] |= (1 << GOLDEN_FRAME) | (1 << LAST_FRAME);
4195 case MAX_REF_FRAMES: assert(0 && "Invalid Reference frame"); break;
4200 if ((ref_frame_skip_mask[0] & (1 << ref_frame)) &&
4201 (ref_frame_skip_mask[1] & (1 << VPXMAX(0, second_ref_frame))))
4204 // Test best rd so far against threshold for trying this mode.
4205 if (!internal_active_edge &&
4206 rd_less_than_thresh(best_rd,
4207 rd_opt->threshes[segment_id][bsize][ref_index],
4208 &rd_thresh_freq_fact[ref_index]))
4211 // This is only used in motion vector unit test.
4212 if (cpi->oxcf.motion_vector_unit_test && ref_frame == INTRA_FRAME) continue;
4214 comp_pred = second_ref_frame > INTRA_FRAME;
4216 if (!cpi->allow_comp_inter_inter) continue;
4218 if (cm->ref_frame_sign_bias[ref_frame] ==
4219 cm->ref_frame_sign_bias[second_ref_frame])
4222 if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue;
4223 // Do not allow compound prediction if the segment level reference frame
4224 // feature is in use as in this case there can only be one reference.
4225 if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue;
4227 if ((sf->mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) &&
4228 best_mbmode.ref_frame[0] == INTRA_FRAME)
4233 mode_excluded = cm->reference_mode == SINGLE_REFERENCE;
4234 else if (ref_frame != INTRA_FRAME)
4235 mode_excluded = cm->reference_mode == COMPOUND_REFERENCE;
4237 // If the segment reference frame feature is enabled....
4238 // then do nothing if the current ref frame is not allowed..
4239 if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) &&
4240 get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) {
4242 // Disable this drop out case if the ref frame
4243 // segment level feature is enabled for this segment. This is to
4244 // prevent the possibility that we end up unable to pick any mode.
4245 } else if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) {
4246 // Only consider ZEROMV/ALTREF_FRAME for alt ref frame,
4247 // unless ARNR filtering is enabled in which case we want
4248 // an unfiltered alternative. We allow near/nearest as well
4249 // because they may result in zero-zero MVs but be cheaper.
4250 if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0))
4254 mi->tx_size = TX_4X4;
4255 mi->uv_mode = DC_PRED;
4256 mi->ref_frame[0] = ref_frame;
4257 mi->ref_frame[1] = second_ref_frame;
4258 // Evaluate all sub-pel filters irrespective of whether we can use
4259 // them for this frame.
4261 cm->interp_filter == SWITCHABLE ? EIGHTTAP : cm->interp_filter;
4263 set_ref_ptrs(cm, xd, ref_frame, second_ref_frame);
4265 // Select prediction reference frames.
4266 for (i = 0; i < MAX_MB_PLANE; i++) {
4267 xd->plane[i].pre[0] = yv12_mb[ref_frame][i];
4268 if (comp_pred) xd->plane[i].pre[1] = yv12_mb[second_ref_frame][i];
4271 if (ref_frame == INTRA_FRAME) {
4273 if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y, &distortion_y,
4274 best_rd) >= best_rd)
4277 rate2 += intra_cost_penalty;
4278 distortion2 += distortion_y;
4280 if (rate_uv_intra == INT_MAX) {
4281 choose_intra_uv_mode(cpi, x, ctx, bsize, TX_4X4, &rate_uv_intra,
4282 &rate_uv_tokenonly, &dist_uv, &skip_uv, &mode_uv);
4284 rate2 += rate_uv_intra;
4285 rate_uv = rate_uv_tokenonly;
4286 distortion2 += dist_uv;
4287 distortion_uv = dist_uv;
4288 mi->uv_mode = mode_uv;
4292 int64_t this_rd_thresh;
4293 int64_t tmp_rd, tmp_best_rd = INT64_MAX, tmp_best_rdu = INT64_MAX;
4294 int tmp_best_rate = INT_MAX, tmp_best_ratey = INT_MAX;
4295 int64_t tmp_best_distortion = INT_MAX, tmp_best_sse, uv_sse;
4296 int tmp_best_skippable = 0;
4297 int switchable_filter_index;
4298 int_mv *second_ref =
4299 comp_pred ? &x->mbmi_ext->ref_mvs[second_ref_frame][0] : NULL;
4300 b_mode_info tmp_best_bmodes[16];
4301 MODE_INFO tmp_best_mbmode;
4302 BEST_SEG_INFO bsi[SWITCHABLE_FILTERS];
4303 int pred_exists = 0;
4306 YV12_BUFFER_CONFIG *scaled_ref_frame[2] = { NULL, NULL };
4309 for (ref = 0; ref < 2; ++ref) {
4310 scaled_ref_frame[ref] =
4311 mi->ref_frame[ref] > INTRA_FRAME
4312 ? vp9_get_scaled_ref_frame(cpi, mi->ref_frame[ref])
4315 if (scaled_ref_frame[ref]) {
4317 // Swap out the reference frame for a version that's been scaled to
4318 // match the resolution of the current frame, allowing the existing
4319 // motion search code to be used without additional modifications.
4320 for (i = 0; i < MAX_MB_PLANE; i++)
4321 backup_yv12[ref][i] = xd->plane[i].pre[ref];
4322 vp9_setup_pre_planes(xd, ref, scaled_ref_frame[ref], mi_row, mi_col,
4327 this_rd_thresh = (ref_frame == LAST_FRAME)
4328 ? rd_opt->threshes[segment_id][bsize][THR_LAST]
4329 : rd_opt->threshes[segment_id][bsize][THR_ALTR];
4330 this_rd_thresh = (ref_frame == GOLDEN_FRAME)
4331 ? rd_opt->threshes[segment_id][bsize][THR_GOLD]
4333 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
4334 filter_cache[i] = INT64_MAX;
4336 if (cm->interp_filter != BILINEAR) {
4337 tmp_best_filter = EIGHTTAP;
4338 if (x->source_variance < sf->disable_filter_search_var_thresh) {
4339 tmp_best_filter = EIGHTTAP;
4340 } else if (sf->adaptive_pred_interp_filter == 1 &&
4341 ctx->pred_interp_filter < SWITCHABLE) {
4342 tmp_best_filter = ctx->pred_interp_filter;
4343 } else if (sf->adaptive_pred_interp_filter == 2) {
4344 tmp_best_filter = ctx->pred_interp_filter < SWITCHABLE
4345 ? ctx->pred_interp_filter
4348 for (switchable_filter_index = 0;
4349 switchable_filter_index < SWITCHABLE_FILTERS;
4350 ++switchable_filter_index) {
4353 MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext;
4354 mi->interp_filter = switchable_filter_index;
4355 tmp_rd = rd_pick_best_sub8x8_mode(
4356 cpi, x, &mbmi_ext->ref_mvs[ref_frame][0], second_ref, best_yrd,
4357 &rate, &rate_y, &distortion, &skippable, &total_sse,
4358 (int)this_rd_thresh, seg_mvs, bsi, switchable_filter_index,
4361 if (tmp_rd == INT64_MAX) continue;
4362 rs = vp9_get_switchable_rate(cpi, xd);
4363 rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0);
4364 filter_cache[switchable_filter_index] = tmp_rd;
4365 filter_cache[SWITCHABLE_FILTERS] =
4366 VPXMIN(filter_cache[SWITCHABLE_FILTERS], tmp_rd + rs_rd);
4367 if (cm->interp_filter == SWITCHABLE) tmp_rd += rs_rd;
4369 mask_filter = VPXMAX(mask_filter, tmp_rd);
4371 newbest = (tmp_rd < tmp_best_rd);
4373 tmp_best_filter = mi->interp_filter;
4374 tmp_best_rd = tmp_rd;
4376 if ((newbest && cm->interp_filter == SWITCHABLE) ||
4377 (mi->interp_filter == cm->interp_filter &&
4378 cm->interp_filter != SWITCHABLE)) {
4379 tmp_best_rdu = tmp_rd;
4380 tmp_best_rate = rate;
4381 tmp_best_ratey = rate_y;
4382 tmp_best_distortion = distortion;
4383 tmp_best_sse = total_sse;
4384 tmp_best_skippable = skippable;
4385 tmp_best_mbmode = *mi;
4386 for (i = 0; i < 4; i++) {
4387 tmp_best_bmodes[i] = xd->mi[0]->bmi[i];
4388 x->zcoeff_blk[TX_4X4][i] = !x->plane[0].eobs[i];
4389 x->sum_y_eobs[TX_4X4] += x->plane[0].eobs[i];
4392 if (switchable_filter_index == 0 && sf->use_rd_breakout &&
4393 best_rd < INT64_MAX) {
4394 if (tmp_best_rdu / 2 > best_rd) {
4395 // skip searching the other filters if the first is
4396 // already substantially larger than the best so far
4397 tmp_best_filter = mi->interp_filter;
4398 tmp_best_rdu = INT64_MAX;
4403 } // switchable_filter_index loop
4407 if (tmp_best_rdu == INT64_MAX && pred_exists) continue;
4409 mi->interp_filter = (cm->interp_filter == SWITCHABLE ? tmp_best_filter
4410 : cm->interp_filter);
4412 // Handles the special case when a filter that is not in the
4413 // switchable list (bilinear, 6-tap) is indicated at the frame level
4414 tmp_rd = rd_pick_best_sub8x8_mode(
4415 cpi, x, &x->mbmi_ext->ref_mvs[ref_frame][0], second_ref, best_yrd,
4416 &rate, &rate_y, &distortion, &skippable, &total_sse,
4417 (int)this_rd_thresh, seg_mvs, bsi, 0, mi_row, mi_col);
4418 if (tmp_rd == INT64_MAX) continue;
4420 total_sse = tmp_best_sse;
4421 rate = tmp_best_rate;
4422 rate_y = tmp_best_ratey;
4423 distortion = tmp_best_distortion;
4424 skippable = tmp_best_skippable;
4425 *mi = tmp_best_mbmode;
4426 for (i = 0; i < 4; i++) xd->mi[0]->bmi[i] = tmp_best_bmodes[i];
4430 distortion2 += distortion;
4432 if (cm->interp_filter == SWITCHABLE)
4433 rate2 += vp9_get_switchable_rate(cpi, xd);
4436 mode_excluded = comp_pred ? cm->reference_mode == SINGLE_REFERENCE
4437 : cm->reference_mode == COMPOUND_REFERENCE;
4439 compmode_cost = vp9_cost_bit(comp_mode_p, comp_pred);
4442 best_rd - VPXMIN(RDCOST(x->rdmult, x->rddiv, rate2, distortion2),
4443 RDCOST(x->rdmult, x->rddiv, 0, total_sse));
4445 if (tmp_best_rdu > 0) {
4446 // If even the 'Y' rd value of split is higher than best so far
4447 // then dont bother looking at UV
4448 vp9_build_inter_predictors_sbuv(&x->e_mbd, mi_row, mi_col, BLOCK_8X8);
4449 memset(x->skip_txfm, SKIP_TXFM_NONE, sizeof(x->skip_txfm));
4450 if (!super_block_uvrd(cpi, x, &rate_uv, &distortion_uv, &uv_skippable,
4451 &uv_sse, BLOCK_8X8, tmp_best_rdu)) {
4452 for (ref = 0; ref < 2; ++ref) {
4453 if (scaled_ref_frame[ref]) {
4455 for (i = 0; i < MAX_MB_PLANE; ++i)
4456 xd->plane[i].pre[ref] = backup_yv12[ref][i];
4463 distortion2 += distortion_uv;
4464 skippable = skippable && uv_skippable;
4465 total_sse += uv_sse;
4468 for (ref = 0; ref < 2; ++ref) {
4469 if (scaled_ref_frame[ref]) {
4470 // Restore the prediction frame pointers to their unscaled versions.
4472 for (i = 0; i < MAX_MB_PLANE; ++i)
4473 xd->plane[i].pre[ref] = backup_yv12[ref][i];
4478 if (cm->reference_mode == REFERENCE_MODE_SELECT) rate2 += compmode_cost;
4480 // Estimate the reference frame signaling cost and add it
4481 // to the rolling cost variable.
4482 if (second_ref_frame > INTRA_FRAME) {
4483 rate2 += ref_costs_comp[ref_frame];
4485 rate2 += ref_costs_single[ref_frame];
4488 if (!disable_skip) {
4489 const vpx_prob skip_prob = vp9_get_skip_prob(cm, xd);
4490 const int skip_cost0 = vp9_cost_bit(skip_prob, 0);
4491 const int skip_cost1 = vp9_cost_bit(skip_prob, 1);
4493 // Skip is never coded at the segment level for sub8x8 blocks and instead
4494 // always coded in the bitstream at the mode info level.
4495 if (ref_frame != INTRA_FRAME && !xd->lossless) {
4496 if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv + skip_cost0,
4498 RDCOST(x->rdmult, x->rddiv, skip_cost1, total_sse)) {
4499 // Add in the cost of the no skip flag.
4500 rate2 += skip_cost0;
4502 // FIXME(rbultje) make this work for splitmv also
4503 rate2 += skip_cost1;
4504 distortion2 = total_sse;
4505 assert(total_sse >= 0);
4506 rate2 -= (rate_y + rate_uv);
4512 // Add in the cost of the no skip flag.
4513 rate2 += skip_cost0;
4516 // Calculate the final RD estimate for this mode.
4517 this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
4520 if (!disable_skip && ref_frame == INTRA_FRAME) {
4521 for (i = 0; i < REFERENCE_MODES; ++i)
4522 best_pred_rd[i] = VPXMIN(best_pred_rd[i], this_rd);
4523 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++)
4524 best_filter_rd[i] = VPXMIN(best_filter_rd[i], this_rd);
4527 // Did this mode help.. i.e. is it the new best mode
4528 if (this_rd < best_rd || x->skip) {
4529 if (!mode_excluded) {
4530 int max_plane = MAX_MB_PLANE;
4531 // Note index of best mode so far
4532 best_ref_index = ref_index;
4534 if (ref_frame == INTRA_FRAME) {
4535 /* required for left and above block mv */
4536 mi->mv[0].as_int = 0;
4538 // Initialize interp_filter here so we do not have to check for
4539 // inter block modes in get_pred_context_switchable_interp()
4540 mi->interp_filter = SWITCHABLE_FILTERS;
4543 rd_cost->rate = rate2;
4544 rd_cost->dist = distortion2;
4545 rd_cost->rdcost = this_rd;
4548 best_rd - RDCOST(x->rdmult, x->rddiv, rate_uv, distortion_uv);
4550 best_skip2 = this_skip2;
4551 if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, max_plane);
4552 memcpy(ctx->zcoeff_blk, x->zcoeff_blk[TX_4X4],
4553 sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
4554 ctx->sum_y_eobs = x->sum_y_eobs[TX_4X4];
4556 for (i = 0; i < 4; i++) best_bmodes[i] = xd->mi[0]->bmi[i];
4558 // TODO(debargha): enhance this test with a better distortion prediction
4559 // based on qp, activity mask and history
4560 if ((sf->mode_search_skip_flags & FLAG_EARLY_TERMINATE) &&
4561 (ref_index > MIN_EARLY_TERM_INDEX)) {
4562 int qstep = xd->plane[0].dequant[1];
4563 // TODO(debargha): Enhance this by specializing for each mode_index
4565 #if CONFIG_VP9_HIGHBITDEPTH
4566 if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
4567 qstep >>= (xd->bd - 8);
4569 #endif // CONFIG_VP9_HIGHBITDEPTH
4570 if (x->source_variance < UINT_MAX) {
4571 const int var_adjust = (x->source_variance < 16);
4572 scale -= var_adjust;
4574 if (ref_frame > INTRA_FRAME && distortion2 * scale < qstep * qstep) {
4581 /* keep record of best compound/single-only prediction */
4582 if (!disable_skip && ref_frame != INTRA_FRAME) {
4583 int64_t single_rd, hybrid_rd, single_rate, hybrid_rate;
4585 if (cm->reference_mode == REFERENCE_MODE_SELECT) {
4586 single_rate = rate2 - compmode_cost;
4587 hybrid_rate = rate2;
4589 single_rate = rate2;
4590 hybrid_rate = rate2 + compmode_cost;
4593 single_rd = RDCOST(x->rdmult, x->rddiv, single_rate, distortion2);
4594 hybrid_rd = RDCOST(x->rdmult, x->rddiv, hybrid_rate, distortion2);
4596 if (!comp_pred && single_rd < best_pred_rd[SINGLE_REFERENCE])
4597 best_pred_rd[SINGLE_REFERENCE] = single_rd;
4598 else if (comp_pred && single_rd < best_pred_rd[COMPOUND_REFERENCE])
4599 best_pred_rd[COMPOUND_REFERENCE] = single_rd;
4601 if (hybrid_rd < best_pred_rd[REFERENCE_MODE_SELECT])
4602 best_pred_rd[REFERENCE_MODE_SELECT] = hybrid_rd;
4605 /* keep record of best filter type */
4606 if (!mode_excluded && !disable_skip && ref_frame != INTRA_FRAME &&
4607 cm->interp_filter != BILINEAR) {
4609 filter_cache[cm->interp_filter == SWITCHABLE ? SWITCHABLE_FILTERS
4610 : cm->interp_filter];
4612 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) {
4613 if (ref == INT64_MAX)
4615 else if (filter_cache[i] == INT64_MAX)
4616 // when early termination is triggered, the encoder does not have
4617 // access to the rate-distortion cost. it only knows that the cost
4618 // should be above the maximum valid value. hence it takes the known
4619 // maximum plus an arbitrary constant as the rate-distortion cost.
4620 adj_rd = mask_filter - ref + 10;
4622 adj_rd = filter_cache[i] - ref;
4625 best_filter_rd[i] = VPXMIN(best_filter_rd[i], adj_rd);
4629 if (early_term) break;
4631 if (x->skip && !comp_pred) break;
4634 if (best_rd >= best_rd_so_far) {
4635 rd_cost->rate = INT_MAX;
4636 rd_cost->rdcost = INT64_MAX;
4640 // If we used an estimate for the uv intra rd in the loop above...
4641 if (sf->use_uv_intra_rd_estimate) {
4642 // Do Intra UV best rd mode selection if best mode choice above was intra.
4643 if (best_mbmode.ref_frame[0] == INTRA_FRAME) {
4645 rd_pick_intra_sbuv_mode(cpi, x, ctx, &rate_uv_intra, &rate_uv_tokenonly,
4646 &dist_uv, &skip_uv, BLOCK_8X8, TX_4X4);
4650 if (best_rd == INT64_MAX) {
4651 rd_cost->rate = INT_MAX;
4652 rd_cost->dist = INT64_MAX;
4653 rd_cost->rdcost = INT64_MAX;
4657 assert((cm->interp_filter == SWITCHABLE) ||
4658 (cm->interp_filter == best_mbmode.interp_filter) ||
4659 !is_inter_block(&best_mbmode));
4661 vp9_update_rd_thresh_fact(tile_data->thresh_freq_fact, sf->adaptive_rd_thresh,
4662 bsize, best_ref_index);
4666 x->skip |= best_skip2;
4667 if (!is_inter_block(&best_mbmode)) {
4668 for (i = 0; i < 4; i++) xd->mi[0]->bmi[i].as_mode = best_bmodes[i].as_mode;
4670 for (i = 0; i < 4; ++i)
4671 memcpy(&xd->mi[0]->bmi[i], &best_bmodes[i], sizeof(b_mode_info));
4673 mi->mv[0].as_int = xd->mi[0]->bmi[3].as_mv[0].as_int;
4674 mi->mv[1].as_int = xd->mi[0]->bmi[3].as_mv[1].as_int;
4677 for (i = 0; i < REFERENCE_MODES; ++i) {
4678 if (best_pred_rd[i] == INT64_MAX)
4679 best_pred_diff[i] = INT_MIN;
4681 best_pred_diff[i] = best_rd - best_pred_rd[i];
4685 for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) {
4686 if (best_filter_rd[i] == INT64_MAX)
4687 best_filter_diff[i] = 0;
4689 best_filter_diff[i] = best_rd - best_filter_rd[i];
4691 if (cm->interp_filter == SWITCHABLE)
4692 assert(best_filter_diff[SWITCHABLE_FILTERS] == 0);
4694 vp9_zero(best_filter_diff);
4697 store_coding_context(x, ctx, best_ref_index, best_pred_diff, best_filter_diff,