From bde2afbe23c6a503d57720cb49f703d110e2aa11 Mon Sep 17 00:00:00 2001 From: Yunqing Wang Date: Wed, 27 Jul 2011 10:37:33 -0400 Subject: [PATCH] Fix range checks in motion search There were some situations that the start motion vectors were out of range. This fix adjusted range checks to make sure they are checked and clamped. Change-Id: Ife83b7fed0882bba6d1fa559b6e63c054fd5065d --- vp8/encoder/mcomp.c | 23 +++++++++++++++++------ vp8/encoder/pickinter.c | 2 -- vp8/encoder/rdopt.c | 3 --- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/vp8/encoder/mcomp.c b/vp8/encoder/mcomp.c index 58b524f..282e5c9 100644 --- a/vp8/encoder/mcomp.c +++ b/vp8/encoder/mcomp.c @@ -15,6 +15,7 @@ #include #include #include +#include "vp8/common/findnearmv.h" #ifdef ENTROPY_STATS static int mv_ref_ct [31] [4] [2]; @@ -866,7 +867,7 @@ int vp8_hex_search unsigned char *what = (*(b->base_src) + b->src); int what_stride = b->src_stride; int in_what_stride = d->pre_stride; - int br = ref_mv->as_mv.row, bc = ref_mv->as_mv.col; + int br, bc; int_mv this_mv; unsigned int bestsad = 0x7fffffff; unsigned int thissad; @@ -880,6 +881,11 @@ int vp8_hex_search fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3; fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3; + // adjust ref_mv to make sure it is within MV range + vp8_clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max); + br = ref_mv->as_mv.row; + bc = ref_mv->as_mv.col; + // Work out the start point for the search base_offset = (unsigned char *)(*(d->base_pre) + d->pre); this_offset = base_offset + (br * (d->pre_stride)) + bc; @@ -1043,8 +1049,8 @@ int vp8_diamond_search_sad int best_site = 0; int last_site = 0; - int ref_row = ref_mv->as_mv.row; - int ref_col = ref_mv->as_mv.col; + int ref_row; + int ref_col; int this_row_offset; int this_col_offset; search_site *ss; @@ -1057,8 +1063,10 @@ int vp8_diamond_search_sad fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3; fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3; + vp8_clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max); + ref_row = ref_mv->as_mv.row; + ref_col = ref_mv->as_mv.col; *num00 = 0; - best_mv->as_mv.row = ref_row; best_mv->as_mv.col = ref_col; @@ -1162,8 +1170,8 @@ int vp8_diamond_search_sadx4 int best_site = 0; int last_site = 0; - int ref_row = ref_mv->as_mv.row; - int ref_col = ref_mv->as_mv.col; + int ref_row; + int ref_col; int this_row_offset; int this_col_offset; search_site *ss; @@ -1176,6 +1184,9 @@ int vp8_diamond_search_sadx4 fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3; fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3; + vp8_clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max); + ref_row = ref_mv->as_mv.row; + ref_col = ref_mv->as_mv.col; *num00 = 0; best_mv->as_mv.row = ref_row; best_mv->as_mv.col = ref_col; diff --git a/vp8/encoder/pickinter.c b/vp8/encoder/pickinter.c index 725e44e..9cdaf7d 100644 --- a/vp8/encoder/pickinter.c +++ b/vp8/encoder/pickinter.c @@ -669,8 +669,6 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, mvp_full.as_mv.col = mvp.as_mv.col>>3; mvp_full.as_mv.row = mvp.as_mv.row>>3; - /* adjust mvp to make sure it is within MV range */ - vp8_clamp_mv(&mvp_full, col_min, col_max, row_min, row_max); }else { mvp.as_int = best_ref_mv.as_int; diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c index 8b18541..356c03d 100644 --- a/vp8/encoder/rdopt.c +++ b/vp8/encoder/rdopt.c @@ -2016,9 +2016,6 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int mvp_full.as_mv.col = mvp.as_mv.col>>3; mvp_full.as_mv.row = mvp.as_mv.row>>3; - /* adjust mvp to make sure it is within MV range */ - vp8_clamp_mv(&mvp_full, col_min, col_max, row_min, row_max); - // Get intersection of UMV window and valid MV window to reduce # of checks in diamond search. if (x->mv_col_min < col_min ) x->mv_col_min = col_min; -- 2.7.4