From 76550dfdc0a898cb9510977f79b43dbac1fd9de0 Mon Sep 17 00:00:00 2001 From: paulwilkins Date: Thu, 9 Feb 2017 16:30:38 +0000 Subject: [PATCH] Bug in scale_sse_threshold() The function scale_sse_threshold() returns a threshold scaled if necessary for use with 10 and 12 bit from an 8 bit baseline. SSE error values would be expected to rise for the 10 and 12 bit cases where there are more bits of precision. Hence the threshold used for the test should also be scaled up. Change-Id: I4009c98b6eecd1bf64c3c38aaa56598e0136b03d --- vp9/encoder/vp9_firstpass.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c index e087740..cdb957a 100644 --- a/vp9/encoder/vp9_firstpass.c +++ b/vp9/encoder/vp9_firstpass.c @@ -469,8 +469,8 @@ static int scale_sse_threshold(VP9_COMMON *cm, int thresh) { if (cm->use_highbitdepth) { switch (cm->bit_depth) { case VPX_BITS_8: ret_val = thresh; break; - case VPX_BITS_10: ret_val = thresh >> 4; break; - case VPX_BITS_12: ret_val = thresh >> 8; break; + case VPX_BITS_10: ret_val = thresh << 4; break; + case VPX_BITS_12: ret_val = thresh << 8; break; default: assert(0 && "cm->bit_depth should be VPX_BITS_8, " -- 2.7.4