From 61709a177aa8ef60dbc52e4409beb0d486095d55 Mon Sep 17 00:00:00 2001 From: James Zern Date: Mon, 10 Apr 2023 13:29:02 -0700 Subject: [PATCH] vp9_quantize_avx2,highbd_get_max_lane_eob: fix mask Pack nz_mask with zero. After the result is permuted this has the effect of ignoring the upper half of the iscan register which is only loaded with 128-bits. Depending on the optimization level and the load used the upper half of the ymm register may contain undefined values which can produce an incorrect eob. If this is large enough it can cause a crash. Bug: chromium:1431729 Change-Id: I4ebae9fa39f228bdd29dcc19935f3f07759d75f5 --- vp9/encoder/x86/vp9_quantize_avx2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vp9/encoder/x86/vp9_quantize_avx2.c b/vp9/encoder/x86/vp9_quantize_avx2.c index da285be..e6aa71d 100644 --- a/vp9/encoder/x86/vp9_quantize_avx2.c +++ b/vp9/encoder/x86/vp9_quantize_avx2.c @@ -295,7 +295,8 @@ static VPX_FORCE_INLINE void highbd_load_fp_values( static VPX_FORCE_INLINE __m256i highbd_get_max_lane_eob( const int16_t *iscan_ptr, __m256i eobmax, __m256i nz_mask) { - const __m256i packed_nz_mask = _mm256_packs_epi32(nz_mask, nz_mask); + const __m256i packed_nz_mask = + _mm256_packs_epi32(nz_mask, _mm256_setzero_si256()); const __m256i packed_nz_mask_perm = _mm256_permute4x64_epi64(packed_nz_mask, 0xD8); const __m256i iscan = -- 2.7.4