From 2d6b5df657ded665594a7c1d2bdaa092610d1c4d Mon Sep 17 00:00:00 2001 From: Johann Date: Thu, 27 Jul 2017 13:25:38 -0700 Subject: [PATCH] neon: vpx_quantize_b With skip block or coeff < zbin it is about twice as fast as C. If most coeff values are > zbin it is about 10-15x as fast as C. BUG=webm:1426 Change-Id: I5d3c007b014a372d5ef0882b39bb48983b4131c7 --- test/vp9_quantize_test.cc | 8 +++ vpx_dsp/arm/quantize_neon.c | 155 +++++++++++++++++++++++++++++++++++++++++++ vpx_dsp/vpx_dsp.mk | 1 + vpx_dsp/vpx_dsp_rtcd_defs.pl | 2 +- 4 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 vpx_dsp/arm/quantize_neon.c diff --git a/test/vp9_quantize_test.cc b/test/vp9_quantize_test.cc index cffcda2..20faa8e 100644 --- a/test/vp9_quantize_test.cc +++ b/test/vp9_quantize_test.cc @@ -405,4 +405,12 @@ INSTANTIATE_TEST_CASE_P(DISABLED_AVX, VP9Quantize32Test, &vpx_quantize_b_32x32_c, VPX_BITS_8))); #endif // HAVE_AVX && ARCH_X86_64 && !CONFIG_VP9_HIGHBITDEPTH + +// TODO(webm:1448): dqcoeff is not handled correctly in HBD builds. +#if HAVE_NEON && !CONFIG_VP9_HIGHBITDEPTH +INSTANTIATE_TEST_CASE_P(NEON, VP9QuantizeTest, + ::testing::Values(make_tuple(&vpx_quantize_b_neon, + &vpx_quantize_b_c, + VPX_BITS_8))); +#endif // HAVE_NEON && !CONFIG_VP9_HIGHBITDEPTH } // namespace diff --git a/vpx_dsp/arm/quantize_neon.c b/vpx_dsp/arm/quantize_neon.c new file mode 100644 index 0000000..8bed057 --- /dev/null +++ b/vpx_dsp/arm/quantize_neon.c @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2017 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/arm/mem_neon.h" + +void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, + uint16_t *eob_ptr, const int16_t *scan_ptr, + const int16_t *iscan_ptr) { + const int16x8_t zero = vdupq_n_s16(0); + const int16x8_t one = vdupq_n_s16(1); + const int16x8_t neg_one = vdupq_n_s16(-1); + uint16x8_t eob_max; + (void)scan_ptr; + + if (skip_block) { + do { + store_s16q_to_tran_low(qcoeff_ptr, zero); + store_s16q_to_tran_low(dqcoeff_ptr, zero); + qcoeff_ptr += 8; + dqcoeff_ptr += 8; + n_coeffs -= 8; + } while (n_coeffs > 0); + + *eob_ptr = 0; + return; + } + + // Process first 8 values which include a dc component. + { + // Only the first element of each vector is DC. + const int16x8_t zbin = vld1q_s16(zbin_ptr); + const int16x8_t round = vld1q_s16(round_ptr); + const int16x8_t quant = vld1q_s16(quant_ptr); + const int16x8_t quant_shift = vld1q_s16(quant_shift_ptr); + const int16x8_t dequant = vld1q_s16(dequant_ptr); + // Add one because the eob does not index from 0. + const uint16x8_t iscan = + vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan_ptr), one)); + + const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr); + const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15); + const int16x8_t coeff_abs = vabsq_s16(coeff); + + const int16x8_t zbin_mask = + vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin)); + + const int16x8_t rounded = vqaddq_s16(coeff_abs, round); + + // (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16 + int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1); + + qcoeff = vaddq_s16(qcoeff, rounded); + + // (qcoeff * quant_shift * 2) >> 16 >> 1 == (qcoeff * quant_shift) >> 16 + qcoeff = vshrq_n_s16(vqdmulhq_s16(qcoeff, quant_shift), 1); + + // Restore the sign bit. + qcoeff = veorq_s16(qcoeff, coeff_sign); + qcoeff = vsubq_s16(qcoeff, coeff_sign); + + qcoeff = vandq_s16(qcoeff, zbin_mask); + + // Set non-zero elements to -1 and use that to extract values for eob. + eob_max = vandq_u16(vtstq_s16(qcoeff, neg_one), iscan); + + coeff_ptr += 8; + iscan_ptr += 8; + + store_s16q_to_tran_low(qcoeff_ptr, qcoeff); + qcoeff_ptr += 8; + + qcoeff = vmulq_s16(qcoeff, dequant); + + store_s16q_to_tran_low(dqcoeff_ptr, qcoeff); + dqcoeff_ptr += 8; + } + + n_coeffs -= 8; + + { + const int16x8_t zbin = vdupq_n_s16(zbin_ptr[1]); + const int16x8_t round = vdupq_n_s16(round_ptr[1]); + const int16x8_t quant = vdupq_n_s16(quant_ptr[1]); + const int16x8_t quant_shift = vdupq_n_s16(quant_shift_ptr[1]); + const int16x8_t dequant = vdupq_n_s16(dequant_ptr[1]); + + do { + // Add one because the eob is not it's index. + const uint16x8_t iscan = + vreinterpretq_u16_s16(vaddq_s16(vld1q_s16(iscan_ptr), one)); + + const int16x8_t coeff = load_tran_low_to_s16q(coeff_ptr); + const int16x8_t coeff_sign = vshrq_n_s16(coeff, 15); + const int16x8_t coeff_abs = vabsq_s16(coeff); + + const int16x8_t zbin_mask = + vreinterpretq_s16_u16(vcgeq_s16(coeff_abs, zbin)); + + const int16x8_t rounded = vqaddq_s16(coeff_abs, round); + + // (round * quant * 2) >> 16 >> 1 == (round * quant) >> 16 + int16x8_t qcoeff = vshrq_n_s16(vqdmulhq_s16(rounded, quant), 1); + + qcoeff = vaddq_s16(qcoeff, rounded); + + // (qcoeff * quant_shift * 2) >> 16 >> 1 == (qcoeff * quant_shift) >> 16 + qcoeff = vshrq_n_s16(vqdmulhq_s16(qcoeff, quant_shift), 1); + + // Restore the sign bit. + qcoeff = veorq_s16(qcoeff, coeff_sign); + qcoeff = vsubq_s16(qcoeff, coeff_sign); + + qcoeff = vandq_s16(qcoeff, zbin_mask); + + // Set non-zero elements to -1 and use that to extract values for eob. + eob_max = + vmaxq_u16(eob_max, vandq_u16(vtstq_s16(qcoeff, neg_one), iscan)); + + coeff_ptr += 8; + iscan_ptr += 8; + + store_s16q_to_tran_low(qcoeff_ptr, qcoeff); + qcoeff_ptr += 8; + + qcoeff = vmulq_s16(qcoeff, dequant); + + store_s16q_to_tran_low(dqcoeff_ptr, qcoeff); + dqcoeff_ptr += 8; + + n_coeffs -= 8; + } while (n_coeffs > 0); + } + + { + const uint16x4_t eob_max_0 = + vmax_u16(vget_low_u16(eob_max), vget_high_u16(eob_max)); + const uint16x4_t eob_max_1 = vpmax_u16(eob_max_0, eob_max_0); + const uint16x4_t eob_max_2 = vpmax_u16(eob_max_1, eob_max_1); + vst1_lane_u16(eob_ptr, eob_max_2, 0); + } +} diff --git a/vpx_dsp/vpx_dsp.mk b/vpx_dsp/vpx_dsp.mk index 4dc1dba..ff61685 100644 --- a/vpx_dsp/vpx_dsp.mk +++ b/vpx_dsp/vpx_dsp.mk @@ -274,6 +274,7 @@ DSP_SRCS-yes += quantize.c DSP_SRCS-yes += quantize.h DSP_SRCS-$(HAVE_SSE2) += x86/quantize_sse2.c +DSP_SRCS-$(HAVE_NEON) += arm/quantize_neon.c ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_SSE2) += x86/highbd_quantize_intrin_sse2.c endif diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl index 358d169..275ad7a 100644 --- a/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl @@ -673,7 +673,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { # if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") { add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; - specialize qw/vpx_quantize_b sse2/, "$ssse3_x86_64", "$avx_x86_64"; + specialize qw/vpx_quantize_b neon sse2/, "$ssse3_x86_64", "$avx_x86_64"; add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_quantize_b_32x32/, "$ssse3_x86_64", "$avx_x86_64"; -- 2.7.4