From d793a2b5606fc76ae0b93894819548489c7085e8 Mon Sep 17 00:00:00 2001 From: Carlos Rafael Giani Date: Mon, 15 Oct 2012 22:07:22 +0200 Subject: [PATCH] audioresample: added ARM NEON support This adds ARM NEON accelerated code paths for 16-bit integer and 32-bit floating point samples. It is a modified combination of patches #3 and #5 from Jyri Sarha ( http://lists.xiph.org/pipermail/speex-dev/2011-September/008240.html & http://lists.xiph.org/pipermail/speex-dev/2011-September/008238.html ) Signed-off-by: Carlos Rafael Giani --- configure.ac | 18 +++ gst/audioresample/Makefile.am | 1 + gst/audioresample/resample.c | 43 ++++++- gst/audioresample/resample_neon.h | 202 ++++++++++++++++++++++++++++++ gst/audioresample/speex_resampler_float.c | 1 + gst/audioresample/speex_resampler_int.c | 1 + 6 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 gst/audioresample/resample_neon.h diff --git a/configure.ac b/configure.ac index 7d6df30..0bb3c97 100644 --- a/configure.ac +++ b/configure.ac @@ -196,6 +196,24 @@ AM_CONDITIONAL(HAVE_SYS_SOCKET_H, test "x$HAVE_SYS_SOCKET_H" = "xyes") dnl used in gst-libs/gst/pbutils and associated unit test AC_CHECK_HEADERS([process.h sys/types.h sys/wait.h sys/stat.h], [], [], [AC_INCLUDES_DEFAULT]) +dnl checks for ARM NEON support +dnl this instruction set is used by the speex resampler code +AC_MSG_CHECKING(for ARM NEON support in current arch/CFLAGS) +AC_LINK_IFELSE([ +AC_LANG_PROGRAM([[ + #include + int32x4_t testfunc(int16_t *a, int16_t *b) { + return vmull_s16(vld1_s16(a), vld1_s16(b)); + } +]])], +[ + AC_DEFINE(HAVE_ARM_NEON,[],[ARM NEON support is enabled]) + AC_MSG_RESULT(yes) +], +[ + AC_MSG_RESULT(no) +]) + dnl also, Windows does not have long long AX_CREATE_STDINT_H diff --git a/gst/audioresample/Makefile.am b/gst/audioresample/Makefile.am index 0c7e871..78b4e24 100644 --- a/gst/audioresample/Makefile.am +++ b/gst/audioresample/Makefile.am @@ -39,6 +39,7 @@ noinst_HEADERS = \ gstaudioresample.h \ resample.c \ resample_sse.h \ + resample_neon.h \ speex_resampler.h \ speex_resampler_wrapper.h diff --git a/gst/audioresample/resample.c b/gst/audioresample/resample.c index b6b1de6..98d006c 100644 --- a/gst/audioresample/resample.c +++ b/gst/audioresample/resample.c @@ -88,6 +88,12 @@ #endif #endif +#ifdef _USE_NEON +#ifndef HAVE_ARM_NEON +#undef _USE_NEON +#endif +#endif + static inline void * speex_alloc (int size) { @@ -134,6 +140,10 @@ speex_free (void *ptr) #include "resample_sse.h" #endif +#ifdef _USE_NEON +#include "resample_neon.h" +#endif + /* Numer of elements to allocate on the stack */ #ifdef VAR_ARRAYS #define FIXED_STACK_ALLOC 8192 @@ -162,6 +172,16 @@ speex_free (void *ptr) #define SSE2_FALLBACK(macro) #endif +#ifdef _USE_NEON +#define NEON_FALLBACK(macro) \ + if (st->use_neon) goto neon_##macro##_neon; { +#define NEON_IMPLEMENTATION(macro) \ + goto neon_##macro##_end; } neon_##macro##_neon: { +#define NEON_END(macro) neon_##macro##_end:; } +#else +#define NEON_FALLBACK(macro) +#endif + typedef int (*resampler_basic_func) (SpeexResamplerState *, spx_uint32_t, const spx_word16_t *, spx_uint32_t *, spx_word16_t *, spx_uint32_t *); @@ -201,6 +221,7 @@ struct SpeexResamplerState_ int use_sse:1; int use_sse2:1; + int use_neon:1; }; static double kaiser12_table[68] = { @@ -457,6 +478,7 @@ resampler_basic_direct_single (SpeexResamplerState * st, const spx_word16_t *iptr = &in[last_sample]; SSE_FALLBACK (INNER_PRODUCT_SINGLE) + NEON_FALLBACK (INNER_PRODUCT_SINGLE) sum = 0; for (j = 0; j < N; j++) sum += MULT16_16 (sinc[j], iptr[j]); @@ -473,7 +495,11 @@ resampler_basic_direct_single (SpeexResamplerState * st, } sum = accum[0] + accum[1] + accum[2] + accum[3]; */ -#ifdef OVERRIDE_INNER_PRODUCT_SINGLE +#if defined(OVERRIDE_INNER_PRODUCT_SINGLE) && defined(_USE_NEON) + NEON_IMPLEMENTATION (INNER_PRODUCT_SINGLE) + sum = inner_product_single (sinc, iptr, N); + NEON_END(INNER_PRODUCT_SINGLE) +#elif defined(OVERRIDE_INNER_PRODUCT_SINGLE) && defined(_USE_SSE) SSE_IMPLEMENTATION (INNER_PRODUCT_SINGLE) sum = inner_product_single (sinc, iptr, N); SSE_END (INNER_PRODUCT_SINGLE) @@ -528,7 +554,7 @@ resampler_basic_direct_double (SpeexResamplerState * st, accum[3] += sinc[j + 3] * iptr[j + 3]; } sum = accum[0] + accum[1] + accum[2] + accum[3]; -#ifdef OVERRIDE_INNER_PRODUCT_DOUBLE +#if defined(OVERRIDE_INNER_PRODUCT_DOUBLE) && defined(_USE_SSE2) SSE2_IMPLEMENTATION (INNER_PRODUCT_DOUBLE) sum = inner_product_double (sinc, iptr, N); SSE2_END (INNER_PRODUCT_DOUBLE) @@ -607,7 +633,7 @@ resampler_basic_interpolate_single (SpeexResamplerState * st, 1)) + MULT16_32_Q15 (interp[1], SHR32 (accum[1], 1)) + MULT16_32_Q15 (interp[2], SHR32 (accum[2], 1)) + MULT16_32_Q15 (interp[3], SHR32 (accum[3], 1)); -#ifdef OVERRIDE_INTERPOLATE_PRODUCT_SINGLE +#if defined(OVERRIDE_INTERPOLATE_PRODUCT_SINGLE) && defined(_USE_SSE) SSE_IMPLEMENTATION (INTERPOLATE_PRODUCT_SINGLE) cubic_coef (frac, interp); sum = @@ -697,7 +723,7 @@ resampler_basic_interpolate_double (SpeexResamplerState * st, MULT16_32_Q15 (interp[0], accum[0]) + MULT16_32_Q15 (interp[1], accum[1]) + MULT16_32_Q15 (interp[2], accum[2]) + MULT16_32_Q15 (interp[3], accum[3]); -#ifdef OVERRIDE_INTERPOLATE_PRODUCT_DOUBLE +#if defined(OVERRIDE_INTERPOLATE_PRODUCT_DOUBLE) && defined(_USE_SSE2) SSE2_IMPLEMENTATION (INTERPOLATE_PRODUCT_DOUBLE) cubic_coef (frac, interp); sum = @@ -933,10 +959,18 @@ check_insn_set (SpeexResamplerState * st, const char *name) { if (!name) return; +#ifdef _USE_SSE if (!strcmp (name, "sse")) st->use_sse = 1; +#endif +#ifdef _USE_SSE2 if (!strcmp (name, "sse2")) st->use_sse = st->use_sse2 = 1; +#endif +#ifdef _USE_NEON + if (!strcmp (name, "neon")) + st->use_neon = 1; +#endif } #endif @@ -997,6 +1031,7 @@ speex_resampler_init_frac (spx_uint32_t nb_channels, spx_uint32_t ratio_num, #endif st->use_sse = st->use_sse2 = 0; + st->use_neon = 0; #if defined HAVE_ORC && !defined DISABLE_ORC orc_init (); { diff --git a/gst/audioresample/resample_neon.h b/gst/audioresample/resample_neon.h new file mode 100644 index 0000000..478488f --- /dev/null +++ b/gst/audioresample/resample_neon.h @@ -0,0 +1,202 @@ +/* Copyright (C) 2007-2008 Jean-Marc Valin + * Copyright (C) 2008 Thorvald Natvig + * Copyright (C) 2011 Texas Instruments + * author Jyri Sarha + */ +/** + @file resample_neon.h + @brief Resampler functions (NEON version) +*/ +/* + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include + +#ifdef FIXED_POINT +#ifdef __thumb2__ +static inline int32_t saturate_32bit_to_16bit(int32_t a) { + int32_t ret; + asm ("ssat %[ret], #16, %[a]" + : [ret] "=&r" (ret) + : [a] "r" (a) + : ); + return ret; +} +#else +static inline int32_t saturate_32bit_to_16bit(int32_t a) { + int32_t ret; + asm ("vmov.s32 d0[0], %[a]\n" + "vqmovn.s32 d0, q0\n" + "vmov.s16 %[ret], d0[0]\n" + : [ret] "=&r" (ret) + : [a] "r" (a) + : "q0"); + return ret; +} +#endif +#undef WORD2INT +#define WORD2INT(x) (saturate_32bit_to_16bit(x)) + +#define OVERRIDE_INNER_PRODUCT_SINGLE +/* Only works when len % 4 == 0 */ +static inline int32_t inner_product_single(const int16_t *a, const int16_t *b, unsigned int len) +{ + int32_t ret; + uint32_t remainder = len % 16; + len = len - remainder; + + asm volatile (" cmp %[len], #0\n" + " bne 1f\n" + " vld1.16 {d16}, [%[b]]!\n" + " vld1.16 {d20}, [%[a]]!\n" + " subs %[remainder], %[remainder], #4\n" + " vmull.s16 q0, d16, d20\n" + " beq 5f\n" + " b 4f\n" + "1:" + " vld1.16 {d16, d17, d18, d19}, [%[b]]!\n" + " vld1.16 {d20, d21, d22, d23}, [%[a]]!\n" + " subs %[len], %[len], #16\n" + " vmull.s16 q0, d16, d20\n" + " vmlal.s16 q0, d17, d21\n" + " vmlal.s16 q0, d18, d22\n" + " vmlal.s16 q0, d19, d23\n" + " beq 3f\n" + "2:" + " vld1.16 {d16, d17, d18, d19}, [%[b]]!\n" + " vld1.16 {d20, d21, d22, d23}, [%[a]]!\n" + " subs %[len], %[len], #16\n" + " vmlal.s16 q0, d16, d20\n" + " vmlal.s16 q0, d17, d21\n" + " vmlal.s16 q0, d18, d22\n" + " vmlal.s16 q0, d19, d23\n" + " bne 2b\n" + "3:" + " cmp %[remainder], #0\n" + " beq 5f\n" + "4:" + " vld1.16 {d16}, [%[b]]!\n" + " vld1.16 {d20}, [%[a]]!\n" + " subs %[remainder], %[remainder], #4\n" + " vmlal.s16 q0, d16, d20\n" + " bne 4b\n" + "5:" + " vaddl.s32 q0, d0, d1\n" + " vadd.s64 d0, d0, d1\n" + " vqmovn.s64 d0, q0\n" + " vqrshrn.s32 d0, q0, #15\n" + " vmov.s16 %[ret], d0[0]\n" + : [ret] "=&r" (ret), [a] "+r" (a), [b] "+r" (b), + [len] "+r" (len), [remainder] "+r" (remainder) + : + : "cc", "q0", + "d16", "d17", "d18", "d19", + "d20", "d21", "d22", "d23"); + + return ret; +} +#elif defined(FLOATING_POINT) + +static inline int32_t saturate_float_to_16bit(float a) { + int32_t ret; + asm ("vmov.f32 d0[0], %[a]\n" + "vcvt.s32.f32 d0, d0, #15\n" + "vqrshrn.s32 d0, q0, #15\n" + "vmov.s16 %[ret], d0[0]\n" + : [ret] "=&r" (ret) + : [a] "r" (a) + : "q0"); + return ret; +} +#undef WORD2INT +#define WORD2INT(x) (saturate_float_to_16bit(x)) + +#define OVERRIDE_INNER_PRODUCT_SINGLE +/* Only works when len % 4 == 0 */ +static inline float inner_product_single(const float *a, const float *b, unsigned int len) +{ + float ret; + uint32_t remainder = len % 16; + len = len - remainder; + + asm volatile (" cmp %[len], #0\n" + " bne 1f\n" + " vld1.32 {q4}, [%[b]]!\n" + " vld1.32 {q8}, [%[a]]!\n" + " subs %[remainder], %[remainder], #4\n" + " vmul.f32 q0, q4, q8\n" + " bne 4f\n" + " b 5f\n" + "1:" + " vld1.32 {q4, q5}, [%[b]]!\n" + " vld1.32 {q8, q9}, [%[a]]!\n" + " vld1.32 {q6, q7}, [%[b]]!\n" + " vld1.32 {q10, q11}, [%[a]]!\n" + " subs %[len], %[len], #16\n" + " vmul.f32 q0, q4, q8\n" + " vmul.f32 q1, q5, q9\n" + " vmul.f32 q2, q6, q10\n" + " vmul.f32 q3, q7, q11\n" + " beq 3f\n" + "2:" + " vld1.32 {q4, q5}, [%[b]]!\n" + " vld1.32 {q8, q9}, [%[a]]!\n" + " vld1.32 {q6, q7}, [%[b]]!\n" + " vld1.32 {q10, q11}, [%[a]]!\n" + " subs %[len], %[len], #16\n" + " vmla.f32 q0, q4, q8\n" + " vmla.f32 q1, q5, q9\n" + " vmla.f32 q2, q6, q10\n" + " vmla.f32 q3, q7, q11\n" + " bne 2b\n" + "3:" + " vadd.f32 q4, q0, q1\n" + " vadd.f32 q5, q2, q3\n" + " cmp %[remainder], #0\n" + " vadd.f32 q0, q4, q5\n" + " beq 5f\n" + "4:" + " vld1.32 {q6}, [%[b]]!\n" + " vld1.32 {q10}, [%[a]]!\n" + " subs %[remainder], %[remainder], #4\n" + " vmla.f32 q0, q6, q10\n" + " bne 4b\n" + "5:" + " vadd.f32 d0, d0, d1\n" + " vpadd.f32 d0, d0, d0\n" + " vmov.f32 %[ret], d0[0]\n" + : [ret] "=&r" (ret), [a] "+r" (a), [b] "+r" (b), + [len] "+l" (len), [remainder] "+l" (remainder) + : + : "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11"); + return ret; +} +#endif + diff --git a/gst/audioresample/speex_resampler_float.c b/gst/audioresample/speex_resampler_float.c index ef3df15..33d02d1 100644 --- a/gst/audioresample/speex_resampler_float.c +++ b/gst/audioresample/speex_resampler_float.c @@ -19,6 +19,7 @@ #define _USE_SSE #define _USE_SSE2 +#define _USE_NEON #define FLOATING_POINT #define OUTSIDE_SPEEX #define RANDOM_PREFIX resample_float diff --git a/gst/audioresample/speex_resampler_int.c b/gst/audioresample/speex_resampler_int.c index 6b85860..02b3aca 100644 --- a/gst/audioresample/speex_resampler_int.c +++ b/gst/audioresample/speex_resampler_int.c @@ -19,6 +19,7 @@ #define FIXED_POINT 1 #define OUTSIDE_SPEEX 1 +#define _USE_NEON #define RANDOM_PREFIX resample_int #include "resample.c" -- 2.7.4