2 * Copyright (C) <2015> Wim Taymans <wim.taymans@gmail.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
20 #if defined (HAVE_XMMINTRIN_H) && defined(__SSE__)
21 #include <xmmintrin.h>
24 inner_product_gfloat_none_1_sse (gfloat * o, const gfloat * a, const gfloat * b, gint len, gpointer icoeff, gint oversample)
27 __m128 sum = _mm_setzero_ps ();
29 for (; i < len; i += 8) {
31 _mm_add_ps (sum, _mm_mul_ps (_mm_loadu_ps (a + i + 0),
32 _mm_load_ps (b + i + 0)));
34 _mm_add_ps (sum, _mm_mul_ps (_mm_loadu_ps (a + i + 4),
35 _mm_load_ps (b + i + 4)));
37 sum = _mm_add_ps (sum, _mm_movehl_ps (sum, sum));
38 sum = _mm_add_ss (sum, _mm_shuffle_ps (sum, sum, 0x55));
39 _mm_store_ss (o, sum);
43 inner_product_gfloat_linear_1_sse (gfloat * o, const gfloat * a, const gfloat * b, gint len, gpointer icoeff, gint oversample)
46 __m128 sum = _mm_setzero_ps (), t, b0;
47 __m128 f = _mm_loadu_ps(icoeff);
49 for (; i < len; i += 4) {
50 t = _mm_loadu_ps (a + i);
52 b0 = _mm_loadh_pi (b0, (__m64 *) (b + (i+0)*oversample));
53 b0 = _mm_loadl_pi (b0, (__m64 *) (b + (i+1)*oversample));
56 _mm_add_ps (sum, _mm_mul_ps (_mm_unpacklo_ps (t, t), b0));
58 b0 = _mm_loadh_pi (b0, (__m64 *) (b + (i+2)*oversample));
59 b0 = _mm_loadl_pi (b0, (__m64 *) (b + (i+3)*oversample));
62 _mm_add_ps (sum, _mm_mul_ps (_mm_unpackhi_ps (t, t), b0));
64 sum = _mm_mul_ps (sum, f);
65 sum = _mm_add_ps (sum, _mm_movehl_ps (sum, sum));
66 sum = _mm_add_ss (sum, _mm_shuffle_ps (sum, sum, 0x55));
67 _mm_store_ss (o, sum);
71 inner_product_gfloat_none_2_sse (gfloat * o, const gfloat * a, const gfloat * b, gint len, gpointer icoeff, gint oversample)
74 __m128 sum = _mm_setzero_ps (), t;
76 for (; i < len; i += 8) {
77 t = _mm_load_ps (b + i);
79 _mm_add_ps (sum, _mm_mul_ps (_mm_loadu_ps (a + 2 * i + 0),
80 _mm_unpacklo_ps (t, t)));
82 _mm_add_ps (sum, _mm_mul_ps (_mm_loadu_ps (a + 2 * i + 4),
83 _mm_unpackhi_ps (t, t)));
85 t = _mm_load_ps (b + i + 4);
87 _mm_add_ps (sum, _mm_mul_ps (_mm_loadu_ps (a + 2 * i + 8),
88 _mm_unpacklo_ps (t, t)));
90 _mm_add_ps (sum, _mm_mul_ps (_mm_loadu_ps (a + 2 * i + 12),
91 _mm_unpackhi_ps (t, t)));
93 sum = _mm_add_ps (sum, _mm_movehl_ps (sum, sum));
94 *(gint64*)o = _mm_cvtsi128_si64 ((__m128i)sum);
97 MAKE_RESAMPLE_FUNC (gfloat, none, 1, sse);
98 MAKE_RESAMPLE_FUNC (gfloat, none, 2, sse);
99 MAKE_RESAMPLE_FUNC (gfloat, linear, 1, sse);
102 #if defined (HAVE_EMMINTRIN_H) && defined(__SSE2__)
103 #include <emmintrin.h>
106 inner_product_gint16_none_1_sse2 (gint16 * o, const gint16 * a, const gint16 * b, gint len, gpointer icoeff, gint oversample)
111 sum = _mm_setzero_si128 ();
113 for (; i < len; i += 8) {
114 ta = _mm_loadu_si128 ((__m128i *) (a + i));
115 tb = _mm_load_si128 ((__m128i *) (b + i));
117 sum = _mm_add_epi32 (sum, _mm_madd_epi16 (ta, tb));
120 _mm_add_epi32 (sum, _mm_shuffle_epi32 (sum, _MM_SHUFFLE (2, 3, 2,
123 _mm_add_epi32 (sum, _mm_shuffle_epi32 (sum, _MM_SHUFFLE (1, 1, 1,
126 sum = _mm_add_epi32 (sum, _mm_set1_epi32 (1 << (PRECISION_S16 - 1)));
127 sum = _mm_srai_epi32 (sum, PRECISION_S16);
128 sum = _mm_packs_epi32 (sum, sum);
129 *o = _mm_extract_epi16 (sum, 0);
133 inner_product_gdouble_none_1_sse2 (gdouble * o, const gdouble * a, const gdouble * b,
134 gint len, gpointer icoeff, gint oversample)
137 __m128d sum = _mm_setzero_pd ();
139 for (; i < len; i += 8) {
141 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + i + 0),
142 _mm_load_pd (b + i + 0)));
144 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + i + 2),
145 _mm_load_pd (b + i + 2)));
147 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + i + 4),
148 _mm_load_pd (b + i + 4)));
150 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + i + 6),
151 _mm_load_pd (b + i + 6)));
153 sum = _mm_add_sd (sum, _mm_unpackhi_pd (sum, sum));
154 _mm_store_sd (o, sum);
158 inner_product_gint16_none_2_sse2 (gint16 * o, const gint16 * a, const gint16 * b, gint len, gpointer icoeff, gint oversample)
161 __m128i sum, ta, tb, t1;
163 sum = _mm_setzero_si128 ();
165 for (; i < len; i += 8) {
166 tb = _mm_load_si128 ((__m128i *) (b + i));
168 t1 = _mm_unpacklo_epi16 (tb, tb);
169 ta = _mm_loadu_si128 ((__m128i *) (a + 2 * i));
171 sum = _mm_add_epi32 (sum, _mm_madd_epi16 (ta, t1));
173 t1 = _mm_unpackhi_epi16 (tb, tb);
174 ta = _mm_loadu_si128 ((__m128i *) (a + 2 * i + 8));
176 sum = _mm_add_epi32 (sum, _mm_madd_epi16 (ta, t1));
179 _mm_add_epi32 (sum, _mm_shuffle_epi32 (sum, _MM_SHUFFLE (2, 3, 2,
182 sum = _mm_add_epi32 (sum, _mm_set1_epi32 (1 << (PRECISION_S16 - 1)));
183 sum = _mm_srai_epi32 (sum, PRECISION_S16);
184 sum = _mm_packs_epi32 (sum, sum);
185 *(gint32*)o = _mm_cvtsi128_si32 (sum);
189 inner_product_gdouble_none_2_sse2 (gdouble * o, const gdouble * a, const gdouble * b,
190 gint len, gpointer icoeff, gint oversample)
193 __m128d sum = _mm_setzero_pd (), t;
195 for (; i < len; i += 4) {
196 t = _mm_load_pd (b + i);
198 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + 2 * i),
199 _mm_unpacklo_pd (t, t)));
201 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + 2 * i + 2),
202 _mm_unpackhi_pd (t, t)));
204 t = _mm_load_pd (b + i + 2);
206 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + 2 * i + 4),
207 _mm_unpacklo_pd (t, t)));
209 _mm_add_pd (sum, _mm_mul_pd (_mm_loadu_pd (a + 2 * i + 6),
210 _mm_unpackhi_pd (t, t)));
212 _mm_store_pd (o, sum);
215 MAKE_RESAMPLE_FUNC (gint16, none, 1, sse2);
216 MAKE_RESAMPLE_FUNC (gdouble, none, 1, sse2);
217 MAKE_RESAMPLE_FUNC (gint16, none, 2, sse2);
218 MAKE_RESAMPLE_FUNC (gdouble, none, 2, sse2);
221 #if defined (HAVE_SMMINTRIN_H) && defined(__SSE4_1__)
222 #include <smmintrin.h>
225 inner_product_gint32_none_1_sse41 (gint32 * o, const gint32 * a, const gint32 * b,
226 gint len, gpointer icoeff, gint oversample)
232 sum = _mm_setzero_si128 ();
234 for (; i < len; i += 8) {
235 ta = _mm_loadu_si128 ((__m128i *) (a + i));
236 tb = _mm_load_si128 ((__m128i *) (b + i));
239 _mm_add_epi64 (sum, _mm_mul_epi32 (_mm_unpacklo_epi32 (ta, ta),
240 _mm_unpacklo_epi32 (tb, tb)));
242 _mm_add_epi64 (sum, _mm_mul_epi32 (_mm_unpackhi_epi32 (ta, ta),
243 _mm_unpackhi_epi32 (tb, tb)));
245 ta = _mm_loadu_si128 ((__m128i *) (a + i + 4));
246 tb = _mm_load_si128 ((__m128i *) (b + i + 4));
249 _mm_add_epi64 (sum, _mm_mul_epi32 (_mm_unpacklo_epi32 (ta, ta),
250 _mm_unpacklo_epi32 (tb, tb)));
252 _mm_add_epi64 (sum, _mm_mul_epi32 (_mm_unpackhi_epi32 (ta, ta),
253 _mm_unpackhi_epi32 (tb, tb)));
255 sum = _mm_add_epi64 (sum, _mm_unpackhi_epi64 (sum, sum));
256 res = _mm_cvtsi128_si64 (sum);
258 res = (res + (1 << (PRECISION_S32 - 1))) >> PRECISION_S32;
259 *o = CLAMP (res, -(1L << 31), (1L << 31) - 1);
262 MAKE_RESAMPLE_FUNC (gint32, none, 1, sse41);
266 audio_resampler_check_x86 (const gchar *option)
268 if (!strcmp (option, "sse")) {
269 #if defined (HAVE_XMMINTRIN_H) && defined(__SSE__)
270 GST_DEBUG ("enable SSE optimisations");
271 resample_gfloat_none_1 = resample_gfloat_none_1_sse;
272 resample_gfloat_none_2 = resample_gfloat_none_2_sse;
273 resample_gfloat_linear_1 = resample_gfloat_linear_1_sse;
275 } else if (!strcmp (option, "sse2")) {
276 #if defined (HAVE_EMMINTRIN_H) && defined(__SSE2__)
277 GST_DEBUG ("enable SSE2 optimisations");
278 resample_gint16_none_1 = resample_gint16_none_1_sse2;
279 resample_gfloat_none_1 = resample_gfloat_none_1_sse;
280 resample_gfloat_linear_1 = resample_gfloat_linear_1_sse;
281 resample_gfloat_none_2 = resample_gfloat_none_2_sse;
282 resample_gdouble_none_1 = resample_gdouble_none_1_sse2;
283 resample_gint16_none_2 = resample_gint16_none_2_sse2;
284 resample_gdouble_none_2 = resample_gdouble_none_2_sse2;
286 } else if (!strcmp (option, "sse41")) {
287 #if defined (HAVE_SMMINTRIN_H) && defined(__SSE4_1__)
288 GST_DEBUG ("enable SSE41 optimisations");
289 resample_gint32_none_1 = resample_gint32_none_1_sse41;