2 * Copyright (c) 2013 Seppo Tomperi
3 * Copyright (c) 2013 - 2014 Pierre-Edouard Lepere
6 * This file is part of FFmpeg.
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 #include "libavutil/cpu.h"
25 #include "libavutil/x86/asm.h"
26 #include "libavutil/x86/cpu.h"
27 #include "libavcodec/get_bits.h" /* required for hevcdsp.h GetBitContext */
28 #include "libavcodec/hevcdsp.h"
29 #include "libavcodec/x86/hevcdsp.h"
31 #define LFC_FUNC(DIR, DEPTH, OPT) \
32 void ff_hevc_ ## DIR ## _loop_filter_chroma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int *tc, uint8_t *no_p, uint8_t *no_q);
34 #define LFL_FUNC(DIR, DEPTH, OPT) \
35 void ff_hevc_ ## DIR ## _loop_filter_luma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int beta, int *tc, uint8_t *no_p, uint8_t *no_q);
37 #define LFC_FUNCS(type, depth, opt) \
38 LFC_FUNC(h, depth, opt) \
39 LFC_FUNC(v, depth, opt)
41 #define LFL_FUNCS(type, depth, opt) \
42 LFL_FUNC(h, depth, opt) \
43 LFL_FUNC(v, depth, opt)
45 LFC_FUNCS(uint8_t, 8, sse2)
46 LFC_FUNCS(uint8_t, 10, sse2)
47 LFC_FUNCS(uint8_t, 12, sse2)
48 LFC_FUNCS(uint8_t, 8, avx)
49 LFC_FUNCS(uint8_t, 10, avx)
50 LFC_FUNCS(uint8_t, 12, avx)
51 LFL_FUNCS(uint8_t, 8, sse2)
52 LFL_FUNCS(uint8_t, 10, sse2)
53 LFL_FUNCS(uint8_t, 12, sse2)
54 LFL_FUNCS(uint8_t, 8, ssse3)
55 LFL_FUNCS(uint8_t, 10, ssse3)
56 LFL_FUNCS(uint8_t, 12, ssse3)
57 LFL_FUNCS(uint8_t, 8, avx)
58 LFL_FUNCS(uint8_t, 10, avx)
59 LFL_FUNCS(uint8_t, 12, avx)
61 #define IDCT_FUNCS(W, opt) \
62 void ff_hevc_idct##W##_dc_8_##opt(int16_t *coeffs); \
63 void ff_hevc_idct##W##_dc_10_##opt(int16_t *coeffs); \
64 void ff_hevc_idct##W##_dc_12_##opt(int16_t *coeffs)
66 IDCT_FUNCS(4x4, mmxext);
67 IDCT_FUNCS(8x8, mmxext);
68 IDCT_FUNCS(8x8, sse2);
69 IDCT_FUNCS(16x16, sse2);
70 IDCT_FUNCS(32x32, sse2);
71 IDCT_FUNCS(16x16, avx2);
72 IDCT_FUNCS(32x32, avx2);
74 #define mc_rep_func(name, bitd, step, W, opt) \
75 void ff_hevc_put_hevc_##name##W##_##bitd##_##opt(int16_t *_dst, ptrdiff_t dststride, \
76 uint8_t *_src, ptrdiff_t _srcstride, int height, \
77 intptr_t mx, intptr_t my, int width) \
82 for (i = 0; i < W; i += step) { \
83 src = _src + (i * ((bitd + 7) / 8)); \
85 ff_hevc_put_hevc_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, height, mx, my, width); \
88 #define mc_rep_uni_func(name, bitd, step, W, opt) \
89 void ff_hevc_put_hevc_uni_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, \
90 uint8_t *_src, ptrdiff_t _srcstride, int height, \
91 intptr_t mx, intptr_t my, int width) \
96 for (i = 0; i < W; i += step) { \
97 src = _src + (i * ((bitd + 7) / 8)); \
98 dst = _dst + (i * ((bitd + 7) / 8)); \
99 ff_hevc_put_hevc_uni_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, \
100 height, mx, my, width); \
103 #define mc_rep_bi_func(name, bitd, step, W, opt) \
104 void ff_hevc_put_hevc_bi_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, uint8_t *_src, \
105 ptrdiff_t _srcstride, int16_t* _src2, ptrdiff_t _src2stride, \
106 int height, intptr_t mx, intptr_t my, int width) \
112 for (i = 0; i < W ; i += step) { \
113 src = _src + (i * ((bitd + 7) / 8)); \
114 dst = _dst + (i * ((bitd + 7) / 8)); \
116 ff_hevc_put_hevc_bi_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, \
117 _src2stride, height, mx, my, width); \
121 #define mc_rep_funcs(name, bitd, step, W, opt) \
122 mc_rep_func(name, bitd, step, W, opt); \
123 mc_rep_uni_func(name, bitd, step, W, opt); \
124 mc_rep_bi_func(name, bitd, step, W, opt)
127 #if ARCH_X86_64 && HAVE_SSE4_EXTERNAL
129 mc_rep_funcs(pel_pixels, 8, 16, 64, sse4);
130 mc_rep_funcs(pel_pixels, 8, 16, 48, sse4);
131 mc_rep_funcs(pel_pixels, 8, 16, 32, sse4);
132 mc_rep_funcs(pel_pixels, 8, 8, 24, sse4);
133 mc_rep_funcs(pel_pixels,10, 8, 64, sse4);
134 mc_rep_funcs(pel_pixels,10, 8, 48, sse4);
135 mc_rep_funcs(pel_pixels,10, 8, 32, sse4);
136 mc_rep_funcs(pel_pixels,10, 8, 24, sse4);
137 mc_rep_funcs(pel_pixels,10, 8, 16, sse4);
138 mc_rep_funcs(pel_pixels,10, 4, 12, sse4);
139 mc_rep_funcs(pel_pixels,12, 8, 64, sse4);
140 mc_rep_funcs(pel_pixels,12, 8, 48, sse4);
141 mc_rep_funcs(pel_pixels,12, 8, 32, sse4);
142 mc_rep_funcs(pel_pixels,12, 8, 24, sse4);
143 mc_rep_funcs(pel_pixels,12, 8, 16, sse4);
144 mc_rep_funcs(pel_pixels,12, 4, 12, sse4);
146 mc_rep_funcs(epel_h, 8, 16, 64, sse4);
147 mc_rep_funcs(epel_h, 8, 16, 48, sse4);
148 mc_rep_funcs(epel_h, 8, 16, 32, sse4);
149 mc_rep_funcs(epel_h, 8, 8, 24, sse4);
150 mc_rep_funcs(epel_h,10, 8, 64, sse4);
151 mc_rep_funcs(epel_h,10, 8, 48, sse4);
152 mc_rep_funcs(epel_h,10, 8, 32, sse4);
153 mc_rep_funcs(epel_h,10, 8, 24, sse4);
154 mc_rep_funcs(epel_h,10, 8, 16, sse4);
155 mc_rep_funcs(epel_h,10, 4, 12, sse4);
156 mc_rep_funcs(epel_h,12, 8, 64, sse4);
157 mc_rep_funcs(epel_h,12, 8, 48, sse4);
158 mc_rep_funcs(epel_h,12, 8, 32, sse4);
159 mc_rep_funcs(epel_h,12, 8, 24, sse4);
160 mc_rep_funcs(epel_h,12, 8, 16, sse4);
161 mc_rep_funcs(epel_h,12, 4, 12, sse4);
162 mc_rep_funcs(epel_v, 8, 16, 64, sse4);
163 mc_rep_funcs(epel_v, 8, 16, 48, sse4);
164 mc_rep_funcs(epel_v, 8, 16, 32, sse4);
165 mc_rep_funcs(epel_v, 8, 8, 24, sse4);
166 mc_rep_funcs(epel_v,10, 8, 64, sse4);
167 mc_rep_funcs(epel_v,10, 8, 48, sse4);
168 mc_rep_funcs(epel_v,10, 8, 32, sse4);
169 mc_rep_funcs(epel_v,10, 8, 24, sse4);
170 mc_rep_funcs(epel_v,10, 8, 16, sse4);
171 mc_rep_funcs(epel_v,10, 4, 12, sse4);
172 mc_rep_funcs(epel_v,12, 8, 64, sse4);
173 mc_rep_funcs(epel_v,12, 8, 48, sse4);
174 mc_rep_funcs(epel_v,12, 8, 32, sse4);
175 mc_rep_funcs(epel_v,12, 8, 24, sse4);
176 mc_rep_funcs(epel_v,12, 8, 16, sse4);
177 mc_rep_funcs(epel_v,12, 4, 12, sse4);
178 mc_rep_funcs(epel_hv, 8, 8, 64, sse4);
179 mc_rep_funcs(epel_hv, 8, 8, 48, sse4);
180 mc_rep_funcs(epel_hv, 8, 8, 32, sse4);
181 mc_rep_funcs(epel_hv, 8, 8, 24, sse4);
182 mc_rep_funcs(epel_hv, 8, 8, 16, sse4);
183 mc_rep_funcs(epel_hv, 8, 4, 12, sse4);
184 mc_rep_funcs(epel_hv,10, 8, 64, sse4);
185 mc_rep_funcs(epel_hv,10, 8, 48, sse4);
186 mc_rep_funcs(epel_hv,10, 8, 32, sse4);
187 mc_rep_funcs(epel_hv,10, 8, 24, sse4);
188 mc_rep_funcs(epel_hv,10, 8, 16, sse4);
189 mc_rep_funcs(epel_hv,10, 4, 12, sse4);
190 mc_rep_funcs(epel_hv,12, 8, 64, sse4);
191 mc_rep_funcs(epel_hv,12, 8, 48, sse4);
192 mc_rep_funcs(epel_hv,12, 8, 32, sse4);
193 mc_rep_funcs(epel_hv,12, 8, 24, sse4);
194 mc_rep_funcs(epel_hv,12, 8, 16, sse4);
195 mc_rep_funcs(epel_hv,12, 4, 12, sse4);
197 mc_rep_funcs(qpel_h, 8, 16, 64, sse4);
198 mc_rep_funcs(qpel_h, 8, 16, 48, sse4);
199 mc_rep_funcs(qpel_h, 8, 16, 32, sse4);
200 mc_rep_funcs(qpel_h, 8, 8, 24, sse4);
201 mc_rep_funcs(qpel_h,10, 8, 64, sse4);
202 mc_rep_funcs(qpel_h,10, 8, 48, sse4);
203 mc_rep_funcs(qpel_h,10, 8, 32, sse4);
204 mc_rep_funcs(qpel_h,10, 8, 24, sse4);
205 mc_rep_funcs(qpel_h,10, 8, 16, sse4);
206 mc_rep_funcs(qpel_h,10, 4, 12, sse4);
207 mc_rep_funcs(qpel_h,12, 8, 64, sse4);
208 mc_rep_funcs(qpel_h,12, 8, 48, sse4);
209 mc_rep_funcs(qpel_h,12, 8, 32, sse4);
210 mc_rep_funcs(qpel_h,12, 8, 24, sse4);
211 mc_rep_funcs(qpel_h,12, 8, 16, sse4);
212 mc_rep_funcs(qpel_h,12, 4, 12, sse4);
213 mc_rep_funcs(qpel_v, 8, 16, 64, sse4);
214 mc_rep_funcs(qpel_v, 8, 16, 48, sse4);
215 mc_rep_funcs(qpel_v, 8, 16, 32, sse4);
216 mc_rep_funcs(qpel_v, 8, 8, 24, sse4);
217 mc_rep_funcs(qpel_v,10, 8, 64, sse4);
218 mc_rep_funcs(qpel_v,10, 8, 48, sse4);
219 mc_rep_funcs(qpel_v,10, 8, 32, sse4);
220 mc_rep_funcs(qpel_v,10, 8, 24, sse4);
221 mc_rep_funcs(qpel_v,10, 8, 16, sse4);
222 mc_rep_funcs(qpel_v,10, 4, 12, sse4);
223 mc_rep_funcs(qpel_v,12, 8, 64, sse4);
224 mc_rep_funcs(qpel_v,12, 8, 48, sse4);
225 mc_rep_funcs(qpel_v,12, 8, 32, sse4);
226 mc_rep_funcs(qpel_v,12, 8, 24, sse4);
227 mc_rep_funcs(qpel_v,12, 8, 16, sse4);
228 mc_rep_funcs(qpel_v,12, 4, 12, sse4);
229 mc_rep_funcs(qpel_hv, 8, 8, 64, sse4);
230 mc_rep_funcs(qpel_hv, 8, 8, 48, sse4);
231 mc_rep_funcs(qpel_hv, 8, 8, 32, sse4);
232 mc_rep_funcs(qpel_hv, 8, 8, 24, sse4);
233 mc_rep_funcs(qpel_hv, 8, 8, 16, sse4);
234 mc_rep_funcs(qpel_hv, 8, 4, 12, sse4);
235 mc_rep_funcs(qpel_hv,10, 8, 64, sse4);
236 mc_rep_funcs(qpel_hv,10, 8, 48, sse4);
237 mc_rep_funcs(qpel_hv,10, 8, 32, sse4);
238 mc_rep_funcs(qpel_hv,10, 8, 24, sse4);
239 mc_rep_funcs(qpel_hv,10, 8, 16, sse4);
240 mc_rep_funcs(qpel_hv,10, 4, 12, sse4);
241 mc_rep_funcs(qpel_hv,12, 8, 64, sse4);
242 mc_rep_funcs(qpel_hv,12, 8, 48, sse4);
243 mc_rep_funcs(qpel_hv,12, 8, 32, sse4);
244 mc_rep_funcs(qpel_hv,12, 8, 24, sse4);
245 mc_rep_funcs(qpel_hv,12, 8, 16, sse4);
246 mc_rep_funcs(qpel_hv,12, 4, 12, sse4);
248 #define mc_rep_uni_w(bitd, step, W, opt) \
249 void ff_hevc_put_hevc_uni_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, int16_t *_src, ptrdiff_t _srcstride,\
250 int height, int denom, int _wx, int _ox) \
255 for (i = 0; i < W; i += step) { \
257 dst= _dst + (i * ((bitd + 7) / 8)); \
258 ff_hevc_put_hevc_uni_w##step##_##bitd##_##opt(dst, dststride, src, _srcstride, \
259 height, denom, _wx, _ox); \
263 mc_rep_uni_w(8, 6, 12, sse4);
264 mc_rep_uni_w(8, 8, 16, sse4);
265 mc_rep_uni_w(8, 8, 24, sse4);
266 mc_rep_uni_w(8, 8, 32, sse4);
267 mc_rep_uni_w(8, 8, 48, sse4);
268 mc_rep_uni_w(8, 8, 64, sse4);
270 mc_rep_uni_w(10, 6, 12, sse4);
271 mc_rep_uni_w(10, 8, 16, sse4);
272 mc_rep_uni_w(10, 8, 24, sse4);
273 mc_rep_uni_w(10, 8, 32, sse4);
274 mc_rep_uni_w(10, 8, 48, sse4);
275 mc_rep_uni_w(10, 8, 64, sse4);
277 mc_rep_uni_w(12, 6, 12, sse4);
278 mc_rep_uni_w(12, 8, 16, sse4);
279 mc_rep_uni_w(12, 8, 24, sse4);
280 mc_rep_uni_w(12, 8, 32, sse4);
281 mc_rep_uni_w(12, 8, 48, sse4);
282 mc_rep_uni_w(12, 8, 64, sse4);
284 #define mc_rep_bi_w(bitd, step, W, opt) \
285 void ff_hevc_put_hevc_bi_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, int16_t *_src, ptrdiff_t _srcstride, \
286 int16_t *_src2, ptrdiff_t _src2stride, int height, \
287 int denom, int _wx0, int _wx1, int _ox0, int _ox1) \
293 for (i = 0; i < W; i += step) { \
296 dst = _dst + (i * ((bitd + 7) / 8)); \
297 ff_hevc_put_hevc_bi_w##step##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, _src2stride, \
298 height, denom, _wx0, _wx1, _ox0, _ox1); \
302 mc_rep_bi_w(8, 6, 12, sse4);
303 mc_rep_bi_w(8, 8, 16, sse4);
304 mc_rep_bi_w(8, 8, 24, sse4);
305 mc_rep_bi_w(8, 8, 32, sse4);
306 mc_rep_bi_w(8, 8, 48, sse4);
307 mc_rep_bi_w(8, 8, 64, sse4);
309 mc_rep_bi_w(10, 6, 12, sse4);
310 mc_rep_bi_w(10, 8, 16, sse4);
311 mc_rep_bi_w(10, 8, 24, sse4);
312 mc_rep_bi_w(10, 8, 32, sse4);
313 mc_rep_bi_w(10, 8, 48, sse4);
314 mc_rep_bi_w(10, 8, 64, sse4);
316 mc_rep_bi_w(12, 6, 12, sse4);
317 mc_rep_bi_w(12, 8, 16, sse4);
318 mc_rep_bi_w(12, 8, 24, sse4);
319 mc_rep_bi_w(12, 8, 32, sse4);
320 mc_rep_bi_w(12, 8, 48, sse4);
321 mc_rep_bi_w(12, 8, 64, sse4);
323 #define mc_uni_w_func(name, bitd, W, opt) \
324 void ff_hevc_put_hevc_uni_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
325 uint8_t *_src, ptrdiff_t _srcstride, \
326 int height, int denom, \
328 intptr_t mx, intptr_t my, int width) \
330 LOCAL_ALIGNED_16(int16_t, temp, [71 * 64]); \
331 ff_hevc_put_hevc_##name##W##_##bitd##_##opt(temp, 64, _src, _srcstride, height, mx, my, width); \
332 ff_hevc_put_hevc_uni_w##W##_##bitd##_##opt(_dst, _dststride, temp, 64, height, denom, _wx, _ox);\
335 #define mc_uni_w_funcs(name, bitd, opt) \
336 mc_uni_w_func(name, bitd, 4, opt); \
337 mc_uni_w_func(name, bitd, 8, opt); \
338 mc_uni_w_func(name, bitd, 12, opt); \
339 mc_uni_w_func(name, bitd, 16, opt); \
340 mc_uni_w_func(name, bitd, 24, opt); \
341 mc_uni_w_func(name, bitd, 32, opt); \
342 mc_uni_w_func(name, bitd, 48, opt); \
343 mc_uni_w_func(name, bitd, 64, opt)
345 mc_uni_w_funcs(pel_pixels, 8, sse4);
346 mc_uni_w_func(pel_pixels, 8, 6, sse4);
347 mc_uni_w_funcs(epel_h, 8, sse4);
348 mc_uni_w_func(epel_h, 8, 6, sse4);
349 mc_uni_w_funcs(epel_v, 8, sse4);
350 mc_uni_w_func(epel_v, 8, 6, sse4);
351 mc_uni_w_funcs(epel_hv, 8, sse4);
352 mc_uni_w_func(epel_hv, 8, 6, sse4);
353 mc_uni_w_funcs(qpel_h, 8, sse4);
354 mc_uni_w_funcs(qpel_v, 8, sse4);
355 mc_uni_w_funcs(qpel_hv, 8, sse4);
357 mc_uni_w_funcs(pel_pixels, 10, sse4);
358 mc_uni_w_func(pel_pixels, 10, 6, sse4);
359 mc_uni_w_funcs(epel_h, 10, sse4);
360 mc_uni_w_func(epel_h, 10, 6, sse4);
361 mc_uni_w_funcs(epel_v, 10, sse4);
362 mc_uni_w_func(epel_v, 10, 6, sse4);
363 mc_uni_w_funcs(epel_hv, 10, sse4);
364 mc_uni_w_func(epel_hv, 10, 6, sse4);
365 mc_uni_w_funcs(qpel_h, 10, sse4);
366 mc_uni_w_funcs(qpel_v, 10, sse4);
367 mc_uni_w_funcs(qpel_hv, 10, sse4);
369 mc_uni_w_funcs(pel_pixels, 12, sse4);
370 mc_uni_w_func(pel_pixels, 12, 6, sse4);
371 mc_uni_w_funcs(epel_h, 12, sse4);
372 mc_uni_w_func(epel_h, 12, 6, sse4);
373 mc_uni_w_funcs(epel_v, 12, sse4);
374 mc_uni_w_func(epel_v, 12, 6, sse4);
375 mc_uni_w_funcs(epel_hv, 12, sse4);
376 mc_uni_w_func(epel_hv, 12, 6, sse4);
377 mc_uni_w_funcs(qpel_h, 12, sse4);
378 mc_uni_w_funcs(qpel_v, 12, sse4);
379 mc_uni_w_funcs(qpel_hv, 12, sse4);
381 #define mc_bi_w_func(name, bitd, W, opt) \
382 void ff_hevc_put_hevc_bi_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
383 uint8_t *_src, ptrdiff_t _srcstride, \
384 int16_t *_src2, ptrdiff_t _src2stride, \
385 int height, int denom, \
386 int _wx0, int _wx1, int _ox0, int _ox1, \
387 intptr_t mx, intptr_t my, int width) \
389 LOCAL_ALIGNED_16(int16_t, temp, [71 * 64]); \
390 ff_hevc_put_hevc_##name##W##_##bitd##_##opt(temp, 64, _src, _srcstride, height, mx, my, width); \
391 ff_hevc_put_hevc_bi_w##W##_##bitd##_##opt(_dst, _dststride, temp, 64, _src2, _src2stride, \
392 height, denom, _wx0, _wx1, _ox0, _ox1); \
395 #define mc_bi_w_funcs(name, bitd, opt) \
396 mc_bi_w_func(name, bitd, 4, opt); \
397 mc_bi_w_func(name, bitd, 8, opt); \
398 mc_bi_w_func(name, bitd, 12, opt); \
399 mc_bi_w_func(name, bitd, 16, opt); \
400 mc_bi_w_func(name, bitd, 24, opt); \
401 mc_bi_w_func(name, bitd, 32, opt); \
402 mc_bi_w_func(name, bitd, 48, opt); \
403 mc_bi_w_func(name, bitd, 64, opt)
405 mc_bi_w_funcs(pel_pixels, 8, sse4);
406 mc_bi_w_func(pel_pixels, 8, 6, sse4);
407 mc_bi_w_funcs(epel_h, 8, sse4);
408 mc_bi_w_func(epel_h, 8, 6, sse4);
409 mc_bi_w_funcs(epel_v, 8, sse4);
410 mc_bi_w_func(epel_v, 8, 6, sse4);
411 mc_bi_w_funcs(epel_hv, 8, sse4);
412 mc_bi_w_func(epel_hv, 8, 6, sse4);
413 mc_bi_w_funcs(qpel_h, 8, sse4);
414 mc_bi_w_funcs(qpel_v, 8, sse4);
415 mc_bi_w_funcs(qpel_hv, 8, sse4);
417 mc_bi_w_funcs(pel_pixels, 10, sse4);
418 mc_bi_w_func(pel_pixels, 10, 6, sse4);
419 mc_bi_w_funcs(epel_h, 10, sse4);
420 mc_bi_w_func(epel_h, 10, 6, sse4);
421 mc_bi_w_funcs(epel_v, 10, sse4);
422 mc_bi_w_func(epel_v, 10, 6, sse4);
423 mc_bi_w_funcs(epel_hv, 10, sse4);
424 mc_bi_w_func(epel_hv, 10, 6, sse4);
425 mc_bi_w_funcs(qpel_h, 10, sse4);
426 mc_bi_w_funcs(qpel_v, 10, sse4);
427 mc_bi_w_funcs(qpel_hv, 10, sse4);
429 mc_bi_w_funcs(pel_pixels, 12, sse4);
430 mc_bi_w_func(pel_pixels, 12, 6, sse4);
431 mc_bi_w_funcs(epel_h, 12, sse4);
432 mc_bi_w_func(epel_h, 12, 6, sse4);
433 mc_bi_w_funcs(epel_v, 12, sse4);
434 mc_bi_w_func(epel_v, 12, 6, sse4);
435 mc_bi_w_funcs(epel_hv, 12, sse4);
436 mc_bi_w_func(epel_hv, 12, 6, sse4);
437 mc_bi_w_funcs(qpel_h, 12, sse4);
438 mc_bi_w_funcs(qpel_v, 12, sse4);
439 mc_bi_w_funcs(qpel_hv, 12, sse4);
440 #endif //ARCH_X86_64 && HAVE_SSE4_EXTERNAL
443 #define EPEL_LINKS(pointer, my, mx, fname, bitd, opt ) \
444 PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
445 PEL_LINK(pointer, 2, my , mx , fname##6 , bitd, opt ); \
446 PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
447 PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
448 PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
449 PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
450 PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
451 PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
452 PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
453 #define QPEL_LINKS(pointer, my, mx, fname, bitd, opt) \
454 PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
455 PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
456 PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
457 PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
458 PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
459 PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
460 PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
461 PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
464 void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
466 int cpu_flags = av_get_cpu_flags();
468 if (bit_depth == 8) {
469 if (EXTERNAL_MMXEXT(cpu_flags)) {
470 c->idct_dc[0] = ff_hevc_idct4x4_dc_8_mmxext;
471 c->idct_dc[1] = ff_hevc_idct8x8_dc_8_mmxext;
473 if (EXTERNAL_SSE2(cpu_flags)) {
474 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_sse2;
475 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_sse2;
477 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_sse2;
478 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_sse2;
481 c->idct_dc[1] = ff_hevc_idct8x8_dc_8_sse2;
482 c->idct_dc[2] = ff_hevc_idct16x16_dc_8_sse2;
483 c->idct_dc[3] = ff_hevc_idct32x32_dc_8_sse2;
485 if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
486 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3;
487 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3;
489 if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
491 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 8, sse4);
492 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 8, sse4);
493 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 8, sse4);
494 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 8, sse4);
496 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 8, sse4);
497 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 8, sse4);
498 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 8, sse4);
499 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 8, sse4);
501 if (EXTERNAL_AVX(cpu_flags)) {
502 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_avx;
503 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_avx;
505 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_avx;
506 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_avx;
509 if (EXTERNAL_AVX2(cpu_flags)) {
510 c->idct_dc[2] = ff_hevc_idct16x16_dc_8_avx2;
511 c->idct_dc[3] = ff_hevc_idct32x32_dc_8_avx2;
513 } else if (bit_depth == 10) {
514 if (EXTERNAL_MMXEXT(cpu_flags)) {
515 c->idct_dc[0] = ff_hevc_idct4x4_dc_10_mmxext;
516 c->idct_dc[1] = ff_hevc_idct8x8_dc_10_mmxext;
518 if (EXTERNAL_SSE2(cpu_flags)) {
519 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_sse2;
520 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_sse2;
522 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_sse2;
523 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_sse2;
526 c->idct_dc[1] = ff_hevc_idct8x8_dc_10_sse2;
527 c->idct_dc[2] = ff_hevc_idct16x16_dc_10_sse2;
528 c->idct_dc[3] = ff_hevc_idct32x32_dc_10_sse2;
530 if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
531 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_ssse3;
532 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_ssse3;
534 if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
535 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 10, sse4);
536 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 10, sse4);
537 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 10, sse4);
538 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 10, sse4);
540 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 10, sse4);
541 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 10, sse4);
542 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 10, sse4);
543 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 10, sse4);
545 if (EXTERNAL_AVX(cpu_flags)) {
546 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_avx;
547 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_avx;
549 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_avx;
550 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_avx;
553 if (EXTERNAL_AVX2(cpu_flags)) {
554 c->idct_dc[2] = ff_hevc_idct16x16_dc_10_avx2;
555 c->idct_dc[3] = ff_hevc_idct32x32_dc_10_avx2;
558 } else if (bit_depth == 12) {
559 if (EXTERNAL_MMXEXT(cpu_flags)) {
560 c->idct_dc[0] = ff_hevc_idct4x4_dc_12_mmxext;
561 c->idct_dc[1] = ff_hevc_idct8x8_dc_12_mmxext;
563 if (EXTERNAL_SSE2(cpu_flags)) {
564 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_sse2;
565 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_sse2;
567 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_sse2;
568 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_sse2;
571 c->idct_dc[1] = ff_hevc_idct8x8_dc_12_sse2;
572 c->idct_dc[2] = ff_hevc_idct16x16_dc_12_sse2;
573 c->idct_dc[3] = ff_hevc_idct32x32_dc_12_sse2;
575 if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
576 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_ssse3;
577 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_ssse3;
579 if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
580 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 12, sse4);
581 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 12, sse4);
582 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 12, sse4);
583 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 12, sse4);
585 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 12, sse4);
586 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 12, sse4);
587 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 12, sse4);
588 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 12, sse4);
590 if (EXTERNAL_AVX(cpu_flags)) {
591 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_avx;
592 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_avx;
594 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_avx;
595 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_avx;
598 if (EXTERNAL_AVX2(cpu_flags)) {
599 c->idct_dc[2] = ff_hevc_idct16x16_dc_12_avx2;
600 c->idct_dc[3] = ff_hevc_idct32x32_dc_12_avx2;