2 * Copyright © 2008 Rodrigo Kumpera
3 * Copyright © 2008 André Tupinambá
5 * Permission to use, copy, modify, distribute, and sell this software and its
6 * documentation for any purpose is hereby granted without fee, provided that
7 * the above copyright notice appear in all copies and that both that
8 * copyright notice and this permission notice appear in supporting
9 * documentation, and that the name of Red Hat not be used in advertising or
10 * publicity pertaining to distribution of the software without specific,
11 * written prior permission. Red Hat makes no representations about the
12 * suitability of this software for any purpose. It is provided "as is"
13 * without express or implied warranty.
15 * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
16 * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
18 * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
20 * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
21 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
24 * Author: Rodrigo Kumpera (kumpera@gmail.com)
25 * André Tupinambá (andrelrt@gmail.com)
27 * Based on work by Owen Taylor and Søren Sandmann
34 #include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */
35 #include <emmintrin.h> /* for SSE2 intrinsics */
36 #include "pixman-private.h"
37 #include "pixman-combine32.h"
38 #include "pixman-fast-path.h"
40 #if defined(_MSC_VER) && defined(_M_AMD64)
41 /* Windows 64 doesn't allow MMX to be used, so
42 * the pixman-x64-mmx-emulation.h file contains
43 * implementations of those MMX intrinsics that
44 * are used in the SSE2 implementation.
46 # include "pixman-x64-mmx-emulation.h"
51 /* --------------------------------------------------------------------
55 static __m64 mask_x0080;
56 static __m64 mask_x00ff;
57 static __m64 mask_x0101;
58 static __m64 mask_x_alpha;
60 static __m64 mask_x565_rgb;
61 static __m64 mask_x565_unpack;
63 static __m128i mask_0080;
64 static __m128i mask_00ff;
65 static __m128i mask_0101;
66 static __m128i mask_ffff;
67 static __m128i mask_ff000000;
68 static __m128i mask_alpha;
70 static __m128i mask_565_r;
71 static __m128i mask_565_g1, mask_565_g2;
72 static __m128i mask_565_b;
73 static __m128i mask_red;
74 static __m128i mask_green;
75 static __m128i mask_blue;
77 static __m128i mask_565_fix_rb;
78 static __m128i mask_565_fix_g;
80 /* ----------------------------------------------------------------------
83 static force_inline __m128i
84 unpack_32_1x128 (uint32_t data)
86 return _mm_unpacklo_epi8 (_mm_cvtsi32_si128 (data), _mm_setzero_si128 ());
89 static force_inline void
90 unpack_128_2x128 (__m128i data, __m128i* data_lo, __m128i* data_hi)
92 *data_lo = _mm_unpacklo_epi8 (data, _mm_setzero_si128 ());
93 *data_hi = _mm_unpackhi_epi8 (data, _mm_setzero_si128 ());
96 static force_inline __m128i
97 unpack_565_to_8888 (__m128i lo)
99 __m128i r, g, b, rb, t;
101 r = _mm_and_si128 (_mm_slli_epi32 (lo, 8), mask_red);
102 g = _mm_and_si128 (_mm_slli_epi32 (lo, 5), mask_green);
103 b = _mm_and_si128 (_mm_slli_epi32 (lo, 3), mask_blue);
105 rb = _mm_or_si128 (r, b);
106 t = _mm_and_si128 (rb, mask_565_fix_rb);
107 t = _mm_srli_epi32 (t, 5);
108 rb = _mm_or_si128 (rb, t);
110 t = _mm_and_si128 (g, mask_565_fix_g);
111 t = _mm_srli_epi32 (t, 6);
112 g = _mm_or_si128 (g, t);
114 return _mm_or_si128 (rb, g);
117 static force_inline void
118 unpack_565_128_4x128 (__m128i data,
126 lo = _mm_unpacklo_epi16 (data, _mm_setzero_si128 ());
127 hi = _mm_unpackhi_epi16 (data, _mm_setzero_si128 ());
129 lo = unpack_565_to_8888 (lo);
130 hi = unpack_565_to_8888 (hi);
132 unpack_128_2x128 (lo, data0, data1);
133 unpack_128_2x128 (hi, data2, data3);
136 static force_inline uint16_t
137 pack_565_32_16 (uint32_t pixel)
139 return (uint16_t) (((pixel >> 8) & 0xf800) |
140 ((pixel >> 5) & 0x07e0) |
141 ((pixel >> 3) & 0x001f));
144 static force_inline __m128i
145 pack_2x128_128 (__m128i lo, __m128i hi)
147 return _mm_packus_epi16 (lo, hi);
150 static force_inline __m128i
151 pack_565_2x128_128 (__m128i lo, __m128i hi)
154 __m128i r, g1, g2, b;
156 data = pack_2x128_128 (lo, hi);
158 r = _mm_and_si128 (data, mask_565_r);
159 g1 = _mm_and_si128 (_mm_slli_epi32 (data, 3), mask_565_g1);
160 g2 = _mm_and_si128 (_mm_srli_epi32 (data, 5), mask_565_g2);
161 b = _mm_and_si128 (_mm_srli_epi32 (data, 3), mask_565_b);
163 return _mm_or_si128 (_mm_or_si128 (_mm_or_si128 (r, g1), g2), b);
166 static force_inline __m128i
167 pack_565_4x128_128 (__m128i* xmm0, __m128i* xmm1, __m128i* xmm2, __m128i* xmm3)
169 return _mm_packus_epi16 (pack_565_2x128_128 (*xmm0, *xmm1),
170 pack_565_2x128_128 (*xmm2, *xmm3));
173 static force_inline int
174 is_opaque (__m128i x)
176 __m128i ffs = _mm_cmpeq_epi8 (x, x);
178 return (_mm_movemask_epi8 (_mm_cmpeq_epi8 (x, ffs)) & 0x8888) == 0x8888;
181 static force_inline int
184 return _mm_movemask_epi8 (
185 _mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) == 0xffff;
188 static force_inline int
189 is_transparent (__m128i x)
191 return (_mm_movemask_epi8 (
192 _mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) & 0x8888) == 0x8888;
195 static force_inline __m128i
196 expand_pixel_32_1x128 (uint32_t data)
198 return _mm_shuffle_epi32 (unpack_32_1x128 (data), _MM_SHUFFLE (1, 0, 1, 0));
201 static force_inline __m128i
202 expand_alpha_1x128 (__m128i data)
204 return _mm_shufflehi_epi16 (_mm_shufflelo_epi16 (data,
205 _MM_SHUFFLE (3, 3, 3, 3)),
206 _MM_SHUFFLE (3, 3, 3, 3));
209 static force_inline void
210 expand_alpha_2x128 (__m128i data_lo,
217 lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 3, 3, 3));
218 hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 3, 3, 3));
220 *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 3, 3, 3));
221 *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 3, 3, 3));
224 static force_inline void
225 expand_alpha_rev_2x128 (__m128i data_lo,
232 lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (0, 0, 0, 0));
233 hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (0, 0, 0, 0));
234 *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (0, 0, 0, 0));
235 *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (0, 0, 0, 0));
238 static force_inline void
239 pix_multiply_2x128 (__m128i* data_lo,
248 lo = _mm_mullo_epi16 (*data_lo, *alpha_lo);
249 hi = _mm_mullo_epi16 (*data_hi, *alpha_hi);
250 lo = _mm_adds_epu16 (lo, mask_0080);
251 hi = _mm_adds_epu16 (hi, mask_0080);
252 *ret_lo = _mm_mulhi_epu16 (lo, mask_0101);
253 *ret_hi = _mm_mulhi_epu16 (hi, mask_0101);
256 static force_inline void
257 pix_add_multiply_2x128 (__m128i* src_lo,
259 __m128i* alpha_dst_lo,
260 __m128i* alpha_dst_hi,
263 __m128i* alpha_src_lo,
264 __m128i* alpha_src_hi,
268 __m128i t1_lo, t1_hi;
269 __m128i t2_lo, t2_hi;
271 pix_multiply_2x128 (src_lo, src_hi, alpha_dst_lo, alpha_dst_hi, &t1_lo, &t1_hi);
272 pix_multiply_2x128 (dst_lo, dst_hi, alpha_src_lo, alpha_src_hi, &t2_lo, &t2_hi);
274 *ret_lo = _mm_adds_epu8 (t1_lo, t2_lo);
275 *ret_hi = _mm_adds_epu8 (t1_hi, t2_hi);
278 static force_inline void
279 negate_2x128 (__m128i data_lo,
284 *neg_lo = _mm_xor_si128 (data_lo, mask_00ff);
285 *neg_hi = _mm_xor_si128 (data_hi, mask_00ff);
288 static force_inline void
289 invert_colors_2x128 (__m128i data_lo,
296 lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 0, 1, 2));
297 hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 0, 1, 2));
298 *inv_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 0, 1, 2));
299 *inv_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 0, 1, 2));
302 static force_inline void
303 over_2x128 (__m128i* src_lo,
312 negate_2x128 (*alpha_lo, *alpha_hi, &t1, &t2);
314 pix_multiply_2x128 (dst_lo, dst_hi, &t1, &t2, dst_lo, dst_hi);
316 *dst_lo = _mm_adds_epu8 (*src_lo, *dst_lo);
317 *dst_hi = _mm_adds_epu8 (*src_hi, *dst_hi);
320 static force_inline void
321 over_rev_non_pre_2x128 (__m128i src_lo,
327 __m128i alpha_lo, alpha_hi;
329 expand_alpha_2x128 (src_lo, src_hi, &alpha_lo, &alpha_hi);
331 lo = _mm_or_si128 (alpha_lo, mask_alpha);
332 hi = _mm_or_si128 (alpha_hi, mask_alpha);
334 invert_colors_2x128 (src_lo, src_hi, &src_lo, &src_hi);
336 pix_multiply_2x128 (&src_lo, &src_hi, &lo, &hi, &lo, &hi);
338 over_2x128 (&lo, &hi, &alpha_lo, &alpha_hi, dst_lo, dst_hi);
341 static force_inline void
342 in_over_2x128 (__m128i* src_lo,
354 pix_multiply_2x128 (src_lo, src_hi, mask_lo, mask_hi, &s_lo, &s_hi);
355 pix_multiply_2x128 (alpha_lo, alpha_hi, mask_lo, mask_hi, &a_lo, &a_hi);
357 over_2x128 (&s_lo, &s_hi, &a_lo, &a_hi, dst_lo, dst_hi);
360 static force_inline void
361 cache_prefetch (__m128i* addr)
363 _mm_prefetch ((void const*)addr, _MM_HINT_T0);
366 static force_inline void
367 cache_prefetch_next (__m128i* addr)
369 _mm_prefetch ((void const *)(addr + 4), _MM_HINT_T0); /* 64 bytes ahead */
372 /* prefetching NULL is very slow on some systems. don't do that. */
374 static force_inline void
375 maybe_prefetch (__m128i* addr)
378 cache_prefetch (addr);
381 static force_inline void
382 maybe_prefetch_next (__m128i* addr)
385 cache_prefetch_next (addr);
388 /* load 4 pixels from a 16-byte boundary aligned address */
389 static force_inline __m128i
390 load_128_aligned (__m128i* src)
392 return _mm_load_si128 (src);
395 /* load 4 pixels from a unaligned address */
396 static force_inline __m128i
397 load_128_unaligned (const __m128i* src)
399 return _mm_loadu_si128 (src);
402 /* save 4 pixels using Write Combining memory on a 16-byte
403 * boundary aligned address
405 static force_inline void
406 save_128_write_combining (__m128i* dst,
409 _mm_stream_si128 (dst, data);
412 /* save 4 pixels on a 16-byte boundary aligned address */
413 static force_inline void
414 save_128_aligned (__m128i* dst,
417 _mm_store_si128 (dst, data);
420 /* save 4 pixels on a unaligned address */
421 static force_inline void
422 save_128_unaligned (__m128i* dst,
425 _mm_storeu_si128 (dst, data);
428 /* ------------------------------------------------------------------
432 static force_inline __m64
433 load_32_1x64 (uint32_t data)
435 return _mm_cvtsi32_si64 (data);
438 static force_inline __m64
439 unpack_32_1x64 (uint32_t data)
441 return _mm_unpacklo_pi8 (load_32_1x64 (data), _mm_setzero_si64 ());
444 static force_inline __m64
445 expand_alpha_1x64 (__m64 data)
447 return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 3, 3, 3));
450 static force_inline __m64
451 expand_alpha_rev_1x64 (__m64 data)
453 return _mm_shuffle_pi16 (data, _MM_SHUFFLE (0, 0, 0, 0));
456 static force_inline __m64
457 expand_pixel_8_1x64 (uint8_t data)
459 return _mm_shuffle_pi16 (
460 unpack_32_1x64 ((uint32_t)data), _MM_SHUFFLE (0, 0, 0, 0));
463 static force_inline __m64
464 pix_multiply_1x64 (__m64 data,
467 return _mm_mulhi_pu16 (_mm_adds_pu16 (_mm_mullo_pi16 (data, alpha),
472 static force_inline __m64
473 pix_add_multiply_1x64 (__m64* src,
478 __m64 t1 = pix_multiply_1x64 (*src, *alpha_dst);
479 __m64 t2 = pix_multiply_1x64 (*dst, *alpha_src);
481 return _mm_adds_pu8 (t1, t2);
484 static force_inline __m64
485 negate_1x64 (__m64 data)
487 return _mm_xor_si64 (data, mask_x00ff);
490 static force_inline __m64
491 invert_colors_1x64 (__m64 data)
493 return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 0, 1, 2));
496 static force_inline __m64
497 over_1x64 (__m64 src, __m64 alpha, __m64 dst)
499 return _mm_adds_pu8 (src, pix_multiply_1x64 (dst, negate_1x64 (alpha)));
502 static force_inline __m64
503 in_over_1x64 (__m64* src, __m64* alpha, __m64* mask, __m64* dst)
505 return over_1x64 (pix_multiply_1x64 (*src, *mask),
506 pix_multiply_1x64 (*alpha, *mask),
510 static force_inline __m64
511 over_rev_non_pre_1x64 (__m64 src, __m64 dst)
513 __m64 alpha = expand_alpha_1x64 (src);
515 return over_1x64 (pix_multiply_1x64 (invert_colors_1x64 (src),
516 _mm_or_si64 (alpha, mask_x_alpha)),
521 static force_inline uint32_t
522 pack_1x64_32 (__m64 data)
524 return _mm_cvtsi64_si32 (_mm_packs_pu16 (data, _mm_setzero_si64 ()));
527 /* Expand 16 bits positioned at @pos (0-3) of a mmx register into
531 * --- Expanding 565 in the low word ---
533 * m = (m << (32 - 3)) | (m << (16 - 5)) | m;
534 * m = m & (01f0003f001f);
535 * m = m * (008404100840);
538 * Note the trick here - the top word is shifted by another nibble to
539 * avoid it bumping into the middle word
541 static force_inline __m64
542 expand565_16_1x64 (uint16_t pixel)
547 p = _mm_cvtsi32_si64 ((uint32_t) pixel);
549 t1 = _mm_slli_si64 (p, 36 - 11);
550 t2 = _mm_slli_si64 (p, 16 - 5);
552 p = _mm_or_si64 (t1, p);
553 p = _mm_or_si64 (t2, p);
554 p = _mm_and_si64 (p, mask_x565_rgb);
555 p = _mm_mullo_pi16 (p, mask_x565_unpack);
557 return _mm_srli_pi16 (p, 8);
560 /* ----------------------------------------------------------------------------
561 * Compose Core transformations
563 static force_inline uint32_t
564 core_combine_over_u_pixel_sse2 (uint32_t src, uint32_t dst)
577 ms = unpack_32_1x64 (src);
578 return pack_1x64_32 (
579 over_1x64 (ms, expand_alpha_1x64 (ms), unpack_32_1x64 (dst)));
585 static force_inline uint32_t
586 combine1 (const uint32_t *ps, const uint32_t *pm)
594 mm = unpack_32_1x64 (*pm);
595 mm = expand_alpha_1x64 (mm);
597 ms = unpack_32_1x64 (s);
598 ms = pix_multiply_1x64 (ms, mm);
600 s = pack_1x64_32 (ms);
606 static force_inline __m128i
607 combine4 (const __m128i *ps, const __m128i *pm)
609 __m128i xmm_src_lo, xmm_src_hi;
610 __m128i xmm_msk_lo, xmm_msk_hi;
615 xmm_msk_lo = load_128_unaligned (pm);
617 if (is_transparent (xmm_msk_lo))
618 return _mm_setzero_si128 ();
621 s = load_128_unaligned (ps);
625 unpack_128_2x128 (s, &xmm_src_lo, &xmm_src_hi);
626 unpack_128_2x128 (xmm_msk_lo, &xmm_msk_lo, &xmm_msk_hi);
628 expand_alpha_2x128 (xmm_msk_lo, xmm_msk_hi, &xmm_msk_lo, &xmm_msk_hi);
630 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
631 &xmm_msk_lo, &xmm_msk_hi,
632 &xmm_src_lo, &xmm_src_hi);
634 s = pack_2x128_128 (xmm_src_lo, xmm_src_hi);
640 static force_inline void
641 core_combine_over_u_sse2 (uint32_t* pd,
648 __m128i xmm_dst_lo, xmm_dst_hi;
649 __m128i xmm_src_lo, xmm_src_hi;
650 __m128i xmm_alpha_lo, xmm_alpha_hi;
652 /* call prefetch hint to optimize cache load*/
653 cache_prefetch ((__m128i*)ps);
654 cache_prefetch ((__m128i*)pd);
655 maybe_prefetch ((__m128i*)pm);
657 /* Align dst on a 16-byte boundary */
658 while (w && ((unsigned long)pd & 15))
661 s = combine1 (ps, pm);
663 *pd++ = core_combine_over_u_pixel_sse2 (s, d);
670 /* call prefetch hint to optimize cache load*/
671 cache_prefetch ((__m128i*)ps);
672 cache_prefetch ((__m128i*)pd);
673 maybe_prefetch ((__m128i*)pm);
677 /* fill cache line with next memory */
678 cache_prefetch_next ((__m128i*)ps);
679 cache_prefetch_next ((__m128i*)pd);
680 maybe_prefetch_next ((__m128i*)pm);
682 /* I'm loading unaligned because I'm not sure about
683 * the address alignment.
685 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
687 if (is_opaque (xmm_src_hi))
689 save_128_aligned ((__m128i*)pd, xmm_src_hi);
691 else if (!is_zero (xmm_src_hi))
693 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
695 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
696 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
699 xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
701 over_2x128 (&xmm_src_lo, &xmm_src_hi,
702 &xmm_alpha_lo, &xmm_alpha_hi,
703 &xmm_dst_lo, &xmm_dst_hi);
705 /* rebuid the 4 pixel data and save*/
706 save_128_aligned ((__m128i*)pd,
707 pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
720 s = combine1 (ps, pm);
722 *pd++ = core_combine_over_u_pixel_sse2 (s, d);
731 static force_inline void
732 core_combine_over_reverse_u_sse2 (uint32_t* pd,
739 __m128i xmm_dst_lo, xmm_dst_hi;
740 __m128i xmm_src_lo, xmm_src_hi;
741 __m128i xmm_alpha_lo, xmm_alpha_hi;
743 /* call prefetch hint to optimize cache load*/
744 cache_prefetch ((__m128i*)ps);
745 cache_prefetch ((__m128i*)pd);
746 maybe_prefetch ((__m128i*)pm);
748 /* Align dst on a 16-byte boundary */
750 ((unsigned long)pd & 15))
753 s = combine1 (ps, pm);
755 *pd++ = core_combine_over_u_pixel_sse2 (d, s);
762 /* call prefetch hint to optimize cache load*/
763 cache_prefetch ((__m128i*)ps);
764 cache_prefetch ((__m128i*)pd);
765 maybe_prefetch ((__m128i*)pm);
769 /* fill cache line with next memory */
770 cache_prefetch_next ((__m128i*)ps);
771 cache_prefetch_next ((__m128i*)pd);
772 maybe_prefetch_next ((__m128i*)pm);
774 /* I'm loading unaligned because I'm not sure
775 * about the address alignment.
777 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
778 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
780 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
781 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
783 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
784 &xmm_alpha_lo, &xmm_alpha_hi);
786 over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
787 &xmm_alpha_lo, &xmm_alpha_hi,
788 &xmm_src_lo, &xmm_src_hi);
790 /* rebuid the 4 pixel data and save*/
791 save_128_aligned ((__m128i*)pd,
792 pack_2x128_128 (xmm_src_lo, xmm_src_hi));
805 s = combine1 (ps, pm);
807 *pd++ = core_combine_over_u_pixel_sse2 (d, s);
815 static force_inline uint32_t
816 core_combine_in_u_pixelsse2 (uint32_t src, uint32_t dst)
818 uint32_t maska = src >> 24;
824 else if (maska != 0xff)
826 return pack_1x64_32 (
827 pix_multiply_1x64 (unpack_32_1x64 (dst),
828 expand_alpha_1x64 (unpack_32_1x64 (src))));
834 static force_inline void
835 core_combine_in_u_sse2 (uint32_t* pd,
842 __m128i xmm_src_lo, xmm_src_hi;
843 __m128i xmm_dst_lo, xmm_dst_hi;
845 /* call prefetch hint to optimize cache load*/
846 cache_prefetch ((__m128i*)ps);
847 cache_prefetch ((__m128i*)pd);
848 maybe_prefetch ((__m128i*)pm);
850 while (w && ((unsigned long) pd & 15))
852 s = combine1 (ps, pm);
855 *pd++ = core_combine_in_u_pixelsse2 (d, s);
862 /* call prefetch hint to optimize cache load*/
863 cache_prefetch ((__m128i*)ps);
864 cache_prefetch ((__m128i*)pd);
865 maybe_prefetch ((__m128i*)pm);
869 /* fill cache line with next memory */
870 cache_prefetch_next ((__m128i*)ps);
871 cache_prefetch_next ((__m128i*)pd);
872 maybe_prefetch_next ((__m128i*)pm);
874 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
875 xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*) pm);
877 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
878 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
880 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
881 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
882 &xmm_dst_lo, &xmm_dst_hi,
883 &xmm_dst_lo, &xmm_dst_hi);
885 save_128_aligned ((__m128i*)pd,
886 pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
897 s = combine1 (ps, pm);
900 *pd++ = core_combine_in_u_pixelsse2 (d, s);
908 static force_inline void
909 core_combine_reverse_in_u_sse2 (uint32_t* pd,
916 __m128i xmm_src_lo, xmm_src_hi;
917 __m128i xmm_dst_lo, xmm_dst_hi;
919 /* call prefetch hint to optimize cache load*/
920 cache_prefetch ((__m128i*)ps);
921 cache_prefetch ((__m128i*)pd);
922 maybe_prefetch ((__m128i*)pm);
924 while (w && ((unsigned long) pd & 15))
926 s = combine1 (ps, pm);
929 *pd++ = core_combine_in_u_pixelsse2 (s, d);
936 /* call prefetch hint to optimize cache load*/
937 cache_prefetch ((__m128i*)ps);
938 cache_prefetch ((__m128i*)pd);
939 maybe_prefetch ((__m128i*)pm);
943 /* fill cache line with next memory */
944 cache_prefetch_next ((__m128i*)ps);
945 cache_prefetch_next ((__m128i*)pd);
946 maybe_prefetch_next ((__m128i*)pm);
948 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
949 xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
951 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
952 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
954 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
955 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
956 &xmm_src_lo, &xmm_src_hi,
957 &xmm_dst_lo, &xmm_dst_hi);
960 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
971 s = combine1 (ps, pm);
974 *pd++ = core_combine_in_u_pixelsse2 (s, d);
982 static force_inline void
983 core_combine_reverse_out_u_sse2 (uint32_t* pd,
988 /* call prefetch hint to optimize cache load*/
989 cache_prefetch ((__m128i*)ps);
990 cache_prefetch ((__m128i*)pd);
991 maybe_prefetch ((__m128i*)pm);
993 while (w && ((unsigned long) pd & 15))
995 uint32_t s = combine1 (ps, pm);
998 *pd++ = pack_1x64_32 (
1000 unpack_32_1x64 (d), negate_1x64 (
1001 expand_alpha_1x64 (unpack_32_1x64 (s)))));
1009 /* call prefetch hint to optimize cache load*/
1010 cache_prefetch ((__m128i*)ps);
1011 cache_prefetch ((__m128i*)pd);
1012 maybe_prefetch ((__m128i*)pm);
1016 __m128i xmm_src_lo, xmm_src_hi;
1017 __m128i xmm_dst_lo, xmm_dst_hi;
1019 /* fill cache line with next memory */
1020 cache_prefetch_next ((__m128i*)ps);
1021 cache_prefetch_next ((__m128i*)pd);
1022 maybe_prefetch_next ((__m128i*)pm);
1024 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1025 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1027 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1028 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1030 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1031 negate_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1033 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1034 &xmm_src_lo, &xmm_src_hi,
1035 &xmm_dst_lo, &xmm_dst_hi);
1038 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1050 uint32_t s = combine1 (ps, pm);
1053 *pd++ = pack_1x64_32 (
1055 unpack_32_1x64 (d), negate_1x64 (
1056 expand_alpha_1x64 (unpack_32_1x64 (s)))));
1064 static force_inline void
1065 core_combine_out_u_sse2 (uint32_t* pd,
1070 /* call prefetch hint to optimize cache load*/
1071 cache_prefetch ((__m128i*)ps);
1072 cache_prefetch ((__m128i*)pd);
1073 maybe_prefetch ((__m128i*)pm);
1075 while (w && ((unsigned long) pd & 15))
1077 uint32_t s = combine1 (ps, pm);
1080 *pd++ = pack_1x64_32 (
1082 unpack_32_1x64 (s), negate_1x64 (
1083 expand_alpha_1x64 (unpack_32_1x64 (d)))));
1090 /* call prefetch hint to optimize cache load*/
1091 cache_prefetch ((__m128i*)ps);
1092 cache_prefetch ((__m128i*)pd);
1093 maybe_prefetch ((__m128i*)pm);
1097 __m128i xmm_src_lo, xmm_src_hi;
1098 __m128i xmm_dst_lo, xmm_dst_hi;
1100 /* fill cache line with next memory */
1101 cache_prefetch_next ((__m128i*)ps);
1102 cache_prefetch_next ((__m128i*)pd);
1103 maybe_prefetch_next ((__m128i*)pm);
1105 xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
1106 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1108 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1109 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1111 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1112 negate_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1114 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1115 &xmm_dst_lo, &xmm_dst_hi,
1116 &xmm_dst_lo, &xmm_dst_hi);
1119 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1130 uint32_t s = combine1 (ps, pm);
1133 *pd++ = pack_1x64_32 (
1135 unpack_32_1x64 (s), negate_1x64 (
1136 expand_alpha_1x64 (unpack_32_1x64 (d)))));
1144 static force_inline uint32_t
1145 core_combine_atop_u_pixel_sse2 (uint32_t src,
1148 __m64 s = unpack_32_1x64 (src);
1149 __m64 d = unpack_32_1x64 (dst);
1151 __m64 sa = negate_1x64 (expand_alpha_1x64 (s));
1152 __m64 da = expand_alpha_1x64 (d);
1154 return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
1157 static force_inline void
1158 core_combine_atop_u_sse2 (uint32_t* pd,
1165 __m128i xmm_src_lo, xmm_src_hi;
1166 __m128i xmm_dst_lo, xmm_dst_hi;
1167 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1168 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1170 /* call prefetch hint to optimize cache load*/
1171 cache_prefetch ((__m128i*)ps);
1172 cache_prefetch ((__m128i*)pd);
1173 maybe_prefetch ((__m128i*)pm);
1175 while (w && ((unsigned long) pd & 15))
1177 s = combine1 (ps, pm);
1180 *pd++ = core_combine_atop_u_pixel_sse2 (s, d);
1187 /* call prefetch hint to optimize cache load*/
1188 cache_prefetch ((__m128i*)ps);
1189 cache_prefetch ((__m128i*)pd);
1190 maybe_prefetch ((__m128i*)pm);
1194 /* fill cache line with next memory */
1195 cache_prefetch_next ((__m128i*)ps);
1196 cache_prefetch_next ((__m128i*)pd);
1197 maybe_prefetch_next ((__m128i*)pm);
1199 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1200 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1202 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1203 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1205 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1206 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1207 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1208 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1210 negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
1211 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1213 pix_add_multiply_2x128 (
1214 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1215 &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1216 &xmm_dst_lo, &xmm_dst_hi);
1219 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1230 s = combine1 (ps, pm);
1233 *pd++ = core_combine_atop_u_pixel_sse2 (s, d);
1241 static force_inline uint32_t
1242 core_combine_reverse_atop_u_pixel_sse2 (uint32_t src,
1245 __m64 s = unpack_32_1x64 (src);
1246 __m64 d = unpack_32_1x64 (dst);
1248 __m64 sa = expand_alpha_1x64 (s);
1249 __m64 da = negate_1x64 (expand_alpha_1x64 (d));
1251 return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
1254 static force_inline void
1255 core_combine_reverse_atop_u_sse2 (uint32_t* pd,
1262 __m128i xmm_src_lo, xmm_src_hi;
1263 __m128i xmm_dst_lo, xmm_dst_hi;
1264 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1265 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1267 /* call prefetch hint to optimize cache load*/
1268 cache_prefetch ((__m128i*)ps);
1269 cache_prefetch ((__m128i*)pd);
1270 maybe_prefetch ((__m128i*)pm);
1272 while (w && ((unsigned long) pd & 15))
1274 s = combine1 (ps, pm);
1277 *pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
1284 /* call prefetch hint to optimize cache load*/
1285 cache_prefetch ((__m128i*)ps);
1286 cache_prefetch ((__m128i*)pd);
1287 maybe_prefetch ((__m128i*)pm);
1291 /* fill cache line with next memory */
1292 cache_prefetch_next ((__m128i*)ps);
1293 cache_prefetch_next ((__m128i*)pd);
1294 maybe_prefetch_next ((__m128i*)pm);
1296 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1297 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1299 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1300 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1302 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1303 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1304 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1305 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1307 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
1308 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1310 pix_add_multiply_2x128 (
1311 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1312 &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1313 &xmm_dst_lo, &xmm_dst_hi);
1316 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1327 s = combine1 (ps, pm);
1330 *pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
1338 static force_inline uint32_t
1339 core_combine_xor_u_pixel_sse2 (uint32_t src,
1342 __m64 s = unpack_32_1x64 (src);
1343 __m64 d = unpack_32_1x64 (dst);
1345 __m64 neg_d = negate_1x64 (expand_alpha_1x64 (d));
1346 __m64 neg_s = negate_1x64 (expand_alpha_1x64 (s));
1348 return pack_1x64_32 (pix_add_multiply_1x64 (&s, &neg_d, &d, &neg_s));
1351 static force_inline void
1352 core_combine_xor_u_sse2 (uint32_t* dst,
1353 const uint32_t* src,
1354 const uint32_t *mask,
1360 const uint32_t* ps = src;
1361 const uint32_t* pm = mask;
1363 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
1364 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
1365 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1366 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1368 /* call prefetch hint to optimize cache load*/
1369 cache_prefetch ((__m128i*)ps);
1370 cache_prefetch ((__m128i*)pd);
1371 maybe_prefetch ((__m128i*)pm);
1373 while (w && ((unsigned long) pd & 15))
1375 s = combine1 (ps, pm);
1378 *pd++ = core_combine_xor_u_pixel_sse2 (s, d);
1385 /* call prefetch hint to optimize cache load*/
1386 cache_prefetch ((__m128i*)ps);
1387 cache_prefetch ((__m128i*)pd);
1388 maybe_prefetch ((__m128i*)pm);
1392 /* fill cache line with next memory */
1393 cache_prefetch_next ((__m128i*)ps);
1394 cache_prefetch_next ((__m128i*)pd);
1395 maybe_prefetch_next ((__m128i*)pm);
1397 xmm_src = combine4 ((__m128i*) ps, (__m128i*) pm);
1398 xmm_dst = load_128_aligned ((__m128i*) pd);
1400 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
1401 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
1403 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1404 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1405 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1406 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1408 negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
1409 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1410 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
1411 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1413 pix_add_multiply_2x128 (
1414 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1415 &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1416 &xmm_dst_lo, &xmm_dst_hi);
1419 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1430 s = combine1 (ps, pm);
1433 *pd++ = core_combine_xor_u_pixel_sse2 (s, d);
1441 static force_inline void
1442 core_combine_add_u_sse2 (uint32_t* dst,
1443 const uint32_t* src,
1444 const uint32_t* mask,
1450 const uint32_t* ps = src;
1451 const uint32_t* pm = mask;
1453 /* call prefetch hint to optimize cache load*/
1454 cache_prefetch ((__m128i*)ps);
1455 cache_prefetch ((__m128i*)pd);
1456 maybe_prefetch ((__m128i*)pm);
1458 while (w && (unsigned long)pd & 15)
1460 s = combine1 (ps, pm);
1466 *pd++ = _mm_cvtsi64_si32 (
1467 _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
1471 /* call prefetch hint to optimize cache load*/
1472 cache_prefetch ((__m128i*)ps);
1473 cache_prefetch ((__m128i*)pd);
1474 maybe_prefetch ((__m128i*)pm);
1480 /* fill cache line with next memory */
1481 cache_prefetch_next ((__m128i*)ps);
1482 cache_prefetch_next ((__m128i*)pd);
1483 maybe_prefetch_next ((__m128i*)pm);
1485 s = combine4 ((__m128i*)ps, (__m128i*)pm);
1488 (__m128i*)pd, _mm_adds_epu8 (s, load_128_aligned ((__m128i*)pd)));
1499 s = combine1 (ps, pm);
1503 *pd++ = _mm_cvtsi64_si32 (
1504 _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
1510 static force_inline uint32_t
1511 core_combine_saturate_u_pixel_sse2 (uint32_t src,
1514 __m64 ms = unpack_32_1x64 (src);
1515 __m64 md = unpack_32_1x64 (dst);
1516 uint32_t sa = src >> 24;
1517 uint32_t da = ~dst >> 24;
1521 ms = pix_multiply_1x64 (
1522 ms, expand_alpha_1x64 (unpack_32_1x64 (DIV_UN8 (da, sa) << 24)));
1525 return pack_1x64_32 (_mm_adds_pu16 (md, ms));
1528 static force_inline void
1529 core_combine_saturate_u_sse2 (uint32_t * pd,
1537 __m128i xmm_src, xmm_dst;
1539 /* call prefetch hint to optimize cache load*/
1540 cache_prefetch ((__m128i*)ps);
1541 cache_prefetch ((__m128i*)pd);
1542 maybe_prefetch ((__m128i*)pm);
1544 while (w && (unsigned long)pd & 15)
1546 s = combine1 (ps, pm);
1549 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1556 /* call prefetch hint to optimize cache load*/
1557 cache_prefetch ((__m128i*)ps);
1558 cache_prefetch ((__m128i*)pd);
1559 maybe_prefetch ((__m128i*)pm);
1563 /* fill cache line with next memory */
1564 cache_prefetch_next ((__m128i*)ps);
1565 cache_prefetch_next ((__m128i*)pd);
1566 maybe_prefetch_next ((__m128i*)pm);
1568 xmm_dst = load_128_aligned ((__m128i*)pd);
1569 xmm_src = combine4 ((__m128i*)ps, (__m128i*)pm);
1571 pack_cmp = _mm_movemask_epi8 (
1573 _mm_srli_epi32 (xmm_src, 24),
1574 _mm_srli_epi32 (_mm_xor_si128 (xmm_dst, mask_ff000000), 24)));
1576 /* if some alpha src is grater than respective ~alpha dst */
1579 s = combine1 (ps++, pm);
1581 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1585 s = combine1 (ps++, pm);
1587 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1591 s = combine1 (ps++, pm);
1593 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1597 s = combine1 (ps++, pm);
1599 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1605 save_128_aligned ((__m128i*)pd, _mm_adds_epu8 (xmm_dst, xmm_src));
1618 s = combine1 (ps, pm);
1621 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1628 static force_inline void
1629 core_combine_src_ca_sse2 (uint32_t* pd,
1636 __m128i xmm_src_lo, xmm_src_hi;
1637 __m128i xmm_mask_lo, xmm_mask_hi;
1638 __m128i xmm_dst_lo, xmm_dst_hi;
1640 /* call prefetch hint to optimize cache load*/
1641 cache_prefetch ((__m128i*)ps);
1642 cache_prefetch ((__m128i*)pd);
1643 cache_prefetch ((__m128i*)pm);
1645 while (w && (unsigned long)pd & 15)
1649 *pd++ = pack_1x64_32 (
1650 pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
1654 /* call prefetch hint to optimize cache load*/
1655 cache_prefetch ((__m128i*)ps);
1656 cache_prefetch ((__m128i*)pd);
1657 cache_prefetch ((__m128i*)pm);
1661 /* fill cache line with next memory */
1662 cache_prefetch_next ((__m128i*)ps);
1663 cache_prefetch_next ((__m128i*)pd);
1664 cache_prefetch_next ((__m128i*)pm);
1666 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1667 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1669 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1670 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1672 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1673 &xmm_mask_lo, &xmm_mask_hi,
1674 &xmm_dst_lo, &xmm_dst_hi);
1677 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1689 *pd++ = pack_1x64_32 (
1690 pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
1695 static force_inline uint32_t
1696 core_combine_over_ca_pixel_sse2 (uint32_t src,
1700 __m64 s = unpack_32_1x64 (src);
1701 __m64 expAlpha = expand_alpha_1x64 (s);
1702 __m64 unpk_mask = unpack_32_1x64 (mask);
1703 __m64 unpk_dst = unpack_32_1x64 (dst);
1705 return pack_1x64_32 (in_over_1x64 (&s, &expAlpha, &unpk_mask, &unpk_dst));
1708 static force_inline void
1709 core_combine_over_ca_sse2 (uint32_t* pd,
1716 __m128i xmm_alpha_lo, xmm_alpha_hi;
1717 __m128i xmm_src_lo, xmm_src_hi;
1718 __m128i xmm_dst_lo, xmm_dst_hi;
1719 __m128i xmm_mask_lo, xmm_mask_hi;
1721 /* call prefetch hint to optimize cache load*/
1722 cache_prefetch ((__m128i*)ps);
1723 cache_prefetch ((__m128i*)pd);
1724 cache_prefetch ((__m128i*)pm);
1726 while (w && (unsigned long)pd & 15)
1732 *pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
1736 /* call prefetch hint to optimize cache load*/
1737 cache_prefetch ((__m128i*)ps);
1738 cache_prefetch ((__m128i*)pd);
1739 cache_prefetch ((__m128i*)pm);
1743 /* fill cache line with next memory */
1744 cache_prefetch_next ((__m128i*)ps);
1745 cache_prefetch_next ((__m128i*)pd);
1746 cache_prefetch_next ((__m128i*)pm);
1748 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1749 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1750 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1752 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1753 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1754 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1756 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1757 &xmm_alpha_lo, &xmm_alpha_hi);
1759 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
1760 &xmm_alpha_lo, &xmm_alpha_hi,
1761 &xmm_mask_lo, &xmm_mask_hi,
1762 &xmm_dst_lo, &xmm_dst_hi);
1765 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1779 *pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
1784 static force_inline uint32_t
1785 core_combine_over_reverse_ca_pixel_sse2 (uint32_t src,
1789 __m64 d = unpack_32_1x64 (dst);
1791 return pack_1x64_32 (
1792 over_1x64 (d, expand_alpha_1x64 (d),
1793 pix_multiply_1x64 (unpack_32_1x64 (src),
1794 unpack_32_1x64 (mask))));
1797 static force_inline void
1798 core_combine_over_reverse_ca_sse2 (uint32_t* pd,
1805 __m128i xmm_alpha_lo, xmm_alpha_hi;
1806 __m128i xmm_src_lo, xmm_src_hi;
1807 __m128i xmm_dst_lo, xmm_dst_hi;
1808 __m128i xmm_mask_lo, xmm_mask_hi;
1810 /* call prefetch hint to optimize cache load*/
1811 cache_prefetch ((__m128i*)ps);
1812 cache_prefetch ((__m128i*)pd);
1813 cache_prefetch ((__m128i*)pm);
1815 while (w && (unsigned long)pd & 15)
1821 *pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
1825 /* call prefetch hint to optimize cache load*/
1826 cache_prefetch ((__m128i*)ps);
1827 cache_prefetch ((__m128i*)pd);
1828 cache_prefetch ((__m128i*)pm);
1832 /* fill cache line with next memory */
1833 cache_prefetch_next ((__m128i*)ps);
1834 cache_prefetch_next ((__m128i*)pd);
1835 cache_prefetch_next ((__m128i*)pm);
1837 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1838 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1839 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1841 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1842 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1843 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1845 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1846 &xmm_alpha_lo, &xmm_alpha_hi);
1847 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1848 &xmm_mask_lo, &xmm_mask_hi,
1849 &xmm_mask_lo, &xmm_mask_hi);
1851 over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1852 &xmm_alpha_lo, &xmm_alpha_hi,
1853 &xmm_mask_lo, &xmm_mask_hi);
1856 (__m128i*)pd, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
1870 *pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
1875 static force_inline void
1876 core_combine_in_ca_sse2 (uint32_t * pd,
1883 __m128i xmm_alpha_lo, xmm_alpha_hi;
1884 __m128i xmm_src_lo, xmm_src_hi;
1885 __m128i xmm_dst_lo, xmm_dst_hi;
1886 __m128i xmm_mask_lo, xmm_mask_hi;
1888 /* call prefetch hint to optimize cache load*/
1889 cache_prefetch ((__m128i*)ps);
1890 cache_prefetch ((__m128i*)pd);
1891 cache_prefetch ((__m128i*)pm);
1893 while (w && (unsigned long)pd & 15)
1899 *pd++ = pack_1x64_32 (
1901 pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
1902 expand_alpha_1x64 (unpack_32_1x64 (d))));
1907 /* call prefetch hint to optimize cache load*/
1908 cache_prefetch ((__m128i*)ps);
1909 cache_prefetch ((__m128i*)pd);
1910 cache_prefetch ((__m128i*)pm);
1914 /* fill cache line with next memory */
1915 cache_prefetch_next ((__m128i*)ps);
1916 cache_prefetch_next ((__m128i*)pd);
1917 cache_prefetch_next ((__m128i*)pm);
1919 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1920 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1921 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1923 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1924 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1925 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1927 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1928 &xmm_alpha_lo, &xmm_alpha_hi);
1930 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1931 &xmm_mask_lo, &xmm_mask_hi,
1932 &xmm_dst_lo, &xmm_dst_hi);
1934 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1935 &xmm_alpha_lo, &xmm_alpha_hi,
1936 &xmm_dst_lo, &xmm_dst_hi);
1939 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1953 *pd++ = pack_1x64_32 (
1956 unpack_32_1x64 (s), unpack_32_1x64 (m)),
1957 expand_alpha_1x64 (unpack_32_1x64 (d))));
1963 static force_inline void
1964 core_combine_in_reverse_ca_sse2 (uint32_t * pd,
1971 __m128i xmm_alpha_lo, xmm_alpha_hi;
1972 __m128i xmm_src_lo, xmm_src_hi;
1973 __m128i xmm_dst_lo, xmm_dst_hi;
1974 __m128i xmm_mask_lo, xmm_mask_hi;
1976 /* call prefetch hint to optimize cache load*/
1977 cache_prefetch ((__m128i*)ps);
1978 cache_prefetch ((__m128i*)pd);
1979 cache_prefetch ((__m128i*)pm);
1981 while (w && (unsigned long)pd & 15)
1987 *pd++ = pack_1x64_32 (
1990 pix_multiply_1x64 (unpack_32_1x64 (m),
1991 expand_alpha_1x64 (unpack_32_1x64 (s)))));
1995 /* call prefetch hint to optimize cache load*/
1996 cache_prefetch ((__m128i*)ps);
1997 cache_prefetch ((__m128i*)pd);
1998 cache_prefetch ((__m128i*)pm);
2002 /* fill cache line with next memory */
2003 cache_prefetch_next ((__m128i*)ps);
2004 cache_prefetch_next ((__m128i*)pd);
2005 cache_prefetch_next ((__m128i*)pm);
2007 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2008 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2009 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2011 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2012 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2013 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2015 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2016 &xmm_alpha_lo, &xmm_alpha_hi);
2017 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2018 &xmm_alpha_lo, &xmm_alpha_hi,
2019 &xmm_alpha_lo, &xmm_alpha_hi);
2021 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
2022 &xmm_alpha_lo, &xmm_alpha_hi,
2023 &xmm_dst_lo, &xmm_dst_hi);
2026 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2040 *pd++ = pack_1x64_32 (
2043 pix_multiply_1x64 (unpack_32_1x64 (m),
2044 expand_alpha_1x64 (unpack_32_1x64 (s)))));
2049 static force_inline void
2050 core_combine_out_ca_sse2 (uint32_t * pd,
2057 __m128i xmm_alpha_lo, xmm_alpha_hi;
2058 __m128i xmm_src_lo, xmm_src_hi;
2059 __m128i xmm_dst_lo, xmm_dst_hi;
2060 __m128i xmm_mask_lo, xmm_mask_hi;
2062 /* call prefetch hint to optimize cache load*/
2063 cache_prefetch ((__m128i*)ps);
2064 cache_prefetch ((__m128i*)pd);
2065 cache_prefetch ((__m128i*)pm);
2067 while (w && (unsigned long)pd & 15)
2073 *pd++ = pack_1x64_32 (
2076 unpack_32_1x64 (s), unpack_32_1x64 (m)),
2077 negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
2081 /* call prefetch hint to optimize cache load*/
2082 cache_prefetch ((__m128i*)ps);
2083 cache_prefetch ((__m128i*)pd);
2084 cache_prefetch ((__m128i*)pm);
2088 /* fill cache line with next memory */
2089 cache_prefetch_next ((__m128i*)ps);
2090 cache_prefetch_next ((__m128i*)pd);
2091 cache_prefetch_next ((__m128i*)pm);
2093 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2094 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2095 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2097 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2098 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2099 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2101 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2102 &xmm_alpha_lo, &xmm_alpha_hi);
2103 negate_2x128 (xmm_alpha_lo, xmm_alpha_hi,
2104 &xmm_alpha_lo, &xmm_alpha_hi);
2106 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2107 &xmm_mask_lo, &xmm_mask_hi,
2108 &xmm_dst_lo, &xmm_dst_hi);
2109 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
2110 &xmm_alpha_lo, &xmm_alpha_hi,
2111 &xmm_dst_lo, &xmm_dst_hi);
2114 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2128 *pd++ = pack_1x64_32 (
2131 unpack_32_1x64 (s), unpack_32_1x64 (m)),
2132 negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
2138 static force_inline void
2139 core_combine_out_reverse_ca_sse2 (uint32_t * pd,
2146 __m128i xmm_alpha_lo, xmm_alpha_hi;
2147 __m128i xmm_src_lo, xmm_src_hi;
2148 __m128i xmm_dst_lo, xmm_dst_hi;
2149 __m128i xmm_mask_lo, xmm_mask_hi;
2151 /* call prefetch hint to optimize cache load*/
2152 cache_prefetch ((__m128i*)ps);
2153 cache_prefetch ((__m128i*)pd);
2154 cache_prefetch ((__m128i*)pm);
2156 while (w && (unsigned long)pd & 15)
2162 *pd++ = pack_1x64_32 (
2165 negate_1x64 (pix_multiply_1x64 (
2167 expand_alpha_1x64 (unpack_32_1x64 (s))))));
2171 /* call prefetch hint to optimize cache load*/
2172 cache_prefetch ((__m128i*)ps);
2173 cache_prefetch ((__m128i*)pd);
2174 cache_prefetch ((__m128i*)pm);
2178 /* fill cache line with next memory */
2179 cache_prefetch_next ((__m128i*)ps);
2180 cache_prefetch_next ((__m128i*)pd);
2181 cache_prefetch_next ((__m128i*)pm);
2183 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2184 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2185 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2187 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2188 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2189 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2191 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2192 &xmm_alpha_lo, &xmm_alpha_hi);
2194 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2195 &xmm_alpha_lo, &xmm_alpha_hi,
2196 &xmm_mask_lo, &xmm_mask_hi);
2198 negate_2x128 (xmm_mask_lo, xmm_mask_hi,
2199 &xmm_mask_lo, &xmm_mask_hi);
2201 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
2202 &xmm_mask_lo, &xmm_mask_hi,
2203 &xmm_dst_lo, &xmm_dst_hi);
2206 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2220 *pd++ = pack_1x64_32 (
2223 negate_1x64 (pix_multiply_1x64 (
2225 expand_alpha_1x64 (unpack_32_1x64 (s))))));
2230 static force_inline uint32_t
2231 core_combine_atop_ca_pixel_sse2 (uint32_t src,
2235 __m64 m = unpack_32_1x64 (mask);
2236 __m64 s = unpack_32_1x64 (src);
2237 __m64 d = unpack_32_1x64 (dst);
2238 __m64 sa = expand_alpha_1x64 (s);
2239 __m64 da = expand_alpha_1x64 (d);
2241 s = pix_multiply_1x64 (s, m);
2242 m = negate_1x64 (pix_multiply_1x64 (m, sa));
2244 return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
2247 static force_inline void
2248 core_combine_atop_ca_sse2 (uint32_t * pd,
2255 __m128i xmm_src_lo, xmm_src_hi;
2256 __m128i xmm_dst_lo, xmm_dst_hi;
2257 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2258 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2259 __m128i xmm_mask_lo, xmm_mask_hi;
2261 /* call prefetch hint to optimize cache load*/
2262 cache_prefetch ((__m128i*)ps);
2263 cache_prefetch ((__m128i*)pd);
2264 cache_prefetch ((__m128i*)pm);
2266 while (w && (unsigned long)pd & 15)
2272 *pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
2276 /* call prefetch hint to optimize cache load*/
2277 cache_prefetch ((__m128i*)ps);
2278 cache_prefetch ((__m128i*)pd);
2279 cache_prefetch ((__m128i*)pm);
2283 /* fill cache line with next memory */
2284 cache_prefetch_next ((__m128i*)ps);
2285 cache_prefetch_next ((__m128i*)pd);
2286 cache_prefetch_next ((__m128i*)pm);
2288 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2289 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2290 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2292 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2293 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2294 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2296 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2297 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2298 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2299 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2301 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2302 &xmm_mask_lo, &xmm_mask_hi,
2303 &xmm_src_lo, &xmm_src_hi);
2304 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2305 &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2306 &xmm_mask_lo, &xmm_mask_hi);
2308 negate_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2310 pix_add_multiply_2x128 (
2311 &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2312 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2313 &xmm_dst_lo, &xmm_dst_hi);
2316 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2330 *pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
2335 static force_inline uint32_t
2336 core_combine_reverse_atop_ca_pixel_sse2 (uint32_t src,
2340 __m64 m = unpack_32_1x64 (mask);
2341 __m64 s = unpack_32_1x64 (src);
2342 __m64 d = unpack_32_1x64 (dst);
2344 __m64 da = negate_1x64 (expand_alpha_1x64 (d));
2345 __m64 sa = expand_alpha_1x64 (s);
2347 s = pix_multiply_1x64 (s, m);
2348 m = pix_multiply_1x64 (m, sa);
2350 return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
2353 static force_inline void
2354 core_combine_reverse_atop_ca_sse2 (uint32_t * pd,
2361 __m128i xmm_src_lo, xmm_src_hi;
2362 __m128i xmm_dst_lo, xmm_dst_hi;
2363 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2364 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2365 __m128i xmm_mask_lo, xmm_mask_hi;
2367 /* call prefetch hint to optimize cache load*/
2368 cache_prefetch ((__m128i*)ps);
2369 cache_prefetch ((__m128i*)pd);
2370 cache_prefetch ((__m128i*)pm);
2372 while (w && (unsigned long)pd & 15)
2378 *pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
2382 /* call prefetch hint to optimize cache load*/
2383 cache_prefetch ((__m128i*)ps);
2384 cache_prefetch ((__m128i*)pd);
2385 cache_prefetch ((__m128i*)pm);
2389 /* fill cache line with next memory */
2390 cache_prefetch_next ((__m128i*)ps);
2391 cache_prefetch_next ((__m128i*)pd);
2392 cache_prefetch_next ((__m128i*)pm);
2394 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2395 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2396 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2398 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2399 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2400 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2402 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2403 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2404 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2405 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2407 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2408 &xmm_mask_lo, &xmm_mask_hi,
2409 &xmm_src_lo, &xmm_src_hi);
2410 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2411 &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2412 &xmm_mask_lo, &xmm_mask_hi);
2414 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
2415 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2417 pix_add_multiply_2x128 (
2418 &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2419 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2420 &xmm_dst_lo, &xmm_dst_hi);
2423 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2437 *pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
2442 static force_inline uint32_t
2443 core_combine_xor_ca_pixel_sse2 (uint32_t src,
2447 __m64 a = unpack_32_1x64 (mask);
2448 __m64 s = unpack_32_1x64 (src);
2449 __m64 d = unpack_32_1x64 (dst);
2451 __m64 alpha_dst = negate_1x64 (pix_multiply_1x64 (
2452 a, expand_alpha_1x64 (s)));
2453 __m64 dest = pix_multiply_1x64 (s, a);
2454 __m64 alpha_src = negate_1x64 (expand_alpha_1x64 (d));
2456 return pack_1x64_32 (pix_add_multiply_1x64 (&d,
2462 static force_inline void
2463 core_combine_xor_ca_sse2 (uint32_t * pd,
2470 __m128i xmm_src_lo, xmm_src_hi;
2471 __m128i xmm_dst_lo, xmm_dst_hi;
2472 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2473 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2474 __m128i xmm_mask_lo, xmm_mask_hi;
2476 /* call prefetch hint to optimize cache load*/
2477 cache_prefetch ((__m128i*)ps);
2478 cache_prefetch ((__m128i*)pd);
2479 cache_prefetch ((__m128i*)pm);
2481 while (w && (unsigned long)pd & 15)
2487 *pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
2491 /* call prefetch hint to optimize cache load*/
2492 cache_prefetch ((__m128i*)ps);
2493 cache_prefetch ((__m128i*)pd);
2494 cache_prefetch ((__m128i*)pm);
2498 /* fill cache line with next memory */
2499 cache_prefetch_next ((__m128i*)ps);
2500 cache_prefetch_next ((__m128i*)pd);
2501 cache_prefetch_next ((__m128i*)pm);
2503 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2504 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2505 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2507 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2508 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2509 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2511 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2512 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2513 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2514 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2516 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2517 &xmm_mask_lo, &xmm_mask_hi,
2518 &xmm_src_lo, &xmm_src_hi);
2519 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2520 &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2521 &xmm_mask_lo, &xmm_mask_hi);
2523 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
2524 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2525 negate_2x128 (xmm_mask_lo, xmm_mask_hi,
2526 &xmm_mask_lo, &xmm_mask_hi);
2528 pix_add_multiply_2x128 (
2529 &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2530 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2531 &xmm_dst_lo, &xmm_dst_hi);
2534 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2548 *pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
2553 static force_inline void
2554 core_combine_add_ca_sse2 (uint32_t * pd,
2561 __m128i xmm_src_lo, xmm_src_hi;
2562 __m128i xmm_dst_lo, xmm_dst_hi;
2563 __m128i xmm_mask_lo, xmm_mask_hi;
2565 /* call prefetch hint to optimize cache load*/
2566 cache_prefetch ((__m128i*)ps);
2567 cache_prefetch ((__m128i*)pd);
2568 cache_prefetch ((__m128i*)pm);
2570 while (w && (unsigned long)pd & 15)
2576 *pd++ = pack_1x64_32 (
2577 _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
2578 unpack_32_1x64 (m)),
2579 unpack_32_1x64 (d)));
2583 /* call prefetch hint to optimize cache load*/
2584 cache_prefetch ((__m128i*)ps);
2585 cache_prefetch ((__m128i*)pd);
2586 cache_prefetch ((__m128i*)pm);
2590 /* fill cache line with next memory */
2591 cache_prefetch_next ((__m128i*)ps);
2592 cache_prefetch_next ((__m128i*)pd);
2593 cache_prefetch_next ((__m128i*)pm);
2595 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2596 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2597 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2599 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2600 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2601 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2603 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2604 &xmm_mask_lo, &xmm_mask_hi,
2605 &xmm_src_lo, &xmm_src_hi);
2608 (__m128i*)pd, pack_2x128_128 (
2609 _mm_adds_epu8 (xmm_src_lo, xmm_dst_lo),
2610 _mm_adds_epu8 (xmm_src_hi, xmm_dst_hi)));
2624 *pd++ = pack_1x64_32 (
2625 _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
2626 unpack_32_1x64 (m)),
2627 unpack_32_1x64 (d)));
2632 /* ---------------------------------------------------
2633 * fb_compose_setup_sSE2
2635 static force_inline __m64
2636 create_mask_16_64 (uint16_t mask)
2638 return _mm_set1_pi16 (mask);
2641 static force_inline __m128i
2642 create_mask_16_128 (uint16_t mask)
2644 return _mm_set1_epi16 (mask);
2647 static force_inline __m64
2648 create_mask_2x32_64 (uint32_t mask0,
2651 return _mm_set_pi32 (mask0, mask1);
2654 /* Work around a code generation bug in Sun Studio 12. */
2655 #if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)
2656 # define create_mask_2x32_128(mask0, mask1) \
2657 (_mm_set_epi32 ((mask0), (mask1), (mask0), (mask1)))
2659 static force_inline __m128i
2660 create_mask_2x32_128 (uint32_t mask0,
2663 return _mm_set_epi32 (mask0, mask1, mask0, mask1);
2667 /* SSE2 code patch for fbcompose.c */
2670 sse2_combine_over_u (pixman_implementation_t *imp,
2673 const uint32_t * src,
2674 const uint32_t * mask,
2677 core_combine_over_u_sse2 (dst, src, mask, width);
2682 sse2_combine_over_reverse_u (pixman_implementation_t *imp,
2685 const uint32_t * src,
2686 const uint32_t * mask,
2689 core_combine_over_reverse_u_sse2 (dst, src, mask, width);
2694 sse2_combine_in_u (pixman_implementation_t *imp,
2697 const uint32_t * src,
2698 const uint32_t * mask,
2701 core_combine_in_u_sse2 (dst, src, mask, width);
2706 sse2_combine_in_reverse_u (pixman_implementation_t *imp,
2709 const uint32_t * src,
2710 const uint32_t * mask,
2713 core_combine_reverse_in_u_sse2 (dst, src, mask, width);
2718 sse2_combine_out_u (pixman_implementation_t *imp,
2721 const uint32_t * src,
2722 const uint32_t * mask,
2725 core_combine_out_u_sse2 (dst, src, mask, width);
2730 sse2_combine_out_reverse_u (pixman_implementation_t *imp,
2733 const uint32_t * src,
2734 const uint32_t * mask,
2737 core_combine_reverse_out_u_sse2 (dst, src, mask, width);
2742 sse2_combine_atop_u (pixman_implementation_t *imp,
2745 const uint32_t * src,
2746 const uint32_t * mask,
2749 core_combine_atop_u_sse2 (dst, src, mask, width);
2754 sse2_combine_atop_reverse_u (pixman_implementation_t *imp,
2757 const uint32_t * src,
2758 const uint32_t * mask,
2761 core_combine_reverse_atop_u_sse2 (dst, src, mask, width);
2766 sse2_combine_xor_u (pixman_implementation_t *imp,
2769 const uint32_t * src,
2770 const uint32_t * mask,
2773 core_combine_xor_u_sse2 (dst, src, mask, width);
2778 sse2_combine_add_u (pixman_implementation_t *imp,
2781 const uint32_t * src,
2782 const uint32_t * mask,
2785 core_combine_add_u_sse2 (dst, src, mask, width);
2790 sse2_combine_saturate_u (pixman_implementation_t *imp,
2793 const uint32_t * src,
2794 const uint32_t * mask,
2797 core_combine_saturate_u_sse2 (dst, src, mask, width);
2802 sse2_combine_src_ca (pixman_implementation_t *imp,
2805 const uint32_t * src,
2806 const uint32_t * mask,
2809 core_combine_src_ca_sse2 (dst, src, mask, width);
2814 sse2_combine_over_ca (pixman_implementation_t *imp,
2817 const uint32_t * src,
2818 const uint32_t * mask,
2821 core_combine_over_ca_sse2 (dst, src, mask, width);
2826 sse2_combine_over_reverse_ca (pixman_implementation_t *imp,
2829 const uint32_t * src,
2830 const uint32_t * mask,
2833 core_combine_over_reverse_ca_sse2 (dst, src, mask, width);
2838 sse2_combine_in_ca (pixman_implementation_t *imp,
2841 const uint32_t * src,
2842 const uint32_t * mask,
2845 core_combine_in_ca_sse2 (dst, src, mask, width);
2850 sse2_combine_in_reverse_ca (pixman_implementation_t *imp,
2853 const uint32_t * src,
2854 const uint32_t * mask,
2857 core_combine_in_reverse_ca_sse2 (dst, src, mask, width);
2862 sse2_combine_out_ca (pixman_implementation_t *imp,
2865 const uint32_t * src,
2866 const uint32_t * mask,
2869 core_combine_out_ca_sse2 (dst, src, mask, width);
2874 sse2_combine_out_reverse_ca (pixman_implementation_t *imp,
2877 const uint32_t * src,
2878 const uint32_t * mask,
2881 core_combine_out_reverse_ca_sse2 (dst, src, mask, width);
2886 sse2_combine_atop_ca (pixman_implementation_t *imp,
2889 const uint32_t * src,
2890 const uint32_t * mask,
2893 core_combine_atop_ca_sse2 (dst, src, mask, width);
2898 sse2_combine_atop_reverse_ca (pixman_implementation_t *imp,
2901 const uint32_t * src,
2902 const uint32_t * mask,
2905 core_combine_reverse_atop_ca_sse2 (dst, src, mask, width);
2910 sse2_combine_xor_ca (pixman_implementation_t *imp,
2913 const uint32_t * src,
2914 const uint32_t * mask,
2917 core_combine_xor_ca_sse2 (dst, src, mask, width);
2922 sse2_combine_add_ca (pixman_implementation_t *imp,
2925 const uint32_t * src,
2926 const uint32_t * mask,
2929 core_combine_add_ca_sse2 (dst, src, mask, width);
2933 /* -------------------------------------------------------------------
2934 * composite_over_n_8888
2938 sse2_composite_over_n_8888 (pixman_implementation_t *imp,
2940 pixman_image_t * src_image,
2941 pixman_image_t * mask_image,
2942 pixman_image_t * dst_image,
2953 uint32_t *dst_line, *dst, d;
2956 __m128i xmm_src, xmm_alpha;
2957 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
2959 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
2964 PIXMAN_IMAGE_GET_LINE (
2965 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2967 xmm_src = expand_pixel_32_1x128 (src);
2968 xmm_alpha = expand_alpha_1x128 (xmm_src);
2974 /* call prefetch hint to optimize cache load*/
2975 cache_prefetch ((__m128i*)dst);
2977 dst_line += dst_stride;
2980 while (w && (unsigned long)dst & 15)
2983 *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
2984 _mm_movepi64_pi64 (xmm_alpha),
2985 unpack_32_1x64 (d)));
2989 cache_prefetch ((__m128i*)dst);
2993 /* fill cache line with next memory */
2994 cache_prefetch_next ((__m128i*)dst);
2996 xmm_dst = load_128_aligned ((__m128i*)dst);
2998 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3000 over_2x128 (&xmm_src, &xmm_src,
3001 &xmm_alpha, &xmm_alpha,
3002 &xmm_dst_lo, &xmm_dst_hi);
3004 /* rebuid the 4 pixel data and save*/
3006 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3015 *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
3016 _mm_movepi64_pi64 (xmm_alpha),
3017 unpack_32_1x64 (d)));
3025 /* ---------------------------------------------------------------------
3026 * composite_over_n_0565
3029 sse2_composite_over_n_0565 (pixman_implementation_t *imp,
3031 pixman_image_t * src_image,
3032 pixman_image_t * mask_image,
3033 pixman_image_t * dst_image,
3044 uint16_t *dst_line, *dst, d;
3047 __m128i xmm_src, xmm_alpha;
3048 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
3050 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3055 PIXMAN_IMAGE_GET_LINE (
3056 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
3058 xmm_src = expand_pixel_32_1x128 (src);
3059 xmm_alpha = expand_alpha_1x128 (xmm_src);
3065 /* call prefetch hint to optimize cache load*/
3066 cache_prefetch ((__m128i*)dst);
3068 dst_line += dst_stride;
3071 while (w && (unsigned long)dst & 15)
3075 *dst++ = pack_565_32_16 (
3076 pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
3077 _mm_movepi64_pi64 (xmm_alpha),
3078 expand565_16_1x64 (d))));
3082 /* call prefetch hint to optimize cache load*/
3083 cache_prefetch ((__m128i*)dst);
3087 /* fill cache line with next memory */
3088 cache_prefetch_next ((__m128i*)dst);
3090 xmm_dst = load_128_aligned ((__m128i*)dst);
3092 unpack_565_128_4x128 (xmm_dst,
3093 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3095 over_2x128 (&xmm_src, &xmm_src,
3096 &xmm_alpha, &xmm_alpha,
3097 &xmm_dst0, &xmm_dst1);
3098 over_2x128 (&xmm_src, &xmm_src,
3099 &xmm_alpha, &xmm_alpha,
3100 &xmm_dst2, &xmm_dst3);
3102 xmm_dst = pack_565_4x128_128 (
3103 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3105 save_128_aligned ((__m128i*)dst, xmm_dst);
3114 *dst++ = pack_565_32_16 (
3115 pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
3116 _mm_movepi64_pi64 (xmm_alpha),
3117 expand565_16_1x64 (d))));
3124 /* ------------------------------
3125 * composite_add_n_8888_8888_ca
3128 sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
3130 pixman_image_t * src_image,
3131 pixman_image_t * mask_image,
3132 pixman_image_t * dst_image,
3143 uint32_t *dst_line, d;
3144 uint32_t *mask_line, m;
3146 int dst_stride, mask_stride;
3148 __m128i xmm_src, xmm_alpha;
3150 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3152 __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3154 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3160 PIXMAN_IMAGE_GET_LINE (
3161 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3162 PIXMAN_IMAGE_GET_LINE (
3163 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
3165 xmm_src = _mm_unpacklo_epi8 (
3166 create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
3167 xmm_alpha = expand_alpha_1x128 (xmm_src);
3168 mmx_src = _mm_movepi64_pi64 (xmm_src);
3169 mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
3174 const uint32_t *pm = (uint32_t *)mask_line;
3175 uint32_t *pd = (uint32_t *)dst_line;
3177 dst_line += dst_stride;
3178 mask_line += mask_stride;
3180 /* call prefetch hint to optimize cache load*/
3181 cache_prefetch ((__m128i*)pd);
3182 cache_prefetch ((__m128i*)pm);
3184 while (w && (unsigned long)pd & 15)
3192 mmx_mask = unpack_32_1x64 (m);
3193 mmx_dest = unpack_32_1x64 (d);
3195 *pd = pack_1x64_32 (
3196 _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
3203 /* call prefetch hint to optimize cache load*/
3204 cache_prefetch ((__m128i*)pd);
3205 cache_prefetch ((__m128i*)pm);
3209 /* fill cache line with next memory */
3210 cache_prefetch_next ((__m128i*)pd);
3211 cache_prefetch_next ((__m128i*)pm);
3213 xmm_mask = load_128_unaligned ((__m128i*)pm);
3217 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
3219 /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
3220 if (pack_cmp != 0xffff)
3222 xmm_dst = load_128_aligned ((__m128i*)pd);
3224 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3226 pix_multiply_2x128 (&xmm_src, &xmm_src,
3227 &xmm_mask_lo, &xmm_mask_hi,
3228 &xmm_mask_lo, &xmm_mask_hi);
3229 xmm_mask_hi = pack_2x128_128 (xmm_mask_lo, xmm_mask_hi);
3232 (__m128i*)pd, _mm_adds_epu8 (xmm_mask_hi, xmm_dst));
3248 mmx_mask = unpack_32_1x64 (m);
3249 mmx_dest = unpack_32_1x64 (d);
3251 *pd = pack_1x64_32 (
3252 _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
3263 /* ---------------------------------------------------------------------------
3264 * composite_over_n_8888_8888_ca
3268 sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
3270 pixman_image_t * src_image,
3271 pixman_image_t * mask_image,
3272 pixman_image_t * dst_image,
3283 uint32_t *dst_line, d;
3284 uint32_t *mask_line, m;
3286 int dst_stride, mask_stride;
3288 __m128i xmm_src, xmm_alpha;
3289 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3290 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3292 __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3294 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3299 PIXMAN_IMAGE_GET_LINE (
3300 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3301 PIXMAN_IMAGE_GET_LINE (
3302 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
3304 xmm_src = _mm_unpacklo_epi8 (
3305 create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
3306 xmm_alpha = expand_alpha_1x128 (xmm_src);
3307 mmx_src = _mm_movepi64_pi64 (xmm_src);
3308 mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
3313 const uint32_t *pm = (uint32_t *)mask_line;
3314 uint32_t *pd = (uint32_t *)dst_line;
3316 dst_line += dst_stride;
3317 mask_line += mask_stride;
3319 /* call prefetch hint to optimize cache load*/
3320 cache_prefetch ((__m128i*)pd);
3321 cache_prefetch ((__m128i*)pm);
3323 while (w && (unsigned long)pd & 15)
3330 mmx_mask = unpack_32_1x64 (m);
3331 mmx_dest = unpack_32_1x64 (d);
3333 *pd = pack_1x64_32 (in_over_1x64 (&mmx_src,
3343 /* call prefetch hint to optimize cache load*/
3344 cache_prefetch ((__m128i*)pd);
3345 cache_prefetch ((__m128i*)pm);
3349 /* fill cache line with next memory */
3350 cache_prefetch_next ((__m128i*)pd);
3351 cache_prefetch_next ((__m128i*)pm);
3353 xmm_mask = load_128_unaligned ((__m128i*)pm);
3357 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
3359 /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
3360 if (pack_cmp != 0xffff)
3362 xmm_dst = load_128_aligned ((__m128i*)pd);
3364 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3365 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3367 in_over_2x128 (&xmm_src, &xmm_src,
3368 &xmm_alpha, &xmm_alpha,
3369 &xmm_mask_lo, &xmm_mask_hi,
3370 &xmm_dst_lo, &xmm_dst_hi);
3373 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3388 mmx_mask = unpack_32_1x64 (m);
3389 mmx_dest = unpack_32_1x64 (d);
3391 *pd = pack_1x64_32 (
3392 in_over_1x64 (&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest));
3403 /*---------------------------------------------------------------------
3404 * composite_over_8888_n_8888
3408 sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
3410 pixman_image_t * src_image,
3411 pixman_image_t * mask_image,
3412 pixman_image_t * dst_image,
3422 uint32_t *dst_line, *dst;
3423 uint32_t *src_line, *src;
3426 int dst_stride, src_stride;
3429 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3430 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3431 __m128i xmm_alpha_lo, xmm_alpha_hi;
3433 PIXMAN_IMAGE_GET_LINE (
3434 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3435 PIXMAN_IMAGE_GET_LINE (
3436 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3438 mask = _pixman_image_get_solid (mask_image, PIXMAN_a8r8g8b8);
3440 xmm_mask = create_mask_16_128 (mask >> 24);
3445 dst_line += dst_stride;
3447 src_line += src_stride;
3450 /* call prefetch hint to optimize cache load*/
3451 cache_prefetch ((__m128i*)dst);
3452 cache_prefetch ((__m128i*)src);
3454 while (w && (unsigned long)dst & 15)
3456 uint32_t s = *src++;
3459 __m64 ms = unpack_32_1x64 (s);
3460 __m64 alpha = expand_alpha_1x64 (ms);
3461 __m64 dest = _mm_movepi64_pi64 (xmm_mask);
3462 __m64 alpha_dst = unpack_32_1x64 (d);
3464 *dst++ = pack_1x64_32 (
3465 in_over_1x64 (&ms, &alpha, &dest, &alpha_dst));
3470 /* call prefetch hint to optimize cache load*/
3471 cache_prefetch ((__m128i*)dst);
3472 cache_prefetch ((__m128i*)src);
3476 /* fill cache line with next memory */
3477 cache_prefetch_next ((__m128i*)dst);
3478 cache_prefetch_next ((__m128i*)src);
3480 xmm_src = load_128_unaligned ((__m128i*)src);
3481 xmm_dst = load_128_aligned ((__m128i*)dst);
3483 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3484 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3485 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3486 &xmm_alpha_lo, &xmm_alpha_hi);
3488 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
3489 &xmm_alpha_lo, &xmm_alpha_hi,
3490 &xmm_mask, &xmm_mask,
3491 &xmm_dst_lo, &xmm_dst_hi);
3494 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3503 uint32_t s = *src++;
3506 __m64 ms = unpack_32_1x64 (s);
3507 __m64 alpha = expand_alpha_1x64 (ms);
3508 __m64 mask = _mm_movepi64_pi64 (xmm_mask);
3509 __m64 dest = unpack_32_1x64 (d);
3511 *dst++ = pack_1x64_32 (
3512 in_over_1x64 (&ms, &alpha, &mask, &dest));
3521 /*---------------------------------------------------------------------
3522 * composite_over_8888_n_8888
3526 sse2_composite_src_x888_8888 (pixman_implementation_t *imp,
3528 pixman_image_t * src_image,
3529 pixman_image_t * mask_image,
3530 pixman_image_t * dst_image,
3540 uint32_t *dst_line, *dst;
3541 uint32_t *src_line, *src;
3543 int dst_stride, src_stride;
3546 PIXMAN_IMAGE_GET_LINE (
3547 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3548 PIXMAN_IMAGE_GET_LINE (
3549 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3554 dst_line += dst_stride;
3556 src_line += src_stride;
3559 /* call prefetch hint to optimize cache load*/
3560 cache_prefetch ((__m128i*)src);
3562 while (w && (unsigned long)dst & 15)
3564 *dst++ = *src++ | 0xff000000;
3568 /* call prefetch hint to optimize cache load*/
3569 cache_prefetch ((__m128i*)src);
3573 __m128i xmm_src1, xmm_src2, xmm_src3, xmm_src4;
3575 /* fill cache line with next memory */
3576 cache_prefetch_next ((__m128i*)src);
3578 xmm_src1 = load_128_unaligned ((__m128i*)src + 0);
3579 xmm_src2 = load_128_unaligned ((__m128i*)src + 1);
3580 xmm_src3 = load_128_unaligned ((__m128i*)src + 2);
3581 xmm_src4 = load_128_unaligned ((__m128i*)src + 3);
3583 save_128_aligned ((__m128i*)dst + 0, _mm_or_si128 (xmm_src1, mask_ff000000));
3584 save_128_aligned ((__m128i*)dst + 1, _mm_or_si128 (xmm_src2, mask_ff000000));
3585 save_128_aligned ((__m128i*)dst + 2, _mm_or_si128 (xmm_src3, mask_ff000000));
3586 save_128_aligned ((__m128i*)dst + 3, _mm_or_si128 (xmm_src4, mask_ff000000));
3595 *dst++ = *src++ | 0xff000000;
3603 /* ---------------------------------------------------------------------
3604 * composite_over_x888_n_8888
3607 sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
3609 pixman_image_t * src_image,
3610 pixman_image_t * mask_image,
3611 pixman_image_t * dst_image,
3621 uint32_t *dst_line, *dst;
3622 uint32_t *src_line, *src;
3624 int dst_stride, src_stride;
3627 __m128i xmm_mask, xmm_alpha;
3628 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3629 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3631 PIXMAN_IMAGE_GET_LINE (
3632 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3633 PIXMAN_IMAGE_GET_LINE (
3634 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3636 mask = _pixman_image_get_solid (mask_image, PIXMAN_a8r8g8b8);
3638 xmm_mask = create_mask_16_128 (mask >> 24);
3639 xmm_alpha = mask_00ff;
3644 dst_line += dst_stride;
3646 src_line += src_stride;
3649 /* call prefetch hint to optimize cache load*/
3650 cache_prefetch ((__m128i*)dst);
3651 cache_prefetch ((__m128i*)src);
3653 while (w && (unsigned long)dst & 15)
3655 uint32_t s = (*src++) | 0xff000000;
3658 __m64 src = unpack_32_1x64 (s);
3659 __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
3660 __m64 mask = _mm_movepi64_pi64 (xmm_mask);
3661 __m64 dest = unpack_32_1x64 (d);
3663 *dst++ = pack_1x64_32 (
3664 in_over_1x64 (&src, &alpha, &mask, &dest));
3669 /* call prefetch hint to optimize cache load*/
3670 cache_prefetch ((__m128i*)dst);
3671 cache_prefetch ((__m128i*)src);
3675 /* fill cache line with next memory */
3676 cache_prefetch_next ((__m128i*)dst);
3677 cache_prefetch_next ((__m128i*)src);
3679 xmm_src = _mm_or_si128 (
3680 load_128_unaligned ((__m128i*)src), mask_ff000000);
3681 xmm_dst = load_128_aligned ((__m128i*)dst);
3683 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3684 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3686 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
3687 &xmm_alpha, &xmm_alpha,
3688 &xmm_mask, &xmm_mask,
3689 &xmm_dst_lo, &xmm_dst_hi);
3692 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3702 uint32_t s = (*src++) | 0xff000000;
3705 __m64 src = unpack_32_1x64 (s);
3706 __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
3707 __m64 mask = _mm_movepi64_pi64 (xmm_mask);
3708 __m64 dest = unpack_32_1x64 (d);
3710 *dst++ = pack_1x64_32 (
3711 in_over_1x64 (&src, &alpha, &mask, &dest));
3720 /* --------------------------------------------------------------------
3721 * composite_over_8888_8888
3724 sse2_composite_over_8888_8888 (pixman_implementation_t *imp,
3726 pixman_image_t * src_image,
3727 pixman_image_t * mask_image,
3728 pixman_image_t * dst_image,
3738 int dst_stride, src_stride;
3739 uint32_t *dst_line, *dst;
3740 uint32_t *src_line, *src;
3742 PIXMAN_IMAGE_GET_LINE (
3743 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3744 PIXMAN_IMAGE_GET_LINE (
3745 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3752 core_combine_over_u_sse2 (dst, src, NULL, width);
3760 /* ------------------------------------------------------------------
3761 * composite_over_8888_0565
3763 static force_inline uint16_t
3764 composite_over_8888_0565pixel (uint32_t src, uint16_t dst)
3768 ms = unpack_32_1x64 (src);
3769 return pack_565_32_16 (
3772 ms, expand_alpha_1x64 (ms), expand565_16_1x64 (dst))));
3776 sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
3778 pixman_image_t * src_image,
3779 pixman_image_t * mask_image,
3780 pixman_image_t * dst_image,
3790 uint16_t *dst_line, *dst, d;
3791 uint32_t *src_line, *src, s;
3792 int dst_stride, src_stride;
3795 __m128i xmm_alpha_lo, xmm_alpha_hi;
3796 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3797 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
3799 PIXMAN_IMAGE_GET_LINE (
3800 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
3801 PIXMAN_IMAGE_GET_LINE (
3802 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3807 * I copy the code from MMX one and keep the fixme.
3808 * If it's a problem there, probably is a problem here.
3810 assert (src_image->drawable == mask_image->drawable);
3818 /* call prefetch hint to optimize cache load*/
3819 cache_prefetch ((__m128i*)src);
3820 cache_prefetch ((__m128i*)dst);
3822 dst_line += dst_stride;
3823 src_line += src_stride;
3826 /* Align dst on a 16-byte boundary */
3828 ((unsigned long)dst & 15))
3833 *dst++ = composite_over_8888_0565pixel (s, d);
3837 /* call prefetch hint to optimize cache load*/
3838 cache_prefetch ((__m128i*)src);
3839 cache_prefetch ((__m128i*)dst);
3841 /* It's a 8 pixel loop */
3844 /* fill cache line with next memory */
3845 cache_prefetch_next ((__m128i*)src);
3846 cache_prefetch_next ((__m128i*)dst);
3848 /* I'm loading unaligned because I'm not sure
3849 * about the address alignment.
3851 xmm_src = load_128_unaligned ((__m128i*) src);
3852 xmm_dst = load_128_aligned ((__m128i*) dst);
3855 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3856 unpack_565_128_4x128 (xmm_dst,
3857 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3858 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3859 &xmm_alpha_lo, &xmm_alpha_hi);
3861 /* I'm loading next 4 pixels from memory
3862 * before to optimze the memory read.
3864 xmm_src = load_128_unaligned ((__m128i*) (src + 4));
3866 over_2x128 (&xmm_src_lo, &xmm_src_hi,
3867 &xmm_alpha_lo, &xmm_alpha_hi,
3868 &xmm_dst0, &xmm_dst1);
3871 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3872 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3873 &xmm_alpha_lo, &xmm_alpha_hi);
3875 over_2x128 (&xmm_src_lo, &xmm_src_hi,
3876 &xmm_alpha_lo, &xmm_alpha_hi,
3877 &xmm_dst2, &xmm_dst3);
3880 (__m128i*)dst, pack_565_4x128_128 (
3881 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
3893 *dst++ = composite_over_8888_0565pixel (s, d);
3900 /* -----------------------------------------------------------------
3901 * composite_over_n_8_8888
3905 sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
3907 pixman_image_t * src_image,
3908 pixman_image_t * mask_image,
3909 pixman_image_t * dst_image,
3920 uint32_t *dst_line, *dst;
3921 uint8_t *mask_line, *mask;
3922 int dst_stride, mask_stride;
3926 __m128i xmm_src, xmm_alpha, xmm_def;
3927 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3928 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3930 __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3932 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3938 PIXMAN_IMAGE_GET_LINE (
3939 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3940 PIXMAN_IMAGE_GET_LINE (
3941 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
3943 xmm_def = create_mask_2x32_128 (src, src);
3944 xmm_src = expand_pixel_32_1x128 (src);
3945 xmm_alpha = expand_alpha_1x128 (xmm_src);
3946 mmx_src = _mm_movepi64_pi64 (xmm_src);
3947 mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
3952 dst_line += dst_stride;
3954 mask_line += mask_stride;
3957 /* call prefetch hint to optimize cache load*/
3958 cache_prefetch ((__m128i*)mask);
3959 cache_prefetch ((__m128i*)dst);
3961 while (w && (unsigned long)dst & 15)
3963 uint8_t m = *mask++;
3968 mmx_mask = expand_pixel_8_1x64 (m);
3969 mmx_dest = unpack_32_1x64 (d);
3971 *dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
3981 /* call prefetch hint to optimize cache load*/
3982 cache_prefetch ((__m128i*)mask);
3983 cache_prefetch ((__m128i*)dst);
3987 /* fill cache line with next memory */
3988 cache_prefetch_next ((__m128i*)mask);
3989 cache_prefetch_next ((__m128i*)dst);
3991 m = *((uint32_t*)mask);
3993 if (srca == 0xff && m == 0xffffffff)
3995 save_128_aligned ((__m128i*)dst, xmm_def);
3999 xmm_dst = load_128_aligned ((__m128i*) dst);
4000 xmm_mask = unpack_32_1x128 (m);
4001 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
4004 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
4005 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4007 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
4008 &xmm_mask_lo, &xmm_mask_hi);
4010 in_over_2x128 (&xmm_src, &xmm_src,
4011 &xmm_alpha, &xmm_alpha,
4012 &xmm_mask_lo, &xmm_mask_hi,
4013 &xmm_dst_lo, &xmm_dst_hi);
4016 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4026 uint8_t m = *mask++;
4031 mmx_mask = expand_pixel_8_1x64 (m);
4032 mmx_dest = unpack_32_1x64 (d);
4034 *dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
4048 /* ----------------------------------------------------------------
4049 * composite_over_n_8_8888
4053 pixman_fill_sse2 (uint32_t *bits,
4062 uint32_t byte_width;
4072 stride = stride * (int) sizeof (uint32_t) / 1;
4073 byte_line = (uint8_t *)(((uint8_t *)bits) + stride * y + x);
4079 data = (w << 16) | w;
4083 stride = stride * (int) sizeof (uint32_t) / 2;
4084 byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
4085 byte_width = 2 * width;
4088 data = (data & 0xffff) * 0x00010001;
4092 stride = stride * (int) sizeof (uint32_t) / 4;
4093 byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
4094 byte_width = 4 * width;
4102 cache_prefetch ((__m128i*)byte_line);
4103 xmm_def = create_mask_2x32_128 (data, data);
4108 uint8_t *d = byte_line;
4109 byte_line += stride;
4112 cache_prefetch_next ((__m128i*)d);
4114 while (w >= 1 && ((unsigned long)d & 1))
4116 *(uint8_t *)d = data;
4121 while (w >= 2 && ((unsigned long)d & 3))
4123 *(uint16_t *)d = data;
4128 while (w >= 4 && ((unsigned long)d & 15))
4130 *(uint32_t *)d = data;
4136 cache_prefetch_next ((__m128i*)d);
4140 cache_prefetch (((__m128i*)d) + 12);
4142 save_128_aligned ((__m128i*)(d), xmm_def);
4143 save_128_aligned ((__m128i*)(d + 16), xmm_def);
4144 save_128_aligned ((__m128i*)(d + 32), xmm_def);
4145 save_128_aligned ((__m128i*)(d + 48), xmm_def);
4146 save_128_aligned ((__m128i*)(d + 64), xmm_def);
4147 save_128_aligned ((__m128i*)(d + 80), xmm_def);
4148 save_128_aligned ((__m128i*)(d + 96), xmm_def);
4149 save_128_aligned ((__m128i*)(d + 112), xmm_def);
4157 cache_prefetch (((__m128i*)d) + 8);
4159 save_128_aligned ((__m128i*)(d), xmm_def);
4160 save_128_aligned ((__m128i*)(d + 16), xmm_def);
4161 save_128_aligned ((__m128i*)(d + 32), xmm_def);
4162 save_128_aligned ((__m128i*)(d + 48), xmm_def);
4168 cache_prefetch_next ((__m128i*)d);
4172 save_128_aligned ((__m128i*)(d), xmm_def);
4173 save_128_aligned ((__m128i*)(d + 16), xmm_def);
4181 save_128_aligned ((__m128i*)(d), xmm_def);
4187 cache_prefetch_next ((__m128i*)d);
4191 *(uint32_t *)d = data;
4199 *(uint16_t *)d = data;
4206 *(uint8_t *)d = data;
4217 sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
4219 pixman_image_t * src_image,
4220 pixman_image_t * mask_image,
4221 pixman_image_t * dst_image,
4232 uint32_t *dst_line, *dst;
4233 uint8_t *mask_line, *mask;
4234 int dst_stride, mask_stride;
4238 __m128i xmm_src, xmm_def;
4239 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4241 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4246 pixman_fill_sse2 (dst_image->bits.bits, dst_image->bits.rowstride,
4247 PIXMAN_FORMAT_BPP (dst_image->bits.format),
4248 dest_x, dest_y, width, height, 0);
4252 PIXMAN_IMAGE_GET_LINE (
4253 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
4254 PIXMAN_IMAGE_GET_LINE (
4255 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4257 xmm_def = create_mask_2x32_128 (src, src);
4258 xmm_src = expand_pixel_32_1x128 (src);
4263 dst_line += dst_stride;
4265 mask_line += mask_stride;
4268 /* call prefetch hint to optimize cache load*/
4269 cache_prefetch ((__m128i*)mask);
4270 cache_prefetch ((__m128i*)dst);
4272 while (w && (unsigned long)dst & 15)
4274 uint8_t m = *mask++;
4278 *dst = pack_1x64_32 (
4280 _mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
4291 /* call prefetch hint to optimize cache load*/
4292 cache_prefetch ((__m128i*)mask);
4293 cache_prefetch ((__m128i*)dst);
4297 /* fill cache line with next memory */
4298 cache_prefetch_next ((__m128i*)mask);
4299 cache_prefetch_next ((__m128i*)dst);
4301 m = *((uint32_t*)mask);
4303 if (srca == 0xff && m == 0xffffffff)
4305 save_128_aligned ((__m128i*)dst, xmm_def);
4309 xmm_mask = unpack_32_1x128 (m);
4310 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
4313 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4315 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
4316 &xmm_mask_lo, &xmm_mask_hi);
4318 pix_multiply_2x128 (&xmm_src, &xmm_src,
4319 &xmm_mask_lo, &xmm_mask_hi,
4320 &xmm_mask_lo, &xmm_mask_hi);
4323 (__m128i*)dst, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
4327 save_128_aligned ((__m128i*)dst, _mm_setzero_si128 ());
4337 uint8_t m = *mask++;
4341 *dst = pack_1x64_32 (
4343 _mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
4358 /*-----------------------------------------------------------------------
4359 * composite_over_n_8_0565
4363 sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
4365 pixman_image_t * src_image,
4366 pixman_image_t * mask_image,
4367 pixman_image_t * dst_image,
4378 uint16_t *dst_line, *dst, d;
4379 uint8_t *mask_line, *mask;
4380 int dst_stride, mask_stride;
4383 __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
4385 __m128i xmm_src, xmm_alpha;
4386 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4387 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
4389 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4395 PIXMAN_IMAGE_GET_LINE (
4396 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
4397 PIXMAN_IMAGE_GET_LINE (
4398 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4400 xmm_src = expand_pixel_32_1x128 (src);
4401 xmm_alpha = expand_alpha_1x128 (xmm_src);
4402 mmx_src = _mm_movepi64_pi64 (xmm_src);
4403 mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
4408 dst_line += dst_stride;
4410 mask_line += mask_stride;
4413 /* call prefetch hint to optimize cache load*/
4414 cache_prefetch ((__m128i*)mask);
4415 cache_prefetch ((__m128i*)dst);
4417 while (w && (unsigned long)dst & 15)
4424 mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
4425 mmx_dest = expand565_16_1x64 (d);
4427 *dst = pack_565_32_16 (
4430 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4437 /* call prefetch hint to optimize cache load*/
4438 cache_prefetch ((__m128i*)mask);
4439 cache_prefetch ((__m128i*)dst);
4443 /* fill cache line with next memory */
4444 cache_prefetch_next ((__m128i*)mask);
4445 cache_prefetch_next ((__m128i*)dst);
4447 xmm_dst = load_128_aligned ((__m128i*) dst);
4448 unpack_565_128_4x128 (xmm_dst,
4449 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
4451 m = *((uint32_t*)mask);
4456 xmm_mask = unpack_32_1x128 (m);
4457 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
4460 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4462 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
4463 &xmm_mask_lo, &xmm_mask_hi);
4465 in_over_2x128 (&xmm_src, &xmm_src,
4466 &xmm_alpha, &xmm_alpha,
4467 &xmm_mask_lo, &xmm_mask_hi,
4468 &xmm_dst0, &xmm_dst1);
4471 m = *((uint32_t*)mask);
4476 xmm_mask = unpack_32_1x128 (m);
4477 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
4480 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4482 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
4483 &xmm_mask_lo, &xmm_mask_hi);
4484 in_over_2x128 (&xmm_src, &xmm_src,
4485 &xmm_alpha, &xmm_alpha,
4486 &xmm_mask_lo, &xmm_mask_hi,
4487 &xmm_dst2, &xmm_dst3);
4491 (__m128i*)dst, pack_565_4x128_128 (
4492 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
4505 mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
4506 mmx_dest = expand565_16_1x64 (d);
4508 *dst = pack_565_32_16 (
4511 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4522 /* -----------------------------------------------------------------------
4523 * composite_over_pixbuf_0565
4527 sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
4529 pixman_image_t * src_image,
4530 pixman_image_t * mask_image,
4531 pixman_image_t * dst_image,
4541 uint16_t *dst_line, *dst, d;
4542 uint32_t *src_line, *src, s;
4543 int dst_stride, src_stride;
4545 uint32_t opaque, zero;
4548 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
4549 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
4551 PIXMAN_IMAGE_GET_LINE (
4552 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
4553 PIXMAN_IMAGE_GET_LINE (
4554 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
4559 * I copy the code from MMX one and keep the fixme.
4560 * If it's a problem there, probably is a problem here.
4562 assert (src_image->drawable == mask_image->drawable);
4568 dst_line += dst_stride;
4570 src_line += src_stride;
4573 /* call prefetch hint to optimize cache load*/
4574 cache_prefetch ((__m128i*)src);
4575 cache_prefetch ((__m128i*)dst);
4577 while (w && (unsigned long)dst & 15)
4582 ms = unpack_32_1x64 (s);
4584 *dst++ = pack_565_32_16 (
4586 over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
4590 /* call prefetch hint to optimize cache load*/
4591 cache_prefetch ((__m128i*)src);
4592 cache_prefetch ((__m128i*)dst);
4596 /* fill cache line with next memory */
4597 cache_prefetch_next ((__m128i*)src);
4598 cache_prefetch_next ((__m128i*)dst);
4601 xmm_src = load_128_unaligned ((__m128i*)src);
4602 xmm_dst = load_128_aligned ((__m128i*)dst);
4604 opaque = is_opaque (xmm_src);
4605 zero = is_zero (xmm_src);
4607 unpack_565_128_4x128 (xmm_dst,
4608 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
4609 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
4611 /* preload next round*/
4612 xmm_src = load_128_unaligned ((__m128i*)(src + 4));
4616 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
4617 &xmm_dst0, &xmm_dst1);
4621 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
4622 &xmm_dst0, &xmm_dst1);
4626 opaque = is_opaque (xmm_src);
4627 zero = is_zero (xmm_src);
4629 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
4633 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
4634 &xmm_dst2, &xmm_dst3);
4638 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
4639 &xmm_dst2, &xmm_dst3);
4643 (__m128i*)dst, pack_565_4x128_128 (
4644 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
4656 ms = unpack_32_1x64 (s);
4658 *dst++ = pack_565_32_16 (
4660 over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
4668 /* -------------------------------------------------------------------------
4669 * composite_over_pixbuf_8888
4673 sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
4675 pixman_image_t * src_image,
4676 pixman_image_t * mask_image,
4677 pixman_image_t * dst_image,
4687 uint32_t *dst_line, *dst, d;
4688 uint32_t *src_line, *src, s;
4689 int dst_stride, src_stride;
4691 uint32_t opaque, zero;
4693 __m128i xmm_src_lo, xmm_src_hi;
4694 __m128i xmm_dst_lo, xmm_dst_hi;
4696 PIXMAN_IMAGE_GET_LINE (
4697 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
4698 PIXMAN_IMAGE_GET_LINE (
4699 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
4704 * I copy the code from MMX one and keep the fixme.
4705 * If it's a problem there, probably is a problem here.
4707 assert (src_image->drawable == mask_image->drawable);
4713 dst_line += dst_stride;
4715 src_line += src_stride;
4718 /* call prefetch hint to optimize cache load*/
4719 cache_prefetch ((__m128i*)src);
4720 cache_prefetch ((__m128i*)dst);
4722 while (w && (unsigned long)dst & 15)
4727 *dst++ = pack_1x64_32 (
4728 over_rev_non_pre_1x64 (
4729 unpack_32_1x64 (s), unpack_32_1x64 (d)));
4734 /* call prefetch hint to optimize cache load*/
4735 cache_prefetch ((__m128i*)src);
4736 cache_prefetch ((__m128i*)dst);
4740 /* fill cache line with next memory */
4741 cache_prefetch_next ((__m128i*)src);
4742 cache_prefetch_next ((__m128i*)dst);
4744 xmm_src_hi = load_128_unaligned ((__m128i*)src);
4746 opaque = is_opaque (xmm_src_hi);
4747 zero = is_zero (xmm_src_hi);
4749 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
4753 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
4754 &xmm_dst_lo, &xmm_dst_hi);
4757 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4761 xmm_dst_hi = load_128_aligned ((__m128i*)dst);
4763 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
4765 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
4766 &xmm_dst_lo, &xmm_dst_hi);
4769 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4782 *dst++ = pack_1x64_32 (
4783 over_rev_non_pre_1x64 (
4784 unpack_32_1x64 (s), unpack_32_1x64 (d)));
4793 /* -------------------------------------------------------------------------------------------------
4794 * composite_over_n_8888_0565_ca
4798 sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
4800 pixman_image_t * src_image,
4801 pixman_image_t * mask_image,
4802 pixman_image_t * dst_image,
4813 uint16_t *dst_line, *dst, d;
4814 uint32_t *mask_line, *mask, m;
4815 int dst_stride, mask_stride;
4819 __m128i xmm_src, xmm_alpha;
4820 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4821 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
4823 __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
4825 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4830 PIXMAN_IMAGE_GET_LINE (
4831 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
4832 PIXMAN_IMAGE_GET_LINE (
4833 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
4835 xmm_src = expand_pixel_32_1x128 (src);
4836 xmm_alpha = expand_alpha_1x128 (xmm_src);
4837 mmx_src = _mm_movepi64_pi64 (xmm_src);
4838 mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
4845 mask_line += mask_stride;
4846 dst_line += dst_stride;
4848 /* call prefetch hint to optimize cache load*/
4849 cache_prefetch ((__m128i*)mask);
4850 cache_prefetch ((__m128i*)dst);
4852 while (w && ((unsigned long)dst & 15))
4854 m = *(uint32_t *) mask;
4859 mmx_mask = unpack_32_1x64 (m);
4860 mmx_dest = expand565_16_1x64 (d);
4862 *dst = pack_565_32_16 (
4865 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4873 /* call prefetch hint to optimize cache load*/
4874 cache_prefetch ((__m128i*)mask);
4875 cache_prefetch ((__m128i*)dst);
4879 /* fill cache line with next memory */
4880 cache_prefetch_next ((__m128i*)mask);
4881 cache_prefetch_next ((__m128i*)dst);
4884 xmm_mask = load_128_unaligned ((__m128i*)mask);
4885 xmm_dst = load_128_aligned ((__m128i*)dst);
4887 pack_cmp = _mm_movemask_epi8 (
4888 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
4890 unpack_565_128_4x128 (xmm_dst,
4891 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
4892 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4894 /* preload next round */
4895 xmm_mask = load_128_unaligned ((__m128i*)(mask + 4));
4897 /* preload next round */
4898 if (pack_cmp != 0xffff)
4900 in_over_2x128 (&xmm_src, &xmm_src,
4901 &xmm_alpha, &xmm_alpha,
4902 &xmm_mask_lo, &xmm_mask_hi,
4903 &xmm_dst0, &xmm_dst1);
4907 pack_cmp = _mm_movemask_epi8 (
4908 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
4910 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4912 if (pack_cmp != 0xffff)
4914 in_over_2x128 (&xmm_src, &xmm_src,
4915 &xmm_alpha, &xmm_alpha,
4916 &xmm_mask_lo, &xmm_mask_hi,
4917 &xmm_dst2, &xmm_dst3);
4921 (__m128i*)dst, pack_565_4x128_128 (
4922 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
4931 m = *(uint32_t *) mask;
4936 mmx_mask = unpack_32_1x64 (m);
4937 mmx_dest = expand565_16_1x64 (d);
4939 *dst = pack_565_32_16 (
4942 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4954 /* -----------------------------------------------------------------------
4955 * composite_in_n_8_8
4959 sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
4961 pixman_image_t * src_image,
4962 pixman_image_t * mask_image,
4963 pixman_image_t * dst_image,
4973 uint8_t *dst_line, *dst;
4974 uint8_t *mask_line, *mask;
4975 int dst_stride, mask_stride;
4982 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4983 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4985 PIXMAN_IMAGE_GET_LINE (
4986 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4987 PIXMAN_IMAGE_GET_LINE (
4988 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4990 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4994 xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
4999 dst_line += dst_stride;
5001 mask_line += mask_stride;
5004 /* call prefetch hint to optimize cache load*/
5005 cache_prefetch ((__m128i*)mask);
5006 cache_prefetch ((__m128i*)dst);
5008 while (w && ((unsigned long)dst & 15))
5010 m = (uint32_t) *mask++;
5011 d = (uint32_t) *dst;
5013 *dst++ = (uint8_t) pack_1x64_32 (
5015 pix_multiply_1x64 (_mm_movepi64_pi64 (xmm_alpha),
5016 unpack_32_1x64 (m)),
5017 unpack_32_1x64 (d)));
5021 /* call prefetch hint to optimize cache load*/
5022 cache_prefetch ((__m128i*)mask);
5023 cache_prefetch ((__m128i*)dst);
5027 /* fill cache line with next memory */
5028 cache_prefetch_next ((__m128i*)mask);
5029 cache_prefetch_next ((__m128i*)dst);
5031 xmm_mask = load_128_unaligned ((__m128i*)mask);
5032 xmm_dst = load_128_aligned ((__m128i*)dst);
5034 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
5035 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5037 pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
5038 &xmm_mask_lo, &xmm_mask_hi,
5039 &xmm_mask_lo, &xmm_mask_hi);
5041 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
5042 &xmm_dst_lo, &xmm_dst_hi,
5043 &xmm_dst_lo, &xmm_dst_hi);
5046 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5055 m = (uint32_t) *mask++;
5056 d = (uint32_t) *dst;
5058 *dst++ = (uint8_t) pack_1x64_32 (
5061 _mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
5062 unpack_32_1x64 (d)));
5070 /* -----------------------------------------------------------------------
5075 sse2_composite_in_n_8 (pixman_implementation_t *imp,
5077 pixman_image_t * src_image,
5078 pixman_image_t * mask_image,
5079 pixman_image_t * dst_image,
5089 uint8_t *dst_line, *dst;
5096 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5098 PIXMAN_IMAGE_GET_LINE (
5099 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
5101 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
5103 xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
5112 pixman_fill (dst_image->bits.bits, dst_image->bits.rowstride,
5113 8, dest_x, dest_y, width, height, src);
5121 dst_line += dst_stride;
5124 /* call prefetch hint to optimize cache load*/
5125 cache_prefetch ((__m128i*)dst);
5127 while (w && ((unsigned long)dst & 15))
5129 d = (uint32_t) *dst;
5131 *dst++ = (uint8_t) pack_1x64_32 (
5133 _mm_movepi64_pi64 (xmm_alpha),
5134 unpack_32_1x64 (d)));
5138 /* call prefetch hint to optimize cache load*/
5139 cache_prefetch ((__m128i*)dst);
5143 /* fill cache line with next memory */
5144 cache_prefetch_next ((__m128i*)dst);
5146 xmm_dst = load_128_aligned ((__m128i*)dst);
5148 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5150 pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
5151 &xmm_dst_lo, &xmm_dst_hi,
5152 &xmm_dst_lo, &xmm_dst_hi);
5155 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5163 d = (uint32_t) *dst;
5165 *dst++ = (uint8_t) pack_1x64_32 (
5167 _mm_movepi64_pi64 (xmm_alpha),
5168 unpack_32_1x64 (d)));
5176 /* ---------------------------------------------------------------------------
5181 sse2_composite_in_8_8 (pixman_implementation_t *imp,
5183 pixman_image_t * src_image,
5184 pixman_image_t * mask_image,
5185 pixman_image_t * dst_image,
5195 uint8_t *dst_line, *dst;
5196 uint8_t *src_line, *src;
5197 int src_stride, dst_stride;
5201 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
5202 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5204 PIXMAN_IMAGE_GET_LINE (
5205 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
5206 PIXMAN_IMAGE_GET_LINE (
5207 src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
5212 dst_line += dst_stride;
5214 src_line += src_stride;
5217 /* call prefetch hint to optimize cache load*/
5218 cache_prefetch ((__m128i*)src);
5219 cache_prefetch ((__m128i*)dst);
5221 while (w && ((unsigned long)dst & 15))
5223 s = (uint32_t) *src++;
5224 d = (uint32_t) *dst;
5226 *dst++ = (uint8_t) pack_1x64_32 (
5228 unpack_32_1x64 (s), unpack_32_1x64 (d)));
5232 /* call prefetch hint to optimize cache load*/
5233 cache_prefetch ((__m128i*)src);
5234 cache_prefetch ((__m128i*)dst);
5238 /* fill cache line with next memory */
5239 cache_prefetch_next ((__m128i*)src);
5240 cache_prefetch_next ((__m128i*)dst);
5242 xmm_src = load_128_unaligned ((__m128i*)src);
5243 xmm_dst = load_128_aligned ((__m128i*)dst);
5245 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
5246 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5248 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
5249 &xmm_dst_lo, &xmm_dst_hi,
5250 &xmm_dst_lo, &xmm_dst_hi);
5253 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5262 s = (uint32_t) *src++;
5263 d = (uint32_t) *dst;
5265 *dst++ = (uint8_t) pack_1x64_32 (
5266 pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (d)));
5274 /* -------------------------------------------------------------------------
5275 * composite_add_n_8_8
5279 sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
5281 pixman_image_t * src_image,
5282 pixman_image_t * mask_image,
5283 pixman_image_t * dst_image,
5293 uint8_t *dst_line, *dst;
5294 uint8_t *mask_line, *mask;
5295 int dst_stride, mask_stride;
5302 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
5303 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5305 PIXMAN_IMAGE_GET_LINE (
5306 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
5307 PIXMAN_IMAGE_GET_LINE (
5308 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
5310 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
5314 xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
5319 dst_line += dst_stride;
5321 mask_line += mask_stride;
5324 /* call prefetch hint to optimize cache load*/
5325 cache_prefetch ((__m128i*)mask);
5326 cache_prefetch ((__m128i*)dst);
5328 while (w && ((unsigned long)dst & 15))
5330 m = (uint32_t) *mask++;
5331 d = (uint32_t) *dst;
5333 *dst++ = (uint8_t) pack_1x64_32 (
5336 _mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
5337 unpack_32_1x64 (d)));
5341 /* call prefetch hint to optimize cache load*/
5342 cache_prefetch ((__m128i*)mask);
5343 cache_prefetch ((__m128i*)dst);
5347 /* fill cache line with next memory */
5348 cache_prefetch_next ((__m128i*)mask);
5349 cache_prefetch_next ((__m128i*)dst);
5351 xmm_mask = load_128_unaligned ((__m128i*)mask);
5352 xmm_dst = load_128_aligned ((__m128i*)dst);
5354 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
5355 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5357 pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
5358 &xmm_mask_lo, &xmm_mask_hi,
5359 &xmm_mask_lo, &xmm_mask_hi);
5361 xmm_dst_lo = _mm_adds_epu16 (xmm_mask_lo, xmm_dst_lo);
5362 xmm_dst_hi = _mm_adds_epu16 (xmm_mask_hi, xmm_dst_hi);
5365 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5374 m = (uint32_t) *mask++;
5375 d = (uint32_t) *dst;
5377 *dst++ = (uint8_t) pack_1x64_32 (
5380 _mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
5381 unpack_32_1x64 (d)));
5390 /* -------------------------------------------------------------------------
5391 * composite_add_n_8_8
5395 sse2_composite_add_n_8 (pixman_implementation_t *imp,
5397 pixman_image_t * src_image,
5398 pixman_image_t * mask_image,
5399 pixman_image_t * dst_image,
5409 uint8_t *dst_line, *dst;
5416 PIXMAN_IMAGE_GET_LINE (
5417 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
5419 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
5428 pixman_fill (dst_image->bits.bits, dst_image->bits.rowstride,
5429 8, dest_x, dest_y, width, height, 0xff);
5434 src = (src << 24) | (src << 16) | (src << 8) | src;
5435 xmm_src = _mm_set_epi32 (src, src, src, src);
5440 dst_line += dst_stride;
5443 /* call prefetch hint to optimize cache load*/
5444 cache_prefetch ((__m128i*)dst);
5446 while (w && ((unsigned long)dst & 15))
5448 *dst = (uint8_t)_mm_cvtsi64_si32 (
5450 _mm_movepi64_pi64 (xmm_src),
5451 _mm_cvtsi32_si64 (*dst)));
5457 /* call prefetch hint to optimize cache load*/
5458 cache_prefetch ((__m128i*)dst);
5462 /* fill cache line with next memory */
5463 cache_prefetch_next ((__m128i*)dst);
5466 (__m128i*)dst, _mm_adds_epu8 (xmm_src, load_128_aligned ((__m128i*)dst)));
5474 *dst = (uint8_t)_mm_cvtsi64_si32 (
5476 _mm_movepi64_pi64 (xmm_src),
5477 _mm_cvtsi32_si64 (*dst)));
5487 /* ----------------------------------------------------------------------
5488 * composite_add_8000_8000
5492 sse2_composite_add_8000_8000 (pixman_implementation_t *imp,
5494 pixman_image_t * src_image,
5495 pixman_image_t * mask_image,
5496 pixman_image_t * dst_image,
5506 uint8_t *dst_line, *dst;
5507 uint8_t *src_line, *src;
5508 int dst_stride, src_stride;
5512 PIXMAN_IMAGE_GET_LINE (
5513 src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
5514 PIXMAN_IMAGE_GET_LINE (
5515 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
5522 /* call prefetch hint to optimize cache load*/
5523 cache_prefetch ((__m128i*)src);
5524 cache_prefetch ((__m128i*)dst);
5526 dst_line += dst_stride;
5527 src_line += src_stride;
5531 while (w && (unsigned long)dst & 3)
5533 t = (*dst) + (*src++);
5534 *dst++ = t | (0 - (t >> 8));
5538 core_combine_add_u_sse2 ((uint32_t*)dst, (uint32_t*)src, NULL, w >> 2);
5548 t = (*dst) + (*src++);
5549 *dst++ = t | (0 - (t >> 8));
5557 /* ---------------------------------------------------------------------
5558 * composite_add_8888_8888
5561 sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
5563 pixman_image_t * src_image,
5564 pixman_image_t * mask_image,
5565 pixman_image_t * dst_image,
5575 uint32_t *dst_line, *dst;
5576 uint32_t *src_line, *src;
5577 int dst_stride, src_stride;
5579 PIXMAN_IMAGE_GET_LINE (
5580 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
5581 PIXMAN_IMAGE_GET_LINE (
5582 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
5587 dst_line += dst_stride;
5589 src_line += src_stride;
5591 core_combine_add_u_sse2 (dst, src, NULL, width);
5597 /* -------------------------------------------------------------------------------------------------
5598 * sse2_composite_copy_area
5601 static pixman_bool_t
5602 pixman_blt_sse2 (uint32_t *src_bits,
5615 uint8_t * src_bytes;
5616 uint8_t * dst_bytes;
5619 if (src_bpp != dst_bpp)
5624 src_stride = src_stride * (int) sizeof (uint32_t) / 2;
5625 dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
5626 src_bytes =(uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
5627 dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
5628 byte_width = 2 * width;
5632 else if (src_bpp == 32)
5634 src_stride = src_stride * (int) sizeof (uint32_t) / 4;
5635 dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
5636 src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
5637 dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
5638 byte_width = 4 * width;
5647 cache_prefetch ((__m128i*)src_bytes);
5648 cache_prefetch ((__m128i*)dst_bytes);
5653 uint8_t *s = src_bytes;
5654 uint8_t *d = dst_bytes;
5655 src_bytes += src_stride;
5656 dst_bytes += dst_stride;
5659 cache_prefetch_next ((__m128i*)s);
5660 cache_prefetch_next ((__m128i*)d);
5662 while (w >= 2 && ((unsigned long)d & 3))
5664 *(uint16_t *)d = *(uint16_t *)s;
5670 while (w >= 4 && ((unsigned long)d & 15))
5672 *(uint32_t *)d = *(uint32_t *)s;
5679 cache_prefetch_next ((__m128i*)s);
5680 cache_prefetch_next ((__m128i*)d);
5684 __m128i xmm0, xmm1, xmm2, xmm3;
5686 /* 128 bytes ahead */
5687 cache_prefetch (((__m128i*)s) + 8);
5688 cache_prefetch (((__m128i*)d) + 8);
5690 xmm0 = load_128_unaligned ((__m128i*)(s));
5691 xmm1 = load_128_unaligned ((__m128i*)(s + 16));
5692 xmm2 = load_128_unaligned ((__m128i*)(s + 32));
5693 xmm3 = load_128_unaligned ((__m128i*)(s + 48));
5695 save_128_aligned ((__m128i*)(d), xmm0);
5696 save_128_aligned ((__m128i*)(d + 16), xmm1);
5697 save_128_aligned ((__m128i*)(d + 32), xmm2);
5698 save_128_aligned ((__m128i*)(d + 48), xmm3);
5705 cache_prefetch_next ((__m128i*)s);
5706 cache_prefetch_next ((__m128i*)d);
5710 save_128_aligned ((__m128i*)d, load_128_unaligned ((__m128i*)s) );
5717 cache_prefetch_next ((__m128i*)s);
5718 cache_prefetch_next ((__m128i*)d);
5722 *(uint32_t *)d = *(uint32_t *)s;
5731 *(uint16_t *)d = *(uint16_t *)s;
5744 sse2_composite_copy_area (pixman_implementation_t *imp,
5746 pixman_image_t * src_image,
5747 pixman_image_t * mask_image,
5748 pixman_image_t * dst_image,
5758 pixman_blt_sse2 (src_image->bits.bits,
5759 dst_image->bits.bits,
5760 src_image->bits.rowstride,
5761 dst_image->bits.rowstride,
5762 PIXMAN_FORMAT_BPP (src_image->bits.format),
5763 PIXMAN_FORMAT_BPP (dst_image->bits.format),
5764 src_x, src_y, dest_x, dest_y, width, height);
5768 sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
5770 pixman_image_t * src_image,
5771 pixman_image_t * mask_image,
5772 pixman_image_t * dst_image,
5782 uint32_t *src, *src_line, s;
5783 uint32_t *dst, *dst_line, d;
5784 uint8_t *mask, *mask_line;
5786 int src_stride, mask_stride, dst_stride;
5790 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
5791 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5792 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
5794 PIXMAN_IMAGE_GET_LINE (
5795 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
5796 PIXMAN_IMAGE_GET_LINE (
5797 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
5798 PIXMAN_IMAGE_GET_LINE (
5799 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
5804 src_line += src_stride;
5806 dst_line += dst_stride;
5808 mask_line += mask_stride;
5812 /* call prefetch hint to optimize cache load*/
5813 cache_prefetch ((__m128i*)src);
5814 cache_prefetch ((__m128i*)dst);
5815 cache_prefetch ((__m128i*)mask);
5817 while (w && (unsigned long)dst & 15)
5819 s = 0xff000000 | *src++;
5820 m = (uint32_t) *mask++;
5822 ms = unpack_32_1x64 (s);
5826 __m64 ma = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
5827 __m64 md = unpack_32_1x64 (d);
5829 ms = in_over_1x64 (&ms, &mask_x00ff, &ma, &md);
5832 *dst++ = pack_1x64_32 (ms);
5836 /* call prefetch hint to optimize cache load*/
5837 cache_prefetch ((__m128i*)src);
5838 cache_prefetch ((__m128i*)dst);
5839 cache_prefetch ((__m128i*)mask);
5843 /* fill cache line with next memory */
5844 cache_prefetch_next ((__m128i*)src);
5845 cache_prefetch_next ((__m128i*)dst);
5846 cache_prefetch_next ((__m128i*)mask);
5848 m = *(uint32_t*) mask;
5849 xmm_src = _mm_or_si128 (load_128_unaligned ((__m128i*)src), mask_ff000000);
5851 if (m == 0xffffffff)
5853 save_128_aligned ((__m128i*)dst, xmm_src);
5857 xmm_dst = load_128_aligned ((__m128i*)dst);
5859 xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
5861 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
5862 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
5863 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5865 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
5867 in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &mask_00ff, &mask_00ff, &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
5869 save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5880 m = (uint32_t) *mask++;
5884 s = 0xff000000 | *src;
5896 ma = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
5897 md = unpack_32_1x64 (d);
5898 ms = unpack_32_1x64 (s);
5900 *dst = pack_1x64_32 (in_over_1x64 (&ms, &mask_x00ff, &ma, &md));
5915 sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
5917 pixman_image_t * src_image,
5918 pixman_image_t * mask_image,
5919 pixman_image_t * dst_image,
5929 uint32_t *src, *src_line, s;
5930 uint32_t *dst, *dst_line, d;
5931 uint8_t *mask, *mask_line;
5933 int src_stride, mask_stride, dst_stride;
5936 __m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
5937 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5938 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
5940 PIXMAN_IMAGE_GET_LINE (
5941 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
5942 PIXMAN_IMAGE_GET_LINE (
5943 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
5944 PIXMAN_IMAGE_GET_LINE (
5945 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
5950 src_line += src_stride;
5952 dst_line += dst_stride;
5954 mask_line += mask_stride;
5958 /* call prefetch hint to optimize cache load*/
5959 cache_prefetch ((__m128i *)src);
5960 cache_prefetch ((__m128i *)dst);
5961 cache_prefetch ((__m128i *)mask);
5963 while (w && (unsigned long)dst & 15)
5968 m = (uint32_t) *mask++;
5975 if (sa == 0xff && m == 0xff)
5981 __m64 ms, md, ma, msa;
5983 ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
5984 ms = unpack_32_1x64 (s);
5985 md = unpack_32_1x64 (d);
5987 msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
5989 *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
5997 /* call prefetch hint to optimize cache load*/
5998 cache_prefetch ((__m128i *)src);
5999 cache_prefetch ((__m128i *)dst);
6000 cache_prefetch ((__m128i *)mask);
6004 /* fill cache line with next memory */
6005 cache_prefetch_next ((__m128i *)src);
6006 cache_prefetch_next ((__m128i *)dst);
6007 cache_prefetch_next ((__m128i *)mask);
6009 m = *(uint32_t *) mask;
6013 xmm_src = load_128_unaligned ((__m128i*)src);
6015 if (m == 0xffffffff && is_opaque (xmm_src))
6017 save_128_aligned ((__m128i *)dst, xmm_src);
6021 xmm_dst = load_128_aligned ((__m128i *)dst);
6023 xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
6025 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
6026 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
6027 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
6029 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
6030 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
6032 in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
6033 &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
6035 save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
6050 m = (uint32_t) *mask++;
6057 if (sa == 0xff && m == 0xff)
6063 __m64 ms, md, ma, msa;
6065 ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
6066 ms = unpack_32_1x64 (s);
6067 md = unpack_32_1x64 (d);
6069 msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
6071 *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
6084 sse2_composite_over_reverse_n_8888 (pixman_implementation_t *imp,
6086 pixman_image_t * src_image,
6087 pixman_image_t * mask_image,
6088 pixman_image_t * dst_image,
6099 uint32_t *dst_line, *dst;
6101 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
6102 __m128i xmm_dsta_hi, xmm_dsta_lo;
6106 src = _pixman_image_get_solid (src_image, dst_image->bits.format);
6111 PIXMAN_IMAGE_GET_LINE (
6112 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
6114 xmm_src = expand_pixel_32_1x128 (src);
6120 /* call prefetch hint to optimize cache load*/
6121 cache_prefetch ((__m128i*)dst);
6123 dst_line += dst_stride;
6126 while (w && (unsigned long)dst & 15)
6130 vd = unpack_32_1x64 (*dst);
6132 *dst = pack_1x64_32 (over_1x64 (vd, expand_alpha_1x64 (vd),
6133 _mm_movepi64_pi64 (xmm_src)));
6138 cache_prefetch ((__m128i*)dst);
6142 __m128i tmp_lo, tmp_hi;
6144 /* fill cache line with next memory */
6145 cache_prefetch_next ((__m128i*)(dst + 4));
6147 xmm_dst = load_128_aligned ((__m128i*)dst);
6149 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
6150 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dsta_lo, &xmm_dsta_hi);
6155 over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
6156 &xmm_dsta_lo, &xmm_dsta_hi,
6160 (__m128i*)dst, pack_2x128_128 (tmp_lo, tmp_hi));
6170 vd = unpack_32_1x64 (*dst);
6172 *dst = pack_1x64_32 (over_1x64 (vd, expand_alpha_1x64 (vd),
6173 _mm_movepi64_pi64 (xmm_src)));
6184 sse2_composite_over_8888_8888_8888 (pixman_implementation_t *imp,
6186 pixman_image_t * src_image,
6187 pixman_image_t * mask_image,
6188 pixman_image_t * dst_image,
6198 uint32_t *src, *src_line, s;
6199 uint32_t *dst, *dst_line, d;
6200 uint32_t *mask, *mask_line;
6202 int src_stride, mask_stride, dst_stride;
6205 __m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
6206 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
6207 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
6209 PIXMAN_IMAGE_GET_LINE (
6210 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
6211 PIXMAN_IMAGE_GET_LINE (
6212 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
6213 PIXMAN_IMAGE_GET_LINE (
6214 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
6219 src_line += src_stride;
6221 dst_line += dst_stride;
6223 mask_line += mask_stride;
6227 /* call prefetch hint to optimize cache load*/
6228 cache_prefetch ((__m128i *)src);
6229 cache_prefetch ((__m128i *)dst);
6230 cache_prefetch ((__m128i *)mask);
6232 while (w && (unsigned long)dst & 15)
6237 m = (*mask++) >> 24;
6244 if (sa == 0xff && m == 0xff)
6250 __m64 ms, md, ma, msa;
6252 ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
6253 ms = unpack_32_1x64 (s);
6254 md = unpack_32_1x64 (d);
6256 msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
6258 *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
6266 /* call prefetch hint to optimize cache load*/
6267 cache_prefetch ((__m128i *)src);
6268 cache_prefetch ((__m128i *)dst);
6269 cache_prefetch ((__m128i *)mask);
6273 /* fill cache line with next memory */
6274 cache_prefetch_next ((__m128i *)src);
6275 cache_prefetch_next ((__m128i *)dst);
6276 cache_prefetch_next ((__m128i *)mask);
6278 xmm_mask = load_128_unaligned ((__m128i*)mask);
6280 if (!is_transparent (xmm_mask))
6282 xmm_src = load_128_unaligned ((__m128i*)src);
6284 if (is_opaque (xmm_mask) && is_opaque (xmm_src))
6286 save_128_aligned ((__m128i *)dst, xmm_src);
6290 xmm_dst = load_128_aligned ((__m128i *)dst);
6292 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
6293 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
6294 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
6296 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
6297 expand_alpha_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
6299 in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
6300 &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
6302 save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
6317 m = (*mask++) >> 24;
6324 if (sa == 0xff && m == 0xff)
6330 __m64 ms, md, ma, msa;
6332 ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
6333 ms = unpack_32_1x64 (s);
6334 md = unpack_32_1x64 (d);
6336 msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
6338 *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
6350 /* A variant of 'core_combine_over_u_sse2' with minor tweaks */
6351 static force_inline void
6352 scaled_nearest_scanline_sse2_8888_8888_OVER (uint32_t* pd,
6356 pixman_fixed_t unit_x,
6357 pixman_fixed_t max_vx)
6360 const uint32_t* pm = NULL;
6362 __m128i xmm_dst_lo, xmm_dst_hi;
6363 __m128i xmm_src_lo, xmm_src_hi;
6364 __m128i xmm_alpha_lo, xmm_alpha_hi;
6366 /* Align dst on a 16-byte boundary */
6367 while (w && ((unsigned long)pd & 15))
6370 s = combine1 (ps + (vx >> 16), pm);
6373 *pd++ = core_combine_over_u_pixel_sse2 (s, d);
6382 uint32_t tmp1, tmp2, tmp3, tmp4;
6384 tmp1 = ps[vx >> 16];
6386 tmp2 = ps[vx >> 16];
6388 tmp3 = ps[vx >> 16];
6390 tmp4 = ps[vx >> 16];
6393 tmp = _mm_set_epi32 (tmp4, tmp3, tmp2, tmp1);
6395 xmm_src_hi = combine4 ((__m128i*)&tmp, (__m128i*)pm);
6397 if (is_opaque (xmm_src_hi))
6399 save_128_aligned ((__m128i*)pd, xmm_src_hi);
6401 else if (!is_zero (xmm_src_hi))
6403 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
6405 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
6406 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
6408 expand_alpha_2x128 (
6409 xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
6411 over_2x128 (&xmm_src_lo, &xmm_src_hi,
6412 &xmm_alpha_lo, &xmm_alpha_hi,
6413 &xmm_dst_lo, &xmm_dst_hi);
6415 /* rebuid the 4 pixel data and save*/
6416 save_128_aligned ((__m128i*)pd,
6417 pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
6429 s = combine1 (ps + (vx >> 16), pm);
6432 *pd++ = core_combine_over_u_pixel_sse2 (s, d);
6441 FAST_NEAREST_MAINLOOP (sse2_8888_8888_cover_OVER,
6442 scaled_nearest_scanline_sse2_8888_8888_OVER,
6443 uint32_t, uint32_t, COVER);
6444 FAST_NEAREST_MAINLOOP (sse2_8888_8888_none_OVER,
6445 scaled_nearest_scanline_sse2_8888_8888_OVER,
6446 uint32_t, uint32_t, NONE);
6447 FAST_NEAREST_MAINLOOP (sse2_8888_8888_pad_OVER,
6448 scaled_nearest_scanline_sse2_8888_8888_OVER,
6449 uint32_t, uint32_t, PAD);
6451 static const pixman_fast_path_t sse2_fast_paths[] =
6453 /* PIXMAN_OP_OVER */
6454 PIXMAN_STD_FAST_PATH (OVER, solid, a8, r5g6b5, sse2_composite_over_n_8_0565),
6455 PIXMAN_STD_FAST_PATH (OVER, solid, a8, b5g6r5, sse2_composite_over_n_8_0565),
6456 PIXMAN_STD_FAST_PATH (OVER, solid, null, a8r8g8b8, sse2_composite_over_n_8888),
6457 PIXMAN_STD_FAST_PATH (OVER, solid, null, x8r8g8b8, sse2_composite_over_n_8888),
6458 PIXMAN_STD_FAST_PATH (OVER, solid, null, r5g6b5, sse2_composite_over_n_0565),
6459 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, sse2_composite_over_8888_8888),
6460 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, sse2_composite_over_8888_8888),
6461 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, sse2_composite_over_8888_8888),
6462 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, x8b8g8r8, sse2_composite_over_8888_8888),
6463 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, r5g6b5, sse2_composite_over_8888_0565),
6464 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, b5g6r5, sse2_composite_over_8888_0565),
6465 PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8r8g8b8, sse2_composite_over_n_8_8888),
6466 PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, sse2_composite_over_n_8_8888),
6467 PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, sse2_composite_over_n_8_8888),
6468 PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, sse2_composite_over_n_8_8888),
6469 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, sse2_composite_over_8888_8888_8888),
6470 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, x8r8g8b8, sse2_composite_over_8888_8_8888),
6471 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, a8r8g8b8, sse2_composite_over_8888_8_8888),
6472 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, x8b8g8r8, sse2_composite_over_8888_8_8888),
6473 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, a8b8g8r8, sse2_composite_over_8888_8_8888),
6474 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, x8r8g8b8, sse2_composite_over_x888_8_8888),
6475 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, a8r8g8b8, sse2_composite_over_x888_8_8888),
6476 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, x8b8g8r8, sse2_composite_over_x888_8_8888),
6477 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, a8b8g8r8, sse2_composite_over_x888_8_8888),
6478 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, a8r8g8b8, sse2_composite_over_x888_n_8888),
6479 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, x8r8g8b8, sse2_composite_over_x888_n_8888),
6480 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, a8b8g8r8, sse2_composite_over_x888_n_8888),
6481 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, x8b8g8r8, sse2_composite_over_x888_n_8888),
6482 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, a8r8g8b8, sse2_composite_over_8888_n_8888),
6483 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, x8r8g8b8, sse2_composite_over_8888_n_8888),
6484 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, a8b8g8r8, sse2_composite_over_8888_n_8888),
6485 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, x8b8g8r8, sse2_composite_over_8888_n_8888),
6486 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, sse2_composite_over_n_8888_8888_ca),
6487 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, sse2_composite_over_n_8888_8888_ca),
6488 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, sse2_composite_over_n_8888_8888_ca),
6489 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, sse2_composite_over_n_8888_8888_ca),
6490 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5, sse2_composite_over_n_8888_0565_ca),
6491 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5, sse2_composite_over_n_8888_0565_ca),
6492 PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, a8r8g8b8, sse2_composite_over_pixbuf_8888),
6493 PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, x8r8g8b8, sse2_composite_over_pixbuf_8888),
6494 PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, a8b8g8r8, sse2_composite_over_pixbuf_8888),
6495 PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, x8b8g8r8, sse2_composite_over_pixbuf_8888),
6496 PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, r5g6b5, sse2_composite_over_pixbuf_0565),
6497 PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, b5g6r5, sse2_composite_over_pixbuf_0565),
6498 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
6499 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
6501 /* PIXMAN_OP_OVER_REVERSE */
6502 PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, sse2_composite_over_reverse_n_8888),
6503 PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, sse2_composite_over_reverse_n_8888),
6506 PIXMAN_STD_FAST_PATH_CA (ADD, solid, a8r8g8b8, a8r8g8b8, sse2_composite_add_n_8888_8888_ca),
6507 PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, sse2_composite_add_8000_8000),
6508 PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, sse2_composite_add_8888_8888),
6509 PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, sse2_composite_add_8888_8888),
6510 PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, sse2_composite_add_n_8_8),
6511 PIXMAN_STD_FAST_PATH (ADD, solid, null, a8, sse2_composite_add_n_8),
6514 PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8r8g8b8, sse2_composite_src_n_8_8888),
6515 PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8r8g8b8, sse2_composite_src_n_8_8888),
6516 PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8b8g8r8, sse2_composite_src_n_8_8888),
6517 PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8b8g8r8, sse2_composite_src_n_8_8888),
6518 PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, a8r8g8b8, sse2_composite_src_x888_8888),
6519 PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, a8b8g8r8, sse2_composite_src_x888_8888),
6520 PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, sse2_composite_copy_area),
6521 PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, sse2_composite_copy_area),
6522 PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
6523 PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
6524 PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
6525 PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
6526 PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, r5g6b5, sse2_composite_copy_area),
6527 PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, b5g6r5, sse2_composite_copy_area),
6530 PIXMAN_STD_FAST_PATH (IN, a8, null, a8, sse2_composite_in_8_8),
6531 PIXMAN_STD_FAST_PATH (IN, solid, a8, a8, sse2_composite_in_n_8_8),
6532 PIXMAN_STD_FAST_PATH (IN, solid, null, a8, sse2_composite_in_n_8),
6534 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
6535 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
6536 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
6537 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
6538 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
6539 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
6540 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
6541 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
6542 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
6543 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
6544 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
6545 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
6550 static pixman_bool_t
6551 sse2_blt (pixman_implementation_t *imp,
6552 uint32_t * src_bits,
6553 uint32_t * dst_bits,
6565 if (!pixman_blt_sse2 (
6566 src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
6567 src_x, src_y, dst_x, dst_y, width, height))
6570 return _pixman_implementation_blt (
6572 src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
6573 src_x, src_y, dst_x, dst_y, width, height);
6579 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
6580 __attribute__((__force_align_arg_pointer__))
6582 static pixman_bool_t
6583 sse2_fill (pixman_implementation_t *imp,
6593 if (!pixman_fill_sse2 (bits, stride, bpp, x, y, width, height, xor))
6595 return _pixman_implementation_fill (
6596 imp->delegate, bits, stride, bpp, x, y, width, height, xor);
6602 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
6603 __attribute__((__force_align_arg_pointer__))
6605 pixman_implementation_t *
6606 _pixman_implementation_create_sse2 (void)
6609 pixman_implementation_t *fallback = _pixman_implementation_create_mmx ();
6611 pixman_implementation_t *fallback = _pixman_implementation_create_fast_path ();
6613 pixman_implementation_t *imp = _pixman_implementation_create (fallback, sse2_fast_paths);
6615 /* SSE2 constants */
6616 mask_565_r = create_mask_2x32_128 (0x00f80000, 0x00f80000);
6617 mask_565_g1 = create_mask_2x32_128 (0x00070000, 0x00070000);
6618 mask_565_g2 = create_mask_2x32_128 (0x000000e0, 0x000000e0);
6619 mask_565_b = create_mask_2x32_128 (0x0000001f, 0x0000001f);
6620 mask_red = create_mask_2x32_128 (0x00f80000, 0x00f80000);
6621 mask_green = create_mask_2x32_128 (0x0000fc00, 0x0000fc00);
6622 mask_blue = create_mask_2x32_128 (0x000000f8, 0x000000f8);
6623 mask_565_fix_rb = create_mask_2x32_128 (0x00e000e0, 0x00e000e0);
6624 mask_565_fix_g = create_mask_2x32_128 (0x0000c000, 0x0000c000);
6625 mask_0080 = create_mask_16_128 (0x0080);
6626 mask_00ff = create_mask_16_128 (0x00ff);
6627 mask_0101 = create_mask_16_128 (0x0101);
6628 mask_ffff = create_mask_16_128 (0xffff);
6629 mask_ff000000 = create_mask_2x32_128 (0xff000000, 0xff000000);
6630 mask_alpha = create_mask_2x32_128 (0x00ff0000, 0x00000000);
6633 mask_x565_rgb = create_mask_2x32_64 (0x000001f0, 0x003f001f);
6634 mask_x565_unpack = create_mask_2x32_64 (0x00000084, 0x04100840);
6636 mask_x0080 = create_mask_16_64 (0x0080);
6637 mask_x00ff = create_mask_16_64 (0x00ff);
6638 mask_x0101 = create_mask_16_64 (0x0101);
6639 mask_x_alpha = create_mask_2x32_64 (0x00ff0000, 0x00000000);
6643 /* Set up function pointers */
6645 /* SSE code patch for fbcompose.c */
6646 imp->combine_32[PIXMAN_OP_OVER] = sse2_combine_over_u;
6647 imp->combine_32[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_u;
6648 imp->combine_32[PIXMAN_OP_IN] = sse2_combine_in_u;
6649 imp->combine_32[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_u;
6650 imp->combine_32[PIXMAN_OP_OUT] = sse2_combine_out_u;
6651 imp->combine_32[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_u;
6652 imp->combine_32[PIXMAN_OP_ATOP] = sse2_combine_atop_u;
6653 imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_u;
6654 imp->combine_32[PIXMAN_OP_XOR] = sse2_combine_xor_u;
6655 imp->combine_32[PIXMAN_OP_ADD] = sse2_combine_add_u;
6657 imp->combine_32[PIXMAN_OP_SATURATE] = sse2_combine_saturate_u;
6659 imp->combine_32_ca[PIXMAN_OP_SRC] = sse2_combine_src_ca;
6660 imp->combine_32_ca[PIXMAN_OP_OVER] = sse2_combine_over_ca;
6661 imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_ca;
6662 imp->combine_32_ca[PIXMAN_OP_IN] = sse2_combine_in_ca;
6663 imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_ca;
6664 imp->combine_32_ca[PIXMAN_OP_OUT] = sse2_combine_out_ca;
6665 imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_ca;
6666 imp->combine_32_ca[PIXMAN_OP_ATOP] = sse2_combine_atop_ca;
6667 imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_ca;
6668 imp->combine_32_ca[PIXMAN_OP_XOR] = sse2_combine_xor_ca;
6669 imp->combine_32_ca[PIXMAN_OP_ADD] = sse2_combine_add_ca;
6671 imp->blt = sse2_blt;
6672 imp->fill = sse2_fill;
6677 #endif /* USE_SSE2 */