2 * Copyright © 2008 Rodrigo Kumpera
3 * Copyright © 2008 André Tupinambá
5 * Permission to use, copy, modify, distribute, and sell this software and its
6 * documentation for any purpose is hereby granted without fee, provided that
7 * the above copyright notice appear in all copies and that both that
8 * copyright notice and this permission notice appear in supporting
9 * documentation, and that the name of Red Hat not be used in advertising or
10 * publicity pertaining to distribution of the software without specific,
11 * written prior permission. Red Hat makes no representations about the
12 * suitability of this software for any purpose. It is provided "as is"
13 * without express or implied warranty.
15 * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
16 * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
17 * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
18 * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
20 * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
21 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
24 * Author: Rodrigo Kumpera (kumpera@gmail.com)
25 * André Tupinambá (andrelrt@gmail.com)
27 * Based on work by Owen Taylor and Søren Sandmann
33 #include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */
34 #include <emmintrin.h> /* for SSE2 intrinsics */
35 #include "pixman-private.h"
36 #include "pixman-combine32.h"
37 #include "pixman-fast-path.h"
39 static __m128i mask_0080;
40 static __m128i mask_00ff;
41 static __m128i mask_0101;
42 static __m128i mask_ffff;
43 static __m128i mask_ff000000;
44 static __m128i mask_alpha;
46 static __m128i mask_565_r;
47 static __m128i mask_565_g1, mask_565_g2;
48 static __m128i mask_565_b;
49 static __m128i mask_red;
50 static __m128i mask_green;
51 static __m128i mask_blue;
53 static __m128i mask_565_fix_rb;
54 static __m128i mask_565_fix_g;
56 static force_inline __m128i
57 unpack_32_1x128 (uint32_t data)
59 return _mm_unpacklo_epi8 (_mm_cvtsi32_si128 (data), _mm_setzero_si128 ());
62 static force_inline void
63 unpack_128_2x128 (__m128i data, __m128i* data_lo, __m128i* data_hi)
65 *data_lo = _mm_unpacklo_epi8 (data, _mm_setzero_si128 ());
66 *data_hi = _mm_unpackhi_epi8 (data, _mm_setzero_si128 ());
69 static force_inline __m128i
70 unpack_565_to_8888 (__m128i lo)
72 __m128i r, g, b, rb, t;
74 r = _mm_and_si128 (_mm_slli_epi32 (lo, 8), mask_red);
75 g = _mm_and_si128 (_mm_slli_epi32 (lo, 5), mask_green);
76 b = _mm_and_si128 (_mm_slli_epi32 (lo, 3), mask_blue);
78 rb = _mm_or_si128 (r, b);
79 t = _mm_and_si128 (rb, mask_565_fix_rb);
80 t = _mm_srli_epi32 (t, 5);
81 rb = _mm_or_si128 (rb, t);
83 t = _mm_and_si128 (g, mask_565_fix_g);
84 t = _mm_srli_epi32 (t, 6);
85 g = _mm_or_si128 (g, t);
87 return _mm_or_si128 (rb, g);
90 static force_inline void
91 unpack_565_128_4x128 (__m128i data,
99 lo = _mm_unpacklo_epi16 (data, _mm_setzero_si128 ());
100 hi = _mm_unpackhi_epi16 (data, _mm_setzero_si128 ());
102 lo = unpack_565_to_8888 (lo);
103 hi = unpack_565_to_8888 (hi);
105 unpack_128_2x128 (lo, data0, data1);
106 unpack_128_2x128 (hi, data2, data3);
109 static force_inline uint16_t
110 pack_565_32_16 (uint32_t pixel)
112 return (uint16_t) (((pixel >> 8) & 0xf800) |
113 ((pixel >> 5) & 0x07e0) |
114 ((pixel >> 3) & 0x001f));
117 static force_inline __m128i
118 pack_2x128_128 (__m128i lo, __m128i hi)
120 return _mm_packus_epi16 (lo, hi);
123 static force_inline __m128i
124 pack_565_2x128_128 (__m128i lo, __m128i hi)
127 __m128i r, g1, g2, b;
129 data = pack_2x128_128 (lo, hi);
131 r = _mm_and_si128 (data, mask_565_r);
132 g1 = _mm_and_si128 (_mm_slli_epi32 (data, 3), mask_565_g1);
133 g2 = _mm_and_si128 (_mm_srli_epi32 (data, 5), mask_565_g2);
134 b = _mm_and_si128 (_mm_srli_epi32 (data, 3), mask_565_b);
136 return _mm_or_si128 (_mm_or_si128 (_mm_or_si128 (r, g1), g2), b);
139 static force_inline __m128i
140 pack_565_4x128_128 (__m128i* xmm0, __m128i* xmm1, __m128i* xmm2, __m128i* xmm3)
142 return _mm_packus_epi16 (pack_565_2x128_128 (*xmm0, *xmm1),
143 pack_565_2x128_128 (*xmm2, *xmm3));
146 static force_inline int
147 is_opaque (__m128i x)
149 __m128i ffs = _mm_cmpeq_epi8 (x, x);
151 return (_mm_movemask_epi8 (_mm_cmpeq_epi8 (x, ffs)) & 0x8888) == 0x8888;
154 static force_inline int
157 return _mm_movemask_epi8 (
158 _mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) == 0xffff;
161 static force_inline int
162 is_transparent (__m128i x)
164 return (_mm_movemask_epi8 (
165 _mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) & 0x8888) == 0x8888;
168 static force_inline __m128i
169 expand_pixel_32_1x128 (uint32_t data)
171 return _mm_shuffle_epi32 (unpack_32_1x128 (data), _MM_SHUFFLE (1, 0, 1, 0));
174 static force_inline __m128i
175 expand_alpha_1x128 (__m128i data)
177 return _mm_shufflehi_epi16 (_mm_shufflelo_epi16 (data,
178 _MM_SHUFFLE (3, 3, 3, 3)),
179 _MM_SHUFFLE (3, 3, 3, 3));
182 static force_inline void
183 expand_alpha_2x128 (__m128i data_lo,
190 lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 3, 3, 3));
191 hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 3, 3, 3));
193 *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 3, 3, 3));
194 *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 3, 3, 3));
197 static force_inline void
198 expand_alpha_rev_2x128 (__m128i data_lo,
205 lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (0, 0, 0, 0));
206 hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (0, 0, 0, 0));
207 *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (0, 0, 0, 0));
208 *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (0, 0, 0, 0));
211 static force_inline void
212 pix_multiply_2x128 (__m128i* data_lo,
221 lo = _mm_mullo_epi16 (*data_lo, *alpha_lo);
222 hi = _mm_mullo_epi16 (*data_hi, *alpha_hi);
223 lo = _mm_adds_epu16 (lo, mask_0080);
224 hi = _mm_adds_epu16 (hi, mask_0080);
225 *ret_lo = _mm_mulhi_epu16 (lo, mask_0101);
226 *ret_hi = _mm_mulhi_epu16 (hi, mask_0101);
229 static force_inline void
230 pix_add_multiply_2x128 (__m128i* src_lo,
232 __m128i* alpha_dst_lo,
233 __m128i* alpha_dst_hi,
236 __m128i* alpha_src_lo,
237 __m128i* alpha_src_hi,
241 __m128i t1_lo, t1_hi;
242 __m128i t2_lo, t2_hi;
244 pix_multiply_2x128 (src_lo, src_hi, alpha_dst_lo, alpha_dst_hi, &t1_lo, &t1_hi);
245 pix_multiply_2x128 (dst_lo, dst_hi, alpha_src_lo, alpha_src_hi, &t2_lo, &t2_hi);
247 *ret_lo = _mm_adds_epu8 (t1_lo, t2_lo);
248 *ret_hi = _mm_adds_epu8 (t1_hi, t2_hi);
251 static force_inline void
252 negate_2x128 (__m128i data_lo,
257 *neg_lo = _mm_xor_si128 (data_lo, mask_00ff);
258 *neg_hi = _mm_xor_si128 (data_hi, mask_00ff);
261 static force_inline void
262 invert_colors_2x128 (__m128i data_lo,
269 lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 0, 1, 2));
270 hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 0, 1, 2));
271 *inv_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 0, 1, 2));
272 *inv_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 0, 1, 2));
275 static force_inline void
276 over_2x128 (__m128i* src_lo,
285 negate_2x128 (*alpha_lo, *alpha_hi, &t1, &t2);
287 pix_multiply_2x128 (dst_lo, dst_hi, &t1, &t2, dst_lo, dst_hi);
289 *dst_lo = _mm_adds_epu8 (*src_lo, *dst_lo);
290 *dst_hi = _mm_adds_epu8 (*src_hi, *dst_hi);
293 static force_inline void
294 over_rev_non_pre_2x128 (__m128i src_lo,
300 __m128i alpha_lo, alpha_hi;
302 expand_alpha_2x128 (src_lo, src_hi, &alpha_lo, &alpha_hi);
304 lo = _mm_or_si128 (alpha_lo, mask_alpha);
305 hi = _mm_or_si128 (alpha_hi, mask_alpha);
307 invert_colors_2x128 (src_lo, src_hi, &src_lo, &src_hi);
309 pix_multiply_2x128 (&src_lo, &src_hi, &lo, &hi, &lo, &hi);
311 over_2x128 (&lo, &hi, &alpha_lo, &alpha_hi, dst_lo, dst_hi);
314 static force_inline void
315 in_over_2x128 (__m128i* src_lo,
327 pix_multiply_2x128 (src_lo, src_hi, mask_lo, mask_hi, &s_lo, &s_hi);
328 pix_multiply_2x128 (alpha_lo, alpha_hi, mask_lo, mask_hi, &a_lo, &a_hi);
330 over_2x128 (&s_lo, &s_hi, &a_lo, &a_hi, dst_lo, dst_hi);
333 /* load 4 pixels from a 16-byte boundary aligned address */
334 static force_inline __m128i
335 load_128_aligned (__m128i* src)
337 return _mm_load_si128 (src);
340 /* load 4 pixels from a unaligned address */
341 static force_inline __m128i
342 load_128_unaligned (const __m128i* src)
344 return _mm_loadu_si128 (src);
347 /* save 4 pixels using Write Combining memory on a 16-byte
348 * boundary aligned address
350 static force_inline void
351 save_128_write_combining (__m128i* dst,
354 _mm_stream_si128 (dst, data);
357 /* save 4 pixels on a 16-byte boundary aligned address */
358 static force_inline void
359 save_128_aligned (__m128i* dst,
362 _mm_store_si128 (dst, data);
365 /* save 4 pixels on a unaligned address */
366 static force_inline void
367 save_128_unaligned (__m128i* dst,
370 _mm_storeu_si128 (dst, data);
373 static force_inline __m128i
374 load_32_1x128 (uint32_t data)
376 return _mm_cvtsi32_si128 (data);
379 static force_inline __m128i
380 expand_alpha_rev_1x128 (__m128i data)
382 return _mm_shufflelo_epi16 (data, _MM_SHUFFLE (0, 0, 0, 0));
385 static force_inline __m128i
386 expand_pixel_8_1x128 (uint8_t data)
388 return _mm_shufflelo_epi16 (
389 unpack_32_1x128 ((uint32_t)data), _MM_SHUFFLE (0, 0, 0, 0));
392 static force_inline __m128i
393 pix_multiply_1x128 (__m128i data,
396 return _mm_mulhi_epu16 (_mm_adds_epu16 (_mm_mullo_epi16 (data, alpha),
401 static force_inline __m128i
402 pix_add_multiply_1x128 (__m128i* src,
407 __m128i t1 = pix_multiply_1x128 (*src, *alpha_dst);
408 __m128i t2 = pix_multiply_1x128 (*dst, *alpha_src);
410 return _mm_adds_epu8 (t1, t2);
413 static force_inline __m128i
414 negate_1x128 (__m128i data)
416 return _mm_xor_si128 (data, mask_00ff);
419 static force_inline __m128i
420 invert_colors_1x128 (__m128i data)
422 return _mm_shufflelo_epi16 (data, _MM_SHUFFLE (3, 0, 1, 2));
425 static force_inline __m128i
426 over_1x128 (__m128i src, __m128i alpha, __m128i dst)
428 return _mm_adds_epu8 (src, pix_multiply_1x128 (dst, negate_1x128 (alpha)));
431 static force_inline __m128i
432 in_over_1x128 (__m128i* src, __m128i* alpha, __m128i* mask, __m128i* dst)
434 return over_1x128 (pix_multiply_1x128 (*src, *mask),
435 pix_multiply_1x128 (*alpha, *mask),
439 static force_inline __m128i
440 over_rev_non_pre_1x128 (__m128i src, __m128i dst)
442 __m128i alpha = expand_alpha_1x128 (src);
444 return over_1x128 (pix_multiply_1x128 (invert_colors_1x128 (src),
445 _mm_or_si128 (alpha, mask_alpha)),
450 static force_inline uint32_t
451 pack_1x128_32 (__m128i data)
453 return _mm_cvtsi128_si32 (_mm_packus_epi16 (data, _mm_setzero_si128 ()));
456 static force_inline __m128i
457 expand565_16_1x128 (uint16_t pixel)
459 __m128i m = _mm_cvtsi32_si128 (pixel);
461 m = unpack_565_to_8888 (m);
463 return _mm_unpacklo_epi8 (m, _mm_setzero_si128 ());
466 static force_inline uint32_t
467 core_combine_over_u_pixel_sse2 (uint32_t src, uint32_t dst)
480 xmms = unpack_32_1x128 (src);
481 return pack_1x128_32 (
482 over_1x128 (xmms, expand_alpha_1x128 (xmms),
483 unpack_32_1x128 (dst)));
489 static force_inline uint32_t
490 combine1 (const uint32_t *ps, const uint32_t *pm)
498 mm = unpack_32_1x128 (*pm);
499 mm = expand_alpha_1x128 (mm);
501 ms = unpack_32_1x128 (s);
502 ms = pix_multiply_1x128 (ms, mm);
504 s = pack_1x128_32 (ms);
510 static force_inline __m128i
511 combine4 (const __m128i *ps, const __m128i *pm)
513 __m128i xmm_src_lo, xmm_src_hi;
514 __m128i xmm_msk_lo, xmm_msk_hi;
519 xmm_msk_lo = load_128_unaligned (pm);
521 if (is_transparent (xmm_msk_lo))
522 return _mm_setzero_si128 ();
525 s = load_128_unaligned (ps);
529 unpack_128_2x128 (s, &xmm_src_lo, &xmm_src_hi);
530 unpack_128_2x128 (xmm_msk_lo, &xmm_msk_lo, &xmm_msk_hi);
532 expand_alpha_2x128 (xmm_msk_lo, xmm_msk_hi, &xmm_msk_lo, &xmm_msk_hi);
534 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
535 &xmm_msk_lo, &xmm_msk_hi,
536 &xmm_src_lo, &xmm_src_hi);
538 s = pack_2x128_128 (xmm_src_lo, xmm_src_hi);
544 static force_inline void
545 core_combine_over_u_sse2_mask (uint32_t * pd,
552 /* Align dst on a 16-byte boundary */
553 while (w && ((unsigned long)pd & 15))
556 s = combine1 (ps, pm);
559 *pd = core_combine_over_u_pixel_sse2 (s, d);
568 __m128i mask = load_128_unaligned ((__m128i *)pm);
573 __m128i src_hi, src_lo;
574 __m128i mask_hi, mask_lo;
575 __m128i alpha_hi, alpha_lo;
577 src = load_128_unaligned ((__m128i *)ps);
579 if (is_opaque (_mm_and_si128 (src, mask)))
581 save_128_aligned ((__m128i *)pd, src);
585 __m128i dst = load_128_aligned ((__m128i *)pd);
586 __m128i dst_hi, dst_lo;
588 unpack_128_2x128 (mask, &mask_lo, &mask_hi);
589 unpack_128_2x128 (src, &src_lo, &src_hi);
591 expand_alpha_2x128 (mask_lo, mask_hi, &mask_lo, &mask_hi);
592 pix_multiply_2x128 (&src_lo, &src_hi,
596 unpack_128_2x128 (dst, &dst_lo, &dst_hi);
598 expand_alpha_2x128 (src_lo, src_hi,
599 &alpha_lo, &alpha_hi);
601 over_2x128 (&src_lo, &src_hi, &alpha_lo, &alpha_hi,
606 pack_2x128_128 (dst_lo, dst_hi));
618 s = combine1 (ps, pm);
621 *pd = core_combine_over_u_pixel_sse2 (s, d);
630 static force_inline void
631 core_combine_over_u_sse2_no_mask (uint32_t * pd,
637 /* Align dst on a 16-byte boundary */
638 while (w && ((unsigned long)pd & 15))
644 *pd = core_combine_over_u_pixel_sse2 (s, d);
653 __m128i src_hi, src_lo, dst_hi, dst_lo;
654 __m128i alpha_hi, alpha_lo;
656 src = load_128_unaligned ((__m128i *)ps);
662 save_128_aligned ((__m128i *)pd, src);
666 __m128i dst = load_128_aligned ((__m128i *)pd);
668 unpack_128_2x128 (src, &src_lo, &src_hi);
669 unpack_128_2x128 (dst, &dst_lo, &dst_hi);
671 expand_alpha_2x128 (src_lo, src_hi,
672 &alpha_lo, &alpha_hi);
673 over_2x128 (&src_lo, &src_hi, &alpha_lo, &alpha_hi,
678 pack_2x128_128 (dst_lo, dst_hi));
692 *pd = core_combine_over_u_pixel_sse2 (s, d);
700 static force_inline void
701 sse2_combine_over_u (pixman_implementation_t *imp,
709 core_combine_over_u_sse2_mask (pd, ps, pm, w);
711 core_combine_over_u_sse2_no_mask (pd, ps, w);
715 sse2_combine_over_reverse_u (pixman_implementation_t *imp,
724 __m128i xmm_dst_lo, xmm_dst_hi;
725 __m128i xmm_src_lo, xmm_src_hi;
726 __m128i xmm_alpha_lo, xmm_alpha_hi;
728 /* Align dst on a 16-byte boundary */
730 ((unsigned long)pd & 15))
733 s = combine1 (ps, pm);
735 *pd++ = core_combine_over_u_pixel_sse2 (d, s);
744 /* I'm loading unaligned because I'm not sure
745 * about the address alignment.
747 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
748 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
750 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
751 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
753 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
754 &xmm_alpha_lo, &xmm_alpha_hi);
756 over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
757 &xmm_alpha_lo, &xmm_alpha_hi,
758 &xmm_src_lo, &xmm_src_hi);
760 /* rebuid the 4 pixel data and save*/
761 save_128_aligned ((__m128i*)pd,
762 pack_2x128_128 (xmm_src_lo, xmm_src_hi));
775 s = combine1 (ps, pm);
777 *pd++ = core_combine_over_u_pixel_sse2 (d, s);
785 static force_inline uint32_t
786 core_combine_in_u_pixel_sse2 (uint32_t src, uint32_t dst)
788 uint32_t maska = src >> 24;
794 else if (maska != 0xff)
796 return pack_1x128_32 (
797 pix_multiply_1x128 (unpack_32_1x128 (dst),
798 expand_alpha_1x128 (unpack_32_1x128 (src))));
805 sse2_combine_in_u (pixman_implementation_t *imp,
814 __m128i xmm_src_lo, xmm_src_hi;
815 __m128i xmm_dst_lo, xmm_dst_hi;
817 while (w && ((unsigned long) pd & 15))
819 s = combine1 (ps, pm);
822 *pd++ = core_combine_in_u_pixel_sse2 (d, s);
831 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
832 xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*) pm);
834 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
835 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
837 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
838 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
839 &xmm_dst_lo, &xmm_dst_hi,
840 &xmm_dst_lo, &xmm_dst_hi);
842 save_128_aligned ((__m128i*)pd,
843 pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
854 s = combine1 (ps, pm);
857 *pd++ = core_combine_in_u_pixel_sse2 (d, s);
866 sse2_combine_in_reverse_u (pixman_implementation_t *imp,
875 __m128i xmm_src_lo, xmm_src_hi;
876 __m128i xmm_dst_lo, xmm_dst_hi;
878 while (w && ((unsigned long) pd & 15))
880 s = combine1 (ps, pm);
883 *pd++ = core_combine_in_u_pixel_sse2 (s, d);
892 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
893 xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
895 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
896 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
898 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
899 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
900 &xmm_src_lo, &xmm_src_hi,
901 &xmm_dst_lo, &xmm_dst_hi);
904 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
915 s = combine1 (ps, pm);
918 *pd++ = core_combine_in_u_pixel_sse2 (s, d);
927 sse2_combine_out_reverse_u (pixman_implementation_t *imp,
934 while (w && ((unsigned long) pd & 15))
936 uint32_t s = combine1 (ps, pm);
939 *pd++ = pack_1x128_32 (
941 unpack_32_1x128 (d), negate_1x128 (
942 expand_alpha_1x128 (unpack_32_1x128 (s)))));
952 __m128i xmm_src_lo, xmm_src_hi;
953 __m128i xmm_dst_lo, xmm_dst_hi;
955 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
956 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
958 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
959 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
961 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
962 negate_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
964 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
965 &xmm_src_lo, &xmm_src_hi,
966 &xmm_dst_lo, &xmm_dst_hi);
969 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
981 uint32_t s = combine1 (ps, pm);
984 *pd++ = pack_1x128_32 (
986 unpack_32_1x128 (d), negate_1x128 (
987 expand_alpha_1x128 (unpack_32_1x128 (s)))));
996 sse2_combine_out_u (pixman_implementation_t *imp,
1000 const uint32_t * pm,
1003 while (w && ((unsigned long) pd & 15))
1005 uint32_t s = combine1 (ps, pm);
1008 *pd++ = pack_1x128_32 (
1009 pix_multiply_1x128 (
1010 unpack_32_1x128 (s), negate_1x128 (
1011 expand_alpha_1x128 (unpack_32_1x128 (d)))));
1020 __m128i xmm_src_lo, xmm_src_hi;
1021 __m128i xmm_dst_lo, xmm_dst_hi;
1023 xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
1024 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1026 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1027 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1029 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1030 negate_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1032 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1033 &xmm_dst_lo, &xmm_dst_hi,
1034 &xmm_dst_lo, &xmm_dst_hi);
1037 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1048 uint32_t s = combine1 (ps, pm);
1051 *pd++ = pack_1x128_32 (
1052 pix_multiply_1x128 (
1053 unpack_32_1x128 (s), negate_1x128 (
1054 expand_alpha_1x128 (unpack_32_1x128 (d)))));
1062 static force_inline uint32_t
1063 core_combine_atop_u_pixel_sse2 (uint32_t src,
1066 __m128i s = unpack_32_1x128 (src);
1067 __m128i d = unpack_32_1x128 (dst);
1069 __m128i sa = negate_1x128 (expand_alpha_1x128 (s));
1070 __m128i da = expand_alpha_1x128 (d);
1072 return pack_1x128_32 (pix_add_multiply_1x128 (&s, &da, &d, &sa));
1076 sse2_combine_atop_u (pixman_implementation_t *imp,
1079 const uint32_t * ps,
1080 const uint32_t * pm,
1085 __m128i xmm_src_lo, xmm_src_hi;
1086 __m128i xmm_dst_lo, xmm_dst_hi;
1087 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1088 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1090 while (w && ((unsigned long) pd & 15))
1092 s = combine1 (ps, pm);
1095 *pd++ = core_combine_atop_u_pixel_sse2 (s, d);
1104 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1105 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1107 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1108 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1110 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1111 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1112 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1113 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1115 negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
1116 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1118 pix_add_multiply_2x128 (
1119 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1120 &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1121 &xmm_dst_lo, &xmm_dst_hi);
1124 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1135 s = combine1 (ps, pm);
1138 *pd++ = core_combine_atop_u_pixel_sse2 (s, d);
1146 static force_inline uint32_t
1147 core_combine_reverse_atop_u_pixel_sse2 (uint32_t src,
1150 __m128i s = unpack_32_1x128 (src);
1151 __m128i d = unpack_32_1x128 (dst);
1153 __m128i sa = expand_alpha_1x128 (s);
1154 __m128i da = negate_1x128 (expand_alpha_1x128 (d));
1156 return pack_1x128_32 (pix_add_multiply_1x128 (&s, &da, &d, &sa));
1160 sse2_combine_atop_reverse_u (pixman_implementation_t *imp,
1163 const uint32_t * ps,
1164 const uint32_t * pm,
1169 __m128i xmm_src_lo, xmm_src_hi;
1170 __m128i xmm_dst_lo, xmm_dst_hi;
1171 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1172 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1174 while (w && ((unsigned long) pd & 15))
1176 s = combine1 (ps, pm);
1179 *pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
1188 xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1189 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1191 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1192 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1194 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1195 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1196 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1197 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1199 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
1200 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1202 pix_add_multiply_2x128 (
1203 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1204 &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1205 &xmm_dst_lo, &xmm_dst_hi);
1208 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1219 s = combine1 (ps, pm);
1222 *pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
1230 static force_inline uint32_t
1231 core_combine_xor_u_pixel_sse2 (uint32_t src,
1234 __m128i s = unpack_32_1x128 (src);
1235 __m128i d = unpack_32_1x128 (dst);
1237 __m128i neg_d = negate_1x128 (expand_alpha_1x128 (d));
1238 __m128i neg_s = negate_1x128 (expand_alpha_1x128 (s));
1240 return pack_1x128_32 (pix_add_multiply_1x128 (&s, &neg_d, &d, &neg_s));
1244 sse2_combine_xor_u (pixman_implementation_t *imp,
1247 const uint32_t * src,
1248 const uint32_t * mask,
1254 const uint32_t* ps = src;
1255 const uint32_t* pm = mask;
1257 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
1258 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
1259 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1260 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1262 while (w && ((unsigned long) pd & 15))
1264 s = combine1 (ps, pm);
1267 *pd++ = core_combine_xor_u_pixel_sse2 (s, d);
1276 xmm_src = combine4 ((__m128i*) ps, (__m128i*) pm);
1277 xmm_dst = load_128_aligned ((__m128i*) pd);
1279 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
1280 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
1282 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1283 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1284 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1285 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1287 negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
1288 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1289 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
1290 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1292 pix_add_multiply_2x128 (
1293 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1294 &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1295 &xmm_dst_lo, &xmm_dst_hi);
1298 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1309 s = combine1 (ps, pm);
1312 *pd++ = core_combine_xor_u_pixel_sse2 (s, d);
1320 static force_inline void
1321 sse2_combine_add_u (pixman_implementation_t *imp,
1324 const uint32_t * src,
1325 const uint32_t * mask,
1331 const uint32_t* ps = src;
1332 const uint32_t* pm = mask;
1334 while (w && (unsigned long)pd & 15)
1336 s = combine1 (ps, pm);
1342 *pd++ = _mm_cvtsi128_si32 (
1343 _mm_adds_epu8 (_mm_cvtsi32_si128 (s), _mm_cvtsi32_si128 (d)));
1351 s = combine4 ((__m128i*)ps, (__m128i*)pm);
1354 (__m128i*)pd, _mm_adds_epu8 (s, load_128_aligned ((__m128i*)pd)));
1365 s = combine1 (ps, pm);
1369 *pd++ = _mm_cvtsi128_si32 (
1370 _mm_adds_epu8 (_mm_cvtsi32_si128 (s), _mm_cvtsi32_si128 (d)));
1376 static force_inline uint32_t
1377 core_combine_saturate_u_pixel_sse2 (uint32_t src,
1380 __m128i ms = unpack_32_1x128 (src);
1381 __m128i md = unpack_32_1x128 (dst);
1382 uint32_t sa = src >> 24;
1383 uint32_t da = ~dst >> 24;
1387 ms = pix_multiply_1x128 (
1388 ms, expand_alpha_1x128 (unpack_32_1x128 (DIV_UN8 (da, sa) << 24)));
1391 return pack_1x128_32 (_mm_adds_epu16 (md, ms));
1395 sse2_combine_saturate_u (pixman_implementation_t *imp,
1398 const uint32_t * ps,
1399 const uint32_t * pm,
1405 __m128i xmm_src, xmm_dst;
1407 while (w && (unsigned long)pd & 15)
1409 s = combine1 (ps, pm);
1412 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1421 xmm_dst = load_128_aligned ((__m128i*)pd);
1422 xmm_src = combine4 ((__m128i*)ps, (__m128i*)pm);
1424 pack_cmp = _mm_movemask_epi8 (
1426 _mm_srli_epi32 (xmm_src, 24),
1427 _mm_srli_epi32 (_mm_xor_si128 (xmm_dst, mask_ff000000), 24)));
1429 /* if some alpha src is grater than respective ~alpha dst */
1432 s = combine1 (ps++, pm);
1434 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1438 s = combine1 (ps++, pm);
1440 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1444 s = combine1 (ps++, pm);
1446 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1450 s = combine1 (ps++, pm);
1452 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1458 save_128_aligned ((__m128i*)pd, _mm_adds_epu8 (xmm_dst, xmm_src));
1471 s = combine1 (ps, pm);
1474 *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1482 sse2_combine_src_ca (pixman_implementation_t *imp,
1485 const uint32_t * ps,
1486 const uint32_t * pm,
1491 __m128i xmm_src_lo, xmm_src_hi;
1492 __m128i xmm_mask_lo, xmm_mask_hi;
1493 __m128i xmm_dst_lo, xmm_dst_hi;
1495 while (w && (unsigned long)pd & 15)
1499 *pd++ = pack_1x128_32 (
1500 pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)));
1506 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1507 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1509 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1510 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1512 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1513 &xmm_mask_lo, &xmm_mask_hi,
1514 &xmm_dst_lo, &xmm_dst_hi);
1517 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1529 *pd++ = pack_1x128_32 (
1530 pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)));
1535 static force_inline uint32_t
1536 core_combine_over_ca_pixel_sse2 (uint32_t src,
1540 __m128i s = unpack_32_1x128 (src);
1541 __m128i expAlpha = expand_alpha_1x128 (s);
1542 __m128i unpk_mask = unpack_32_1x128 (mask);
1543 __m128i unpk_dst = unpack_32_1x128 (dst);
1545 return pack_1x128_32 (in_over_1x128 (&s, &expAlpha, &unpk_mask, &unpk_dst));
1549 sse2_combine_over_ca (pixman_implementation_t *imp,
1552 const uint32_t * ps,
1553 const uint32_t * pm,
1558 __m128i xmm_alpha_lo, xmm_alpha_hi;
1559 __m128i xmm_src_lo, xmm_src_hi;
1560 __m128i xmm_dst_lo, xmm_dst_hi;
1561 __m128i xmm_mask_lo, xmm_mask_hi;
1563 while (w && (unsigned long)pd & 15)
1569 *pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
1575 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1576 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1577 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1579 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1580 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1581 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1583 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1584 &xmm_alpha_lo, &xmm_alpha_hi);
1586 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
1587 &xmm_alpha_lo, &xmm_alpha_hi,
1588 &xmm_mask_lo, &xmm_mask_hi,
1589 &xmm_dst_lo, &xmm_dst_hi);
1592 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1606 *pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
1611 static force_inline uint32_t
1612 core_combine_over_reverse_ca_pixel_sse2 (uint32_t src,
1616 __m128i d = unpack_32_1x128 (dst);
1618 return pack_1x128_32 (
1619 over_1x128 (d, expand_alpha_1x128 (d),
1620 pix_multiply_1x128 (unpack_32_1x128 (src),
1621 unpack_32_1x128 (mask))));
1625 sse2_combine_over_reverse_ca (pixman_implementation_t *imp,
1628 const uint32_t * ps,
1629 const uint32_t * pm,
1634 __m128i xmm_alpha_lo, xmm_alpha_hi;
1635 __m128i xmm_src_lo, xmm_src_hi;
1636 __m128i xmm_dst_lo, xmm_dst_hi;
1637 __m128i xmm_mask_lo, xmm_mask_hi;
1639 while (w && (unsigned long)pd & 15)
1645 *pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
1651 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1652 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1653 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1655 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1656 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1657 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1659 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1660 &xmm_alpha_lo, &xmm_alpha_hi);
1661 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1662 &xmm_mask_lo, &xmm_mask_hi,
1663 &xmm_mask_lo, &xmm_mask_hi);
1665 over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1666 &xmm_alpha_lo, &xmm_alpha_hi,
1667 &xmm_mask_lo, &xmm_mask_hi);
1670 (__m128i*)pd, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
1684 *pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
1690 sse2_combine_in_ca (pixman_implementation_t *imp,
1693 const uint32_t * ps,
1694 const uint32_t * pm,
1699 __m128i xmm_alpha_lo, xmm_alpha_hi;
1700 __m128i xmm_src_lo, xmm_src_hi;
1701 __m128i xmm_dst_lo, xmm_dst_hi;
1702 __m128i xmm_mask_lo, xmm_mask_hi;
1704 while (w && (unsigned long)pd & 15)
1710 *pd++ = pack_1x128_32 (
1711 pix_multiply_1x128 (
1712 pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (m)),
1713 expand_alpha_1x128 (unpack_32_1x128 (d))));
1720 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1721 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1722 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1724 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1725 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1726 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1728 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1729 &xmm_alpha_lo, &xmm_alpha_hi);
1731 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1732 &xmm_mask_lo, &xmm_mask_hi,
1733 &xmm_dst_lo, &xmm_dst_hi);
1735 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1736 &xmm_alpha_lo, &xmm_alpha_hi,
1737 &xmm_dst_lo, &xmm_dst_hi);
1740 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1754 *pd++ = pack_1x128_32 (
1755 pix_multiply_1x128 (
1756 pix_multiply_1x128 (
1757 unpack_32_1x128 (s), unpack_32_1x128 (m)),
1758 expand_alpha_1x128 (unpack_32_1x128 (d))));
1765 sse2_combine_in_reverse_ca (pixman_implementation_t *imp,
1768 const uint32_t * ps,
1769 const uint32_t * pm,
1774 __m128i xmm_alpha_lo, xmm_alpha_hi;
1775 __m128i xmm_src_lo, xmm_src_hi;
1776 __m128i xmm_dst_lo, xmm_dst_hi;
1777 __m128i xmm_mask_lo, xmm_mask_hi;
1779 while (w && (unsigned long)pd & 15)
1785 *pd++ = pack_1x128_32 (
1786 pix_multiply_1x128 (
1787 unpack_32_1x128 (d),
1788 pix_multiply_1x128 (unpack_32_1x128 (m),
1789 expand_alpha_1x128 (unpack_32_1x128 (s)))));
1795 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1796 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1797 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1799 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1800 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1801 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1803 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1804 &xmm_alpha_lo, &xmm_alpha_hi);
1805 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
1806 &xmm_alpha_lo, &xmm_alpha_hi,
1807 &xmm_alpha_lo, &xmm_alpha_hi);
1809 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1810 &xmm_alpha_lo, &xmm_alpha_hi,
1811 &xmm_dst_lo, &xmm_dst_hi);
1814 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1828 *pd++ = pack_1x128_32 (
1829 pix_multiply_1x128 (
1830 unpack_32_1x128 (d),
1831 pix_multiply_1x128 (unpack_32_1x128 (m),
1832 expand_alpha_1x128 (unpack_32_1x128 (s)))));
1838 sse2_combine_out_ca (pixman_implementation_t *imp,
1841 const uint32_t * ps,
1842 const uint32_t * pm,
1847 __m128i xmm_alpha_lo, xmm_alpha_hi;
1848 __m128i xmm_src_lo, xmm_src_hi;
1849 __m128i xmm_dst_lo, xmm_dst_hi;
1850 __m128i xmm_mask_lo, xmm_mask_hi;
1852 while (w && (unsigned long)pd & 15)
1858 *pd++ = pack_1x128_32 (
1859 pix_multiply_1x128 (
1860 pix_multiply_1x128 (
1861 unpack_32_1x128 (s), unpack_32_1x128 (m)),
1862 negate_1x128 (expand_alpha_1x128 (unpack_32_1x128 (d)))));
1868 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1869 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1870 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1872 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1873 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1874 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1876 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1877 &xmm_alpha_lo, &xmm_alpha_hi);
1878 negate_2x128 (xmm_alpha_lo, xmm_alpha_hi,
1879 &xmm_alpha_lo, &xmm_alpha_hi);
1881 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1882 &xmm_mask_lo, &xmm_mask_hi,
1883 &xmm_dst_lo, &xmm_dst_hi);
1884 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1885 &xmm_alpha_lo, &xmm_alpha_hi,
1886 &xmm_dst_lo, &xmm_dst_hi);
1889 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1903 *pd++ = pack_1x128_32 (
1904 pix_multiply_1x128 (
1905 pix_multiply_1x128 (
1906 unpack_32_1x128 (s), unpack_32_1x128 (m)),
1907 negate_1x128 (expand_alpha_1x128 (unpack_32_1x128 (d)))));
1914 sse2_combine_out_reverse_ca (pixman_implementation_t *imp,
1917 const uint32_t * ps,
1918 const uint32_t * pm,
1923 __m128i xmm_alpha_lo, xmm_alpha_hi;
1924 __m128i xmm_src_lo, xmm_src_hi;
1925 __m128i xmm_dst_lo, xmm_dst_hi;
1926 __m128i xmm_mask_lo, xmm_mask_hi;
1928 while (w && (unsigned long)pd & 15)
1934 *pd++ = pack_1x128_32 (
1935 pix_multiply_1x128 (
1936 unpack_32_1x128 (d),
1937 negate_1x128 (pix_multiply_1x128 (
1938 unpack_32_1x128 (m),
1939 expand_alpha_1x128 (unpack_32_1x128 (s))))));
1945 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1946 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1947 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1949 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1950 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1951 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1953 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1954 &xmm_alpha_lo, &xmm_alpha_hi);
1956 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
1957 &xmm_alpha_lo, &xmm_alpha_hi,
1958 &xmm_mask_lo, &xmm_mask_hi);
1960 negate_2x128 (xmm_mask_lo, xmm_mask_hi,
1961 &xmm_mask_lo, &xmm_mask_hi);
1963 pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1964 &xmm_mask_lo, &xmm_mask_hi,
1965 &xmm_dst_lo, &xmm_dst_hi);
1968 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1982 *pd++ = pack_1x128_32 (
1983 pix_multiply_1x128 (
1984 unpack_32_1x128 (d),
1985 negate_1x128 (pix_multiply_1x128 (
1986 unpack_32_1x128 (m),
1987 expand_alpha_1x128 (unpack_32_1x128 (s))))));
1992 static force_inline uint32_t
1993 core_combine_atop_ca_pixel_sse2 (uint32_t src,
1997 __m128i m = unpack_32_1x128 (mask);
1998 __m128i s = unpack_32_1x128 (src);
1999 __m128i d = unpack_32_1x128 (dst);
2000 __m128i sa = expand_alpha_1x128 (s);
2001 __m128i da = expand_alpha_1x128 (d);
2003 s = pix_multiply_1x128 (s, m);
2004 m = negate_1x128 (pix_multiply_1x128 (m, sa));
2006 return pack_1x128_32 (pix_add_multiply_1x128 (&d, &m, &s, &da));
2010 sse2_combine_atop_ca (pixman_implementation_t *imp,
2013 const uint32_t * ps,
2014 const uint32_t * pm,
2019 __m128i xmm_src_lo, xmm_src_hi;
2020 __m128i xmm_dst_lo, xmm_dst_hi;
2021 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2022 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2023 __m128i xmm_mask_lo, xmm_mask_hi;
2025 while (w && (unsigned long)pd & 15)
2031 *pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
2037 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2038 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2039 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2041 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2042 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2043 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2045 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2046 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2047 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2048 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2050 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2051 &xmm_mask_lo, &xmm_mask_hi,
2052 &xmm_src_lo, &xmm_src_hi);
2053 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2054 &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2055 &xmm_mask_lo, &xmm_mask_hi);
2057 negate_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2059 pix_add_multiply_2x128 (
2060 &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2061 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2062 &xmm_dst_lo, &xmm_dst_hi);
2065 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2079 *pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
2084 static force_inline uint32_t
2085 core_combine_reverse_atop_ca_pixel_sse2 (uint32_t src,
2089 __m128i m = unpack_32_1x128 (mask);
2090 __m128i s = unpack_32_1x128 (src);
2091 __m128i d = unpack_32_1x128 (dst);
2093 __m128i da = negate_1x128 (expand_alpha_1x128 (d));
2094 __m128i sa = expand_alpha_1x128 (s);
2096 s = pix_multiply_1x128 (s, m);
2097 m = pix_multiply_1x128 (m, sa);
2099 return pack_1x128_32 (pix_add_multiply_1x128 (&d, &m, &s, &da));
2103 sse2_combine_atop_reverse_ca (pixman_implementation_t *imp,
2106 const uint32_t * ps,
2107 const uint32_t * pm,
2112 __m128i xmm_src_lo, xmm_src_hi;
2113 __m128i xmm_dst_lo, xmm_dst_hi;
2114 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2115 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2116 __m128i xmm_mask_lo, xmm_mask_hi;
2118 while (w && (unsigned long)pd & 15)
2124 *pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
2130 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2131 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2132 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2134 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2135 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2136 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2138 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2139 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2140 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2141 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2143 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2144 &xmm_mask_lo, &xmm_mask_hi,
2145 &xmm_src_lo, &xmm_src_hi);
2146 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2147 &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2148 &xmm_mask_lo, &xmm_mask_hi);
2150 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
2151 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2153 pix_add_multiply_2x128 (
2154 &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2155 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2156 &xmm_dst_lo, &xmm_dst_hi);
2159 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2173 *pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
2178 static force_inline uint32_t
2179 core_combine_xor_ca_pixel_sse2 (uint32_t src,
2183 __m128i a = unpack_32_1x128 (mask);
2184 __m128i s = unpack_32_1x128 (src);
2185 __m128i d = unpack_32_1x128 (dst);
2187 __m128i alpha_dst = negate_1x128 (pix_multiply_1x128 (
2188 a, expand_alpha_1x128 (s)));
2189 __m128i dest = pix_multiply_1x128 (s, a);
2190 __m128i alpha_src = negate_1x128 (expand_alpha_1x128 (d));
2192 return pack_1x128_32 (pix_add_multiply_1x128 (&d,
2199 sse2_combine_xor_ca (pixman_implementation_t *imp,
2202 const uint32_t * ps,
2203 const uint32_t * pm,
2208 __m128i xmm_src_lo, xmm_src_hi;
2209 __m128i xmm_dst_lo, xmm_dst_hi;
2210 __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2211 __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2212 __m128i xmm_mask_lo, xmm_mask_hi;
2214 while (w && (unsigned long)pd & 15)
2220 *pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
2226 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2227 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2228 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2230 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2231 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2232 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2234 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2235 &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2236 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2237 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2239 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2240 &xmm_mask_lo, &xmm_mask_hi,
2241 &xmm_src_lo, &xmm_src_hi);
2242 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2243 &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2244 &xmm_mask_lo, &xmm_mask_hi);
2246 negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
2247 &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2248 negate_2x128 (xmm_mask_lo, xmm_mask_hi,
2249 &xmm_mask_lo, &xmm_mask_hi);
2251 pix_add_multiply_2x128 (
2252 &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2253 &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2254 &xmm_dst_lo, &xmm_dst_hi);
2257 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2271 *pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
2277 sse2_combine_add_ca (pixman_implementation_t *imp,
2280 const uint32_t * ps,
2281 const uint32_t * pm,
2286 __m128i xmm_src_lo, xmm_src_hi;
2287 __m128i xmm_dst_lo, xmm_dst_hi;
2288 __m128i xmm_mask_lo, xmm_mask_hi;
2290 while (w && (unsigned long)pd & 15)
2296 *pd++ = pack_1x128_32 (
2297 _mm_adds_epu8 (pix_multiply_1x128 (unpack_32_1x128 (s),
2298 unpack_32_1x128 (m)),
2299 unpack_32_1x128 (d)));
2305 xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2306 xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2307 xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2309 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2310 unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2311 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2313 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2314 &xmm_mask_lo, &xmm_mask_hi,
2315 &xmm_src_lo, &xmm_src_hi);
2318 (__m128i*)pd, pack_2x128_128 (
2319 _mm_adds_epu8 (xmm_src_lo, xmm_dst_lo),
2320 _mm_adds_epu8 (xmm_src_hi, xmm_dst_hi)));
2334 *pd++ = pack_1x128_32 (
2335 _mm_adds_epu8 (pix_multiply_1x128 (unpack_32_1x128 (s),
2336 unpack_32_1x128 (m)),
2337 unpack_32_1x128 (d)));
2342 static force_inline __m128i
2343 create_mask_16_128 (uint16_t mask)
2345 return _mm_set1_epi16 (mask);
2348 /* Work around a code generation bug in Sun Studio 12. */
2349 #if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)
2350 # define create_mask_2x32_128(mask0, mask1) \
2351 (_mm_set_epi32 ((mask0), (mask1), (mask0), (mask1)))
2353 static force_inline __m128i
2354 create_mask_2x32_128 (uint32_t mask0,
2357 return _mm_set_epi32 (mask0, mask1, mask0, mask1);
2362 sse2_composite_over_n_8888 (pixman_implementation_t *imp,
2364 pixman_image_t * src_image,
2365 pixman_image_t * mask_image,
2366 pixman_image_t * dst_image,
2377 uint32_t *dst_line, *dst, d;
2380 __m128i xmm_src, xmm_alpha;
2381 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
2383 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
2388 PIXMAN_IMAGE_GET_LINE (
2389 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2391 xmm_src = expand_pixel_32_1x128 (src);
2392 xmm_alpha = expand_alpha_1x128 (xmm_src);
2398 dst_line += dst_stride;
2401 while (w && (unsigned long)dst & 15)
2404 *dst++ = pack_1x128_32 (over_1x128 (xmm_src,
2406 unpack_32_1x128 (d)));
2412 xmm_dst = load_128_aligned ((__m128i*)dst);
2414 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
2416 over_2x128 (&xmm_src, &xmm_src,
2417 &xmm_alpha, &xmm_alpha,
2418 &xmm_dst_lo, &xmm_dst_hi);
2420 /* rebuid the 4 pixel data and save*/
2422 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2431 *dst++ = pack_1x128_32 (over_1x128 (xmm_src,
2433 unpack_32_1x128 (d)));
2441 sse2_composite_over_n_0565 (pixman_implementation_t *imp,
2443 pixman_image_t * src_image,
2444 pixman_image_t * mask_image,
2445 pixman_image_t * dst_image,
2456 uint16_t *dst_line, *dst, d;
2459 __m128i xmm_src, xmm_alpha;
2460 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
2462 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
2467 PIXMAN_IMAGE_GET_LINE (
2468 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
2470 xmm_src = expand_pixel_32_1x128 (src);
2471 xmm_alpha = expand_alpha_1x128 (xmm_src);
2477 dst_line += dst_stride;
2480 while (w && (unsigned long)dst & 15)
2484 *dst++ = pack_565_32_16 (
2485 pack_1x128_32 (over_1x128 (xmm_src,
2487 expand565_16_1x128 (d))));
2493 xmm_dst = load_128_aligned ((__m128i*)dst);
2495 unpack_565_128_4x128 (xmm_dst,
2496 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
2498 over_2x128 (&xmm_src, &xmm_src,
2499 &xmm_alpha, &xmm_alpha,
2500 &xmm_dst0, &xmm_dst1);
2501 over_2x128 (&xmm_src, &xmm_src,
2502 &xmm_alpha, &xmm_alpha,
2503 &xmm_dst2, &xmm_dst3);
2505 xmm_dst = pack_565_4x128_128 (
2506 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
2508 save_128_aligned ((__m128i*)dst, xmm_dst);
2517 *dst++ = pack_565_32_16 (
2518 pack_1x128_32 (over_1x128 (xmm_src, xmm_alpha,
2519 expand565_16_1x128 (d))));
2526 sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
2528 pixman_image_t * src_image,
2529 pixman_image_t * mask_image,
2530 pixman_image_t * dst_image,
2541 uint32_t *dst_line, d;
2542 uint32_t *mask_line, m;
2544 int dst_stride, mask_stride;
2548 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
2550 __m128i mmx_src, mmx_mask, mmx_dest;
2552 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
2557 PIXMAN_IMAGE_GET_LINE (
2558 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2559 PIXMAN_IMAGE_GET_LINE (
2560 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
2562 xmm_src = _mm_unpacklo_epi8 (
2563 create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
2569 const uint32_t *pm = (uint32_t *)mask_line;
2570 uint32_t *pd = (uint32_t *)dst_line;
2572 dst_line += dst_stride;
2573 mask_line += mask_stride;
2575 while (w && (unsigned long)pd & 15)
2583 mmx_mask = unpack_32_1x128 (m);
2584 mmx_dest = unpack_32_1x128 (d);
2586 *pd = pack_1x128_32 (
2587 _mm_adds_epu8 (pix_multiply_1x128 (mmx_mask, mmx_src),
2597 xmm_mask = load_128_unaligned ((__m128i*)pm);
2601 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
2603 /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
2604 if (pack_cmp != 0xffff)
2606 xmm_dst = load_128_aligned ((__m128i*)pd);
2608 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
2610 pix_multiply_2x128 (&xmm_src, &xmm_src,
2611 &xmm_mask_lo, &xmm_mask_hi,
2612 &xmm_mask_lo, &xmm_mask_hi);
2613 xmm_mask_hi = pack_2x128_128 (xmm_mask_lo, xmm_mask_hi);
2616 (__m128i*)pd, _mm_adds_epu8 (xmm_mask_hi, xmm_dst));
2632 mmx_mask = unpack_32_1x128 (m);
2633 mmx_dest = unpack_32_1x128 (d);
2635 *pd = pack_1x128_32 (
2636 _mm_adds_epu8 (pix_multiply_1x128 (mmx_mask, mmx_src),
2648 sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
2650 pixman_image_t * src_image,
2651 pixman_image_t * mask_image,
2652 pixman_image_t * dst_image,
2663 uint32_t *dst_line, d;
2664 uint32_t *mask_line, m;
2666 int dst_stride, mask_stride;
2668 __m128i xmm_src, xmm_alpha;
2669 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
2670 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
2672 __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
2674 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
2679 PIXMAN_IMAGE_GET_LINE (
2680 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2681 PIXMAN_IMAGE_GET_LINE (
2682 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
2684 xmm_src = _mm_unpacklo_epi8 (
2685 create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
2686 xmm_alpha = expand_alpha_1x128 (xmm_src);
2688 mmx_alpha = xmm_alpha;
2693 const uint32_t *pm = (uint32_t *)mask_line;
2694 uint32_t *pd = (uint32_t *)dst_line;
2696 dst_line += dst_stride;
2697 mask_line += mask_stride;
2699 while (w && (unsigned long)pd & 15)
2706 mmx_mask = unpack_32_1x128 (m);
2707 mmx_dest = unpack_32_1x128 (d);
2709 *pd = pack_1x128_32 (in_over_1x128 (&mmx_src,
2721 xmm_mask = load_128_unaligned ((__m128i*)pm);
2725 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
2727 /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
2728 if (pack_cmp != 0xffff)
2730 xmm_dst = load_128_aligned ((__m128i*)pd);
2732 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
2733 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
2735 in_over_2x128 (&xmm_src, &xmm_src,
2736 &xmm_alpha, &xmm_alpha,
2737 &xmm_mask_lo, &xmm_mask_hi,
2738 &xmm_dst_lo, &xmm_dst_hi);
2741 (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2756 mmx_mask = unpack_32_1x128 (m);
2757 mmx_dest = unpack_32_1x128 (d);
2759 *pd = pack_1x128_32 (
2760 in_over_1x128 (&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest));
2771 sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
2773 pixman_image_t * src_image,
2774 pixman_image_t * mask_image,
2775 pixman_image_t * dst_image,
2785 uint32_t *dst_line, *dst;
2786 uint32_t *src_line, *src;
2789 int dst_stride, src_stride;
2792 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
2793 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
2794 __m128i xmm_alpha_lo, xmm_alpha_hi;
2796 PIXMAN_IMAGE_GET_LINE (
2797 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2798 PIXMAN_IMAGE_GET_LINE (
2799 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
2801 mask = _pixman_image_get_solid (imp, mask_image, PIXMAN_a8r8g8b8);
2803 xmm_mask = create_mask_16_128 (mask >> 24);
2808 dst_line += dst_stride;
2810 src_line += src_stride;
2813 while (w && (unsigned long)dst & 15)
2815 uint32_t s = *src++;
2821 __m128i ms = unpack_32_1x128 (s);
2822 __m128i alpha = expand_alpha_1x128 (ms);
2823 __m128i dest = xmm_mask;
2824 __m128i alpha_dst = unpack_32_1x128 (d);
2826 *dst = pack_1x128_32 (
2827 in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
2835 xmm_src = load_128_unaligned ((__m128i*)src);
2837 if (!is_zero (xmm_src))
2839 xmm_dst = load_128_aligned ((__m128i*)dst);
2841 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
2842 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
2843 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2844 &xmm_alpha_lo, &xmm_alpha_hi);
2846 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
2847 &xmm_alpha_lo, &xmm_alpha_hi,
2848 &xmm_mask, &xmm_mask,
2849 &xmm_dst_lo, &xmm_dst_hi);
2852 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2862 uint32_t s = *src++;
2868 __m128i ms = unpack_32_1x128 (s);
2869 __m128i alpha = expand_alpha_1x128 (ms);
2870 __m128i mask = xmm_mask;
2871 __m128i dest = unpack_32_1x128 (d);
2873 *dst = pack_1x128_32 (
2874 in_over_1x128 (&ms, &alpha, &mask, &dest));
2885 sse2_composite_src_x888_8888 (pixman_implementation_t *imp,
2887 pixman_image_t * src_image,
2888 pixman_image_t * mask_image,
2889 pixman_image_t * dst_image,
2899 uint32_t *dst_line, *dst;
2900 uint32_t *src_line, *src;
2902 int dst_stride, src_stride;
2905 PIXMAN_IMAGE_GET_LINE (
2906 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2907 PIXMAN_IMAGE_GET_LINE (
2908 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
2913 dst_line += dst_stride;
2915 src_line += src_stride;
2918 while (w && (unsigned long)dst & 15)
2920 *dst++ = *src++ | 0xff000000;
2926 __m128i xmm_src1, xmm_src2, xmm_src3, xmm_src4;
2928 xmm_src1 = load_128_unaligned ((__m128i*)src + 0);
2929 xmm_src2 = load_128_unaligned ((__m128i*)src + 1);
2930 xmm_src3 = load_128_unaligned ((__m128i*)src + 2);
2931 xmm_src4 = load_128_unaligned ((__m128i*)src + 3);
2933 save_128_aligned ((__m128i*)dst + 0, _mm_or_si128 (xmm_src1, mask_ff000000));
2934 save_128_aligned ((__m128i*)dst + 1, _mm_or_si128 (xmm_src2, mask_ff000000));
2935 save_128_aligned ((__m128i*)dst + 2, _mm_or_si128 (xmm_src3, mask_ff000000));
2936 save_128_aligned ((__m128i*)dst + 3, _mm_or_si128 (xmm_src4, mask_ff000000));
2945 *dst++ = *src++ | 0xff000000;
2953 sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
2955 pixman_image_t * src_image,
2956 pixman_image_t * mask_image,
2957 pixman_image_t * dst_image,
2967 uint32_t *dst_line, *dst;
2968 uint32_t *src_line, *src;
2970 int dst_stride, src_stride;
2973 __m128i xmm_mask, xmm_alpha;
2974 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
2975 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
2977 PIXMAN_IMAGE_GET_LINE (
2978 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2979 PIXMAN_IMAGE_GET_LINE (
2980 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
2982 mask = _pixman_image_get_solid (imp, mask_image, PIXMAN_a8r8g8b8);
2984 xmm_mask = create_mask_16_128 (mask >> 24);
2985 xmm_alpha = mask_00ff;
2990 dst_line += dst_stride;
2992 src_line += src_stride;
2995 while (w && (unsigned long)dst & 15)
2997 uint32_t s = (*src++) | 0xff000000;
3000 __m128i src = unpack_32_1x128 (s);
3001 __m128i alpha = xmm_alpha;
3002 __m128i mask = xmm_mask;
3003 __m128i dest = unpack_32_1x128 (d);
3005 *dst++ = pack_1x128_32 (
3006 in_over_1x128 (&src, &alpha, &mask, &dest));
3013 xmm_src = _mm_or_si128 (
3014 load_128_unaligned ((__m128i*)src), mask_ff000000);
3015 xmm_dst = load_128_aligned ((__m128i*)dst);
3017 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3018 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3020 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
3021 &xmm_alpha, &xmm_alpha,
3022 &xmm_mask, &xmm_mask,
3023 &xmm_dst_lo, &xmm_dst_hi);
3026 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3036 uint32_t s = (*src++) | 0xff000000;
3039 __m128i src = unpack_32_1x128 (s);
3040 __m128i alpha = xmm_alpha;
3041 __m128i mask = xmm_mask;
3042 __m128i dest = unpack_32_1x128 (d);
3044 *dst++ = pack_1x128_32 (
3045 in_over_1x128 (&src, &alpha, &mask, &dest));
3054 sse2_composite_over_8888_8888 (pixman_implementation_t *imp,
3056 pixman_image_t * src_image,
3057 pixman_image_t * mask_image,
3058 pixman_image_t * dst_image,
3068 int dst_stride, src_stride;
3069 uint32_t *dst_line, *dst;
3070 uint32_t *src_line, *src;
3072 PIXMAN_IMAGE_GET_LINE (
3073 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3074 PIXMAN_IMAGE_GET_LINE (
3075 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3082 sse2_combine_over_u (imp, op, dst, src, NULL, width);
3089 static force_inline uint16_t
3090 composite_over_8888_0565pixel (uint32_t src, uint16_t dst)
3094 ms = unpack_32_1x128 (src);
3095 return pack_565_32_16 (
3098 ms, expand_alpha_1x128 (ms), expand565_16_1x128 (dst))));
3102 sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
3104 pixman_image_t * src_image,
3105 pixman_image_t * mask_image,
3106 pixman_image_t * dst_image,
3116 uint16_t *dst_line, *dst, d;
3117 uint32_t *src_line, *src, s;
3118 int dst_stride, src_stride;
3121 __m128i xmm_alpha_lo, xmm_alpha_hi;
3122 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3123 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
3125 PIXMAN_IMAGE_GET_LINE (
3126 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
3127 PIXMAN_IMAGE_GET_LINE (
3128 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3135 dst_line += dst_stride;
3136 src_line += src_stride;
3139 /* Align dst on a 16-byte boundary */
3141 ((unsigned long)dst & 15))
3146 *dst++ = composite_over_8888_0565pixel (s, d);
3150 /* It's a 8 pixel loop */
3153 /* I'm loading unaligned because I'm not sure
3154 * about the address alignment.
3156 xmm_src = load_128_unaligned ((__m128i*) src);
3157 xmm_dst = load_128_aligned ((__m128i*) dst);
3160 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3161 unpack_565_128_4x128 (xmm_dst,
3162 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3163 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3164 &xmm_alpha_lo, &xmm_alpha_hi);
3166 /* I'm loading next 4 pixels from memory
3167 * before to optimze the memory read.
3169 xmm_src = load_128_unaligned ((__m128i*) (src + 4));
3171 over_2x128 (&xmm_src_lo, &xmm_src_hi,
3172 &xmm_alpha_lo, &xmm_alpha_hi,
3173 &xmm_dst0, &xmm_dst1);
3176 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3177 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3178 &xmm_alpha_lo, &xmm_alpha_hi);
3180 over_2x128 (&xmm_src_lo, &xmm_src_hi,
3181 &xmm_alpha_lo, &xmm_alpha_hi,
3182 &xmm_dst2, &xmm_dst3);
3185 (__m128i*)dst, pack_565_4x128_128 (
3186 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
3198 *dst++ = composite_over_8888_0565pixel (s, d);
3205 sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
3207 pixman_image_t * src_image,
3208 pixman_image_t * mask_image,
3209 pixman_image_t * dst_image,
3220 uint32_t *dst_line, *dst;
3221 uint8_t *mask_line, *mask;
3222 int dst_stride, mask_stride;
3226 __m128i xmm_src, xmm_alpha, xmm_def;
3227 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3228 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3230 __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3232 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
3238 PIXMAN_IMAGE_GET_LINE (
3239 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3240 PIXMAN_IMAGE_GET_LINE (
3241 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
3243 xmm_def = create_mask_2x32_128 (src, src);
3244 xmm_src = expand_pixel_32_1x128 (src);
3245 xmm_alpha = expand_alpha_1x128 (xmm_src);
3247 mmx_alpha = xmm_alpha;
3252 dst_line += dst_stride;
3254 mask_line += mask_stride;
3257 while (w && (unsigned long)dst & 15)
3259 uint8_t m = *mask++;
3264 mmx_mask = expand_pixel_8_1x128 (m);
3265 mmx_dest = unpack_32_1x128 (d);
3267 *dst = pack_1x128_32 (in_over_1x128 (&mmx_src,
3279 m = *((uint32_t*)mask);
3281 if (srca == 0xff && m == 0xffffffff)
3283 save_128_aligned ((__m128i*)dst, xmm_def);
3287 xmm_dst = load_128_aligned ((__m128i*) dst);
3288 xmm_mask = unpack_32_1x128 (m);
3289 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
3292 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3293 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3295 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
3296 &xmm_mask_lo, &xmm_mask_hi);
3298 in_over_2x128 (&xmm_src, &xmm_src,
3299 &xmm_alpha, &xmm_alpha,
3300 &xmm_mask_lo, &xmm_mask_hi,
3301 &xmm_dst_lo, &xmm_dst_hi);
3304 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3314 uint8_t m = *mask++;
3319 mmx_mask = expand_pixel_8_1x128 (m);
3320 mmx_dest = unpack_32_1x128 (d);
3322 *dst = pack_1x128_32 (in_over_1x128 (&mmx_src,
3335 static pixman_bool_t
3336 pixman_fill_sse2 (uint32_t *bits,
3345 uint32_t byte_width;
3355 stride = stride * (int) sizeof (uint32_t) / 1;
3356 byte_line = (uint8_t *)(((uint8_t *)bits) + stride * y + x);
3362 data = (w << 16) | w;
3366 stride = stride * (int) sizeof (uint32_t) / 2;
3367 byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
3368 byte_width = 2 * width;
3371 data = (data & 0xffff) * 0x00010001;
3375 stride = stride * (int) sizeof (uint32_t) / 4;
3376 byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
3377 byte_width = 4 * width;
3385 xmm_def = create_mask_2x32_128 (data, data);
3390 uint8_t *d = byte_line;
3391 byte_line += stride;
3394 while (w >= 1 && ((unsigned long)d & 1))
3396 *(uint8_t *)d = data;
3401 while (w >= 2 && ((unsigned long)d & 3))
3403 *(uint16_t *)d = data;
3408 while (w >= 4 && ((unsigned long)d & 15))
3410 *(uint32_t *)d = data;
3418 save_128_aligned ((__m128i*)(d), xmm_def);
3419 save_128_aligned ((__m128i*)(d + 16), xmm_def);
3420 save_128_aligned ((__m128i*)(d + 32), xmm_def);
3421 save_128_aligned ((__m128i*)(d + 48), xmm_def);
3422 save_128_aligned ((__m128i*)(d + 64), xmm_def);
3423 save_128_aligned ((__m128i*)(d + 80), xmm_def);
3424 save_128_aligned ((__m128i*)(d + 96), xmm_def);
3425 save_128_aligned ((__m128i*)(d + 112), xmm_def);
3433 save_128_aligned ((__m128i*)(d), xmm_def);
3434 save_128_aligned ((__m128i*)(d + 16), xmm_def);
3435 save_128_aligned ((__m128i*)(d + 32), xmm_def);
3436 save_128_aligned ((__m128i*)(d + 48), xmm_def);
3444 save_128_aligned ((__m128i*)(d), xmm_def);
3445 save_128_aligned ((__m128i*)(d + 16), xmm_def);
3453 save_128_aligned ((__m128i*)(d), xmm_def);
3461 *(uint32_t *)d = data;
3469 *(uint16_t *)d = data;
3476 *(uint8_t *)d = data;
3486 sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
3488 pixman_image_t * src_image,
3489 pixman_image_t * mask_image,
3490 pixman_image_t * dst_image,
3501 uint32_t *dst_line, *dst;
3502 uint8_t *mask_line, *mask;
3503 int dst_stride, mask_stride;
3507 __m128i xmm_src, xmm_def;
3508 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3510 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
3515 pixman_fill_sse2 (dst_image->bits.bits, dst_image->bits.rowstride,
3516 PIXMAN_FORMAT_BPP (dst_image->bits.format),
3517 dest_x, dest_y, width, height, 0);
3521 PIXMAN_IMAGE_GET_LINE (
3522 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3523 PIXMAN_IMAGE_GET_LINE (
3524 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
3526 xmm_def = create_mask_2x32_128 (src, src);
3527 xmm_src = expand_pixel_32_1x128 (src);
3532 dst_line += dst_stride;
3534 mask_line += mask_stride;
3537 while (w && (unsigned long)dst & 15)
3539 uint8_t m = *mask++;
3543 *dst = pack_1x128_32 (
3544 pix_multiply_1x128 (xmm_src, expand_pixel_8_1x128 (m)));
3557 m = *((uint32_t*)mask);
3559 if (srca == 0xff && m == 0xffffffff)
3561 save_128_aligned ((__m128i*)dst, xmm_def);
3565 xmm_mask = unpack_32_1x128 (m);
3566 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
3569 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3571 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
3572 &xmm_mask_lo, &xmm_mask_hi);
3574 pix_multiply_2x128 (&xmm_src, &xmm_src,
3575 &xmm_mask_lo, &xmm_mask_hi,
3576 &xmm_mask_lo, &xmm_mask_hi);
3579 (__m128i*)dst, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
3583 save_128_aligned ((__m128i*)dst, _mm_setzero_si128 ());
3593 uint8_t m = *mask++;
3597 *dst = pack_1x128_32 (
3598 pix_multiply_1x128 (
3599 xmm_src, expand_pixel_8_1x128 (m)));
3614 sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
3616 pixman_image_t * src_image,
3617 pixman_image_t * mask_image,
3618 pixman_image_t * dst_image,
3629 uint16_t *dst_line, *dst, d;
3630 uint8_t *mask_line, *mask;
3631 int dst_stride, mask_stride;
3634 __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3636 __m128i xmm_src, xmm_alpha;
3637 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3638 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
3640 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
3645 PIXMAN_IMAGE_GET_LINE (
3646 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
3647 PIXMAN_IMAGE_GET_LINE (
3648 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
3650 xmm_src = expand_pixel_32_1x128 (src);
3651 xmm_alpha = expand_alpha_1x128 (xmm_src);
3653 mmx_alpha = xmm_alpha;
3658 dst_line += dst_stride;
3660 mask_line += mask_stride;
3663 while (w && (unsigned long)dst & 15)
3670 mmx_mask = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
3671 mmx_dest = expand565_16_1x128 (d);
3673 *dst = pack_565_32_16 (
3676 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
3685 xmm_dst = load_128_aligned ((__m128i*) dst);
3686 unpack_565_128_4x128 (xmm_dst,
3687 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3689 m = *((uint32_t*)mask);
3694 xmm_mask = unpack_32_1x128 (m);
3695 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
3698 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3700 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
3701 &xmm_mask_lo, &xmm_mask_hi);
3703 in_over_2x128 (&xmm_src, &xmm_src,
3704 &xmm_alpha, &xmm_alpha,
3705 &xmm_mask_lo, &xmm_mask_hi,
3706 &xmm_dst0, &xmm_dst1);
3709 m = *((uint32_t*)mask);
3714 xmm_mask = unpack_32_1x128 (m);
3715 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
3718 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3720 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
3721 &xmm_mask_lo, &xmm_mask_hi);
3722 in_over_2x128 (&xmm_src, &xmm_src,
3723 &xmm_alpha, &xmm_alpha,
3724 &xmm_mask_lo, &xmm_mask_hi,
3725 &xmm_dst2, &xmm_dst3);
3729 (__m128i*)dst, pack_565_4x128_128 (
3730 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
3743 mmx_mask = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
3744 mmx_dest = expand565_16_1x128 (d);
3746 *dst = pack_565_32_16 (
3749 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
3760 sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
3762 pixman_image_t * src_image,
3763 pixman_image_t * mask_image,
3764 pixman_image_t * dst_image,
3774 uint16_t *dst_line, *dst, d;
3775 uint32_t *src_line, *src, s;
3776 int dst_stride, src_stride;
3778 uint32_t opaque, zero;
3781 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3782 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
3784 PIXMAN_IMAGE_GET_LINE (
3785 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
3786 PIXMAN_IMAGE_GET_LINE (
3787 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3792 dst_line += dst_stride;
3794 src_line += src_stride;
3797 while (w && (unsigned long)dst & 15)
3802 ms = unpack_32_1x128 (s);
3804 *dst++ = pack_565_32_16 (
3806 over_rev_non_pre_1x128 (ms, expand565_16_1x128 (d))));
3813 xmm_src = load_128_unaligned ((__m128i*)src);
3814 xmm_dst = load_128_aligned ((__m128i*)dst);
3816 opaque = is_opaque (xmm_src);
3817 zero = is_zero (xmm_src);
3819 unpack_565_128_4x128 (xmm_dst,
3820 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3821 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3823 /* preload next round*/
3824 xmm_src = load_128_unaligned ((__m128i*)(src + 4));
3828 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
3829 &xmm_dst0, &xmm_dst1);
3833 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
3834 &xmm_dst0, &xmm_dst1);
3838 opaque = is_opaque (xmm_src);
3839 zero = is_zero (xmm_src);
3841 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3845 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
3846 &xmm_dst2, &xmm_dst3);
3850 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
3851 &xmm_dst2, &xmm_dst3);
3855 (__m128i*)dst, pack_565_4x128_128 (
3856 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
3868 ms = unpack_32_1x128 (s);
3870 *dst++ = pack_565_32_16 (
3872 over_rev_non_pre_1x128 (ms, expand565_16_1x128 (d))));
3880 sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
3882 pixman_image_t * src_image,
3883 pixman_image_t * mask_image,
3884 pixman_image_t * dst_image,
3894 uint32_t *dst_line, *dst, d;
3895 uint32_t *src_line, *src, s;
3896 int dst_stride, src_stride;
3898 uint32_t opaque, zero;
3900 __m128i xmm_src_lo, xmm_src_hi;
3901 __m128i xmm_dst_lo, xmm_dst_hi;
3903 PIXMAN_IMAGE_GET_LINE (
3904 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3905 PIXMAN_IMAGE_GET_LINE (
3906 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3911 dst_line += dst_stride;
3913 src_line += src_stride;
3916 while (w && (unsigned long)dst & 15)
3921 *dst++ = pack_1x128_32 (
3922 over_rev_non_pre_1x128 (
3923 unpack_32_1x128 (s), unpack_32_1x128 (d)));
3930 xmm_src_hi = load_128_unaligned ((__m128i*)src);
3932 opaque = is_opaque (xmm_src_hi);
3933 zero = is_zero (xmm_src_hi);
3935 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
3939 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
3940 &xmm_dst_lo, &xmm_dst_hi);
3943 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3947 xmm_dst_hi = load_128_aligned ((__m128i*)dst);
3949 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
3951 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
3952 &xmm_dst_lo, &xmm_dst_hi);
3955 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3968 *dst++ = pack_1x128_32 (
3969 over_rev_non_pre_1x128 (
3970 unpack_32_1x128 (s), unpack_32_1x128 (d)));
3979 sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
3981 pixman_image_t * src_image,
3982 pixman_image_t * mask_image,
3983 pixman_image_t * dst_image,
3994 uint16_t *dst_line, *dst, d;
3995 uint32_t *mask_line, *mask, m;
3996 int dst_stride, mask_stride;
4000 __m128i xmm_src, xmm_alpha;
4001 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4002 __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
4004 __m128i mmx_src, mmx_alpha, mmx_mask, mmx_dest;
4006 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
4011 PIXMAN_IMAGE_GET_LINE (
4012 dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
4013 PIXMAN_IMAGE_GET_LINE (
4014 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
4016 xmm_src = expand_pixel_32_1x128 (src);
4017 xmm_alpha = expand_alpha_1x128 (xmm_src);
4019 mmx_alpha = xmm_alpha;
4026 mask_line += mask_stride;
4027 dst_line += dst_stride;
4029 while (w && ((unsigned long)dst & 15))
4031 m = *(uint32_t *) mask;
4036 mmx_mask = unpack_32_1x128 (m);
4037 mmx_dest = expand565_16_1x128 (d);
4039 *dst = pack_565_32_16 (
4042 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4053 xmm_mask = load_128_unaligned ((__m128i*)mask);
4054 xmm_dst = load_128_aligned ((__m128i*)dst);
4056 pack_cmp = _mm_movemask_epi8 (
4057 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
4059 unpack_565_128_4x128 (xmm_dst,
4060 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
4061 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4063 /* preload next round */
4064 xmm_mask = load_128_unaligned ((__m128i*)(mask + 4));
4066 /* preload next round */
4067 if (pack_cmp != 0xffff)
4069 in_over_2x128 (&xmm_src, &xmm_src,
4070 &xmm_alpha, &xmm_alpha,
4071 &xmm_mask_lo, &xmm_mask_hi,
4072 &xmm_dst0, &xmm_dst1);
4076 pack_cmp = _mm_movemask_epi8 (
4077 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
4079 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4081 if (pack_cmp != 0xffff)
4083 in_over_2x128 (&xmm_src, &xmm_src,
4084 &xmm_alpha, &xmm_alpha,
4085 &xmm_mask_lo, &xmm_mask_hi,
4086 &xmm_dst2, &xmm_dst3);
4090 (__m128i*)dst, pack_565_4x128_128 (
4091 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
4100 m = *(uint32_t *) mask;
4105 mmx_mask = unpack_32_1x128 (m);
4106 mmx_dest = expand565_16_1x128 (d);
4108 *dst = pack_565_32_16 (
4111 &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4123 sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
4125 pixman_image_t * src_image,
4126 pixman_image_t * mask_image,
4127 pixman_image_t * dst_image,
4137 uint8_t *dst_line, *dst;
4138 uint8_t *mask_line, *mask;
4139 int dst_stride, mask_stride;
4145 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4146 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4148 PIXMAN_IMAGE_GET_LINE (
4149 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4150 PIXMAN_IMAGE_GET_LINE (
4151 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4153 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
4155 xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
4160 dst_line += dst_stride;
4162 mask_line += mask_stride;
4165 while (w && ((unsigned long)dst & 15))
4167 m = (uint32_t) *mask++;
4168 d = (uint32_t) *dst;
4170 *dst++ = (uint8_t) pack_1x128_32 (
4171 pix_multiply_1x128 (
4172 pix_multiply_1x128 (xmm_alpha,
4173 unpack_32_1x128 (m)),
4174 unpack_32_1x128 (d)));
4180 xmm_mask = load_128_unaligned ((__m128i*)mask);
4181 xmm_dst = load_128_aligned ((__m128i*)dst);
4183 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4184 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
4186 pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
4187 &xmm_mask_lo, &xmm_mask_hi,
4188 &xmm_mask_lo, &xmm_mask_hi);
4190 pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
4191 &xmm_dst_lo, &xmm_dst_hi,
4192 &xmm_dst_lo, &xmm_dst_hi);
4195 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4204 m = (uint32_t) *mask++;
4205 d = (uint32_t) *dst;
4207 *dst++ = (uint8_t) pack_1x128_32 (
4208 pix_multiply_1x128 (
4209 pix_multiply_1x128 (
4210 xmm_alpha, unpack_32_1x128 (m)),
4211 unpack_32_1x128 (d)));
4219 sse2_composite_in_n_8 (pixman_implementation_t *imp,
4221 pixman_image_t * src_image,
4222 pixman_image_t * mask_image,
4223 pixman_image_t * dst_image,
4233 uint8_t *dst_line, *dst;
4240 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4242 PIXMAN_IMAGE_GET_LINE (
4243 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4245 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
4247 xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
4256 pixman_fill (dst_image->bits.bits, dst_image->bits.rowstride,
4257 8, dest_x, dest_y, width, height, src);
4265 dst_line += dst_stride;
4268 while (w && ((unsigned long)dst & 15))
4270 d = (uint32_t) *dst;
4272 *dst++ = (uint8_t) pack_1x128_32 (
4273 pix_multiply_1x128 (
4275 unpack_32_1x128 (d)));
4281 xmm_dst = load_128_aligned ((__m128i*)dst);
4283 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
4285 pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
4286 &xmm_dst_lo, &xmm_dst_hi,
4287 &xmm_dst_lo, &xmm_dst_hi);
4290 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4298 d = (uint32_t) *dst;
4300 *dst++ = (uint8_t) pack_1x128_32 (
4301 pix_multiply_1x128 (
4303 unpack_32_1x128 (d)));
4311 sse2_composite_in_8_8 (pixman_implementation_t *imp,
4313 pixman_image_t * src_image,
4314 pixman_image_t * mask_image,
4315 pixman_image_t * dst_image,
4325 uint8_t *dst_line, *dst;
4326 uint8_t *src_line, *src;
4327 int src_stride, dst_stride;
4331 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
4332 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4334 PIXMAN_IMAGE_GET_LINE (
4335 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4336 PIXMAN_IMAGE_GET_LINE (
4337 src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
4342 dst_line += dst_stride;
4344 src_line += src_stride;
4347 while (w && ((unsigned long)dst & 15))
4349 s = (uint32_t) *src++;
4350 d = (uint32_t) *dst;
4352 *dst++ = (uint8_t) pack_1x128_32 (
4353 pix_multiply_1x128 (
4354 unpack_32_1x128 (s), unpack_32_1x128 (d)));
4360 xmm_src = load_128_unaligned ((__m128i*)src);
4361 xmm_dst = load_128_aligned ((__m128i*)dst);
4363 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
4364 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
4366 pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
4367 &xmm_dst_lo, &xmm_dst_hi,
4368 &xmm_dst_lo, &xmm_dst_hi);
4371 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4380 s = (uint32_t) *src++;
4381 d = (uint32_t) *dst;
4383 *dst++ = (uint8_t) pack_1x128_32 (
4384 pix_multiply_1x128 (unpack_32_1x128 (s), unpack_32_1x128 (d)));
4392 sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
4394 pixman_image_t * src_image,
4395 pixman_image_t * mask_image,
4396 pixman_image_t * dst_image,
4406 uint8_t *dst_line, *dst;
4407 uint8_t *mask_line, *mask;
4408 int dst_stride, mask_stride;
4414 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4415 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4417 PIXMAN_IMAGE_GET_LINE (
4418 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4419 PIXMAN_IMAGE_GET_LINE (
4420 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4422 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
4424 xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
4429 dst_line += dst_stride;
4431 mask_line += mask_stride;
4434 while (w && ((unsigned long)dst & 15))
4436 m = (uint32_t) *mask++;
4437 d = (uint32_t) *dst;
4439 *dst++ = (uint8_t) pack_1x128_32 (
4441 pix_multiply_1x128 (
4442 xmm_alpha, unpack_32_1x128 (m)),
4443 unpack_32_1x128 (d)));
4449 xmm_mask = load_128_unaligned ((__m128i*)mask);
4450 xmm_dst = load_128_aligned ((__m128i*)dst);
4452 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4453 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
4455 pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
4456 &xmm_mask_lo, &xmm_mask_hi,
4457 &xmm_mask_lo, &xmm_mask_hi);
4459 xmm_dst_lo = _mm_adds_epu16 (xmm_mask_lo, xmm_dst_lo);
4460 xmm_dst_hi = _mm_adds_epu16 (xmm_mask_hi, xmm_dst_hi);
4463 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4472 m = (uint32_t) *mask++;
4473 d = (uint32_t) *dst;
4475 *dst++ = (uint8_t) pack_1x128_32 (
4477 pix_multiply_1x128 (
4478 xmm_alpha, unpack_32_1x128 (m)),
4479 unpack_32_1x128 (d)));
4488 sse2_composite_add_n_8 (pixman_implementation_t *imp,
4490 pixman_image_t * src_image,
4491 pixman_image_t * mask_image,
4492 pixman_image_t * dst_image,
4502 uint8_t *dst_line, *dst;
4509 PIXMAN_IMAGE_GET_LINE (
4510 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4512 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
4521 pixman_fill (dst_image->bits.bits, dst_image->bits.rowstride,
4522 8, dest_x, dest_y, width, height, 0xff);
4527 src = (src << 24) | (src << 16) | (src << 8) | src;
4528 xmm_src = _mm_set_epi32 (src, src, src, src);
4533 dst_line += dst_stride;
4536 while (w && ((unsigned long)dst & 15))
4538 *dst = (uint8_t)_mm_cvtsi128_si32 (
4541 _mm_cvtsi32_si128 (*dst)));
4550 (__m128i*)dst, _mm_adds_epu8 (xmm_src, load_128_aligned ((__m128i*)dst)));
4558 *dst = (uint8_t)_mm_cvtsi128_si32 (
4561 _mm_cvtsi32_si128 (*dst)));
4571 sse2_composite_add_8_8 (pixman_implementation_t *imp,
4573 pixman_image_t * src_image,
4574 pixman_image_t * mask_image,
4575 pixman_image_t * dst_image,
4585 uint8_t *dst_line, *dst;
4586 uint8_t *src_line, *src;
4587 int dst_stride, src_stride;
4591 PIXMAN_IMAGE_GET_LINE (
4592 src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
4593 PIXMAN_IMAGE_GET_LINE (
4594 dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4601 dst_line += dst_stride;
4602 src_line += src_stride;
4606 while (w && (unsigned long)dst & 3)
4608 t = (*dst) + (*src++);
4609 *dst++ = t | (0 - (t >> 8));
4613 sse2_combine_add_u (imp, op,
4614 (uint32_t*)dst, (uint32_t*)src, NULL, w >> 2);
4624 t = (*dst) + (*src++);
4625 *dst++ = t | (0 - (t >> 8));
4633 sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
4635 pixman_image_t * src_image,
4636 pixman_image_t * mask_image,
4637 pixman_image_t * dst_image,
4647 uint32_t *dst_line, *dst;
4648 uint32_t *src_line, *src;
4649 int dst_stride, src_stride;
4651 PIXMAN_IMAGE_GET_LINE (
4652 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
4653 PIXMAN_IMAGE_GET_LINE (
4654 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
4659 dst_line += dst_stride;
4661 src_line += src_stride;
4663 sse2_combine_add_u (imp, op, dst, src, NULL, width);
4668 static pixman_bool_t
4669 pixman_blt_sse2 (uint32_t *src_bits,
4682 uint8_t * src_bytes;
4683 uint8_t * dst_bytes;
4686 if (src_bpp != dst_bpp)
4691 src_stride = src_stride * (int) sizeof (uint32_t) / 2;
4692 dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
4693 src_bytes =(uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
4694 dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
4695 byte_width = 2 * width;
4699 else if (src_bpp == 32)
4701 src_stride = src_stride * (int) sizeof (uint32_t) / 4;
4702 dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
4703 src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
4704 dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
4705 byte_width = 4 * width;
4717 uint8_t *s = src_bytes;
4718 uint8_t *d = dst_bytes;
4719 src_bytes += src_stride;
4720 dst_bytes += dst_stride;
4723 while (w >= 2 && ((unsigned long)d & 3))
4725 *(uint16_t *)d = *(uint16_t *)s;
4731 while (w >= 4 && ((unsigned long)d & 15))
4733 *(uint32_t *)d = *(uint32_t *)s;
4742 __m128i xmm0, xmm1, xmm2, xmm3;
4744 xmm0 = load_128_unaligned ((__m128i*)(s));
4745 xmm1 = load_128_unaligned ((__m128i*)(s + 16));
4746 xmm2 = load_128_unaligned ((__m128i*)(s + 32));
4747 xmm3 = load_128_unaligned ((__m128i*)(s + 48));
4749 save_128_aligned ((__m128i*)(d), xmm0);
4750 save_128_aligned ((__m128i*)(d + 16), xmm1);
4751 save_128_aligned ((__m128i*)(d + 32), xmm2);
4752 save_128_aligned ((__m128i*)(d + 48), xmm3);
4761 save_128_aligned ((__m128i*)d, load_128_unaligned ((__m128i*)s) );
4770 *(uint32_t *)d = *(uint32_t *)s;
4779 *(uint16_t *)d = *(uint16_t *)s;
4791 sse2_composite_copy_area (pixman_implementation_t *imp,
4793 pixman_image_t * src_image,
4794 pixman_image_t * mask_image,
4795 pixman_image_t * dst_image,
4805 pixman_blt_sse2 (src_image->bits.bits,
4806 dst_image->bits.bits,
4807 src_image->bits.rowstride,
4808 dst_image->bits.rowstride,
4809 PIXMAN_FORMAT_BPP (src_image->bits.format),
4810 PIXMAN_FORMAT_BPP (dst_image->bits.format),
4811 src_x, src_y, dest_x, dest_y, width, height);
4815 sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
4817 pixman_image_t * src_image,
4818 pixman_image_t * mask_image,
4819 pixman_image_t * dst_image,
4829 uint32_t *src, *src_line, s;
4830 uint32_t *dst, *dst_line, d;
4831 uint8_t *mask, *mask_line;
4833 int src_stride, mask_stride, dst_stride;
4837 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
4838 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4839 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4841 PIXMAN_IMAGE_GET_LINE (
4842 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
4843 PIXMAN_IMAGE_GET_LINE (
4844 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4845 PIXMAN_IMAGE_GET_LINE (
4846 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
4851 src_line += src_stride;
4853 dst_line += dst_stride;
4855 mask_line += mask_stride;
4859 while (w && (unsigned long)dst & 15)
4861 s = 0xff000000 | *src++;
4862 m = (uint32_t) *mask++;
4864 ms = unpack_32_1x128 (s);
4868 __m128i ma = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
4869 __m128i md = unpack_32_1x128 (d);
4871 ms = in_over_1x128 (&ms, &mask_00ff, &ma, &md);
4874 *dst++ = pack_1x128_32 (ms);
4880 m = *(uint32_t*) mask;
4881 xmm_src = _mm_or_si128 (
4882 load_128_unaligned ((__m128i*)src), mask_ff000000);
4884 if (m == 0xffffffff)
4886 save_128_aligned ((__m128i*)dst, xmm_src);
4890 xmm_dst = load_128_aligned ((__m128i*)dst);
4892 xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
4894 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
4895 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4896 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
4898 expand_alpha_rev_2x128 (
4899 xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
4901 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
4902 &mask_00ff, &mask_00ff, &xmm_mask_lo, &xmm_mask_hi,
4903 &xmm_dst_lo, &xmm_dst_hi);
4905 save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4916 m = (uint32_t) *mask++;
4920 s = 0xff000000 | *src;
4932 ma = expand_alpha_rev_1x128 (unpack_32_1x128 (m));
4933 md = unpack_32_1x128 (d);
4934 ms = unpack_32_1x128 (s);
4936 *dst = pack_1x128_32 (in_over_1x128 (&ms, &mask_00ff, &ma, &md));
4950 sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
4952 pixman_image_t * src_image,
4953 pixman_image_t * mask_image,
4954 pixman_image_t * dst_image,
4964 uint32_t *src, *src_line, s;
4965 uint32_t *dst, *dst_line, d;
4966 uint8_t *mask, *mask_line;
4968 int src_stride, mask_stride, dst_stride;
4971 __m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
4972 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4973 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4975 PIXMAN_IMAGE_GET_LINE (
4976 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
4977 PIXMAN_IMAGE_GET_LINE (
4978 mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4979 PIXMAN_IMAGE_GET_LINE (
4980 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
4985 src_line += src_stride;
4987 dst_line += dst_stride;
4989 mask_line += mask_stride;
4993 while (w && (unsigned long)dst & 15)
4998 m = (uint32_t) *mask++;
5005 if (sa == 0xff && m == 0xff)
5011 __m128i ms, md, ma, msa;
5013 ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
5014 ms = unpack_32_1x128 (s);
5015 md = unpack_32_1x128 (d);
5017 msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
5019 *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
5029 m = *(uint32_t *) mask;
5033 xmm_src = load_128_unaligned ((__m128i*)src);
5035 if (m == 0xffffffff && is_opaque (xmm_src))
5037 save_128_aligned ((__m128i *)dst, xmm_src);
5041 xmm_dst = load_128_aligned ((__m128i *)dst);
5043 xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
5045 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
5046 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
5047 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5049 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
5050 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
5052 in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
5053 &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
5055 save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5070 m = (uint32_t) *mask++;
5077 if (sa == 0xff && m == 0xff)
5083 __m128i ms, md, ma, msa;
5085 ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
5086 ms = unpack_32_1x128 (s);
5087 md = unpack_32_1x128 (d);
5089 msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
5091 *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
5103 sse2_composite_over_reverse_n_8888 (pixman_implementation_t *imp,
5105 pixman_image_t * src_image,
5106 pixman_image_t * mask_image,
5107 pixman_image_t * dst_image,
5118 uint32_t *dst_line, *dst;
5120 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5121 __m128i xmm_dsta_hi, xmm_dsta_lo;
5125 src = _pixman_image_get_solid (imp, src_image, dst_image->bits.format);
5130 PIXMAN_IMAGE_GET_LINE (
5131 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
5133 xmm_src = expand_pixel_32_1x128 (src);
5139 dst_line += dst_stride;
5142 while (w && (unsigned long)dst & 15)
5146 vd = unpack_32_1x128 (*dst);
5148 *dst = pack_1x128_32 (over_1x128 (vd, expand_alpha_1x128 (vd),
5156 __m128i tmp_lo, tmp_hi;
5158 xmm_dst = load_128_aligned ((__m128i*)dst);
5160 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5161 expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dsta_lo, &xmm_dsta_hi);
5166 over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
5167 &xmm_dsta_lo, &xmm_dsta_hi,
5171 (__m128i*)dst, pack_2x128_128 (tmp_lo, tmp_hi));
5181 vd = unpack_32_1x128 (*dst);
5183 *dst = pack_1x128_32 (over_1x128 (vd, expand_alpha_1x128 (vd),
5194 sse2_composite_over_8888_8888_8888 (pixman_implementation_t *imp,
5196 pixman_image_t * src_image,
5197 pixman_image_t * mask_image,
5198 pixman_image_t * dst_image,
5208 uint32_t *src, *src_line, s;
5209 uint32_t *dst, *dst_line, d;
5210 uint32_t *mask, *mask_line;
5212 int src_stride, mask_stride, dst_stride;
5215 __m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
5216 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5217 __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
5219 PIXMAN_IMAGE_GET_LINE (
5220 dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
5221 PIXMAN_IMAGE_GET_LINE (
5222 mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
5223 PIXMAN_IMAGE_GET_LINE (
5224 src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
5229 src_line += src_stride;
5231 dst_line += dst_stride;
5233 mask_line += mask_stride;
5237 while (w && (unsigned long)dst & 15)
5242 m = (*mask++) >> 24;
5249 if (sa == 0xff && m == 0xff)
5255 __m128i ms, md, ma, msa;
5257 ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
5258 ms = unpack_32_1x128 (s);
5259 md = unpack_32_1x128 (d);
5261 msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
5263 *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
5273 xmm_mask = load_128_unaligned ((__m128i*)mask);
5275 if (!is_transparent (xmm_mask))
5277 xmm_src = load_128_unaligned ((__m128i*)src);
5279 if (is_opaque (xmm_mask) && is_opaque (xmm_src))
5281 save_128_aligned ((__m128i *)dst, xmm_src);
5285 xmm_dst = load_128_aligned ((__m128i *)dst);
5287 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
5288 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
5289 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5291 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
5292 expand_alpha_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
5294 in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
5295 &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
5297 save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5312 m = (*mask++) >> 24;
5319 if (sa == 0xff && m == 0xff)
5325 __m128i ms, md, ma, msa;
5327 ma = expand_alpha_rev_1x128 (load_32_1x128 (m));
5328 ms = unpack_32_1x128 (s);
5329 md = unpack_32_1x128 (d);
5331 msa = expand_alpha_rev_1x128 (load_32_1x128 (sa));
5333 *dst = pack_1x128_32 (in_over_1x128 (&ms, &msa, &ma, &md));
5344 /* A variant of 'sse2_combine_over_u' with minor tweaks */
5345 static force_inline void
5346 scaled_nearest_scanline_sse2_8888_8888_OVER (uint32_t* pd,
5350 pixman_fixed_t unit_x,
5351 pixman_fixed_t max_vx,
5352 pixman_bool_t fully_transparent_src)
5355 const uint32_t* pm = NULL;
5357 __m128i xmm_dst_lo, xmm_dst_hi;
5358 __m128i xmm_src_lo, xmm_src_hi;
5359 __m128i xmm_alpha_lo, xmm_alpha_hi;
5361 if (fully_transparent_src)
5364 /* Align dst on a 16-byte boundary */
5365 while (w && ((unsigned long)pd & 15))
5368 s = combine1 (ps + (vx >> 16), pm);
5371 *pd++ = core_combine_over_u_pixel_sse2 (s, d);
5380 uint32_t tmp1, tmp2, tmp3, tmp4;
5382 tmp1 = ps[vx >> 16];
5384 tmp2 = ps[vx >> 16];
5386 tmp3 = ps[vx >> 16];
5388 tmp4 = ps[vx >> 16];
5391 tmp = _mm_set_epi32 (tmp4, tmp3, tmp2, tmp1);
5393 xmm_src_hi = combine4 ((__m128i*)&tmp, (__m128i*)pm);
5395 if (is_opaque (xmm_src_hi))
5397 save_128_aligned ((__m128i*)pd, xmm_src_hi);
5399 else if (!is_zero (xmm_src_hi))
5401 xmm_dst_hi = load_128_aligned ((__m128i*) pd);
5403 unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
5404 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
5406 expand_alpha_2x128 (
5407 xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
5409 over_2x128 (&xmm_src_lo, &xmm_src_hi,
5410 &xmm_alpha_lo, &xmm_alpha_hi,
5411 &xmm_dst_lo, &xmm_dst_hi);
5413 /* rebuid the 4 pixel data and save*/
5414 save_128_aligned ((__m128i*)pd,
5415 pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5427 s = combine1 (ps + (vx >> 16), pm);
5430 *pd++ = core_combine_over_u_pixel_sse2 (s, d);
5438 FAST_NEAREST_MAINLOOP (sse2_8888_8888_cover_OVER,
5439 scaled_nearest_scanline_sse2_8888_8888_OVER,
5440 uint32_t, uint32_t, COVER)
5441 FAST_NEAREST_MAINLOOP (sse2_8888_8888_none_OVER,
5442 scaled_nearest_scanline_sse2_8888_8888_OVER,
5443 uint32_t, uint32_t, NONE)
5444 FAST_NEAREST_MAINLOOP (sse2_8888_8888_pad_OVER,
5445 scaled_nearest_scanline_sse2_8888_8888_OVER,
5446 uint32_t, uint32_t, PAD)
5448 static force_inline void
5449 scaled_nearest_scanline_sse2_8888_n_8888_OVER (const uint32_t * mask,
5451 const uint32_t * src,
5454 pixman_fixed_t unit_x,
5455 pixman_fixed_t max_vx,
5456 pixman_bool_t zero_src)
5459 __m128i xmm_src, xmm_src_lo, xmm_src_hi;
5460 __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5461 __m128i xmm_alpha_lo, xmm_alpha_hi;
5463 if (zero_src || (*mask >> 24) == 0)
5466 xmm_mask = create_mask_16_128 (*mask >> 24);
5468 while (w && (unsigned long)dst & 15)
5470 uint32_t s = src[pixman_fixed_to_int (vx)];
5477 __m128i ms = unpack_32_1x128 (s);
5478 __m128i alpha = expand_alpha_1x128 (ms);
5479 __m128i dest = xmm_mask;
5480 __m128i alpha_dst = unpack_32_1x128 (d);
5482 *dst = pack_1x128_32 (
5483 in_over_1x128 (&ms, &alpha, &dest, &alpha_dst));
5491 uint32_t tmp1, tmp2, tmp3, tmp4;
5493 tmp1 = src[pixman_fixed_to_int (vx)];
5495 tmp2 = src[pixman_fixed_to_int (vx)];
5497 tmp3 = src[pixman_fixed_to_int (vx)];
5499 tmp4 = src[pixman_fixed_to_int (vx)];
5502 xmm_src = _mm_set_epi32 (tmp4, tmp3, tmp2, tmp1);
5504 if (!is_zero (xmm_src))
5506 xmm_dst = load_128_aligned ((__m128i*)dst);
5508 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
5509 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5510 expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
5511 &xmm_alpha_lo, &xmm_alpha_hi);
5513 in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
5514 &xmm_alpha_lo, &xmm_alpha_hi,
5515 &xmm_mask, &xmm_mask,
5516 &xmm_dst_lo, &xmm_dst_hi);
5519 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5528 uint32_t s = src[pixman_fixed_to_int (vx)];
5535 __m128i ms = unpack_32_1x128 (s);
5536 __m128i alpha = expand_alpha_1x128 (ms);
5537 __m128i mask = xmm_mask;
5538 __m128i dest = unpack_32_1x128 (d);
5540 *dst = pack_1x128_32 (
5541 in_over_1x128 (&ms, &alpha, &mask, &dest));
5550 FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_cover_OVER,
5551 scaled_nearest_scanline_sse2_8888_n_8888_OVER,
5552 uint32_t, uint32_t, uint32_t, COVER, TRUE, TRUE)
5553 FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_pad_OVER,
5554 scaled_nearest_scanline_sse2_8888_n_8888_OVER,
5555 uint32_t, uint32_t, uint32_t, PAD, TRUE, TRUE)
5556 FAST_NEAREST_MAINLOOP_COMMON (sse2_8888_n_8888_none_OVER,
5557 scaled_nearest_scanline_sse2_8888_n_8888_OVER,
5558 uint32_t, uint32_t, uint32_t, NONE, TRUE, TRUE)
5561 bilinear_interpolate_line_sse2 (uint32_t * out,
5562 const uint32_t * top,
5563 const uint32_t * bottom,
5570 const __m128i xmm_wt = _mm_set_epi16 (wt, wt, wt, wt, wt, wt, wt, wt);
5571 const __m128i xmm_wb = _mm_set_epi16 (wb, wb, wb, wb, wb, wb, wb, wb);
5572 const __m128i xmm_xorc = _mm_set_epi16 (0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff);
5573 const __m128i xmm_addc = _mm_set_epi16 (0, 0, 0, 0, 1, 1, 1, 1);
5574 const __m128i xmm_ux = _mm_set_epi16 (ux, ux, ux, ux, ux, ux, ux, ux);
5575 const __m128i xmm_zero = _mm_setzero_si128 ();
5576 __m128i xmm_x = _mm_set_epi16 (x, x, x, x, x, x, x, x);
5577 uint32_t pix1, pix2, pix3, pix4;
5579 #define INTERPOLATE_ONE_PIXEL(pix) \
5581 __m128i xmm_wh, xmm_lo, xmm_hi, a; \
5582 /* fetch 2x2 pixel block into sse2 register */ \
5583 uint32_t tl = top [pixman_fixed_to_int (x)]; \
5584 uint32_t tr = top [pixman_fixed_to_int (x) + 1]; \
5585 uint32_t bl = bottom [pixman_fixed_to_int (x)]; \
5586 uint32_t br = bottom [pixman_fixed_to_int (x) + 1]; \
5587 a = _mm_set_epi32 (tr, tl, br, bl); \
5589 /* vertical interpolation */ \
5590 a = _mm_add_epi16 (_mm_mullo_epi16 (_mm_unpackhi_epi8 (a, xmm_zero), \
5592 _mm_mullo_epi16 (_mm_unpacklo_epi8 (a, xmm_zero), \
5594 /* calculate horizontal weights */ \
5595 xmm_wh = _mm_add_epi16 (xmm_addc, \
5596 _mm_xor_si128 (xmm_xorc, \
5597 _mm_srli_epi16 (xmm_x, 8))); \
5598 xmm_x = _mm_add_epi16 (xmm_x, xmm_ux); \
5599 /* horizontal interpolation */ \
5600 xmm_lo = _mm_mullo_epi16 (a, xmm_wh); \
5601 xmm_hi = _mm_mulhi_epu16 (a, xmm_wh); \
5602 a = _mm_add_epi32 (_mm_unpacklo_epi16 (xmm_lo, xmm_hi), \
5603 _mm_unpackhi_epi16 (xmm_lo, xmm_hi)); \
5604 /* shift and pack the result */ \
5605 a = _mm_srli_epi32 (a, 16); \
5606 a = _mm_packs_epi32 (a, a); \
5607 a = _mm_packus_epi16 (a, a); \
5608 pix = _mm_cvtsi128_si32 (a); \
5611 while ((width -= 4) >= 0)
5613 INTERPOLATE_ONE_PIXEL (pix1);
5614 INTERPOLATE_ONE_PIXEL (pix2);
5615 INTERPOLATE_ONE_PIXEL (pix3);
5616 INTERPOLATE_ONE_PIXEL (pix4);
5624 INTERPOLATE_ONE_PIXEL (pix1);
5625 INTERPOLATE_ONE_PIXEL (pix2);
5631 INTERPOLATE_ONE_PIXEL (pix1);
5635 #undef INTERPOLATE_ONE_PIXEL
5638 static force_inline void
5639 scaled_bilinear_scanline_sse2_8888_8888_SRC (uint32_t * dst,
5640 const uint32_t * mask,
5641 const uint32_t * src_top,
5642 const uint32_t * src_bottom,
5647 pixman_fixed_t unit_x,
5648 pixman_fixed_t max_vx,
5649 pixman_bool_t zero_src)
5651 bilinear_interpolate_line_sse2 (dst, src_top, src_bottom,
5652 wt, wb, vx, unit_x, w);
5655 FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_cover_SRC,
5656 scaled_bilinear_scanline_sse2_8888_8888_SRC,
5657 uint32_t, uint32_t, uint32_t,
5658 COVER, FALSE, FALSE)
5659 FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_pad_SRC,
5660 scaled_bilinear_scanline_sse2_8888_8888_SRC,
5661 uint32_t, uint32_t, uint32_t,
5663 FAST_BILINEAR_MAINLOOP_COMMON (sse2_8888_8888_none_SRC,
5664 scaled_bilinear_scanline_sse2_8888_8888_SRC,
5665 uint32_t, uint32_t, uint32_t,
5668 static const pixman_fast_path_t sse2_fast_paths[] =
5670 /* PIXMAN_OP_OVER */
5671 PIXMAN_STD_FAST_PATH (OVER, solid, a8, r5g6b5, sse2_composite_over_n_8_0565),
5672 PIXMAN_STD_FAST_PATH (OVER, solid, a8, b5g6r5, sse2_composite_over_n_8_0565),
5673 PIXMAN_STD_FAST_PATH (OVER, solid, null, a8r8g8b8, sse2_composite_over_n_8888),
5674 PIXMAN_STD_FAST_PATH (OVER, solid, null, x8r8g8b8, sse2_composite_over_n_8888),
5675 PIXMAN_STD_FAST_PATH (OVER, solid, null, r5g6b5, sse2_composite_over_n_0565),
5676 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, sse2_composite_over_8888_8888),
5677 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, sse2_composite_over_8888_8888),
5678 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, sse2_composite_over_8888_8888),
5679 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, x8b8g8r8, sse2_composite_over_8888_8888),
5680 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, r5g6b5, sse2_composite_over_8888_0565),
5681 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, b5g6r5, sse2_composite_over_8888_0565),
5682 PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8r8g8b8, sse2_composite_over_n_8_8888),
5683 PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, sse2_composite_over_n_8_8888),
5684 PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, sse2_composite_over_n_8_8888),
5685 PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, sse2_composite_over_n_8_8888),
5686 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, sse2_composite_over_8888_8888_8888),
5687 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, x8r8g8b8, sse2_composite_over_8888_8_8888),
5688 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, a8r8g8b8, sse2_composite_over_8888_8_8888),
5689 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, x8b8g8r8, sse2_composite_over_8888_8_8888),
5690 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, a8b8g8r8, sse2_composite_over_8888_8_8888),
5691 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, x8r8g8b8, sse2_composite_over_x888_8_8888),
5692 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, a8r8g8b8, sse2_composite_over_x888_8_8888),
5693 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, x8b8g8r8, sse2_composite_over_x888_8_8888),
5694 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, a8b8g8r8, sse2_composite_over_x888_8_8888),
5695 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, a8r8g8b8, sse2_composite_over_x888_n_8888),
5696 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, x8r8g8b8, sse2_composite_over_x888_n_8888),
5697 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, a8b8g8r8, sse2_composite_over_x888_n_8888),
5698 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, x8b8g8r8, sse2_composite_over_x888_n_8888),
5699 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, a8r8g8b8, sse2_composite_over_8888_n_8888),
5700 PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, x8r8g8b8, sse2_composite_over_8888_n_8888),
5701 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, a8b8g8r8, sse2_composite_over_8888_n_8888),
5702 PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, x8b8g8r8, sse2_composite_over_8888_n_8888),
5703 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, sse2_composite_over_n_8888_8888_ca),
5704 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, sse2_composite_over_n_8888_8888_ca),
5705 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, sse2_composite_over_n_8888_8888_ca),
5706 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, sse2_composite_over_n_8888_8888_ca),
5707 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5, sse2_composite_over_n_8888_0565_ca),
5708 PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5, sse2_composite_over_n_8888_0565_ca),
5709 PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, a8r8g8b8, sse2_composite_over_pixbuf_8888),
5710 PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, x8r8g8b8, sse2_composite_over_pixbuf_8888),
5711 PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, a8b8g8r8, sse2_composite_over_pixbuf_8888),
5712 PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, x8b8g8r8, sse2_composite_over_pixbuf_8888),
5713 PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, r5g6b5, sse2_composite_over_pixbuf_0565),
5714 PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, b5g6r5, sse2_composite_over_pixbuf_0565),
5715 PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
5716 PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
5718 /* PIXMAN_OP_OVER_REVERSE */
5719 PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, sse2_composite_over_reverse_n_8888),
5720 PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, sse2_composite_over_reverse_n_8888),
5723 PIXMAN_STD_FAST_PATH_CA (ADD, solid, a8r8g8b8, a8r8g8b8, sse2_composite_add_n_8888_8888_ca),
5724 PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, sse2_composite_add_8_8),
5725 PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, sse2_composite_add_8888_8888),
5726 PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, sse2_composite_add_8888_8888),
5727 PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, sse2_composite_add_n_8_8),
5728 PIXMAN_STD_FAST_PATH (ADD, solid, null, a8, sse2_composite_add_n_8),
5731 PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8r8g8b8, sse2_composite_src_n_8_8888),
5732 PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8r8g8b8, sse2_composite_src_n_8_8888),
5733 PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8b8g8r8, sse2_composite_src_n_8_8888),
5734 PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8b8g8r8, sse2_composite_src_n_8_8888),
5735 PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, a8r8g8b8, sse2_composite_src_x888_8888),
5736 PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, a8b8g8r8, sse2_composite_src_x888_8888),
5737 PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, sse2_composite_copy_area),
5738 PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, sse2_composite_copy_area),
5739 PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
5740 PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
5741 PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
5742 PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
5743 PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, r5g6b5, sse2_composite_copy_area),
5744 PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, b5g6r5, sse2_composite_copy_area),
5747 PIXMAN_STD_FAST_PATH (IN, a8, null, a8, sse2_composite_in_8_8),
5748 PIXMAN_STD_FAST_PATH (IN, solid, a8, a8, sse2_composite_in_n_8_8),
5749 PIXMAN_STD_FAST_PATH (IN, solid, null, a8, sse2_composite_in_n_8),
5751 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
5752 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
5753 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
5754 SIMPLE_NEAREST_FAST_PATH_COVER (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
5755 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
5756 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
5757 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
5758 SIMPLE_NEAREST_FAST_PATH_NONE (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
5759 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
5760 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_8888),
5761 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
5762 SIMPLE_NEAREST_FAST_PATH_PAD (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_8888),
5764 SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, sse2_8888_n_8888),
5765 SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, sse2_8888_n_8888),
5766 SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, sse2_8888_n_8888),
5767 SIMPLE_NEAREST_SOLID_MASK_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, sse2_8888_n_8888),
5769 SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, sse2_8888_8888),
5770 SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, sse2_8888_8888),
5771 SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, sse2_8888_8888),
5776 static pixman_bool_t
5777 sse2_blt (pixman_implementation_t *imp,
5778 uint32_t * src_bits,
5779 uint32_t * dst_bits,
5791 if (!pixman_blt_sse2 (
5792 src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
5793 src_x, src_y, dst_x, dst_y, width, height))
5796 return _pixman_implementation_blt (
5798 src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
5799 src_x, src_y, dst_x, dst_y, width, height);
5805 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
5806 __attribute__((__force_align_arg_pointer__))
5808 static pixman_bool_t
5809 sse2_fill (pixman_implementation_t *imp,
5819 if (!pixman_fill_sse2 (bits, stride, bpp, x, y, width, height, xor))
5821 return _pixman_implementation_fill (
5822 imp->delegate, bits, stride, bpp, x, y, width, height, xor);
5829 sse2_fetch_x8r8g8b8 (pixman_iter_t *iter, const uint32_t *mask)
5831 int w = iter->width;
5832 __m128i ff000000 = mask_ff000000;
5833 uint32_t *dst = iter->buffer;
5834 uint32_t *src = (uint32_t *)iter->bits;
5836 iter->bits += iter->stride;
5838 while (w && ((unsigned long)dst) & 0x0f)
5840 *dst++ = (*src++) | 0xff000000;
5847 (__m128i *)dst, _mm_or_si128 (
5848 load_128_unaligned ((__m128i *)src), ff000000));
5857 *dst++ = (*src++) | 0xff000000;
5861 return iter->buffer;
5865 sse2_fetch_r5g6b5 (pixman_iter_t *iter, const uint32_t *mask)
5867 int w = iter->width;
5868 uint32_t *dst = iter->buffer;
5869 uint16_t *src = (uint16_t *)iter->bits;
5870 __m128i ff000000 = mask_ff000000;
5872 iter->bits += iter->stride;
5874 while (w && ((unsigned long)dst) & 0x0f)
5876 uint16_t s = *src++;
5878 *dst++ = CONVERT_0565_TO_8888 (s);
5886 s = _mm_loadu_si128 ((__m128i *)src);
5888 lo = unpack_565_to_8888 (_mm_unpacklo_epi16 (s, _mm_setzero_si128 ()));
5889 hi = unpack_565_to_8888 (_mm_unpackhi_epi16 (s, _mm_setzero_si128 ()));
5891 save_128_aligned ((__m128i *)(dst + 0), _mm_or_si128 (lo, ff000000));
5892 save_128_aligned ((__m128i *)(dst + 4), _mm_or_si128 (hi, ff000000));
5901 uint16_t s = *src++;
5903 *dst++ = CONVERT_0565_TO_8888 (s);
5907 return iter->buffer;
5911 sse2_fetch_a8 (pixman_iter_t *iter, const uint32_t *mask)
5913 int w = iter->width;
5914 uint32_t *dst = iter->buffer;
5915 uint8_t *src = iter->bits;
5916 __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6;
5918 iter->bits += iter->stride;
5920 while (w && (((unsigned long)dst) & 15))
5922 *dst++ = *(src++) << 24;
5928 xmm0 = _mm_loadu_si128((__m128i *)src);
5930 xmm1 = _mm_unpacklo_epi8 (_mm_setzero_si128(), xmm0);
5931 xmm2 = _mm_unpackhi_epi8 (_mm_setzero_si128(), xmm0);
5932 xmm3 = _mm_unpacklo_epi16 (_mm_setzero_si128(), xmm1);
5933 xmm4 = _mm_unpackhi_epi16 (_mm_setzero_si128(), xmm1);
5934 xmm5 = _mm_unpacklo_epi16 (_mm_setzero_si128(), xmm2);
5935 xmm6 = _mm_unpackhi_epi16 (_mm_setzero_si128(), xmm2);
5937 _mm_store_si128(((__m128i *)(dst + 0)), xmm3);
5938 _mm_store_si128(((__m128i *)(dst + 4)), xmm4);
5939 _mm_store_si128(((__m128i *)(dst + 8)), xmm5);
5940 _mm_store_si128(((__m128i *)(dst + 12)), xmm6);
5949 *dst++ = *(src++) << 24;
5953 return iter->buffer;
5958 pixman_format_code_t format;
5959 pixman_iter_get_scanline_t get_scanline;
5962 static const fetcher_info_t fetchers[] =
5964 { PIXMAN_x8r8g8b8, sse2_fetch_x8r8g8b8 },
5965 { PIXMAN_r5g6b5, sse2_fetch_r5g6b5 },
5966 { PIXMAN_a8, sse2_fetch_a8 },
5971 sse2_src_iter_init (pixman_implementation_t *imp, pixman_iter_t *iter)
5973 pixman_image_t *image = iter->image;
5976 int width = iter->width;
5977 int height = iter->height;
5980 (FAST_PATH_STANDARD_FLAGS | FAST_PATH_ID_TRANSFORM)
5982 if ((iter->flags & ITER_NARROW) &&
5983 (image->common.flags & FLAGS) == FLAGS &&
5985 x + width <= image->bits.width &&
5986 y + height <= image->bits.height)
5988 const fetcher_info_t *f;
5990 for (f = &fetchers[0]; f->format != PIXMAN_null; f++)
5992 if (image->common.extended_format_code == f->format)
5994 uint8_t *b = (uint8_t *)image->bits.bits;
5995 int s = image->bits.rowstride * 4;
5997 iter->bits = b + s * iter->y + x * PIXMAN_FORMAT_BPP (f->format) / 8;
6000 iter->get_scanline = f->get_scanline;
6006 imp->delegate->src_iter_init (imp->delegate, iter);
6009 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
6010 __attribute__((__force_align_arg_pointer__))
6012 pixman_implementation_t *
6013 _pixman_implementation_create_sse2 (pixman_implementation_t *fallback)
6015 pixman_implementation_t *imp = _pixman_implementation_create (fallback, sse2_fast_paths);
6017 /* SSE2 constants */
6018 mask_565_r = create_mask_2x32_128 (0x00f80000, 0x00f80000);
6019 mask_565_g1 = create_mask_2x32_128 (0x00070000, 0x00070000);
6020 mask_565_g2 = create_mask_2x32_128 (0x000000e0, 0x000000e0);
6021 mask_565_b = create_mask_2x32_128 (0x0000001f, 0x0000001f);
6022 mask_red = create_mask_2x32_128 (0x00f80000, 0x00f80000);
6023 mask_green = create_mask_2x32_128 (0x0000fc00, 0x0000fc00);
6024 mask_blue = create_mask_2x32_128 (0x000000f8, 0x000000f8);
6025 mask_565_fix_rb = create_mask_2x32_128 (0x00e000e0, 0x00e000e0);
6026 mask_565_fix_g = create_mask_2x32_128 (0x0000c000, 0x0000c000);
6027 mask_0080 = create_mask_16_128 (0x0080);
6028 mask_00ff = create_mask_16_128 (0x00ff);
6029 mask_0101 = create_mask_16_128 (0x0101);
6030 mask_ffff = create_mask_16_128 (0xffff);
6031 mask_ff000000 = create_mask_2x32_128 (0xff000000, 0xff000000);
6032 mask_alpha = create_mask_2x32_128 (0x00ff0000, 0x00000000);
6034 /* Set up function pointers */
6035 imp->combine_32[PIXMAN_OP_OVER] = sse2_combine_over_u;
6036 imp->combine_32[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_u;
6037 imp->combine_32[PIXMAN_OP_IN] = sse2_combine_in_u;
6038 imp->combine_32[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_u;
6039 imp->combine_32[PIXMAN_OP_OUT] = sse2_combine_out_u;
6040 imp->combine_32[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_u;
6041 imp->combine_32[PIXMAN_OP_ATOP] = sse2_combine_atop_u;
6042 imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_u;
6043 imp->combine_32[PIXMAN_OP_XOR] = sse2_combine_xor_u;
6044 imp->combine_32[PIXMAN_OP_ADD] = sse2_combine_add_u;
6046 imp->combine_32[PIXMAN_OP_SATURATE] = sse2_combine_saturate_u;
6048 imp->combine_32_ca[PIXMAN_OP_SRC] = sse2_combine_src_ca;
6049 imp->combine_32_ca[PIXMAN_OP_OVER] = sse2_combine_over_ca;
6050 imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_ca;
6051 imp->combine_32_ca[PIXMAN_OP_IN] = sse2_combine_in_ca;
6052 imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_ca;
6053 imp->combine_32_ca[PIXMAN_OP_OUT] = sse2_combine_out_ca;
6054 imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_ca;
6055 imp->combine_32_ca[PIXMAN_OP_ATOP] = sse2_combine_atop_ca;
6056 imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_ca;
6057 imp->combine_32_ca[PIXMAN_OP_XOR] = sse2_combine_xor_ca;
6058 imp->combine_32_ca[PIXMAN_OP_ADD] = sse2_combine_add_ca;
6060 imp->blt = sse2_blt;
6061 imp->fill = sse2_fill;
6063 imp->src_iter_init = sse2_src_iter_init;