pixman/pixman-sse2.c

   1 /*
   2  * Copyright © 2008 Rodrigo Kumpera
   3  * Copyright © 2008 André Tupinambá
   4  *
   5  * Permission to use, copy, modify, distribute, and sell this software and its
   6  * documentation for any purpose is hereby granted without fee, provided that
   7  * the above copyright notice appear in all copies and that both that
   8  * copyright notice and this permission notice appear in supporting
   9  * documentation, and that the name of Red Hat not be used in advertising or
  10  * publicity pertaining to distribution of the software without specific,
  11  * written prior permission.  Red Hat makes no representations about the
  12  * suitability of this software for any purpose.  It is provided "as is"
  13  * without express or implied warranty.
  14  *
  15  * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
  16  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  17  * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
  18  * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  19  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  20  * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
  21  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
  22  * SOFTWARE.
  23  *
  24  * Author:  Rodrigo Kumpera (kumpera@gmail.com)
  25  *          André Tupinambá (andrelrt@gmail.com)
  26  *
  27  * Based on work by Owen Taylor and Søren Sandmann
  28  */
  29 #ifdef HAVE_CONFIG_H
  30 #include <config.h>
  31 #endif
  32
  33 #include <mmintrin.h>
  34 #include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */
  35 #include <emmintrin.h> /* for SSE2 intrinsics */
  36 #include "pixman-private.h"
  37 #include "pixman-combine32.h"
  38
  39 #if defined(_MSC_VER) && defined(_M_AMD64)
  40 /* Windows 64 doesn't allow MMX to be used, so
  41  * the pixman-x64-mmx-emulation.h file contains
  42  * implementations of those MMX intrinsics that
  43  * are used in the SSE2 implementation.
  44  */
  45 #   include "pixman-x64-mmx-emulation.h"
  46 #endif
  47
  48 #ifdef USE_SSE2
  49
  50 /* --------------------------------------------------------------------
  51  * Locals
  52  */
  53
  54 static __m64 mask_x0080;
  55 static __m64 mask_x00ff;
  56 static __m64 mask_x0101;
  57 static __m64 mask_x_alpha;
  58
  59 static __m64 mask_x565_rgb;
  60 static __m64 mask_x565_unpack;
  61
  62 static __m128i mask_0080;
  63 static __m128i mask_00ff;
  64 static __m128i mask_0101;
  65 static __m128i mask_ffff;
  66 static __m128i mask_ff000000;
  67 static __m128i mask_alpha;
  68
  69 static __m128i mask_565_r;
  70 static __m128i mask_565_g1, mask_565_g2;
  71 static __m128i mask_565_b;
  72 static __m128i mask_red;
  73 static __m128i mask_green;
  74 static __m128i mask_blue;
  75
  76 static __m128i mask_565_fix_rb;
  77 static __m128i mask_565_fix_g;
  78
  79 /* ----------------------------------------------------------------------
  80  * SSE2 Inlines
  81  */
  82 static force_inline __m128i
  83 unpack_32_1x128 (uint32_t data)
  84 {
  85     return _mm_unpacklo_epi8 (_mm_cvtsi32_si128 (data), _mm_setzero_si128 ());
  86 }
  87
  88 static force_inline void
  89 unpack_128_2x128 (__m128i data, __m128i* data_lo, __m128i* data_hi)
  90 {
  91     *data_lo = _mm_unpacklo_epi8 (data, _mm_setzero_si128 ());
  92     *data_hi = _mm_unpackhi_epi8 (data, _mm_setzero_si128 ());
  93 }
  94
  95 static force_inline __m128i
  96 unpack_565_to_8888 (__m128i lo)
  97 {
  98     __m128i r, g, b, rb, t;
  99
 100     r = _mm_and_si128 (_mm_slli_epi32 (lo, 8), mask_red);
 101     g = _mm_and_si128 (_mm_slli_epi32 (lo, 5), mask_green);
 102     b = _mm_and_si128 (_mm_slli_epi32 (lo, 3), mask_blue);
 103
 104     rb = _mm_or_si128 (r, b);
 105     t  = _mm_and_si128 (rb, mask_565_fix_rb);
 106     t  = _mm_srli_epi32 (t, 5);
 107     rb = _mm_or_si128 (rb, t);
 108
 109     t  = _mm_and_si128 (g, mask_565_fix_g);
 110     t  = _mm_srli_epi32 (t, 6);
 111     g  = _mm_or_si128 (g, t);
 112
 113     return _mm_or_si128 (rb, g);
 114 }
 115
 116 static force_inline void
 117 unpack_565_128_4x128 (__m128i  data,
 118                       __m128i* data0,
 119                       __m128i* data1,
 120                       __m128i* data2,
 121                       __m128i* data3)
 122 {
 123     __m128i lo, hi;
 124
 125     lo = _mm_unpacklo_epi16 (data, _mm_setzero_si128 ());
 126     hi = _mm_unpackhi_epi16 (data, _mm_setzero_si128 ());
 127
 128     lo = unpack_565_to_8888 (lo);
 129     hi = unpack_565_to_8888 (hi);
 130
 131     unpack_128_2x128 (lo, data0, data1);
 132     unpack_128_2x128 (hi, data2, data3);
 133 }
 134
 135 static force_inline uint16_t
 136 pack_565_32_16 (uint32_t pixel)
 137 {
 138     return (uint16_t) (((pixel >> 8) & 0xf800) |
 139                        ((pixel >> 5) & 0x07e0) |
 140                        ((pixel >> 3) & 0x001f));
 141 }
 142
 143 static force_inline __m128i
 144 pack_2x128_128 (__m128i lo, __m128i hi)
 145 {
 146     return _mm_packus_epi16 (lo, hi);
 147 }
 148
 149 static force_inline __m128i
 150 pack_565_2x128_128 (__m128i lo, __m128i hi)
 151 {
 152     __m128i data;
 153     __m128i r, g1, g2, b;
 154
 155     data = pack_2x128_128 (lo, hi);
 156
 157     r  = _mm_and_si128 (data, mask_565_r);
 158     g1 = _mm_and_si128 (_mm_slli_epi32 (data, 3), mask_565_g1);
 159     g2 = _mm_and_si128 (_mm_srli_epi32 (data, 5), mask_565_g2);
 160     b  = _mm_and_si128 (_mm_srli_epi32 (data, 3), mask_565_b);
 161
 162     return _mm_or_si128 (_mm_or_si128 (_mm_or_si128 (r, g1), g2), b);
 163 }
 164
 165 static force_inline __m128i
 166 pack_565_4x128_128 (__m128i* xmm0, __m128i* xmm1, __m128i* xmm2, __m128i* xmm3)
 167 {
 168     return _mm_packus_epi16 (pack_565_2x128_128 (*xmm0, *xmm1),
 169                              pack_565_2x128_128 (*xmm2, *xmm3));
 170 }
 171
 172 static force_inline int
 173 is_opaque (__m128i x)
 174 {
 175     __m128i ffs = _mm_cmpeq_epi8 (x, x);
 176
 177     return (_mm_movemask_epi8 (_mm_cmpeq_epi8 (x, ffs)) & 0x8888) == 0x8888;
 178 }
 179
 180 static force_inline int
 181 is_zero (__m128i x)
 182 {
 183     return _mm_movemask_epi8 (
 184         _mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) == 0xffff;
 185 }
 186
 187 static force_inline int
 188 is_transparent (__m128i x)
 189 {
 190     return (_mm_movemask_epi8 (
 191                 _mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) & 0x8888) == 0x8888;
 192 }
 193
 194 static force_inline __m128i
 195 expand_pixel_32_1x128 (uint32_t data)
 196 {
 197     return _mm_shuffle_epi32 (unpack_32_1x128 (data), _MM_SHUFFLE (1, 0, 1, 0));
 198 }
 199
 200 static force_inline __m128i
 201 expand_alpha_1x128 (__m128i data)
 202 {
 203     return _mm_shufflehi_epi16 (_mm_shufflelo_epi16 (data,
 204                                                      _MM_SHUFFLE (3, 3, 3, 3)),
 205                                 _MM_SHUFFLE (3, 3, 3, 3));
 206 }
 207
 208 static force_inline void
 209 expand_alpha_2x128 (__m128i  data_lo,
 210                     __m128i  data_hi,
 211                     __m128i* alpha_lo,
 212                     __m128i* alpha_hi)
 213 {
 214     __m128i lo, hi;
 215
 216     lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 3, 3, 3));
 217     hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 3, 3, 3));
 218
 219     *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 3, 3, 3));
 220     *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 3, 3, 3));
 221 }
 222
 223 static force_inline void
 224 expand_alpha_rev_2x128 (__m128i  data_lo,
 225                         __m128i  data_hi,
 226                         __m128i* alpha_lo,
 227                         __m128i* alpha_hi)
 228 {
 229     __m128i lo, hi;
 230
 231     lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (0, 0, 0, 0));
 232     hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (0, 0, 0, 0));
 233     *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (0, 0, 0, 0));
 234     *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (0, 0, 0, 0));
 235 }
 236
 237 static force_inline void
 238 pix_multiply_2x128 (__m128i* data_lo,
 239                     __m128i* data_hi,
 240                     __m128i* alpha_lo,
 241                     __m128i* alpha_hi,
 242                     __m128i* ret_lo,
 243                     __m128i* ret_hi)
 244 {
 245     __m128i lo, hi;
 246
 247     lo = _mm_mullo_epi16 (*data_lo, *alpha_lo);
 248     hi = _mm_mullo_epi16 (*data_hi, *alpha_hi);
 249     lo = _mm_adds_epu16 (lo, mask_0080);
 250     hi = _mm_adds_epu16 (hi, mask_0080);
 251     *ret_lo = _mm_mulhi_epu16 (lo, mask_0101);
 252     *ret_hi = _mm_mulhi_epu16 (hi, mask_0101);
 253 }
 254
 255 static force_inline void
 256 pix_add_multiply_2x128 (__m128i* src_lo,
 257                         __m128i* src_hi,
 258                         __m128i* alpha_dst_lo,
 259                         __m128i* alpha_dst_hi,
 260                         __m128i* dst_lo,
 261                         __m128i* dst_hi,
 262                         __m128i* alpha_src_lo,
 263                         __m128i* alpha_src_hi,
 264                         __m128i* ret_lo,
 265                         __m128i* ret_hi)
 266 {
 267     __m128i t1_lo, t1_hi;
 268     __m128i t2_lo, t2_hi;
 269
 270     pix_multiply_2x128 (src_lo, src_hi, alpha_dst_lo, alpha_dst_hi, &t1_lo, &t1_hi);
 271     pix_multiply_2x128 (dst_lo, dst_hi, alpha_src_lo, alpha_src_hi, &t2_lo, &t2_hi);
 272
 273     *ret_lo = _mm_adds_epu8 (t1_lo, t2_lo);
 274     *ret_hi = _mm_adds_epu8 (t1_hi, t2_hi);
 275 }
 276
 277 static force_inline void
 278 negate_2x128 (__m128i  data_lo,
 279               __m128i  data_hi,
 280               __m128i* neg_lo,
 281               __m128i* neg_hi)
 282 {
 283     *neg_lo = _mm_xor_si128 (data_lo, mask_00ff);
 284     *neg_hi = _mm_xor_si128 (data_hi, mask_00ff);
 285 }
 286
 287 static force_inline void
 288 invert_colors_2x128 (__m128i  data_lo,
 289                      __m128i  data_hi,
 290                      __m128i* inv_lo,
 291                      __m128i* inv_hi)
 292 {
 293     __m128i lo, hi;
 294
 295     lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 0, 1, 2));
 296     hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 0, 1, 2));
 297     *inv_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 0, 1, 2));
 298     *inv_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 0, 1, 2));
 299 }
 300
 301 static force_inline void
 302 over_2x128 (__m128i* src_lo,
 303             __m128i* src_hi,
 304             __m128i* alpha_lo,
 305             __m128i* alpha_hi,
 306             __m128i* dst_lo,
 307             __m128i* dst_hi)
 308 {
 309     __m128i t1, t2;
 310
 311     negate_2x128 (*alpha_lo, *alpha_hi, &t1, &t2);
 312
 313     pix_multiply_2x128 (dst_lo, dst_hi, &t1, &t2, dst_lo, dst_hi);
 314
 315     *dst_lo = _mm_adds_epu8 (*src_lo, *dst_lo);
 316     *dst_hi = _mm_adds_epu8 (*src_hi, *dst_hi);
 317 }
 318
 319 static force_inline void
 320 over_rev_non_pre_2x128 (__m128i  src_lo,
 321                         __m128i  src_hi,
 322                         __m128i* dst_lo,
 323                         __m128i* dst_hi)
 324 {
 325     __m128i lo, hi;
 326     __m128i alpha_lo, alpha_hi;
 327
 328     expand_alpha_2x128 (src_lo, src_hi, &alpha_lo, &alpha_hi);
 329
 330     lo = _mm_or_si128 (alpha_lo, mask_alpha);
 331     hi = _mm_or_si128 (alpha_hi, mask_alpha);
 332
 333     invert_colors_2x128 (src_lo, src_hi, &src_lo, &src_hi);
 334
 335     pix_multiply_2x128 (&src_lo, &src_hi, &lo, &hi, &lo, &hi);
 336
 337     over_2x128 (&lo, &hi, &alpha_lo, &alpha_hi, dst_lo, dst_hi);
 338 }
 339
 340 static force_inline void
 341 in_over_2x128 (__m128i* src_lo,
 342                __m128i* src_hi,
 343                __m128i* alpha_lo,
 344                __m128i* alpha_hi,
 345                __m128i* mask_lo,
 346                __m128i* mask_hi,
 347                __m128i* dst_lo,
 348                __m128i* dst_hi)
 349 {
 350     __m128i s_lo, s_hi;
 351     __m128i a_lo, a_hi;
 352
 353     pix_multiply_2x128 (src_lo,   src_hi, mask_lo, mask_hi, &s_lo, &s_hi);
 354     pix_multiply_2x128 (alpha_lo, alpha_hi, mask_lo, mask_hi, &a_lo, &a_hi);
 355
 356     over_2x128 (&s_lo, &s_hi, &a_lo, &a_hi, dst_lo, dst_hi);
 357 }
 358
 359 static force_inline void
 360 cache_prefetch (__m128i* addr)
 361 {
 362     _mm_prefetch (addr, _MM_HINT_T0);
 363 }
 364
 365 static force_inline void
 366 cache_prefetch_next (__m128i* addr)
 367 {
 368     _mm_prefetch (addr + 4, _MM_HINT_T0); /* 64 bytes ahead */
 369 }
 370
 371 /* load 4 pixels from a 16-byte boundary aligned address */
 372 static force_inline __m128i
 373 load_128_aligned (__m128i* src)
 374 {
 375     return _mm_load_si128 (src);
 376 }
 377
 378 /* load 4 pixels from a unaligned address */
 379 static force_inline __m128i
 380 load_128_unaligned (const __m128i* src)
 381 {
 382     return _mm_loadu_si128 (src);
 383 }
 384
 385 /* save 4 pixels using Write Combining memory on a 16-byte
 386  * boundary aligned address
 387  */
 388 static force_inline void
 389 save_128_write_combining (__m128i* dst,
 390                           __m128i  data)
 391 {
 392     _mm_stream_si128 (dst, data);
 393 }
 394
 395 /* save 4 pixels on a 16-byte boundary aligned address */
 396 static force_inline void
 397 save_128_aligned (__m128i* dst,
 398                   __m128i  data)
 399 {
 400     _mm_store_si128 (dst, data);
 401 }
 402
 403 /* save 4 pixels on a unaligned address */
 404 static force_inline void
 405 save_128_unaligned (__m128i* dst,
 406                     __m128i  data)
 407 {
 408     _mm_storeu_si128 (dst, data);
 409 }
 410
 411 /* ------------------------------------------------------------------
 412  * MMX inlines
 413  */
 414
 415 static force_inline __m64
 416 unpack_32_1x64 (uint32_t data)
 417 {
 418     return _mm_unpacklo_pi8 (_mm_cvtsi32_si64 (data), _mm_setzero_si64 ());
 419 }
 420
 421 static force_inline __m64
 422 expand_alpha_1x64 (__m64 data)
 423 {
 424     return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 3, 3, 3));
 425 }
 426
 427 static force_inline __m64
 428 expand_alpha_rev_1x64 (__m64 data)
 429 {
 430     return _mm_shuffle_pi16 (data, _MM_SHUFFLE (0, 0, 0, 0));
 431 }
 432
 433 static force_inline __m64
 434 expand_pixel_8_1x64 (uint8_t data)
 435 {
 436     return _mm_shuffle_pi16 (
 437         unpack_32_1x64 ((uint32_t)data), _MM_SHUFFLE (0, 0, 0, 0));
 438 }
 439
 440 static force_inline __m64
 441 pix_multiply_1x64 (__m64 data,
 442                    __m64 alpha)
 443 {
 444     return _mm_mulhi_pu16 (_mm_adds_pu16 (_mm_mullo_pi16 (data, alpha),
 445                                           mask_x0080),
 446                            mask_x0101);
 447 }
 448
 449 static force_inline __m64
 450 pix_add_multiply_1x64 (__m64* src,
 451                        __m64* alpha_dst,
 452                        __m64* dst,
 453                        __m64* alpha_src)
 454 {
 455     __m64 t1 = pix_multiply_1x64 (*src, *alpha_dst);
 456     __m64 t2 = pix_multiply_1x64 (*dst, *alpha_src);
 457
 458     return _mm_adds_pu8 (t1, t2);
 459 }
 460
 461 static force_inline __m64
 462 negate_1x64 (__m64 data)
 463 {
 464     return _mm_xor_si64 (data, mask_x00ff);
 465 }
 466
 467 static force_inline __m64
 468 invert_colors_1x64 (__m64 data)
 469 {
 470     return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 0, 1, 2));
 471 }
 472
 473 static force_inline __m64
 474 over_1x64 (__m64 src, __m64 alpha, __m64 dst)
 475 {
 476     return _mm_adds_pu8 (src, pix_multiply_1x64 (dst, negate_1x64 (alpha)));
 477 }
 478
 479 static force_inline __m64
 480 in_over_1x64 (__m64* src, __m64* alpha, __m64* mask, __m64* dst)
 481 {
 482     return over_1x64 (pix_multiply_1x64 (*src, *mask),
 483                       pix_multiply_1x64 (*alpha, *mask),
 484                       *dst);
 485 }
 486
 487 static force_inline __m64
 488 over_rev_non_pre_1x64 (__m64 src, __m64 dst)
 489 {
 490     __m64 alpha = expand_alpha_1x64 (src);
 491
 492     return over_1x64 (pix_multiply_1x64 (invert_colors_1x64 (src),
 493                                          _mm_or_si64 (alpha, mask_x_alpha)),
 494                       alpha,
 495                       dst);
 496 }
 497
 498 static force_inline uint32_t
 499 pack_1x64_32 (__m64 data)
 500 {
 501     return _mm_cvtsi64_si32 (_mm_packs_pu16 (data, _mm_setzero_si64 ()));
 502 }
 503
 504 /* Expand 16 bits positioned at @pos (0-3) of a mmx register into
 505  *
 506  *    00RR00GG00BB
 507  *
 508  * --- Expanding 565 in the low word ---
 509  *
 510  * m = (m << (32 - 3)) | (m << (16 - 5)) | m;
 511  * m = m & (01f0003f001f);
 512  * m = m * (008404100840);
 513  * m = m >> 8;
 514  *
 515  * Note the trick here - the top word is shifted by another nibble to
 516  * avoid it bumping into the middle word
 517  */
 518 static force_inline __m64
 519 expand565_16_1x64 (uint16_t pixel)
 520 {
 521     __m64 p;
 522     __m64 t1, t2;
 523
 524     p = _mm_cvtsi32_si64 ((uint32_t) pixel);
 525
 526     t1 = _mm_slli_si64 (p, 36 - 11);
 527     t2 = _mm_slli_si64 (p, 16 - 5);
 528
 529     p = _mm_or_si64 (t1, p);
 530     p = _mm_or_si64 (t2, p);
 531     p = _mm_and_si64 (p, mask_x565_rgb);
 532     p = _mm_mullo_pi16 (p, mask_x565_unpack);
 533
 534     return _mm_srli_pi16 (p, 8);
 535 }
 536
 537 /* ----------------------------------------------------------------------------
 538  * Compose Core transformations
 539  */
 540 static force_inline uint32_t
 541 core_combine_over_u_pixel_sse2 (uint32_t src, uint32_t dst)
 542 {
 543     uint8_t a;
 544     __m64 ms;
 545
 546     a = src >> 24;
 547
 548     if (a == 0xff)
 549     {
 550         return src;
 551     }
 552     else if (src)
 553     {
 554         ms = unpack_32_1x64 (src);
 555         return pack_1x64_32 (
 556             over_1x64 (ms, expand_alpha_1x64 (ms), unpack_32_1x64 (dst)));
 557     }
 558
 559     return dst;
 560 }
 561
 562 static force_inline uint32_t
 563 combine1 (const uint32_t *ps, const uint32_t *pm)
 564 {
 565     uint32_t s = *ps;
 566
 567     if (pm)
 568     {
 569         __m64 ms, mm;
 570
 571         mm = unpack_32_1x64 (*pm);
 572         mm = expand_alpha_1x64 (mm);
 573
 574         ms = unpack_32_1x64 (s);
 575         ms = pix_multiply_1x64 (ms, mm);
 576
 577         s = pack_1x64_32 (ms);
 578     }
 579
 580     return s;
 581 }
 582
 583 static force_inline __m128i
 584 combine4 (const __m128i *ps, const __m128i *pm)
 585 {
 586     __m128i xmm_src_lo, xmm_src_hi;
 587     __m128i xmm_msk_lo, xmm_msk_hi;
 588     __m128i s;
 589
 590     if (pm)
 591     {
 592         xmm_msk_lo = load_128_unaligned (pm);
 593
 594         if (is_transparent (xmm_msk_lo))
 595             return _mm_setzero_si128 ();
 596     }
 597
 598     s = load_128_unaligned (ps);
 599
 600     if (pm)
 601     {
 602         unpack_128_2x128 (s, &xmm_src_lo, &xmm_src_hi);
 603         unpack_128_2x128 (xmm_msk_lo, &xmm_msk_lo, &xmm_msk_hi);
 604
 605         expand_alpha_2x128 (xmm_msk_lo, xmm_msk_hi, &xmm_msk_lo, &xmm_msk_hi);
 606
 607         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
 608                             &xmm_msk_lo, &xmm_msk_hi,
 609                             &xmm_src_lo, &xmm_src_hi);
 610
 611         s = pack_2x128_128 (xmm_src_lo, xmm_src_hi);
 612     }
 613
 614     return s;
 615 }
 616
 617 static force_inline void
 618 core_combine_over_u_sse2 (uint32_t*       pd,
 619                           const uint32_t* ps,
 620                           const uint32_t* pm,
 621                           int             w)
 622 {
 623     uint32_t s, d;
 624
 625     __m128i xmm_dst_lo, xmm_dst_hi;
 626     __m128i xmm_src_lo, xmm_src_hi;
 627     __m128i xmm_alpha_lo, xmm_alpha_hi;
 628
 629     /* call prefetch hint to optimize cache load*/
 630     cache_prefetch ((__m128i*)ps);
 631     cache_prefetch ((__m128i*)pd);
 632     cache_prefetch ((__m128i*)pm);
 633
 634     /* Align dst on a 16-byte boundary */
 635     while (w && ((unsigned long)pd & 15))
 636     {
 637         d = *pd;
 638         s = combine1 (ps, pm);
 639
 640         *pd++ = core_combine_over_u_pixel_sse2 (s, d);
 641         ps++;
 642         if (pm)
 643             pm++;
 644         w--;
 645     }
 646
 647     /* call prefetch hint to optimize cache load*/
 648     cache_prefetch ((__m128i*)ps);
 649     cache_prefetch ((__m128i*)pd);
 650     cache_prefetch ((__m128i*)pm);
 651
 652     while (w >= 4)
 653     {
 654         /* fill cache line with next memory */
 655         cache_prefetch_next ((__m128i*)ps);
 656         cache_prefetch_next ((__m128i*)pd);
 657         cache_prefetch_next ((__m128i*)pm);
 658
 659         /* I'm loading unaligned because I'm not sure about
 660          * the address alignment.
 661          */
 662         xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
 663
 664         if (is_opaque (xmm_src_hi))
 665         {
 666             save_128_aligned ((__m128i*)pd, xmm_src_hi);
 667         }
 668         else if (!is_zero (xmm_src_hi))
 669         {
 670             xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 671
 672             unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
 673             unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
 674
 675             expand_alpha_2x128 (
 676                 xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
 677
 678             over_2x128 (&xmm_src_lo, &xmm_src_hi,
 679                         &xmm_alpha_lo, &xmm_alpha_hi,
 680                         &xmm_dst_lo, &xmm_dst_hi);
 681
 682             /* rebuid the 4 pixel data and save*/
 683             save_128_aligned ((__m128i*)pd,
 684                               pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
 685         }
 686
 687         w -= 4;
 688         ps += 4;
 689         pd += 4;
 690         if (pm)
 691             pm += 4;
 692     }
 693
 694     while (w)
 695     {
 696         d = *pd;
 697         s = combine1 (ps, pm);
 698
 699         *pd++ = core_combine_over_u_pixel_sse2 (s, d);
 700         ps++;
 701         if (pm)
 702             pm++;
 703
 704         w--;
 705     }
 706 }
 707
 708 static force_inline void
 709 core_combine_over_reverse_u_sse2 (uint32_t*       pd,
 710                                   const uint32_t* ps,
 711                                   const uint32_t* pm,
 712                                   int             w)
 713 {
 714     uint32_t s, d;
 715
 716     __m128i xmm_dst_lo, xmm_dst_hi;
 717     __m128i xmm_src_lo, xmm_src_hi;
 718     __m128i xmm_alpha_lo, xmm_alpha_hi;
 719
 720     /* call prefetch hint to optimize cache load*/
 721     cache_prefetch ((__m128i*)ps);
 722     cache_prefetch ((__m128i*)pd);
 723     cache_prefetch ((__m128i*)pm);
 724
 725     /* Align dst on a 16-byte boundary */
 726     while (w &&
 727            ((unsigned long)pd & 15))
 728     {
 729         d = *pd;
 730         s = combine1 (ps, pm);
 731
 732         *pd++ = core_combine_over_u_pixel_sse2 (d, s);
 733         w--;
 734         ps++;
 735         if (pm)
 736             pm++;
 737     }
 738
 739     /* call prefetch hint to optimize cache load*/
 740     cache_prefetch ((__m128i*)ps);
 741     cache_prefetch ((__m128i*)pd);
 742     cache_prefetch ((__m128i*)pm);
 743
 744     while (w >= 4)
 745     {
 746         /* fill cache line with next memory */
 747         cache_prefetch_next ((__m128i*)ps);
 748         cache_prefetch_next ((__m128i*)pd);
 749         cache_prefetch_next ((__m128i*)pm);
 750
 751         /* I'm loading unaligned because I'm not sure
 752          * about the address alignment.
 753          */
 754         xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
 755         xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 756
 757         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
 758         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
 759
 760         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
 761                             &xmm_alpha_lo, &xmm_alpha_hi);
 762
 763         over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
 764                     &xmm_alpha_lo, &xmm_alpha_hi,
 765                     &xmm_src_lo, &xmm_src_hi);
 766
 767         /* rebuid the 4 pixel data and save*/
 768         save_128_aligned ((__m128i*)pd,
 769                           pack_2x128_128 (xmm_src_lo, xmm_src_hi));
 770
 771         w -= 4;
 772         ps += 4;
 773         pd += 4;
 774
 775         if (pm)
 776             pm += 4;
 777     }
 778
 779     while (w)
 780     {
 781         d = *pd;
 782         s = combine1 (ps, pm);
 783
 784         *pd++ = core_combine_over_u_pixel_sse2 (d, s);
 785         ps++;
 786         w--;
 787         if (pm)
 788             pm++;
 789     }
 790 }
 791
 792 static force_inline uint32_t
 793 core_combine_in_u_pixelsse2 (uint32_t src, uint32_t dst)
 794 {
 795     uint32_t maska = src >> 24;
 796
 797     if (maska == 0)
 798     {
 799         return 0;
 800     }
 801     else if (maska != 0xff)
 802     {
 803         return pack_1x64_32 (
 804             pix_multiply_1x64 (unpack_32_1x64 (dst),
 805                                expand_alpha_1x64 (unpack_32_1x64 (src))));
 806     }
 807
 808     return dst;
 809 }
 810
 811 static force_inline void
 812 core_combine_in_u_sse2 (uint32_t*       pd,
 813                         const uint32_t* ps,
 814                         const uint32_t* pm,
 815                         int             w)
 816 {
 817     uint32_t s, d;
 818
 819     __m128i xmm_src_lo, xmm_src_hi;
 820     __m128i xmm_dst_lo, xmm_dst_hi;
 821
 822     /* call prefetch hint to optimize cache load*/
 823     cache_prefetch ((__m128i*)ps);
 824     cache_prefetch ((__m128i*)pd);
 825     cache_prefetch ((__m128i*)pm);
 826
 827     while (w && ((unsigned long) pd & 15))
 828     {
 829         s = combine1 (ps, pm);
 830         d = *pd;
 831
 832         *pd++ = core_combine_in_u_pixelsse2 (d, s);
 833         w--;
 834         ps++;
 835         if (pm)
 836             pm++;
 837     }
 838
 839     /* call prefetch hint to optimize cache load*/
 840     cache_prefetch ((__m128i*)ps);
 841     cache_prefetch ((__m128i*)pd);
 842     cache_prefetch ((__m128i*)pm);
 843
 844     while (w >= 4)
 845     {
 846         /* fill cache line with next memory */
 847         cache_prefetch_next ((__m128i*)ps);
 848         cache_prefetch_next ((__m128i*)pd);
 849         cache_prefetch_next ((__m128i*)pm);
 850
 851         xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 852         xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*) pm);
 853
 854         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
 855         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
 856
 857         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
 858         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
 859                             &xmm_dst_lo, &xmm_dst_hi,
 860                             &xmm_dst_lo, &xmm_dst_hi);
 861
 862         save_128_aligned ((__m128i*)pd,
 863                           pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
 864
 865         ps += 4;
 866         pd += 4;
 867         w -= 4;
 868         if (pm)
 869             pm += 4;
 870     }
 871
 872     while (w)
 873     {
 874         s = combine1 (ps, pm);
 875         d = *pd;
 876
 877         *pd++ = core_combine_in_u_pixelsse2 (d, s);
 878         w--;
 879         ps++;
 880         if (pm)
 881             pm++;
 882     }
 883 }
 884
 885 static force_inline void
 886 core_combine_reverse_in_u_sse2 (uint32_t*       pd,
 887                                 const uint32_t* ps,
 888                                 const uint32_t *pm,
 889                                 int             w)
 890 {
 891     uint32_t s, d;
 892
 893     __m128i xmm_src_lo, xmm_src_hi;
 894     __m128i xmm_dst_lo, xmm_dst_hi;
 895
 896     /* call prefetch hint to optimize cache load*/
 897     cache_prefetch ((__m128i*)ps);
 898     cache_prefetch ((__m128i*)pd);
 899     cache_prefetch ((__m128i*)pm);
 900
 901     while (w && ((unsigned long) pd & 15))
 902     {
 903         s = combine1 (ps, pm);
 904         d = *pd;
 905
 906         *pd++ = core_combine_in_u_pixelsse2 (s, d);
 907         ps++;
 908         w--;
 909         if (pm)
 910             pm++;
 911     }
 912
 913     /* call prefetch hint to optimize cache load*/
 914     cache_prefetch ((__m128i*)ps);
 915     cache_prefetch ((__m128i*)pd);
 916     cache_prefetch ((__m128i*)pm);
 917
 918     while (w >= 4)
 919     {
 920         /* fill cache line with next memory */
 921         cache_prefetch_next ((__m128i*)ps);
 922         cache_prefetch_next ((__m128i*)pd);
 923         cache_prefetch_next ((__m128i*)pm);
 924
 925         xmm_dst_hi = load_128_aligned ((__m128i*) pd);
 926         xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
 927
 928         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
 929         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
 930
 931         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
 932         pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
 933                             &xmm_src_lo, &xmm_src_hi,
 934                             &xmm_dst_lo, &xmm_dst_hi);
 935
 936         save_128_aligned (
 937             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
 938
 939         ps += 4;
 940         pd += 4;
 941         w -= 4;
 942         if (pm)
 943             pm += 4;
 944     }
 945
 946     while (w)
 947     {
 948         s = combine1 (ps, pm);
 949         d = *pd;
 950
 951         *pd++ = core_combine_in_u_pixelsse2 (s, d);
 952         w--;
 953         ps++;
 954         if (pm)
 955             pm++;
 956     }
 957 }
 958
 959 static force_inline void
 960 core_combine_reverse_out_u_sse2 (uint32_t*       pd,
 961                                  const uint32_t* ps,
 962                                  const uint32_t* pm,
 963                                  int             w)
 964 {
 965     /* call prefetch hint to optimize cache load*/
 966     cache_prefetch ((__m128i*)ps);
 967     cache_prefetch ((__m128i*)pd);
 968     cache_prefetch ((__m128i*)pm);
 969
 970     while (w && ((unsigned long) pd & 15))
 971     {
 972         uint32_t s = combine1 (ps, pm);
 973         uint32_t d = *pd;
 974
 975         *pd++ = pack_1x64_32 (
 976             pix_multiply_1x64 (
 977                 unpack_32_1x64 (d), negate_1x64 (
 978                     expand_alpha_1x64 (unpack_32_1x64 (s)))));
 979
 980         if (pm)
 981             pm++;
 982         ps++;
 983         w--;
 984     }
 985
 986     /* call prefetch hint to optimize cache load*/
 987     cache_prefetch ((__m128i*)ps);
 988     cache_prefetch ((__m128i*)pd);
 989     cache_prefetch ((__m128i*)pm);
 990
 991     while (w >= 4)
 992     {
 993         __m128i xmm_src_lo, xmm_src_hi;
 994         __m128i xmm_dst_lo, xmm_dst_hi;
 995
 996         /* fill cache line with next memory */
 997         cache_prefetch_next ((__m128i*)ps);
 998         cache_prefetch_next ((__m128i*)pd);
 999         cache_prefetch_next ((__m128i*)pm);
1000
1001         xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1002         xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1003
1004         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1005         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1006
1007         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1008         negate_2x128       (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1009
1010         pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1011                             &xmm_src_lo, &xmm_src_hi,
1012                             &xmm_dst_lo, &xmm_dst_hi);
1013
1014         save_128_aligned (
1015             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1016
1017         ps += 4;
1018         pd += 4;
1019         if (pm)
1020             pm += 4;
1021
1022         w -= 4;
1023     }
1024
1025     while (w)
1026     {
1027         uint32_t s = combine1 (ps, pm);
1028         uint32_t d = *pd;
1029
1030         *pd++ = pack_1x64_32 (
1031             pix_multiply_1x64 (
1032                 unpack_32_1x64 (d), negate_1x64 (
1033                     expand_alpha_1x64 (unpack_32_1x64 (s)))));
1034         ps++;
1035         if (pm)
1036             pm++;
1037         w--;
1038     }
1039 }
1040
1041 static force_inline void
1042 core_combine_out_u_sse2 (uint32_t*       pd,
1043                          const uint32_t* ps,
1044                          const uint32_t* pm,
1045                          int             w)
1046 {
1047     /* call prefetch hint to optimize cache load*/
1048     cache_prefetch ((__m128i*)ps);
1049     cache_prefetch ((__m128i*)pd);
1050     cache_prefetch ((__m128i*)pm);
1051
1052     while (w && ((unsigned long) pd & 15))
1053     {
1054         uint32_t s = combine1 (ps, pm);
1055         uint32_t d = *pd;
1056
1057         *pd++ = pack_1x64_32 (
1058             pix_multiply_1x64 (
1059                 unpack_32_1x64 (s), negate_1x64 (
1060                     expand_alpha_1x64 (unpack_32_1x64 (d)))));
1061         w--;
1062         ps++;
1063         if (pm)
1064             pm++;
1065     }
1066
1067     /* call prefetch hint to optimize cache load*/
1068     cache_prefetch ((__m128i*)ps);
1069     cache_prefetch ((__m128i*)pd);
1070     cache_prefetch ((__m128i*)pm);
1071
1072     while (w >= 4)
1073     {
1074         __m128i xmm_src_lo, xmm_src_hi;
1075         __m128i xmm_dst_lo, xmm_dst_hi;
1076
1077         /* fill cache line with next memory */
1078         cache_prefetch_next ((__m128i*)ps);
1079         cache_prefetch_next ((__m128i*)pd);
1080         cache_prefetch_next ((__m128i*)pm);
1081
1082         xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
1083         xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1084
1085         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1086         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1087
1088         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1089         negate_2x128       (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1090
1091         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1092                             &xmm_dst_lo, &xmm_dst_hi,
1093                             &xmm_dst_lo, &xmm_dst_hi);
1094
1095         save_128_aligned (
1096             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1097
1098         ps += 4;
1099         pd += 4;
1100         w -= 4;
1101         if (pm)
1102             pm += 4;
1103     }
1104
1105     while (w)
1106     {
1107         uint32_t s = combine1 (ps, pm);
1108         uint32_t d = *pd;
1109
1110         *pd++ = pack_1x64_32 (
1111             pix_multiply_1x64 (
1112                 unpack_32_1x64 (s), negate_1x64 (
1113                     expand_alpha_1x64 (unpack_32_1x64 (d)))));
1114         w--;
1115         ps++;
1116         if (pm)
1117             pm++;
1118     }
1119 }
1120
1121 static force_inline uint32_t
1122 core_combine_atop_u_pixel_sse2 (uint32_t src,
1123                                 uint32_t dst)
1124 {
1125     __m64 s = unpack_32_1x64 (src);
1126     __m64 d = unpack_32_1x64 (dst);
1127
1128     __m64 sa = negate_1x64 (expand_alpha_1x64 (s));
1129     __m64 da = expand_alpha_1x64 (d);
1130
1131     return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
1132 }
1133
1134 static force_inline void
1135 core_combine_atop_u_sse2 (uint32_t*       pd,
1136                           const uint32_t* ps,
1137                           const uint32_t* pm,
1138                           int             w)
1139 {
1140     uint32_t s, d;
1141
1142     __m128i xmm_src_lo, xmm_src_hi;
1143     __m128i xmm_dst_lo, xmm_dst_hi;
1144     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1145     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1146
1147     /* call prefetch hint to optimize cache load*/
1148     cache_prefetch ((__m128i*)ps);
1149     cache_prefetch ((__m128i*)pd);
1150     cache_prefetch ((__m128i*)pm);
1151
1152     while (w && ((unsigned long) pd & 15))
1153     {
1154         s = combine1 (ps, pm);
1155         d = *pd;
1156
1157         *pd++ = core_combine_atop_u_pixel_sse2 (s, d);
1158         w--;
1159         ps++;
1160         if (pm)
1161             pm++;
1162     }
1163
1164     /* call prefetch hint to optimize cache load*/
1165     cache_prefetch ((__m128i*)ps);
1166     cache_prefetch ((__m128i*)pd);
1167     cache_prefetch ((__m128i*)pm);
1168
1169     while (w >= 4)
1170     {
1171         /* fill cache line with next memory */
1172         cache_prefetch_next ((__m128i*)ps);
1173         cache_prefetch_next ((__m128i*)pd);
1174         cache_prefetch_next ((__m128i*)pm);
1175
1176         xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1177         xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1178
1179         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1180         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1181
1182         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1183                             &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1184         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1185                             &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1186
1187         negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
1188                       &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1189
1190         pix_add_multiply_2x128 (
1191             &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1192             &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1193             &xmm_dst_lo, &xmm_dst_hi);
1194
1195         save_128_aligned (
1196             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1197
1198         ps += 4;
1199         pd += 4;
1200         w -= 4;
1201         if (pm)
1202             pm += 4;
1203     }
1204
1205     while (w)
1206     {
1207         s = combine1 (ps, pm);
1208         d = *pd;
1209
1210         *pd++ = core_combine_atop_u_pixel_sse2 (s, d);
1211         w--;
1212         ps++;
1213         if (pm)
1214             pm++;
1215     }
1216 }
1217
1218 static force_inline uint32_t
1219 core_combine_reverse_atop_u_pixel_sse2 (uint32_t src,
1220                                         uint32_t dst)
1221 {
1222     __m64 s = unpack_32_1x64 (src);
1223     __m64 d = unpack_32_1x64 (dst);
1224
1225     __m64 sa = expand_alpha_1x64 (s);
1226     __m64 da = negate_1x64 (expand_alpha_1x64 (d));
1227
1228     return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
1229 }
1230
1231 static force_inline void
1232 core_combine_reverse_atop_u_sse2 (uint32_t*       pd,
1233                                   const uint32_t* ps,
1234                                   const uint32_t* pm,
1235                                   int             w)
1236 {
1237     uint32_t s, d;
1238
1239     __m128i xmm_src_lo, xmm_src_hi;
1240     __m128i xmm_dst_lo, xmm_dst_hi;
1241     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1242     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1243
1244     /* call prefetch hint to optimize cache load*/
1245     cache_prefetch ((__m128i*)ps);
1246     cache_prefetch ((__m128i*)pd);
1247     cache_prefetch ((__m128i*)pm);
1248
1249     while (w && ((unsigned long) pd & 15))
1250     {
1251         s = combine1 (ps, pm);
1252         d = *pd;
1253
1254         *pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
1255         ps++;
1256         w--;
1257         if (pm)
1258             pm++;
1259     }
1260
1261     /* call prefetch hint to optimize cache load*/
1262     cache_prefetch ((__m128i*)ps);
1263     cache_prefetch ((__m128i*)pd);
1264     cache_prefetch ((__m128i*)pm);
1265
1266     while (w >= 4)
1267     {
1268         /* fill cache line with next memory */
1269         cache_prefetch_next ((__m128i*)ps);
1270         cache_prefetch_next ((__m128i*)pd);
1271         cache_prefetch_next ((__m128i*)pm);
1272
1273         xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
1274         xmm_dst_hi = load_128_aligned ((__m128i*) pd);
1275
1276         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1277         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1278
1279         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1280                             &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1281         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1282                             &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1283
1284         negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
1285                       &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1286
1287         pix_add_multiply_2x128 (
1288             &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1289             &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1290             &xmm_dst_lo, &xmm_dst_hi);
1291
1292         save_128_aligned (
1293             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1294
1295         ps += 4;
1296         pd += 4;
1297         w -= 4;
1298         if (pm)
1299             pm += 4;
1300     }
1301
1302     while (w)
1303     {
1304         s = combine1 (ps, pm);
1305         d = *pd;
1306
1307         *pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
1308         ps++;
1309         w--;
1310         if (pm)
1311             pm++;
1312     }
1313 }
1314
1315 static force_inline uint32_t
1316 core_combine_xor_u_pixel_sse2 (uint32_t src,
1317                                uint32_t dst)
1318 {
1319     __m64 s = unpack_32_1x64 (src);
1320     __m64 d = unpack_32_1x64 (dst);
1321
1322     __m64 neg_d = negate_1x64 (expand_alpha_1x64 (d));
1323     __m64 neg_s = negate_1x64 (expand_alpha_1x64 (s));
1324
1325     return pack_1x64_32 (pix_add_multiply_1x64 (&s, &neg_d, &d, &neg_s));
1326 }
1327
1328 static force_inline void
1329 core_combine_xor_u_sse2 (uint32_t*       dst,
1330                          const uint32_t* src,
1331                          const uint32_t *mask,
1332                          int             width)
1333 {
1334     int w = width;
1335     uint32_t s, d;
1336     uint32_t* pd = dst;
1337     const uint32_t* ps = src;
1338     const uint32_t* pm = mask;
1339
1340     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
1341     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
1342     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
1343     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
1344
1345     /* call prefetch hint to optimize cache load*/
1346     cache_prefetch ((__m128i*)ps);
1347     cache_prefetch ((__m128i*)pd);
1348     cache_prefetch ((__m128i*)pm);
1349
1350     while (w && ((unsigned long) pd & 15))
1351     {
1352         s = combine1 (ps, pm);
1353         d = *pd;
1354
1355         *pd++ = core_combine_xor_u_pixel_sse2 (s, d);
1356         w--;
1357         ps++;
1358         if (pm)
1359             pm++;
1360     }
1361
1362     /* call prefetch hint to optimize cache load*/
1363     cache_prefetch ((__m128i*)ps);
1364     cache_prefetch ((__m128i*)pd);
1365     cache_prefetch ((__m128i*)pm);
1366
1367     while (w >= 4)
1368     {
1369         /* fill cache line with next memory */
1370         cache_prefetch_next ((__m128i*)ps);
1371         cache_prefetch_next ((__m128i*)pd);
1372         cache_prefetch_next ((__m128i*)pm);
1373
1374         xmm_src = combine4 ((__m128i*) ps, (__m128i*) pm);
1375         xmm_dst = load_128_aligned ((__m128i*) pd);
1376
1377         unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
1378         unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
1379
1380         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1381                             &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1382         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1383                             &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1384
1385         negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
1386                       &xmm_alpha_src_lo, &xmm_alpha_src_hi);
1387         negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
1388                       &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
1389
1390         pix_add_multiply_2x128 (
1391             &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
1392             &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
1393             &xmm_dst_lo, &xmm_dst_hi);
1394
1395         save_128_aligned (
1396             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1397
1398         ps += 4;
1399         pd += 4;
1400         w -= 4;
1401         if (pm)
1402             pm += 4;
1403     }
1404
1405     while (w)
1406     {
1407         s = combine1 (ps, pm);
1408         d = *pd;
1409
1410         *pd++ = core_combine_xor_u_pixel_sse2 (s, d);
1411         w--;
1412         ps++;
1413         if (pm)
1414             pm++;
1415     }
1416 }
1417
1418 static force_inline void
1419 core_combine_add_u_sse2 (uint32_t*       dst,
1420                          const uint32_t* src,
1421                          const uint32_t* mask,
1422                          int             width)
1423 {
1424     int w = width;
1425     uint32_t s, d;
1426     uint32_t* pd = dst;
1427     const uint32_t* ps = src;
1428     const uint32_t* pm = mask;
1429
1430     /* call prefetch hint to optimize cache load*/
1431     cache_prefetch ((__m128i*)ps);
1432     cache_prefetch ((__m128i*)pd);
1433     cache_prefetch ((__m128i*)pm);
1434
1435     while (w && (unsigned long)pd & 15)
1436     {
1437         s = combine1 (ps, pm);
1438         d = *pd;
1439
1440         ps++;
1441         if (pm)
1442             pm++;
1443         *pd++ = _mm_cvtsi64_si32 (
1444             _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
1445         w--;
1446     }
1447
1448     /* call prefetch hint to optimize cache load*/
1449     cache_prefetch ((__m128i*)ps);
1450     cache_prefetch ((__m128i*)pd);
1451     cache_prefetch ((__m128i*)pm);
1452
1453     while (w >= 4)
1454     {
1455         __m128i s;
1456
1457         /* fill cache line with next memory */
1458         cache_prefetch_next ((__m128i*)ps);
1459         cache_prefetch_next ((__m128i*)pd);
1460         cache_prefetch_next ((__m128i*)pm);
1461
1462         s = combine4 ((__m128i*)ps, (__m128i*)pm);
1463
1464         save_128_aligned (
1465             (__m128i*)pd, _mm_adds_epu8 (s, load_128_aligned  ((__m128i*)pd)));
1466
1467         pd += 4;
1468         ps += 4;
1469         if (pm)
1470             pm += 4;
1471         w -= 4;
1472     }
1473
1474     while (w--)
1475     {
1476         s = combine1 (ps, pm);
1477         d = *pd;
1478
1479         ps++;
1480         *pd++ = _mm_cvtsi64_si32 (
1481             _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
1482         if (pm)
1483             pm++;
1484     }
1485 }
1486
1487 static force_inline uint32_t
1488 core_combine_saturate_u_pixel_sse2 (uint32_t src,
1489                                     uint32_t dst)
1490 {
1491     __m64 ms = unpack_32_1x64 (src);
1492     __m64 md = unpack_32_1x64 (dst);
1493     uint32_t sa = src >> 24;
1494     uint32_t da = ~dst >> 24;
1495
1496     if (sa > da)
1497     {
1498         ms = pix_multiply_1x64 (
1499             ms, expand_alpha_1x64 (unpack_32_1x64 (DIV_UN8 (da, sa) << 24)));
1500     }
1501
1502     return pack_1x64_32 (_mm_adds_pu16 (md, ms));
1503 }
1504
1505 static force_inline void
1506 core_combine_saturate_u_sse2 (uint32_t *      pd,
1507                               const uint32_t *ps,
1508                               const uint32_t *pm,
1509                               int             w)
1510 {
1511     uint32_t s, d;
1512
1513     uint32_t pack_cmp;
1514     __m128i xmm_src, xmm_dst;
1515
1516     /* call prefetch hint to optimize cache load*/
1517     cache_prefetch ((__m128i*)ps);
1518     cache_prefetch ((__m128i*)pd);
1519     cache_prefetch ((__m128i*)pm);
1520
1521     while (w && (unsigned long)pd & 15)
1522     {
1523         s = combine1 (ps, pm);
1524         d = *pd;
1525
1526         *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1527         w--;
1528         ps++;
1529         if (pm)
1530             pm++;
1531     }
1532
1533     /* call prefetch hint to optimize cache load*/
1534     cache_prefetch ((__m128i*)ps);
1535     cache_prefetch ((__m128i*)pd);
1536     cache_prefetch ((__m128i*)pm);
1537
1538     while (w >= 4)
1539     {
1540         /* fill cache line with next memory */
1541         cache_prefetch_next ((__m128i*)ps);
1542         cache_prefetch_next ((__m128i*)pd);
1543         cache_prefetch_next ((__m128i*)pm);
1544
1545         xmm_dst = load_128_aligned  ((__m128i*)pd);
1546         xmm_src = combine4 ((__m128i*)ps, (__m128i*)pm);
1547
1548         pack_cmp = _mm_movemask_epi8 (
1549             _mm_cmpgt_epi32 (
1550                 _mm_srli_epi32 (xmm_src, 24),
1551                 _mm_srli_epi32 (_mm_xor_si128 (xmm_dst, mask_ff000000), 24)));
1552
1553         /* if some alpha src is grater than respective ~alpha dst */
1554         if (pack_cmp)
1555         {
1556             s = combine1 (ps++, pm);
1557             d = *pd;
1558             *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1559             if (pm)
1560                 pm++;
1561
1562             s = combine1 (ps++, pm);
1563             d = *pd;
1564             *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1565             if (pm)
1566                 pm++;
1567
1568             s = combine1 (ps++, pm);
1569             d = *pd;
1570             *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1571             if (pm)
1572                 pm++;
1573
1574             s = combine1 (ps++, pm);
1575             d = *pd;
1576             *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1577             if (pm)
1578                 pm++;
1579         }
1580         else
1581         {
1582             save_128_aligned ((__m128i*)pd, _mm_adds_epu8 (xmm_dst, xmm_src));
1583
1584             pd += 4;
1585             ps += 4;
1586             if (pm)
1587                 pm += 4;
1588         }
1589
1590         w -= 4;
1591     }
1592
1593     while (w--)
1594     {
1595         s = combine1 (ps, pm);
1596         d = *pd;
1597
1598         *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
1599         ps++;
1600         if (pm)
1601             pm++;
1602     }
1603 }
1604
1605 static force_inline void
1606 core_combine_src_ca_sse2 (uint32_t*       pd,
1607                           const uint32_t* ps,
1608                           const uint32_t *pm,
1609                           int             w)
1610 {
1611     uint32_t s, m;
1612
1613     __m128i xmm_src_lo, xmm_src_hi;
1614     __m128i xmm_mask_lo, xmm_mask_hi;
1615     __m128i xmm_dst_lo, xmm_dst_hi;
1616
1617     /* call prefetch hint to optimize cache load*/
1618     cache_prefetch ((__m128i*)ps);
1619     cache_prefetch ((__m128i*)pd);
1620     cache_prefetch ((__m128i*)pm);
1621
1622     while (w && (unsigned long)pd & 15)
1623     {
1624         s = *ps++;
1625         m = *pm++;
1626         *pd++ = pack_1x64_32 (
1627             pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
1628         w--;
1629     }
1630
1631     /* call prefetch hint to optimize cache load*/
1632     cache_prefetch ((__m128i*)ps);
1633     cache_prefetch ((__m128i*)pd);
1634     cache_prefetch ((__m128i*)pm);
1635
1636     while (w >= 4)
1637     {
1638         /* fill cache line with next memory */
1639         cache_prefetch_next ((__m128i*)ps);
1640         cache_prefetch_next ((__m128i*)pd);
1641         cache_prefetch_next ((__m128i*)pm);
1642
1643         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1644         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1645
1646         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1647         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1648
1649         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1650                             &xmm_mask_lo, &xmm_mask_hi,
1651                             &xmm_dst_lo, &xmm_dst_hi);
1652
1653         save_128_aligned (
1654             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1655
1656         ps += 4;
1657         pd += 4;
1658         pm += 4;
1659         w -= 4;
1660     }
1661
1662     while (w)
1663     {
1664         s = *ps++;
1665         m = *pm++;
1666         *pd++ = pack_1x64_32 (
1667             pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
1668         w--;
1669     }
1670 }
1671
1672 static force_inline uint32_t
1673 core_combine_over_ca_pixel_sse2 (uint32_t src,
1674                                  uint32_t mask,
1675                                  uint32_t dst)
1676 {
1677     __m64 s = unpack_32_1x64 (src);
1678     __m64 expAlpha = expand_alpha_1x64 (s);
1679     __m64 unpk_mask = unpack_32_1x64 (mask);
1680     __m64 unpk_dst  = unpack_32_1x64 (dst);
1681
1682     return pack_1x64_32 (in_over_1x64 (&s, &expAlpha, &unpk_mask, &unpk_dst));
1683 }
1684
1685 static force_inline void
1686 core_combine_over_ca_sse2 (uint32_t*       pd,
1687                            const uint32_t* ps,
1688                            const uint32_t *pm,
1689                            int             w)
1690 {
1691     uint32_t s, m, d;
1692
1693     __m128i xmm_alpha_lo, xmm_alpha_hi;
1694     __m128i xmm_src_lo, xmm_src_hi;
1695     __m128i xmm_dst_lo, xmm_dst_hi;
1696     __m128i xmm_mask_lo, xmm_mask_hi;
1697
1698     /* call prefetch hint to optimize cache load*/
1699     cache_prefetch ((__m128i*)ps);
1700     cache_prefetch ((__m128i*)pd);
1701     cache_prefetch ((__m128i*)pm);
1702
1703     while (w && (unsigned long)pd & 15)
1704     {
1705         s = *ps++;
1706         m = *pm++;
1707         d = *pd;
1708
1709         *pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
1710         w--;
1711     }
1712
1713     /* call prefetch hint to optimize cache load*/
1714     cache_prefetch ((__m128i*)ps);
1715     cache_prefetch ((__m128i*)pd);
1716     cache_prefetch ((__m128i*)pm);
1717
1718     while (w >= 4)
1719     {
1720         /* fill cache line with next memory */
1721         cache_prefetch_next ((__m128i*)ps);
1722         cache_prefetch_next ((__m128i*)pd);
1723         cache_prefetch_next ((__m128i*)pm);
1724
1725         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1726         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1727         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1728
1729         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1730         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1731         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1732
1733         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1734                             &xmm_alpha_lo, &xmm_alpha_hi);
1735
1736         in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
1737                        &xmm_alpha_lo, &xmm_alpha_hi,
1738                        &xmm_mask_lo, &xmm_mask_hi,
1739                        &xmm_dst_lo, &xmm_dst_hi);
1740
1741         save_128_aligned (
1742             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1743
1744         ps += 4;
1745         pd += 4;
1746         pm += 4;
1747         w -= 4;
1748     }
1749
1750     while (w)
1751     {
1752         s = *ps++;
1753         m = *pm++;
1754         d = *pd;
1755
1756         *pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
1757         w--;
1758     }
1759 }
1760
1761 static force_inline uint32_t
1762 core_combine_over_reverse_ca_pixel_sse2 (uint32_t src,
1763                                          uint32_t mask,
1764                                          uint32_t dst)
1765 {
1766     __m64 d = unpack_32_1x64 (dst);
1767
1768     return pack_1x64_32 (
1769         over_1x64 (d, expand_alpha_1x64 (d),
1770                    pix_multiply_1x64 (unpack_32_1x64 (src),
1771                                       unpack_32_1x64 (mask))));
1772 }
1773
1774 static force_inline void
1775 core_combine_over_reverse_ca_sse2 (uint32_t*       pd,
1776                                    const uint32_t* ps,
1777                                    const uint32_t *pm,
1778                                    int             w)
1779 {
1780     uint32_t s, m, d;
1781
1782     __m128i xmm_alpha_lo, xmm_alpha_hi;
1783     __m128i xmm_src_lo, xmm_src_hi;
1784     __m128i xmm_dst_lo, xmm_dst_hi;
1785     __m128i xmm_mask_lo, xmm_mask_hi;
1786
1787     /* call prefetch hint to optimize cache load*/
1788     cache_prefetch ((__m128i*)ps);
1789     cache_prefetch ((__m128i*)pd);
1790     cache_prefetch ((__m128i*)pm);
1791
1792     while (w && (unsigned long)pd & 15)
1793     {
1794         s = *ps++;
1795         m = *pm++;
1796         d = *pd;
1797
1798         *pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
1799         w--;
1800     }
1801
1802     /* call prefetch hint to optimize cache load*/
1803     cache_prefetch ((__m128i*)ps);
1804     cache_prefetch ((__m128i*)pd);
1805     cache_prefetch ((__m128i*)pm);
1806
1807     while (w >= 4)
1808     {
1809         /* fill cache line with next memory */
1810         cache_prefetch_next ((__m128i*)ps);
1811         cache_prefetch_next ((__m128i*)pd);
1812         cache_prefetch_next ((__m128i*)pm);
1813
1814         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1815         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1816         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1817
1818         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1819         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1820         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1821
1822         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1823                             &xmm_alpha_lo, &xmm_alpha_hi);
1824         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1825                             &xmm_mask_lo, &xmm_mask_hi,
1826                             &xmm_mask_lo, &xmm_mask_hi);
1827
1828         over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1829                     &xmm_alpha_lo, &xmm_alpha_hi,
1830                     &xmm_mask_lo, &xmm_mask_hi);
1831
1832         save_128_aligned (
1833             (__m128i*)pd, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
1834
1835         ps += 4;
1836         pd += 4;
1837         pm += 4;
1838         w -= 4;
1839     }
1840
1841     while (w)
1842     {
1843         s = *ps++;
1844         m = *pm++;
1845         d = *pd;
1846
1847         *pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
1848         w--;
1849     }
1850 }
1851
1852 static force_inline void
1853 core_combine_in_ca_sse2 (uint32_t *      pd,
1854                          const uint32_t *ps,
1855                          const uint32_t *pm,
1856                          int             w)
1857 {
1858     uint32_t s, m, d;
1859
1860     __m128i xmm_alpha_lo, xmm_alpha_hi;
1861     __m128i xmm_src_lo, xmm_src_hi;
1862     __m128i xmm_dst_lo, xmm_dst_hi;
1863     __m128i xmm_mask_lo, xmm_mask_hi;
1864
1865     /* call prefetch hint to optimize cache load*/
1866     cache_prefetch ((__m128i*)ps);
1867     cache_prefetch ((__m128i*)pd);
1868     cache_prefetch ((__m128i*)pm);
1869
1870     while (w && (unsigned long)pd & 15)
1871     {
1872         s = *ps++;
1873         m = *pm++;
1874         d = *pd;
1875
1876         *pd++ = pack_1x64_32 (
1877             pix_multiply_1x64 (
1878                 pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
1879                 expand_alpha_1x64 (unpack_32_1x64 (d))));
1880
1881         w--;
1882     }
1883
1884     /* call prefetch hint to optimize cache load*/
1885     cache_prefetch ((__m128i*)ps);
1886     cache_prefetch ((__m128i*)pd);
1887     cache_prefetch ((__m128i*)pm);
1888
1889     while (w >= 4)
1890     {
1891         /* fill cache line with next memory */
1892         cache_prefetch_next ((__m128i*)ps);
1893         cache_prefetch_next ((__m128i*)pd);
1894         cache_prefetch_next ((__m128i*)pm);
1895
1896         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1897         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1898         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1899
1900         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1901         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1902         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1903
1904         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
1905                             &xmm_alpha_lo, &xmm_alpha_hi);
1906
1907         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
1908                             &xmm_mask_lo, &xmm_mask_hi,
1909                             &xmm_dst_lo, &xmm_dst_hi);
1910
1911         pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1912                             &xmm_alpha_lo, &xmm_alpha_hi,
1913                             &xmm_dst_lo, &xmm_dst_hi);
1914
1915         save_128_aligned (
1916             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
1917
1918         ps += 4;
1919         pd += 4;
1920         pm += 4;
1921         w -= 4;
1922     }
1923
1924     while (w)
1925     {
1926         s = *ps++;
1927         m = *pm++;
1928         d = *pd;
1929
1930         *pd++ = pack_1x64_32 (
1931             pix_multiply_1x64 (
1932                 pix_multiply_1x64 (
1933                     unpack_32_1x64 (s), unpack_32_1x64 (m)),
1934                 expand_alpha_1x64 (unpack_32_1x64 (d))));
1935
1936         w--;
1937     }
1938 }
1939
1940 static force_inline void
1941 core_combine_in_reverse_ca_sse2 (uint32_t *      pd,
1942                                  const uint32_t *ps,
1943                                  const uint32_t *pm,
1944                                  int             w)
1945 {
1946     uint32_t s, m, d;
1947
1948     __m128i xmm_alpha_lo, xmm_alpha_hi;
1949     __m128i xmm_src_lo, xmm_src_hi;
1950     __m128i xmm_dst_lo, xmm_dst_hi;
1951     __m128i xmm_mask_lo, xmm_mask_hi;
1952
1953     /* call prefetch hint to optimize cache load*/
1954     cache_prefetch ((__m128i*)ps);
1955     cache_prefetch ((__m128i*)pd);
1956     cache_prefetch ((__m128i*)pm);
1957
1958     while (w && (unsigned long)pd & 15)
1959     {
1960         s = *ps++;
1961         m = *pm++;
1962         d = *pd;
1963
1964         *pd++ = pack_1x64_32 (
1965             pix_multiply_1x64 (
1966                 unpack_32_1x64 (d),
1967                 pix_multiply_1x64 (unpack_32_1x64 (m),
1968                                    expand_alpha_1x64 (unpack_32_1x64 (s)))));
1969         w--;
1970     }
1971
1972     /* call prefetch hint to optimize cache load*/
1973     cache_prefetch ((__m128i*)ps);
1974     cache_prefetch ((__m128i*)pd);
1975     cache_prefetch ((__m128i*)pm);
1976
1977     while (w >= 4)
1978     {
1979         /* fill cache line with next memory */
1980         cache_prefetch_next ((__m128i*)ps);
1981         cache_prefetch_next ((__m128i*)pd);
1982         cache_prefetch_next ((__m128i*)pm);
1983
1984         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
1985         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
1986         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
1987
1988         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
1989         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
1990         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
1991
1992         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
1993                             &xmm_alpha_lo, &xmm_alpha_hi);
1994         pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
1995                             &xmm_alpha_lo, &xmm_alpha_hi,
1996                             &xmm_alpha_lo, &xmm_alpha_hi);
1997
1998         pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
1999                             &xmm_alpha_lo, &xmm_alpha_hi,
2000                             &xmm_dst_lo, &xmm_dst_hi);
2001
2002         save_128_aligned (
2003             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2004
2005         ps += 4;
2006         pd += 4;
2007         pm += 4;
2008         w -= 4;
2009     }
2010
2011     while (w)
2012     {
2013         s = *ps++;
2014         m = *pm++;
2015         d = *pd;
2016
2017         *pd++ = pack_1x64_32 (
2018             pix_multiply_1x64 (
2019                 unpack_32_1x64 (d),
2020                 pix_multiply_1x64 (unpack_32_1x64 (m),
2021                                    expand_alpha_1x64 (unpack_32_1x64 (s)))));
2022         w--;
2023     }
2024 }
2025
2026 static force_inline void
2027 core_combine_out_ca_sse2 (uint32_t *      pd,
2028                           const uint32_t *ps,
2029                           const uint32_t *pm,
2030                           int             w)
2031 {
2032     uint32_t s, m, d;
2033
2034     __m128i xmm_alpha_lo, xmm_alpha_hi;
2035     __m128i xmm_src_lo, xmm_src_hi;
2036     __m128i xmm_dst_lo, xmm_dst_hi;
2037     __m128i xmm_mask_lo, xmm_mask_hi;
2038
2039     /* call prefetch hint to optimize cache load*/
2040     cache_prefetch ((__m128i*)ps);
2041     cache_prefetch ((__m128i*)pd);
2042     cache_prefetch ((__m128i*)pm);
2043
2044     while (w && (unsigned long)pd & 15)
2045     {
2046         s = *ps++;
2047         m = *pm++;
2048         d = *pd;
2049
2050         *pd++ = pack_1x64_32 (
2051             pix_multiply_1x64 (
2052                 pix_multiply_1x64 (
2053                     unpack_32_1x64 (s), unpack_32_1x64 (m)),
2054                 negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
2055         w--;
2056     }
2057
2058     /* call prefetch hint to optimize cache load*/
2059     cache_prefetch ((__m128i*)ps);
2060     cache_prefetch ((__m128i*)pd);
2061     cache_prefetch ((__m128i*)pm);
2062
2063     while (w >= 4)
2064     {
2065         /* fill cache line with next memory */
2066         cache_prefetch_next ((__m128i*)ps);
2067         cache_prefetch_next ((__m128i*)pd);
2068         cache_prefetch_next ((__m128i*)pm);
2069
2070         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2071         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2072         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2073
2074         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2075         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2076         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2077
2078         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2079                             &xmm_alpha_lo, &xmm_alpha_hi);
2080         negate_2x128 (xmm_alpha_lo, xmm_alpha_hi,
2081                       &xmm_alpha_lo, &xmm_alpha_hi);
2082
2083         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2084                             &xmm_mask_lo, &xmm_mask_hi,
2085                             &xmm_dst_lo, &xmm_dst_hi);
2086         pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
2087                             &xmm_alpha_lo, &xmm_alpha_hi,
2088                             &xmm_dst_lo, &xmm_dst_hi);
2089
2090         save_128_aligned (
2091             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2092
2093         ps += 4;
2094         pd += 4;
2095         pm += 4;
2096         w -= 4;
2097     }
2098
2099     while (w)
2100     {
2101         s = *ps++;
2102         m = *pm++;
2103         d = *pd;
2104
2105         *pd++ = pack_1x64_32 (
2106             pix_multiply_1x64 (
2107                 pix_multiply_1x64 (
2108                     unpack_32_1x64 (s), unpack_32_1x64 (m)),
2109                 negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
2110
2111         w--;
2112     }
2113 }
2114
2115 static force_inline void
2116 core_combine_out_reverse_ca_sse2 (uint32_t *      pd,
2117                                   const uint32_t *ps,
2118                                   const uint32_t *pm,
2119                                   int             w)
2120 {
2121     uint32_t s, m, d;
2122
2123     __m128i xmm_alpha_lo, xmm_alpha_hi;
2124     __m128i xmm_src_lo, xmm_src_hi;
2125     __m128i xmm_dst_lo, xmm_dst_hi;
2126     __m128i xmm_mask_lo, xmm_mask_hi;
2127
2128     /* call prefetch hint to optimize cache load*/
2129     cache_prefetch ((__m128i*)ps);
2130     cache_prefetch ((__m128i*)pd);
2131     cache_prefetch ((__m128i*)pm);
2132
2133     while (w && (unsigned long)pd & 15)
2134     {
2135         s = *ps++;
2136         m = *pm++;
2137         d = *pd;
2138
2139         *pd++ = pack_1x64_32 (
2140             pix_multiply_1x64 (
2141                 unpack_32_1x64 (d),
2142                 negate_1x64 (pix_multiply_1x64 (
2143                                  unpack_32_1x64 (m),
2144                                  expand_alpha_1x64 (unpack_32_1x64 (s))))));
2145         w--;
2146     }
2147
2148     /* call prefetch hint to optimize cache load*/
2149     cache_prefetch ((__m128i*)ps);
2150     cache_prefetch ((__m128i*)pd);
2151     cache_prefetch ((__m128i*)pm);
2152
2153     while (w >= 4)
2154     {
2155         /* fill cache line with next memory */
2156         cache_prefetch_next ((__m128i*)ps);
2157         cache_prefetch_next ((__m128i*)pd);
2158         cache_prefetch_next ((__m128i*)pm);
2159
2160         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2161         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2162         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2163
2164         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2165         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2166         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2167
2168         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2169                             &xmm_alpha_lo, &xmm_alpha_hi);
2170
2171         pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2172                             &xmm_alpha_lo, &xmm_alpha_hi,
2173                             &xmm_mask_lo, &xmm_mask_hi);
2174
2175         negate_2x128 (xmm_mask_lo, xmm_mask_hi,
2176                       &xmm_mask_lo, &xmm_mask_hi);
2177
2178         pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
2179                             &xmm_mask_lo, &xmm_mask_hi,
2180                             &xmm_dst_lo, &xmm_dst_hi);
2181
2182         save_128_aligned (
2183             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2184
2185         ps += 4;
2186         pd += 4;
2187         pm += 4;
2188         w -= 4;
2189     }
2190
2191     while (w)
2192     {
2193         s = *ps++;
2194         m = *pm++;
2195         d = *pd;
2196
2197         *pd++ = pack_1x64_32 (
2198             pix_multiply_1x64 (
2199                 unpack_32_1x64 (d),
2200                 negate_1x64 (pix_multiply_1x64 (
2201                                  unpack_32_1x64 (m),
2202                                  expand_alpha_1x64 (unpack_32_1x64 (s))))));
2203         w--;
2204     }
2205 }
2206
2207 static force_inline uint32_t
2208 core_combine_atop_ca_pixel_sse2 (uint32_t src,
2209                                  uint32_t mask,
2210                                  uint32_t dst)
2211 {
2212     __m64 m = unpack_32_1x64 (mask);
2213     __m64 s = unpack_32_1x64 (src);
2214     __m64 d = unpack_32_1x64 (dst);
2215     __m64 sa = expand_alpha_1x64 (s);
2216     __m64 da = expand_alpha_1x64 (d);
2217
2218     s = pix_multiply_1x64 (s, m);
2219     m = negate_1x64 (pix_multiply_1x64 (m, sa));
2220
2221     return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
2222 }
2223
2224 static force_inline void
2225 core_combine_atop_ca_sse2 (uint32_t *      pd,
2226                            const uint32_t *ps,
2227                            const uint32_t *pm,
2228                            int             w)
2229 {
2230     uint32_t s, m, d;
2231
2232     __m128i xmm_src_lo, xmm_src_hi;
2233     __m128i xmm_dst_lo, xmm_dst_hi;
2234     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2235     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2236     __m128i xmm_mask_lo, xmm_mask_hi;
2237
2238     /* call prefetch hint to optimize cache load*/
2239     cache_prefetch ((__m128i*)ps);
2240     cache_prefetch ((__m128i*)pd);
2241     cache_prefetch ((__m128i*)pm);
2242
2243     while (w && (unsigned long)pd & 15)
2244     {
2245         s = *ps++;
2246         m = *pm++;
2247         d = *pd;
2248
2249         *pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
2250         w--;
2251     }
2252
2253     /* call prefetch hint to optimize cache load*/
2254     cache_prefetch ((__m128i*)ps);
2255     cache_prefetch ((__m128i*)pd);
2256     cache_prefetch ((__m128i*)pm);
2257
2258     while (w >= 4)
2259     {
2260         /* fill cache line with next memory */
2261         cache_prefetch_next ((__m128i*)ps);
2262         cache_prefetch_next ((__m128i*)pd);
2263         cache_prefetch_next ((__m128i*)pm);
2264
2265         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2266         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2267         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2268
2269         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2270         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2271         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2272
2273         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2274                             &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2275         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2276                             &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2277
2278         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2279                             &xmm_mask_lo, &xmm_mask_hi,
2280                             &xmm_src_lo, &xmm_src_hi);
2281         pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2282                             &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2283                             &xmm_mask_lo, &xmm_mask_hi);
2284
2285         negate_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2286
2287         pix_add_multiply_2x128 (
2288             &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2289             &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2290             &xmm_dst_lo, &xmm_dst_hi);
2291
2292         save_128_aligned (
2293             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2294
2295         ps += 4;
2296         pd += 4;
2297         pm += 4;
2298         w -= 4;
2299     }
2300
2301     while (w)
2302     {
2303         s = *ps++;
2304         m = *pm++;
2305         d = *pd;
2306
2307         *pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
2308         w--;
2309     }
2310 }
2311
2312 static force_inline uint32_t
2313 core_combine_reverse_atop_ca_pixel_sse2 (uint32_t src,
2314                                          uint32_t mask,
2315                                          uint32_t dst)
2316 {
2317     __m64 m = unpack_32_1x64 (mask);
2318     __m64 s = unpack_32_1x64 (src);
2319     __m64 d = unpack_32_1x64 (dst);
2320
2321     __m64 da = negate_1x64 (expand_alpha_1x64 (d));
2322     __m64 sa = expand_alpha_1x64 (s);
2323
2324     s = pix_multiply_1x64 (s, m);
2325     m = pix_multiply_1x64 (m, sa);
2326
2327     return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
2328 }
2329
2330 static force_inline void
2331 core_combine_reverse_atop_ca_sse2 (uint32_t *      pd,
2332                                    const uint32_t *ps,
2333                                    const uint32_t *pm,
2334                                    int             w)
2335 {
2336     uint32_t s, m, d;
2337
2338     __m128i xmm_src_lo, xmm_src_hi;
2339     __m128i xmm_dst_lo, xmm_dst_hi;
2340     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2341     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2342     __m128i xmm_mask_lo, xmm_mask_hi;
2343
2344     /* call prefetch hint to optimize cache load*/
2345     cache_prefetch ((__m128i*)ps);
2346     cache_prefetch ((__m128i*)pd);
2347     cache_prefetch ((__m128i*)pm);
2348
2349     while (w && (unsigned long)pd & 15)
2350     {
2351         s = *ps++;
2352         m = *pm++;
2353         d = *pd;
2354
2355         *pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
2356         w--;
2357     }
2358
2359     /* call prefetch hint to optimize cache load*/
2360     cache_prefetch ((__m128i*)ps);
2361     cache_prefetch ((__m128i*)pd);
2362     cache_prefetch ((__m128i*)pm);
2363
2364     while (w >= 4)
2365     {
2366         /* fill cache line with next memory */
2367         cache_prefetch_next ((__m128i*)ps);
2368         cache_prefetch_next ((__m128i*)pd);
2369         cache_prefetch_next ((__m128i*)pm);
2370
2371         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2372         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2373         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2374
2375         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2376         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2377         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2378
2379         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2380                             &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2381         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2382                             &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2383
2384         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2385                             &xmm_mask_lo, &xmm_mask_hi,
2386                             &xmm_src_lo, &xmm_src_hi);
2387         pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2388                             &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2389                             &xmm_mask_lo, &xmm_mask_hi);
2390
2391         negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
2392                       &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2393
2394         pix_add_multiply_2x128 (
2395             &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2396             &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2397             &xmm_dst_lo, &xmm_dst_hi);
2398
2399         save_128_aligned (
2400             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2401
2402         ps += 4;
2403         pd += 4;
2404         pm += 4;
2405         w -= 4;
2406     }
2407
2408     while (w)
2409     {
2410         s = *ps++;
2411         m = *pm++;
2412         d = *pd;
2413
2414         *pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
2415         w--;
2416     }
2417 }
2418
2419 static force_inline uint32_t
2420 core_combine_xor_ca_pixel_sse2 (uint32_t src,
2421                                 uint32_t mask,
2422                                 uint32_t dst)
2423 {
2424     __m64 a = unpack_32_1x64 (mask);
2425     __m64 s = unpack_32_1x64 (src);
2426     __m64 d = unpack_32_1x64 (dst);
2427
2428     __m64 alpha_dst = negate_1x64 (pix_multiply_1x64 (
2429                                        a, expand_alpha_1x64 (s)));
2430     __m64 dest      = pix_multiply_1x64 (s, a);
2431     __m64 alpha_src = negate_1x64 (expand_alpha_1x64 (d));
2432
2433     return pack_1x64_32 (pix_add_multiply_1x64 (&d,
2434                                                 &alpha_dst,
2435                                                 &dest,
2436                                                 &alpha_src));
2437 }
2438
2439 static force_inline void
2440 core_combine_xor_ca_sse2 (uint32_t *      pd,
2441                           const uint32_t *ps,
2442                           const uint32_t *pm,
2443                           int             w)
2444 {
2445     uint32_t s, m, d;
2446
2447     __m128i xmm_src_lo, xmm_src_hi;
2448     __m128i xmm_dst_lo, xmm_dst_hi;
2449     __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
2450     __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
2451     __m128i xmm_mask_lo, xmm_mask_hi;
2452
2453     /* call prefetch hint to optimize cache load*/
2454     cache_prefetch ((__m128i*)ps);
2455     cache_prefetch ((__m128i*)pd);
2456     cache_prefetch ((__m128i*)pm);
2457
2458     while (w && (unsigned long)pd & 15)
2459     {
2460         s = *ps++;
2461         m = *pm++;
2462         d = *pd;
2463
2464         *pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
2465         w--;
2466     }
2467
2468     /* call prefetch hint to optimize cache load*/
2469     cache_prefetch ((__m128i*)ps);
2470     cache_prefetch ((__m128i*)pd);
2471     cache_prefetch ((__m128i*)pm);
2472
2473     while (w >= 4)
2474     {
2475         /* fill cache line with next memory */
2476         cache_prefetch_next ((__m128i*)ps);
2477         cache_prefetch_next ((__m128i*)pd);
2478         cache_prefetch_next ((__m128i*)pm);
2479
2480         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2481         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2482         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2483
2484         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2485         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2486         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2487
2488         expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
2489                             &xmm_alpha_src_lo, &xmm_alpha_src_hi);
2490         expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
2491                             &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2492
2493         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2494                             &xmm_mask_lo, &xmm_mask_hi,
2495                             &xmm_src_lo, &xmm_src_hi);
2496         pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
2497                             &xmm_alpha_src_lo, &xmm_alpha_src_hi,
2498                             &xmm_mask_lo, &xmm_mask_hi);
2499
2500         negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
2501                       &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
2502         negate_2x128 (xmm_mask_lo, xmm_mask_hi,
2503                       &xmm_mask_lo, &xmm_mask_hi);
2504
2505         pix_add_multiply_2x128 (
2506             &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
2507             &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
2508             &xmm_dst_lo, &xmm_dst_hi);
2509
2510         save_128_aligned (
2511             (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2512
2513         ps += 4;
2514         pd += 4;
2515         pm += 4;
2516         w -= 4;
2517     }
2518
2519     while (w)
2520     {
2521         s = *ps++;
2522         m = *pm++;
2523         d = *pd;
2524
2525         *pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
2526         w--;
2527     }
2528 }
2529
2530 static force_inline void
2531 core_combine_add_ca_sse2 (uint32_t *      pd,
2532                           const uint32_t *ps,
2533                           const uint32_t *pm,
2534                           int             w)
2535 {
2536     uint32_t s, m, d;
2537
2538     __m128i xmm_src_lo, xmm_src_hi;
2539     __m128i xmm_dst_lo, xmm_dst_hi;
2540     __m128i xmm_mask_lo, xmm_mask_hi;
2541
2542     /* call prefetch hint to optimize cache load*/
2543     cache_prefetch ((__m128i*)ps);
2544     cache_prefetch ((__m128i*)pd);
2545     cache_prefetch ((__m128i*)pm);
2546
2547     while (w && (unsigned long)pd & 15)
2548     {
2549         s = *ps++;
2550         m = *pm++;
2551         d = *pd;
2552
2553         *pd++ = pack_1x64_32 (
2554             _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
2555                                              unpack_32_1x64 (m)),
2556                           unpack_32_1x64 (d)));
2557         w--;
2558     }
2559
2560     /* call prefetch hint to optimize cache load*/
2561     cache_prefetch ((__m128i*)ps);
2562     cache_prefetch ((__m128i*)pd);
2563     cache_prefetch ((__m128i*)pm);
2564
2565     while (w >= 4)
2566     {
2567         /* fill cache line with next memory */
2568         cache_prefetch_next ((__m128i*)ps);
2569         cache_prefetch_next ((__m128i*)pd);
2570         cache_prefetch_next ((__m128i*)pm);
2571
2572         xmm_src_hi = load_128_unaligned ((__m128i*)ps);
2573         xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
2574         xmm_dst_hi = load_128_aligned ((__m128i*)pd);
2575
2576         unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
2577         unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
2578         unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
2579
2580         pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
2581                             &xmm_mask_lo, &xmm_mask_hi,
2582                             &xmm_src_lo, &xmm_src_hi);
2583
2584         save_128_aligned (
2585             (__m128i*)pd, pack_2x128_128 (
2586                 _mm_adds_epu8 (xmm_src_lo, xmm_dst_lo),
2587                 _mm_adds_epu8 (xmm_src_hi, xmm_dst_hi)));
2588
2589         ps += 4;
2590         pd += 4;
2591         pm += 4;
2592         w -= 4;
2593     }
2594
2595     while (w)
2596     {
2597         s = *ps++;
2598         m = *pm++;
2599         d = *pd;
2600
2601         *pd++ = pack_1x64_32 (
2602             _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
2603                                              unpack_32_1x64 (m)),
2604                           unpack_32_1x64 (d)));
2605         w--;
2606     }
2607 }
2608
2609 /* ---------------------------------------------------
2610  * fb_compose_setup_sSE2
2611  */
2612 static force_inline __m64
2613 create_mask_16_64 (uint16_t mask)
2614 {
2615     return _mm_set1_pi16 (mask);
2616 }
2617
2618 static force_inline __m128i
2619 create_mask_16_128 (uint16_t mask)
2620 {
2621     return _mm_set1_epi16 (mask);
2622 }
2623
2624 static force_inline __m64
2625 create_mask_2x32_64 (uint32_t mask0,
2626                      uint32_t mask1)
2627 {
2628     return _mm_set_pi32 (mask0, mask1);
2629 }
2630
2631 static force_inline __m128i
2632 create_mask_2x32_128 (uint32_t mask0,
2633                       uint32_t mask1)
2634 {
2635     return _mm_set_epi32 (mask0, mask1, mask0, mask1);
2636 }
2637
2638 /* SSE2 code patch for fbcompose.c */
2639
2640 static void
2641 sse2_combine_over_u (pixman_implementation_t *imp,
2642                      pixman_op_t              op,
2643                      uint32_t *               dst,
2644                      const uint32_t *         src,
2645                      const uint32_t *         mask,
2646                      int                      width)
2647 {
2648     core_combine_over_u_sse2 (dst, src, mask, width);
2649     _mm_empty ();
2650 }
2651
2652 static void
2653 sse2_combine_over_reverse_u (pixman_implementation_t *imp,
2654                              pixman_op_t              op,
2655                              uint32_t *               dst,
2656                              const uint32_t *         src,
2657                              const uint32_t *         mask,
2658                              int                      width)
2659 {
2660     core_combine_over_reverse_u_sse2 (dst, src, mask, width);
2661     _mm_empty ();
2662 }
2663
2664 static void
2665 sse2_combine_in_u (pixman_implementation_t *imp,
2666                    pixman_op_t              op,
2667                    uint32_t *               dst,
2668                    const uint32_t *         src,
2669                    const uint32_t *         mask,
2670                    int                      width)
2671 {
2672     core_combine_in_u_sse2 (dst, src, mask, width);
2673     _mm_empty ();
2674 }
2675
2676 static void
2677 sse2_combine_in_reverse_u (pixman_implementation_t *imp,
2678                            pixman_op_t              op,
2679                            uint32_t *               dst,
2680                            const uint32_t *         src,
2681                            const uint32_t *         mask,
2682                            int                      width)
2683 {
2684     core_combine_reverse_in_u_sse2 (dst, src, mask, width);
2685     _mm_empty ();
2686 }
2687
2688 static void
2689 sse2_combine_out_u (pixman_implementation_t *imp,
2690                     pixman_op_t              op,
2691                     uint32_t *               dst,
2692                     const uint32_t *         src,
2693                     const uint32_t *         mask,
2694                     int                      width)
2695 {
2696     core_combine_out_u_sse2 (dst, src, mask, width);
2697     _mm_empty ();
2698 }
2699
2700 static void
2701 sse2_combine_out_reverse_u (pixman_implementation_t *imp,
2702                             pixman_op_t              op,
2703                             uint32_t *               dst,
2704                             const uint32_t *         src,
2705                             const uint32_t *         mask,
2706                             int                      width)
2707 {
2708     core_combine_reverse_out_u_sse2 (dst, src, mask, width);
2709     _mm_empty ();
2710 }
2711
2712 static void
2713 sse2_combine_atop_u (pixman_implementation_t *imp,
2714                      pixman_op_t              op,
2715                      uint32_t *               dst,
2716                      const uint32_t *         src,
2717                      const uint32_t *         mask,
2718                      int                      width)
2719 {
2720     core_combine_atop_u_sse2 (dst, src, mask, width);
2721     _mm_empty ();
2722 }
2723
2724 static void
2725 sse2_combine_atop_reverse_u (pixman_implementation_t *imp,
2726                              pixman_op_t              op,
2727                              uint32_t *               dst,
2728                              const uint32_t *         src,
2729                              const uint32_t *         mask,
2730                              int                      width)
2731 {
2732     core_combine_reverse_atop_u_sse2 (dst, src, mask, width);
2733     _mm_empty ();
2734 }
2735
2736 static void
2737 sse2_combine_xor_u (pixman_implementation_t *imp,
2738                     pixman_op_t              op,
2739                     uint32_t *               dst,
2740                     const uint32_t *         src,
2741                     const uint32_t *         mask,
2742                     int                      width)
2743 {
2744     core_combine_xor_u_sse2 (dst, src, mask, width);
2745     _mm_empty ();
2746 }
2747
2748 static void
2749 sse2_combine_add_u (pixman_implementation_t *imp,
2750                     pixman_op_t              op,
2751                     uint32_t *               dst,
2752                     const uint32_t *         src,
2753                     const uint32_t *         mask,
2754                     int                      width)
2755 {
2756     core_combine_add_u_sse2 (dst, src, mask, width);
2757     _mm_empty ();
2758 }
2759
2760 static void
2761 sse2_combine_saturate_u (pixman_implementation_t *imp,
2762                          pixman_op_t              op,
2763                          uint32_t *               dst,
2764                          const uint32_t *         src,
2765                          const uint32_t *         mask,
2766                          int                      width)
2767 {
2768     core_combine_saturate_u_sse2 (dst, src, mask, width);
2769     _mm_empty ();
2770 }
2771
2772 static void
2773 sse2_combine_src_ca (pixman_implementation_t *imp,
2774                      pixman_op_t              op,
2775                      uint32_t *               dst,
2776                      const uint32_t *         src,
2777                      const uint32_t *         mask,
2778                      int                      width)
2779 {
2780     core_combine_src_ca_sse2 (dst, src, mask, width);
2781     _mm_empty ();
2782 }
2783
2784 static void
2785 sse2_combine_over_ca (pixman_implementation_t *imp,
2786                       pixman_op_t              op,
2787                       uint32_t *               dst,
2788                       const uint32_t *         src,
2789                       const uint32_t *         mask,
2790                       int                      width)
2791 {
2792     core_combine_over_ca_sse2 (dst, src, mask, width);
2793     _mm_empty ();
2794 }
2795
2796 static void
2797 sse2_combine_over_reverse_ca (pixman_implementation_t *imp,
2798                               pixman_op_t              op,
2799                               uint32_t *               dst,
2800                               const uint32_t *         src,
2801                               const uint32_t *         mask,
2802                               int                      width)
2803 {
2804     core_combine_over_reverse_ca_sse2 (dst, src, mask, width);
2805     _mm_empty ();
2806 }
2807
2808 static void
2809 sse2_combine_in_ca (pixman_implementation_t *imp,
2810                     pixman_op_t              op,
2811                     uint32_t *               dst,
2812                     const uint32_t *         src,
2813                     const uint32_t *         mask,
2814                     int                      width)
2815 {
2816     core_combine_in_ca_sse2 (dst, src, mask, width);
2817     _mm_empty ();
2818 }
2819
2820 static void
2821 sse2_combine_in_reverse_ca (pixman_implementation_t *imp,
2822                             pixman_op_t              op,
2823                             uint32_t *               dst,
2824                             const uint32_t *         src,
2825                             const uint32_t *         mask,
2826                             int                      width)
2827 {
2828     core_combine_in_reverse_ca_sse2 (dst, src, mask, width);
2829     _mm_empty ();
2830 }
2831
2832 static void
2833 sse2_combine_out_ca (pixman_implementation_t *imp,
2834                      pixman_op_t              op,
2835                      uint32_t *               dst,
2836                      const uint32_t *         src,
2837                      const uint32_t *         mask,
2838                      int                      width)
2839 {
2840     core_combine_out_ca_sse2 (dst, src, mask, width);
2841     _mm_empty ();
2842 }
2843
2844 static void
2845 sse2_combine_out_reverse_ca (pixman_implementation_t *imp,
2846                              pixman_op_t              op,
2847                              uint32_t *               dst,
2848                              const uint32_t *         src,
2849                              const uint32_t *         mask,
2850                              int                      width)
2851 {
2852     core_combine_out_reverse_ca_sse2 (dst, src, mask, width);
2853     _mm_empty ();
2854 }
2855
2856 static void
2857 sse2_combine_atop_ca (pixman_implementation_t *imp,
2858                       pixman_op_t              op,
2859                       uint32_t *               dst,
2860                       const uint32_t *         src,
2861                       const uint32_t *         mask,
2862                       int                      width)
2863 {
2864     core_combine_atop_ca_sse2 (dst, src, mask, width);
2865     _mm_empty ();
2866 }
2867
2868 static void
2869 sse2_combine_atop_reverse_ca (pixman_implementation_t *imp,
2870                               pixman_op_t              op,
2871                               uint32_t *               dst,
2872                               const uint32_t *         src,
2873                               const uint32_t *         mask,
2874                               int                      width)
2875 {
2876     core_combine_reverse_atop_ca_sse2 (dst, src, mask, width);
2877     _mm_empty ();
2878 }
2879
2880 static void
2881 sse2_combine_xor_ca (pixman_implementation_t *imp,
2882                      pixman_op_t              op,
2883                      uint32_t *               dst,
2884                      const uint32_t *         src,
2885                      const uint32_t *         mask,
2886                      int                      width)
2887 {
2888     core_combine_xor_ca_sse2 (dst, src, mask, width);
2889     _mm_empty ();
2890 }
2891
2892 static void
2893 sse2_combine_add_ca (pixman_implementation_t *imp,
2894                      pixman_op_t              op,
2895                      uint32_t *               dst,
2896                      const uint32_t *         src,
2897                      const uint32_t *         mask,
2898                      int                      width)
2899 {
2900     core_combine_add_ca_sse2 (dst, src, mask, width);
2901     _mm_empty ();
2902 }
2903
2904 /* -------------------------------------------------------------------
2905  * composite_over_n_8888
2906  */
2907
2908 static void
2909 sse2_composite_over_n_8888 (pixman_implementation_t *imp,
2910                             pixman_op_t              op,
2911                             pixman_image_t *         src_image,
2912                             pixman_image_t *         mask_image,
2913                             pixman_image_t *         dst_image,
2914                             int32_t                  src_x,
2915                             int32_t                  src_y,
2916                             int32_t                  mask_x,
2917                             int32_t                  mask_y,
2918                             int32_t                  dest_x,
2919                             int32_t                  dest_y,
2920                             int32_t                  width,
2921                             int32_t                  height)
2922 {
2923     uint32_t src;
2924     uint32_t    *dst_line, *dst, d;
2925     uint16_t w;
2926     int dst_stride;
2927     __m128i xmm_src, xmm_alpha;
2928     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
2929
2930     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
2931
2932     if (src == 0)
2933         return;
2934
2935     PIXMAN_IMAGE_GET_LINE (
2936         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
2937
2938     xmm_src = expand_pixel_32_1x128 (src);
2939     xmm_alpha = expand_alpha_1x128 (xmm_src);
2940
2941     while (height--)
2942     {
2943         dst = dst_line;
2944
2945         /* call prefetch hint to optimize cache load*/
2946         cache_prefetch ((__m128i*)dst);
2947
2948         dst_line += dst_stride;
2949         w = width;
2950
2951         while (w && (unsigned long)dst & 15)
2952         {
2953             d = *dst;
2954             *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
2955                                               _mm_movepi64_pi64 (xmm_alpha),
2956                                               unpack_32_1x64 (d)));
2957             w--;
2958         }
2959
2960         cache_prefetch ((__m128i*)dst);
2961
2962         while (w >= 4)
2963         {
2964             /* fill cache line with next memory */
2965             cache_prefetch_next ((__m128i*)dst);
2966
2967             xmm_dst = load_128_aligned ((__m128i*)dst);
2968
2969             unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
2970
2971             over_2x128 (&xmm_src, &xmm_src,
2972                         &xmm_alpha, &xmm_alpha,
2973                         &xmm_dst_lo, &xmm_dst_hi);
2974
2975             /* rebuid the 4 pixel data and save*/
2976             save_128_aligned (
2977                 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
2978
2979             w -= 4;
2980             dst += 4;
2981         }
2982
2983         while (w)
2984         {
2985             d = *dst;
2986             *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
2987                                               _mm_movepi64_pi64 (xmm_alpha),
2988                                               unpack_32_1x64 (d)));
2989             w--;
2990         }
2991
2992     }
2993     _mm_empty ();
2994 }
2995
2996 /* ---------------------------------------------------------------------
2997  * composite_over_n_0565
2998  */
2999 static void
3000 sse2_composite_over_n_0565 (pixman_implementation_t *imp,
3001                             pixman_op_t              op,
3002                             pixman_image_t *         src_image,
3003                             pixman_image_t *         mask_image,
3004                             pixman_image_t *         dst_image,
3005                             int32_t                  src_x,
3006                             int32_t                  src_y,
3007                             int32_t                  mask_x,
3008                             int32_t                  mask_y,
3009                             int32_t                  dest_x,
3010                             int32_t                  dest_y,
3011                             int32_t                  width,
3012                             int32_t                  height)
3013 {
3014     uint32_t src;
3015     uint16_t    *dst_line, *dst, d;
3016     uint16_t w;
3017     int dst_stride;
3018     __m128i xmm_src, xmm_alpha;
3019     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
3020
3021     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3022
3023     if (src == 0)
3024         return;
3025
3026     PIXMAN_IMAGE_GET_LINE (
3027         dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
3028
3029     xmm_src = expand_pixel_32_1x128 (src);
3030     xmm_alpha = expand_alpha_1x128 (xmm_src);
3031
3032     while (height--)
3033     {
3034         dst = dst_line;
3035
3036         /* call prefetch hint to optimize cache load*/
3037         cache_prefetch ((__m128i*)dst);
3038
3039         dst_line += dst_stride;
3040         w = width;
3041
3042         while (w && (unsigned long)dst & 15)
3043         {
3044             d = *dst;
3045
3046             *dst++ = pack_565_32_16 (
3047                 pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
3048                                          _mm_movepi64_pi64 (xmm_alpha),
3049                                          expand565_16_1x64 (d))));
3050             w--;
3051         }
3052
3053         /* call prefetch hint to optimize cache load*/
3054         cache_prefetch ((__m128i*)dst);
3055
3056         while (w >= 8)
3057         {
3058             /* fill cache line with next memory */
3059             cache_prefetch_next ((__m128i*)dst);
3060
3061             xmm_dst = load_128_aligned ((__m128i*)dst);
3062
3063             unpack_565_128_4x128 (xmm_dst,
3064                                   &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3065
3066             over_2x128 (&xmm_src, &xmm_src,
3067                         &xmm_alpha, &xmm_alpha,
3068                         &xmm_dst0, &xmm_dst1);
3069             over_2x128 (&xmm_src, &xmm_src,
3070                         &xmm_alpha, &xmm_alpha,
3071                         &xmm_dst2, &xmm_dst3);
3072
3073             xmm_dst = pack_565_4x128_128 (
3074                 &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3075
3076             save_128_aligned ((__m128i*)dst, xmm_dst);
3077
3078             dst += 8;
3079             w -= 8;
3080         }
3081
3082         while (w--)
3083         {
3084             d = *dst;
3085             *dst++ = pack_565_32_16 (
3086                 pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
3087                                          _mm_movepi64_pi64 (xmm_alpha),
3088                                          expand565_16_1x64 (d))));
3089         }
3090     }
3091
3092     _mm_empty ();
3093 }
3094
3095 /* ------------------------------
3096  * composite_add_n_8888_8888_ca
3097  */
3098 static void
3099 sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
3100                                    pixman_op_t              op,
3101                                    pixman_image_t *         src_image,
3102                                    pixman_image_t *         mask_image,
3103                                    pixman_image_t *         dst_image,
3104                                    int32_t                  src_x,
3105                                    int32_t                  src_y,
3106                                    int32_t                  mask_x,
3107                                    int32_t                  mask_y,
3108                                    int32_t                  dest_x,
3109                                    int32_t                  dest_y,
3110                                    int32_t                  width,
3111                                    int32_t                  height)
3112 {
3113     uint32_t src, srca;
3114     uint32_t    *dst_line, d;
3115     uint32_t    *mask_line, m;
3116     uint32_t pack_cmp;
3117     int dst_stride, mask_stride;
3118
3119     __m128i xmm_src, xmm_alpha;
3120     __m128i xmm_dst;
3121     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3122
3123     __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3124
3125     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3126     srca = src >> 24;
3127
3128     if (src == 0)
3129         return;
3130
3131     PIXMAN_IMAGE_GET_LINE (
3132         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3133     PIXMAN_IMAGE_GET_LINE (
3134         mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
3135
3136     xmm_src = _mm_unpacklo_epi8 (
3137         create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
3138     xmm_alpha = expand_alpha_1x128 (xmm_src);
3139     mmx_src   = _mm_movepi64_pi64 (xmm_src);
3140     mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
3141
3142     while (height--)
3143     {
3144         int w = width;
3145         const uint32_t *pm = (uint32_t *)mask_line;
3146         uint32_t *pd = (uint32_t *)dst_line;
3147
3148         dst_line += dst_stride;
3149         mask_line += mask_stride;
3150
3151         /* call prefetch hint to optimize cache load*/
3152         cache_prefetch ((__m128i*)pd);
3153         cache_prefetch ((__m128i*)pm);
3154
3155         while (w && (unsigned long)pd & 15)
3156         {
3157             m = *pm++;
3158
3159             if (m)
3160             {
3161                 d = *pd;
3162
3163                 mmx_mask = unpack_32_1x64 (m);
3164                 mmx_dest = unpack_32_1x64 (d);
3165
3166                 *pd = pack_1x64_32 (
3167                     _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
3168             }
3169
3170             pd++;
3171             w--;
3172         }
3173
3174         /* call prefetch hint to optimize cache load*/
3175         cache_prefetch ((__m128i*)pd);
3176         cache_prefetch ((__m128i*)pm);
3177
3178         while (w >= 4)
3179         {
3180             /* fill cache line with next memory */
3181             cache_prefetch_next ((__m128i*)pd);
3182             cache_prefetch_next ((__m128i*)pm);
3183
3184             xmm_mask = load_128_unaligned ((__m128i*)pm);
3185
3186             pack_cmp =
3187                 _mm_movemask_epi8 (
3188                     _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
3189
3190             /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
3191             if (pack_cmp != 0xffff)
3192             {
3193                 xmm_dst = load_128_aligned ((__m128i*)pd);
3194
3195                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3196
3197                 pix_multiply_2x128 (&xmm_src, &xmm_src,
3198                                     &xmm_mask_lo, &xmm_mask_hi,
3199                                     &xmm_mask_lo, &xmm_mask_hi);
3200                 xmm_mask_hi = pack_2x128_128 (xmm_mask_lo, xmm_mask_hi);
3201
3202                 save_128_aligned (
3203                     (__m128i*)pd, _mm_adds_epu8 (xmm_mask_hi, xmm_dst));
3204             }
3205
3206             pd += 4;
3207             pm += 4;
3208             w -= 4;
3209         }
3210
3211         while (w)
3212         {
3213             m = *pm++;
3214
3215             if (m)
3216             {
3217                 d = *pd;
3218
3219                 mmx_mask = unpack_32_1x64 (m);
3220                 mmx_dest = unpack_32_1x64 (d);
3221
3222                 *pd = pack_1x64_32 (
3223                     _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
3224             }
3225
3226             pd++;
3227             w--;
3228         }
3229     }
3230
3231     _mm_empty ();
3232 }
3233
3234 /* ---------------------------------------------------------------------------
3235  * composite_over_n_8888_8888_ca
3236  */
3237
3238 static void
3239 sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
3240                                     pixman_op_t              op,
3241                                     pixman_image_t *         src_image,
3242                                     pixman_image_t *         mask_image,
3243                                     pixman_image_t *         dst_image,
3244                                     int32_t                  src_x,
3245                                     int32_t                  src_y,
3246                                     int32_t                  mask_x,
3247                                     int32_t                  mask_y,
3248                                     int32_t                  dest_x,
3249                                     int32_t                  dest_y,
3250                                     int32_t                  width,
3251                                     int32_t                  height)
3252 {
3253     uint32_t src;
3254     uint32_t    *dst_line, d;
3255     uint32_t    *mask_line, m;
3256     uint32_t pack_cmp;
3257     int dst_stride, mask_stride;
3258
3259     __m128i xmm_src, xmm_alpha;
3260     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3261     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3262
3263     __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3264
3265     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3266
3267     if (src == 0)
3268         return;
3269
3270     PIXMAN_IMAGE_GET_LINE (
3271         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3272     PIXMAN_IMAGE_GET_LINE (
3273         mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
3274
3275     xmm_src = _mm_unpacklo_epi8 (
3276         create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
3277     xmm_alpha = expand_alpha_1x128 (xmm_src);
3278     mmx_src   = _mm_movepi64_pi64 (xmm_src);
3279     mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
3280
3281     while (height--)
3282     {
3283         int w = width;
3284         const uint32_t *pm = (uint32_t *)mask_line;
3285         uint32_t *pd = (uint32_t *)dst_line;
3286
3287         dst_line += dst_stride;
3288         mask_line += mask_stride;
3289
3290         /* call prefetch hint to optimize cache load*/
3291         cache_prefetch ((__m128i*)pd);
3292         cache_prefetch ((__m128i*)pm);
3293
3294         while (w && (unsigned long)pd & 15)
3295         {
3296             m = *pm++;
3297
3298             if (m)
3299             {
3300                 d = *pd;
3301                 mmx_mask = unpack_32_1x64 (m);
3302                 mmx_dest = unpack_32_1x64 (d);
3303
3304                 *pd = pack_1x64_32 (in_over_1x64 (&mmx_src,
3305                                                   &mmx_alpha,
3306                                                   &mmx_mask,
3307                                                   &mmx_dest));
3308             }
3309
3310             pd++;
3311             w--;
3312         }
3313
3314         /* call prefetch hint to optimize cache load*/
3315         cache_prefetch ((__m128i*)pd);
3316         cache_prefetch ((__m128i*)pm);
3317
3318         while (w >= 4)
3319         {
3320             /* fill cache line with next memory */
3321             cache_prefetch_next ((__m128i*)pd);
3322             cache_prefetch_next ((__m128i*)pm);
3323
3324             xmm_mask = load_128_unaligned ((__m128i*)pm);
3325
3326             pack_cmp =
3327                 _mm_movemask_epi8 (
3328                     _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
3329
3330             /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
3331             if (pack_cmp != 0xffff)
3332             {
3333                 xmm_dst = load_128_aligned ((__m128i*)pd);
3334
3335                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3336                 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3337
3338                 in_over_2x128 (&xmm_src, &xmm_src,
3339                                &xmm_alpha, &xmm_alpha,
3340                                &xmm_mask_lo, &xmm_mask_hi,
3341                                &xmm_dst_lo, &xmm_dst_hi);
3342
3343                 save_128_aligned (
3344                     (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3345             }
3346
3347             pd += 4;
3348             pm += 4;
3349             w -= 4;
3350         }
3351
3352         while (w)
3353         {
3354             m = *pm++;
3355
3356             if (m)
3357             {
3358                 d = *pd;
3359                 mmx_mask = unpack_32_1x64 (m);
3360                 mmx_dest = unpack_32_1x64 (d);
3361
3362                 *pd = pack_1x64_32 (
3363                     in_over_1x64 (&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest));
3364             }
3365
3366             pd++;
3367             w--;
3368         }
3369     }
3370
3371     _mm_empty ();
3372 }
3373
3374 /*---------------------------------------------------------------------
3375  * composite_over_8888_n_8888
3376  */
3377
3378 static void
3379 sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
3380                                  pixman_op_t              op,
3381                                  pixman_image_t *         src_image,
3382                                  pixman_image_t *         mask_image,
3383                                  pixman_image_t *         dst_image,
3384                                  int32_t                  src_x,
3385                                  int32_t                  src_y,
3386                                  int32_t                  mask_x,
3387                                  int32_t                  mask_y,
3388                                  int32_t                  dest_x,
3389                                  int32_t                  dest_y,
3390                                  int32_t                  width,
3391                                  int32_t                  height)
3392 {
3393     uint32_t    *dst_line, *dst;
3394     uint32_t    *src_line, *src;
3395     uint32_t mask;
3396     uint16_t w;
3397     int dst_stride, src_stride;
3398
3399     __m128i xmm_mask;
3400     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3401     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3402     __m128i xmm_alpha_lo, xmm_alpha_hi;
3403
3404     PIXMAN_IMAGE_GET_LINE (
3405         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3406     PIXMAN_IMAGE_GET_LINE (
3407         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3408
3409     mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);
3410
3411     xmm_mask = create_mask_16_128 (mask >> 24);
3412
3413     while (height--)
3414     {
3415         dst = dst_line;
3416         dst_line += dst_stride;
3417         src = src_line;
3418         src_line += src_stride;
3419         w = width;
3420
3421         /* call prefetch hint to optimize cache load*/
3422         cache_prefetch ((__m128i*)dst);
3423         cache_prefetch ((__m128i*)src);
3424
3425         while (w && (unsigned long)dst & 15)
3426         {
3427             uint32_t s = *src++;
3428             uint32_t d = *dst;
3429
3430             __m64 ms = unpack_32_1x64 (s);
3431             __m64 alpha    = expand_alpha_1x64 (ms);
3432             __m64 dest     = _mm_movepi64_pi64 (xmm_mask);
3433             __m64 alpha_dst = unpack_32_1x64 (d);
3434
3435             *dst++ = pack_1x64_32 (
3436                 in_over_1x64 (&ms, &alpha, &dest, &alpha_dst));
3437
3438             w--;
3439         }
3440
3441         /* call prefetch hint to optimize cache load*/
3442         cache_prefetch ((__m128i*)dst);
3443         cache_prefetch ((__m128i*)src);
3444
3445         while (w >= 4)
3446         {
3447             /* fill cache line with next memory */
3448             cache_prefetch_next ((__m128i*)dst);
3449             cache_prefetch_next ((__m128i*)src);
3450
3451             xmm_src = load_128_unaligned ((__m128i*)src);
3452             xmm_dst = load_128_aligned ((__m128i*)dst);
3453
3454             unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3455             unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3456             expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3457                                 &xmm_alpha_lo, &xmm_alpha_hi);
3458
3459             in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
3460                            &xmm_alpha_lo, &xmm_alpha_hi,
3461                            &xmm_mask, &xmm_mask,
3462                            &xmm_dst_lo, &xmm_dst_hi);
3463
3464             save_128_aligned (
3465                 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3466
3467             dst += 4;
3468             src += 4;
3469             w -= 4;
3470         }
3471
3472         while (w)
3473         {
3474             uint32_t s = *src++;
3475             uint32_t d = *dst;
3476
3477             __m64 ms = unpack_32_1x64 (s);
3478             __m64 alpha = expand_alpha_1x64 (ms);
3479             __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
3480             __m64 dest  = unpack_32_1x64 (d);
3481
3482             *dst++ = pack_1x64_32 (
3483                 in_over_1x64 (&ms, &alpha, &mask, &dest));
3484
3485             w--;
3486         }
3487     }
3488
3489     _mm_empty ();
3490 }
3491
3492 /* ---------------------------------------------------------------------
3493  * composite_over_x888_n_8888
3494  */
3495 static void
3496 sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
3497                                  pixman_op_t              op,
3498                                  pixman_image_t *         src_image,
3499                                  pixman_image_t *         mask_image,
3500                                  pixman_image_t *         dst_image,
3501                                  int32_t                  src_x,
3502                                  int32_t                  src_y,
3503                                  int32_t                  mask_x,
3504                                  int32_t                  mask_y,
3505                                  int32_t                  dest_x,
3506                                  int32_t                  dest_y,
3507                                  int32_t                  width,
3508                                  int32_t                  height)
3509 {
3510     uint32_t    *dst_line, *dst;
3511     uint32_t    *src_line, *src;
3512     uint32_t mask;
3513     int dst_stride, src_stride;
3514     uint16_t w;
3515
3516     __m128i xmm_mask, xmm_alpha;
3517     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3518     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3519
3520     PIXMAN_IMAGE_GET_LINE (
3521         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3522     PIXMAN_IMAGE_GET_LINE (
3523         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3524
3525     mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);
3526
3527     xmm_mask = create_mask_16_128 (mask >> 24);
3528     xmm_alpha = mask_00ff;
3529
3530     while (height--)
3531     {
3532         dst = dst_line;
3533         dst_line += dst_stride;
3534         src = src_line;
3535         src_line += src_stride;
3536         w = width;
3537
3538         /* call prefetch hint to optimize cache load*/
3539         cache_prefetch ((__m128i*)dst);
3540         cache_prefetch ((__m128i*)src);
3541
3542         while (w && (unsigned long)dst & 15)
3543         {
3544             uint32_t s = (*src++) | 0xff000000;
3545             uint32_t d = *dst;
3546
3547             __m64 src   = unpack_32_1x64 (s);
3548             __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
3549             __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
3550             __m64 dest  = unpack_32_1x64 (d);
3551
3552             *dst++ = pack_1x64_32 (
3553                 in_over_1x64 (&src, &alpha, &mask, &dest));
3554
3555             w--;
3556         }
3557
3558         /* call prefetch hint to optimize cache load*/
3559         cache_prefetch ((__m128i*)dst);
3560         cache_prefetch ((__m128i*)src);
3561
3562         while (w >= 4)
3563         {
3564             /* fill cache line with next memory */
3565             cache_prefetch_next ((__m128i*)dst);
3566             cache_prefetch_next ((__m128i*)src);
3567
3568             xmm_src = _mm_or_si128 (
3569                 load_128_unaligned ((__m128i*)src), mask_ff000000);
3570             xmm_dst = load_128_aligned ((__m128i*)dst);
3571
3572             unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3573             unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3574
3575             in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
3576                            &xmm_alpha, &xmm_alpha,
3577                            &xmm_mask, &xmm_mask,
3578                            &xmm_dst_lo, &xmm_dst_hi);
3579
3580             save_128_aligned (
3581                 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3582
3583             dst += 4;
3584             src += 4;
3585             w -= 4;
3586
3587         }
3588
3589         while (w)
3590         {
3591             uint32_t s = (*src++) | 0xff000000;
3592             uint32_t d = *dst;
3593
3594             __m64 src  = unpack_32_1x64 (s);
3595             __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
3596             __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
3597             __m64 dest  = unpack_32_1x64 (d);
3598
3599             *dst++ = pack_1x64_32 (
3600                 in_over_1x64 (&src, &alpha, &mask, &dest));
3601
3602             w--;
3603         }
3604     }
3605
3606     _mm_empty ();
3607 }
3608
3609 /* --------------------------------------------------------------------
3610  * composite_over_8888_8888
3611  */
3612 static void
3613 sse2_composite_over_8888_8888 (pixman_implementation_t *imp,
3614                                pixman_op_t              op,
3615                                pixman_image_t *         src_image,
3616                                pixman_image_t *         mask_image,
3617                                pixman_image_t *         dst_image,
3618                                int32_t                  src_x,
3619                                int32_t                  src_y,
3620                                int32_t                  mask_x,
3621                                int32_t                  mask_y,
3622                                int32_t                  dest_x,
3623                                int32_t                  dest_y,
3624                                int32_t                  width,
3625                                int32_t                  height)
3626 {
3627     int dst_stride, src_stride;
3628     uint32_t    *dst_line, *dst;
3629     uint32_t    *src_line, *src;
3630
3631     PIXMAN_IMAGE_GET_LINE (
3632         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3633     PIXMAN_IMAGE_GET_LINE (
3634         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3635
3636     dst = dst_line;
3637     src = src_line;
3638
3639     while (height--)
3640     {
3641         core_combine_over_u_sse2 (dst, src, NULL, width);
3642
3643         dst += dst_stride;
3644         src += src_stride;
3645     }
3646     _mm_empty ();
3647 }
3648
3649 /* ------------------------------------------------------------------
3650  * composite_over_8888_0565
3651  */
3652 static force_inline uint16_t
3653 composite_over_8888_0565pixel (uint32_t src, uint16_t dst)
3654 {
3655     __m64 ms;
3656
3657     ms = unpack_32_1x64 (src);
3658     return pack_565_32_16 (
3659         pack_1x64_32 (
3660             over_1x64 (
3661                 ms, expand_alpha_1x64 (ms), expand565_16_1x64 (dst))));
3662 }
3663
3664 static void
3665 sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
3666                                pixman_op_t              op,
3667                                pixman_image_t *         src_image,
3668                                pixman_image_t *         mask_image,
3669                                pixman_image_t *         dst_image,
3670                                int32_t                  src_x,
3671                                int32_t                  src_y,
3672                                int32_t                  mask_x,
3673                                int32_t                  mask_y,
3674                                int32_t                  dest_x,
3675                                int32_t                  dest_y,
3676                                int32_t                  width,
3677                                int32_t                  height)
3678 {
3679     uint16_t    *dst_line, *dst, d;
3680     uint32_t    *src_line, *src, s;
3681     int dst_stride, src_stride;
3682     uint16_t w;
3683
3684     __m128i xmm_alpha_lo, xmm_alpha_hi;
3685     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
3686     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
3687
3688     PIXMAN_IMAGE_GET_LINE (
3689         dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
3690     PIXMAN_IMAGE_GET_LINE (
3691         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
3692
3693 #if 0
3694     /* FIXME
3695      *
3696      * I copy the code from MMX one and keep the fixme.
3697      * If it's a problem there, probably is a problem here.
3698      */
3699     assert (src_image->drawable == mask_image->drawable);
3700 #endif
3701
3702     while (height--)
3703     {
3704         dst = dst_line;
3705         src = src_line;
3706
3707         /* call prefetch hint to optimize cache load*/
3708         cache_prefetch ((__m128i*)src);
3709         cache_prefetch ((__m128i*)dst);
3710
3711         dst_line += dst_stride;
3712         src_line += src_stride;
3713         w = width;
3714
3715         /* Align dst on a 16-byte boundary */
3716         while (w &&
3717                ((unsigned long)dst & 15))
3718         {
3719             s = *src++;
3720             d = *dst;
3721
3722             *dst++ = composite_over_8888_0565pixel (s, d);
3723             w--;
3724         }
3725
3726         /* call prefetch hint to optimize cache load*/
3727         cache_prefetch ((__m128i*)src);
3728         cache_prefetch ((__m128i*)dst);
3729
3730         /* It's a 8 pixel loop */
3731         while (w >= 8)
3732         {
3733             /* fill cache line with next memory */
3734             cache_prefetch_next ((__m128i*)src);
3735             cache_prefetch_next ((__m128i*)dst);
3736
3737             /* I'm loading unaligned because I'm not sure
3738              * about the address alignment.
3739              */
3740             xmm_src = load_128_unaligned ((__m128i*) src);
3741             xmm_dst = load_128_aligned ((__m128i*) dst);
3742
3743             /* Unpacking */
3744             unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3745             unpack_565_128_4x128 (xmm_dst,
3746                                   &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
3747             expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3748                                 &xmm_alpha_lo, &xmm_alpha_hi);
3749
3750             /* I'm loading next 4 pixels from memory
3751              * before to optimze the memory read.
3752              */
3753             xmm_src = load_128_unaligned ((__m128i*) (src + 4));
3754
3755             over_2x128 (&xmm_src_lo, &xmm_src_hi,
3756                         &xmm_alpha_lo, &xmm_alpha_hi,
3757                         &xmm_dst0, &xmm_dst1);
3758
3759             /* Unpacking */
3760             unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
3761             expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
3762                                 &xmm_alpha_lo, &xmm_alpha_hi);
3763
3764             over_2x128 (&xmm_src_lo, &xmm_src_hi,
3765                         &xmm_alpha_lo, &xmm_alpha_hi,
3766                         &xmm_dst2, &xmm_dst3);
3767
3768             save_128_aligned (
3769                 (__m128i*)dst, pack_565_4x128_128 (
3770                     &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
3771
3772             w -= 8;
3773             dst += 8;
3774             src += 8;
3775         }
3776
3777         while (w--)
3778         {
3779             s = *src++;
3780             d = *dst;
3781
3782             *dst++ = composite_over_8888_0565pixel (s, d);
3783         }
3784     }
3785
3786     _mm_empty ();
3787 }
3788
3789 /* -----------------------------------------------------------------
3790  * composite_over_n_8_8888
3791  */
3792
3793 static void
3794 sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
3795                               pixman_op_t              op,
3796                               pixman_image_t *         src_image,
3797                               pixman_image_t *         mask_image,
3798                               pixman_image_t *         dst_image,
3799                               int32_t                  src_x,
3800                               int32_t                  src_y,
3801                               int32_t                  mask_x,
3802                               int32_t                  mask_y,
3803                               int32_t                  dest_x,
3804                               int32_t                  dest_y,
3805                               int32_t                  width,
3806                               int32_t                  height)
3807 {
3808     uint32_t src, srca;
3809     uint32_t *dst_line, *dst;
3810     uint8_t *mask_line, *mask;
3811     int dst_stride, mask_stride;
3812     uint16_t w;
3813     uint32_t m, d;
3814
3815     __m128i xmm_src, xmm_alpha, xmm_def;
3816     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
3817     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
3818
3819     __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
3820
3821     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
3822
3823     srca = src >> 24;
3824     if (src == 0)
3825         return;
3826
3827     PIXMAN_IMAGE_GET_LINE (
3828         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
3829     PIXMAN_IMAGE_GET_LINE (
3830         mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
3831
3832     xmm_def = create_mask_2x32_128 (src, src);
3833     xmm_src = expand_pixel_32_1x128 (src);
3834     xmm_alpha = expand_alpha_1x128 (xmm_src);
3835     mmx_src   = _mm_movepi64_pi64 (xmm_src);
3836     mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
3837
3838     while (height--)
3839     {
3840         dst = dst_line;
3841         dst_line += dst_stride;
3842         mask = mask_line;
3843         mask_line += mask_stride;
3844         w = width;
3845
3846         /* call prefetch hint to optimize cache load*/
3847         cache_prefetch ((__m128i*)mask);
3848         cache_prefetch ((__m128i*)dst);
3849
3850         while (w && (unsigned long)dst & 15)
3851         {
3852             uint8_t m = *mask++;
3853
3854             if (m)
3855             {
3856                 d = *dst;
3857                 mmx_mask = expand_pixel_8_1x64 (m);
3858                 mmx_dest = unpack_32_1x64 (d);
3859
3860                 *dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
3861                                                    &mmx_alpha,
3862                                                    &mmx_mask,
3863                                                    &mmx_dest));
3864             }
3865
3866             w--;
3867             dst++;
3868         }
3869
3870         /* call prefetch hint to optimize cache load*/
3871         cache_prefetch ((__m128i*)mask);
3872         cache_prefetch ((__m128i*)dst);
3873
3874         while (w >= 4)
3875         {
3876             /* fill cache line with next memory */
3877             cache_prefetch_next ((__m128i*)mask);
3878             cache_prefetch_next ((__m128i*)dst);
3879
3880             m = *((uint32_t*)mask);
3881
3882             if (srca == 0xff && m == 0xffffffff)
3883             {
3884                 save_128_aligned ((__m128i*)dst, xmm_def);
3885             }
3886             else if (m)
3887             {
3888                 xmm_dst = load_128_aligned ((__m128i*) dst);
3889                 xmm_mask = unpack_32_1x128 (m);
3890                 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
3891
3892                 /* Unpacking */
3893                 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
3894                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
3895
3896                 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
3897                                         &xmm_mask_lo, &xmm_mask_hi);
3898
3899                 in_over_2x128 (&xmm_src, &xmm_src,
3900                                &xmm_alpha, &xmm_alpha,
3901                                &xmm_mask_lo, &xmm_mask_hi,
3902                                &xmm_dst_lo, &xmm_dst_hi);
3903
3904                 save_128_aligned (
3905                     (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
3906             }
3907
3908             w -= 4;
3909             dst += 4;
3910             mask += 4;
3911         }
3912
3913         while (w)
3914         {
3915             uint8_t m = *mask++;
3916
3917             if (m)
3918             {
3919                 d = *dst;
3920                 mmx_mask = expand_pixel_8_1x64 (m);
3921                 mmx_dest = unpack_32_1x64 (d);
3922
3923                 *dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
3924                                                    &mmx_alpha,
3925                                                    &mmx_mask,
3926                                                    &mmx_dest));
3927             }
3928
3929             w--;
3930             dst++;
3931         }
3932     }
3933
3934     _mm_empty ();
3935 }
3936
3937 /* ----------------------------------------------------------------
3938  * composite_over_n_8_8888
3939  */
3940
3941 pixman_bool_t
3942 pixman_fill_sse2 (uint32_t *bits,
3943                   int       stride,
3944                   int       bpp,
3945                   int       x,
3946                   int       y,
3947                   int       width,
3948                   int       height,
3949                   uint32_t  data)
3950 {
3951     uint32_t byte_width;
3952     uint8_t         *byte_line;
3953
3954     __m128i xmm_def;
3955
3956     if (bpp == 16 && (data >> 16 != (data & 0xffff)))
3957         return FALSE;
3958
3959     if (bpp != 16 && bpp != 32)
3960         return FALSE;
3961
3962     if (bpp == 16)
3963     {
3964         stride = stride * (int) sizeof (uint32_t) / 2;
3965         byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
3966         byte_width = 2 * width;
3967         stride *= 2;
3968     }
3969     else
3970     {
3971         stride = stride * (int) sizeof (uint32_t) / 4;
3972         byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
3973         byte_width = 4 * width;
3974         stride *= 4;
3975     }
3976
3977     cache_prefetch ((__m128i*)byte_line);
3978     xmm_def = create_mask_2x32_128 (data, data);
3979
3980     while (height--)
3981     {
3982         int w;
3983         uint8_t *d = byte_line;
3984         byte_line += stride;
3985         w = byte_width;
3986
3987
3988         cache_prefetch_next ((__m128i*)d);
3989
3990         while (w >= 2 && ((unsigned long)d & 3))
3991         {
3992             *(uint16_t *)d = data;
3993             w -= 2;
3994             d += 2;
3995         }
3996
3997         while (w >= 4 && ((unsigned long)d & 15))
3998         {
3999             *(uint32_t *)d = data;
4000
4001             w -= 4;
4002             d += 4;
4003         }
4004
4005         cache_prefetch_next ((__m128i*)d);
4006
4007         while (w >= 128)
4008         {
4009             cache_prefetch (((__m128i*)d) + 12);
4010
4011             save_128_aligned ((__m128i*)(d),     xmm_def);
4012             save_128_aligned ((__m128i*)(d + 16),  xmm_def);
4013             save_128_aligned ((__m128i*)(d + 32),  xmm_def);
4014             save_128_aligned ((__m128i*)(d + 48),  xmm_def);
4015             save_128_aligned ((__m128i*)(d + 64),  xmm_def);
4016             save_128_aligned ((__m128i*)(d + 80),  xmm_def);
4017             save_128_aligned ((__m128i*)(d + 96),  xmm_def);
4018             save_128_aligned ((__m128i*)(d + 112), xmm_def);
4019
4020             d += 128;
4021             w -= 128;
4022         }
4023
4024         if (w >= 64)
4025         {
4026             cache_prefetch (((__m128i*)d) + 8);
4027
4028             save_128_aligned ((__m128i*)(d),     xmm_def);
4029             save_128_aligned ((__m128i*)(d + 16),  xmm_def);
4030             save_128_aligned ((__m128i*)(d + 32),  xmm_def);
4031             save_128_aligned ((__m128i*)(d + 48),  xmm_def);
4032
4033             d += 64;
4034             w -= 64;
4035         }
4036
4037         cache_prefetch_next ((__m128i*)d);
4038
4039         if (w >= 32)
4040         {
4041             save_128_aligned ((__m128i*)(d),     xmm_def);
4042             save_128_aligned ((__m128i*)(d + 16),  xmm_def);
4043
4044             d += 32;
4045             w -= 32;
4046         }
4047
4048         if (w >= 16)
4049         {
4050             save_128_aligned ((__m128i*)(d),     xmm_def);
4051
4052             d += 16;
4053             w -= 16;
4054         }
4055
4056         cache_prefetch_next ((__m128i*)d);
4057
4058         while (w >= 4)
4059         {
4060             *(uint32_t *)d = data;
4061
4062             w -= 4;
4063             d += 4;
4064         }
4065
4066         if (w >= 2)
4067         {
4068             *(uint16_t *)d = data;
4069             w -= 2;
4070             d += 2;
4071         }
4072     }
4073
4074     _mm_empty ();
4075     return TRUE;
4076 }
4077
4078 static void
4079 sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
4080                              pixman_op_t              op,
4081                              pixman_image_t *         src_image,
4082                              pixman_image_t *         mask_image,
4083                              pixman_image_t *         dst_image,
4084                              int32_t                  src_x,
4085                              int32_t                  src_y,
4086                              int32_t                  mask_x,
4087                              int32_t                  mask_y,
4088                              int32_t                  dest_x,
4089                              int32_t                  dest_y,
4090                              int32_t                  width,
4091                              int32_t                  height)
4092 {
4093     uint32_t src, srca;
4094     uint32_t    *dst_line, *dst;
4095     uint8_t     *mask_line, *mask;
4096     int dst_stride, mask_stride;
4097     uint16_t w;
4098     uint32_t m;
4099
4100     __m128i xmm_src, xmm_def;
4101     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4102
4103     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4104
4105     srca = src >> 24;
4106     if (src == 0)
4107     {
4108         pixman_fill_sse2 (dst_image->bits.bits, dst_image->bits.rowstride,
4109                           PIXMAN_FORMAT_BPP (dst_image->bits.format),
4110                           dest_x, dest_y, width, height, 0);
4111         return;
4112     }
4113
4114     PIXMAN_IMAGE_GET_LINE (
4115         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
4116     PIXMAN_IMAGE_GET_LINE (
4117         mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4118
4119     xmm_def = create_mask_2x32_128 (src, src);
4120     xmm_src = expand_pixel_32_1x128 (src);
4121
4122     while (height--)
4123     {
4124         dst = dst_line;
4125         dst_line += dst_stride;
4126         mask = mask_line;
4127         mask_line += mask_stride;
4128         w = width;
4129
4130         /* call prefetch hint to optimize cache load*/
4131         cache_prefetch ((__m128i*)mask);
4132         cache_prefetch ((__m128i*)dst);
4133
4134         while (w && (unsigned long)dst & 15)
4135         {
4136             uint8_t m = *mask++;
4137
4138             if (m)
4139             {
4140                 *dst = pack_1x64_32 (
4141                     pix_multiply_1x64 (
4142                         _mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
4143             }
4144             else
4145             {
4146                 *dst = 0;
4147             }
4148
4149             w--;
4150             dst++;
4151         }
4152
4153         /* call prefetch hint to optimize cache load*/
4154         cache_prefetch ((__m128i*)mask);
4155         cache_prefetch ((__m128i*)dst);
4156
4157         while (w >= 4)
4158         {
4159             /* fill cache line with next memory */
4160             cache_prefetch_next ((__m128i*)mask);
4161             cache_prefetch_next ((__m128i*)dst);
4162
4163             m = *((uint32_t*)mask);
4164
4165             if (srca == 0xff && m == 0xffffffff)
4166             {
4167                 save_128_aligned ((__m128i*)dst, xmm_def);
4168             }
4169             else if (m)
4170             {
4171                 xmm_mask = unpack_32_1x128 (m);
4172                 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
4173
4174                 /* Unpacking */
4175                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4176
4177                 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
4178                                         &xmm_mask_lo, &xmm_mask_hi);
4179
4180                 pix_multiply_2x128 (&xmm_src, &xmm_src,
4181                                     &xmm_mask_lo, &xmm_mask_hi,
4182                                     &xmm_mask_lo, &xmm_mask_hi);
4183
4184                 save_128_aligned (
4185                     (__m128i*)dst, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
4186             }
4187             else
4188             {
4189                 save_128_aligned ((__m128i*)dst, _mm_setzero_si128 ());
4190             }
4191
4192             w -= 4;
4193             dst += 4;
4194             mask += 4;
4195         }
4196
4197         while (w)
4198         {
4199             uint8_t m = *mask++;
4200
4201             if (m)
4202             {
4203                 *dst = pack_1x64_32 (
4204                     pix_multiply_1x64 (
4205                         _mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
4206             }
4207             else
4208             {
4209                 *dst = 0;
4210             }
4211
4212             w--;
4213             dst++;
4214         }
4215     }
4216
4217     _mm_empty ();
4218 }
4219
4220 /*-----------------------------------------------------------------------
4221  * composite_over_n_8_0565
4222  */
4223
4224 static void
4225 sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
4226                               pixman_op_t              op,
4227                               pixman_image_t *         src_image,
4228                               pixman_image_t *         mask_image,
4229                               pixman_image_t *         dst_image,
4230                               int32_t                  src_x,
4231                               int32_t                  src_y,
4232                               int32_t                  mask_x,
4233                               int32_t                  mask_y,
4234                               int32_t                  dest_x,
4235                               int32_t                  dest_y,
4236                               int32_t                  width,
4237                               int32_t                  height)
4238 {
4239     uint32_t src, srca;
4240     uint16_t    *dst_line, *dst, d;
4241     uint8_t     *mask_line, *mask;
4242     int dst_stride, mask_stride;
4243     uint16_t w;
4244     uint32_t m;
4245     __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
4246
4247     __m128i xmm_src, xmm_alpha;
4248     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4249     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
4250
4251     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4252
4253     srca = src >> 24;
4254     if (src == 0)
4255         return;
4256
4257     PIXMAN_IMAGE_GET_LINE (
4258         dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
4259     PIXMAN_IMAGE_GET_LINE (
4260         mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4261
4262     xmm_src = expand_pixel_32_1x128 (src);
4263     xmm_alpha = expand_alpha_1x128 (xmm_src);
4264     mmx_src = _mm_movepi64_pi64 (xmm_src);
4265     mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
4266
4267     while (height--)
4268     {
4269         dst = dst_line;
4270         dst_line += dst_stride;
4271         mask = mask_line;
4272         mask_line += mask_stride;
4273         w = width;
4274
4275         /* call prefetch hint to optimize cache load*/
4276         cache_prefetch ((__m128i*)mask);
4277         cache_prefetch ((__m128i*)dst);
4278
4279         while (w && (unsigned long)dst & 15)
4280         {
4281             m = *mask++;
4282
4283             if (m)
4284             {
4285                 d = *dst;
4286                 mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
4287                 mmx_dest = expand565_16_1x64 (d);
4288
4289                 *dst = pack_565_32_16 (
4290                     pack_1x64_32 (
4291                         in_over_1x64 (
4292                             &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4293             }
4294
4295             w--;
4296             dst++;
4297         }
4298
4299         /* call prefetch hint to optimize cache load*/
4300         cache_prefetch ((__m128i*)mask);
4301         cache_prefetch ((__m128i*)dst);
4302
4303         while (w >= 8)
4304         {
4305             /* fill cache line with next memory */
4306             cache_prefetch_next ((__m128i*)mask);
4307             cache_prefetch_next ((__m128i*)dst);
4308
4309             xmm_dst = load_128_aligned ((__m128i*) dst);
4310             unpack_565_128_4x128 (xmm_dst,
4311                                   &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
4312
4313             m = *((uint32_t*)mask);
4314             mask += 4;
4315
4316             if (m)
4317             {
4318                 xmm_mask = unpack_32_1x128 (m);
4319                 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
4320
4321                 /* Unpacking */
4322                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4323
4324                 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
4325                                         &xmm_mask_lo, &xmm_mask_hi);
4326
4327                 in_over_2x128 (&xmm_src, &xmm_src,
4328                                &xmm_alpha, &xmm_alpha,
4329                                &xmm_mask_lo, &xmm_mask_hi,
4330                                &xmm_dst0, &xmm_dst1);
4331             }
4332
4333             m = *((uint32_t*)mask);
4334             mask += 4;
4335
4336             if (m)
4337             {
4338                 xmm_mask = unpack_32_1x128 (m);
4339                 xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
4340
4341                 /* Unpacking */
4342                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4343
4344                 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
4345                                         &xmm_mask_lo, &xmm_mask_hi);
4346                 in_over_2x128 (&xmm_src, &xmm_src,
4347                                &xmm_alpha, &xmm_alpha,
4348                                &xmm_mask_lo, &xmm_mask_hi,
4349                                &xmm_dst2, &xmm_dst3);
4350             }
4351
4352             save_128_aligned (
4353                 (__m128i*)dst, pack_565_4x128_128 (
4354                     &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
4355
4356             w -= 8;
4357             dst += 8;
4358         }
4359
4360         while (w)
4361         {
4362             m = *mask++;
4363
4364             if (m)
4365             {
4366                 d = *dst;
4367                 mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
4368                 mmx_dest = expand565_16_1x64 (d);
4369
4370                 *dst = pack_565_32_16 (
4371                     pack_1x64_32 (
4372                         in_over_1x64 (
4373                             &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4374             }
4375
4376             w--;
4377             dst++;
4378         }
4379     }
4380
4381     _mm_empty ();
4382 }
4383
4384 /* -----------------------------------------------------------------------
4385  * composite_over_pixbuf_0565
4386  */
4387
4388 static void
4389 sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
4390                                  pixman_op_t              op,
4391                                  pixman_image_t *         src_image,
4392                                  pixman_image_t *         mask_image,
4393                                  pixman_image_t *         dst_image,
4394                                  int32_t                  src_x,
4395                                  int32_t                  src_y,
4396                                  int32_t                  mask_x,
4397                                  int32_t                  mask_y,
4398                                  int32_t                  dest_x,
4399                                  int32_t                  dest_y,
4400                                  int32_t                  width,
4401                                  int32_t                  height)
4402 {
4403     uint16_t    *dst_line, *dst, d;
4404     uint32_t    *src_line, *src, s;
4405     int dst_stride, src_stride;
4406     uint16_t w;
4407     uint32_t opaque, zero;
4408
4409     __m64 ms;
4410     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
4411     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
4412
4413     PIXMAN_IMAGE_GET_LINE (
4414         dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
4415     PIXMAN_IMAGE_GET_LINE (
4416         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
4417
4418 #if 0
4419     /* FIXME
4420      *
4421      * I copy the code from MMX one and keep the fixme.
4422      * If it's a problem there, probably is a problem here.
4423      */
4424     assert (src_image->drawable == mask_image->drawable);
4425 #endif
4426
4427     while (height--)
4428     {
4429         dst = dst_line;
4430         dst_line += dst_stride;
4431         src = src_line;
4432         src_line += src_stride;
4433         w = width;
4434
4435         /* call prefetch hint to optimize cache load*/
4436         cache_prefetch ((__m128i*)src);
4437         cache_prefetch ((__m128i*)dst);
4438
4439         while (w && (unsigned long)dst & 15)
4440         {
4441             s = *src++;
4442             d = *dst;
4443
4444             ms = unpack_32_1x64 (s);
4445
4446             *dst++ = pack_565_32_16 (
4447                 pack_1x64_32 (
4448                     over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
4449             w--;
4450         }
4451
4452         /* call prefetch hint to optimize cache load*/
4453         cache_prefetch ((__m128i*)src);
4454         cache_prefetch ((__m128i*)dst);
4455
4456         while (w >= 8)
4457         {
4458             /* fill cache line with next memory */
4459             cache_prefetch_next ((__m128i*)src);
4460             cache_prefetch_next ((__m128i*)dst);
4461
4462             /* First round */
4463             xmm_src = load_128_unaligned ((__m128i*)src);
4464             xmm_dst = load_128_aligned  ((__m128i*)dst);
4465
4466             opaque = is_opaque (xmm_src);
4467             zero = is_zero (xmm_src);
4468
4469             unpack_565_128_4x128 (xmm_dst,
4470                                   &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
4471             unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
4472
4473             /* preload next round*/
4474             xmm_src = load_128_unaligned ((__m128i*)(src + 4));
4475
4476             if (opaque)
4477             {
4478                 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
4479                                      &xmm_dst0, &xmm_dst1);
4480             }
4481             else if (!zero)
4482             {
4483                 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
4484                                         &xmm_dst0, &xmm_dst1);
4485             }
4486
4487             /* Second round */
4488             opaque = is_opaque (xmm_src);
4489             zero = is_zero (xmm_src);
4490
4491             unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
4492
4493             if (opaque)
4494             {
4495                 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
4496                                      &xmm_dst2, &xmm_dst3);
4497             }
4498             else if (!zero)
4499             {
4500                 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
4501                                         &xmm_dst2, &xmm_dst3);
4502             }
4503
4504             save_128_aligned (
4505                 (__m128i*)dst, pack_565_4x128_128 (
4506                     &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
4507
4508             w -= 8;
4509             src += 8;
4510             dst += 8;
4511         }
4512
4513         while (w)
4514         {
4515             s = *src++;
4516             d = *dst;
4517
4518             ms = unpack_32_1x64 (s);
4519
4520             *dst++ = pack_565_32_16 (
4521                 pack_1x64_32 (
4522                     over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
4523             w--;
4524         }
4525     }
4526
4527     _mm_empty ();
4528 }
4529
4530 /* -------------------------------------------------------------------------
4531  * composite_over_pixbuf_8888
4532  */
4533
4534 static void
4535 sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
4536                                  pixman_op_t              op,
4537                                  pixman_image_t *         src_image,
4538                                  pixman_image_t *         mask_image,
4539                                  pixman_image_t *         dst_image,
4540                                  int32_t                  src_x,
4541                                  int32_t                  src_y,
4542                                  int32_t                  mask_x,
4543                                  int32_t                  mask_y,
4544                                  int32_t                  dest_x,
4545                                  int32_t                  dest_y,
4546                                  int32_t                  width,
4547                                  int32_t                  height)
4548 {
4549     uint32_t    *dst_line, *dst, d;
4550     uint32_t    *src_line, *src, s;
4551     int dst_stride, src_stride;
4552     uint16_t w;
4553     uint32_t opaque, zero;
4554
4555     __m128i xmm_src_lo, xmm_src_hi;
4556     __m128i xmm_dst_lo, xmm_dst_hi;
4557
4558     PIXMAN_IMAGE_GET_LINE (
4559         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
4560     PIXMAN_IMAGE_GET_LINE (
4561         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
4562
4563 #if 0
4564     /* FIXME
4565      *
4566      * I copy the code from MMX one and keep the fixme.
4567      * If it's a problem there, probably is a problem here.
4568      */
4569     assert (src_image->drawable == mask_image->drawable);
4570 #endif
4571
4572     while (height--)
4573     {
4574         dst = dst_line;
4575         dst_line += dst_stride;
4576         src = src_line;
4577         src_line += src_stride;
4578         w = width;
4579
4580         /* call prefetch hint to optimize cache load*/
4581         cache_prefetch ((__m128i*)src);
4582         cache_prefetch ((__m128i*)dst);
4583
4584         while (w && (unsigned long)dst & 15)
4585         {
4586             s = *src++;
4587             d = *dst;
4588
4589             *dst++ = pack_1x64_32 (
4590                 over_rev_non_pre_1x64 (
4591                     unpack_32_1x64 (s), unpack_32_1x64 (d)));
4592
4593             w--;
4594         }
4595
4596         /* call prefetch hint to optimize cache load*/
4597         cache_prefetch ((__m128i*)src);
4598         cache_prefetch ((__m128i*)dst);
4599
4600         while (w >= 4)
4601         {
4602             /* fill cache line with next memory */
4603             cache_prefetch_next ((__m128i*)src);
4604             cache_prefetch_next ((__m128i*)dst);
4605
4606             xmm_src_hi = load_128_unaligned ((__m128i*)src);
4607
4608             opaque = is_opaque (xmm_src_hi);
4609             zero = is_zero (xmm_src_hi);
4610
4611             unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
4612
4613             if (opaque)
4614             {
4615                 invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
4616                                      &xmm_dst_lo, &xmm_dst_hi);
4617
4618                 save_128_aligned (
4619                     (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4620             }
4621             else if (!zero)
4622             {
4623                 xmm_dst_hi = load_128_aligned  ((__m128i*)dst);
4624
4625                 unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
4626
4627                 over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
4628                                         &xmm_dst_lo, &xmm_dst_hi);
4629
4630                 save_128_aligned (
4631                     (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4632             }
4633
4634             w -= 4;
4635             dst += 4;
4636             src += 4;
4637         }
4638
4639         while (w)
4640         {
4641             s = *src++;
4642             d = *dst;
4643
4644             *dst++ = pack_1x64_32 (
4645                 over_rev_non_pre_1x64 (
4646                     unpack_32_1x64 (s), unpack_32_1x64 (d)));
4647
4648             w--;
4649         }
4650     }
4651
4652     _mm_empty ();
4653 }
4654
4655 /* -------------------------------------------------------------------------------------------------
4656  * composite_over_n_8888_0565_ca
4657  */
4658
4659 static void
4660 sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
4661                                     pixman_op_t              op,
4662                                     pixman_image_t *         src_image,
4663                                     pixman_image_t *         mask_image,
4664                                     pixman_image_t *         dst_image,
4665                                     int32_t                  src_x,
4666                                     int32_t                  src_y,
4667                                     int32_t                  mask_x,
4668                                     int32_t                  mask_y,
4669                                     int32_t                  dest_x,
4670                                     int32_t                  dest_y,
4671                                     int32_t                  width,
4672                                     int32_t                  height)
4673 {
4674     uint32_t src;
4675     uint16_t    *dst_line, *dst, d;
4676     uint32_t    *mask_line, *mask, m;
4677     int dst_stride, mask_stride;
4678     int w;
4679     uint32_t pack_cmp;
4680
4681     __m128i xmm_src, xmm_alpha;
4682     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4683     __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
4684
4685     __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
4686
4687     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4688
4689     if (src == 0)
4690         return;
4691
4692     PIXMAN_IMAGE_GET_LINE (
4693         dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
4694     PIXMAN_IMAGE_GET_LINE (
4695         mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
4696
4697     xmm_src = expand_pixel_32_1x128 (src);
4698     xmm_alpha = expand_alpha_1x128 (xmm_src);
4699     mmx_src = _mm_movepi64_pi64 (xmm_src);
4700     mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
4701
4702     while (height--)
4703     {
4704         w = width;
4705         mask = mask_line;
4706         dst = dst_line;
4707         mask_line += mask_stride;
4708         dst_line += dst_stride;
4709
4710         /* call prefetch hint to optimize cache load*/
4711         cache_prefetch ((__m128i*)mask);
4712         cache_prefetch ((__m128i*)dst);
4713
4714         while (w && ((unsigned long)dst & 15))
4715         {
4716             m = *(uint32_t *) mask;
4717
4718             if (m)
4719             {
4720                 d = *dst;
4721                 mmx_mask = unpack_32_1x64 (m);
4722                 mmx_dest = expand565_16_1x64 (d);
4723
4724                 *dst = pack_565_32_16 (
4725                     pack_1x64_32 (
4726                         in_over_1x64 (
4727                             &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4728             }
4729
4730             w--;
4731             dst++;
4732             mask++;
4733         }
4734
4735         /* call prefetch hint to optimize cache load*/
4736         cache_prefetch ((__m128i*)mask);
4737         cache_prefetch ((__m128i*)dst);
4738
4739         while (w >= 8)
4740         {
4741             /* fill cache line with next memory */
4742             cache_prefetch_next ((__m128i*)mask);
4743             cache_prefetch_next ((__m128i*)dst);
4744
4745             /* First round */
4746             xmm_mask = load_128_unaligned ((__m128i*)mask);
4747             xmm_dst = load_128_aligned ((__m128i*)dst);
4748
4749             pack_cmp = _mm_movemask_epi8 (
4750                 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
4751
4752             unpack_565_128_4x128 (xmm_dst,
4753                                   &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
4754             unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4755
4756             /* preload next round */
4757             xmm_mask = load_128_unaligned ((__m128i*)(mask + 4));
4758
4759             /* preload next round */
4760             if (pack_cmp != 0xffff)
4761             {
4762                 in_over_2x128 (&xmm_src, &xmm_src,
4763                                &xmm_alpha, &xmm_alpha,
4764                                &xmm_mask_lo, &xmm_mask_hi,
4765                                &xmm_dst0, &xmm_dst1);
4766             }
4767
4768             /* Second round */
4769             pack_cmp = _mm_movemask_epi8 (
4770                 _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
4771
4772             unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4773
4774             if (pack_cmp != 0xffff)
4775             {
4776                 in_over_2x128 (&xmm_src, &xmm_src,
4777                                &xmm_alpha, &xmm_alpha,
4778                                &xmm_mask_lo, &xmm_mask_hi,
4779                                &xmm_dst2, &xmm_dst3);
4780             }
4781
4782             save_128_aligned (
4783                 (__m128i*)dst, pack_565_4x128_128 (
4784                     &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
4785
4786             w -= 8;
4787             dst += 8;
4788             mask += 8;
4789         }
4790
4791         while (w)
4792         {
4793             m = *(uint32_t *) mask;
4794
4795             if (m)
4796             {
4797                 d = *dst;
4798                 mmx_mask = unpack_32_1x64 (m);
4799                 mmx_dest = expand565_16_1x64 (d);
4800
4801                 *dst = pack_565_32_16 (
4802                     pack_1x64_32 (
4803                         in_over_1x64 (
4804                             &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
4805             }
4806
4807             w--;
4808             dst++;
4809             mask++;
4810         }
4811     }
4812
4813     _mm_empty ();
4814 }
4815
4816 /* -----------------------------------------------------------------------
4817  * composite_in_n_8_8
4818  */
4819
4820 static void
4821 sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
4822                          pixman_op_t              op,
4823                          pixman_image_t *         src_image,
4824                          pixman_image_t *         mask_image,
4825                          pixman_image_t *         dst_image,
4826                          int32_t                  src_x,
4827                          int32_t                  src_y,
4828                          int32_t                  mask_x,
4829                          int32_t                  mask_y,
4830                          int32_t                  dest_x,
4831                          int32_t                  dest_y,
4832                          int32_t                  width,
4833                          int32_t                  height)
4834 {
4835     uint8_t     *dst_line, *dst;
4836     uint8_t     *mask_line, *mask;
4837     int dst_stride, mask_stride;
4838     uint16_t w, d, m;
4839     uint32_t src;
4840     uint8_t sa;
4841
4842     __m128i xmm_alpha;
4843     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
4844     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4845
4846     PIXMAN_IMAGE_GET_LINE (
4847         dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4848     PIXMAN_IMAGE_GET_LINE (
4849         mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
4850
4851     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
4852
4853     sa = src >> 24;
4854
4855     xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
4856
4857     while (height--)
4858     {
4859         dst = dst_line;
4860         dst_line += dst_stride;
4861         mask = mask_line;
4862         mask_line += mask_stride;
4863         w = width;
4864
4865         /* call prefetch hint to optimize cache load*/
4866         cache_prefetch ((__m128i*)mask);
4867         cache_prefetch ((__m128i*)dst);
4868
4869         while (w && ((unsigned long)dst & 15))
4870         {
4871             m = (uint32_t) *mask++;
4872             d = (uint32_t) *dst;
4873
4874             *dst++ = (uint8_t) pack_1x64_32 (
4875                 pix_multiply_1x64 (
4876                     pix_multiply_1x64 (_mm_movepi64_pi64 (xmm_alpha),
4877                                        unpack_32_1x64 (m)),
4878                     unpack_32_1x64 (d)));
4879             w--;
4880         }
4881
4882         /* call prefetch hint to optimize cache load*/
4883         cache_prefetch ((__m128i*)mask);
4884         cache_prefetch ((__m128i*)dst);
4885
4886         while (w >= 16)
4887         {
4888             /* fill cache line with next memory */
4889             cache_prefetch_next ((__m128i*)mask);
4890             cache_prefetch_next ((__m128i*)dst);
4891
4892             xmm_mask = load_128_unaligned ((__m128i*)mask);
4893             xmm_dst = load_128_aligned ((__m128i*)dst);
4894
4895             unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
4896             unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
4897
4898             pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
4899                                 &xmm_mask_lo, &xmm_mask_hi,
4900                                 &xmm_mask_lo, &xmm_mask_hi);
4901
4902             pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
4903                                 &xmm_dst_lo, &xmm_dst_hi,
4904                                 &xmm_dst_lo, &xmm_dst_hi);
4905
4906             save_128_aligned (
4907                 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
4908
4909             mask += 16;
4910             dst += 16;
4911             w -= 16;
4912         }
4913
4914         while (w)
4915         {
4916             m = (uint32_t) *mask++;
4917             d = (uint32_t) *dst;
4918
4919             *dst++ = (uint8_t) pack_1x64_32 (
4920                 pix_multiply_1x64 (
4921                     pix_multiply_1x64 (
4922                         _mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
4923                     unpack_32_1x64 (d)));
4924             w--;
4925         }
4926     }
4927
4928     _mm_empty ();
4929 }
4930
4931 /* ---------------------------------------------------------------------------
4932  * composite_in_8_8
4933  */
4934
4935 static void
4936 sse2_composite_in_8_8 (pixman_implementation_t *imp,
4937                        pixman_op_t              op,
4938                        pixman_image_t *         src_image,
4939                        pixman_image_t *         mask_image,
4940                        pixman_image_t *         dst_image,
4941                        int32_t                  src_x,
4942                        int32_t                  src_y,
4943                        int32_t                  mask_x,
4944                        int32_t                  mask_y,
4945                        int32_t                  dest_x,
4946                        int32_t                  dest_y,
4947                        int32_t                  width,
4948                        int32_t                  height)
4949 {
4950     uint8_t     *dst_line, *dst;
4951     uint8_t     *src_line, *src;
4952     int src_stride, dst_stride;
4953     uint16_t w;
4954     uint32_t s, d;
4955
4956     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
4957     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
4958
4959     PIXMAN_IMAGE_GET_LINE (
4960         dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
4961     PIXMAN_IMAGE_GET_LINE (
4962         src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
4963
4964     while (height--)
4965     {
4966         dst = dst_line;
4967         dst_line += dst_stride;
4968         src = src_line;
4969         src_line += src_stride;
4970         w = width;
4971
4972         /* call prefetch hint to optimize cache load*/
4973         cache_prefetch ((__m128i*)src);
4974         cache_prefetch ((__m128i*)dst);
4975
4976         while (w && ((unsigned long)dst & 15))
4977         {
4978             s = (uint32_t) *src++;
4979             d = (uint32_t) *dst;
4980
4981             *dst++ = (uint8_t) pack_1x64_32 (
4982                 pix_multiply_1x64 (
4983                     unpack_32_1x64 (s), unpack_32_1x64 (d)));
4984             w--;
4985         }
4986
4987         /* call prefetch hint to optimize cache load*/
4988         cache_prefetch ((__m128i*)src);
4989         cache_prefetch ((__m128i*)dst);
4990
4991         while (w >= 16)
4992         {
4993             /* fill cache line with next memory */
4994             cache_prefetch_next ((__m128i*)src);
4995             cache_prefetch_next ((__m128i*)dst);
4996
4997             xmm_src = load_128_unaligned ((__m128i*)src);
4998             xmm_dst = load_128_aligned ((__m128i*)dst);
4999
5000             unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
5001             unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5002
5003             pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
5004                                 &xmm_dst_lo, &xmm_dst_hi,
5005                                 &xmm_dst_lo, &xmm_dst_hi);
5006
5007             save_128_aligned (
5008                 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5009
5010             src += 16;
5011             dst += 16;
5012             w -= 16;
5013         }
5014
5015         while (w)
5016         {
5017             s = (uint32_t) *src++;
5018             d = (uint32_t) *dst;
5019
5020             *dst++ = (uint8_t) pack_1x64_32 (
5021                 pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (d)));
5022             w--;
5023         }
5024     }
5025
5026     _mm_empty ();
5027 }
5028
5029 /* -------------------------------------------------------------------------
5030  * composite_add_8888_8_8
5031  */
5032
5033 static void
5034 sse2_composite_add_8888_8_8 (pixman_implementation_t *imp,
5035                              pixman_op_t              op,
5036                              pixman_image_t *         src_image,
5037                              pixman_image_t *         mask_image,
5038                              pixman_image_t *         dst_image,
5039                              int32_t                  src_x,
5040                              int32_t                  src_y,
5041                              int32_t                  mask_x,
5042                              int32_t                  mask_y,
5043                              int32_t                  dest_x,
5044                              int32_t                  dest_y,
5045                              int32_t                  width,
5046                              int32_t                  height)
5047 {
5048     uint8_t     *dst_line, *dst;
5049     uint8_t     *mask_line, *mask;
5050     int dst_stride, mask_stride;
5051     uint16_t w;
5052     uint32_t src;
5053     uint8_t sa;
5054     uint32_t m, d;
5055
5056     __m128i xmm_alpha;
5057     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
5058     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5059
5060     PIXMAN_IMAGE_GET_LINE (
5061         dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
5062     PIXMAN_IMAGE_GET_LINE (
5063         mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
5064
5065     src = _pixman_image_get_solid (src_image, dst_image->bits.format);
5066
5067     sa = src >> 24;
5068
5069     xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
5070
5071     while (height--)
5072     {
5073         dst = dst_line;
5074         dst_line += dst_stride;
5075         mask = mask_line;
5076         mask_line += mask_stride;
5077         w = width;
5078
5079         /* call prefetch hint to optimize cache load*/
5080         cache_prefetch ((__m128i*)mask);
5081         cache_prefetch ((__m128i*)dst);
5082
5083         while (w && ((unsigned long)dst & 15))
5084         {
5085             m = (uint32_t) *mask++;
5086             d = (uint32_t) *dst;
5087
5088             *dst++ = (uint8_t) pack_1x64_32 (
5089                 _mm_adds_pu16 (
5090                     pix_multiply_1x64 (
5091                         _mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
5092                     unpack_32_1x64 (d)));
5093             w--;
5094         }
5095
5096         /* call prefetch hint to optimize cache load*/
5097         cache_prefetch ((__m128i*)mask);
5098         cache_prefetch ((__m128i*)dst);
5099
5100         while (w >= 16)
5101         {
5102             /* fill cache line with next memory */
5103             cache_prefetch_next ((__m128i*)mask);
5104             cache_prefetch_next ((__m128i*)dst);
5105
5106             xmm_mask = load_128_unaligned ((__m128i*)mask);
5107             xmm_dst = load_128_aligned ((__m128i*)dst);
5108
5109             unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
5110             unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5111
5112             pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
5113                                 &xmm_mask_lo, &xmm_mask_hi,
5114                                 &xmm_mask_lo, &xmm_mask_hi);
5115
5116             xmm_dst_lo = _mm_adds_epu16 (xmm_mask_lo, xmm_dst_lo);
5117             xmm_dst_hi = _mm_adds_epu16 (xmm_mask_hi, xmm_dst_hi);
5118
5119             save_128_aligned (
5120                 (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5121
5122             mask += 16;
5123             dst += 16;
5124             w -= 16;
5125         }
5126
5127         while (w)
5128         {
5129             m = (uint32_t) *mask++;
5130             d = (uint32_t) *dst;
5131
5132             *dst++ = (uint8_t) pack_1x64_32 (
5133                 _mm_adds_pu16 (
5134                     pix_multiply_1x64 (
5135                         _mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
5136                     unpack_32_1x64 (d)));
5137
5138             w--;
5139         }
5140     }
5141
5142     _mm_empty ();
5143 }
5144
5145 /* ----------------------------------------------------------------------
5146  * composite_add_8000_8000
5147  */
5148
5149 static void
5150 sse2_composite_add_8000_8000 (pixman_implementation_t *imp,
5151                               pixman_op_t              op,
5152                               pixman_image_t *         src_image,
5153                               pixman_image_t *         mask_image,
5154                               pixman_image_t *         dst_image,
5155                               int32_t                  src_x,
5156                               int32_t                  src_y,
5157                               int32_t                  mask_x,
5158                               int32_t                  mask_y,
5159                               int32_t                  dest_x,
5160                               int32_t                  dest_y,
5161                               int32_t                  width,
5162                               int32_t                  height)
5163 {
5164     uint8_t     *dst_line, *dst;
5165     uint8_t     *src_line, *src;
5166     int dst_stride, src_stride;
5167     uint16_t w;
5168     uint16_t t;
5169
5170     PIXMAN_IMAGE_GET_LINE (
5171         src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
5172     PIXMAN_IMAGE_GET_LINE (
5173         dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
5174
5175     while (height--)
5176     {
5177         dst = dst_line;
5178         src = src_line;
5179
5180         /* call prefetch hint to optimize cache load*/
5181         cache_prefetch ((__m128i*)src);
5182         cache_prefetch ((__m128i*)dst);
5183
5184         dst_line += dst_stride;
5185         src_line += src_stride;
5186         w = width;
5187
5188         /* Small head */
5189         while (w && (unsigned long)dst & 3)
5190         {
5191             t = (*dst) + (*src++);
5192             *dst++ = t | (0 - (t >> 8));
5193             w--;
5194         }
5195
5196         core_combine_add_u_sse2 ((uint32_t*)dst, (uint32_t*)src, NULL, w >> 2);
5197
5198         /* Small tail */
5199         dst += w & 0xfffc;
5200         src += w & 0xfffc;
5201
5202         w &= 3;
5203
5204         while (w)
5205         {
5206             t = (*dst) + (*src++);
5207             *dst++ = t | (0 - (t >> 8));
5208             w--;
5209         }
5210     }
5211
5212     _mm_empty ();
5213 }
5214
5215 /* ---------------------------------------------------------------------
5216  * composite_add_8888_8888
5217  */
5218 static void
5219 sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
5220                               pixman_op_t              op,
5221                               pixman_image_t *         src_image,
5222                               pixman_image_t *         mask_image,
5223                               pixman_image_t *         dst_image,
5224                               int32_t                  src_x,
5225                               int32_t                  src_y,
5226                               int32_t                  mask_x,
5227                               int32_t                  mask_y,
5228                               int32_t                  dest_x,
5229                               int32_t                  dest_y,
5230                               int32_t                  width,
5231                               int32_t                  height)
5232 {
5233     uint32_t    *dst_line, *dst;
5234     uint32_t    *src_line, *src;
5235     int dst_stride, src_stride;
5236
5237     PIXMAN_IMAGE_GET_LINE (
5238         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
5239     PIXMAN_IMAGE_GET_LINE (
5240         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
5241
5242     while (height--)
5243     {
5244         dst = dst_line;
5245         dst_line += dst_stride;
5246         src = src_line;
5247         src_line += src_stride;
5248
5249         core_combine_add_u_sse2 (dst, src, NULL, width);
5250     }
5251
5252     _mm_empty ();
5253 }
5254
5255 /* -------------------------------------------------------------------------------------------------
5256  * sse2_composite_copy_area
5257  */
5258
5259 static pixman_bool_t
5260 pixman_blt_sse2 (uint32_t *src_bits,
5261                  uint32_t *dst_bits,
5262                  int       src_stride,
5263                  int       dst_stride,
5264                  int       src_bpp,
5265                  int       dst_bpp,
5266                  int       src_x,
5267                  int       src_y,
5268                  int       dst_x,
5269                  int       dst_y,
5270                  int       width,
5271                  int       height)
5272 {
5273     uint8_t *   src_bytes;
5274     uint8_t *   dst_bytes;
5275     int byte_width;
5276
5277     if (src_bpp != dst_bpp)
5278         return FALSE;
5279
5280     if (src_bpp == 16)
5281     {
5282         src_stride = src_stride * (int) sizeof (uint32_t) / 2;
5283         dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
5284         src_bytes =(uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
5285         dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
5286         byte_width = 2 * width;
5287         src_stride *= 2;
5288         dst_stride *= 2;
5289     }
5290     else if (src_bpp == 32)
5291     {
5292         src_stride = src_stride * (int) sizeof (uint32_t) / 4;
5293         dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
5294         src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
5295         dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
5296         byte_width = 4 * width;
5297         src_stride *= 4;
5298         dst_stride *= 4;
5299     }
5300     else
5301     {
5302         return FALSE;
5303     }
5304
5305     cache_prefetch ((__m128i*)src_bytes);
5306     cache_prefetch ((__m128i*)dst_bytes);
5307
5308     while (height--)
5309     {
5310         int w;
5311         uint8_t *s = src_bytes;
5312         uint8_t *d = dst_bytes;
5313         src_bytes += src_stride;
5314         dst_bytes += dst_stride;
5315         w = byte_width;
5316
5317         cache_prefetch_next ((__m128i*)s);
5318         cache_prefetch_next ((__m128i*)d);
5319
5320         while (w >= 2 && ((unsigned long)d & 3))
5321         {
5322             *(uint16_t *)d = *(uint16_t *)s;
5323             w -= 2;
5324             s += 2;
5325             d += 2;
5326         }
5327
5328         while (w >= 4 && ((unsigned long)d & 15))
5329         {
5330             *(uint32_t *)d = *(uint32_t *)s;
5331
5332             w -= 4;
5333             s += 4;
5334             d += 4;
5335         }
5336
5337         cache_prefetch_next ((__m128i*)s);
5338         cache_prefetch_next ((__m128i*)d);
5339
5340         while (w >= 64)
5341         {
5342             __m128i xmm0, xmm1, xmm2, xmm3;
5343
5344             /* 128 bytes ahead */
5345             cache_prefetch (((__m128i*)s) + 8);
5346             cache_prefetch (((__m128i*)d) + 8);
5347
5348             xmm0 = load_128_unaligned ((__m128i*)(s));
5349             xmm1 = load_128_unaligned ((__m128i*)(s + 16));
5350             xmm2 = load_128_unaligned ((__m128i*)(s + 32));
5351             xmm3 = load_128_unaligned ((__m128i*)(s + 48));
5352
5353             save_128_aligned ((__m128i*)(d),    xmm0);
5354             save_128_aligned ((__m128i*)(d + 16), xmm1);
5355             save_128_aligned ((__m128i*)(d + 32), xmm2);
5356             save_128_aligned ((__m128i*)(d + 48), xmm3);
5357
5358             s += 64;
5359             d += 64;
5360             w -= 64;
5361         }
5362
5363         cache_prefetch_next ((__m128i*)s);
5364         cache_prefetch_next ((__m128i*)d);
5365
5366         while (w >= 16)
5367         {
5368             save_128_aligned ((__m128i*)d, load_128_unaligned ((__m128i*)s) );
5369
5370             w -= 16;
5371             d += 16;
5372             s += 16;
5373         }
5374
5375         cache_prefetch_next ((__m128i*)s);
5376         cache_prefetch_next ((__m128i*)d);
5377
5378         while (w >= 4)
5379         {
5380             *(uint32_t *)d = *(uint32_t *)s;
5381
5382             w -= 4;
5383             s += 4;
5384             d += 4;
5385         }
5386
5387         if (w >= 2)
5388         {
5389             *(uint16_t *)d = *(uint16_t *)s;
5390             w -= 2;
5391             s += 2;
5392             d += 2;
5393         }
5394     }
5395
5396     _mm_empty ();
5397
5398     return TRUE;
5399 }
5400
5401 static void
5402 sse2_composite_copy_area (pixman_implementation_t *imp,
5403                           pixman_op_t              op,
5404                           pixman_image_t *         src_image,
5405                           pixman_image_t *         mask_image,
5406                           pixman_image_t *         dst_image,
5407                           int32_t                  src_x,
5408                           int32_t                  src_y,
5409                           int32_t                  mask_x,
5410                           int32_t                  mask_y,
5411                           int32_t                  dest_x,
5412                           int32_t                  dest_y,
5413                           int32_t                  width,
5414                           int32_t                  height)
5415 {
5416     pixman_blt_sse2 (src_image->bits.bits,
5417                      dst_image->bits.bits,
5418                      src_image->bits.rowstride,
5419                      dst_image->bits.rowstride,
5420                      PIXMAN_FORMAT_BPP (src_image->bits.format),
5421                      PIXMAN_FORMAT_BPP (dst_image->bits.format),
5422                      src_x, src_y, dest_x, dest_y, width, height);
5423 }
5424
5425 #if 0
5426 /* This code are buggy in MMX version, now the bug was translated to SSE2 version */
5427 void
5428 sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
5429                                  pixman_op_t              op,
5430                                  pixman_image_t *         src_image,
5431                                  pixman_image_t *         mask_image,
5432                                  pixman_image_t *         dst_image,
5433                                  int32_t                  src_x,
5434                                  int32_t                  src_y,
5435                                  int32_t                  mask_x,
5436                                  int32_t                  mask_y,
5437                                  int32_t                  dest_x,
5438                                  int32_t                  dest_y,
5439                                  int32_t                  width,
5440                                  int32_t                  height)
5441 {
5442     uint32_t    *src, *src_line, s;
5443     uint32_t    *dst, *dst_line, d;
5444     uint8_t         *mask, *mask_line;
5445     uint32_t m;
5446     int src_stride, mask_stride, dst_stride;
5447     uint16_t w;
5448
5449     __m128i xmm_src, xmm_src_lo, xmm_src_hi;
5450     __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
5451     __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
5452
5453     PIXMAN_IMAGE_GET_LINE (
5454         dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
5455     PIXMAN_IMAGE_GET_LINE (
5456         mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
5457     PIXMAN_IMAGE_GET_LINE (
5458         src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
5459
5460     while (height--)
5461     {
5462         src = src_line;
5463         src_line += src_stride;
5464         dst = dst_line;
5465         dst_line += dst_stride;
5466         mask = mask_line;
5467         mask_line += mask_stride;
5468
5469         w = width;
5470
5471         /* call prefetch hint to optimize cache load*/
5472         cache_prefetch ((__m128i*)src);
5473         cache_prefetch ((__m128i*)dst);
5474         cache_prefetch ((__m128i*)mask);
5475
5476         while (w && (unsigned long)dst & 15)
5477         {
5478             s = 0xff000000 | *src++;
5479             m = (uint32_t) *mask++;
5480             d = *dst;
5481
5482             __m64 ms = unpack_32_1x64 (s);
5483
5484             if (m != 0xff)
5485             {
5486                 ms = in_over_1x64 (ms,
5487                                    mask_x00ff,
5488                                    expand_alpha_rev_1x64 (unpack_32_1x64 (m)),
5489                                    unpack_32_1x64 (d));
5490             }
5491
5492             *dst++ = pack_1x64_32 (ms);
5493             w--;
5494         }
5495
5496         /* call prefetch hint to optimize cache load*/
5497         cache_prefetch ((__m128i*)src);
5498         cache_prefetch ((__m128i*)dst);
5499         cache_prefetch ((__m128i*)mask);
5500
5501         while (w >= 4)
5502         {
5503             /* fill cache line with next memory */
5504             cache_prefetch_next ((__m128i*)src);
5505             cache_prefetch_next ((__m128i*)dst);
5506             cache_prefetch_next ((__m128i*)mask);
5507
5508             m = *(uint32_t*) mask;
5509             xmm_src = _mm_or_si128 (load_128_unaligned ((__m128i*)src), mask_ff000000);
5510
5511             if (m == 0xffffffff)
5512             {
5513                 save_128_aligned ((__m128i*)dst, xmm_src);
5514             }
5515             else
5516             {
5517                 xmm_dst = load_128_aligned ((__m128i*)dst);
5518
5519                 xmm_mask = _mm_unpacklo_epi16 (
5520                     unpack_32_1x128 (m), _mm_setzero_si128 ());
5521
5522                 unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
5523                 unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
5524                 unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
5525
5526                 expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
5527                                         &xmm_mask_lo, &xmm_mask_hi);
5528
5529                 in_over_2x128 (xmm_src_lo, xmm_src_hi,
5530                                mask_00ff, mask_00ff,
5531                                xmm_mask_lo, xmm_mask_hi,
5532                                &xmm_dst_lo, &xmm_dst_hi);
5533
5534                 save_128_aligned (
5535                     (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
5536             }
5537
5538             src += 4;
5539             dst += 4;
5540             mask += 4;
5541             w -= 4;
5542         }
5543
5544         while (w)
5545         {
5546             m = (uint32_t) *mask++;
5547
5548             if (m)
5549             {
5550                 s = 0xff000000 | *src;
5551
5552                 if (m == 0xff)
5553                 {
5554                     *dst = s;
5555                 }
5556                 else
5557                 {
5558                     d = *dst;
5559
5560                     *dst = pack_1x64_32 (
5561                         in_over_1x64 (
5562                             unpack_32_1x64 (s),
5563                             mask_x00ff,
5564                             expand_alpha_rev_1x64 (unpack_32_1x64 (m)),
5565                             unpack_32_1x64 (d)));
5566                 }
5567
5568             }
5569
5570             src++;
5571             dst++;
5572             w--;
5573         }
5574     }
5575
5576     _mm_empty ();
5577 }
5578
5579 #endif
5580
5581 static const pixman_fast_path_t sse2_fast_paths[] =
5582 {
5583     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_r5g6b5,   sse2_composite_over_n_8_0565,       0 },
5584     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_b5g6r5,   sse2_composite_over_n_8_0565,       0 },
5585     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_null,     PIXMAN_a8r8g8b8, sse2_composite_over_n_8888,         0 },
5586     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_null,     PIXMAN_x8r8g8b8, sse2_composite_over_n_8888,         0 },
5587     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_null,     PIXMAN_r5g6b5,   sse2_composite_over_n_0565,         0 },
5588     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_a8r8g8b8, sse2_composite_over_8888_8888,      0 },
5589     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_x8r8g8b8, sse2_composite_over_8888_8888,      0 },
5590     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_a8b8g8r8, sse2_composite_over_8888_8888,      0 },
5591     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_x8b8g8r8, sse2_composite_over_8888_8888,      0 },
5592     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_r5g6b5,   sse2_composite_over_8888_0565,      0 },
5593     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_b5g6r5,   sse2_composite_over_8888_0565,      0 },
5594     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_a8r8g8b8, sse2_composite_over_n_8_8888,       0 },
5595     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_x8r8g8b8, sse2_composite_over_n_8_8888,       0 },
5596     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_a8b8g8r8, sse2_composite_over_n_8_8888,       0 },
5597     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_x8b8g8r8, sse2_composite_over_n_8_8888,       0 },
5598 #if 0
5599     /* FIXME: This code are buggy in MMX version, now the bug was translated to SSE2 version */
5600     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8,       PIXMAN_x8r8g8b8, sse2_composite_over_x888_8_8888,    0 },
5601     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8,       PIXMAN_a8r8g8b8, sse2_composite_over_x888_8_8888,    0 },
5602     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8,       PIXMAN_x8b8g8r8, sse2_composite_over_x888_8_8888,    0 },
5603     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8,       PIXMAN_a8r8g8b8, sse2_composite_over_x888_8_8888,    0 },
5604 #endif
5605     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8,       PIXMAN_a8r8g8b8, sse2_composite_over_x888_n_8888,    NEED_SOLID_MASK },
5606     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8,       PIXMAN_x8r8g8b8, sse2_composite_over_x888_n_8888,    NEED_SOLID_MASK },
5607     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8,       PIXMAN_a8b8g8r8, sse2_composite_over_x888_n_8888,    NEED_SOLID_MASK },
5608     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8,       PIXMAN_x8b8g8r8, sse2_composite_over_x888_n_8888,    NEED_SOLID_MASK },
5609     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_a8,       PIXMAN_a8r8g8b8, sse2_composite_over_8888_n_8888,    NEED_SOLID_MASK },
5610     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_a8,       PIXMAN_x8r8g8b8, sse2_composite_over_8888_n_8888,    NEED_SOLID_MASK },
5611     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_a8,       PIXMAN_a8b8g8r8, sse2_composite_over_8888_n_8888,    NEED_SOLID_MASK },
5612     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_a8,       PIXMAN_x8b8g8r8, sse2_composite_over_8888_n_8888,    NEED_SOLID_MASK },
5613     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8r8g8b8, PIXMAN_a8r8g8b8, sse2_composite_over_n_8888_8888_ca, NEED_COMPONENT_ALPHA },
5614     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8r8g8b8, PIXMAN_x8r8g8b8, sse2_composite_over_n_8888_8888_ca, NEED_COMPONENT_ALPHA },
5615     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8b8g8r8, PIXMAN_a8b8g8r8, sse2_composite_over_n_8888_8888_ca, NEED_COMPONENT_ALPHA },
5616     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8b8g8r8, PIXMAN_x8b8g8r8, sse2_composite_over_n_8888_8888_ca, NEED_COMPONENT_ALPHA },
5617     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8r8g8b8, PIXMAN_r5g6b5,   sse2_composite_over_n_8888_0565_ca, NEED_COMPONENT_ALPHA },
5618     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8b8g8r8, PIXMAN_b5g6r5,   sse2_composite_over_n_8888_0565_ca, NEED_COMPONENT_ALPHA },
5619     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8r8g8b8, PIXMAN_a8r8g8b8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5620     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8b8g8r8, PIXMAN_a8r8g8b8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5621     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8r8g8b8, PIXMAN_x8r8g8b8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5622     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8b8g8r8, PIXMAN_x8r8g8b8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5623     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8r8g8b8, PIXMAN_a8b8g8r8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5624     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8b8g8r8, PIXMAN_a8b8g8r8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5625     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8r8g8b8, PIXMAN_x8b8g8r8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5626     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8b8g8r8, PIXMAN_x8b8g8r8, sse2_composite_over_pixbuf_8888,    NEED_PIXBUF },
5627     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8r8g8b8, PIXMAN_r5g6b5,   sse2_composite_over_pixbuf_0565,    NEED_PIXBUF },
5628     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_a8b8g8r8, PIXMAN_r5g6b5,   sse2_composite_over_pixbuf_0565,    NEED_PIXBUF },
5629     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8r8g8b8, PIXMAN_b5g6r5,   sse2_composite_over_pixbuf_0565,    NEED_PIXBUF },
5630     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_a8b8g8r8, PIXMAN_b5g6r5,   sse2_composite_over_pixbuf_0565,    NEED_PIXBUF },
5631     { PIXMAN_OP_OVER, PIXMAN_x8r8g8b8, PIXMAN_null,     PIXMAN_x8r8g8b8, sse2_composite_copy_area,           0 },
5632     { PIXMAN_OP_OVER, PIXMAN_x8b8g8r8, PIXMAN_null,     PIXMAN_x8b8g8r8, sse2_composite_copy_area,           0 },
5633
5634     { PIXMAN_OP_ADD,  PIXMAN_solid,    PIXMAN_a8r8g8b8, PIXMAN_a8r8g8b8, sse2_composite_add_n_8888_8888_ca,  NEED_COMPONENT_ALPHA },
5635     { PIXMAN_OP_ADD,  PIXMAN_a8,       PIXMAN_null,     PIXMAN_a8,       sse2_composite_add_8000_8000,       0 },
5636     { PIXMAN_OP_ADD,  PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_a8r8g8b8, sse2_composite_add_8888_8888,       0 },
5637     { PIXMAN_OP_ADD,  PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_a8b8g8r8, sse2_composite_add_8888_8888,       0 },
5638     { PIXMAN_OP_ADD,  PIXMAN_solid,    PIXMAN_a8,       PIXMAN_a8,       sse2_composite_add_8888_8_8,        0 },
5639
5640     { PIXMAN_OP_SRC, PIXMAN_solid,     PIXMAN_a8,       PIXMAN_a8r8g8b8, sse2_composite_src_n_8_8888,        0 },
5641     { PIXMAN_OP_SRC, PIXMAN_solid,     PIXMAN_a8,       PIXMAN_x8r8g8b8, sse2_composite_src_n_8_8888,        0 },
5642     { PIXMAN_OP_SRC, PIXMAN_solid,     PIXMAN_a8,       PIXMAN_a8b8g8r8, sse2_composite_src_n_8_8888,        0 },
5643     { PIXMAN_OP_SRC, PIXMAN_solid,     PIXMAN_a8,       PIXMAN_x8b8g8r8, sse2_composite_src_n_8_8888,        0 },
5644     { PIXMAN_OP_SRC, PIXMAN_a8r8g8b8,  PIXMAN_null,     PIXMAN_a8r8g8b8, sse2_composite_copy_area,           0 },
5645     { PIXMAN_OP_SRC, PIXMAN_a8b8g8r8,  PIXMAN_null,     PIXMAN_a8b8g8r8, sse2_composite_copy_area,           0 },
5646     { PIXMAN_OP_SRC, PIXMAN_a8r8g8b8,  PIXMAN_null,     PIXMAN_x8r8g8b8, sse2_composite_copy_area,           0 },
5647     { PIXMAN_OP_SRC, PIXMAN_a8b8g8r8,  PIXMAN_null,     PIXMAN_x8b8g8r8, sse2_composite_copy_area,           0 },
5648     { PIXMAN_OP_SRC, PIXMAN_x8r8g8b8,  PIXMAN_null,     PIXMAN_x8r8g8b8, sse2_composite_copy_area,           0 },
5649     { PIXMAN_OP_SRC, PIXMAN_x8b8g8r8,  PIXMAN_null,     PIXMAN_x8b8g8r8, sse2_composite_copy_area,           0 },
5650     { PIXMAN_OP_SRC, PIXMAN_r5g6b5,    PIXMAN_null,     PIXMAN_r5g6b5,   sse2_composite_copy_area,           0 },
5651     { PIXMAN_OP_SRC, PIXMAN_b5g6r5,    PIXMAN_null,     PIXMAN_b5g6r5,   sse2_composite_copy_area,           0 },
5652
5653     { PIXMAN_OP_IN,  PIXMAN_a8,        PIXMAN_null,     PIXMAN_a8,       sse2_composite_in_8_8,              0 },
5654     { PIXMAN_OP_IN,  PIXMAN_solid,     PIXMAN_a8,       PIXMAN_a8,       sse2_composite_in_n_8_8,            0 },
5655
5656     { PIXMAN_OP_NONE },
5657 };
5658
5659 /*
5660  * Work around GCC bug causing crashes in Mozilla with SSE2
5661  *
5662  * When using -msse, gcc generates movdqa instructions assuming that
5663  * the stack is 16 byte aligned. Unfortunately some applications, such
5664  * as Mozilla and Mono, end up aligning the stack to 4 bytes, which
5665  * causes the movdqa instructions to fail.
5666  *
5667  * The __force_align_arg_pointer__ makes gcc generate a prologue that
5668  * realigns the stack pointer to 16 bytes.
5669  *
5670  * On x86-64 this is not necessary because the standard ABI already
5671  * calls for a 16 byte aligned stack.
5672  *
5673  * See https://bugs.freedesktop.org/show_bug.cgi?id=15693
5674  */
5675 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
5676 __attribute__((__force_align_arg_pointer__))
5677 #endif
5678 static void
5679 sse2_composite (pixman_implementation_t *imp,
5680                 pixman_op_t              op,
5681                 pixman_image_t *         src,
5682                 pixman_image_t *         mask,
5683                 pixman_image_t *         dest,
5684                 int32_t                  src_x,
5685                 int32_t                  src_y,
5686                 int32_t                  mask_x,
5687                 int32_t                  mask_y,
5688                 int32_t                  dest_x,
5689                 int32_t                  dest_y,
5690                 int32_t                  width,
5691                 int32_t                  height)
5692 {
5693     if (_pixman_run_fast_path (sse2_fast_paths, imp,
5694                                op, src, mask, dest,
5695                                src_x, src_y,
5696                                mask_x, mask_y,
5697                                dest_x, dest_y,
5698                                width, height))
5699     {
5700         return;
5701     }
5702
5703     _pixman_implementation_composite (imp->delegate, op,
5704                                       src, mask, dest,
5705                                       src_x, src_y,
5706                                       mask_x, mask_y,
5707                                       dest_x, dest_y,
5708                                       width, height);
5709 }
5710
5711 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
5712 __attribute__((__force_align_arg_pointer__))
5713 #endif
5714 static pixman_bool_t
5715 sse2_blt (pixman_implementation_t *imp,
5716           uint32_t *               src_bits,
5717           uint32_t *               dst_bits,
5718           int                      src_stride,
5719           int                      dst_stride,
5720           int                      src_bpp,
5721           int                      dst_bpp,
5722           int                      src_x,
5723           int                      src_y,
5724           int                      dst_x,
5725           int                      dst_y,
5726           int                      width,
5727           int                      height)
5728 {
5729     if (!pixman_blt_sse2 (
5730             src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
5731             src_x, src_y, dst_x, dst_y, width, height))
5732
5733     {
5734         return _pixman_implementation_blt (
5735             imp->delegate,
5736             src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
5737             src_x, src_y, dst_x, dst_y, width, height);
5738     }
5739
5740     return TRUE;
5741 }
5742
5743 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
5744 __attribute__((__force_align_arg_pointer__))
5745 #endif
5746 static pixman_bool_t
5747 sse2_fill (pixman_implementation_t *imp,
5748            uint32_t *               bits,
5749            int                      stride,
5750            int                      bpp,
5751            int                      x,
5752            int                      y,
5753            int                      width,
5754            int                      height,
5755            uint32_t xor)
5756 {
5757     if (!pixman_fill_sse2 (bits, stride, bpp, x, y, width, height, xor))
5758     {
5759         return _pixman_implementation_fill (
5760             imp->delegate, bits, stride, bpp, x, y, width, height, xor);
5761     }
5762
5763     return TRUE;
5764 }
5765
5766 #if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
5767 __attribute__((__force_align_arg_pointer__))
5768 #endif
5769 pixman_implementation_t *
5770 _pixman_implementation_create_sse2 (void)
5771 {
5772     pixman_implementation_t *mmx = _pixman_implementation_create_mmx ();
5773     pixman_implementation_t *imp = _pixman_implementation_create (mmx);
5774
5775     /* SSE2 constants */
5776     mask_565_r  = create_mask_2x32_128 (0x00f80000, 0x00f80000);
5777     mask_565_g1 = create_mask_2x32_128 (0x00070000, 0x00070000);
5778     mask_565_g2 = create_mask_2x32_128 (0x000000e0, 0x000000e0);
5779     mask_565_b  = create_mask_2x32_128 (0x0000001f, 0x0000001f);
5780     mask_red   = create_mask_2x32_128 (0x00f80000, 0x00f80000);
5781     mask_green = create_mask_2x32_128 (0x0000fc00, 0x0000fc00);
5782     mask_blue  = create_mask_2x32_128 (0x000000f8, 0x000000f8);
5783     mask_565_fix_rb = create_mask_2x32_128 (0x00e000e0, 0x00e000e0);
5784     mask_565_fix_g = create_mask_2x32_128  (0x0000c000, 0x0000c000);
5785     mask_0080 = create_mask_16_128 (0x0080);
5786     mask_00ff = create_mask_16_128 (0x00ff);
5787     mask_0101 = create_mask_16_128 (0x0101);
5788     mask_ffff = create_mask_16_128 (0xffff);
5789     mask_ff000000 = create_mask_2x32_128 (0xff000000, 0xff000000);
5790     mask_alpha = create_mask_2x32_128 (0x00ff0000, 0x00000000);
5791
5792     /* MMX constants */
5793     mask_x565_rgb = create_mask_2x32_64 (0x000001f0, 0x003f001f);
5794     mask_x565_unpack = create_mask_2x32_64 (0x00000084, 0x04100840);
5795
5796     mask_x0080 = create_mask_16_64 (0x0080);
5797     mask_x00ff = create_mask_16_64 (0x00ff);
5798     mask_x0101 = create_mask_16_64 (0x0101);
5799     mask_x_alpha = create_mask_2x32_64 (0x00ff0000, 0x00000000);
5800
5801     _mm_empty ();
5802
5803     /* Set up function pointers */
5804
5805     /* SSE code patch for fbcompose.c */
5806     imp->combine_32[PIXMAN_OP_OVER] = sse2_combine_over_u;
5807     imp->combine_32[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_u;
5808     imp->combine_32[PIXMAN_OP_IN] = sse2_combine_in_u;
5809     imp->combine_32[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_u;
5810     imp->combine_32[PIXMAN_OP_OUT] = sse2_combine_out_u;
5811     imp->combine_32[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_u;
5812     imp->combine_32[PIXMAN_OP_ATOP] = sse2_combine_atop_u;
5813     imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_u;
5814     imp->combine_32[PIXMAN_OP_XOR] = sse2_combine_xor_u;
5815     imp->combine_32[PIXMAN_OP_ADD] = sse2_combine_add_u;
5816
5817     imp->combine_32[PIXMAN_OP_SATURATE] = sse2_combine_saturate_u;
5818
5819     imp->combine_32_ca[PIXMAN_OP_SRC] = sse2_combine_src_ca;
5820     imp->combine_32_ca[PIXMAN_OP_OVER] = sse2_combine_over_ca;
5821     imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_ca;
5822     imp->combine_32_ca[PIXMAN_OP_IN] = sse2_combine_in_ca;
5823     imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_ca;
5824     imp->combine_32_ca[PIXMAN_OP_OUT] = sse2_combine_out_ca;
5825     imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_ca;
5826     imp->combine_32_ca[PIXMAN_OP_ATOP] = sse2_combine_atop_ca;
5827     imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_ca;
5828     imp->combine_32_ca[PIXMAN_OP_XOR] = sse2_combine_xor_ca;
5829     imp->combine_32_ca[PIXMAN_OP_ADD] = sse2_combine_add_ca;
5830
5831     imp->composite = sse2_composite;
5832     imp->blt = sse2_blt;
5833     imp->fill = sse2_fill;
5834
5835     return imp;
5836 }
5837
5838 #endif /* USE_SSE2 */