pixman/pixman-arm-neon.c

   1 /*
   2  * Copyright © 2009 ARM Ltd, Movial Creative Technologies Oy
   3  *
   4  * Permission to use, copy, modify, distribute, and sell this software and its
   5  * documentation for any purpose is hereby granted without fee, provided that
   6  * the above copyright notice appear in all copies and that both that
   7  * copyright notice and this permission notice appear in supporting
   8  * documentation, and that the name of ARM Ltd not be used in
   9  * advertising or publicity pertaining to distribution of the software without
  10  * specific, written prior permission.  ARM Ltd makes no
  11  * representations about the suitability of this software for any purpose.  It
  12  * is provided "as is" without express or implied warranty.
  13  *
  14  * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
  15  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  16  * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
  17  * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  18  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  19  * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
  20  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
  21  * SOFTWARE.
  22  *
  23  * Author:  Ian Rickards (ian.rickards@arm.com)
  24  * Author:  Jonathan Morton (jonathan.morton@movial.com)
  25  * Author:  Markku Vire (markku.vire@movial.com)
  26  *
  27  */
  28
  29 #ifdef HAVE_CONFIG_H
  30 #include <config.h>
  31 #endif
  32
  33 #include <string.h>
  34 #include "pixman-private.h"
  35 #include "pixman-arm-common.h"
  36
  37 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_8888,
  38                                    uint32_t, 1, uint32_t, 1)
  39 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_x888_8888,
  40                                    uint32_t, 1, uint32_t, 1)
  41 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_0565,
  42                                    uint16_t, 1, uint16_t, 1)
  43 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0888,
  44                                    uint8_t, 3, uint8_t, 3)
  45 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_0565,
  46                                    uint32_t, 1, uint16_t, 1)
  47 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_8888,
  48                                    uint16_t, 1, uint32_t, 1)
  49 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_8888_rev,
  50                                    uint8_t, 3, uint32_t, 1)
  51 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0565_rev,
  52                                    uint8_t, 3, uint16_t, 1)
  53 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_pixbuf_8888,
  54                                    uint32_t, 1, uint32_t, 1)
  55 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_rpixbuf_8888,
  56                                    uint32_t, 1, uint32_t, 1)
  57 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8_8,
  58                                    uint8_t, 1, uint8_t, 1)
  59 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8888_8888,
  60                                    uint32_t, 1, uint32_t, 1)
  61 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_0565,
  62                                    uint32_t, 1, uint16_t, 1)
  63 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_8888,
  64                                    uint32_t, 1, uint32_t, 1)
  65 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, out_reverse_8_0565,
  66                                    uint8_t, 1, uint16_t, 1)
  67 PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, out_reverse_8_8888,
  68                                    uint8_t, 1, uint32_t, 1)
  69
  70 PIXMAN_ARM_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, neon, over_n_0565,
  71                                  uint16_t, 1)
  72 PIXMAN_ARM_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, neon, over_n_8888,
  73                                  uint32_t, 1)
  74 PIXMAN_ARM_BIND_FAST_PATH_N_DST (SKIP_ZERO_SRC, neon, over_reverse_n_8888,
  75                                  uint32_t, 1)
  76 PIXMAN_ARM_BIND_FAST_PATH_N_DST (0, neon, in_n_8,
  77                                  uint8_t, 1)
  78
  79 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8_0565,
  80                                       uint8_t, 1, uint16_t, 1)
  81 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8_8888,
  82                                       uint8_t, 1, uint32_t, 1)
  83 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8888_8888_ca,
  84                                       uint32_t, 1, uint32_t, 1)
  85 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8888_0565_ca,
  86                                       uint32_t, 1, uint16_t, 1)
  87 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, over_n_8_8,
  88                                       uint8_t, 1, uint8_t, 1)
  89 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, add_n_8_8,
  90                                       uint8_t, 1, uint8_t, 1)
  91 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (SKIP_ZERO_SRC, neon, add_n_8_8888,
  92                                       uint8_t, 1, uint32_t, 1)
  93 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (0, neon, src_n_8_8888,
  94                                       uint8_t, 1, uint32_t, 1)
  95 PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (0, neon, src_n_8_8,
  96                                       uint8_t, 1, uint8_t, 1)
  97
  98 PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, over_8888_n_8888,
  99                                      uint32_t, 1, uint32_t, 1)
 100 PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, over_8888_n_0565,
 101                                      uint32_t, 1, uint16_t, 1)
 102 PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, over_0565_n_0565,
 103                                      uint16_t, 1, uint16_t, 1)
 104 PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (SKIP_ZERO_MASK, neon, add_8888_n_8888,
 105                                      uint32_t, 1, uint32_t, 1)
 106
 107 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8_8_8,
 108                                         uint8_t, 1, uint8_t, 1, uint8_t, 1)
 109 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_0565_8_0565,
 110                                         uint16_t, 1, uint8_t, 1, uint16_t, 1)
 111 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8888_8_8888,
 112                                         uint32_t, 1, uint8_t, 1, uint32_t, 1)
 113 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8888_8888_8888,
 114                                         uint32_t, 1, uint32_t, 1, uint32_t, 1)
 115 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8_8888,
 116                                         uint32_t, 1, uint8_t, 1, uint32_t, 1)
 117 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8888_8888,
 118                                         uint32_t, 1, uint32_t, 1, uint32_t, 1)
 119 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8_0565,
 120                                         uint32_t, 1, uint8_t, 1, uint16_t, 1)
 121 PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_0565_8_0565,
 122                                         uint16_t, 1, uint8_t, 1, uint16_t, 1)
 123
 124 PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 8888_8888, OVER,
 125                                         uint32_t, uint32_t)
 126 PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 8888_0565, OVER,
 127                                         uint32_t, uint16_t)
 128 PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 8888_0565, SRC,
 129                                         uint32_t, uint16_t)
 130 PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_DST (neon, 0565_8888, SRC,
 131                                         uint16_t, uint32_t)
 132
 133 PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_0565,
 134                                            OVER, uint32_t, uint16_t)
 135 PIXMAN_ARM_BIND_SCALED_NEAREST_SRC_A8_DST (SKIP_ZERO_SRC, neon, 0565_8_0565,
 136                                            OVER, uint16_t, uint16_t)
 137
 138 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 8888_8888, SRC,
 139                                          uint32_t, uint32_t)
 140 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 8888_0565, SRC,
 141                                          uint32_t, uint16_t)
 142 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 0565_x888, SRC,
 143                                          uint16_t, uint32_t)
 144 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 0565_0565, SRC,
 145                                          uint16_t, uint16_t)
 146 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, neon, 8888_8888, OVER,
 147                                          uint32_t, uint32_t)
 148 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, neon, 8888_8888, ADD,
 149                                          uint32_t, uint32_t)
 150
 151 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 8888_8_8888, SRC,
 152                                             uint32_t, uint32_t)
 153 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 8888_8_0565, SRC,
 154                                             uint32_t, uint16_t)
 155 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 0565_8_x888, SRC,
 156                                             uint16_t, uint32_t)
 157 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (0, neon, 0565_8_0565, SRC,
 158                                             uint16_t, uint16_t)
 159 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_8888, OVER,
 160                                             uint32_t, uint32_t)
 161 PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_8888, ADD,
 162                                             uint32_t, uint32_t)
 163
 164 void
 165 pixman_composite_src_n_8_asm_neon (int32_t   w,
 166                                    int32_t   h,
 167                                    uint8_t  *dst,
 168                                    int32_t   dst_stride,
 169                                    uint8_t   src);
 170
 171 void
 172 pixman_composite_src_n_0565_asm_neon (int32_t   w,
 173                                       int32_t   h,
 174                                       uint16_t *dst,
 175                                       int32_t   dst_stride,
 176                                       uint16_t  src);
 177
 178 void
 179 pixman_composite_src_n_8888_asm_neon (int32_t   w,
 180                                       int32_t   h,
 181                                       uint32_t *dst,
 182                                       int32_t   dst_stride,
 183                                       uint32_t  src);
 184
 185 static pixman_bool_t
 186 pixman_fill_neon (uint32_t *bits,
 187                   int       stride,
 188                   int       bpp,
 189                   int       x,
 190                   int       y,
 191                   int       width,
 192                   int       height,
 193                   uint32_t  _xor)
 194 {
 195     /* stride is always multiple of 32bit units in pixman */
 196     uint32_t byte_stride = stride * sizeof(uint32_t);
 197
 198     switch (bpp)
 199     {
 200     case 8:
 201         pixman_composite_src_n_8_asm_neon (
 202                 width,
 203                 height,
 204                 (uint8_t *)(((char *) bits) + y * byte_stride + x),
 205                 byte_stride,
 206                 _xor & 0xff);
 207         return TRUE;
 208     case 16:
 209         pixman_composite_src_n_0565_asm_neon (
 210                 width,
 211                 height,
 212                 (uint16_t *)(((char *) bits) + y * byte_stride + x * 2),
 213                 byte_stride / 2,
 214                 _xor & 0xffff);
 215         return TRUE;
 216     case 32:
 217         pixman_composite_src_n_8888_asm_neon (
 218                 width,
 219                 height,
 220                 (uint32_t *)(((char *) bits) + y * byte_stride + x * 4),
 221                 byte_stride / 4,
 222                 _xor);
 223         return TRUE;
 224     default:
 225         return FALSE;
 226     }
 227 }
 228
 229 static pixman_bool_t
 230 pixman_blt_neon (uint32_t *src_bits,
 231                  uint32_t *dst_bits,
 232                  int       src_stride,
 233                  int       dst_stride,
 234                  int       src_bpp,
 235                  int       dst_bpp,
 236                  int       src_x,
 237                  int       src_y,
 238                  int       dest_x,
 239                  int       dest_y,
 240                  int       width,
 241                  int       height)
 242 {
 243     if (src_bpp != dst_bpp)
 244         return FALSE;
 245
 246     switch (src_bpp)
 247     {
 248     case 16:
 249         pixman_composite_src_0565_0565_asm_neon (
 250                 width, height,
 251                 (uint16_t *)(((char *) dst_bits) +
 252                 dest_y * dst_stride * 4 + dest_x * 2), dst_stride * 2,
 253                 (uint16_t *)(((char *) src_bits) +
 254                 src_y * src_stride * 4 + src_x * 2), src_stride * 2);
 255         return TRUE;
 256     case 32:
 257         pixman_composite_src_8888_8888_asm_neon (
 258                 width, height,
 259                 (uint32_t *)(((char *) dst_bits) +
 260                 dest_y * dst_stride * 4 + dest_x * 4), dst_stride,
 261                 (uint32_t *)(((char *) src_bits) +
 262                 src_y * src_stride * 4 + src_x * 4), src_stride);
 263         return TRUE;
 264     default:
 265         return FALSE;
 266     }
 267 }
 268
 269 static const pixman_fast_path_t arm_neon_fast_paths[] =
 270 {
 271     PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     r5g6b5,   neon_composite_src_0565_0565),
 272     PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     b5g6r5,   neon_composite_src_0565_0565),
 273     PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),
 274     PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),
 275     PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),
 276     PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),
 277     PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     a8r8g8b8, neon_composite_src_0565_8888),
 278     PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     x8r8g8b8, neon_composite_src_0565_8888),
 279     PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     a8b8g8r8, neon_composite_src_0565_8888),
 280     PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     x8b8g8r8, neon_composite_src_0565_8888),
 281     PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),
 282     PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),
 283     PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),
 284     PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),
 285     PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     a8r8g8b8, neon_composite_src_8888_8888),
 286     PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     a8b8g8r8, neon_composite_src_8888_8888),
 287     PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),
 288     PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),
 289     PIXMAN_STD_FAST_PATH (SRC,  r8g8b8,   null,     r8g8b8,   neon_composite_src_0888_0888),
 290     PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     x8r8g8b8, neon_composite_src_0888_8888_rev),
 291     PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     r5g6b5,   neon_composite_src_0888_0565_rev),
 292     PIXMAN_STD_FAST_PATH (SRC,  pixbuf,   pixbuf,   a8r8g8b8, neon_composite_src_pixbuf_8888),
 293     PIXMAN_STD_FAST_PATH (SRC,  pixbuf,   pixbuf,   a8b8g8r8, neon_composite_src_rpixbuf_8888),
 294     PIXMAN_STD_FAST_PATH (SRC,  rpixbuf,  rpixbuf,  a8r8g8b8, neon_composite_src_rpixbuf_8888),
 295     PIXMAN_STD_FAST_PATH (SRC,  rpixbuf,  rpixbuf,  a8b8g8r8, neon_composite_src_pixbuf_8888),
 296     PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       a8r8g8b8, neon_composite_src_n_8_8888),
 297     PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       x8r8g8b8, neon_composite_src_n_8_8888),
 298     PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       a8b8g8r8, neon_composite_src_n_8_8888),
 299     PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       x8b8g8r8, neon_composite_src_n_8_8888),
 300     PIXMAN_STD_FAST_PATH (SRC,  solid,    a8,       a8,       neon_composite_src_n_8_8),
 301
 302     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8,       neon_composite_over_n_8_8),
 303     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       r5g6b5,   neon_composite_over_n_8_0565),
 304     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       b5g6r5,   neon_composite_over_n_8_0565),
 305     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8r8g8b8, neon_composite_over_n_8_8888),
 306     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8r8g8b8, neon_composite_over_n_8_8888),
 307     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8b8g8r8, neon_composite_over_n_8_8888),
 308     PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8b8g8r8, neon_composite_over_n_8_8888),
 309     PIXMAN_STD_FAST_PATH (OVER, solid,    null,     r5g6b5,   neon_composite_over_n_0565),
 310     PIXMAN_STD_FAST_PATH (OVER, solid,    null,     a8r8g8b8, neon_composite_over_n_8888),
 311     PIXMAN_STD_FAST_PATH (OVER, solid,    null,     x8r8g8b8, neon_composite_over_n_8888),
 312     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, neon_composite_over_n_8888_8888_ca),
 313     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, neon_composite_over_n_8888_8888_ca),
 314     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, neon_composite_over_n_8888_8888_ca),
 315     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, neon_composite_over_n_8888_8888_ca),
 316     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5,   neon_composite_over_n_8888_0565_ca),
 317     PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5,   neon_composite_over_n_8888_0565_ca),
 318     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    a8r8g8b8, neon_composite_over_8888_n_8888),
 319     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    x8r8g8b8, neon_composite_over_8888_n_8888),
 320     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    r5g6b5,   neon_composite_over_8888_n_0565),
 321     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid,    b5g6r5,   neon_composite_over_8888_n_0565),
 322     PIXMAN_STD_FAST_PATH (OVER, r5g6b5,   solid,    r5g6b5,   neon_composite_over_0565_n_0565),
 323     PIXMAN_STD_FAST_PATH (OVER, b5g6r5,   solid,    b5g6r5,   neon_composite_over_0565_n_0565),
 324     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       a8r8g8b8, neon_composite_over_8888_8_8888),
 325     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       x8r8g8b8, neon_composite_over_8888_8_8888),
 326     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       a8b8g8r8, neon_composite_over_8888_8_8888),
 327     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       x8b8g8r8, neon_composite_over_8888_8_8888),
 328     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       r5g6b5,   neon_composite_over_8888_8_0565),
 329     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       b5g6r5,   neon_composite_over_8888_8_0565),
 330     PIXMAN_STD_FAST_PATH (OVER, r5g6b5,   a8,       r5g6b5,   neon_composite_over_0565_8_0565),
 331     PIXMAN_STD_FAST_PATH (OVER, b5g6r5,   a8,       b5g6r5,   neon_composite_over_0565_8_0565),
 332     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_over_8888_8888_8888),
 333     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     r5g6b5,   neon_composite_over_8888_0565),
 334     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     b5g6r5,   neon_composite_over_8888_0565),
 335     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     a8r8g8b8, neon_composite_over_8888_8888),
 336     PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     x8r8g8b8, neon_composite_over_8888_8888),
 337     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     a8b8g8r8, neon_composite_over_8888_8888),
 338     PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     x8b8g8r8, neon_composite_over_8888_8888),
 339     PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),
 340     PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),
 341     PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8,       neon_composite_add_n_8_8),
 342     PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8r8g8b8, neon_composite_add_n_8_8888),
 343     PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8b8g8r8, neon_composite_add_n_8_8888),
 344     PIXMAN_STD_FAST_PATH (ADD,  a8,       a8,       a8,       neon_composite_add_8_8_8),
 345     PIXMAN_STD_FAST_PATH (ADD,  r5g6b5,   a8,       r5g6b5,   neon_composite_add_0565_8_0565),
 346     PIXMAN_STD_FAST_PATH (ADD,  b5g6r5,   a8,       b5g6r5,   neon_composite_add_0565_8_0565),
 347     PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8,       a8r8g8b8, neon_composite_add_8888_8_8888),
 348     PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, a8,       a8b8g8r8, neon_composite_add_8888_8_8888),
 349     PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_add_8888_8888_8888),
 350     PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, solid,    a8r8g8b8, neon_composite_add_8888_n_8888),
 351     PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, solid,    a8b8g8r8, neon_composite_add_8888_n_8888),
 352     PIXMAN_STD_FAST_PATH (ADD,  a8,       null,     a8,       neon_composite_add_8_8),
 353     PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, null,     a8r8g8b8, neon_composite_add_8888_8888),
 354     PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, null,     a8b8g8r8, neon_composite_add_8888_8888),
 355     PIXMAN_STD_FAST_PATH (IN,   solid,    null,     a8,       neon_composite_in_n_8),
 356     PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, neon_composite_over_reverse_n_8888),
 357     PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, neon_composite_over_reverse_n_8888),
 358     PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, r5g6b5,   neon_composite_out_reverse_8_0565),
 359     PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, b5g6r5,   neon_composite_out_reverse_8_0565),
 360     PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, a8r8g8b8, neon_composite_out_reverse_8_8888),
 361     PIXMAN_STD_FAST_PATH (OUT_REVERSE,  a8,    null, a8b8g8r8, neon_composite_out_reverse_8_8888),
 362
 363     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8888),
 364     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, neon_8888_8888),
 365     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, neon_8888_8888),
 366     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, neon_8888_8888),
 367
 368     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, r5g6b5, neon_8888_0565),
 369     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, b5g6r5, neon_8888_0565),
 370
 371     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, r5g6b5, neon_8888_0565),
 372     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, r5g6b5, neon_8888_0565),
 373     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, b5g6r5, neon_8888_0565),
 374     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, x8b8g8r8, b5g6r5, neon_8888_0565),
 375
 376     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, b5g6r5, x8b8g8r8, neon_0565_8888),
 377     PIXMAN_ARM_SIMPLE_NEAREST_FAST_PATH (SRC, r5g6b5, x8r8g8b8, neon_0565_8888),
 378     /* Note: NONE repeat is not supported yet */
 379     SIMPLE_NEAREST_FAST_PATH_COVER (SRC, r5g6b5, a8r8g8b8, neon_0565_8888),
 380     SIMPLE_NEAREST_FAST_PATH_COVER (SRC, b5g6r5, a8b8g8r8, neon_0565_8888),
 381     SIMPLE_NEAREST_FAST_PATH_PAD (SRC, r5g6b5, a8r8g8b8, neon_0565_8888),
 382     SIMPLE_NEAREST_FAST_PATH_PAD (SRC, b5g6r5, a8b8g8r8, neon_0565_8888),
 383
 384     PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, a8r8g8b8, r5g6b5, neon_8888_8_0565),
 385     PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, a8b8g8r8, b5g6r5, neon_8888_8_0565),
 386
 387     PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, r5g6b5, r5g6b5, neon_0565_8_0565),
 388     PIXMAN_ARM_SIMPLE_NEAREST_A8_MASK_FAST_PATH (OVER, b5g6r5, b5g6r5, neon_0565_8_0565),
 389
 390     SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, neon_8888_8888),
 391     SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, neon_8888_8888),
 392     SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, neon_8888_8888),
 393
 394     SIMPLE_BILINEAR_FAST_PATH (SRC, a8r8g8b8, r5g6b5, neon_8888_0565),
 395     SIMPLE_BILINEAR_FAST_PATH (SRC, x8r8g8b8, r5g6b5, neon_8888_0565),
 396
 397     SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, x8r8g8b8, neon_0565_x888),
 398     SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, r5g6b5, neon_0565_0565),
 399
 400     SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8888),
 401     SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, neon_8888_8888),
 402
 403     SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, neon_8888_8888),
 404     SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, neon_8888_8888),
 405
 406     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, neon_8888_8_8888),
 407     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, neon_8888_8_8888),
 408     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, neon_8888_8_8888),
 409
 410     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, r5g6b5, neon_8888_8_0565),
 411     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, x8r8g8b8, r5g6b5, neon_8888_8_0565),
 412
 413     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, r5g6b5, x8r8g8b8, neon_0565_8_x888),
 414     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, r5g6b5, r5g6b5, neon_0565_8_0565),
 415
 416     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8_8888),
 417     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, neon_8888_8_8888),
 418
 419     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, neon_8888_8_8888),
 420     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, neon_8888_8_8888),
 421
 422     { PIXMAN_OP_NONE },
 423 };
 424
 425 static pixman_bool_t
 426 arm_neon_blt (pixman_implementation_t *imp,
 427               uint32_t *               src_bits,
 428               uint32_t *               dst_bits,
 429               int                      src_stride,
 430               int                      dst_stride,
 431               int                      src_bpp,
 432               int                      dst_bpp,
 433               int                      src_x,
 434               int                      src_y,
 435               int                      dest_x,
 436               int                      dest_y,
 437               int                      width,
 438               int                      height)
 439 {
 440     if (!pixman_blt_neon (
 441             src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
 442             src_x, src_y, dest_x, dest_y, width, height))
 443
 444     {
 445         return _pixman_implementation_blt (
 446             imp->delegate,
 447             src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
 448             src_x, src_y, dest_x, dest_y, width, height);
 449     }
 450
 451     return TRUE;
 452 }
 453
 454 static pixman_bool_t
 455 arm_neon_fill (pixman_implementation_t *imp,
 456                uint32_t *               bits,
 457                int                      stride,
 458                int                      bpp,
 459                int                      x,
 460                int                      y,
 461                int                      width,
 462                int                      height,
 463                uint32_t xor)
 464 {
 465     if (pixman_fill_neon (bits, stride, bpp, x, y, width, height, xor))
 466         return TRUE;
 467
 468     return _pixman_implementation_fill (
 469         imp->delegate, bits, stride, bpp, x, y, width, height, xor);
 470 }
 471
 472 #define BIND_COMBINE_U(name)                                             \
 473 void                                                                     \
 474 pixman_composite_scanline_##name##_mask_asm_neon (int32_t         w,     \
 475                                                   const uint32_t *dst,   \
 476                                                   const uint32_t *src,   \
 477                                                   const uint32_t *mask); \
 478                                                                          \
 479 void                                                                     \
 480 pixman_composite_scanline_##name##_asm_neon (int32_t         w,          \
 481                                              const uint32_t *dst,        \
 482                                              const uint32_t *src);       \
 483                                                                          \
 484 static void                                                              \
 485 neon_combine_##name##_u (pixman_implementation_t *imp,                   \
 486                          pixman_op_t              op,                    \
 487                          uint32_t *               dest,                  \
 488                          const uint32_t *         src,                   \
 489                          const uint32_t *         mask,                  \
 490                          int                      width)                 \
 491 {                                                                        \
 492     if (mask)                                                            \
 493         pixman_composite_scanline_##name##_mask_asm_neon (width, dest,   \
 494                                                           src, mask);    \
 495     else                                                                 \
 496         pixman_composite_scanline_##name##_asm_neon (width, dest, src);  \
 497 }
 498
 499 BIND_COMBINE_U (over)
 500 BIND_COMBINE_U (add)
 501 BIND_COMBINE_U (out_reverse)
 502
 503 pixman_implementation_t *
 504 _pixman_implementation_create_arm_neon (pixman_implementation_t *fallback)
 505 {
 506     pixman_implementation_t *imp =
 507         _pixman_implementation_create (fallback, arm_neon_fast_paths);
 508
 509     imp->combine_32[PIXMAN_OP_OVER] = neon_combine_over_u;
 510     imp->combine_32[PIXMAN_OP_ADD] = neon_combine_add_u;
 511     imp->combine_32[PIXMAN_OP_OUT_REVERSE] = neon_combine_out_reverse_u;
 512
 513     imp->blt = arm_neon_blt;
 514     imp->fill = arm_neon_fill;
 515
 516     return imp;
 517 }