pixman/pixman-arm-neon.c

   1 /*
   2  * Copyright © 2009 ARM Ltd, Movial Creative Technologies Oy
   3  *
   4  * Permission to use, copy, modify, distribute, and sell this software and its
   5  * documentation for any purpose is hereby granted without fee, provided that
   6  * the above copyright notice appear in all copies and that both that
   7  * copyright notice and this permission notice appear in supporting
   8  * documentation, and that the name of ARM Ltd not be used in
   9  * advertising or publicity pertaining to distribution of the software without
  10  * specific, written prior permission.  ARM Ltd makes no
  11  * representations about the suitability of this software for any purpose.  It
  12  * is provided "as is" without express or implied warranty.
  13  *
  14  * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
  15  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  16  * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
  17  * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  18  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  19  * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
  20  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
  21  * SOFTWARE.
  22  *
  23  * Author:  Ian Rickards (ian.rickards@arm.com)
  24  * Author:  Jonathan Morton (jonathan.morton@movial.com)
  25  * Author:  Markku Vire (markku.vire@movial.com)
  26  *
  27  */
  28
  29 #ifdef HAVE_CONFIG_H
  30 #include <config.h>
  31 #endif
  32
  33 #include <string.h>
  34 #include "pixman-private.h"
  35
  36 #define BIND_SRC_NULL_DST(name, src_type, src_cnt, dst_type, dst_cnt)   \
  37 void                                                                    \
  38 pixman_composite_##name##_asm_neon (int32_t   w,                        \
  39                                     int32_t   h,                        \
  40                                     dst_type *dst,                      \
  41                                     int32_t   dst_stride,               \
  42                                     src_type *src,                      \
  43                                     int32_t   src_stride);              \
  44                                                                         \
  45 static void                                                             \
  46 neon_composite_##name (pixman_implementation_t *imp,                    \
  47                        pixman_op_t              op,                     \
  48                        pixman_image_t *         src_image,              \
  49                        pixman_image_t *         mask_image,             \
  50                        pixman_image_t *         dst_image,              \
  51                        int32_t                  src_x,                  \
  52                        int32_t                  src_y,                  \
  53                        int32_t                  mask_x,                 \
  54                        int32_t                  mask_y,                 \
  55                        int32_t                  dest_x,                 \
  56                        int32_t                  dest_y,                 \
  57                        int32_t                  width,                  \
  58                        int32_t                  height)                 \
  59 {                                                                       \
  60     dst_type *dst_line;                                                 \
  61     src_type *src_line;                                                 \
  62     int32_t dst_stride, src_stride;                                     \
  63                                                                         \
  64     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
  65                            src_stride, src_line, src_cnt);              \
  66     PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
  67                            dst_stride, dst_line, dst_cnt);              \
  68                                                                         \
  69     pixman_composite_##name##_asm_neon (width, height,                  \
  70                                         dst_line, dst_stride,           \
  71                                         src_line, src_stride);          \
  72 }
  73
  74 #define BIND_N_NULL_DST(name, dst_type, dst_cnt)                        \
  75 void                                                                    \
  76 pixman_composite_##name##_asm_neon (int32_t    w,                       \
  77                                     int32_t    h,                       \
  78                                     dst_type  *dst,                     \
  79                                     int32_t    dst_stride,              \
  80                                     uint32_t   src);                    \
  81                                                                         \
  82 static void                                                             \
  83 neon_composite_##name (pixman_implementation_t *imp,                    \
  84                        pixman_op_t              op,                     \
  85                        pixman_image_t *         src_image,              \
  86                        pixman_image_t *         mask_image,             \
  87                        pixman_image_t *         dst_image,              \
  88                        int32_t                  src_x,                  \
  89                        int32_t                  src_y,                  \
  90                        int32_t                  mask_x,                 \
  91                        int32_t                  mask_y,                 \
  92                        int32_t                  dest_x,                 \
  93                        int32_t                  dest_y,                 \
  94                        int32_t                  width,                  \
  95                        int32_t                  height)                 \
  96 {                                                                       \
  97     dst_type  *dst_line;                                                \
  98     int32_t    dst_stride;                                              \
  99     uint32_t   src;                                                     \
 100                                                                         \
 101     src = _pixman_image_get_solid (src_image, dst_image->bits.format);  \
 102                                                                         \
 103     if (src == 0)                                                       \
 104         return;                                                         \
 105                                                                         \
 106     PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
 107                            dst_stride, dst_line, dst_cnt);              \
 108                                                                         \
 109     pixman_composite_##name##_asm_neon (width, height,                  \
 110                                         dst_line, dst_stride,           \
 111                                         src);                           \
 112 }
 113
 114 #define BIND_N_MASK_DST(name, mask_type, mask_cnt, dst_type, dst_cnt)   \
 115 void                                                                    \
 116 pixman_composite_##name##_asm_neon (int32_t    w,                       \
 117                                     int32_t    h,                       \
 118                                     dst_type  *dst,                     \
 119                                     int32_t    dst_stride,              \
 120                                     uint32_t   src,                     \
 121                                     int32_t    unused,                  \
 122                                     mask_type *mask,                    \
 123                                     int32_t    mask_stride);            \
 124                                                                         \
 125 static void                                                             \
 126 neon_composite_##name (pixman_implementation_t *imp,                    \
 127                        pixman_op_t              op,                     \
 128                        pixman_image_t *         src_image,              \
 129                        pixman_image_t *         mask_image,             \
 130                        pixman_image_t *         dst_image,              \
 131                        int32_t                  src_x,                  \
 132                        int32_t                  src_y,                  \
 133                        int32_t                  mask_x,                 \
 134                        int32_t                  mask_y,                 \
 135                        int32_t                  dest_x,                 \
 136                        int32_t                  dest_y,                 \
 137                        int32_t                  width,                  \
 138                        int32_t                  height)                 \
 139 {                                                                       \
 140     dst_type  *dst_line;                                                \
 141     mask_type *mask_line;                                               \
 142     int32_t    dst_stride, mask_stride;                                 \
 143     uint32_t   src;                                                     \
 144                                                                         \
 145     src = _pixman_image_get_solid (src_image, dst_image->bits.format);  \
 146                                                                         \
 147     if (src == 0)                                                       \
 148         return;                                                         \
 149                                                                         \
 150     PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
 151                            dst_stride, dst_line, dst_cnt);              \
 152     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type,       \
 153                            mask_stride, mask_line, mask_cnt);           \
 154                                                                         \
 155     pixman_composite_##name##_asm_neon (width, height,                  \
 156                                         dst_line, dst_stride,           \
 157                                         src, 0,                         \
 158                                         mask_line, mask_stride);        \
 159 }
 160
 161 #define BIND_SRC_N_DST(name, src_type, src_cnt, dst_type, dst_cnt)      \
 162 void                                                                    \
 163 pixman_composite_##name##_asm_neon (int32_t    w,                       \
 164                                     int32_t    h,                       \
 165                                     dst_type  *dst,                     \
 166                                     int32_t    dst_stride,              \
 167                                     src_type  *src,                     \
 168                                     int32_t    src_stride,              \
 169                                     uint32_t   mask);                   \
 170                                                                         \
 171 static void                                                             \
 172 neon_composite_##name (pixman_implementation_t *imp,                    \
 173                        pixman_op_t              op,                     \
 174                        pixman_image_t *         src_image,              \
 175                        pixman_image_t *         mask_image,             \
 176                        pixman_image_t *         dst_image,              \
 177                        int32_t                  src_x,                  \
 178                        int32_t                  src_y,                  \
 179                        int32_t                  mask_x,                 \
 180                        int32_t                  mask_y,                 \
 181                        int32_t                  dest_x,                 \
 182                        int32_t                  dest_y,                 \
 183                        int32_t                  width,                  \
 184                        int32_t                  height)                 \
 185 {                                                                       \
 186     dst_type  *dst_line;                                                \
 187     src_type  *src_line;                                                \
 188     int32_t    dst_stride, src_stride;                                  \
 189     uint32_t   mask;                                                    \
 190                                                                         \
 191     mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);\
 192                                                                         \
 193     if (mask == 0)                                                      \
 194         return;                                                         \
 195                                                                         \
 196     PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
 197                            dst_stride, dst_line, dst_cnt);              \
 198     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
 199                            src_stride, src_line, src_cnt);              \
 200                                                                         \
 201     pixman_composite_##name##_asm_neon (width, height,                  \
 202                                         dst_line, dst_stride,           \
 203                                         src_line, src_stride,           \
 204                                         mask);                          \
 205 }
 206
 207 #define BIND_SRC_MASK_DST(name, src_type, src_cnt, mask_type, mask_cnt, \
 208                           dst_type, dst_cnt)                            \
 209 void                                                                    \
 210 pixman_composite_##name##_asm_neon (int32_t    w,                       \
 211                                     int32_t    h,                       \
 212                                     dst_type  *dst,                     \
 213                                     int32_t    dst_stride,              \
 214                                     src_type  *src,                     \
 215                                     int32_t    src_stride,              \
 216                                     mask_type *mask,                    \
 217                                     int32_t    mask_stride);            \
 218                                                                         \
 219 static void                                                             \
 220 neon_composite_##name (pixman_implementation_t *imp,                    \
 221                        pixman_op_t              op,                     \
 222                        pixman_image_t *         src_image,              \
 223                        pixman_image_t *         mask_image,             \
 224                        pixman_image_t *         dst_image,              \
 225                        int32_t                  src_x,                  \
 226                        int32_t                  src_y,                  \
 227                        int32_t                  mask_x,                 \
 228                        int32_t                  mask_y,                 \
 229                        int32_t                  dest_x,                 \
 230                        int32_t                  dest_y,                 \
 231                        int32_t                  width,                  \
 232                        int32_t                  height)                 \
 233 {                                                                       \
 234     dst_type  *dst_line;                                                \
 235     src_type  *src_line;                                                \
 236     mask_type *mask_line;                                               \
 237     int32_t    dst_stride, src_stride, mask_stride;                     \
 238                                                                         \
 239     PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
 240                            dst_stride, dst_line, dst_cnt);              \
 241     PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
 242                            src_stride, src_line, src_cnt);              \
 243     PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type,       \
 244                            mask_stride, mask_line, mask_cnt);           \
 245                                                                         \
 246     pixman_composite_##name##_asm_neon (width, height,                  \
 247                                         dst_line, dst_stride,           \
 248                                         src_line, src_stride,           \
 249                                         mask_line, mask_stride);        \
 250 }
 251
 252
 253 BIND_SRC_NULL_DST(src_8888_8888, uint32_t, 1, uint32_t, 1)
 254 BIND_SRC_NULL_DST(src_0565_0565, uint16_t, 1, uint16_t, 1)
 255 BIND_SRC_NULL_DST(src_0888_0888, uint8_t, 3, uint8_t, 3)
 256 BIND_SRC_NULL_DST(src_8888_0565, uint32_t, 1, uint16_t, 1)
 257 BIND_SRC_NULL_DST(src_0565_8888, uint16_t, 1, uint32_t, 1)
 258 BIND_SRC_NULL_DST(src_0888_8888_rev, uint8_t, 3, uint32_t, 1)
 259 BIND_SRC_NULL_DST(src_0888_0565_rev, uint8_t, 3, uint16_t, 1)
 260 BIND_SRC_NULL_DST(src_pixbuf_8888, uint32_t, 1, uint32_t, 1)
 261 BIND_SRC_NULL_DST(add_8000_8000, uint8_t, 1, uint8_t, 1)
 262 BIND_SRC_NULL_DST(add_8888_8888, uint32_t, 1, uint32_t, 1)
 263
 264 BIND_N_NULL_DST(over_n_0565, uint16_t, 1)
 265 BIND_N_NULL_DST(over_n_8888, uint32_t, 1)
 266
 267 BIND_SRC_NULL_DST(over_8888_0565, uint32_t, 1, uint16_t, 1)
 268 BIND_SRC_NULL_DST(over_8888_8888, uint32_t, 1, uint32_t, 1)
 269
 270 BIND_N_MASK_DST(over_n_8_0565, uint8_t, 1, uint16_t, 1)
 271 BIND_N_MASK_DST(over_n_8_8888, uint8_t, 1, uint32_t, 1)
 272 BIND_N_MASK_DST(add_n_8_8, uint8_t, 1, uint8_t, 1)
 273
 274 BIND_SRC_N_DST(over_8888_n_8888, uint32_t, 1, uint32_t, 1)
 275
 276 BIND_SRC_MASK_DST(add_8_8_8, uint8_t, 1, uint8_t, 1, uint8_t, 1)
 277 BIND_SRC_MASK_DST(add_8888_8888_8888, uint32_t, 1, uint32_t, 1, uint32_t, 1)
 278 BIND_SRC_MASK_DST(over_8888_8_8888, uint32_t, 1, uint8_t, 1, uint32_t, 1)
 279 BIND_SRC_MASK_DST(over_8888_8888_8888, uint32_t, 1, uint32_t, 1, uint32_t, 1)
 280
 281 void
 282 pixman_composite_src_n_8_asm_neon (int32_t   w,
 283                                    int32_t   h,
 284                                    uint8_t  *dst,
 285                                    int32_t   dst_stride,
 286                                    uint8_t   src);
 287
 288 void
 289 pixman_composite_src_n_0565_asm_neon (int32_t   w,
 290                                       int32_t   h,
 291                                       uint16_t *dst,
 292                                       int32_t   dst_stride,
 293                                       uint16_t  src);
 294
 295 void
 296 pixman_composite_src_n_8888_asm_neon (int32_t   w,
 297                                       int32_t   h,
 298                                       uint32_t *dst,
 299                                       int32_t   dst_stride,
 300                                       uint32_t  src);
 301
 302 static pixman_bool_t
 303 pixman_fill_neon (uint32_t *bits,
 304                   int       stride,
 305                   int       bpp,
 306                   int       x,
 307                   int       y,
 308                   int       width,
 309                   int       height,
 310                   uint32_t  _xor)
 311 {
 312     /* stride is always multiple of 32bit units in pixman */
 313     uint32_t byte_stride = stride * sizeof(uint32_t);
 314
 315     switch (bpp)
 316     {
 317     case 8:
 318         pixman_composite_src_n_8_asm_neon (
 319                 width,
 320                 height,
 321                 (uint8_t *)(((char *) bits) + y * byte_stride + x),
 322                 byte_stride,
 323                 _xor & 0xff);
 324         return TRUE;
 325     case 16:
 326         pixman_composite_src_n_0565_asm_neon (
 327                 width,
 328                 height,
 329                 (uint16_t *)(((char *) bits) + y * byte_stride + x * 2),
 330                 byte_stride / 2,
 331                 _xor & 0xffff);
 332         return TRUE;
 333     case 32:
 334         pixman_composite_src_n_8888_asm_neon (
 335                 width,
 336                 height,
 337                 (uint32_t *)(((char *) bits) + y * byte_stride + x * 4),
 338                 byte_stride / 4,
 339                 _xor);
 340         return TRUE;
 341     default:
 342         return FALSE;
 343     }
 344 }
 345
 346 static pixman_bool_t
 347 pixman_blt_neon (uint32_t *src_bits,
 348                  uint32_t *dst_bits,
 349                  int       src_stride,
 350                  int       dst_stride,
 351                  int       src_bpp,
 352                  int       dst_bpp,
 353                  int       src_x,
 354                  int       src_y,
 355                  int       dst_x,
 356                  int       dst_y,
 357                  int       width,
 358                  int       height)
 359 {
 360     if (src_bpp != dst_bpp)
 361         return FALSE;
 362
 363     switch (src_bpp)
 364     {
 365     case 16:
 366         pixman_composite_src_0565_0565_asm_neon (
 367                 width, height,
 368                 (uint16_t *)(((char *) dst_bits) +
 369                 dst_y * dst_stride * 4 + dst_x * 2), dst_stride * 2,
 370                 (uint16_t *)(((char *) src_bits) +
 371                 src_y * src_stride * 4 + src_x * 2), src_stride * 2);
 372         return TRUE;
 373     case 32:
 374         pixman_composite_src_8888_8888_asm_neon (
 375                 width, height,
 376                 (uint32_t *)(((char *) dst_bits) +
 377                 dst_y * dst_stride * 4 + dst_x * 4), dst_stride,
 378                 (uint32_t *)(((char *) src_bits) +
 379                 src_y * src_stride * 4 + src_x * 4), src_stride);
 380         return TRUE;
 381     default:
 382         return FALSE;
 383     }
 384 }
 385
 386 static const pixman_fast_path_t arm_neon_fast_path_array[] =
 387 {
 388     { PIXMAN_OP_SRC,  PIXMAN_r5g6b5,   PIXMAN_null,     PIXMAN_r5g6b5,   neon_composite_src_0565_0565    },
 389     { PIXMAN_OP_SRC,  PIXMAN_b5g6r5,   PIXMAN_null,     PIXMAN_b5g6r5,   neon_composite_src_0565_0565    },
 390     { PIXMAN_OP_SRC,  PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_r5g6b5,   neon_composite_src_8888_0565    },
 391     { PIXMAN_OP_SRC,  PIXMAN_x8r8g8b8, PIXMAN_null,     PIXMAN_r5g6b5,   neon_composite_src_8888_0565    },
 392     { PIXMAN_OP_SRC,  PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_b5g6r5,   neon_composite_src_8888_0565    },
 393     { PIXMAN_OP_SRC,  PIXMAN_x8b8g8r8, PIXMAN_null,     PIXMAN_b5g6r5,   neon_composite_src_8888_0565    },
 394     { PIXMAN_OP_SRC,  PIXMAN_r5g6b5,   PIXMAN_null,     PIXMAN_a8r8g8b8, neon_composite_src_0565_8888    },
 395     { PIXMAN_OP_SRC,  PIXMAN_r5g6b5,   PIXMAN_null,     PIXMAN_x8r8g8b8, neon_composite_src_0565_8888    },
 396     { PIXMAN_OP_SRC,  PIXMAN_b5g6r5,   PIXMAN_null,     PIXMAN_a8b8g8r8, neon_composite_src_0565_8888    },
 397     { PIXMAN_OP_SRC,  PIXMAN_b5g6r5,   PIXMAN_null,     PIXMAN_x8b8g8r8, neon_composite_src_0565_8888    },
 398     { PIXMAN_OP_SRC,  PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_x8r8g8b8, neon_composite_src_8888_8888    },
 399     { PIXMAN_OP_SRC,  PIXMAN_x8r8g8b8, PIXMAN_null,     PIXMAN_x8r8g8b8, neon_composite_src_8888_8888    },
 400     { PIXMAN_OP_SRC,  PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_x8b8g8r8, neon_composite_src_8888_8888    },
 401     { PIXMAN_OP_SRC,  PIXMAN_x8b8g8r8, PIXMAN_null,     PIXMAN_x8b8g8r8, neon_composite_src_8888_8888    },
 402     { PIXMAN_OP_SRC,  PIXMAN_r8g8b8,   PIXMAN_null,     PIXMAN_r8g8b8,   neon_composite_src_0888_0888    },
 403     { PIXMAN_OP_SRC,  PIXMAN_b8g8r8,   PIXMAN_null,     PIXMAN_x8r8g8b8, neon_composite_src_0888_8888_rev },
 404     { PIXMAN_OP_SRC,  PIXMAN_b8g8r8,   PIXMAN_null,     PIXMAN_r5g6b5,   neon_composite_src_0888_0565_rev },
 405     { PIXMAN_OP_SRC,  PIXMAN_pixbuf,   PIXMAN_pixbuf,   PIXMAN_a8r8g8b8, neon_composite_src_pixbuf_8888  },
 406     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_r5g6b5,   neon_composite_over_n_8_0565    },
 407     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_b5g6r5,   neon_composite_over_n_8_0565    },
 408     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_a8r8g8b8, neon_composite_over_n_8_8888    },
 409     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_x8r8g8b8, neon_composite_over_n_8_8888    },
 410     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_a8b8g8r8, neon_composite_over_n_8_8888    },
 411     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_a8,       PIXMAN_x8b8g8r8, neon_composite_over_n_8_8888    },
 412     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_null,     PIXMAN_r5g6b5,   neon_composite_over_n_0565      },
 413     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_null,     PIXMAN_a8r8g8b8, neon_composite_over_n_8888      },
 414     { PIXMAN_OP_OVER, PIXMAN_solid,    PIXMAN_null,     PIXMAN_x8r8g8b8, neon_composite_over_n_8888      },
 415     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_solid,    PIXMAN_a8r8g8b8, neon_composite_over_8888_n_8888 },
 416     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_solid,    PIXMAN_x8r8g8b8, neon_composite_over_8888_n_8888 },
 417     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_a8,       PIXMAN_a8r8g8b8, neon_composite_over_8888_8_8888 },
 418     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_a8,       PIXMAN_x8r8g8b8, neon_composite_over_8888_8_8888 },
 419     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_a8,       PIXMAN_a8b8g8r8, neon_composite_over_8888_8_8888 },
 420     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_a8,       PIXMAN_x8b8g8r8, neon_composite_over_8888_8_8888 },
 421     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_a8r8g8b8, PIXMAN_a8r8g8b8, neon_composite_over_8888_8888_8888 },
 422     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_r5g6b5,   neon_composite_over_8888_0565   },
 423     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_b5g6r5,   neon_composite_over_8888_0565   },
 424     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_a8r8g8b8, neon_composite_over_8888_8888   },
 425     { PIXMAN_OP_OVER, PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_x8r8g8b8, neon_composite_over_8888_8888   },
 426     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_a8b8g8r8, neon_composite_over_8888_8888   },
 427     { PIXMAN_OP_OVER, PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_x8b8g8r8, neon_composite_over_8888_8888   },
 428     { PIXMAN_OP_ADD,  PIXMAN_solid,    PIXMAN_a8,       PIXMAN_a8,       neon_composite_add_n_8_8        },
 429     { PIXMAN_OP_ADD,  PIXMAN_a8,       PIXMAN_a8,       PIXMAN_a8,       neon_composite_add_8_8_8        },
 430     { PIXMAN_OP_ADD,  PIXMAN_a8r8g8b8, PIXMAN_a8r8g8b8, PIXMAN_a8r8g8b8, neon_composite_add_8888_8888_8888 },
 431     { PIXMAN_OP_ADD,  PIXMAN_a8,       PIXMAN_null,     PIXMAN_a8,       neon_composite_add_8000_8000    },
 432     { PIXMAN_OP_ADD,  PIXMAN_a8r8g8b8, PIXMAN_null,     PIXMAN_a8r8g8b8, neon_composite_add_8888_8888    },
 433     { PIXMAN_OP_ADD,  PIXMAN_a8b8g8r8, PIXMAN_null,     PIXMAN_a8b8g8r8, neon_composite_add_8888_8888    },
 434     { PIXMAN_OP_NONE },
 435 };
 436
 437 const pixman_fast_path_t *const arm_neon_fast_paths = arm_neon_fast_path_array;
 438
 439 static void
 440 arm_neon_composite (pixman_implementation_t *imp,
 441                     pixman_op_t              op,
 442                     pixman_image_t *         src,
 443                     pixman_image_t *         mask,
 444                     pixman_image_t *         dest,
 445                     int32_t                  src_x,
 446                     int32_t                  src_y,
 447                     int32_t                  mask_x,
 448                     int32_t                  mask_y,
 449                     int32_t                  dest_x,
 450                     int32_t                  dest_y,
 451                     int32_t                  width,
 452                     int32_t                  height)
 453 {
 454     if (_pixman_run_fast_path (arm_neon_fast_paths, imp,
 455                                op, src, mask, dest,
 456                                src_x, src_y,
 457                                mask_x, mask_y,
 458                                dest_x, dest_y,
 459                                width, height))
 460     {
 461         return;
 462     }
 463
 464     _pixman_implementation_composite (imp->delegate, op,
 465                                       src, mask, dest,
 466                                       src_x, src_y,
 467                                       mask_x, mask_y,
 468                                       dest_x, dest_y,
 469                                       width, height);
 470 }
 471
 472 static pixman_bool_t
 473 arm_neon_blt (pixman_implementation_t *imp,
 474               uint32_t *               src_bits,
 475               uint32_t *               dst_bits,
 476               int                      src_stride,
 477               int                      dst_stride,
 478               int                      src_bpp,
 479               int                      dst_bpp,
 480               int                      src_x,
 481               int                      src_y,
 482               int                      dst_x,
 483               int                      dst_y,
 484               int                      width,
 485               int                      height)
 486 {
 487     if (!pixman_blt_neon (
 488             src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
 489             src_x, src_y, dst_x, dst_y, width, height))
 490
 491     {
 492         return _pixman_implementation_blt (
 493             imp->delegate,
 494             src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
 495             src_x, src_y, dst_x, dst_y, width, height);
 496     }
 497
 498     return TRUE;
 499 }
 500
 501 static pixman_bool_t
 502 arm_neon_fill (pixman_implementation_t *imp,
 503                uint32_t *               bits,
 504                int                      stride,
 505                int                      bpp,
 506                int                      x,
 507                int                      y,
 508                int                      width,
 509                int                      height,
 510                uint32_t xor)
 511 {
 512     if (pixman_fill_neon (bits, stride, bpp, x, y, width, height, xor))
 513         return TRUE;
 514
 515     return _pixman_implementation_fill (
 516         imp->delegate, bits, stride, bpp, x, y, width, height, xor);
 517 }
 518
 519 #define BIND_COMBINE_U(name)                                             \
 520 void                                                                     \
 521 pixman_composite_scanline_##name##_mask_asm_neon (int32_t         w,     \
 522                                                   const uint32_t *dst,   \
 523                                                   const uint32_t *src,   \
 524                                                   const uint32_t *mask); \
 525                                                                          \
 526 void                                                                     \
 527 pixman_composite_scanline_##name##_asm_neon (int32_t         w,          \
 528                                              const uint32_t *dst,        \
 529                                              const uint32_t *src);       \
 530                                                                          \
 531 static void                                                              \
 532 neon_combine_##name##_u (pixman_implementation_t *imp,                   \
 533                          pixman_op_t              op,                    \
 534                          uint32_t *               dest,                  \
 535                          const uint32_t *         src,                   \
 536                          const uint32_t *         mask,                  \
 537                          int                      width)                 \
 538 {                                                                        \
 539     if (mask)                                                            \
 540         pixman_composite_scanline_##name##_mask_asm_neon (width, dest,   \
 541                                                           src, mask);    \
 542     else                                                                 \
 543         pixman_composite_scanline_##name##_asm_neon (width, dest, src);  \
 544 }
 545
 546 BIND_COMBINE_U (over)
 547 BIND_COMBINE_U (add)
 548
 549 pixman_implementation_t *
 550 _pixman_implementation_create_arm_neon (void)
 551 {
 552     pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
 553     pixman_implementation_t *imp = _pixman_implementation_create (general);
 554
 555     imp->combine_32[PIXMAN_OP_OVER] = neon_combine_over_u;
 556     imp->combine_32[PIXMAN_OP_ADD] = neon_combine_add_u;
 557
 558     imp->composite = arm_neon_composite;
 559     imp->blt = arm_neon_blt;
 560     imp->fill = arm_neon_fill;
 561
 562     return imp;
 563 }