src/third_party/libyuv/source/rotate_neon.cc

   1 /*
   2  *  Copyright 2011 The LibYuv Project Authors. All rights reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS. All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "libyuv/row.h"
  12
  13 #include "libyuv/basic_types.h"
  14
  15 #ifdef __cplusplus
  16 namespace libyuv {
  17 extern "C" {
  18 #endif
  19
  20 #if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__)
  21
  22 static uvec8 kVTbl4x4Transpose =
  23   { 0,  4,  8, 12,  1,  5,  9, 13,  2,  6, 10, 14,  3,  7, 11, 15 };
  24
  25 void TransposeWx8_NEON(const uint8* src, int src_stride,
  26                        uint8* dst, int dst_stride,
  27                        int width) {
  28   const uint8* src_temp = NULL;
  29   asm volatile (
  30     // loops are on blocks of 8. loop will stop when
  31     // counter gets to or below 0. starting the counter
  32     // at w-8 allow for this
  33     "sub         %5, #8                        \n"
  34
  35     // handle 8x8 blocks. this should be the majority of the plane
  36     ".p2align  2                               \n"
  37     "1:                                        \n"
  38       "mov         %0, %1                      \n"
  39
  40       MEMACCESS(0)
  41       "vld1.8      {d0}, [%0], %2              \n"
  42       MEMACCESS(0)
  43       "vld1.8      {d1}, [%0], %2              \n"
  44       MEMACCESS(0)
  45       "vld1.8      {d2}, [%0], %2              \n"
  46       MEMACCESS(0)
  47       "vld1.8      {d3}, [%0], %2              \n"
  48       MEMACCESS(0)
  49       "vld1.8      {d4}, [%0], %2              \n"
  50       MEMACCESS(0)
  51       "vld1.8      {d5}, [%0], %2              \n"
  52       MEMACCESS(0)
  53       "vld1.8      {d6}, [%0], %2              \n"
  54       MEMACCESS(0)
  55       "vld1.8      {d7}, [%0]                  \n"
  56
  57       "vtrn.8      d1, d0                      \n"
  58       "vtrn.8      d3, d2                      \n"
  59       "vtrn.8      d5, d4                      \n"
  60       "vtrn.8      d7, d6                      \n"
  61
  62       "vtrn.16     d1, d3                      \n"
  63       "vtrn.16     d0, d2                      \n"
  64       "vtrn.16     d5, d7                      \n"
  65       "vtrn.16     d4, d6                      \n"
  66
  67       "vtrn.32     d1, d5                      \n"
  68       "vtrn.32     d0, d4                      \n"
  69       "vtrn.32     d3, d7                      \n"
  70       "vtrn.32     d2, d6                      \n"
  71
  72       "vrev16.8    q0, q0                      \n"
  73       "vrev16.8    q1, q1                      \n"
  74       "vrev16.8    q2, q2                      \n"
  75       "vrev16.8    q3, q3                      \n"
  76
  77       "mov         %0, %3                      \n"
  78
  79     MEMACCESS(0)
  80       "vst1.8      {d1}, [%0], %4              \n"
  81     MEMACCESS(0)
  82       "vst1.8      {d0}, [%0], %4              \n"
  83     MEMACCESS(0)
  84       "vst1.8      {d3}, [%0], %4              \n"
  85     MEMACCESS(0)
  86       "vst1.8      {d2}, [%0], %4              \n"
  87     MEMACCESS(0)
  88       "vst1.8      {d5}, [%0], %4              \n"
  89     MEMACCESS(0)
  90       "vst1.8      {d4}, [%0], %4              \n"
  91     MEMACCESS(0)
  92       "vst1.8      {d7}, [%0], %4              \n"
  93     MEMACCESS(0)
  94       "vst1.8      {d6}, [%0]                  \n"
  95
  96       "add         %1, #8                      \n"  // src += 8
  97       "add         %3, %3, %4, lsl #3          \n"  // dst += 8 * dst_stride
  98       "subs        %5,  #8                     \n"  // w   -= 8
  99       "bge         1b                          \n"
 100
 101     // add 8 back to counter. if the result is 0 there are
 102     // no residuals.
 103     "adds        %5, #8                        \n"
 104     "beq         4f                            \n"
 105
 106     // some residual, so between 1 and 7 lines left to transpose
 107     "cmp         %5, #2                        \n"
 108     "blt         3f                            \n"
 109
 110     "cmp         %5, #4                        \n"
 111     "blt         2f                            \n"
 112
 113     // 4x8 block
 114     "mov         %0, %1                        \n"
 115     MEMACCESS(0)
 116     "vld1.32     {d0[0]}, [%0], %2             \n"
 117     MEMACCESS(0)
 118     "vld1.32     {d0[1]}, [%0], %2             \n"
 119     MEMACCESS(0)
 120     "vld1.32     {d1[0]}, [%0], %2             \n"
 121     MEMACCESS(0)
 122     "vld1.32     {d1[1]}, [%0], %2             \n"
 123     MEMACCESS(0)
 124     "vld1.32     {d2[0]}, [%0], %2             \n"
 125     MEMACCESS(0)
 126     "vld1.32     {d2[1]}, [%0], %2             \n"
 127     MEMACCESS(0)
 128     "vld1.32     {d3[0]}, [%0], %2             \n"
 129     MEMACCESS(0)
 130     "vld1.32     {d3[1]}, [%0]                 \n"
 131
 132     "mov         %0, %3                        \n"
 133
 134     MEMACCESS(6)
 135     "vld1.8      {q3}, [%6]                    \n"
 136
 137     "vtbl.8      d4, {d0, d1}, d6              \n"
 138     "vtbl.8      d5, {d0, d1}, d7              \n"
 139     "vtbl.8      d0, {d2, d3}, d6              \n"
 140     "vtbl.8      d1, {d2, d3}, d7              \n"
 141
 142     // TODO(frkoenig): Rework shuffle above to
 143     // write out with 4 instead of 8 writes.
 144     MEMACCESS(0)
 145     "vst1.32     {d4[0]}, [%0], %4             \n"
 146     MEMACCESS(0)
 147     "vst1.32     {d4[1]}, [%0], %4             \n"
 148     MEMACCESS(0)
 149     "vst1.32     {d5[0]}, [%0], %4             \n"
 150     MEMACCESS(0)
 151     "vst1.32     {d5[1]}, [%0]                 \n"
 152
 153     "add         %0, %3, #4                    \n"
 154     MEMACCESS(0)
 155     "vst1.32     {d0[0]}, [%0], %4             \n"
 156     MEMACCESS(0)
 157     "vst1.32     {d0[1]}, [%0], %4             \n"
 158     MEMACCESS(0)
 159     "vst1.32     {d1[0]}, [%0], %4             \n"
 160     MEMACCESS(0)
 161     "vst1.32     {d1[1]}, [%0]                 \n"
 162
 163     "add         %1, #4                        \n"  // src += 4
 164     "add         %3, %3, %4, lsl #2            \n"  // dst += 4 * dst_stride
 165     "subs        %5,  #4                       \n"  // w   -= 4
 166     "beq         4f                            \n"
 167
 168     // some residual, check to see if it includes a 2x8 block,
 169     // or less
 170     "cmp         %5, #2                        \n"
 171     "blt         3f                            \n"
 172
 173     // 2x8 block
 174     "2:                                        \n"
 175     "mov         %0, %1                        \n"
 176     MEMACCESS(0)
 177     "vld1.16     {d0[0]}, [%0], %2             \n"
 178     MEMACCESS(0)
 179     "vld1.16     {d1[0]}, [%0], %2             \n"
 180     MEMACCESS(0)
 181     "vld1.16     {d0[1]}, [%0], %2             \n"
 182     MEMACCESS(0)
 183     "vld1.16     {d1[1]}, [%0], %2             \n"
 184     MEMACCESS(0)
 185     "vld1.16     {d0[2]}, [%0], %2             \n"
 186     MEMACCESS(0)
 187     "vld1.16     {d1[2]}, [%0], %2             \n"
 188     MEMACCESS(0)
 189     "vld1.16     {d0[3]}, [%0], %2             \n"
 190     MEMACCESS(0)
 191     "vld1.16     {d1[3]}, [%0]                 \n"
 192
 193     "vtrn.8      d0, d1                        \n"
 194
 195     "mov         %0, %3                        \n"
 196
 197     MEMACCESS(0)
 198     "vst1.64     {d0}, [%0], %4                \n"
 199     MEMACCESS(0)
 200     "vst1.64     {d1}, [%0]                    \n"
 201
 202     "add         %1, #2                        \n"  // src += 2
 203     "add         %3, %3, %4, lsl #1            \n"  // dst += 2 * dst_stride
 204     "subs        %5,  #2                       \n"  // w   -= 2
 205     "beq         4f                            \n"
 206
 207     // 1x8 block
 208     "3:                                        \n"
 209     MEMACCESS(1)
 210     "vld1.8      {d0[0]}, [%1], %2             \n"
 211     MEMACCESS(1)
 212     "vld1.8      {d0[1]}, [%1], %2             \n"
 213     MEMACCESS(1)
 214     "vld1.8      {d0[2]}, [%1], %2             \n"
 215     MEMACCESS(1)
 216     "vld1.8      {d0[3]}, [%1], %2             \n"
 217     MEMACCESS(1)
 218     "vld1.8      {d0[4]}, [%1], %2             \n"
 219     MEMACCESS(1)
 220     "vld1.8      {d0[5]}, [%1], %2             \n"
 221     MEMACCESS(1)
 222     "vld1.8      {d0[6]}, [%1], %2             \n"
 223     MEMACCESS(1)
 224     "vld1.8      {d0[7]}, [%1]                 \n"
 225
 226     MEMACCESS(3)
 227     "vst1.64     {d0}, [%3]                    \n"
 228
 229     "4:                                        \n"
 230
 231     : "+r"(src_temp),          // %0
 232       "+r"(src),               // %1
 233       "+r"(src_stride),        // %2
 234       "+r"(dst),               // %3
 235       "+r"(dst_stride),        // %4
 236       "+r"(width)              // %5
 237     : "r"(&kVTbl4x4Transpose)  // %6
 238     : "memory", "cc", "q0", "q1", "q2", "q3"
 239   );
 240 }
 241
 242 static uvec8 kVTbl4x4TransposeDi =
 243   { 0,  8,  1,  9,  2, 10,  3, 11,  4, 12,  5, 13,  6, 14,  7, 15 };
 244
 245 void TransposeUVWx8_NEON(const uint8* src, int src_stride,
 246                          uint8* dst_a, int dst_stride_a,
 247                          uint8* dst_b, int dst_stride_b,
 248                          int width) {
 249   const uint8* src_temp = NULL;
 250   asm volatile (
 251     // loops are on blocks of 8. loop will stop when
 252     // counter gets to or below 0. starting the counter
 253     // at w-8 allow for this
 254     "sub         %7, #8                        \n"
 255
 256     // handle 8x8 blocks. this should be the majority of the plane
 257     ".p2align  2                               \n"
 258     "1:                                        \n"
 259       "mov         %0, %1                      \n"
 260
 261       MEMACCESS(0)
 262       "vld2.8      {d0,  d1},  [%0], %2        \n"
 263       MEMACCESS(0)
 264       "vld2.8      {d2,  d3},  [%0], %2        \n"
 265       MEMACCESS(0)
 266       "vld2.8      {d4,  d5},  [%0], %2        \n"
 267       MEMACCESS(0)
 268       "vld2.8      {d6,  d7},  [%0], %2        \n"
 269       MEMACCESS(0)
 270       "vld2.8      {d16, d17}, [%0], %2        \n"
 271       MEMACCESS(0)
 272       "vld2.8      {d18, d19}, [%0], %2        \n"
 273       MEMACCESS(0)
 274       "vld2.8      {d20, d21}, [%0], %2        \n"
 275       MEMACCESS(0)
 276       "vld2.8      {d22, d23}, [%0]            \n"
 277
 278       "vtrn.8      q1, q0                      \n"
 279       "vtrn.8      q3, q2                      \n"
 280       "vtrn.8      q9, q8                      \n"
 281       "vtrn.8      q11, q10                    \n"
 282
 283       "vtrn.16     q1, q3                      \n"
 284       "vtrn.16     q0, q2                      \n"
 285       "vtrn.16     q9, q11                     \n"
 286       "vtrn.16     q8, q10                     \n"
 287
 288       "vtrn.32     q1, q9                      \n"
 289       "vtrn.32     q0, q8                      \n"
 290       "vtrn.32     q3, q11                     \n"
 291       "vtrn.32     q2, q10                     \n"
 292
 293       "vrev16.8    q0, q0                      \n"
 294       "vrev16.8    q1, q1                      \n"
 295       "vrev16.8    q2, q2                      \n"
 296       "vrev16.8    q3, q3                      \n"
 297       "vrev16.8    q8, q8                      \n"
 298       "vrev16.8    q9, q9                      \n"
 299       "vrev16.8    q10, q10                    \n"
 300       "vrev16.8    q11, q11                    \n"
 301
 302       "mov         %0, %3                      \n"
 303
 304     MEMACCESS(0)
 305       "vst1.8      {d2},  [%0], %4             \n"
 306     MEMACCESS(0)
 307       "vst1.8      {d0},  [%0], %4             \n"
 308     MEMACCESS(0)
 309       "vst1.8      {d6},  [%0], %4             \n"
 310     MEMACCESS(0)
 311       "vst1.8      {d4},  [%0], %4             \n"
 312     MEMACCESS(0)
 313       "vst1.8      {d18}, [%0], %4             \n"
 314     MEMACCESS(0)
 315       "vst1.8      {d16}, [%0], %4             \n"
 316     MEMACCESS(0)
 317       "vst1.8      {d22}, [%0], %4             \n"
 318     MEMACCESS(0)
 319       "vst1.8      {d20}, [%0]                 \n"
 320
 321       "mov         %0, %5                      \n"
 322
 323     MEMACCESS(0)
 324       "vst1.8      {d3},  [%0], %6             \n"
 325     MEMACCESS(0)
 326       "vst1.8      {d1},  [%0], %6             \n"
 327     MEMACCESS(0)
 328       "vst1.8      {d7},  [%0], %6             \n"
 329     MEMACCESS(0)
 330       "vst1.8      {d5},  [%0], %6             \n"
 331     MEMACCESS(0)
 332       "vst1.8      {d19}, [%0], %6             \n"
 333     MEMACCESS(0)
 334       "vst1.8      {d17}, [%0], %6             \n"
 335     MEMACCESS(0)
 336       "vst1.8      {d23}, [%0], %6             \n"
 337     MEMACCESS(0)
 338       "vst1.8      {d21}, [%0]                 \n"
 339
 340       "add         %1, #8*2                    \n"  // src   += 8*2
 341       "add         %3, %3, %4, lsl #3          \n"  // dst_a += 8 * dst_stride_a
 342       "add         %5, %5, %6, lsl #3          \n"  // dst_b += 8 * dst_stride_b
 343       "subs        %7,  #8                     \n"  // w     -= 8
 344       "bge         1b                          \n"
 345
 346     // add 8 back to counter. if the result is 0 there are
 347     // no residuals.
 348     "adds        %7, #8                        \n"
 349     "beq         4f                            \n"
 350
 351     // some residual, so between 1 and 7 lines left to transpose
 352     "cmp         %7, #2                        \n"
 353     "blt         3f                            \n"
 354
 355     "cmp         %7, #4                        \n"
 356     "blt         2f                            \n"
 357
 358     // TODO(frkoenig): Clean this up
 359     // 4x8 block
 360     "mov         %0, %1                        \n"
 361     MEMACCESS(0)
 362     "vld1.64     {d0}, [%0], %2                \n"
 363     MEMACCESS(0)
 364     "vld1.64     {d1}, [%0], %2                \n"
 365     MEMACCESS(0)
 366     "vld1.64     {d2}, [%0], %2                \n"
 367     MEMACCESS(0)
 368     "vld1.64     {d3}, [%0], %2                \n"
 369     MEMACCESS(0)
 370     "vld1.64     {d4}, [%0], %2                \n"
 371     MEMACCESS(0)
 372     "vld1.64     {d5}, [%0], %2                \n"
 373     MEMACCESS(0)
 374     "vld1.64     {d6}, [%0], %2                \n"
 375     MEMACCESS(0)
 376     "vld1.64     {d7}, [%0]                    \n"
 377
 378     MEMACCESS(8)
 379     "vld1.8      {q15}, [%8]                   \n"
 380
 381     "vtrn.8      q0, q1                        \n"
 382     "vtrn.8      q2, q3                        \n"
 383
 384     "vtbl.8      d16, {d0, d1}, d30            \n"
 385     "vtbl.8      d17, {d0, d1}, d31            \n"
 386     "vtbl.8      d18, {d2, d3}, d30            \n"
 387     "vtbl.8      d19, {d2, d3}, d31            \n"
 388     "vtbl.8      d20, {d4, d5}, d30            \n"
 389     "vtbl.8      d21, {d4, d5}, d31            \n"
 390     "vtbl.8      d22, {d6, d7}, d30            \n"
 391     "vtbl.8      d23, {d6, d7}, d31            \n"
 392
 393     "mov         %0, %3                        \n"
 394
 395     MEMACCESS(0)
 396     "vst1.32     {d16[0]},  [%0], %4           \n"
 397     MEMACCESS(0)
 398     "vst1.32     {d16[1]},  [%0], %4           \n"
 399     MEMACCESS(0)
 400     "vst1.32     {d17[0]},  [%0], %4           \n"
 401     MEMACCESS(0)
 402     "vst1.32     {d17[1]},  [%0], %4           \n"
 403
 404     "add         %0, %3, #4                    \n"
 405     MEMACCESS(0)
 406     "vst1.32     {d20[0]}, [%0], %4            \n"
 407     MEMACCESS(0)
 408     "vst1.32     {d20[1]}, [%0], %4            \n"
 409     MEMACCESS(0)
 410     "vst1.32     {d21[0]}, [%0], %4            \n"
 411     MEMACCESS(0)
 412     "vst1.32     {d21[1]}, [%0]                \n"
 413
 414     "mov         %0, %5                        \n"
 415
 416     MEMACCESS(0)
 417     "vst1.32     {d18[0]}, [%0], %6            \n"
 418     MEMACCESS(0)
 419     "vst1.32     {d18[1]}, [%0], %6            \n"
 420     MEMACCESS(0)
 421     "vst1.32     {d19[0]}, [%0], %6            \n"
 422     MEMACCESS(0)
 423     "vst1.32     {d19[1]}, [%0], %6            \n"
 424
 425     "add         %0, %5, #4                    \n"
 426     MEMACCESS(0)
 427     "vst1.32     {d22[0]},  [%0], %6           \n"
 428     MEMACCESS(0)
 429     "vst1.32     {d22[1]},  [%0], %6           \n"
 430     MEMACCESS(0)
 431     "vst1.32     {d23[0]},  [%0], %6           \n"
 432     MEMACCESS(0)
 433     "vst1.32     {d23[1]},  [%0]               \n"
 434
 435     "add         %1, #4*2                      \n"  // src   += 4 * 2
 436     "add         %3, %3, %4, lsl #2            \n"  // dst_a += 4 * dst_stride_a
 437     "add         %5, %5, %6, lsl #2            \n"  // dst_b += 4 * dst_stride_b
 438     "subs        %7,  #4                       \n"  // w     -= 4
 439     "beq         4f                            \n"
 440
 441     // some residual, check to see if it includes a 2x8 block,
 442     // or less
 443     "cmp         %7, #2                        \n"
 444     "blt         3f                            \n"
 445
 446     // 2x8 block
 447     "2:                                        \n"
 448     "mov         %0, %1                        \n"
 449     MEMACCESS(0)
 450     "vld2.16     {d0[0], d2[0]}, [%0], %2      \n"
 451     MEMACCESS(0)
 452     "vld2.16     {d1[0], d3[0]}, [%0], %2      \n"
 453     MEMACCESS(0)
 454     "vld2.16     {d0[1], d2[1]}, [%0], %2      \n"
 455     MEMACCESS(0)
 456     "vld2.16     {d1[1], d3[1]}, [%0], %2      \n"
 457     MEMACCESS(0)
 458     "vld2.16     {d0[2], d2[2]}, [%0], %2      \n"
 459     MEMACCESS(0)
 460     "vld2.16     {d1[2], d3[2]}, [%0], %2      \n"
 461     MEMACCESS(0)
 462     "vld2.16     {d0[3], d2[3]}, [%0], %2      \n"
 463     MEMACCESS(0)
 464     "vld2.16     {d1[3], d3[3]}, [%0]          \n"
 465
 466     "vtrn.8      d0, d1                        \n"
 467     "vtrn.8      d2, d3                        \n"
 468
 469     "mov         %0, %3                        \n"
 470
 471     MEMACCESS(0)
 472     "vst1.64     {d0}, [%0], %4                \n"
 473     MEMACCESS(0)
 474     "vst1.64     {d2}, [%0]                    \n"
 475
 476     "mov         %0, %5                        \n"
 477
 478     MEMACCESS(0)
 479     "vst1.64     {d1}, [%0], %6                \n"
 480     MEMACCESS(0)
 481     "vst1.64     {d3}, [%0]                    \n"
 482
 483     "add         %1, #2*2                      \n"  // src   += 2 * 2
 484     "add         %3, %3, %4, lsl #1            \n"  // dst_a += 2 * dst_stride_a
 485     "add         %5, %5, %6, lsl #1            \n"  // dst_b += 2 * dst_stride_b
 486     "subs        %7,  #2                       \n"  // w     -= 2
 487     "beq         4f                            \n"
 488
 489     // 1x8 block
 490     "3:                                        \n"
 491     MEMACCESS(1)
 492     "vld2.8      {d0[0], d1[0]}, [%1], %2      \n"
 493     MEMACCESS(1)
 494     "vld2.8      {d0[1], d1[1]}, [%1], %2      \n"
 495     MEMACCESS(1)
 496     "vld2.8      {d0[2], d1[2]}, [%1], %2      \n"
 497     MEMACCESS(1)
 498     "vld2.8      {d0[3], d1[3]}, [%1], %2      \n"
 499     MEMACCESS(1)
 500     "vld2.8      {d0[4], d1[4]}, [%1], %2      \n"
 501     MEMACCESS(1)
 502     "vld2.8      {d0[5], d1[5]}, [%1], %2      \n"
 503     MEMACCESS(1)
 504     "vld2.8      {d0[6], d1[6]}, [%1], %2      \n"
 505     MEMACCESS(1)
 506     "vld2.8      {d0[7], d1[7]}, [%1]          \n"
 507
 508     MEMACCESS(3)
 509     "vst1.64     {d0}, [%3]                    \n"
 510     MEMACCESS(5)
 511     "vst1.64     {d1}, [%5]                    \n"
 512
 513     "4:                                        \n"
 514
 515     : "+r"(src_temp),            // %0
 516       "+r"(src),                 // %1
 517       "+r"(src_stride),          // %2
 518       "+r"(dst_a),               // %3
 519       "+r"(dst_stride_a),        // %4
 520       "+r"(dst_b),               // %5
 521       "+r"(dst_stride_b),        // %6
 522       "+r"(width)                // %7
 523     : "r"(&kVTbl4x4TransposeDi)  // %8
 524     : "memory", "cc",
 525       "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"
 526   );
 527 }
 528 #endif
 529
 530 #ifdef __cplusplus
 531 }  // extern "C"
 532 }  // namespace libyuv
 533 #endif