2 * ARM NEON vector operations.
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
7 * This code is licenced under the GNU GPL v2.
16 #define SIGNBIT (uint32_t)0x80000000
17 #define SIGNBIT64 ((uint64_t)1 << 63)
19 #define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q
21 static float_status neon_float_status;
22 #define NFS &neon_float_status
24 /* Helper routines to perform bitwise copies between float and int. */
25 static inline float32 vfp_itos(uint32_t i)
36 static inline uint32_t vfp_stoi(float32 s)
47 #define NEON_TYPE1(name, type) \
52 #ifdef HOST_WORDS_BIGENDIAN
53 #define NEON_TYPE2(name, type) \
59 #define NEON_TYPE4(name, type) \
68 #define NEON_TYPE2(name, type) \
74 #define NEON_TYPE4(name, type) \
84 NEON_TYPE4(s8, int8_t)
85 NEON_TYPE4(u8, uint8_t)
86 NEON_TYPE2(s16, int16_t)
87 NEON_TYPE2(u16, uint16_t)
88 NEON_TYPE1(s32, int32_t)
89 NEON_TYPE1(u32, uint32_t)
94 /* Copy from a uint32_t to a vector structure type. */
95 #define NEON_UNPACK(vtype, dest, val) do { \
104 /* Copy from a vector structure type to a uint32_t. */
105 #define NEON_PACK(vtype, dest, val) do { \
115 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
117 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
118 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
120 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
121 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
122 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
123 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
125 #define NEON_VOP_BODY(vtype, n) \
131 NEON_UNPACK(vtype, vsrc1, arg1); \
132 NEON_UNPACK(vtype, vsrc2, arg2); \
134 NEON_PACK(vtype, res, vdest); \
138 #define NEON_VOP(name, vtype, n) \
139 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
140 NEON_VOP_BODY(vtype, n)
142 #define NEON_VOP_ENV(name, vtype, n) \
143 uint32_t HELPER(glue(neon_,name))(CPUState *env, uint32_t arg1, uint32_t arg2) \
144 NEON_VOP_BODY(vtype, n)
146 /* Pairwise operations. */
147 /* For 32-bit elements each segment only contains a single element, so
148 the elementwise and pairwise operations are the same. */
150 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
151 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
153 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
154 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
155 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
156 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
158 #define NEON_POP(name, vtype, n) \
159 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
165 NEON_UNPACK(vtype, vsrc1, arg1); \
166 NEON_UNPACK(vtype, vsrc2, arg2); \
168 NEON_PACK(vtype, res, vdest); \
172 /* Unary operators. */
173 #define NEON_VOP1(name, vtype, n) \
174 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
178 NEON_UNPACK(vtype, vsrc1, arg); \
180 NEON_PACK(vtype, arg, vdest); \
185 #define NEON_USAT(dest, src1, src2, type) do { \
186 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
187 if (tmp != (type)tmp) { \
193 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
194 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
196 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
197 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
201 uint32_t HELPER(neon_qadd_u32)(CPUState *env, uint32_t a, uint32_t b)
203 uint32_t res = a + b;
211 uint64_t HELPER(neon_qadd_u64)(CPUState *env, uint64_t src1, uint64_t src2)
223 #define NEON_SSAT(dest, src1, src2, type) do { \
224 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
225 if (tmp != (type)tmp) { \
228 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
230 tmp = 1 << (sizeof(type) * 8 - 1); \
235 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
236 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
238 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
239 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
243 uint32_t HELPER(neon_qadd_s32)(CPUState *env, uint32_t a, uint32_t b)
245 uint32_t res = a + b;
246 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
248 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
253 uint64_t HELPER(neon_qadd_s64)(CPUState *env, uint64_t src1, uint64_t src2)
258 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
260 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
265 #define NEON_USAT(dest, src1, src2, type) do { \
266 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
267 if (tmp != (type)tmp) { \
273 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
274 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
276 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
277 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
281 uint32_t HELPER(neon_qsub_u32)(CPUState *env, uint32_t a, uint32_t b)
283 uint32_t res = a - b;
291 uint64_t HELPER(neon_qsub_u64)(CPUState *env, uint64_t src1, uint64_t src2)
304 #define NEON_SSAT(dest, src1, src2, type) do { \
305 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
306 if (tmp != (type)tmp) { \
309 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
311 tmp = 1 << (sizeof(type) * 8 - 1); \
316 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
317 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
319 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
320 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
324 uint32_t HELPER(neon_qsub_s32)(CPUState *env, uint32_t a, uint32_t b)
326 uint32_t res = a - b;
327 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
329 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
334 uint64_t HELPER(neon_qsub_s64)(CPUState *env, uint64_t src1, uint64_t src2)
339 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
341 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
346 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
347 NEON_VOP(hadd_s8, neon_s8, 4)
348 NEON_VOP(hadd_u8, neon_u8, 4)
349 NEON_VOP(hadd_s16, neon_s16, 2)
350 NEON_VOP(hadd_u16, neon_u16, 2)
353 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
357 dest = (src1 >> 1) + (src2 >> 1);
363 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
367 dest = (src1 >> 1) + (src2 >> 1);
373 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
374 NEON_VOP(rhadd_s8, neon_s8, 4)
375 NEON_VOP(rhadd_u8, neon_u8, 4)
376 NEON_VOP(rhadd_s16, neon_s16, 2)
377 NEON_VOP(rhadd_u16, neon_u16, 2)
380 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
384 dest = (src1 >> 1) + (src2 >> 1);
385 if ((src1 | src2) & 1)
390 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
394 dest = (src1 >> 1) + (src2 >> 1);
395 if ((src1 | src2) & 1)
400 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
401 NEON_VOP(hsub_s8, neon_s8, 4)
402 NEON_VOP(hsub_u8, neon_u8, 4)
403 NEON_VOP(hsub_s16, neon_s16, 2)
404 NEON_VOP(hsub_u16, neon_u16, 2)
407 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
411 dest = (src1 >> 1) - (src2 >> 1);
412 if ((~src1) & src2 & 1)
417 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
421 dest = (src1 >> 1) - (src2 >> 1);
422 if ((~src1) & src2 & 1)
427 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
428 NEON_VOP(cgt_s8, neon_s8, 4)
429 NEON_VOP(cgt_u8, neon_u8, 4)
430 NEON_VOP(cgt_s16, neon_s16, 2)
431 NEON_VOP(cgt_u16, neon_u16, 2)
432 NEON_VOP(cgt_s32, neon_s32, 1)
433 NEON_VOP(cgt_u32, neon_u32, 1)
436 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
437 NEON_VOP(cge_s8, neon_s8, 4)
438 NEON_VOP(cge_u8, neon_u8, 4)
439 NEON_VOP(cge_s16, neon_s16, 2)
440 NEON_VOP(cge_u16, neon_u16, 2)
441 NEON_VOP(cge_s32, neon_s32, 1)
442 NEON_VOP(cge_u32, neon_u32, 1)
445 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
446 NEON_VOP(min_s8, neon_s8, 4)
447 NEON_VOP(min_u8, neon_u8, 4)
448 NEON_VOP(min_s16, neon_s16, 2)
449 NEON_VOP(min_u16, neon_u16, 2)
450 NEON_VOP(min_s32, neon_s32, 1)
451 NEON_VOP(min_u32, neon_u32, 1)
452 NEON_POP(pmin_s8, neon_s8, 4)
453 NEON_POP(pmin_u8, neon_u8, 4)
454 NEON_POP(pmin_s16, neon_s16, 2)
455 NEON_POP(pmin_u16, neon_u16, 2)
458 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
459 NEON_VOP(max_s8, neon_s8, 4)
460 NEON_VOP(max_u8, neon_u8, 4)
461 NEON_VOP(max_s16, neon_s16, 2)
462 NEON_VOP(max_u16, neon_u16, 2)
463 NEON_VOP(max_s32, neon_s32, 1)
464 NEON_VOP(max_u32, neon_u32, 1)
465 NEON_POP(pmax_s8, neon_s8, 4)
466 NEON_POP(pmax_u8, neon_u8, 4)
467 NEON_POP(pmax_s16, neon_s16, 2)
468 NEON_POP(pmax_u16, neon_u16, 2)
471 #define NEON_FN(dest, src1, src2) \
472 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
473 NEON_VOP(abd_s8, neon_s8, 4)
474 NEON_VOP(abd_u8, neon_u8, 4)
475 NEON_VOP(abd_s16, neon_s16, 2)
476 NEON_VOP(abd_u16, neon_u16, 2)
477 NEON_VOP(abd_s32, neon_s32, 1)
478 NEON_VOP(abd_u32, neon_u32, 1)
481 #define NEON_FN(dest, src1, src2) do { \
483 tmp = (int8_t)src2; \
484 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
485 tmp <= -(ssize_t)sizeof(src1) * 8) { \
487 } else if (tmp < 0) { \
488 dest = src1 >> -tmp; \
490 dest = src1 << tmp; \
492 NEON_VOP(shl_u8, neon_u8, 4)
493 NEON_VOP(shl_u16, neon_u16, 2)
494 NEON_VOP(shl_u32, neon_u32, 1)
497 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
499 int8_t shift = (int8_t)shiftop;
500 if (shift >= 64 || shift <= -64) {
502 } else if (shift < 0) {
510 #define NEON_FN(dest, src1, src2) do { \
512 tmp = (int8_t)src2; \
513 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
515 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
516 dest = src1 >> (sizeof(src1) * 8 - 1); \
517 } else if (tmp < 0) { \
518 dest = src1 >> -tmp; \
520 dest = src1 << tmp; \
522 NEON_VOP(shl_s8, neon_s8, 4)
523 NEON_VOP(shl_s16, neon_s16, 2)
524 NEON_VOP(shl_s32, neon_s32, 1)
527 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
529 int8_t shift = (int8_t)shiftop;
533 } else if (shift <= -64) {
535 } else if (shift < 0) {
543 #define NEON_FN(dest, src1, src2) do { \
545 tmp = (int8_t)src2; \
546 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
548 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
549 dest = src1 >> (sizeof(src1) * 8 - 1); \
550 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
551 dest = src1 >> (tmp - 1); \
554 } else if (tmp < 0) { \
555 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
557 dest = src1 << tmp; \
559 NEON_VOP(rshl_s8, neon_s8, 4)
560 NEON_VOP(rshl_s16, neon_s16, 2)
561 NEON_VOP(rshl_s32, neon_s32, 1)
564 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
566 int8_t shift = (int8_t)shiftop;
570 } else if (shift < -64) {
572 } else if (shift == -63) {
576 } else if (shift < 0) {
577 val = (val + ((int64_t)1 << (-1 - shift))) >> -shift;
584 #define NEON_FN(dest, src1, src2) do { \
586 tmp = (int8_t)src2; \
587 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
588 tmp < -(ssize_t)sizeof(src1) * 8) { \
590 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
591 dest = src1 >> (tmp - 1); \
592 } else if (tmp < 0) { \
593 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
595 dest = src1 << tmp; \
597 NEON_VOP(rshl_u8, neon_u8, 4)
598 NEON_VOP(rshl_u16, neon_u16, 2)
599 NEON_VOP(rshl_u32, neon_u32, 1)
602 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
604 int8_t shift = (uint8_t)shiftop;
605 if (shift >= 64 || shift < 64) {
607 } else if (shift == -64) {
608 /* Rounding a 1-bit result just preserves that bit. */
611 val = (val + ((uint64_t)1 << (-1 - shift))) >> -shift;
619 #define NEON_FN(dest, src1, src2) do { \
621 tmp = (int8_t)src2; \
622 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
629 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
631 } else if (tmp < 0) { \
632 dest = src1 >> -tmp; \
634 dest = src1 << tmp; \
635 if ((dest >> tmp) != src1) { \
640 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
641 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
642 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
645 uint64_t HELPER(neon_qshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
647 int8_t shift = (int8_t)shiftop;
653 } else if (shift <= -64) {
655 } else if (shift < 0) {
660 if ((val >> shift) != tmp) {
668 #define NEON_FN(dest, src1, src2) do { \
670 tmp = (int8_t)src2; \
671 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
674 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
681 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
683 } else if (tmp < 0) { \
684 dest = src1 >> -tmp; \
686 dest = src1 << tmp; \
687 if ((dest >> tmp) != src1) { \
689 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
695 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
696 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
697 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
700 uint64_t HELPER(neon_qshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
702 int8_t shift = (uint8_t)shiftop;
707 val = (val >> 63) ^ ~SIGNBIT64;
709 } else if (shift <= -64) {
711 } else if (shift < 0) {
716 if ((val >> shift) != tmp) {
718 val = (tmp >> 63) ^ ~SIGNBIT64;
724 #define NEON_FN(dest, src1, src2) do { \
725 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
730 tmp = (int8_t)src2; \
731 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
738 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
740 } else if (tmp < 0) { \
741 dest = src1 >> -tmp; \
743 dest = src1 << tmp; \
744 if ((dest >> tmp) != src1) { \
750 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
751 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
754 uint32_t HELPER(neon_qshlu_s32)(CPUState *env, uint32_t valop, uint32_t shiftop)
756 if ((int32_t)valop < 0) {
760 return helper_neon_qshl_u32(env, valop, shiftop);
763 uint64_t HELPER(neon_qshlu_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
765 if ((int64_t)valop < 0) {
769 return helper_neon_qshl_u64(env, valop, shiftop);
772 /* FIXME: This is wrong. */
773 #define NEON_FN(dest, src1, src2) do { \
775 tmp = (int8_t)src2; \
777 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
779 dest = src1 << tmp; \
780 if ((dest >> tmp) != src1) { \
785 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
786 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
787 NEON_VOP_ENV(qrshl_u32, neon_u32, 1)
790 uint64_t HELPER(neon_qrshl_u64)(CPUState *env, uint64_t val, uint64_t shiftop)
792 int8_t shift = (int8_t)shiftop;
794 val = (val + (1 << (-1 - shift))) >> -shift;
798 if ((val >> shift) != tmp) {
806 #define NEON_FN(dest, src1, src2) do { \
808 tmp = (int8_t)src2; \
810 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
812 dest = src1 << tmp; \
813 if ((dest >> tmp) != src1) { \
818 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
819 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
820 NEON_VOP_ENV(qrshl_s32, neon_s32, 1)
823 uint64_t HELPER(neon_qrshl_s64)(CPUState *env, uint64_t valop, uint64_t shiftop)
825 int8_t shift = (uint8_t)shiftop;
829 val = (val + (1 << (-1 - shift))) >> -shift;
833 if ((val >> shift) != tmp) {
841 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
844 mask = (a ^ b) & 0x80808080u;
847 return (a + b) ^ mask;
850 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
853 mask = (a ^ b) & 0x80008000u;
856 return (a + b) ^ mask;
859 #define NEON_FN(dest, src1, src2) dest = src1 + src2
860 NEON_POP(padd_u8, neon_u8, 4)
861 NEON_POP(padd_u16, neon_u16, 2)
864 #define NEON_FN(dest, src1, src2) dest = src1 - src2
865 NEON_VOP(sub_u8, neon_u8, 4)
866 NEON_VOP(sub_u16, neon_u16, 2)
869 #define NEON_FN(dest, src1, src2) dest = src1 * src2
870 NEON_VOP(mul_u8, neon_u8, 4)
871 NEON_VOP(mul_u16, neon_u16, 2)
874 /* Polynomial multiplication is like integer multiplication except the
875 partial products are XORed, not added. */
876 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
888 mask |= (0xff << 16);
890 mask |= (0xff << 24);
891 result ^= op2 & mask;
892 op1 = (op1 >> 1) & 0x7f7f7f7f;
893 op2 = (op2 << 1) & 0xfefefefe;
898 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
899 NEON_VOP(tst_u8, neon_u8, 4)
900 NEON_VOP(tst_u16, neon_u16, 2)
901 NEON_VOP(tst_u32, neon_u32, 1)
904 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
905 NEON_VOP(ceq_u8, neon_u8, 4)
906 NEON_VOP(ceq_u16, neon_u16, 2)
907 NEON_VOP(ceq_u32, neon_u32, 1)
910 #define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src
911 NEON_VOP1(abs_s8, neon_s8, 4)
912 NEON_VOP1(abs_s16, neon_s16, 2)
915 /* Count Leading Sign/Zero Bits. */
916 static inline int do_clz8(uint8_t x)
924 static inline int do_clz16(uint16_t x)
932 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
933 NEON_VOP1(clz_u8, neon_u8, 4)
936 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
937 NEON_VOP1(clz_u16, neon_u16, 2)
940 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
941 NEON_VOP1(cls_s8, neon_s8, 4)
944 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
945 NEON_VOP1(cls_s16, neon_s16, 2)
948 uint32_t HELPER(neon_cls_s32)(uint32_t x)
953 for (count = 32; x; count--)
959 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
961 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
962 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
963 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
967 #define NEON_QDMULH16(dest, src1, src2, round) do { \
968 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
969 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
971 tmp = (tmp >> 31) ^ ~SIGNBIT; \
978 if ((int32_t)tmp < old) { \
985 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
986 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
988 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
989 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
993 #define NEON_QDMULH32(dest, src1, src2, round) do { \
994 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
995 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
997 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1002 int64_t old = tmp; \
1003 tmp += (int64_t)1 << 31; \
1004 if ((int64_t)tmp < old) { \
1006 tmp = SIGNBIT64 - 1; \
1011 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1012 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1014 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1015 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1017 #undef NEON_QDMULH32
1019 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1021 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1022 | ((x >> 24) & 0xff000000u);
1025 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1027 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1030 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1032 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1033 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1036 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1038 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1041 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1043 x &= 0xff80ff80ff80ff80ull;
1044 x += 0x0080008000800080ull;
1045 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1046 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1049 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1051 x &= 0xffff8000ffff8000ull;
1052 x += 0x0000800000008000ull;
1053 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1056 uint32_t HELPER(neon_narrow_sat_u8)(CPUState *env, uint64_t x)
1069 res |= (uint32_t)d << (n / 2);
1079 uint32_t HELPER(neon_narrow_sat_s8)(CPUState *env, uint64_t x)
1086 if (s != (int8_t)s) { \
1087 d = (s >> 15) ^ 0x7f; \
1092 res |= (uint32_t)d << (n / 2);
1102 uint32_t HELPER(neon_narrow_sat_u16)(CPUState *env, uint64_t x)
1112 if (high > 0xffff) {
1116 return low | (high << 16);
1119 uint32_t HELPER(neon_narrow_sat_s16)(CPUState *env, uint64_t x)
1124 if (low != (int16_t)low) {
1125 low = (low >> 31) ^ 0x7fff;
1129 if (high != (int16_t)high) {
1130 high = (high >> 31) ^ 0x7fff;
1133 return (uint16_t)low | (high << 16);
1136 uint32_t HELPER(neon_narrow_sat_u32)(CPUState *env, uint64_t x)
1138 if (x > 0xffffffffu) {
1145 uint32_t HELPER(neon_narrow_sat_s32)(CPUState *env, uint64_t x)
1147 if ((int64_t)x != (int32_t)x) {
1149 return (x >> 63) ^ 0x7fffffff;
1154 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1159 tmp = (uint8_t)(x >> 8);
1161 tmp = (uint8_t)(x >> 16);
1163 tmp = (uint8_t)(x >> 24);
1168 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1172 ret = (uint16_t)(int8_t)x;
1173 tmp = (uint16_t)(int8_t)(x >> 8);
1175 tmp = (uint16_t)(int8_t)(x >> 16);
1177 tmp = (uint16_t)(int8_t)(x >> 24);
1182 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1184 uint64_t high = (uint16_t)(x >> 16);
1185 return ((uint16_t)x) | (high << 32);
1188 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1190 uint64_t high = (int16_t)(x >> 16);
1191 return ((uint32_t)(int16_t)x) | (high << 32);
1194 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1197 mask = (a ^ b) & 0x8000800080008000ull;
1198 a &= ~0x8000800080008000ull;
1199 b &= ~0x8000800080008000ull;
1200 return (a + b) ^ mask;
1203 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1206 mask = (a ^ b) & 0x8000000080000000ull;
1207 a &= ~0x8000000080000000ull;
1208 b &= ~0x8000000080000000ull;
1209 return (a + b) ^ mask;
1212 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1217 tmp = a & 0x0000ffff0000ffffull;
1218 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1219 tmp2 = b & 0xffff0000ffff0000ull;
1220 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1221 return ( tmp & 0xffff)
1222 | ((tmp >> 16) & 0xffff0000ull)
1223 | ((tmp2 << 16) & 0xffff00000000ull)
1224 | ( tmp2 & 0xffff000000000000ull);
1227 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1229 uint32_t low = a + (a >> 32);
1230 uint32_t high = b + (b >> 32);
1231 return low + ((uint64_t)high << 32);
1234 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1237 mask = (a ^ ~b) & 0x8000800080008000ull;
1238 a |= 0x8000800080008000ull;
1239 b &= ~0x8000800080008000ull;
1240 return (a - b) ^ mask;
1243 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1246 mask = (a ^ ~b) & 0x8000000080000000ull;
1247 a |= 0x8000000080000000ull;
1248 b &= ~0x8000000080000000ull;
1249 return (a - b) ^ mask;
1252 uint64_t HELPER(neon_addl_saturate_s32)(CPUState *env, uint64_t a, uint64_t b)
1260 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1262 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1267 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1269 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1271 return low | ((uint64_t)high << 32);
1274 uint64_t HELPER(neon_addl_saturate_s64)(CPUState *env, uint64_t a, uint64_t b)
1279 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1281 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1286 #define DO_ABD(dest, x, y, type) do { \
1289 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1292 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1296 DO_ABD(result, a, b, uint8_t);
1297 DO_ABD(tmp, a >> 8, b >> 8, uint8_t);
1298 result |= tmp << 16;
1299 DO_ABD(tmp, a >> 16, b >> 16, uint8_t);
1300 result |= tmp << 32;
1301 DO_ABD(tmp, a >> 24, b >> 24, uint8_t);
1302 result |= tmp << 48;
1306 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1310 DO_ABD(result, a, b, int8_t);
1311 DO_ABD(tmp, a >> 8, b >> 8, int8_t);
1312 result |= tmp << 16;
1313 DO_ABD(tmp, a >> 16, b >> 16, int8_t);
1314 result |= tmp << 32;
1315 DO_ABD(tmp, a >> 24, b >> 24, int8_t);
1316 result |= tmp << 48;
1320 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1324 DO_ABD(result, a, b, uint16_t);
1325 DO_ABD(tmp, a >> 16, b >> 16, uint16_t);
1326 return result | (tmp << 32);
1329 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1333 DO_ABD(result, a, b, int16_t);
1334 DO_ABD(tmp, a >> 16, b >> 16, int16_t);
1335 return result | (tmp << 32);
1338 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1341 DO_ABD(result, a, b, uint32_t);
1345 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1348 DO_ABD(result, a, b, int32_t);
1353 /* Widening multiply. Named type is the source type. */
1354 #define DO_MULL(dest, x, y, type1, type2) do { \
1357 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1360 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1365 DO_MULL(result, a, b, uint8_t, uint16_t);
1366 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1367 result |= tmp << 16;
1368 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1369 result |= tmp << 32;
1370 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1371 result |= tmp << 48;
1375 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1380 DO_MULL(result, a, b, int8_t, uint16_t);
1381 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1382 result |= tmp << 16;
1383 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1384 result |= tmp << 32;
1385 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1386 result |= tmp << 48;
1390 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1395 DO_MULL(result, a, b, uint16_t, uint32_t);
1396 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1397 return result | (tmp << 32);
1400 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1405 DO_MULL(result, a, b, int16_t, uint32_t);
1406 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1407 return result | (tmp << 32);
1410 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1414 result = (uint16_t)-x;
1416 result |= (uint64_t)tmp << 16;
1418 result |= (uint64_t)tmp << 32;
1420 result |= (uint64_t)tmp << 48;
1425 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1428 uint32_t high = -(x >> 32);
1429 return low | ((uint64_t)high << 32);
1432 /* FIXME: There should be a native op for this. */
1433 uint64_t HELPER(neon_negl_u64)(uint64_t x)
1438 /* Saturnating sign manuipulation. */
1439 /* ??? Make these use NEON_VOP1 */
1440 #define DO_QABS8(x) do { \
1441 if (x == (int8_t)0x80) { \
1444 } else if (x < 0) { \
1447 uint32_t HELPER(neon_qabs_s8)(CPUState *env, uint32_t x)
1450 NEON_UNPACK(neon_s8, vec, x);
1455 NEON_PACK(neon_s8, x, vec);
1460 #define DO_QNEG8(x) do { \
1461 if (x == (int8_t)0x80) { \
1467 uint32_t HELPER(neon_qneg_s8)(CPUState *env, uint32_t x)
1470 NEON_UNPACK(neon_s8, vec, x);
1475 NEON_PACK(neon_s8, x, vec);
1480 #define DO_QABS16(x) do { \
1481 if (x == (int16_t)0x8000) { \
1484 } else if (x < 0) { \
1487 uint32_t HELPER(neon_qabs_s16)(CPUState *env, uint32_t x)
1490 NEON_UNPACK(neon_s16, vec, x);
1493 NEON_PACK(neon_s16, x, vec);
1498 #define DO_QNEG16(x) do { \
1499 if (x == (int16_t)0x8000) { \
1505 uint32_t HELPER(neon_qneg_s16)(CPUState *env, uint32_t x)
1508 NEON_UNPACK(neon_s16, vec, x);
1511 NEON_PACK(neon_s16, x, vec);
1516 uint32_t HELPER(neon_qabs_s32)(CPUState *env, uint32_t x)
1521 } else if ((int32_t)x < 0) {
1527 uint32_t HELPER(neon_qneg_s32)(CPUState *env, uint32_t x)
1538 /* NEON Float helpers. */
1539 uint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b)
1541 float32 f0 = vfp_itos(a);
1542 float32 f1 = vfp_itos(b);
1543 return (float32_compare_quiet(f0, f1, NFS) == -1) ? a : b;
1546 uint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b)
1548 float32 f0 = vfp_itos(a);
1549 float32 f1 = vfp_itos(b);
1550 return (float32_compare_quiet(f0, f1, NFS) == 1) ? a : b;
1553 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b)
1555 float32 f0 = vfp_itos(a);
1556 float32 f1 = vfp_itos(b);
1557 return vfp_stoi((float32_compare_quiet(f0, f1, NFS) == 1)
1558 ? float32_sub(f0, f1, NFS)
1559 : float32_sub(f1, f0, NFS));
1562 uint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b)
1564 return vfp_stoi(float32_add(vfp_itos(a), vfp_itos(b), NFS));
1567 uint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b)
1569 return vfp_stoi(float32_sub(vfp_itos(a), vfp_itos(b), NFS));
1572 uint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b)
1574 return vfp_stoi(float32_mul(vfp_itos(a), vfp_itos(b), NFS));
1577 /* Floating point comparisons produce an integer result. */
1578 #define NEON_VOP_FCMP(name, cmp) \
1579 uint32_t HELPER(neon_##name)(uint32_t a, uint32_t b) \
1581 if (float32_compare_quiet(vfp_itos(a), vfp_itos(b), NFS) cmp 0) \
1587 NEON_VOP_FCMP(ceq_f32, ==)
1588 NEON_VOP_FCMP(cge_f32, >=)
1589 NEON_VOP_FCMP(cgt_f32, >)
1591 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b)
1593 float32 f0 = float32_abs(vfp_itos(a));
1594 float32 f1 = float32_abs(vfp_itos(b));
1595 return (float32_compare_quiet(f0, f1,NFS) >= 0) ? ~0 : 0;
1598 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b)
1600 float32 f0 = float32_abs(vfp_itos(a));
1601 float32 f1 = float32_abs(vfp_itos(b));
1602 return (float32_compare_quiet(f0, f1, NFS) > 0) ? ~0 : 0;