#define dst x3
#define srcend x4
#define dstend x5
-#define tmp2 x6
-#define tmp3 x7
+#define tmp2 x6
+#define tmp3 x7
#define tmp3w w7
#define A_l x6
#define A_lw w6
#define G_h dst
#define tmp1 x14
-#define A_q q0
-#define B_q q1
-#define C_q q2
-#define D_q q3
-#define E_q q4
-#define F_q q5
-#define G_q q6
+#define A_q q0
+#define B_q q1
+#define C_q q2
+#define D_q q3
+#define E_q q4
+#define F_q q5
+#define G_q q6
#define H_q q7
#define I_q q16
#define J_q q17
-#define A_v v0
-#define B_v v1
-#define C_v v2
-#define D_v v3
-#define E_v v4
-#define F_v v5
-#define G_v v6
-#define H_v v7
-#define I_v v16
+#define A_v v0
+#define B_v v1
+#define C_v v2
+#define D_v v3
+#define E_v v4
+#define F_v v5
+#define G_v v6
+#define H_v v7
+#define I_v v16
#define J_v v17
#ifndef MEMMOVE
#if IS_IN (libc)
#undef MEMCPY
-#undef MEMMOVE
#define MEMCPY __memcpy_thunderx2
+#undef MEMMOVE
#define MEMMOVE __memmove_thunderx2
-/* Moves are split into 3 main cases: small copies of up to 16 bytes,
- medium copies of 17..96 bytes which are fully unrolled. Large copies
- of more than 96 bytes align the destination and use an unrolled loop
- processing 64 bytes per iteration.
- Overlapping large forward memmoves use a loop that copies backwards.
+/* Overlapping large forward memmoves use a loop that copies backwards.
+ Otherwise memcpy is used. Small moves branch to memcopy16 directly.
+ The longer memcpy cases fall through to the memcpy head.
*/
ENTRY_ALIGN (MEMMOVE, 6)
DELOUSE (1)
DELOUSE (2)
+ add srcend, src, count
+ cmp count, 16
+ b.ls L(memcopy16)
sub tmp1, dstin, src
cmp count, 96
ccmp tmp1, count, 2, hi
b.lo L(move_long)
- prfm PLDL1KEEP, [src]
- add srcend, src, count
- add dstend, dstin, count
- cmp count, 16
- b.ls L(copy16)
- cmp count, 96
- b.hi L(copy_long)
-
- /* Medium copies: 17..96 bytes. */
- sub tmp1, count, 1
- ldp A_l, A_h, [src]
- tbnz tmp1, 6, L(copy96)
- ldp D_l, D_h, [srcend, -16]
- tbz tmp1, 5, 1f
- ldp B_l, B_h, [src, 16]
- ldp C_l, C_h, [srcend, -32]
- stp B_l, B_h, [dstin, 16]
- stp C_l, C_h, [dstend, -32]
-1:
- stp A_l, A_h, [dstin]
- stp D_l, D_h, [dstend, -16]
- ret
-
- .p2align 4
- /* Small copies: 0..16 bytes. */
-L(copy16):
- cmp count, 8
- b.lo 1f
- ldr A_l, [src]
- ldr A_h, [srcend, -8]
- str A_l, [dstin]
- str A_h, [dstend, -8]
- ret
- .p2align 4
-1:
- tbz count, 2, 1f
- ldr A_lw, [src]
- ldr A_hw, [srcend, -4]
- str A_lw, [dstin]
- str A_hw, [dstend, -4]
- ret
-
- /* Copy 0..3 bytes. Use a branchless sequence that copies the same
- byte 3 times if count==1, or the 2nd byte twice if count==2. */
-1:
- cbz count, 2f
- lsr tmp1, count, 1
- ldrb A_lw, [src]
- ldrb A_hw, [srcend, -1]
- ldrb B_lw, [src, tmp1]
- strb A_lw, [dstin]
- strb B_lw, [dstin, tmp1]
- strb A_hw, [dstend, -1]
-2: ret
-
- .p2align 4
- /* Copy 64..96 bytes. Copy 64 bytes from the start and
- 32 bytes from the end. */
-L(copy96):
- ldp B_l, B_h, [src, 16]
- ldp C_l, C_h, [src, 32]
- ldp D_l, D_h, [src, 48]
- ldp E_l, E_h, [srcend, -32]
- ldp F_l, F_h, [srcend, -16]
- stp A_l, A_h, [dstin]
- stp B_l, B_h, [dstin, 16]
- stp C_l, C_h, [dstin, 32]
- stp D_l, D_h, [dstin, 48]
- stp E_l, E_h, [dstend, -32]
- stp F_l, F_h, [dstend, -16]
- ret
-
- /* Align DST to 16 byte alignment so that we don't cross cache line
- boundaries on both loads and stores. There are at least 96 bytes
- to copy, so copy 16 bytes unaligned and then align. The loop
- copies 64 bytes per iteration and prefetches one iteration ahead. */
-
- .p2align 4
-L(copy_long):
- and tmp1, dstin, 15
- bic dst, dstin, 15
- ldp D_l, D_h, [src]
- sub src, src, tmp1
- add count, count, tmp1 /* Count is now 16 too large. */
- ldp A_l, A_h, [src, 16]
- stp D_l, D_h, [dstin]
- ldp B_l, B_h, [src, 32]
- ldp C_l, C_h, [src, 48]
- ldp D_l, D_h, [src, 64]!
- subs count, count, 128 + 16 /* Test and readjust count. */
- b.ls L(last64)
-L(loop64):
- stp A_l, A_h, [dst, 16]
- ldp A_l, A_h, [src, 16]
- stp B_l, B_h, [dst, 32]
- ldp B_l, B_h, [src, 32]
- stp C_l, C_h, [dst, 48]
- ldp C_l, C_h, [src, 48]
- stp D_l, D_h, [dst, 64]!
- ldp D_l, D_h, [src, 64]!
- subs count, count, 64
- b.hi L(loop64)
-
- /* Write the last full set of 64 bytes. The remainder is at most 64
- bytes, so it is safe to always copy 64 bytes from the end even if
- there is just 1 byte left. */
-L(last64):
- ldp E_l, E_h, [srcend, -64]
- stp A_l, A_h, [dst, 16]
- ldp A_l, A_h, [srcend, -48]
- stp B_l, B_h, [dst, 32]
- ldp B_l, B_h, [srcend, -32]
- stp C_l, C_h, [dst, 48]
- ldp C_l, C_h, [srcend, -16]
- stp D_l, D_h, [dst, 64]
- stp E_l, E_h, [dstend, -64]
- stp A_l, A_h, [dstend, -48]
- stp B_l, B_h, [dstend, -32]
- stp C_l, C_h, [dstend, -16]
- ret
-
- .p2align 4
-L(move_long):
- cbz tmp1, 3f
-
- add srcend, src, count
- add dstend, dstin, count
-
- /* Align dstend to 16 byte alignment so that we don't cross cache line
- boundaries on both loads and stores. There are at least 96 bytes
- to copy, so copy 16 bytes unaligned and then align. The loop
- copies 64 bytes per iteration and prefetches one iteration ahead. */
-
- and tmp1, dstend, 15
- ldp D_l, D_h, [srcend, -16]
- sub srcend, srcend, tmp1
- sub count, count, tmp1
- ldp A_l, A_h, [srcend, -16]
- stp D_l, D_h, [dstend, -16]
- ldp B_l, B_h, [srcend, -32]
- ldp C_l, C_h, [srcend, -48]
- ldp D_l, D_h, [srcend, -64]!
- sub dstend, dstend, tmp1
- subs count, count, 128
- b.ls 2f
-
- nop
-1:
- stp A_l, A_h, [dstend, -16]
- ldp A_l, A_h, [srcend, -16]
- stp B_l, B_h, [dstend, -32]
- ldp B_l, B_h, [srcend, -32]
- stp C_l, C_h, [dstend, -48]
- ldp C_l, C_h, [srcend, -48]
- stp D_l, D_h, [dstend, -64]!
- ldp D_l, D_h, [srcend, -64]!
- subs count, count, 64
- b.hi 1b
-
- /* Write the last full set of 64 bytes. The remainder is at most 64
- bytes, so it is safe to always copy 64 bytes from the start even if
- there is just 1 byte left. */
-2:
- ldp G_l, G_h, [src, 48]
- stp A_l, A_h, [dstend, -16]
- ldp A_l, A_h, [src, 32]
- stp B_l, B_h, [dstend, -32]
- ldp B_l, B_h, [src, 16]
- stp C_l, C_h, [dstend, -48]
- ldp C_l, C_h, [src]
- stp D_l, D_h, [dstend, -64]
- stp G_l, G_h, [dstin, 48]
- stp A_l, A_h, [dstin, 32]
- stp B_l, B_h, [dstin, 16]
- stp C_l, C_h, [dstin]
-3: ret
-
END (MEMMOVE)
libc_hidden_builtin_def (MEMMOVE)
medium copies of 17..96 bytes which are fully unrolled. Large copies
of more than 96 bytes align the destination and use load-and-merge
approach in the case src and dst addresses are unaligned not evenly,
- so that, loads and stores are always aligned.
- Large copies use an unrolled loop processing 64 bytes per iteration.
- The current optimized memcpy implementation is not compatible with
- memmove and is separated from it completely.
-
- memcpy implementation below is not compatible with memmove
- because of pipelined loads/stores, which are faster, but they
- can't be used in the case of overlapping memmove arrays */
+ so that, actual loads and stores are always aligned.
+ Large copies use the loops processing 64 bytes per iteration for
+ unaligned case and 128 bytes per iteration for aligned ones.
+*/
#define MEMCPY_PREFETCH_LDR 640
+ .p2align 4
ENTRY (MEMCPY)
+
DELOUSE (0)
DELOUSE (1)
DELOUSE (2)
- add srcend, src, count
- cmp count, 16
- b.ls L(memcopy16)
- ldr A_q, [src], #16
- add dstend, dstin, count
- and tmp1, src, 15
- cmp count, 96
- b.hi L(memcopy_long)
+ add srcend, src, count
+ cmp count, 16
+ b.ls L(memcopy16)
+ ldr A_q, [src], #16
+ add dstend, dstin, count
+ and tmp1, src, 15
+ cmp count, 96
+ b.hi L(memcopy_long)
/* Medium copies: 17..96 bytes. */
- ldr E_q, [srcend, -16]
- cmp count, 64
- b.gt L(memcpy_copy96)
- cmp count, 48
- b.le L(bytes_17_to_48)
+ ldr E_q, [srcend, -16]
+ cmp count, 64
+ b.gt L(memcpy_copy96)
+ cmp count, 48
+ b.le L(bytes_17_to_48)
/* 49..64 bytes */
- ldp B_q, C_q, [src]
- str E_q, [dstend, -16]
- stp A_q, B_q, [dstin]
- str C_q, [dstin, 32]
+ ldp B_q, C_q, [src]
+ str E_q, [dstend, -16]
+ stp A_q, B_q, [dstin]
+ str C_q, [dstin, 32]
ret
L(bytes_17_to_48):
/* 17..48 bytes*/
- cmp count, 32
- b.gt L(bytes_32_to_48)
+ cmp count, 32
+ b.gt L(bytes_32_to_48)
/* 17..32 bytes*/
- str A_q, [dstin]
- str E_q, [dstend, -16]
+ str A_q, [dstin]
+ str E_q, [dstend, -16]
ret
L(bytes_32_to_48):
/* 32..48 */
- ldr B_q, [src]
- str A_q, [dstin]
- str E_q, [dstend, -16]
- str B_q, [dstin, 16]
+ ldr B_q, [src]
+ str A_q, [dstin]
+ str E_q, [dstend, -16]
+ str B_q, [dstin, 16]
ret
.p2align 4
/* Small copies: 0..16 bytes. */
L(memcopy16):
- cmp count, 8
- b.lo L(bytes_0_to_8)
- ldr A_l, [src]
- ldr A_h, [srcend, -8]
- add dstend, dstin, count
- str A_l, [dstin]
- str A_h, [dstend, -8]
+ cmp count, 8
+ b.lo L(bytes_0_to_8)
+ ldr A_l, [src]
+ ldr A_h, [srcend, -8]
+ add dstend, dstin, count
+ str A_l, [dstin]
+ str A_h, [dstend, -8]
ret
.p2align 4
L(bytes_0_to_8):
- tbz count, 2, L(bytes_0_to_3)
- ldr A_lw, [src]
- ldr A_hw, [srcend, -4]
- add dstend, dstin, count
- str A_lw, [dstin]
- str A_hw, [dstend, -4]
+ tbz count, 2, L(bytes_0_to_3)
+ ldr A_lw, [src]
+ ldr A_hw, [srcend, -4]
+ add dstend, dstin, count
+ str A_lw, [dstin]
+ str A_hw, [dstend, -4]
ret
/* Copy 0..3 bytes. Use a branchless sequence that copies the same
byte 3 times if count==1, or the 2nd byte twice if count==2. */
L(bytes_0_to_3):
- cbz count, L(end)
- lsr tmp1, count, 1
- ldrb A_lw, [src]
- ldrb A_hw, [srcend, -1]
- add dstend, dstin, count
- ldrb B_lw, [src, tmp1]
- strb A_lw, [dstin]
- strb B_lw, [dstin, tmp1]
- strb A_hw, [dstend, -1]
-L(end):
+ cbz count, 1f
+ lsr tmp1, count, 1
+ ldrb A_lw, [src]
+ ldrb A_hw, [srcend, -1]
+ add dstend, dstin, count
+ ldrb B_lw, [src, tmp1]
+ strb B_lw, [dstin, tmp1]
+ strb A_hw, [dstend, -1]
+ strb A_lw, [dstin]
+1:
ret
.p2align 4
L(memcpy_copy96):
/* Copying 65..96 bytes. A_q (first 16 bytes) and
- E_q(last 16 bytes) are already loaded.
-
- The size is large enough to benefit from aligned
- loads */
- bic src, src, 15
- ldp B_q, C_q, [src]
- str A_q, [dstin]
+ E_q(last 16 bytes) are already loaded. The size
+ is large enough to benefit from aligned loads */
+ bic src, src, 15
+ ldp B_q, C_q, [src]
/* Loaded 64 bytes, second 16-bytes chunk can be
overlapping with the first chunk by tmp1 bytes.
Stored 16 bytes. */
- sub dst, dstin, tmp1
- add count, count, tmp1
+ sub dst, dstin, tmp1
+ add count, count, tmp1
/* The range of count being [65..96] becomes [65..111]
after tmp [0..15] gets added to it,
count now is <bytes-left-to-load>+48 */
- cmp count, 80
- b.gt L(copy96_medium)
- ldr D_q, [src, 32]
- stp B_q, C_q, [dst, 16]
- str E_q, [dstend, -16]
- str D_q, [dst, 48]
+ cmp count, 80
+ b.gt L(copy96_medium)
+ ldr D_q, [src, 32]
+ stp B_q, C_q, [dst, 16]
+ str D_q, [dst, 48]
+ str A_q, [dstin]
+ str E_q, [dstend, -16]
ret
.p2align 4
L(copy96_medium):
- ldp D_q, A_q, [src, 32]
- str B_q, [dst, 16]
- cmp count, 96
- b.gt L(copy96_large)
- str E_q, [dstend, -16]
- stp C_q, D_q, [dst, 32]
- str A_q, [dst, 64]
+ ldp D_q, G_q, [src, 32]
+ cmp count, 96
+ b.gt L(copy96_large)
+ stp B_q, C_q, [dst, 16]
+ stp D_q, G_q, [dst, 48]
+ str A_q, [dstin]
+ str E_q, [dstend, -16]
ret
L(copy96_large):
- ldr F_q, [src, 64]
- stp C_q, D_q, [dst, 32]
- str E_q, [dstend, -16]
- stp A_q, F_q, [dst, 64]
+ ldr F_q, [src, 64]
+ str B_q, [dst, 16]
+ stp C_q, D_q, [dst, 32]
+ stp G_q, F_q, [dst, 64]
+ str A_q, [dstin]
+ str E_q, [dstend, -16]
ret
.p2align 4
L(memcopy_long):
- bic src, src, 15
- ldp B_q, C_q, [src], #32
- str A_q, [dstin]
- sub dst, dstin, tmp1
- add count, count, tmp1
- add dst, dst, 16
+ bic src, src, 15
+ ldp B_q, C_q, [src], #32
+ sub dst, dstin, tmp1
+ add count, count, tmp1
+ add dst, dst, 16
and tmp1, dst, 15
- ldp D_q, E_q, [src], #32
- str B_q, [dst], #16
+ ldp D_q, E_q, [src], #32
+ str A_q, [dstin]
/* Already loaded 64+16 bytes. Check if at
least 64 more bytes left */
- subs count, count, 64+64+16
- b.lt L(loop128_exit2)
- cmp count, MEMCPY_PREFETCH_LDR + 64 + 32
- b.lt L(loop128)
+ subs count, count, 64+64+16
+ b.lt L(loop128_exit0)
+ cmp count, MEMCPY_PREFETCH_LDR + 64 + 32
+ b.lt L(loop128)
cbnz tmp1, L(dst_unaligned)
- sub count, count, MEMCPY_PREFETCH_LDR + 64 + 32
+ sub count, count, MEMCPY_PREFETCH_LDR + 64 + 32
.p2align 4
L(loop128_prefetch):
- str C_q, [dst], #16
- prfm pldl1strm, [src, MEMCPY_PREFETCH_LDR]
- str D_q, [dst], #16
- ldp F_q, G_q, [src], #32
- str E_q, [dst], #16
- ldp H_q, A_q, [src], #32
- str F_q, [dst], #16
- prfm pldl1strm, [src, MEMCPY_PREFETCH_LDR]
- str G_q, [dst], #16
- ldp B_q, C_q, [src], #32
- str H_q, [dst], #16
- ldp D_q, E_q, [src], #32
- stp A_q, B_q, [dst], #32
+ prfm pldl1strm, [src, MEMCPY_PREFETCH_LDR]
+ ldp F_q, G_q, [src], #32
+ stp B_q, C_q, [dst], #32
+ ldp H_q, I_q, [src], #32
+ prfm pldl1strm, [src, MEMCPY_PREFETCH_LDR]
+ ldp B_q, C_q, [src], #32
+ stp D_q, E_q, [dst], #32
+ ldp D_q, E_q, [src], #32
+ stp F_q, G_q, [dst], #32
+ stp H_q, I_q, [dst], #32
subs count, count, 128
- b.ge L(loop128_prefetch)
+ b.ge L(loop128_prefetch)
-L(preloop128):
add count, count, MEMCPY_PREFETCH_LDR + 64 + 32
.p2align 4
L(loop128):
- ldp F_q, G_q, [src], #32
- str C_q, [dst], #16
- ldp B_q, A_q, [src], #32
- str D_q, [dst], #16
- stp E_q, F_q, [dst], #32
- stp G_q, B_q, [dst], #32
- subs count, count, 64
- b.lt L(loop128_exit1)
-L(loop128_proceed):
- ldp B_q, C_q, [src], #32
- str A_q, [dst], #16
- ldp D_q, E_q, [src], #32
- str B_q, [dst], #16
- subs count, count, 64
- b.ge L(loop128)
-
- .p2align 4
-L(loop128_exit2):
- stp C_q, D_q, [dst], #32
- str E_q, [dst], #16
- b L(copy_long_check32);
-
+ ldp F_q, G_q, [src], #32
+ ldp H_q, I_q, [src], #32
+ stp B_q, C_q, [dst], #32
+ stp D_q, E_q, [dst], #32
+ subs count, count, 64
+ b.lt L(loop128_exit1)
+ ldp B_q, C_q, [src], #32
+ ldp D_q, E_q, [src], #32
+ stp F_q, G_q, [dst], #32
+ stp H_q, I_q, [dst], #32
+ subs count, count, 64
+ b.ge L(loop128)
+L(loop128_exit0):
+ ldp F_q, G_q, [srcend, -64]
+ ldp H_q, I_q, [srcend, -32]
+ stp B_q, C_q, [dst], #32
+ stp D_q, E_q, [dst]
+ stp F_q, G_q, [dstend, -64]
+ stp H_q, I_q, [dstend, -32]
+ ret
L(loop128_exit1):
- /* A_q is still not stored and 0..63 bytes left,
- so, count is -64..-1.
- Check if less than 32 bytes left (count < -32) */
- str A_q, [dst], #16
-L(copy_long_check32):
- cmn count, 64
- b.eq L(copy_long_done)
- cmn count, 32
- b.le L(copy_long_last32)
- ldp B_q, C_q, [src]
- stp B_q, C_q, [dst]
-
-L(copy_long_last32):
- ldp F_q, G_q, [srcend, -32]
- stp F_q, G_q, [dstend, -32]
-
-L(copy_long_done):
+ ldp B_q, C_q, [srcend, -64]
+ ldp D_q, E_q, [srcend, -32]
+ stp F_q, G_q, [dst], #32
+ stp H_q, I_q, [dst]
+ stp B_q, C_q, [dstend, -64]
+ stp D_q, E_q, [dstend, -32]
+ ret
+
+L(dst_unaligned_tail):
+ ldp C_q, D_q, [srcend, -64]
+ ldp E_q, F_q, [srcend, -32]
+ stp A_q, B_q, [dst], #32
+ stp H_q, I_q, [dst], #16
+ str G_q, [dst, tmp1]
+ stp C_q, D_q, [dstend, -64]
+ stp E_q, F_q, [dstend, -32]
ret
L(dst_unaligned):
/* Store the 16 bytes to dst and align dst for further
operations, several bytes will be stored at this
address once more */
- str C_q, [dst], #16
- ldp F_q, G_q, [src], #32
+
+ ldp F_q, G_q, [src], #32
+ stp B_q, C_q, [dst], #32
bic dst, dst, 15
- subs count, count, 32
+ sub count, count, 32
adrp tmp2, L(ext_table)
add tmp2, tmp2, :lo12:L(ext_table)
add tmp2, tmp2, tmp1, LSL #2
ldr tmp3w, [tmp2]
add tmp2, tmp2, tmp3w, SXTW
br tmp2
-.p2align 4 ;\
+
+.p2align 4
+ /* to make the loop in each chunk 16-bytes aligned */
nop
#define EXT_CHUNK(shft) \
L(ext_size_ ## shft):;\
b.ge 1b;\
2:;\
ext I_v.16b, F_v.16b, G_v.16b, 16-shft;\
- b L(ext_tail);
+ b L(dst_unaligned_tail);
EXT_CHUNK(1)
EXT_CHUNK(2)
EXT_CHUNK(14)
EXT_CHUNK(15)
-L(ext_tail):
- stp A_q, B_q, [dst], #32
- stp H_q, I_q, [dst], #16
- add dst, dst, tmp1
- str G_q, [dst], #16
- b L(copy_long_check32)
+L(move_long):
+ .p2align 4
+1:
+ cbz tmp1, 3f
+
+ add srcend, src, count
+ add dstend, dstin, count
+
+ and tmp1, srcend, 15
+ ldr D_q, [srcend, -16]
+ sub srcend, srcend, tmp1
+ sub count, count, tmp1
+ ldp A_q, B_q, [srcend, -32]
+ str D_q, [dstend, -16]
+ ldp C_q, D_q, [srcend, -64]!
+ sub dstend, dstend, tmp1
+ subs count, count, 128
+ b.ls 2f
+
+ .p2align 4
+1:
+ subs count, count, 64
+ stp A_q, B_q, [dstend, -32]
+ ldp A_q, B_q, [srcend, -32]
+ stp C_q, D_q, [dstend, -64]!
+ ldp C_q, D_q, [srcend, -64]!
+ b.hi 1b
+
+ /* Write the last full set of 64 bytes. The remainder is at most 64
+ bytes, so it is safe to always copy 64 bytes from the start even if
+ there is just 1 byte left. */
+2:
+ ldp E_q, F_q, [src, 32]
+ ldp G_q, H_q, [src]
+ stp A_q, B_q, [dstend, -32]
+ stp C_q, D_q, [dstend, -64]
+ stp E_q, F_q, [dstin, 32]
+ stp G_q, H_q, [dstin]
+3: ret
END (MEMCPY)
projects / external / glibc.git / commitdiff