1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright (c) 2013-2021, Arm Limited.
5 * Adapted from the original at:
6 * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/strncmp.S
9 #include <linux/linkage.h>
10 #include <asm/assembler.h>
17 #define L(label) .L ## label
19 #define REP8_01 0x0101010101010101
20 #define REP8_7f 0x7f7f7f7f7f7f7f7f
21 #define REP8_80 0x8080808080808080
23 /* Parameters and result. */
29 /* Internal variables. */
47 SYM_FUNC_START_WEAK_PI(strncmp)
50 mov zeroones, #REP8_01
54 cbnz count, L(mutual_align)
55 /* Calculate the number of full and partial words -1. */
56 sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
57 lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */
59 /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
60 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
61 can be done in parallel across the entire word. */
67 subs limit_wd, limit_wd, #1
68 sub tmp1, data1, zeroones
69 orr tmp2, data1, #REP8_7f
70 eor diff, data1, data2 /* Non-zero if differences found. */
71 csinv endloop, diff, xzr, pl /* Last Dword or differences. */
72 bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
73 ccmp endloop, #0, #0, eq
75 /* End of main loop */
77 /* Not reached the limit, must have found the end or a diff. */
78 tbz limit_wd, #63, L(not_limit)
80 /* Limit % 8 == 0 => all bytes significant. */
84 lsl limit, limit, #3 /* Bits -> bytes. */
91 bic data1, data1, mask
92 bic data2, data2, mask
94 /* Make sure that the NUL byte is marked in the syndrome. */
95 orr has_nul, has_nul, mask
98 orr syndrome, diff, has_nul
100 #ifndef __AARCH64EB__
101 rev syndrome, syndrome
103 /* The MS-non-zero bit of the syndrome marks either the first bit
104 that is different, or the top bit of the first zero byte.
105 Shifting left now will bring the critical information into the
109 lsl data1, data1, pos
110 lsl data2, data2, pos
111 /* But we need to zero-extend (char is unsigned) the value and then
112 perform a signed 32-bit subtraction. */
113 lsr data1, data1, #56
114 sub result, data1, data2, lsr #56
117 /* For big-endian we cannot use the trick with the syndrome value
118 as carry-propagation can corrupt the upper bits if the trailing
119 bytes in the string contain 0x01. */
120 /* However, if there is no NUL byte in the dword, we can generate
121 the result directly. We can't just subtract the bytes as the
122 MSB might be significant. */
126 cneg result, result, lo
129 /* Re-compute the NUL-byte detection, using a byte-reversed value. */
131 sub tmp1, tmp3, zeroones
132 orr tmp2, tmp3, #REP8_7f
133 bic has_nul, tmp1, tmp2
135 orr syndrome, diff, has_nul
137 /* The MS-non-zero bit of the syndrome marks either the first bit
138 that is different, or the top bit of the first zero byte.
139 Shifting left now will bring the critical information into the
141 lsl data1, data1, pos
142 lsl data2, data2, pos
143 /* But we need to zero-extend (char is unsigned) the value and then
144 perform a signed 32-bit subtraction. */
145 lsr data1, data1, #56
146 sub result, data1, data2, lsr #56
151 /* Sources are mutually aligned, but are not currently at an
152 alignment boundary. Round down the addresses and then mask off
153 the bytes that precede the start point.
154 We also need to adjust the limit calculations, but without
155 overflowing if the limit is near ULONG_MAX. */
158 ldr data1, [src1], #8
159 neg tmp3, count, lsl #3 /* 64 - bits(bytes beyond align). */
160 ldr data2, [src2], #8
162 sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
164 /* Big-endian. Early bytes are at MSB. */
165 lsl tmp2, tmp2, tmp3 /* Shift (count & 63). */
167 /* Little-endian. Early bytes are at LSB. */
168 lsr tmp2, tmp2, tmp3 /* Shift (count & 63). */
170 and tmp3, limit_wd, #7
171 lsr limit_wd, limit_wd, #3
172 /* Adjust the limit. Only low 3 bits used, so overflow irrelevant. */
173 add limit, limit, count
174 add tmp3, tmp3, count
175 orr data1, data1, tmp2
176 orr data2, data2, tmp2
177 add limit_wd, limit_wd, tmp3, lsr #3
181 /* Don't bother with dwords for up to 16 bytes. */
184 b.hs L(try_misaligned_words)
187 /* Perhaps we can do better than this. */
188 ldrb data1w, [src1], #1
189 ldrb data2w, [src2], #1
190 subs limit, limit, #1
191 ccmp data1w, #1, #0, hi /* NZCV = 0b0000. */
192 ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
195 sub result, data1, data2
197 /* Align the SRC1 to a dword by doing a bytewise compare and then do
199 L(try_misaligned_words):
200 lsr limit_wd, limit, #3
201 cbz count, L(do_misaligned)
205 sub limit, limit, count
206 lsr limit_wd, limit, #3
209 ldrb data1w, [src1], #1
210 ldrb data2w, [src2], #1
212 ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
214 subs count, count, #1
215 b.hi L(page_end_loop)
218 /* Prepare ourselves for the next page crossing. Unlike the aligned
219 loop, we fetch 1 less dword because we risk crossing bounds on
222 subs limit_wd, limit_wd, #1
225 and tmp2, src2, #0xff8
226 eor tmp2, tmp2, #0xff8
227 cbz tmp2, L(page_end_loop)
229 ldr data1, [src1], #8
230 ldr data2, [src2], #8
231 sub tmp1, data1, zeroones
232 orr tmp2, data1, #REP8_7f
233 eor diff, data1, data2 /* Non-zero if differences found. */
234 bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
235 ccmp diff, #0, #0, eq
237 subs limit_wd, limit_wd, #1
238 b.pl L(loop_misaligned)
241 /* We found a difference or a NULL before the limit was reached. */
243 cbz limit, L(not_limit)
244 /* Read the last word. */
247 ldr data1, [src1, limit]
248 ldr data2, [src2, limit]
249 sub tmp1, data1, zeroones
250 orr tmp2, data1, #REP8_7f
251 eor diff, data1, data2 /* Non-zero if differences found. */
252 bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
253 ccmp diff, #0, #0, eq
260 SYM_FUNC_END_PI(strncmp)
261 EXPORT_SYMBOL_NOHWKASAN(strncmp)