1 /* SPDX-License-Identifier: GPL-2.0-only */
2 #include <linux/linkage.h>
4 #include <asm/export.h>
7 * Most CPUs support enhanced REP MOVSB/STOSB instructions. It is
8 * recommended to use this when possible and we do use them by default.
9 * If enhanced REP MOVSB/STOSB is not available, try to use fast string.
10 * Otherwise, use original.
17 SYM_FUNC_START(clear_page_rep)
22 SYM_FUNC_END(clear_page_rep)
23 EXPORT_SYMBOL_GPL(clear_page_rep)
25 SYM_FUNC_START(clear_page_orig)
31 #define PUT(x) movq %rax,x*8(%rdi)
44 SYM_FUNC_END(clear_page_orig)
45 EXPORT_SYMBOL_GPL(clear_page_orig)
47 SYM_FUNC_START(clear_page_erms)
52 SYM_FUNC_END(clear_page_erms)
53 EXPORT_SYMBOL_GPL(clear_page_erms)
56 * Default clear user-space.
63 * rcx: uncleared bytes or 0 if successful.
65 SYM_FUNC_START(rep_stos_alternative)
83 _ASM_EXTABLE_UA( 0b, .Lexit)
98 12: movq %rax,16(%rdi)
99 13: movq %rax,24(%rdi)
100 14: movq %rax,32(%rdi)
101 15: movq %rax,40(%rdi)
102 16: movq %rax,48(%rdi)
103 17: movq %rax,56(%rdi)
111 jne .Lclear_user_tail
115 * If we take an exception on any of the
116 * word stores, we know that %rcx isn't zero,
117 * so we can just go to the tail clearing to
118 * get the exact count.
120 * The unrolled case might end up clearing
121 * some bytes twice. Don't care.
123 * We could use the value in %rdi to avoid
124 * a second fault on the exact count case,
125 * but do we really care? No.
127 * Finally, we could try to align %rdi at the
128 * top of the unrolling. But unaligned stores
129 * just aren't that common or expensive.
131 _ASM_EXTABLE_UA( 1b, .Lclear_user_tail)
132 _ASM_EXTABLE_UA(10b, .Lclear_user_tail)
133 _ASM_EXTABLE_UA(11b, .Lclear_user_tail)
134 _ASM_EXTABLE_UA(12b, .Lclear_user_tail)
135 _ASM_EXTABLE_UA(13b, .Lclear_user_tail)
136 _ASM_EXTABLE_UA(14b, .Lclear_user_tail)
137 _ASM_EXTABLE_UA(15b, .Lclear_user_tail)
138 _ASM_EXTABLE_UA(16b, .Lclear_user_tail)
139 _ASM_EXTABLE_UA(17b, .Lclear_user_tail)
140 SYM_FUNC_END(rep_stos_alternative)
141 EXPORT_SYMBOL(rep_stos_alternative)