__must_check unsigned long
clear_user_original(void __user *addr, unsigned long len);
-__must_check unsigned long
-clear_user_rep_good(void __user *addr, unsigned long len);
-__must_check unsigned long
-clear_user_erms(void __user *addr, unsigned long len);
static __always_inline __must_check unsigned long __clear_user(void __user *addr, unsigned long size)
{
*/
asm volatile(
"1:\n\t"
- ALTERNATIVE_3("rep stosb",
- "call clear_user_erms", ALT_NOT(X86_FEATURE_FSRM),
- "call clear_user_rep_good", ALT_NOT(X86_FEATURE_ERMS),
- "call clear_user_original", ALT_NOT(X86_FEATURE_REP_GOOD))
+ ALTERNATIVE("rep stosb",
+ "call clear_user_original", ALT_NOT(X86_FEATURE_FSRS))
"2:\n"
_ASM_EXTABLE_UA(1b, 2b)
: "+c" (size), "+D" (addr), ASM_CALL_CONSTRAINT
- : "a" (0)
- /* rep_good clobbers %rdx */
- : "rdx");
+ : "a" (0));
clac();
_ASM_EXTABLE_UA(.Lbytes, .Lbytes_exception)
SYM_FUNC_END(clear_user_original)
EXPORT_SYMBOL(clear_user_original)
-
-/*
- * Alternative clear user-space when CPU feature X86_FEATURE_REP_GOOD is
- * present.
- * Input:
- * rdi destination
- * rcx count
- *
- * Output:
- * rcx: uncleared bytes or 0 if successful.
- */
-SYM_FUNC_START(clear_user_rep_good)
- # call the original thing for less than a cacheline
- cmp $64, %rcx
- jb clear_user_original
-
-.Lprep:
- # copy lower 32-bits for rest bytes
- mov %ecx, %edx
- shr $3, %rcx
- jz .Lrep_good_rest_bytes
-
-.Lrep_good_qwords:
- rep stosq
-
-.Lrep_good_rest_bytes:
- and $7, %edx
- jz .Lrep_good_exit
-
-.Lrep_good_bytes:
- mov %edx, %ecx
- rep stosb
-
-.Lrep_good_exit:
- # see .Lexit comment above
- xor %eax, %eax
- RET
-
-.Lrep_good_qwords_exception:
- # convert remaining qwords back into bytes to return to caller
- shl $3, %rcx
- and $7, %edx
- add %rdx, %rcx
- jmp .Lrep_good_exit
-
- _ASM_EXTABLE_UA(.Lrep_good_qwords, .Lrep_good_qwords_exception)
- _ASM_EXTABLE_UA(.Lrep_good_bytes, .Lrep_good_exit)
-SYM_FUNC_END(clear_user_rep_good)
-EXPORT_SYMBOL(clear_user_rep_good)
-
-/*
- * Alternative clear user-space when CPU feature X86_FEATURE_ERMS is present.
- * Input:
- * rdi destination
- * rcx count
- *
- * Output:
- * rcx: uncleared bytes or 0 if successful.
- *
- */
-SYM_FUNC_START(clear_user_erms)
- # call the original thing for less than a cacheline
- cmp $64, %rcx
- jb clear_user_original
-
-.Lerms_bytes:
- rep stosb
-
-.Lerms_exit:
- xorl %eax,%eax
- RET
-
- _ASM_EXTABLE_UA(.Lerms_bytes, .Lerms_exit)
-SYM_FUNC_END(clear_user_erms)
-EXPORT_SYMBOL(clear_user_erms)
"copy_mc_fragile_handle_tail",
"copy_mc_enhanced_fast_string",
"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
- "clear_user_erms",
- "clear_user_rep_good",
"clear_user_original",
NULL
};