x86: Make __put_user() generate an out-of-line call

Instead of inlining the stac/mov/clac sequence (which also requires
individual exception table entries and several asm instruction
alternatives entries), just generate "call __put_user_nocheck_X" for the
__put_user() cases, the same way we changed __get_user earlier.

Unlike the get_user() case, we didn't have the same nice infrastructure
to just generate the call with a single case, so this actually has to
change some of the infrastructure in order to do this.  But that only
cleans up the code further.

So now, instead of using a case statement for the sizes, we just do the
same thing we've done on the get_user() side for a long time: use the
size as an immediate constant to the asm, and generate the asm that way
directly.

In order to handle the special case of 64-bit data on a 32-bit kernel, I
needed to change the calling convention slightly: the data is passed in
%eax[:%edx], the pointer in %ecx, and the return value is also returned
in %ecx.  It used to be returned in %eax, but because of how %eax can
now be a double register input, we don't want mix that with a
single-register output.

The actual low-level asm is easier to handle: we'll just share the code
between the checking and non-checking case, with the non-checking case
jumping into the middle of the function.  That may sound a bit too
special, but this code is all very very special anyway, so...

Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 600565f..4c2f8bb 100644 (file)
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -201,11 +201,6 @@ extern int __get_user_bad(void);
  */
 #define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)
 
-#define __put_user_x(size, x, ptr, __ret_pu)                   \
-       asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
-                    : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
-
-
 
 #ifdef CONFIG_X86_32
 #define __put_user_goto_u64(x, addr, label)                    \
@@ -217,25 +212,41 @@ extern int __get_user_bad(void);
                     : : "A" (x), "r" (addr)                    \
                     : : label)
 
-#define __put_user_x8(x, ptr, __ret_pu)                                \
-       asm volatile("call __put_user_8" : "=a" (__ret_pu)      \
-                    : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
 #else
 #define __put_user_goto_u64(x, ptr, label) \
        __put_user_goto(x, ptr, "q", "er", label)
-#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
 #endif
 
 extern void __put_user_bad(void);
 
 /*
  * Strange magic calling convention: pointer in %ecx,
- * value in %eax(:%edx), return value in %eax. clobbers %rbx
+ * value in %eax(:%edx), return value in %ecx. clobbers %rbx
  */
 extern void __put_user_1(void);
 extern void __put_user_2(void);
 extern void __put_user_4(void);
 extern void __put_user_8(void);
+extern void __put_user_nocheck_1(void);
+extern void __put_user_nocheck_2(void);
+extern void __put_user_nocheck_4(void);
+extern void __put_user_nocheck_8(void);
+
+#define do_put_user_call(fn,x,ptr)                                     \
+({                                                                     \
+       int __ret_pu;                                                   \
+       register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX);           \
+       __chk_user_ptr(ptr);                                            \
+       __val_pu = (x);                                                 \
+       asm volatile("call __" #fn "_%P[size]"                          \
+                    : "=c" (__ret_pu),                                 \
+                       ASM_CALL_CONSTRAINT                             \
+                    : "0" (ptr),                                       \
+                      "r" (__val_pu),                                  \
+                      [size] "i" (sizeof(*(ptr)))                      \
+                    :"ebx");                                           \
+       __builtin_expect(__ret_pu, 0);                                  \
+})
 
 /**
  * put_user - Write a simple value into user space.
@@ -254,32 +265,29 @@ extern void __put_user_8(void);
  *
  * Return: zero on success, or -EFAULT on error.
  */
-#define put_user(x, ptr)                                       \
-({                                                             \
-       int __ret_pu;                                           \
-       __typeof__(*(ptr)) __pu_val;                            \
-       __chk_user_ptr(ptr);                                    \
-       might_fault();                                          \
-       __pu_val = x;                                           \
-       switch (sizeof(*(ptr))) {                               \
-       case 1:                                                 \
-               __put_user_x(1, __pu_val, ptr, __ret_pu);       \
-               break;                                          \
-       case 2:                                                 \
-               __put_user_x(2, __pu_val, ptr, __ret_pu);       \
-               break;                                          \
-       case 4:                                                 \
-               __put_user_x(4, __pu_val, ptr, __ret_pu);       \
-               break;                                          \
-       case 8:                                                 \
-               __put_user_x8(__pu_val, ptr, __ret_pu);         \
-               break;                                          \
-       default:                                                \
-               __put_user_x(X, __pu_val, ptr, __ret_pu);       \
-               break;                                          \
-       }                                                       \
-       __builtin_expect(__ret_pu, 0);                          \
-})
+#define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })
+
+/**
+ * __put_user - Write a simple value into user space, with less checking.
+ * @x:   Value to copy to user space.
+ * @ptr: Destination address, in user space.
+ *
+ * Context: User context only. This function may sleep if pagefaults are
+ *          enabled.
+ *
+ * This macro copies a single simple value from kernel space to user
+ * space.  It supports simple types like char and int, but not larger
+ * data types like structures or arrays.
+ *
+ * @ptr must have pointer-to-simple-variable type, and @x must be assignable
+ * to the result of dereferencing @ptr.
+ *
+ * Caller must check the pointer with access_ok() before calling this
+ * function.
+ *
+ * Return: zero on success, or -EFAULT on error.
+ */
+#define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)
 
 #define __put_user_size(x, ptr, size, label)                           \
 do {                                                                   \
@@ -370,21 +378,6 @@ do {                                                                       \
                     : [umem] "m" (__m(addr)),                          \
                       [efault] "i" (-EFAULT), "0" (err))
 
-#define __put_user_nocheck(x, ptr, size)                       \
-({                                                             \
-       __label__ __pu_label;                                   \
-       int __pu_err = -EFAULT;                                 \
-       __typeof__(*(ptr)) __pu_val = (x);                      \
-       __typeof__(ptr) __pu_ptr = (ptr);                       \
-       __typeof__(size) __pu_size = (size);                    \
-       __uaccess_begin();                                      \
-       __put_user_size(__pu_val, __pu_ptr, __pu_size, __pu_label);     \
-       __pu_err = 0;                                           \
-__pu_label:                                                    \
-       __uaccess_end();                                        \
-       __builtin_expect(__pu_err, 0);                          \
-})
-
 /* FIXME: this hack is definitely wrong -AK */
 struct __large_struct { unsigned long buf[100]; };
 #define __m(x) (*(struct __large_struct __user *)(x))
@@ -401,30 +394,6 @@ struct __large_struct { unsigned long buf[100]; };
                : : ltype(x), "m" (__m(addr))                           \
                : : label)
 
-/**
- * __put_user - Write a simple value into user space, with less checking.
- * @x:   Value to copy to user space.
- * @ptr: Destination address, in user space.
- *
- * Context: User context only. This function may sleep if pagefaults are
- *          enabled.
- *
- * This macro copies a single simple value from kernel space to user
- * space.  It supports simple types like char and int, but not larger
- * data types like structures or arrays.
- *
- * @ptr must have pointer-to-simple-variable type, and @x must be assignable
- * to the result of dereferencing @ptr.
- *
- * Caller must check the pointer with access_ok() before calling this
- * function.
- *
- * Return: zero on success, or -EFAULT on error.
- */
-
-#define __put_user(x, ptr)                                             \
-       __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
-
 extern unsigned long
 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
 extern __must_check long

diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S
index 7c7c92d..b34a177 100644 (file)
--- a/arch/x86/lib/putuser.S
+++ b/arch/x86/lib/putuser.S
@@ -25,7 +25,9 @@
  * Inputs:     %eax[:%edx] contains the data
  *             %ecx contains the address
  *
- * Outputs:    %eax is error code (0 or -EFAULT)
+ * Outputs:    %ecx is error code (0 or -EFAULT)
+ *
+ * Clobbers:   %ebx needed for task pointer
  *
  * These functions should not modify any other registers,
  * as they get called from within inline assembly.
@@ -38,13 +40,15 @@ SYM_FUNC_START(__put_user_1)
        ENTER
        cmp TASK_addr_limit(%_ASM_BX),%_ASM_CX
        jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
        ASM_STAC
 1:     movb %al,(%_ASM_CX)
-       xor %eax,%eax
+       xor %ecx,%ecx
        ASM_CLAC
        ret
 SYM_FUNC_END(__put_user_1)
 EXPORT_SYMBOL(__put_user_1)
+EXPORT_SYMBOL(__put_user_nocheck_1)
 
 SYM_FUNC_START(__put_user_2)
        ENTER
@@ -52,13 +56,15 @@ SYM_FUNC_START(__put_user_2)
        sub $1,%_ASM_BX
        cmp %_ASM_BX,%_ASM_CX
        jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
        ASM_STAC
 2:     movw %ax,(%_ASM_CX)
-       xor %eax,%eax
+       xor %ecx,%ecx
        ASM_CLAC
        ret
 SYM_FUNC_END(__put_user_2)
 EXPORT_SYMBOL(__put_user_2)
+EXPORT_SYMBOL(__put_user_nocheck_2)
 
 SYM_FUNC_START(__put_user_4)
        ENTER
@@ -66,13 +72,15 @@ SYM_FUNC_START(__put_user_4)
        sub $3,%_ASM_BX
        cmp %_ASM_BX,%_ASM_CX
        jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
        ASM_STAC
 3:     movl %eax,(%_ASM_CX)
-       xor %eax,%eax
+       xor %ecx,%ecx
        ASM_CLAC
        ret
 SYM_FUNC_END(__put_user_4)
 EXPORT_SYMBOL(__put_user_4)
+EXPORT_SYMBOL(__put_user_nocheck_4)
 
 SYM_FUNC_START(__put_user_8)
        ENTER
@@ -80,21 +88,23 @@ SYM_FUNC_START(__put_user_8)
        sub $7,%_ASM_BX
        cmp %_ASM_BX,%_ASM_CX
        jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)
        ASM_STAC
 4:     mov %_ASM_AX,(%_ASM_CX)
 #ifdef CONFIG_X86_32
 5:     movl %edx,4(%_ASM_CX)
 #endif
-       xor %eax,%eax
+       xor %ecx,%ecx
        ASM_CLAC
        RET
 SYM_FUNC_END(__put_user_8)
 EXPORT_SYMBOL(__put_user_8)
+EXPORT_SYMBOL(__put_user_nocheck_8)
 
 SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
        ASM_CLAC
 .Lbad_put_user:
-       movl $-EFAULT,%eax
+       movl $-EFAULT,%ecx
        RET
 SYM_CODE_END(.Lbad_put_user_clac)