arch/x86/include/asm/uaccess_64.h

   1 #ifndef _ASM_X86_UACCESS_64_H
   2 #define _ASM_X86_UACCESS_64_H
   3
   4 /*
   5  * User space memory access functions
   6  */
   7 #include <linux/compiler.h>
   8 #include <linux/errno.h>
   9 #include <linux/lockdep.h>
  10 #include <asm/alternative.h>
  11 #include <asm/cpufeature.h>
  12 #include <asm/page.h>
  13
  14 /*
  15  * Copy To/From Userspace
  16  */
  17
  18 /* Handles exceptions in both to and from, but doesn't do access_ok */
  19 __must_check unsigned long
  20 copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
  21 __must_check unsigned long
  22 copy_user_generic_string(void *to, const void *from, unsigned len);
  23 __must_check unsigned long
  24 copy_user_generic_unrolled(void *to, const void *from, unsigned len);
  25
  26 static __always_inline __must_check unsigned long
  27 copy_user_generic(void *to, const void *from, unsigned len)
  28 {
  29         unsigned ret;
  30
  31         /*
  32          * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
  33          * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
  34          * Otherwise, use copy_user_generic_unrolled.
  35          */
  36         alternative_call_2(copy_user_generic_unrolled,
  37                          copy_user_generic_string,
  38                          X86_FEATURE_REP_GOOD,
  39                          copy_user_enhanced_fast_string,
  40                          X86_FEATURE_ERMS,
  41                          ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
  42                                      "=d" (len)),
  43                          "1" (to), "2" (from), "3" (len)
  44                          : "memory", "rcx", "r8", "r9", "r10", "r11");
  45         return ret;
  46 }
  47
  48 __must_check unsigned long
  49 copy_in_user(void __user *to, const void __user *from, unsigned len);
  50
  51 static __always_inline __must_check
  52 int __copy_from_user(void *dst, const void __user *src, unsigned size)
  53 {
  54         int ret = 0;
  55
  56         might_fault();
  57         if (!__builtin_constant_p(size))
  58                 return copy_user_generic(dst, (__force void *)src, size);
  59         switch (size) {
  60         case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
  61                               ret, "b", "b", "=q", 1);
  62                 return ret;
  63         case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
  64                               ret, "w", "w", "=r", 2);
  65                 return ret;
  66         case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
  67                               ret, "l", "k", "=r", 4);
  68                 return ret;
  69         case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
  70                               ret, "q", "", "=r", 8);
  71                 return ret;
  72         case 10:
  73                 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
  74                                ret, "q", "", "=r", 10);
  75                 if (unlikely(ret))
  76                         return ret;
  77                 __get_user_asm(*(u16 *)(8 + (char *)dst),
  78                                (u16 __user *)(8 + (char __user *)src),
  79                                ret, "w", "w", "=r", 2);
  80                 return ret;
  81         case 16:
  82                 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
  83                                ret, "q", "", "=r", 16);
  84                 if (unlikely(ret))
  85                         return ret;
  86                 __get_user_asm(*(u64 *)(8 + (char *)dst),
  87                                (u64 __user *)(8 + (char __user *)src),
  88                                ret, "q", "", "=r", 8);
  89                 return ret;
  90         default:
  91                 return copy_user_generic(dst, (__force void *)src, size);
  92         }
  93 }
  94
  95 static __always_inline __must_check
  96 int __copy_to_user(void __user *dst, const void *src, unsigned size)
  97 {
  98         int ret = 0;
  99
 100         might_fault();
 101         if (!__builtin_constant_p(size))
 102                 return copy_user_generic((__force void *)dst, src, size);
 103         switch (size) {
 104         case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
 105                               ret, "b", "b", "iq", 1);
 106                 return ret;
 107         case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
 108                               ret, "w", "w", "ir", 2);
 109                 return ret;
 110         case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
 111                               ret, "l", "k", "ir", 4);
 112                 return ret;
 113         case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
 114                               ret, "q", "", "er", 8);
 115                 return ret;
 116         case 10:
 117                 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
 118                                ret, "q", "", "er", 10);
 119                 if (unlikely(ret))
 120                         return ret;
 121                 asm("":::"memory");
 122                 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
 123                                ret, "w", "w", "ir", 2);
 124                 return ret;
 125         case 16:
 126                 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
 127                                ret, "q", "", "er", 16);
 128                 if (unlikely(ret))
 129                         return ret;
 130                 asm("":::"memory");
 131                 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
 132                                ret, "q", "", "er", 8);
 133                 return ret;
 134         default:
 135                 return copy_user_generic((__force void *)dst, src, size);
 136         }
 137 }
 138
 139 static __always_inline __must_check
 140 int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
 141 {
 142         int ret = 0;
 143
 144         might_fault();
 145         if (!__builtin_constant_p(size))
 146                 return copy_user_generic((__force void *)dst,
 147                                          (__force void *)src, size);
 148         switch (size) {
 149         case 1: {
 150                 u8 tmp;
 151                 __get_user_asm(tmp, (u8 __user *)src,
 152                                ret, "b", "b", "=q", 1);
 153                 if (likely(!ret))
 154                         __put_user_asm(tmp, (u8 __user *)dst,
 155                                        ret, "b", "b", "iq", 1);
 156                 return ret;
 157         }
 158         case 2: {
 159                 u16 tmp;
 160                 __get_user_asm(tmp, (u16 __user *)src,
 161                                ret, "w", "w", "=r", 2);
 162                 if (likely(!ret))
 163                         __put_user_asm(tmp, (u16 __user *)dst,
 164                                        ret, "w", "w", "ir", 2);
 165                 return ret;
 166         }
 167
 168         case 4: {
 169                 u32 tmp;
 170                 __get_user_asm(tmp, (u32 __user *)src,
 171                                ret, "l", "k", "=r", 4);
 172                 if (likely(!ret))
 173                         __put_user_asm(tmp, (u32 __user *)dst,
 174                                        ret, "l", "k", "ir", 4);
 175                 return ret;
 176         }
 177         case 8: {
 178                 u64 tmp;
 179                 __get_user_asm(tmp, (u64 __user *)src,
 180                                ret, "q", "", "=r", 8);
 181                 if (likely(!ret))
 182                         __put_user_asm(tmp, (u64 __user *)dst,
 183                                        ret, "q", "", "er", 8);
 184                 return ret;
 185         }
 186         default:
 187                 return copy_user_generic((__force void *)dst,
 188                                          (__force void *)src, size);
 189         }
 190 }
 191
 192 static __must_check __always_inline int
 193 __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
 194 {
 195         return copy_user_generic(dst, (__force const void *)src, size);
 196 }
 197
 198 static __must_check __always_inline int
 199 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
 200 {
 201         return copy_user_generic((__force void *)dst, src, size);
 202 }
 203
 204 extern long __copy_user_nocache(void *dst, const void __user *src,
 205                                 unsigned size, int zerorest);
 206
 207 static inline int
 208 __copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
 209 {
 210         might_fault();
 211         return __copy_user_nocache(dst, src, size, 1);
 212 }
 213
 214 static inline int
 215 __copy_from_user_inatomic_nocache(void *dst, const void __user *src,
 216                                   unsigned size)
 217 {
 218         return __copy_user_nocache(dst, src, size, 0);
 219 }
 220
 221 unsigned long
 222 copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest);
 223
 224 #endif /* _ASM_X86_UACCESS_64_H */