arch/x86/lib/usercopy_32.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * User address space access functions.
   4  * The non inlined parts of asm-i386/uaccess.h are here.
   5  *
   6  * Copyright 1997 Andi Kleen <ak@muc.de>
   7  * Copyright 1997 Linus Torvalds
   8  */
   9 #include <linux/export.h>
  10 #include <linux/uaccess.h>
  11 #include <asm/asm.h>
  12
  13 #ifdef CONFIG_X86_INTEL_USERCOPY
  14 /*
  15  * Alignment at which movsl is preferred for bulk memory copies.
  16  */
  17 struct movsl_mask movsl_mask __read_mostly;
  18 #endif
  19
  20 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
  21 {
  22 #ifdef CONFIG_X86_INTEL_USERCOPY
  23         if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
  24                 return 0;
  25 #endif
  26         return 1;
  27 }
  28 #define movsl_is_ok(a1, a2, n) \
  29         __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
  30
  31 /*
  32  * Zero Userspace
  33  */
  34
  35 #define __do_clear_user(addr,size)                                      \
  36 do {                                                                    \
  37         int __d0;                                                       \
  38         might_fault();                                                  \
  39         __asm__ __volatile__(                                           \
  40                 ASM_STAC "\n"                                           \
  41                 "0:     rep; stosl\n"                                   \
  42                 "       movl %2,%0\n"                                   \
  43                 "1:     rep; stosb\n"                                   \
  44                 "2: " ASM_CLAC "\n"                                     \
  45                 _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %2)   \
  46                 _ASM_EXTABLE_UA(1b, 2b)                                 \
  47                 : "=&c"(size), "=&D" (__d0)                             \
  48                 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));     \
  49 } while (0)
  50
  51 /**
  52  * clear_user - Zero a block of memory in user space.
  53  * @to:   Destination address, in user space.
  54  * @n:    Number of bytes to zero.
  55  *
  56  * Zero a block of memory in user space.
  57  *
  58  * Return: number of bytes that could not be cleared.
  59  * On success, this will be zero.
  60  */
  61 unsigned long
  62 clear_user(void __user *to, unsigned long n)
  63 {
  64         might_fault();
  65         if (access_ok(to, n))
  66                 __do_clear_user(to, n);
  67         return n;
  68 }
  69 EXPORT_SYMBOL(clear_user);
  70
  71 /**
  72  * __clear_user - Zero a block of memory in user space, with less checking.
  73  * @to:   Destination address, in user space.
  74  * @n:    Number of bytes to zero.
  75  *
  76  * Zero a block of memory in user space.  Caller must check
  77  * the specified block with access_ok() before calling this function.
  78  *
  79  * Return: number of bytes that could not be cleared.
  80  * On success, this will be zero.
  81  */
  82 unsigned long
  83 __clear_user(void __user *to, unsigned long n)
  84 {
  85         __do_clear_user(to, n);
  86         return n;
  87 }
  88 EXPORT_SYMBOL(__clear_user);
  89
  90 #ifdef CONFIG_X86_INTEL_USERCOPY
  91 static unsigned long
  92 __copy_user_intel(void __user *to, const void *from, unsigned long size)
  93 {
  94         int d0, d1;
  95         __asm__ __volatile__(
  96                        "       .align 2,0x90\n"
  97                        "1:     movl 32(%4), %%eax\n"
  98                        "       cmpl $67, %0\n"
  99                        "       jbe 3f\n"
 100                        "2:     movl 64(%4), %%eax\n"
 101                        "       .align 2,0x90\n"
 102                        "3:     movl 0(%4), %%eax\n"
 103                        "4:     movl 4(%4), %%edx\n"
 104                        "5:     movl %%eax, 0(%3)\n"
 105                        "6:     movl %%edx, 4(%3)\n"
 106                        "7:     movl 8(%4), %%eax\n"
 107                        "8:     movl 12(%4),%%edx\n"
 108                        "9:     movl %%eax, 8(%3)\n"
 109                        "10:    movl %%edx, 12(%3)\n"
 110                        "11:    movl 16(%4), %%eax\n"
 111                        "12:    movl 20(%4), %%edx\n"
 112                        "13:    movl %%eax, 16(%3)\n"
 113                        "14:    movl %%edx, 20(%3)\n"
 114                        "15:    movl 24(%4), %%eax\n"
 115                        "16:    movl 28(%4), %%edx\n"
 116                        "17:    movl %%eax, 24(%3)\n"
 117                        "18:    movl %%edx, 28(%3)\n"
 118                        "19:    movl 32(%4), %%eax\n"
 119                        "20:    movl 36(%4), %%edx\n"
 120                        "21:    movl %%eax, 32(%3)\n"
 121                        "22:    movl %%edx, 36(%3)\n"
 122                        "23:    movl 40(%4), %%eax\n"
 123                        "24:    movl 44(%4), %%edx\n"
 124                        "25:    movl %%eax, 40(%3)\n"
 125                        "26:    movl %%edx, 44(%3)\n"
 126                        "27:    movl 48(%4), %%eax\n"
 127                        "28:    movl 52(%4), %%edx\n"
 128                        "29:    movl %%eax, 48(%3)\n"
 129                        "30:    movl %%edx, 52(%3)\n"
 130                        "31:    movl 56(%4), %%eax\n"
 131                        "32:    movl 60(%4), %%edx\n"
 132                        "33:    movl %%eax, 56(%3)\n"
 133                        "34:    movl %%edx, 60(%3)\n"
 134                        "       addl $-64, %0\n"
 135                        "       addl $64, %4\n"
 136                        "       addl $64, %3\n"
 137                        "       cmpl $63, %0\n"
 138                        "       ja  1b\n"
 139                        "35:    movl  %0, %%eax\n"
 140                        "       shrl  $2, %0\n"
 141                        "       andl  $3, %%eax\n"
 142                        "       cld\n"
 143                        "99:    rep; movsl\n"
 144                        "36:    movl %%eax, %0\n"
 145                        "37:    rep; movsb\n"
 146                        "100:\n"
 147                        _ASM_EXTABLE_UA(1b, 100b)
 148                        _ASM_EXTABLE_UA(2b, 100b)
 149                        _ASM_EXTABLE_UA(3b, 100b)
 150                        _ASM_EXTABLE_UA(4b, 100b)
 151                        _ASM_EXTABLE_UA(5b, 100b)
 152                        _ASM_EXTABLE_UA(6b, 100b)
 153                        _ASM_EXTABLE_UA(7b, 100b)
 154                        _ASM_EXTABLE_UA(8b, 100b)
 155                        _ASM_EXTABLE_UA(9b, 100b)
 156                        _ASM_EXTABLE_UA(10b, 100b)
 157                        _ASM_EXTABLE_UA(11b, 100b)
 158                        _ASM_EXTABLE_UA(12b, 100b)
 159                        _ASM_EXTABLE_UA(13b, 100b)
 160                        _ASM_EXTABLE_UA(14b, 100b)
 161                        _ASM_EXTABLE_UA(15b, 100b)
 162                        _ASM_EXTABLE_UA(16b, 100b)
 163                        _ASM_EXTABLE_UA(17b, 100b)
 164                        _ASM_EXTABLE_UA(18b, 100b)
 165                        _ASM_EXTABLE_UA(19b, 100b)
 166                        _ASM_EXTABLE_UA(20b, 100b)
 167                        _ASM_EXTABLE_UA(21b, 100b)
 168                        _ASM_EXTABLE_UA(22b, 100b)
 169                        _ASM_EXTABLE_UA(23b, 100b)
 170                        _ASM_EXTABLE_UA(24b, 100b)
 171                        _ASM_EXTABLE_UA(25b, 100b)
 172                        _ASM_EXTABLE_UA(26b, 100b)
 173                        _ASM_EXTABLE_UA(27b, 100b)
 174                        _ASM_EXTABLE_UA(28b, 100b)
 175                        _ASM_EXTABLE_UA(29b, 100b)
 176                        _ASM_EXTABLE_UA(30b, 100b)
 177                        _ASM_EXTABLE_UA(31b, 100b)
 178                        _ASM_EXTABLE_UA(32b, 100b)
 179                        _ASM_EXTABLE_UA(33b, 100b)
 180                        _ASM_EXTABLE_UA(34b, 100b)
 181                        _ASM_EXTABLE_UA(35b, 100b)
 182                        _ASM_EXTABLE_UA(36b, 100b)
 183                        _ASM_EXTABLE_UA(37b, 100b)
 184                        _ASM_EXTABLE_TYPE_REG(99b, 100b, EX_TYPE_UCOPY_LEN4, %%eax)
 185                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 186                        :  "1"(to), "2"(from), "0"(size)
 187                        : "eax", "edx", "memory");
 188         return size;
 189 }
 190
 191 static unsigned long __copy_user_intel_nocache(void *to,
 192                                 const void __user *from, unsigned long size)
 193 {
 194         int d0, d1;
 195
 196         __asm__ __volatile__(
 197                "        .align 2,0x90\n"
 198                "0:      movl 32(%4), %%eax\n"
 199                "        cmpl $67, %0\n"
 200                "        jbe 2f\n"
 201                "1:      movl 64(%4), %%eax\n"
 202                "        .align 2,0x90\n"
 203                "2:      movl 0(%4), %%eax\n"
 204                "21:     movl 4(%4), %%edx\n"
 205                "        movnti %%eax, 0(%3)\n"
 206                "        movnti %%edx, 4(%3)\n"
 207                "3:      movl 8(%4), %%eax\n"
 208                "31:     movl 12(%4),%%edx\n"
 209                "        movnti %%eax, 8(%3)\n"
 210                "        movnti %%edx, 12(%3)\n"
 211                "4:      movl 16(%4), %%eax\n"
 212                "41:     movl 20(%4), %%edx\n"
 213                "        movnti %%eax, 16(%3)\n"
 214                "        movnti %%edx, 20(%3)\n"
 215                "10:     movl 24(%4), %%eax\n"
 216                "51:     movl 28(%4), %%edx\n"
 217                "        movnti %%eax, 24(%3)\n"
 218                "        movnti %%edx, 28(%3)\n"
 219                "11:     movl 32(%4), %%eax\n"
 220                "61:     movl 36(%4), %%edx\n"
 221                "        movnti %%eax, 32(%3)\n"
 222                "        movnti %%edx, 36(%3)\n"
 223                "12:     movl 40(%4), %%eax\n"
 224                "71:     movl 44(%4), %%edx\n"
 225                "        movnti %%eax, 40(%3)\n"
 226                "        movnti %%edx, 44(%3)\n"
 227                "13:     movl 48(%4), %%eax\n"
 228                "81:     movl 52(%4), %%edx\n"
 229                "        movnti %%eax, 48(%3)\n"
 230                "        movnti %%edx, 52(%3)\n"
 231                "14:     movl 56(%4), %%eax\n"
 232                "91:     movl 60(%4), %%edx\n"
 233                "        movnti %%eax, 56(%3)\n"
 234                "        movnti %%edx, 60(%3)\n"
 235                "        addl $-64, %0\n"
 236                "        addl $64, %4\n"
 237                "        addl $64, %3\n"
 238                "        cmpl $63, %0\n"
 239                "        ja  0b\n"
 240                "        sfence \n"
 241                "5:      movl  %0, %%eax\n"
 242                "        shrl  $2, %0\n"
 243                "        andl $3, %%eax\n"
 244                "        cld\n"
 245                "6:      rep; movsl\n"
 246                "        movl %%eax,%0\n"
 247                "7:      rep; movsb\n"
 248                "8:\n"
 249                _ASM_EXTABLE_UA(0b, 8b)
 250                _ASM_EXTABLE_UA(1b, 8b)
 251                _ASM_EXTABLE_UA(2b, 8b)
 252                _ASM_EXTABLE_UA(21b, 8b)
 253                _ASM_EXTABLE_UA(3b, 8b)
 254                _ASM_EXTABLE_UA(31b, 8b)
 255                _ASM_EXTABLE_UA(4b, 8b)
 256                _ASM_EXTABLE_UA(41b, 8b)
 257                _ASM_EXTABLE_UA(10b, 8b)
 258                _ASM_EXTABLE_UA(51b, 8b)
 259                _ASM_EXTABLE_UA(11b, 8b)
 260                _ASM_EXTABLE_UA(61b, 8b)
 261                _ASM_EXTABLE_UA(12b, 8b)
 262                _ASM_EXTABLE_UA(71b, 8b)
 263                _ASM_EXTABLE_UA(13b, 8b)
 264                _ASM_EXTABLE_UA(81b, 8b)
 265                _ASM_EXTABLE_UA(14b, 8b)
 266                _ASM_EXTABLE_UA(91b, 8b)
 267                _ASM_EXTABLE_TYPE_REG(6b, 8b, EX_TYPE_UCOPY_LEN4, %%eax)
 268                _ASM_EXTABLE_UA(7b, 8b)
 269                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 270                :  "1"(to), "2"(from), "0"(size)
 271                : "eax", "edx", "memory");
 272         return size;
 273 }
 274
 275 #else
 276
 277 /*
 278  * Leave these declared but undefined.  They should not be any references to
 279  * them
 280  */
 281 unsigned long __copy_user_intel(void __user *to, const void *from,
 282                                         unsigned long size);
 283 #endif /* CONFIG_X86_INTEL_USERCOPY */
 284
 285 /* Generic arbitrary sized copy.  */
 286 #define __copy_user(to, from, size)                                     \
 287 do {                                                                    \
 288         int __d0, __d1, __d2;                                           \
 289         __asm__ __volatile__(                                           \
 290                 "       cmp  $7,%0\n"                                   \
 291                 "       jbe  1f\n"                                      \
 292                 "       movl %1,%0\n"                                   \
 293                 "       negl %0\n"                                      \
 294                 "       andl $7,%0\n"                                   \
 295                 "       subl %0,%3\n"                                   \
 296                 "4:     rep; movsb\n"                                   \
 297                 "       movl %3,%0\n"                                   \
 298                 "       shrl $2,%0\n"                                   \
 299                 "       andl $3,%3\n"                                   \
 300                 "       .align 2,0x90\n"                                \
 301                 "0:     rep; movsl\n"                                   \
 302                 "       movl %3,%0\n"                                   \
 303                 "1:     rep; movsb\n"                                   \
 304                 "2:\n"                                                  \
 305                 _ASM_EXTABLE_TYPE_REG(4b, 2b, EX_TYPE_UCOPY_LEN1, %3)   \
 306                 _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN4, %3)   \
 307                 _ASM_EXTABLE_UA(1b, 2b)                                 \
 308                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 309                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 310                 : "memory");                                            \
 311 } while (0)
 312
 313 unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
 314 {
 315         __uaccess_begin_nospec();
 316         if (movsl_is_ok(to, from, n))
 317                 __copy_user(to, from, n);
 318         else
 319                 n = __copy_user_intel(to, from, n);
 320         __uaccess_end();
 321         return n;
 322 }
 323 EXPORT_SYMBOL(__copy_user_ll);
 324
 325 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
 326                                         unsigned long n)
 327 {
 328         __uaccess_begin_nospec();
 329 #ifdef CONFIG_X86_INTEL_USERCOPY
 330         if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
 331                 n = __copy_user_intel_nocache(to, from, n);
 332         else
 333                 __copy_user(to, from, n);
 334 #else
 335         __copy_user(to, from, n);
 336 #endif
 337         __uaccess_end();
 338         return n;
 339 }
 340 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);