arch/mips/include/asm/uaccess.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  * Copyright (C) 2007  Maciej W. Rozycki
   9  * Copyright (C) 2014, Imagination Technologies Ltd.
  10  */
  11 #ifndef _ASM_UACCESS_H
  12 #define _ASM_UACCESS_H
  13
  14 #include <linux/kernel.h>
  15 #include <linux/errno.h>
  16 #include <linux/thread_info.h>
  17 #include <asm/asm-eva.h>
  18
  19 /*
  20  * The fs value determines whether argument validity checking should be
  21  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  22  * get_fs() == KERNEL_DS, checking is bypassed.
  23  *
  24  * For historical reasons, these macros are grossly misnamed.
  25  */
  26 #ifdef CONFIG_32BIT
  27
  28 #ifdef CONFIG_KVM_GUEST
  29 #define __UA_LIMIT 0x40000000UL
  30 #else
  31 #define __UA_LIMIT 0x80000000UL
  32 #endif
  33
  34 #define __UA_ADDR       ".word"
  35 #define __UA_LA         "la"
  36 #define __UA_ADDU       "addu"
  37 #define __UA_t0         "$8"
  38 #define __UA_t1         "$9"
  39
  40 #endif /* CONFIG_32BIT */
  41
  42 #ifdef CONFIG_64BIT
  43
  44 extern u64 __ua_limit;
  45
  46 #define __UA_LIMIT      __ua_limit
  47
  48 #define __UA_ADDR       ".dword"
  49 #define __UA_LA         "dla"
  50 #define __UA_ADDU       "daddu"
  51 #define __UA_t0         "$12"
  52 #define __UA_t1         "$13"
  53
  54 #endif /* CONFIG_64BIT */
  55
  56 /*
  57  * USER_DS is a bitmask that has the bits set that may not be set in a valid
  58  * userspace address.  Note that we limit 32-bit userspace to 0x7fff8000 but
  59  * the arithmetic we're doing only works if the limit is a power of two, so
  60  * we use 0x80000000 here on 32-bit kernels.  If a process passes an invalid
  61  * address in this range it's the process's problem, not ours :-)
  62  */
  63
  64 #ifdef CONFIG_KVM_GUEST
  65 #define KERNEL_DS       ((mm_segment_t) { 0x80000000UL })
  66 #define USER_DS         ((mm_segment_t) { 0xC0000000UL })
  67 #else
  68 #define KERNEL_DS       ((mm_segment_t) { 0UL })
  69 #define USER_DS         ((mm_segment_t) { __UA_LIMIT })
  70 #endif
  71
  72 #define VERIFY_READ    0
  73 #define VERIFY_WRITE   1
  74
  75 #define get_ds()        (KERNEL_DS)
  76 #define get_fs()        (current_thread_info()->addr_limit)
  77 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
  78
  79 #define segment_eq(a, b)        ((a).seg == (b).seg)
  80
  81
  82 /*
  83  * Is a address valid? This does a straighforward calculation rather
  84  * than tests.
  85  *
  86  * Address valid if:
  87  *  - "addr" doesn't have any high-bits set
  88  *  - AND "size" doesn't have any high-bits set
  89  *  - AND "addr+size" doesn't have any high-bits set
  90  *  - OR we are in kernel mode.
  91  *
  92  * __ua_size() is a trick to avoid runtime checking of positive constant
  93  * sizes; for those we already know at compile time that the size is ok.
  94  */
  95 #define __ua_size(size)                                                 \
  96         ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
  97
  98 /*
  99  * access_ok: - Checks if a user space pointer is valid
 100  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
 101  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
 102  *        to write to a block, it is always safe to read from it.
 103  * @addr: User space pointer to start of block to check
 104  * @size: Size of block to check
 105  *
 106  * Context: User context only. This function may sleep if pagefaults are
 107  *          enabled.
 108  *
 109  * Checks if a pointer to a block of memory in user space is valid.
 110  *
 111  * Returns true (nonzero) if the memory block may be valid, false (zero)
 112  * if it is definitely invalid.
 113  *
 114  * Note that, depending on architecture, this function probably just
 115  * checks that the pointer is in the user space range - after calling
 116  * this function, memory access functions may still return -EFAULT.
 117  */
 118
 119 #define __access_mask get_fs().seg
 120
 121 #define __access_ok(addr, size, mask)                                   \
 122 ({                                                                      \
 123         unsigned long __addr = (unsigned long) (addr);                  \
 124         unsigned long __size = size;                                    \
 125         unsigned long __mask = mask;                                    \
 126         unsigned long __ok;                                             \
 127                                                                         \
 128         __chk_user_ptr(addr);                                           \
 129         __ok = (signed long)(__mask & (__addr | (__addr + __size) |     \
 130                 __ua_size(__size)));                                    \
 131         __ok == 0;                                                      \
 132 })
 133
 134 #define access_ok(type, addr, size)                                     \
 135         likely(__access_ok((addr), (size), __access_mask))
 136
 137 /*
 138  * put_user: - Write a simple value into user space.
 139  * @x:   Value to copy to user space.
 140  * @ptr: Destination address, in user space.
 141  *
 142  * Context: User context only. This function may sleep if pagefaults are
 143  *          enabled.
 144  *
 145  * This macro copies a single simple value from kernel space to user
 146  * space.  It supports simple types like char and int, but not larger
 147  * data types like structures or arrays.
 148  *
 149  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 150  * to the result of dereferencing @ptr.
 151  *
 152  * Returns zero on success, or -EFAULT on error.
 153  */
 154 #define put_user(x,ptr) \
 155         __put_user_check((x), (ptr), sizeof(*(ptr)))
 156
 157 /*
 158  * get_user: - Get a simple variable from user space.
 159  * @x:   Variable to store result.
 160  * @ptr: Source address, in user space.
 161  *
 162  * Context: User context only. This function may sleep if pagefaults are
 163  *          enabled.
 164  *
 165  * This macro copies a single simple variable from user space to kernel
 166  * space.  It supports simple types like char and int, but not larger
 167  * data types like structures or arrays.
 168  *
 169  * @ptr must have pointer-to-simple-variable type, and the result of
 170  * dereferencing @ptr must be assignable to @x without a cast.
 171  *
 172  * Returns zero on success, or -EFAULT on error.
 173  * On error, the variable @x is set to zero.
 174  */
 175 #define get_user(x,ptr) \
 176         __get_user_check((x), (ptr), sizeof(*(ptr)))
 177
 178 /*
 179  * __put_user: - Write a simple value into user space, with less checking.
 180  * @x:   Value to copy to user space.
 181  * @ptr: Destination address, in user space.
 182  *
 183  * Context: User context only. This function may sleep if pagefaults are
 184  *          enabled.
 185  *
 186  * This macro copies a single simple value from kernel space to user
 187  * space.  It supports simple types like char and int, but not larger
 188  * data types like structures or arrays.
 189  *
 190  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 191  * to the result of dereferencing @ptr.
 192  *
 193  * Caller must check the pointer with access_ok() before calling this
 194  * function.
 195  *
 196  * Returns zero on success, or -EFAULT on error.
 197  */
 198 #define __put_user(x,ptr) \
 199         __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
 200
 201 /*
 202  * __get_user: - Get a simple variable from user space, with less checking.
 203  * @x:   Variable to store result.
 204  * @ptr: Source address, in user space.
 205  *
 206  * Context: User context only. This function may sleep if pagefaults are
 207  *          enabled.
 208  *
 209  * This macro copies a single simple variable from user space to kernel
 210  * space.  It supports simple types like char and int, but not larger
 211  * data types like structures or arrays.
 212  *
 213  * @ptr must have pointer-to-simple-variable type, and the result of
 214  * dereferencing @ptr must be assignable to @x without a cast.
 215  *
 216  * Caller must check the pointer with access_ok() before calling this
 217  * function.
 218  *
 219  * Returns zero on success, or -EFAULT on error.
 220  * On error, the variable @x is set to zero.
 221  */
 222 #define __get_user(x,ptr) \
 223         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
 224
 225 struct __large_struct { unsigned long buf[100]; };
 226 #define __m(x) (*(struct __large_struct __user *)(x))
 227
 228 /*
 229  * Yuck.  We need two variants, one for 64bit operation and one
 230  * for 32 bit mode and old iron.
 231  */
 232 #ifndef CONFIG_EVA
 233 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr)
 234 #else
 235 /*
 236  * Kernel specific functions for EVA. We need to use normal load instructions
 237  * to read data from kernel when operating in EVA mode. We use these macros to
 238  * avoid redefining __get_user_asm for EVA.
 239  */
 240 #undef _loadd
 241 #undef _loadw
 242 #undef _loadh
 243 #undef _loadb
 244 #ifdef CONFIG_32BIT
 245 #define _loadd                  _loadw
 246 #else
 247 #define _loadd(reg, addr)       "ld " reg ", " addr
 248 #endif
 249 #define _loadw(reg, addr)       "lw " reg ", " addr
 250 #define _loadh(reg, addr)       "lh " reg ", " addr
 251 #define _loadb(reg, addr)       "lb " reg ", " addr
 252
 253 #define __get_kernel_common(val, size, ptr)                             \
 254 do {                                                                    \
 255         switch (size) {                                                 \
 256         case 1: __get_data_asm(val, _loadb, ptr); break;                \
 257         case 2: __get_data_asm(val, _loadh, ptr); break;                \
 258         case 4: __get_data_asm(val, _loadw, ptr); break;                \
 259         case 8: __GET_DW(val, _loadd, ptr); break;                      \
 260         default: __get_user_unknown(); break;                           \
 261         }                                                               \
 262 } while (0)
 263 #endif
 264
 265 #ifdef CONFIG_32BIT
 266 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr)
 267 #endif
 268 #ifdef CONFIG_64BIT
 269 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr)
 270 #endif
 271
 272 extern void __get_user_unknown(void);
 273
 274 #define __get_user_common(val, size, ptr)                               \
 275 do {                                                                    \
 276         switch (size) {                                                 \
 277         case 1: __get_data_asm(val, user_lb, ptr); break;               \
 278         case 2: __get_data_asm(val, user_lh, ptr); break;               \
 279         case 4: __get_data_asm(val, user_lw, ptr); break;               \
 280         case 8: __GET_DW(val, user_ld, ptr); break;                     \
 281         default: __get_user_unknown(); break;                           \
 282         }                                                               \
 283 } while (0)
 284
 285 #define __get_user_nocheck(x, ptr, size)                                \
 286 ({                                                                      \
 287         int __gu_err;                                                   \
 288                                                                         \
 289         if (segment_eq(get_fs(), get_ds())) {                           \
 290                 __get_kernel_common((x), size, ptr);                    \
 291         } else {                                                        \
 292                 __chk_user_ptr(ptr);                                    \
 293                 __get_user_common((x), size, ptr);                      \
 294         }                                                               \
 295         __gu_err;                                                       \
 296 })
 297
 298 #define __get_user_check(x, ptr, size)                                  \
 299 ({                                                                      \
 300         int __gu_err = -EFAULT;                                         \
 301         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 302                                                                         \
 303         might_fault();                                                  \
 304         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size))) {          \
 305                 if (segment_eq(get_fs(), get_ds()))                     \
 306                         __get_kernel_common((x), size, __gu_ptr);       \
 307                 else                                                    \
 308                         __get_user_common((x), size, __gu_ptr);         \
 309         } else                                                          \
 310                 (x) = 0;                                                \
 311                                                                         \
 312         __gu_err;                                                       \
 313 })
 314
 315 #define __get_data_asm(val, insn, addr)                                 \
 316 {                                                                       \
 317         long __gu_tmp;                                                  \
 318                                                                         \
 319         __asm__ __volatile__(                                           \
 320         "1:     "insn("%1", "%3")"                              \n"     \
 321         "2:                                                     \n"     \
 322         "       .insn                                           \n"     \
 323         "       .section .fixup,\"ax\"                          \n"     \
 324         "3:     li      %0, %4                                  \n"     \
 325         "       move    %1, $0                                  \n"     \
 326         "       j       2b                                      \n"     \
 327         "       .previous                                       \n"     \
 328         "       .section __ex_table,\"a\"                       \n"     \
 329         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 330         "       .previous                                       \n"     \
 331         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 332         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 333                                                                         \
 334         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 335 }
 336
 337 /*
 338  * Get a long long 64 using 32 bit registers.
 339  */
 340 #define __get_data_asm_ll32(val, insn, addr)                            \
 341 {                                                                       \
 342         union {                                                         \
 343                 unsigned long long      l;                              \
 344                 __typeof__(*(addr))     t;                              \
 345         } __gu_tmp;                                                     \
 346                                                                         \
 347         __asm__ __volatile__(                                           \
 348         "1:     " insn("%1", "(%3)")"                           \n"     \
 349         "2:     " insn("%D1", "4(%3)")"                         \n"     \
 350         "3:                                                     \n"     \
 351         "       .insn                                           \n"     \
 352         "       .section        .fixup,\"ax\"                   \n"     \
 353         "4:     li      %0, %4                                  \n"     \
 354         "       move    %1, $0                                  \n"     \
 355         "       move    %D1, $0                                 \n"     \
 356         "       j       3b                                      \n"     \
 357         "       .previous                                       \n"     \
 358         "       .section        __ex_table,\"a\"                \n"     \
 359         "       " __UA_ADDR "   1b, 4b                          \n"     \
 360         "       " __UA_ADDR "   2b, 4b                          \n"     \
 361         "       .previous                                       \n"     \
 362         : "=r" (__gu_err), "=&r" (__gu_tmp.l)                           \
 363         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 364                                                                         \
 365         (val) = __gu_tmp.t;                                             \
 366 }
 367
 368 #ifndef CONFIG_EVA
 369 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size)
 370 #else
 371 /*
 372  * Kernel specific functions for EVA. We need to use normal load instructions
 373  * to read data from kernel when operating in EVA mode. We use these macros to
 374  * avoid redefining __get_data_asm for EVA.
 375  */
 376 #undef _stored
 377 #undef _storew
 378 #undef _storeh
 379 #undef _storeb
 380 #ifdef CONFIG_32BIT
 381 #define _stored                 _storew
 382 #else
 383 #define _stored(reg, addr)      "ld " reg ", " addr
 384 #endif
 385
 386 #define _storew(reg, addr)      "sw " reg ", " addr
 387 #define _storeh(reg, addr)      "sh " reg ", " addr
 388 #define _storeb(reg, addr)      "sb " reg ", " addr
 389
 390 #define __put_kernel_common(ptr, size)                                  \
 391 do {                                                                    \
 392         switch (size) {                                                 \
 393         case 1: __put_data_asm(_storeb, ptr); break;                    \
 394         case 2: __put_data_asm(_storeh, ptr); break;                    \
 395         case 4: __put_data_asm(_storew, ptr); break;                    \
 396         case 8: __PUT_DW(_stored, ptr); break;                          \
 397         default: __put_user_unknown(); break;                           \
 398         }                                                               \
 399 } while(0)
 400 #endif
 401
 402 /*
 403  * Yuck.  We need two variants, one for 64bit operation and one
 404  * for 32 bit mode and old iron.
 405  */
 406 #ifdef CONFIG_32BIT
 407 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr)
 408 #endif
 409 #ifdef CONFIG_64BIT
 410 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr)
 411 #endif
 412
 413 #define __put_user_common(ptr, size)                                    \
 414 do {                                                                    \
 415         switch (size) {                                                 \
 416         case 1: __put_data_asm(user_sb, ptr); break;                    \
 417         case 2: __put_data_asm(user_sh, ptr); break;                    \
 418         case 4: __put_data_asm(user_sw, ptr); break;                    \
 419         case 8: __PUT_DW(user_sd, ptr); break;                          \
 420         default: __put_user_unknown(); break;                           \
 421         }                                                               \
 422 } while (0)
 423
 424 #define __put_user_nocheck(x, ptr, size)                                \
 425 ({                                                                      \
 426         __typeof__(*(ptr)) __pu_val;                                    \
 427         int __pu_err = 0;                                               \
 428                                                                         \
 429         __pu_val = (x);                                                 \
 430         if (segment_eq(get_fs(), get_ds())) {                           \
 431                 __put_kernel_common(ptr, size);                         \
 432         } else {                                                        \
 433                 __chk_user_ptr(ptr);                                    \
 434                 __put_user_common(ptr, size);                           \
 435         }                                                               \
 436         __pu_err;                                                       \
 437 })
 438
 439 #define __put_user_check(x, ptr, size)                                  \
 440 ({                                                                      \
 441         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 442         __typeof__(*(ptr)) __pu_val = (x);                              \
 443         int __pu_err = -EFAULT;                                         \
 444                                                                         \
 445         might_fault();                                                  \
 446         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size))) {        \
 447                 if (segment_eq(get_fs(), get_ds()))                     \
 448                         __put_kernel_common(__pu_addr, size);           \
 449                 else                                                    \
 450                         __put_user_common(__pu_addr, size);             \
 451         }                                                               \
 452                                                                         \
 453         __pu_err;                                                       \
 454 })
 455
 456 #define __put_data_asm(insn, ptr)                                       \
 457 {                                                                       \
 458         __asm__ __volatile__(                                           \
 459         "1:     "insn("%z2", "%3")"     # __put_data_asm        \n"     \
 460         "2:                                                     \n"     \
 461         "       .insn                                           \n"     \
 462         "       .section        .fixup,\"ax\"                   \n"     \
 463         "3:     li      %0, %4                                  \n"     \
 464         "       j       2b                                      \n"     \
 465         "       .previous                                       \n"     \
 466         "       .section        __ex_table,\"a\"                \n"     \
 467         "       " __UA_ADDR "   1b, 3b                          \n"     \
 468         "       .previous                                       \n"     \
 469         : "=r" (__pu_err)                                               \
 470         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 471           "i" (-EFAULT));                                               \
 472 }
 473
 474 #define __put_data_asm_ll32(insn, ptr)                                  \
 475 {                                                                       \
 476         __asm__ __volatile__(                                           \
 477         "1:     "insn("%2", "(%3)")"    # __put_data_asm_ll32   \n"     \
 478         "2:     "insn("%D2", "4(%3)")"                          \n"     \
 479         "3:                                                     \n"     \
 480         "       .insn                                           \n"     \
 481         "       .section        .fixup,\"ax\"                   \n"     \
 482         "4:     li      %0, %4                                  \n"     \
 483         "       j       3b                                      \n"     \
 484         "       .previous                                       \n"     \
 485         "       .section        __ex_table,\"a\"                \n"     \
 486         "       " __UA_ADDR "   1b, 4b                          \n"     \
 487         "       " __UA_ADDR "   2b, 4b                          \n"     \
 488         "       .previous"                                              \
 489         : "=r" (__pu_err)                                               \
 490         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 491           "i" (-EFAULT));                                               \
 492 }
 493
 494 extern void __put_user_unknown(void);
 495
 496 /*
 497  * ul{b,h,w} are macros and there are no equivalent macros for EVA.
 498  * EVA unaligned access is handled in the ADE exception handler.
 499  */
 500 #ifndef CONFIG_EVA
 501 /*
 502  * put_user_unaligned: - Write a simple value into user space.
 503  * @x:   Value to copy to user space.
 504  * @ptr: Destination address, in user space.
 505  *
 506  * Context: User context only. This function may sleep if pagefaults are
 507  *          enabled.
 508  *
 509  * This macro copies a single simple value from kernel space to user
 510  * space.  It supports simple types like char and int, but not larger
 511  * data types like structures or arrays.
 512  *
 513  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 514  * to the result of dereferencing @ptr.
 515  *
 516  * Returns zero on success, or -EFAULT on error.
 517  */
 518 #define put_user_unaligned(x,ptr)       \
 519         __put_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 520
 521 /*
 522  * get_user_unaligned: - Get a simple variable from user space.
 523  * @x:   Variable to store result.
 524  * @ptr: Source address, in user space.
 525  *
 526  * Context: User context only. This function may sleep if pagefaults are
 527  *          enabled.
 528  *
 529  * This macro copies a single simple variable from user space to kernel
 530  * space.  It supports simple types like char and int, but not larger
 531  * data types like structures or arrays.
 532  *
 533  * @ptr must have pointer-to-simple-variable type, and the result of
 534  * dereferencing @ptr must be assignable to @x without a cast.
 535  *
 536  * Returns zero on success, or -EFAULT on error.
 537  * On error, the variable @x is set to zero.
 538  */
 539 #define get_user_unaligned(x,ptr) \
 540         __get_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 541
 542 /*
 543  * __put_user_unaligned: - Write a simple value into user space, with less checking.
 544  * @x:   Value to copy to user space.
 545  * @ptr: Destination address, in user space.
 546  *
 547  * Context: User context only. This function may sleep if pagefaults are
 548  *          enabled.
 549  *
 550  * This macro copies a single simple value from kernel space to user
 551  * space.  It supports simple types like char and int, but not larger
 552  * data types like structures or arrays.
 553  *
 554  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 555  * to the result of dereferencing @ptr.
 556  *
 557  * Caller must check the pointer with access_ok() before calling this
 558  * function.
 559  *
 560  * Returns zero on success, or -EFAULT on error.
 561  */
 562 #define __put_user_unaligned(x,ptr) \
 563         __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
 564
 565 /*
 566  * __get_user_unaligned: - Get a simple variable from user space, with less checking.
 567  * @x:   Variable to store result.
 568  * @ptr: Source address, in user space.
 569  *
 570  * Context: User context only. This function may sleep if pagefaults are
 571  *          enabled.
 572  *
 573  * This macro copies a single simple variable from user space to kernel
 574  * space.  It supports simple types like char and int, but not larger
 575  * data types like structures or arrays.
 576  *
 577  * @ptr must have pointer-to-simple-variable type, and the result of
 578  * dereferencing @ptr must be assignable to @x without a cast.
 579  *
 580  * Caller must check the pointer with access_ok() before calling this
 581  * function.
 582  *
 583  * Returns zero on success, or -EFAULT on error.
 584  * On error, the variable @x is set to zero.
 585  */
 586 #define __get_user_unaligned(x,ptr) \
 587         __get_user__unalignednocheck((x),(ptr),sizeof(*(ptr)))
 588
 589 /*
 590  * Yuck.  We need two variants, one for 64bit operation and one
 591  * for 32 bit mode and old iron.
 592  */
 593 #ifdef CONFIG_32BIT
 594 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 595         __get_user_unaligned_asm_ll32(val, ptr)
 596 #endif
 597 #ifdef CONFIG_64BIT
 598 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 599         __get_user_unaligned_asm(val, "uld", ptr)
 600 #endif
 601
 602 extern void __get_user_unaligned_unknown(void);
 603
 604 #define __get_user_unaligned_common(val, size, ptr)                     \
 605 do {                                                                    \
 606         switch (size) {                                                 \
 607         case 1: __get_data_asm(val, "lb", ptr); break;                  \
 608         case 2: __get_user_unaligned_asm(val, "ulh", ptr); break;       \
 609         case 4: __get_user_unaligned_asm(val, "ulw", ptr); break;       \
 610         case 8: __GET_USER_UNALIGNED_DW(val, ptr); break;               \
 611         default: __get_user_unaligned_unknown(); break;                 \
 612         }                                                               \
 613 } while (0)
 614
 615 #define __get_user_unaligned_nocheck(x,ptr,size)                        \
 616 ({                                                                      \
 617         int __gu_err;                                                   \
 618                                                                         \
 619         __get_user_unaligned_common((x), size, ptr);                    \
 620         __gu_err;                                                       \
 621 })
 622
 623 #define __get_user_unaligned_check(x,ptr,size)                          \
 624 ({                                                                      \
 625         int __gu_err = -EFAULT;                                         \
 626         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 627                                                                         \
 628         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size)))            \
 629                 __get_user_unaligned_common((x), size, __gu_ptr);       \
 630                                                                         \
 631         __gu_err;                                                       \
 632 })
 633
 634 #define __get_data_unaligned_asm(val, insn, addr)                       \
 635 {                                                                       \
 636         long __gu_tmp;                                                  \
 637                                                                         \
 638         __asm__ __volatile__(                                           \
 639         "1:     " insn "        %1, %3                          \n"     \
 640         "2:                                                     \n"     \
 641         "       .insn                                           \n"     \
 642         "       .section .fixup,\"ax\"                          \n"     \
 643         "3:     li      %0, %4                                  \n"     \
 644         "       move    %1, $0                                  \n"     \
 645         "       j       2b                                      \n"     \
 646         "       .previous                                       \n"     \
 647         "       .section __ex_table,\"a\"                       \n"     \
 648         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 649         "       "__UA_ADDR "\t1b + 4, 3b                        \n"     \
 650         "       .previous                                       \n"     \
 651         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 652         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 653                                                                         \
 654         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 655 }
 656
 657 /*
 658  * Get a long long 64 using 32 bit registers.
 659  */
 660 #define __get_user_unaligned_asm_ll32(val, addr)                        \
 661 {                                                                       \
 662         unsigned long long __gu_tmp;                                    \
 663                                                                         \
 664         __asm__ __volatile__(                                           \
 665         "1:     ulw     %1, (%3)                                \n"     \
 666         "2:     ulw     %D1, 4(%3)                              \n"     \
 667         "       move    %0, $0                                  \n"     \
 668         "3:                                                     \n"     \
 669         "       .insn                                           \n"     \
 670         "       .section        .fixup,\"ax\"                   \n"     \
 671         "4:     li      %0, %4                                  \n"     \
 672         "       move    %1, $0                                  \n"     \
 673         "       move    %D1, $0                                 \n"     \
 674         "       j       3b                                      \n"     \
 675         "       .previous                                       \n"     \
 676         "       .section        __ex_table,\"a\"                \n"     \
 677         "       " __UA_ADDR "   1b, 4b                          \n"     \
 678         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 679         "       " __UA_ADDR "   2b, 4b                          \n"     \
 680         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 681         "       .previous                                       \n"     \
 682         : "=r" (__gu_err), "=&r" (__gu_tmp)                             \
 683         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 684         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 685 }
 686
 687 /*
 688  * Yuck.  We need two variants, one for 64bit operation and one
 689  * for 32 bit mode and old iron.
 690  */
 691 #ifdef CONFIG_32BIT
 692 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr)
 693 #endif
 694 #ifdef CONFIG_64BIT
 695 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr)
 696 #endif
 697
 698 #define __put_user_unaligned_common(ptr, size)                          \
 699 do {                                                                    \
 700         switch (size) {                                                 \
 701         case 1: __put_data_asm("sb", ptr); break;                       \
 702         case 2: __put_user_unaligned_asm("ush", ptr); break;            \
 703         case 4: __put_user_unaligned_asm("usw", ptr); break;            \
 704         case 8: __PUT_USER_UNALIGNED_DW(ptr); break;                    \
 705         default: __put_user_unaligned_unknown(); break;                 \
 706 } while (0)
 707
 708 #define __put_user_unaligned_nocheck(x,ptr,size)                        \
 709 ({                                                                      \
 710         __typeof__(*(ptr)) __pu_val;                                    \
 711         int __pu_err = 0;                                               \
 712                                                                         \
 713         __pu_val = (x);                                                 \
 714         __put_user_unaligned_common(ptr, size);                         \
 715         __pu_err;                                                       \
 716 })
 717
 718 #define __put_user_unaligned_check(x,ptr,size)                          \
 719 ({                                                                      \
 720         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 721         __typeof__(*(ptr)) __pu_val = (x);                              \
 722         int __pu_err = -EFAULT;                                         \
 723                                                                         \
 724         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size)))          \
 725                 __put_user_unaligned_common(__pu_addr, size);           \
 726                                                                         \
 727         __pu_err;                                                       \
 728 })
 729
 730 #define __put_user_unaligned_asm(insn, ptr)                             \
 731 {                                                                       \
 732         __asm__ __volatile__(                                           \
 733         "1:     " insn "        %z2, %3         # __put_user_unaligned_asm\n" \
 734         "2:                                                     \n"     \
 735         "       .insn                                           \n"     \
 736         "       .section        .fixup,\"ax\"                   \n"     \
 737         "3:     li      %0, %4                                  \n"     \
 738         "       j       2b                                      \n"     \
 739         "       .previous                                       \n"     \
 740         "       .section        __ex_table,\"a\"                \n"     \
 741         "       " __UA_ADDR "   1b, 3b                          \n"     \
 742         "       .previous                                       \n"     \
 743         : "=r" (__pu_err)                                               \
 744         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 745           "i" (-EFAULT));                                               \
 746 }
 747
 748 #define __put_user_unaligned_asm_ll32(ptr)                              \
 749 {                                                                       \
 750         __asm__ __volatile__(                                           \
 751         "1:     sw      %2, (%3)        # __put_user_unaligned_asm_ll32 \n" \
 752         "2:     sw      %D2, 4(%3)                              \n"     \
 753         "3:                                                     \n"     \
 754         "       .insn                                           \n"     \
 755         "       .section        .fixup,\"ax\"                   \n"     \
 756         "4:     li      %0, %4                                  \n"     \
 757         "       j       3b                                      \n"     \
 758         "       .previous                                       \n"     \
 759         "       .section        __ex_table,\"a\"                \n"     \
 760         "       " __UA_ADDR "   1b, 4b                          \n"     \
 761         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 762         "       " __UA_ADDR "   2b, 4b                          \n"     \
 763         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 764         "       .previous"                                              \
 765         : "=r" (__pu_err)                                               \
 766         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 767           "i" (-EFAULT));                                               \
 768 }
 769
 770 extern void __put_user_unaligned_unknown(void);
 771 #endif
 772
 773 /*
 774  * We're generating jump to subroutines which will be outside the range of
 775  * jump instructions
 776  */
 777 #ifdef MODULE
 778 #define __MODULE_JAL(destination)                                       \
 779         ".set\tnoat\n\t"                                                \
 780         __UA_LA "\t$1, " #destination "\n\t"                            \
 781         "jalr\t$1\n\t"                                                  \
 782         ".set\tat\n\t"
 783 #else
 784 #define __MODULE_JAL(destination)                                       \
 785         "jal\t" #destination "\n\t"
 786 #endif
 787
 788 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) &&    \
 789                                               defined(CONFIG_CPU_HAS_PREFETCH))
 790 #define DADDI_SCRATCH "$3"
 791 #else
 792 #define DADDI_SCRATCH "$0"
 793 #endif
 794
 795 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
 796
 797 #ifndef CONFIG_EVA
 798 #define __invoke_copy_to_user(to, from, n)                              \
 799 ({                                                                      \
 800         register void __user *__cu_to_r __asm__("$4");                  \
 801         register const void *__cu_from_r __asm__("$5");                 \
 802         register long __cu_len_r __asm__("$6");                         \
 803                                                                         \
 804         __cu_to_r = (to);                                               \
 805         __cu_from_r = (from);                                           \
 806         __cu_len_r = (n);                                               \
 807         __asm__ __volatile__(                                           \
 808         __MODULE_JAL(__copy_user)                                       \
 809         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 810         :                                                               \
 811         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 812           DADDI_SCRATCH, "memory");                                     \
 813         __cu_len_r;                                                     \
 814 })
 815
 816 #define __invoke_copy_to_kernel(to, from, n)                            \
 817         __invoke_copy_to_user(to, from, n)
 818
 819 #endif
 820
 821 /*
 822  * __copy_to_user: - Copy a block of data into user space, with less checking.
 823  * @to:   Destination address, in user space.
 824  * @from: Source address, in kernel space.
 825  * @n:    Number of bytes to copy.
 826  *
 827  * Context: User context only. This function may sleep if pagefaults are
 828  *          enabled.
 829  *
 830  * Copy data from kernel space to user space.  Caller must check
 831  * the specified block with access_ok() before calling this function.
 832  *
 833  * Returns number of bytes that could not be copied.
 834  * On success, this will be zero.
 835  */
 836 #define __copy_to_user(to, from, n)                                     \
 837 ({                                                                      \
 838         void __user *__cu_to;                                           \
 839         const void *__cu_from;                                          \
 840         long __cu_len;                                                  \
 841                                                                         \
 842         __cu_to = (to);                                                 \
 843         __cu_from = (from);                                             \
 844         __cu_len = (n);                                                 \
 845         might_fault();                                                  \
 846         if (segment_eq(get_fs(), get_ds()))                             \
 847                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 848                                                    __cu_len);           \
 849         else                                                            \
 850                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 851                                                  __cu_len);             \
 852         __cu_len;                                                       \
 853 })
 854
 855 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
 856
 857 #define __copy_to_user_inatomic(to, from, n)                            \
 858 ({                                                                      \
 859         void __user *__cu_to;                                           \
 860         const void *__cu_from;                                          \
 861         long __cu_len;                                                  \
 862                                                                         \
 863         __cu_to = (to);                                                 \
 864         __cu_from = (from);                                             \
 865         __cu_len = (n);                                                 \
 866         if (segment_eq(get_fs(), get_ds()))                             \
 867                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 868                                                    __cu_len);           \
 869         else                                                            \
 870                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 871                                                  __cu_len);             \
 872         __cu_len;                                                       \
 873 })
 874
 875 #define __copy_from_user_inatomic(to, from, n)                          \
 876 ({                                                                      \
 877         void *__cu_to;                                                  \
 878         const void __user *__cu_from;                                   \
 879         long __cu_len;                                                  \
 880                                                                         \
 881         __cu_to = (to);                                                 \
 882         __cu_from = (from);                                             \
 883         __cu_len = (n);                                                 \
 884         if (segment_eq(get_fs(), get_ds()))                             \
 885                 __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to,  \
 886                                                               __cu_from,\
 887                                                               __cu_len);\
 888         else                                                            \
 889                 __cu_len = __invoke_copy_from_user_inatomic(__cu_to,    \
 890                                                             __cu_from,  \
 891                                                             __cu_len);  \
 892         __cu_len;                                                       \
 893 })
 894
 895 /*
 896  * copy_to_user: - Copy a block of data into user space.
 897  * @to:   Destination address, in user space.
 898  * @from: Source address, in kernel space.
 899  * @n:    Number of bytes to copy.
 900  *
 901  * Context: User context only. This function may sleep if pagefaults are
 902  *          enabled.
 903  *
 904  * Copy data from kernel space to user space.
 905  *
 906  * Returns number of bytes that could not be copied.
 907  * On success, this will be zero.
 908  */
 909 #define copy_to_user(to, from, n)                                       \
 910 ({                                                                      \
 911         void __user *__cu_to;                                           \
 912         const void *__cu_from;                                          \
 913         long __cu_len;                                                  \
 914                                                                         \
 915         __cu_to = (to);                                                 \
 916         __cu_from = (from);                                             \
 917         __cu_len = (n);                                                 \
 918         if (segment_eq(get_fs(), get_ds())) {                           \
 919                 __cu_len = __invoke_copy_to_kernel(__cu_to,             \
 920                                                    __cu_from,           \
 921                                                    __cu_len);           \
 922         } else {                                                        \
 923                 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) {       \
 924                         might_fault();                                  \
 925                         __cu_len = __invoke_copy_to_user(__cu_to,       \
 926                                                          __cu_from,     \
 927                                                          __cu_len);     \
 928                 }                                                       \
 929         }                                                               \
 930         __cu_len;                                                       \
 931 })
 932
 933 #ifndef CONFIG_EVA
 934
 935 #define __invoke_copy_from_user(to, from, n)                            \
 936 ({                                                                      \
 937         register void *__cu_to_r __asm__("$4");                         \
 938         register const void __user *__cu_from_r __asm__("$5");          \
 939         register long __cu_len_r __asm__("$6");                         \
 940                                                                         \
 941         __cu_to_r = (to);                                               \
 942         __cu_from_r = (from);                                           \
 943         __cu_len_r = (n);                                               \
 944         __asm__ __volatile__(                                           \
 945         ".set\tnoreorder\n\t"                                           \
 946         __MODULE_JAL(__copy_user)                                       \
 947         ".set\tnoat\n\t"                                                \
 948         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 949         ".set\tat\n\t"                                                  \
 950         ".set\treorder"                                                 \
 951         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 952         :                                                               \
 953         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 954           DADDI_SCRATCH, "memory");                                     \
 955         __cu_len_r;                                                     \
 956 })
 957
 958 #define __invoke_copy_from_kernel(to, from, n)                          \
 959         __invoke_copy_from_user(to, from, n)
 960
 961 /* For userland <-> userland operations */
 962 #define ___invoke_copy_in_user(to, from, n)                             \
 963         __invoke_copy_from_user(to, from, n)
 964
 965 /* For kernel <-> kernel operations */
 966 #define ___invoke_copy_in_kernel(to, from, n)                           \
 967         __invoke_copy_from_user(to, from, n)
 968
 969 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
 970 ({                                                                      \
 971         register void *__cu_to_r __asm__("$4");                         \
 972         register const void __user *__cu_from_r __asm__("$5");          \
 973         register long __cu_len_r __asm__("$6");                         \
 974                                                                         \
 975         __cu_to_r = (to);                                               \
 976         __cu_from_r = (from);                                           \
 977         __cu_len_r = (n);                                               \
 978         __asm__ __volatile__(                                           \
 979         ".set\tnoreorder\n\t"                                           \
 980         __MODULE_JAL(__copy_user_inatomic)                              \
 981         ".set\tnoat\n\t"                                                \
 982         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 983         ".set\tat\n\t"                                                  \
 984         ".set\treorder"                                                 \
 985         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 986         :                                                               \
 987         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 988           DADDI_SCRATCH, "memory");                                     \
 989         __cu_len_r;                                                     \
 990 })
 991
 992 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
 993         __invoke_copy_from_user_inatomic(to, from, n)                   \
 994
 995 #else
 996
 997 /* EVA specific functions */
 998
 999 extern size_t __copy_user_inatomic_eva(void *__to, const void *__from,
1000                                        size_t __n);
1001 extern size_t __copy_from_user_eva(void *__to, const void *__from,
1002                                    size_t __n);
1003 extern size_t __copy_to_user_eva(void *__to, const void *__from,
1004                                  size_t __n);
1005 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
1006
1007 #define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr)      \
1008 ({                                                                      \
1009         register void *__cu_to_r __asm__("$4");                         \
1010         register const void __user *__cu_from_r __asm__("$5");          \
1011         register long __cu_len_r __asm__("$6");                         \
1012                                                                         \
1013         __cu_to_r = (to);                                               \
1014         __cu_from_r = (from);                                           \
1015         __cu_len_r = (n);                                               \
1016         __asm__ __volatile__(                                           \
1017         ".set\tnoreorder\n\t"                                           \
1018         __MODULE_JAL(func_ptr)                                          \
1019         ".set\tnoat\n\t"                                                \
1020         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
1021         ".set\tat\n\t"                                                  \
1022         ".set\treorder"                                                 \
1023         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1024         :                                                               \
1025         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1026           DADDI_SCRATCH, "memory");                                     \
1027         __cu_len_r;                                                     \
1028 })
1029
1030 #define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr)        \
1031 ({                                                                      \
1032         register void *__cu_to_r __asm__("$4");                         \
1033         register const void __user *__cu_from_r __asm__("$5");          \
1034         register long __cu_len_r __asm__("$6");                         \
1035                                                                         \
1036         __cu_to_r = (to);                                               \
1037         __cu_from_r = (from);                                           \
1038         __cu_len_r = (n);                                               \
1039         __asm__ __volatile__(                                           \
1040         __MODULE_JAL(func_ptr)                                          \
1041         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1042         :                                                               \
1043         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1044           DADDI_SCRATCH, "memory");                                     \
1045         __cu_len_r;                                                     \
1046 })
1047
1048 /*
1049  * Source or destination address is in userland. We need to go through
1050  * the TLB
1051  */
1052 #define __invoke_copy_from_user(to, from, n)                            \
1053         __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva)
1054
1055 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
1056         __invoke_copy_from_user_eva_generic(to, from, n,                \
1057                                             __copy_user_inatomic_eva)
1058
1059 #define __invoke_copy_to_user(to, from, n)                              \
1060         __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva)
1061
1062 #define ___invoke_copy_in_user(to, from, n)                             \
1063         __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva)
1064
1065 /*
1066  * Source or destination address in the kernel. We are not going through
1067  * the TLB
1068  */
1069 #define __invoke_copy_from_kernel(to, from, n)                          \
1070         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1071
1072 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
1073         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic)
1074
1075 #define __invoke_copy_to_kernel(to, from, n)                            \
1076         __invoke_copy_to_user_eva_generic(to, from, n, __copy_user)
1077
1078 #define ___invoke_copy_in_kernel(to, from, n)                           \
1079         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1080
1081 #endif /* CONFIG_EVA */
1082
1083 /*
1084  * __copy_from_user: - Copy a block of data from user space, with less checking.
1085  * @to:   Destination address, in kernel space.
1086  * @from: Source address, in user space.
1087  * @n:    Number of bytes to copy.
1088  *
1089  * Context: User context only. This function may sleep if pagefaults are
1090  *          enabled.
1091  *
1092  * Copy data from user space to kernel space.  Caller must check
1093  * the specified block with access_ok() before calling this function.
1094  *
1095  * Returns number of bytes that could not be copied.
1096  * On success, this will be zero.
1097  *
1098  * If some data could not be copied, this function will pad the copied
1099  * data to the requested size using zero bytes.
1100  */
1101 #define __copy_from_user(to, from, n)                                   \
1102 ({                                                                      \
1103         void *__cu_to;                                                  \
1104         const void __user *__cu_from;                                   \
1105         long __cu_len;                                                  \
1106                                                                         \
1107         __cu_to = (to);                                                 \
1108         __cu_from = (from);                                             \
1109         __cu_len = (n);                                                 \
1110         might_fault();                                                  \
1111         __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,          \
1112                                            __cu_len);                   \
1113         __cu_len;                                                       \
1114 })
1115
1116 /*
1117  * copy_from_user: - Copy a block of data from user space.
1118  * @to:   Destination address, in kernel space.
1119  * @from: Source address, in user space.
1120  * @n:    Number of bytes to copy.
1121  *
1122  * Context: User context only. This function may sleep if pagefaults are
1123  *          enabled.
1124  *
1125  * Copy data from user space to kernel space.
1126  *
1127  * Returns number of bytes that could not be copied.
1128  * On success, this will be zero.
1129  *
1130  * If some data could not be copied, this function will pad the copied
1131  * data to the requested size using zero bytes.
1132  */
1133 #define copy_from_user(to, from, n)                                     \
1134 ({                                                                      \
1135         void *__cu_to;                                                  \
1136         const void __user *__cu_from;                                   \
1137         long __cu_len;                                                  \
1138                                                                         \
1139         __cu_to = (to);                                                 \
1140         __cu_from = (from);                                             \
1141         __cu_len = (n);                                                 \
1142         if (segment_eq(get_fs(), get_ds())) {                           \
1143                 __cu_len = __invoke_copy_from_kernel(__cu_to,           \
1144                                                      __cu_from,         \
1145                                                      __cu_len);         \
1146         } else {                                                        \
1147                 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) {      \
1148                         might_fault();                                  \
1149                         __cu_len = __invoke_copy_from_user(__cu_to,     \
1150                                                            __cu_from,   \
1151                                                            __cu_len);   \
1152                 }                                                       \
1153         }                                                               \
1154         __cu_len;                                                       \
1155 })
1156
1157 #define __copy_in_user(to, from, n)                                     \
1158 ({                                                                      \
1159         void __user *__cu_to;                                           \
1160         const void __user *__cu_from;                                   \
1161         long __cu_len;                                                  \
1162                                                                         \
1163         __cu_to = (to);                                                 \
1164         __cu_from = (from);                                             \
1165         __cu_len = (n);                                                 \
1166         if (segment_eq(get_fs(), get_ds())) {                           \
1167                 __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \
1168                                                     __cu_len);          \
1169         } else {                                                        \
1170                 might_fault();                                          \
1171                 __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from,   \
1172                                                   __cu_len);            \
1173         }                                                               \
1174         __cu_len;                                                       \
1175 })
1176
1177 #define copy_in_user(to, from, n)                                       \
1178 ({                                                                      \
1179         void __user *__cu_to;                                           \
1180         const void __user *__cu_from;                                   \
1181         long __cu_len;                                                  \
1182                                                                         \
1183         __cu_to = (to);                                                 \
1184         __cu_from = (from);                                             \
1185         __cu_len = (n);                                                 \
1186         if (segment_eq(get_fs(), get_ds())) {                           \
1187                 __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from,  \
1188                                                     __cu_len);          \
1189         } else {                                                        \
1190                 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\
1191                            access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\
1192                         might_fault();                                  \
1193                         __cu_len = ___invoke_copy_in_user(__cu_to,      \
1194                                                           __cu_from,    \
1195                                                           __cu_len);    \
1196                 }                                                       \
1197         }                                                               \
1198         __cu_len;                                                       \
1199 })
1200
1201 /*
1202  * __clear_user: - Zero a block of memory in user space, with less checking.
1203  * @to:   Destination address, in user space.
1204  * @n:    Number of bytes to zero.
1205  *
1206  * Zero a block of memory in user space.  Caller must check
1207  * the specified block with access_ok() before calling this function.
1208  *
1209  * Returns number of bytes that could not be cleared.
1210  * On success, this will be zero.
1211  */
1212 static inline __kernel_size_t
1213 __clear_user(void __user *addr, __kernel_size_t size)
1214 {
1215         __kernel_size_t res;
1216
1217         might_fault();
1218         __asm__ __volatile__(
1219                 "move\t$4, %1\n\t"
1220                 "move\t$5, $0\n\t"
1221                 "move\t$6, %2\n\t"
1222                 __MODULE_JAL(__bzero)
1223                 "move\t%0, $6"
1224                 : "=r" (res)
1225                 : "r" (addr), "r" (size)
1226                 : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31");
1227
1228         return res;
1229 }
1230
1231 #define clear_user(addr,n)                                              \
1232 ({                                                                      \
1233         void __user * __cl_addr = (addr);                               \
1234         unsigned long __cl_size = (n);                                  \
1235         if (__cl_size && access_ok(VERIFY_WRITE,                        \
1236                                         __cl_addr, __cl_size))          \
1237                 __cl_size = __clear_user(__cl_addr, __cl_size);         \
1238         __cl_size;                                                      \
1239 })
1240
1241 /*
1242  * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
1243  * @dst:   Destination address, in kernel space.  This buffer must be at
1244  *         least @count bytes long.
1245  * @src:   Source address, in user space.
1246  * @count: Maximum number of bytes to copy, including the trailing NUL.
1247  *
1248  * Copies a NUL-terminated string from userspace to kernel space.
1249  * Caller must check the specified block with access_ok() before calling
1250  * this function.
1251  *
1252  * On success, returns the length of the string (not including the trailing
1253  * NUL).
1254  *
1255  * If access to userspace fails, returns -EFAULT (some data may have been
1256  * copied).
1257  *
1258  * If @count is smaller than the length of the string, copies @count bytes
1259  * and returns @count.
1260  */
1261 static inline long
1262 __strncpy_from_user(char *__to, const char __user *__from, long __len)
1263 {
1264         long res;
1265
1266         if (segment_eq(get_fs(), get_ds())) {
1267                 __asm__ __volatile__(
1268                         "move\t$4, %1\n\t"
1269                         "move\t$5, %2\n\t"
1270                         "move\t$6, %3\n\t"
1271                         __MODULE_JAL(__strncpy_from_kernel_nocheck_asm)
1272                         "move\t%0, $2"
1273                         : "=r" (res)
1274                         : "r" (__to), "r" (__from), "r" (__len)
1275                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1276         } else {
1277                 might_fault();
1278                 __asm__ __volatile__(
1279                         "move\t$4, %1\n\t"
1280                         "move\t$5, %2\n\t"
1281                         "move\t$6, %3\n\t"
1282                         __MODULE_JAL(__strncpy_from_user_nocheck_asm)
1283                         "move\t%0, $2"
1284                         : "=r" (res)
1285                         : "r" (__to), "r" (__from), "r" (__len)
1286                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1287         }
1288
1289         return res;
1290 }
1291
1292 /*
1293  * strncpy_from_user: - Copy a NUL terminated string from userspace.
1294  * @dst:   Destination address, in kernel space.  This buffer must be at
1295  *         least @count bytes long.
1296  * @src:   Source address, in user space.
1297  * @count: Maximum number of bytes to copy, including the trailing NUL.
1298  *
1299  * Copies a NUL-terminated string from userspace to kernel space.
1300  *
1301  * On success, returns the length of the string (not including the trailing
1302  * NUL).
1303  *
1304  * If access to userspace fails, returns -EFAULT (some data may have been
1305  * copied).
1306  *
1307  * If @count is smaller than the length of the string, copies @count bytes
1308  * and returns @count.
1309  */
1310 static inline long
1311 strncpy_from_user(char *__to, const char __user *__from, long __len)
1312 {
1313         long res;
1314
1315         if (segment_eq(get_fs(), get_ds())) {
1316                 __asm__ __volatile__(
1317                         "move\t$4, %1\n\t"
1318                         "move\t$5, %2\n\t"
1319                         "move\t$6, %3\n\t"
1320                         __MODULE_JAL(__strncpy_from_kernel_asm)
1321                         "move\t%0, $2"
1322                         : "=r" (res)
1323                         : "r" (__to), "r" (__from), "r" (__len)
1324                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1325         } else {
1326                 might_fault();
1327                 __asm__ __volatile__(
1328                         "move\t$4, %1\n\t"
1329                         "move\t$5, %2\n\t"
1330                         "move\t$6, %3\n\t"
1331                         __MODULE_JAL(__strncpy_from_user_asm)
1332                         "move\t%0, $2"
1333                         : "=r" (res)
1334                         : "r" (__to), "r" (__from), "r" (__len)
1335                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1336         }
1337
1338         return res;
1339 }
1340
1341 /*
1342  * strlen_user: - Get the size of a string in user space.
1343  * @str: The string to measure.
1344  *
1345  * Context: User context only. This function may sleep if pagefaults are
1346  *          enabled.
1347  *
1348  * Get the size of a NUL-terminated string in user space.
1349  *
1350  * Returns the size of the string INCLUDING the terminating NUL.
1351  * On exception, returns 0.
1352  *
1353  * If there is a limit on the length of a valid string, you may wish to
1354  * consider using strnlen_user() instead.
1355  */
1356 static inline long strlen_user(const char __user *s)
1357 {
1358         long res;
1359
1360         if (segment_eq(get_fs(), get_ds())) {
1361                 __asm__ __volatile__(
1362                         "move\t$4, %1\n\t"
1363                         __MODULE_JAL(__strlen_kernel_asm)
1364                         "move\t%0, $2"
1365                         : "=r" (res)
1366                         : "r" (s)
1367                         : "$2", "$4", __UA_t0, "$31");
1368         } else {
1369                 might_fault();
1370                 __asm__ __volatile__(
1371                         "move\t$4, %1\n\t"
1372                         __MODULE_JAL(__strlen_kernel_asm)
1373                         "move\t%0, $2"
1374                         : "=r" (res)
1375                         : "r" (s)
1376                         : "$2", "$4", __UA_t0, "$31");
1377         }
1378
1379         return res;
1380 }
1381
1382 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */
1383 static inline long __strnlen_user(const char __user *s, long n)
1384 {
1385         long res;
1386
1387         if (segment_eq(get_fs(), get_ds())) {
1388                 __asm__ __volatile__(
1389                         "move\t$4, %1\n\t"
1390                         "move\t$5, %2\n\t"
1391                         __MODULE_JAL(__strnlen_kernel_nocheck_asm)
1392                         "move\t%0, $2"
1393                         : "=r" (res)
1394                         : "r" (s), "r" (n)
1395                         : "$2", "$4", "$5", __UA_t0, "$31");
1396         } else {
1397                 might_fault();
1398                 __asm__ __volatile__(
1399                         "move\t$4, %1\n\t"
1400                         "move\t$5, %2\n\t"
1401                         __MODULE_JAL(__strnlen_user_nocheck_asm)
1402                         "move\t%0, $2"
1403                         : "=r" (res)
1404                         : "r" (s), "r" (n)
1405                         : "$2", "$4", "$5", __UA_t0, "$31");
1406         }
1407
1408         return res;
1409 }
1410
1411 /*
1412  * strnlen_user: - Get the size of a string in user space.
1413  * @str: The string to measure.
1414  *
1415  * Context: User context only. This function may sleep if pagefaults are
1416  *          enabled.
1417  *
1418  * Get the size of a NUL-terminated string in user space.
1419  *
1420  * Returns the size of the string INCLUDING the terminating NUL.
1421  * On exception, returns 0.
1422  * If the string is too long, returns a value greater than @n.
1423  */
1424 static inline long strnlen_user(const char __user *s, long n)
1425 {
1426         long res;
1427
1428         might_fault();
1429         if (segment_eq(get_fs(), get_ds())) {
1430                 __asm__ __volatile__(
1431                         "move\t$4, %1\n\t"
1432                         "move\t$5, %2\n\t"
1433                         __MODULE_JAL(__strnlen_kernel_asm)
1434                         "move\t%0, $2"
1435                         : "=r" (res)
1436                         : "r" (s), "r" (n)
1437                         : "$2", "$4", "$5", __UA_t0, "$31");
1438         } else {
1439                 __asm__ __volatile__(
1440                         "move\t$4, %1\n\t"
1441                         "move\t$5, %2\n\t"
1442                         __MODULE_JAL(__strnlen_user_asm)
1443                         "move\t%0, $2"
1444                         : "=r" (res)
1445                         : "r" (s), "r" (n)
1446                         : "$2", "$4", "$5", __UA_t0, "$31");
1447         }
1448
1449         return res;
1450 }
1451
1452 struct exception_table_entry
1453 {
1454         unsigned long insn;
1455         unsigned long nextinsn;
1456 };
1457
1458 extern int fixup_exception(struct pt_regs *regs);
1459
1460 #endif /* _ASM_UACCESS_H */