arch/mips/include/asm/uaccess.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  * Copyright (C) 2007  Maciej W. Rozycki
   9  * Copyright (C) 2014, Imagination Technologies Ltd.
  10  */
  11 #ifndef _ASM_UACCESS_H
  12 #define _ASM_UACCESS_H
  13
  14 #include <linux/kernel.h>
  15 #include <linux/errno.h>
  16 #include <linux/thread_info.h>
  17 #include <asm/asm-eva.h>
  18
  19 /*
  20  * The fs value determines whether argument validity checking should be
  21  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  22  * get_fs() == KERNEL_DS, checking is bypassed.
  23  *
  24  * For historical reasons, these macros are grossly misnamed.
  25  */
  26 #ifdef CONFIG_32BIT
  27
  28 #ifdef CONFIG_KVM_GUEST
  29 #define __UA_LIMIT 0x40000000UL
  30 #else
  31 #define __UA_LIMIT 0x80000000UL
  32 #endif
  33
  34 #define __UA_ADDR       ".word"
  35 #define __UA_LA         "la"
  36 #define __UA_ADDU       "addu"
  37 #define __UA_t0         "$8"
  38 #define __UA_t1         "$9"
  39
  40 #endif /* CONFIG_32BIT */
  41
  42 #ifdef CONFIG_64BIT
  43
  44 extern u64 __ua_limit;
  45
  46 #define __UA_LIMIT      __ua_limit
  47
  48 #define __UA_ADDR       ".dword"
  49 #define __UA_LA         "dla"
  50 #define __UA_ADDU       "daddu"
  51 #define __UA_t0         "$12"
  52 #define __UA_t1         "$13"
  53
  54 #endif /* CONFIG_64BIT */
  55
  56 /*
  57  * USER_DS is a bitmask that has the bits set that may not be set in a valid
  58  * userspace address.  Note that we limit 32-bit userspace to 0x7fff8000 but
  59  * the arithmetic we're doing only works if the limit is a power of two, so
  60  * we use 0x80000000 here on 32-bit kernels.  If a process passes an invalid
  61  * address in this range it's the process's problem, not ours :-)
  62  */
  63
  64 #ifdef CONFIG_KVM_GUEST
  65 #define KERNEL_DS       ((mm_segment_t) { 0x80000000UL })
  66 #define USER_DS         ((mm_segment_t) { 0xC0000000UL })
  67 #else
  68 #define KERNEL_DS       ((mm_segment_t) { 0UL })
  69 #define USER_DS         ((mm_segment_t) { __UA_LIMIT })
  70 #endif
  71
  72 #define VERIFY_READ    0
  73 #define VERIFY_WRITE   1
  74
  75 #define get_ds()        (KERNEL_DS)
  76 #define get_fs()        (current_thread_info()->addr_limit)
  77 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
  78
  79 #define segment_eq(a, b)        ((a).seg == (b).seg)
  80
  81
  82 /*
  83  * Is a address valid? This does a straighforward calculation rather
  84  * than tests.
  85  *
  86  * Address valid if:
  87  *  - "addr" doesn't have any high-bits set
  88  *  - AND "size" doesn't have any high-bits set
  89  *  - AND "addr+size" doesn't have any high-bits set
  90  *  - OR we are in kernel mode.
  91  *
  92  * __ua_size() is a trick to avoid runtime checking of positive constant
  93  * sizes; for those we already know at compile time that the size is ok.
  94  */
  95 #define __ua_size(size)                                                 \
  96         ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
  97
  98 /*
  99  * access_ok: - Checks if a user space pointer is valid
 100  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
 101  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
 102  *        to write to a block, it is always safe to read from it.
 103  * @addr: User space pointer to start of block to check
 104  * @size: Size of block to check
 105  *
 106  * Context: User context only.  This function may sleep.
 107  *
 108  * Checks if a pointer to a block of memory in user space is valid.
 109  *
 110  * Returns true (nonzero) if the memory block may be valid, false (zero)
 111  * if it is definitely invalid.
 112  *
 113  * Note that, depending on architecture, this function probably just
 114  * checks that the pointer is in the user space range - after calling
 115  * this function, memory access functions may still return -EFAULT.
 116  */
 117
 118 #define __access_mask get_fs().seg
 119
 120 #define __access_ok(addr, size, mask)                                   \
 121 ({                                                                      \
 122         unsigned long __addr = (unsigned long) (addr);                  \
 123         unsigned long __size = size;                                    \
 124         unsigned long __mask = mask;                                    \
 125         unsigned long __ok;                                             \
 126                                                                         \
 127         __chk_user_ptr(addr);                                           \
 128         __ok = (signed long)(__mask & (__addr | (__addr + __size) |     \
 129                 __ua_size(__size)));                                    \
 130         __ok == 0;                                                      \
 131 })
 132
 133 #define access_ok(type, addr, size)                                     \
 134         likely(__access_ok((addr), (size), __access_mask))
 135
 136 /*
 137  * put_user: - Write a simple value into user space.
 138  * @x:   Value to copy to user space.
 139  * @ptr: Destination address, in user space.
 140  *
 141  * Context: User context only.  This function may sleep.
 142  *
 143  * This macro copies a single simple value from kernel space to user
 144  * space.  It supports simple types like char and int, but not larger
 145  * data types like structures or arrays.
 146  *
 147  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 148  * to the result of dereferencing @ptr.
 149  *
 150  * Returns zero on success, or -EFAULT on error.
 151  */
 152 #define put_user(x,ptr) \
 153         __put_user_check((x), (ptr), sizeof(*(ptr)))
 154
 155 /*
 156  * get_user: - Get a simple variable from user space.
 157  * @x:   Variable to store result.
 158  * @ptr: Source address, in user space.
 159  *
 160  * Context: User context only.  This function may sleep.
 161  *
 162  * This macro copies a single simple variable from user space to kernel
 163  * space.  It supports simple types like char and int, but not larger
 164  * data types like structures or arrays.
 165  *
 166  * @ptr must have pointer-to-simple-variable type, and the result of
 167  * dereferencing @ptr must be assignable to @x without a cast.
 168  *
 169  * Returns zero on success, or -EFAULT on error.
 170  * On error, the variable @x is set to zero.
 171  */
 172 #define get_user(x,ptr) \
 173         __get_user_check((x), (ptr), sizeof(*(ptr)))
 174
 175 /*
 176  * __put_user: - Write a simple value into user space, with less checking.
 177  * @x:   Value to copy to user space.
 178  * @ptr: Destination address, in user space.
 179  *
 180  * Context: User context only.  This function may sleep.
 181  *
 182  * This macro copies a single simple value from kernel space to user
 183  * space.  It supports simple types like char and int, but not larger
 184  * data types like structures or arrays.
 185  *
 186  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 187  * to the result of dereferencing @ptr.
 188  *
 189  * Caller must check the pointer with access_ok() before calling this
 190  * function.
 191  *
 192  * Returns zero on success, or -EFAULT on error.
 193  */
 194 #define __put_user(x,ptr) \
 195         __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
 196
 197 /*
 198  * __get_user: - Get a simple variable from user space, with less checking.
 199  * @x:   Variable to store result.
 200  * @ptr: Source address, in user space.
 201  *
 202  * Context: User context only.  This function may sleep.
 203  *
 204  * This macro copies a single simple variable from user space to kernel
 205  * space.  It supports simple types like char and int, but not larger
 206  * data types like structures or arrays.
 207  *
 208  * @ptr must have pointer-to-simple-variable type, and the result of
 209  * dereferencing @ptr must be assignable to @x without a cast.
 210  *
 211  * Caller must check the pointer with access_ok() before calling this
 212  * function.
 213  *
 214  * Returns zero on success, or -EFAULT on error.
 215  * On error, the variable @x is set to zero.
 216  */
 217 #define __get_user(x,ptr) \
 218         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
 219
 220 struct __large_struct { unsigned long buf[100]; };
 221 #define __m(x) (*(struct __large_struct __user *)(x))
 222
 223 /*
 224  * Yuck.  We need two variants, one for 64bit operation and one
 225  * for 32 bit mode and old iron.
 226  */
 227 #ifndef CONFIG_EVA
 228 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr)
 229 #else
 230 /*
 231  * Kernel specific functions for EVA. We need to use normal load instructions
 232  * to read data from kernel when operating in EVA mode. We use these macros to
 233  * avoid redefining __get_user_asm for EVA.
 234  */
 235 #undef _loadd
 236 #undef _loadw
 237 #undef _loadh
 238 #undef _loadb
 239 #ifdef CONFIG_32BIT
 240 #define _loadd                  _loadw
 241 #else
 242 #define _loadd(reg, addr)       "ld " reg ", " addr
 243 #endif
 244 #define _loadw(reg, addr)       "lw " reg ", " addr
 245 #define _loadh(reg, addr)       "lh " reg ", " addr
 246 #define _loadb(reg, addr)       "lb " reg ", " addr
 247
 248 #define __get_kernel_common(val, size, ptr)                             \
 249 do {                                                                    \
 250         switch (size) {                                                 \
 251         case 1: __get_data_asm(val, _loadb, ptr); break;                \
 252         case 2: __get_data_asm(val, _loadh, ptr); break;                \
 253         case 4: __get_data_asm(val, _loadw, ptr); break;                \
 254         case 8: __GET_DW(val, _loadd, ptr); break;                      \
 255         default: __get_user_unknown(); break;                           \
 256         }                                                               \
 257 } while (0)
 258 #endif
 259
 260 #ifdef CONFIG_32BIT
 261 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr)
 262 #endif
 263 #ifdef CONFIG_64BIT
 264 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr)
 265 #endif
 266
 267 extern void __get_user_unknown(void);
 268
 269 #define __get_user_common(val, size, ptr)                               \
 270 do {                                                                    \
 271         switch (size) {                                                 \
 272         case 1: __get_data_asm(val, user_lb, ptr); break;               \
 273         case 2: __get_data_asm(val, user_lh, ptr); break;               \
 274         case 4: __get_data_asm(val, user_lw, ptr); break;               \
 275         case 8: __GET_DW(val, user_ld, ptr); break;                     \
 276         default: __get_user_unknown(); break;                           \
 277         }                                                               \
 278 } while (0)
 279
 280 #define __get_user_nocheck(x, ptr, size)                                \
 281 ({                                                                      \
 282         int __gu_err;                                                   \
 283                                                                         \
 284         if (segment_eq(get_fs(), get_ds())) {                           \
 285                 __get_kernel_common((x), size, ptr);                    \
 286         } else {                                                        \
 287                 __chk_user_ptr(ptr);                                    \
 288                 __get_user_common((x), size, ptr);                      \
 289         }                                                               \
 290         __gu_err;                                                       \
 291 })
 292
 293 #define __get_user_check(x, ptr, size)                                  \
 294 ({                                                                      \
 295         int __gu_err = -EFAULT;                                         \
 296         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 297                                                                         \
 298         might_fault();                                                  \
 299         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size))) {          \
 300                 if (segment_eq(get_fs(), get_ds()))                     \
 301                         __get_kernel_common((x), size, __gu_ptr);       \
 302                 else                                                    \
 303                         __get_user_common((x), size, __gu_ptr);         \
 304         } else                                                          \
 305                 (x) = 0;                                                \
 306                                                                         \
 307         __gu_err;                                                       \
 308 })
 309
 310 #define __get_data_asm(val, insn, addr)                                 \
 311 {                                                                       \
 312         long __gu_tmp;                                                  \
 313                                                                         \
 314         __asm__ __volatile__(                                           \
 315         "1:     "insn("%1", "%3")"                              \n"     \
 316         "2:                                                     \n"     \
 317         "       .insn                                           \n"     \
 318         "       .section .fixup,\"ax\"                          \n"     \
 319         "3:     li      %0, %4                                  \n"     \
 320         "       move    %1, $0                                  \n"     \
 321         "       j       2b                                      \n"     \
 322         "       .previous                                       \n"     \
 323         "       .section __ex_table,\"a\"                       \n"     \
 324         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 325         "       .previous                                       \n"     \
 326         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 327         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 328                                                                         \
 329         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 330 }
 331
 332 /*
 333  * Get a long long 64 using 32 bit registers.
 334  */
 335 #define __get_data_asm_ll32(val, insn, addr)                            \
 336 {                                                                       \
 337         union {                                                         \
 338                 unsigned long long      l;                              \
 339                 __typeof__(*(addr))     t;                              \
 340         } __gu_tmp;                                                     \
 341                                                                         \
 342         __asm__ __volatile__(                                           \
 343         "1:     " insn("%1", "(%3)")"                           \n"     \
 344         "2:     " insn("%D1", "4(%3)")"                         \n"     \
 345         "3:                                                     \n"     \
 346         "       .insn                                           \n"     \
 347         "       .section        .fixup,\"ax\"                   \n"     \
 348         "4:     li      %0, %4                                  \n"     \
 349         "       move    %1, $0                                  \n"     \
 350         "       move    %D1, $0                                 \n"     \
 351         "       j       3b                                      \n"     \
 352         "       .previous                                       \n"     \
 353         "       .section        __ex_table,\"a\"                \n"     \
 354         "       " __UA_ADDR "   1b, 4b                          \n"     \
 355         "       " __UA_ADDR "   2b, 4b                          \n"     \
 356         "       .previous                                       \n"     \
 357         : "=r" (__gu_err), "=&r" (__gu_tmp.l)                           \
 358         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 359                                                                         \
 360         (val) = __gu_tmp.t;                                             \
 361 }
 362
 363 #ifndef CONFIG_EVA
 364 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size)
 365 #else
 366 /*
 367  * Kernel specific functions for EVA. We need to use normal load instructions
 368  * to read data from kernel when operating in EVA mode. We use these macros to
 369  * avoid redefining __get_data_asm for EVA.
 370  */
 371 #undef _stored
 372 #undef _storew
 373 #undef _storeh
 374 #undef _storeb
 375 #ifdef CONFIG_32BIT
 376 #define _stored                 _storew
 377 #else
 378 #define _stored(reg, addr)      "ld " reg ", " addr
 379 #endif
 380
 381 #define _storew(reg, addr)      "sw " reg ", " addr
 382 #define _storeh(reg, addr)      "sh " reg ", " addr
 383 #define _storeb(reg, addr)      "sb " reg ", " addr
 384
 385 #define __put_kernel_common(ptr, size)                                  \
 386 do {                                                                    \
 387         switch (size) {                                                 \
 388         case 1: __put_data_asm(_storeb, ptr); break;                    \
 389         case 2: __put_data_asm(_storeh, ptr); break;                    \
 390         case 4: __put_data_asm(_storew, ptr); break;                    \
 391         case 8: __PUT_DW(_stored, ptr); break;                          \
 392         default: __put_user_unknown(); break;                           \
 393         }                                                               \
 394 } while(0)
 395 #endif
 396
 397 /*
 398  * Yuck.  We need two variants, one for 64bit operation and one
 399  * for 32 bit mode and old iron.
 400  */
 401 #ifdef CONFIG_32BIT
 402 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr)
 403 #endif
 404 #ifdef CONFIG_64BIT
 405 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr)
 406 #endif
 407
 408 #define __put_user_common(ptr, size)                                    \
 409 do {                                                                    \
 410         switch (size) {                                                 \
 411         case 1: __put_data_asm(user_sb, ptr); break;                    \
 412         case 2: __put_data_asm(user_sh, ptr); break;                    \
 413         case 4: __put_data_asm(user_sw, ptr); break;                    \
 414         case 8: __PUT_DW(user_sd, ptr); break;                          \
 415         default: __put_user_unknown(); break;                           \
 416         }                                                               \
 417 } while (0)
 418
 419 #define __put_user_nocheck(x, ptr, size)                                \
 420 ({                                                                      \
 421         __typeof__(*(ptr)) __pu_val;                                    \
 422         int __pu_err = 0;                                               \
 423                                                                         \
 424         __pu_val = (x);                                                 \
 425         if (segment_eq(get_fs(), get_ds())) {                           \
 426                 __put_kernel_common(ptr, size);                         \
 427         } else {                                                        \
 428                 __chk_user_ptr(ptr);                                    \
 429                 __put_user_common(ptr, size);                           \
 430         }                                                               \
 431         __pu_err;                                                       \
 432 })
 433
 434 #define __put_user_check(x, ptr, size)                                  \
 435 ({                                                                      \
 436         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 437         __typeof__(*(ptr)) __pu_val = (x);                              \
 438         int __pu_err = -EFAULT;                                         \
 439                                                                         \
 440         might_fault();                                                  \
 441         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size))) {        \
 442                 if (segment_eq(get_fs(), get_ds()))                     \
 443                         __put_kernel_common(__pu_addr, size);           \
 444                 else                                                    \
 445                         __put_user_common(__pu_addr, size);             \
 446         }                                                               \
 447                                                                         \
 448         __pu_err;                                                       \
 449 })
 450
 451 #define __put_data_asm(insn, ptr)                                       \
 452 {                                                                       \
 453         __asm__ __volatile__(                                           \
 454         "1:     "insn("%z2", "%3")"     # __put_data_asm        \n"     \
 455         "2:                                                     \n"     \
 456         "       .insn                                           \n"     \
 457         "       .section        .fixup,\"ax\"                   \n"     \
 458         "3:     li      %0, %4                                  \n"     \
 459         "       j       2b                                      \n"     \
 460         "       .previous                                       \n"     \
 461         "       .section        __ex_table,\"a\"                \n"     \
 462         "       " __UA_ADDR "   1b, 3b                          \n"     \
 463         "       .previous                                       \n"     \
 464         : "=r" (__pu_err)                                               \
 465         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 466           "i" (-EFAULT));                                               \
 467 }
 468
 469 #define __put_data_asm_ll32(insn, ptr)                                  \
 470 {                                                                       \
 471         __asm__ __volatile__(                                           \
 472         "1:     "insn("%2", "(%3)")"    # __put_data_asm_ll32   \n"     \
 473         "2:     "insn("%D2", "4(%3)")"                          \n"     \
 474         "3:                                                     \n"     \
 475         "       .insn                                           \n"     \
 476         "       .section        .fixup,\"ax\"                   \n"     \
 477         "4:     li      %0, %4                                  \n"     \
 478         "       j       3b                                      \n"     \
 479         "       .previous                                       \n"     \
 480         "       .section        __ex_table,\"a\"                \n"     \
 481         "       " __UA_ADDR "   1b, 4b                          \n"     \
 482         "       " __UA_ADDR "   2b, 4b                          \n"     \
 483         "       .previous"                                              \
 484         : "=r" (__pu_err)                                               \
 485         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 486           "i" (-EFAULT));                                               \
 487 }
 488
 489 extern void __put_user_unknown(void);
 490
 491 /*
 492  * ul{b,h,w} are macros and there are no equivalent macros for EVA.
 493  * EVA unaligned access is handled in the ADE exception handler.
 494  */
 495 #ifndef CONFIG_EVA
 496 /*
 497  * put_user_unaligned: - Write a simple value into user space.
 498  * @x:   Value to copy to user space.
 499  * @ptr: Destination address, in user space.
 500  *
 501  * Context: User context only.  This function may sleep.
 502  *
 503  * This macro copies a single simple value from kernel space to user
 504  * space.  It supports simple types like char and int, but not larger
 505  * data types like structures or arrays.
 506  *
 507  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 508  * to the result of dereferencing @ptr.
 509  *
 510  * Returns zero on success, or -EFAULT on error.
 511  */
 512 #define put_user_unaligned(x,ptr)       \
 513         __put_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 514
 515 /*
 516  * get_user_unaligned: - Get a simple variable from user space.
 517  * @x:   Variable to store result.
 518  * @ptr: Source address, in user space.
 519  *
 520  * Context: User context only.  This function may sleep.
 521  *
 522  * This macro copies a single simple variable from user space to kernel
 523  * space.  It supports simple types like char and int, but not larger
 524  * data types like structures or arrays.
 525  *
 526  * @ptr must have pointer-to-simple-variable type, and the result of
 527  * dereferencing @ptr must be assignable to @x without a cast.
 528  *
 529  * Returns zero on success, or -EFAULT on error.
 530  * On error, the variable @x is set to zero.
 531  */
 532 #define get_user_unaligned(x,ptr) \
 533         __get_user_unaligned_check((x),(ptr),sizeof(*(ptr)))
 534
 535 /*
 536  * __put_user_unaligned: - Write a simple value into user space, with less checking.
 537  * @x:   Value to copy to user space.
 538  * @ptr: Destination address, in user space.
 539  *
 540  * Context: User context only.  This function may sleep.
 541  *
 542  * This macro copies a single simple value from kernel space to user
 543  * space.  It supports simple types like char and int, but not larger
 544  * data types like structures or arrays.
 545  *
 546  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 547  * to the result of dereferencing @ptr.
 548  *
 549  * Caller must check the pointer with access_ok() before calling this
 550  * function.
 551  *
 552  * Returns zero on success, or -EFAULT on error.
 553  */
 554 #define __put_user_unaligned(x,ptr) \
 555         __put_user_unaligned_nocheck((x),(ptr),sizeof(*(ptr)))
 556
 557 /*
 558  * __get_user_unaligned: - Get a simple variable from user space, with less checking.
 559  * @x:   Variable to store result.
 560  * @ptr: Source address, in user space.
 561  *
 562  * Context: User context only.  This function may sleep.
 563  *
 564  * This macro copies a single simple variable from user space to kernel
 565  * space.  It supports simple types like char and int, but not larger
 566  * data types like structures or arrays.
 567  *
 568  * @ptr must have pointer-to-simple-variable type, and the result of
 569  * dereferencing @ptr must be assignable to @x without a cast.
 570  *
 571  * Caller must check the pointer with access_ok() before calling this
 572  * function.
 573  *
 574  * Returns zero on success, or -EFAULT on error.
 575  * On error, the variable @x is set to zero.
 576  */
 577 #define __get_user_unaligned(x,ptr) \
 578         __get_user__unalignednocheck((x),(ptr),sizeof(*(ptr)))
 579
 580 /*
 581  * Yuck.  We need two variants, one for 64bit operation and one
 582  * for 32 bit mode and old iron.
 583  */
 584 #ifdef CONFIG_32BIT
 585 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 586         __get_user_unaligned_asm_ll32(val, ptr)
 587 #endif
 588 #ifdef CONFIG_64BIT
 589 #define __GET_USER_UNALIGNED_DW(val, ptr)                               \
 590         __get_user_unaligned_asm(val, "uld", ptr)
 591 #endif
 592
 593 extern void __get_user_unaligned_unknown(void);
 594
 595 #define __get_user_unaligned_common(val, size, ptr)                     \
 596 do {                                                                    \
 597         switch (size) {                                                 \
 598         case 1: __get_data_asm(val, "lb", ptr); break;                  \
 599         case 2: __get_user_unaligned_asm(val, "ulh", ptr); break;       \
 600         case 4: __get_user_unaligned_asm(val, "ulw", ptr); break;       \
 601         case 8: __GET_USER_UNALIGNED_DW(val, ptr); break;               \
 602         default: __get_user_unaligned_unknown(); break;                 \
 603         }                                                               \
 604 } while (0)
 605
 606 #define __get_user_unaligned_nocheck(x,ptr,size)                        \
 607 ({                                                                      \
 608         int __gu_err;                                                   \
 609                                                                         \
 610         __get_user_unaligned_common((x), size, ptr);                    \
 611         __gu_err;                                                       \
 612 })
 613
 614 #define __get_user_unaligned_check(x,ptr,size)                          \
 615 ({                                                                      \
 616         int __gu_err = -EFAULT;                                         \
 617         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 618                                                                         \
 619         if (likely(access_ok(VERIFY_READ,  __gu_ptr, size)))            \
 620                 __get_user_unaligned_common((x), size, __gu_ptr);       \
 621                                                                         \
 622         __gu_err;                                                       \
 623 })
 624
 625 #define __get_data_unaligned_asm(val, insn, addr)                       \
 626 {                                                                       \
 627         long __gu_tmp;                                                  \
 628                                                                         \
 629         __asm__ __volatile__(                                           \
 630         "1:     " insn "        %1, %3                          \n"     \
 631         "2:                                                     \n"     \
 632         "       .insn                                           \n"     \
 633         "       .section .fixup,\"ax\"                          \n"     \
 634         "3:     li      %0, %4                                  \n"     \
 635         "       move    %1, $0                                  \n"     \
 636         "       j       2b                                      \n"     \
 637         "       .previous                                       \n"     \
 638         "       .section __ex_table,\"a\"                       \n"     \
 639         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 640         "       "__UA_ADDR "\t1b + 4, 3b                        \n"     \
 641         "       .previous                                       \n"     \
 642         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 643         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 644                                                                         \
 645         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 646 }
 647
 648 /*
 649  * Get a long long 64 using 32 bit registers.
 650  */
 651 #define __get_user_unaligned_asm_ll32(val, addr)                        \
 652 {                                                                       \
 653         unsigned long long __gu_tmp;                                    \
 654                                                                         \
 655         __asm__ __volatile__(                                           \
 656         "1:     ulw     %1, (%3)                                \n"     \
 657         "2:     ulw     %D1, 4(%3)                              \n"     \
 658         "       move    %0, $0                                  \n"     \
 659         "3:                                                     \n"     \
 660         "       .insn                                           \n"     \
 661         "       .section        .fixup,\"ax\"                   \n"     \
 662         "4:     li      %0, %4                                  \n"     \
 663         "       move    %1, $0                                  \n"     \
 664         "       move    %D1, $0                                 \n"     \
 665         "       j       3b                                      \n"     \
 666         "       .previous                                       \n"     \
 667         "       .section        __ex_table,\"a\"                \n"     \
 668         "       " __UA_ADDR "   1b, 4b                          \n"     \
 669         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 670         "       " __UA_ADDR "   2b, 4b                          \n"     \
 671         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 672         "       .previous                                       \n"     \
 673         : "=r" (__gu_err), "=&r" (__gu_tmp)                             \
 674         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 675         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 676 }
 677
 678 /*
 679  * Yuck.  We need two variants, one for 64bit operation and one
 680  * for 32 bit mode and old iron.
 681  */
 682 #ifdef CONFIG_32BIT
 683 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm_ll32(ptr)
 684 #endif
 685 #ifdef CONFIG_64BIT
 686 #define __PUT_USER_UNALIGNED_DW(ptr) __put_user_unaligned_asm("usd", ptr)
 687 #endif
 688
 689 #define __put_user_unaligned_common(ptr, size)                          \
 690 do {                                                                    \
 691         switch (size) {                                                 \
 692         case 1: __put_data_asm("sb", ptr); break;                       \
 693         case 2: __put_user_unaligned_asm("ush", ptr); break;            \
 694         case 4: __put_user_unaligned_asm("usw", ptr); break;            \
 695         case 8: __PUT_USER_UNALIGNED_DW(ptr); break;                    \
 696         default: __put_user_unaligned_unknown(); break;                 \
 697 } while (0)
 698
 699 #define __put_user_unaligned_nocheck(x,ptr,size)                        \
 700 ({                                                                      \
 701         __typeof__(*(ptr)) __pu_val;                                    \
 702         int __pu_err = 0;                                               \
 703                                                                         \
 704         __pu_val = (x);                                                 \
 705         __put_user_unaligned_common(ptr, size);                         \
 706         __pu_err;                                                       \
 707 })
 708
 709 #define __put_user_unaligned_check(x,ptr,size)                          \
 710 ({                                                                      \
 711         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 712         __typeof__(*(ptr)) __pu_val = (x);                              \
 713         int __pu_err = -EFAULT;                                         \
 714                                                                         \
 715         if (likely(access_ok(VERIFY_WRITE,  __pu_addr, size)))          \
 716                 __put_user_unaligned_common(__pu_addr, size);           \
 717                                                                         \
 718         __pu_err;                                                       \
 719 })
 720
 721 #define __put_user_unaligned_asm(insn, ptr)                             \
 722 {                                                                       \
 723         __asm__ __volatile__(                                           \
 724         "1:     " insn "        %z2, %3         # __put_user_unaligned_asm\n" \
 725         "2:                                                     \n"     \
 726         "       .insn                                           \n"     \
 727         "       .section        .fixup,\"ax\"                   \n"     \
 728         "3:     li      %0, %4                                  \n"     \
 729         "       j       2b                                      \n"     \
 730         "       .previous                                       \n"     \
 731         "       .section        __ex_table,\"a\"                \n"     \
 732         "       " __UA_ADDR "   1b, 3b                          \n"     \
 733         "       .previous                                       \n"     \
 734         : "=r" (__pu_err)                                               \
 735         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 736           "i" (-EFAULT));                                               \
 737 }
 738
 739 #define __put_user_unaligned_asm_ll32(ptr)                              \
 740 {                                                                       \
 741         __asm__ __volatile__(                                           \
 742         "1:     sw      %2, (%3)        # __put_user_unaligned_asm_ll32 \n" \
 743         "2:     sw      %D2, 4(%3)                              \n"     \
 744         "3:                                                     \n"     \
 745         "       .insn                                           \n"     \
 746         "       .section        .fixup,\"ax\"                   \n"     \
 747         "4:     li      %0, %4                                  \n"     \
 748         "       j       3b                                      \n"     \
 749         "       .previous                                       \n"     \
 750         "       .section        __ex_table,\"a\"                \n"     \
 751         "       " __UA_ADDR "   1b, 4b                          \n"     \
 752         "       " __UA_ADDR "   1b + 4, 4b                      \n"     \
 753         "       " __UA_ADDR "   2b, 4b                          \n"     \
 754         "       " __UA_ADDR "   2b + 4, 4b                      \n"     \
 755         "       .previous"                                              \
 756         : "=r" (__pu_err)                                               \
 757         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 758           "i" (-EFAULT));                                               \
 759 }
 760
 761 extern void __put_user_unaligned_unknown(void);
 762 #endif
 763
 764 /*
 765  * We're generating jump to subroutines which will be outside the range of
 766  * jump instructions
 767  */
 768 #ifdef MODULE
 769 #define __MODULE_JAL(destination)                                       \
 770         ".set\tnoat\n\t"                                                \
 771         __UA_LA "\t$1, " #destination "\n\t"                            \
 772         "jalr\t$1\n\t"                                                  \
 773         ".set\tat\n\t"
 774 #else
 775 #define __MODULE_JAL(destination)                                       \
 776         "jal\t" #destination "\n\t"
 777 #endif
 778
 779 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) &&    \
 780                                               defined(CONFIG_CPU_HAS_PREFETCH))
 781 #define DADDI_SCRATCH "$3"
 782 #else
 783 #define DADDI_SCRATCH "$0"
 784 #endif
 785
 786 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
 787
 788 #ifndef CONFIG_EVA
 789 #define __invoke_copy_to_user(to, from, n)                              \
 790 ({                                                                      \
 791         register void __user *__cu_to_r __asm__("$4");                  \
 792         register const void *__cu_from_r __asm__("$5");                 \
 793         register long __cu_len_r __asm__("$6");                         \
 794                                                                         \
 795         __cu_to_r = (to);                                               \
 796         __cu_from_r = (from);                                           \
 797         __cu_len_r = (n);                                               \
 798         __asm__ __volatile__(                                           \
 799         __MODULE_JAL(__copy_user)                                       \
 800         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 801         :                                                               \
 802         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 803           DADDI_SCRATCH, "memory");                                     \
 804         __cu_len_r;                                                     \
 805 })
 806
 807 #define __invoke_copy_to_kernel(to, from, n)                            \
 808         __invoke_copy_to_user(to, from, n)
 809
 810 #endif
 811
 812 /*
 813  * __copy_to_user: - Copy a block of data into user space, with less checking.
 814  * @to:   Destination address, in user space.
 815  * @from: Source address, in kernel space.
 816  * @n:    Number of bytes to copy.
 817  *
 818  * Context: User context only.  This function may sleep.
 819  *
 820  * Copy data from kernel space to user space.  Caller must check
 821  * the specified block with access_ok() before calling this function.
 822  *
 823  * Returns number of bytes that could not be copied.
 824  * On success, this will be zero.
 825  */
 826 #define __copy_to_user(to, from, n)                                     \
 827 ({                                                                      \
 828         void __user *__cu_to;                                           \
 829         const void *__cu_from;                                          \
 830         long __cu_len;                                                  \
 831                                                                         \
 832         __cu_to = (to);                                                 \
 833         __cu_from = (from);                                             \
 834         __cu_len = (n);                                                 \
 835         might_fault();                                                  \
 836         if (segment_eq(get_fs(), get_ds()))                             \
 837                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 838                                                    __cu_len);           \
 839         else                                                            \
 840                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 841                                                  __cu_len);             \
 842         __cu_len;                                                       \
 843 })
 844
 845 extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
 846
 847 #define __copy_to_user_inatomic(to, from, n)                            \
 848 ({                                                                      \
 849         void __user *__cu_to;                                           \
 850         const void *__cu_from;                                          \
 851         long __cu_len;                                                  \
 852                                                                         \
 853         __cu_to = (to);                                                 \
 854         __cu_from = (from);                                             \
 855         __cu_len = (n);                                                 \
 856         if (segment_eq(get_fs(), get_ds()))                             \
 857                 __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from,  \
 858                                                    __cu_len);           \
 859         else                                                            \
 860                 __cu_len = __invoke_copy_to_user(__cu_to, __cu_from,    \
 861                                                  __cu_len);             \
 862         __cu_len;                                                       \
 863 })
 864
 865 #define __copy_from_user_inatomic(to, from, n)                          \
 866 ({                                                                      \
 867         void *__cu_to;                                                  \
 868         const void __user *__cu_from;                                   \
 869         long __cu_len;                                                  \
 870                                                                         \
 871         __cu_to = (to);                                                 \
 872         __cu_from = (from);                                             \
 873         __cu_len = (n);                                                 \
 874         if (segment_eq(get_fs(), get_ds()))                             \
 875                 __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to,  \
 876                                                               __cu_from,\
 877                                                               __cu_len);\
 878         else                                                            \
 879                 __cu_len = __invoke_copy_from_user_inatomic(__cu_to,    \
 880                                                             __cu_from,  \
 881                                                             __cu_len);  \
 882         __cu_len;                                                       \
 883 })
 884
 885 /*
 886  * copy_to_user: - Copy a block of data into user space.
 887  * @to:   Destination address, in user space.
 888  * @from: Source address, in kernel space.
 889  * @n:    Number of bytes to copy.
 890  *
 891  * Context: User context only.  This function may sleep.
 892  *
 893  * Copy data from kernel space to user space.
 894  *
 895  * Returns number of bytes that could not be copied.
 896  * On success, this will be zero.
 897  */
 898 #define copy_to_user(to, from, n)                                       \
 899 ({                                                                      \
 900         void __user *__cu_to;                                           \
 901         const void *__cu_from;                                          \
 902         long __cu_len;                                                  \
 903                                                                         \
 904         __cu_to = (to);                                                 \
 905         __cu_from = (from);                                             \
 906         __cu_len = (n);                                                 \
 907         if (segment_eq(get_fs(), get_ds())) {                           \
 908                 __cu_len = __invoke_copy_to_kernel(__cu_to,             \
 909                                                    __cu_from,           \
 910                                                    __cu_len);           \
 911         } else {                                                        \
 912                 if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) {       \
 913                         might_fault();                                  \
 914                         __cu_len = __invoke_copy_to_user(__cu_to,       \
 915                                                          __cu_from,     \
 916                                                          __cu_len);     \
 917                 }                                                       \
 918         }                                                               \
 919         __cu_len;                                                       \
 920 })
 921
 922 #ifndef CONFIG_EVA
 923
 924 #define __invoke_copy_from_user(to, from, n)                            \
 925 ({                                                                      \
 926         register void *__cu_to_r __asm__("$4");                         \
 927         register const void __user *__cu_from_r __asm__("$5");          \
 928         register long __cu_len_r __asm__("$6");                         \
 929                                                                         \
 930         __cu_to_r = (to);                                               \
 931         __cu_from_r = (from);                                           \
 932         __cu_len_r = (n);                                               \
 933         __asm__ __volatile__(                                           \
 934         ".set\tnoreorder\n\t"                                           \
 935         __MODULE_JAL(__copy_user)                                       \
 936         ".set\tnoat\n\t"                                                \
 937         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 938         ".set\tat\n\t"                                                  \
 939         ".set\treorder"                                                 \
 940         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 941         :                                                               \
 942         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 943           DADDI_SCRATCH, "memory");                                     \
 944         __cu_len_r;                                                     \
 945 })
 946
 947 #define __invoke_copy_from_kernel(to, from, n)                          \
 948         __invoke_copy_from_user(to, from, n)
 949
 950 /* For userland <-> userland operations */
 951 #define ___invoke_copy_in_user(to, from, n)                             \
 952         __invoke_copy_from_user(to, from, n)
 953
 954 /* For kernel <-> kernel operations */
 955 #define ___invoke_copy_in_kernel(to, from, n)                           \
 956         __invoke_copy_from_user(to, from, n)
 957
 958 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
 959 ({                                                                      \
 960         register void *__cu_to_r __asm__("$4");                         \
 961         register const void __user *__cu_from_r __asm__("$5");          \
 962         register long __cu_len_r __asm__("$6");                         \
 963                                                                         \
 964         __cu_to_r = (to);                                               \
 965         __cu_from_r = (from);                                           \
 966         __cu_len_r = (n);                                               \
 967         __asm__ __volatile__(                                           \
 968         ".set\tnoreorder\n\t"                                           \
 969         __MODULE_JAL(__copy_user_inatomic)                              \
 970         ".set\tnoat\n\t"                                                \
 971         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 972         ".set\tat\n\t"                                                  \
 973         ".set\treorder"                                                 \
 974         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 975         :                                                               \
 976         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 977           DADDI_SCRATCH, "memory");                                     \
 978         __cu_len_r;                                                     \
 979 })
 980
 981 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
 982         __invoke_copy_from_user_inatomic(to, from, n)                   \
 983
 984 #else
 985
 986 /* EVA specific functions */
 987
 988 extern size_t __copy_user_inatomic_eva(void *__to, const void *__from,
 989                                        size_t __n);
 990 extern size_t __copy_from_user_eva(void *__to, const void *__from,
 991                                    size_t __n);
 992 extern size_t __copy_to_user_eva(void *__to, const void *__from,
 993                                  size_t __n);
 994 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
 995
 996 #define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr)      \
 997 ({                                                                      \
 998         register void *__cu_to_r __asm__("$4");                         \
 999         register const void __user *__cu_from_r __asm__("$5");          \
1000         register long __cu_len_r __asm__("$6");                         \
1001                                                                         \
1002         __cu_to_r = (to);                                               \
1003         __cu_from_r = (from);                                           \
1004         __cu_len_r = (n);                                               \
1005         __asm__ __volatile__(                                           \
1006         ".set\tnoreorder\n\t"                                           \
1007         __MODULE_JAL(func_ptr)                                          \
1008         ".set\tnoat\n\t"                                                \
1009         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
1010         ".set\tat\n\t"                                                  \
1011         ".set\treorder"                                                 \
1012         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1013         :                                                               \
1014         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1015           DADDI_SCRATCH, "memory");                                     \
1016         __cu_len_r;                                                     \
1017 })
1018
1019 #define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr)        \
1020 ({                                                                      \
1021         register void *__cu_to_r __asm__("$4");                         \
1022         register const void __user *__cu_from_r __asm__("$5");          \
1023         register long __cu_len_r __asm__("$6");                         \
1024                                                                         \
1025         __cu_to_r = (to);                                               \
1026         __cu_from_r = (from);                                           \
1027         __cu_len_r = (n);                                               \
1028         __asm__ __volatile__(                                           \
1029         __MODULE_JAL(func_ptr)                                          \
1030         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
1031         :                                                               \
1032         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
1033           DADDI_SCRATCH, "memory");                                     \
1034         __cu_len_r;                                                     \
1035 })
1036
1037 /*
1038  * Source or destination address is in userland. We need to go through
1039  * the TLB
1040  */
1041 #define __invoke_copy_from_user(to, from, n)                            \
1042         __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva)
1043
1044 #define __invoke_copy_from_user_inatomic(to, from, n)                   \
1045         __invoke_copy_from_user_eva_generic(to, from, n,                \
1046                                             __copy_user_inatomic_eva)
1047
1048 #define __invoke_copy_to_user(to, from, n)                              \
1049         __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva)
1050
1051 #define ___invoke_copy_in_user(to, from, n)                             \
1052         __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva)
1053
1054 /*
1055  * Source or destination address in the kernel. We are not going through
1056  * the TLB
1057  */
1058 #define __invoke_copy_from_kernel(to, from, n)                          \
1059         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1060
1061 #define __invoke_copy_from_kernel_inatomic(to, from, n)                 \
1062         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic)
1063
1064 #define __invoke_copy_to_kernel(to, from, n)                            \
1065         __invoke_copy_to_user_eva_generic(to, from, n, __copy_user)
1066
1067 #define ___invoke_copy_in_kernel(to, from, n)                           \
1068         __invoke_copy_from_user_eva_generic(to, from, n, __copy_user)
1069
1070 #endif /* CONFIG_EVA */
1071
1072 /*
1073  * __copy_from_user: - Copy a block of data from user space, with less checking.
1074  * @to:   Destination address, in kernel space.
1075  * @from: Source address, in user space.
1076  * @n:    Number of bytes to copy.
1077  *
1078  * Context: User context only.  This function may sleep.
1079  *
1080  * Copy data from user space to kernel space.  Caller must check
1081  * the specified block with access_ok() before calling this function.
1082  *
1083  * Returns number of bytes that could not be copied.
1084  * On success, this will be zero.
1085  *
1086  * If some data could not be copied, this function will pad the copied
1087  * data to the requested size using zero bytes.
1088  */
1089 #define __copy_from_user(to, from, n)                                   \
1090 ({                                                                      \
1091         void *__cu_to;                                                  \
1092         const void __user *__cu_from;                                   \
1093         long __cu_len;                                                  \
1094                                                                         \
1095         __cu_to = (to);                                                 \
1096         __cu_from = (from);                                             \
1097         __cu_len = (n);                                                 \
1098         might_fault();                                                  \
1099         __cu_len = __invoke_copy_from_user(__cu_to, __cu_from,          \
1100                                            __cu_len);                   \
1101         __cu_len;                                                       \
1102 })
1103
1104 /*
1105  * copy_from_user: - Copy a block of data from user space.
1106  * @to:   Destination address, in kernel space.
1107  * @from: Source address, in user space.
1108  * @n:    Number of bytes to copy.
1109  *
1110  * Context: User context only.  This function may sleep.
1111  *
1112  * Copy data from user space to kernel space.
1113  *
1114  * Returns number of bytes that could not be copied.
1115  * On success, this will be zero.
1116  *
1117  * If some data could not be copied, this function will pad the copied
1118  * data to the requested size using zero bytes.
1119  */
1120 #define copy_from_user(to, from, n)                                     \
1121 ({                                                                      \
1122         void *__cu_to;                                                  \
1123         const void __user *__cu_from;                                   \
1124         long __cu_len;                                                  \
1125                                                                         \
1126         __cu_to = (to);                                                 \
1127         __cu_from = (from);                                             \
1128         __cu_len = (n);                                                 \
1129         if (segment_eq(get_fs(), get_ds())) {                           \
1130                 __cu_len = __invoke_copy_from_kernel(__cu_to,           \
1131                                                      __cu_from,         \
1132                                                      __cu_len);         \
1133         } else {                                                        \
1134                 if (access_ok(VERIFY_READ, __cu_from, __cu_len)) {      \
1135                         might_fault();                                  \
1136                         __cu_len = __invoke_copy_from_user(__cu_to,     \
1137                                                            __cu_from,   \
1138                                                            __cu_len);   \
1139                 }                                                       \
1140         }                                                               \
1141         __cu_len;                                                       \
1142 })
1143
1144 #define __copy_in_user(to, from, n)                                     \
1145 ({                                                                      \
1146         void __user *__cu_to;                                           \
1147         const void __user *__cu_from;                                   \
1148         long __cu_len;                                                  \
1149                                                                         \
1150         __cu_to = (to);                                                 \
1151         __cu_from = (from);                                             \
1152         __cu_len = (n);                                                 \
1153         if (segment_eq(get_fs(), get_ds())) {                           \
1154                 __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \
1155                                                     __cu_len);          \
1156         } else {                                                        \
1157                 might_fault();                                          \
1158                 __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from,   \
1159                                                   __cu_len);            \
1160         }                                                               \
1161         __cu_len;                                                       \
1162 })
1163
1164 #define copy_in_user(to, from, n)                                       \
1165 ({                                                                      \
1166         void __user *__cu_to;                                           \
1167         const void __user *__cu_from;                                   \
1168         long __cu_len;                                                  \
1169                                                                         \
1170         __cu_to = (to);                                                 \
1171         __cu_from = (from);                                             \
1172         __cu_len = (n);                                                 \
1173         if (segment_eq(get_fs(), get_ds())) {                           \
1174                 __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from,  \
1175                                                     __cu_len);          \
1176         } else {                                                        \
1177                 if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\
1178                            access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\
1179                         might_fault();                                  \
1180                         __cu_len = ___invoke_copy_in_user(__cu_to,      \
1181                                                           __cu_from,    \
1182                                                           __cu_len);    \
1183                 }                                                       \
1184         }                                                               \
1185         __cu_len;                                                       \
1186 })
1187
1188 /*
1189  * __clear_user: - Zero a block of memory in user space, with less checking.
1190  * @to:   Destination address, in user space.
1191  * @n:    Number of bytes to zero.
1192  *
1193  * Zero a block of memory in user space.  Caller must check
1194  * the specified block with access_ok() before calling this function.
1195  *
1196  * Returns number of bytes that could not be cleared.
1197  * On success, this will be zero.
1198  */
1199 static inline __kernel_size_t
1200 __clear_user(void __user *addr, __kernel_size_t size)
1201 {
1202         __kernel_size_t res;
1203
1204         might_fault();
1205         __asm__ __volatile__(
1206                 "move\t$4, %1\n\t"
1207                 "move\t$5, $0\n\t"
1208                 "move\t$6, %2\n\t"
1209                 __MODULE_JAL(__bzero)
1210                 "move\t%0, $6"
1211                 : "=r" (res)
1212                 : "r" (addr), "r" (size)
1213                 : "$4", "$5", "$6", __UA_t0, __UA_t1, "$31");
1214
1215         return res;
1216 }
1217
1218 #define clear_user(addr,n)                                              \
1219 ({                                                                      \
1220         void __user * __cl_addr = (addr);                               \
1221         unsigned long __cl_size = (n);                                  \
1222         if (__cl_size && access_ok(VERIFY_WRITE,                        \
1223                                         __cl_addr, __cl_size))          \
1224                 __cl_size = __clear_user(__cl_addr, __cl_size);         \
1225         __cl_size;                                                      \
1226 })
1227
1228 /*
1229  * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
1230  * @dst:   Destination address, in kernel space.  This buffer must be at
1231  *         least @count bytes long.
1232  * @src:   Source address, in user space.
1233  * @count: Maximum number of bytes to copy, including the trailing NUL.
1234  *
1235  * Copies a NUL-terminated string from userspace to kernel space.
1236  * Caller must check the specified block with access_ok() before calling
1237  * this function.
1238  *
1239  * On success, returns the length of the string (not including the trailing
1240  * NUL).
1241  *
1242  * If access to userspace fails, returns -EFAULT (some data may have been
1243  * copied).
1244  *
1245  * If @count is smaller than the length of the string, copies @count bytes
1246  * and returns @count.
1247  */
1248 static inline long
1249 __strncpy_from_user(char *__to, const char __user *__from, long __len)
1250 {
1251         long res;
1252
1253         if (segment_eq(get_fs(), get_ds())) {
1254                 __asm__ __volatile__(
1255                         "move\t$4, %1\n\t"
1256                         "move\t$5, %2\n\t"
1257                         "move\t$6, %3\n\t"
1258                         __MODULE_JAL(__strncpy_from_kernel_nocheck_asm)
1259                         "move\t%0, $2"
1260                         : "=r" (res)
1261                         : "r" (__to), "r" (__from), "r" (__len)
1262                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1263         } else {
1264                 might_fault();
1265                 __asm__ __volatile__(
1266                         "move\t$4, %1\n\t"
1267                         "move\t$5, %2\n\t"
1268                         "move\t$6, %3\n\t"
1269                         __MODULE_JAL(__strncpy_from_user_nocheck_asm)
1270                         "move\t%0, $2"
1271                         : "=r" (res)
1272                         : "r" (__to), "r" (__from), "r" (__len)
1273                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1274         }
1275
1276         return res;
1277 }
1278
1279 /*
1280  * strncpy_from_user: - Copy a NUL terminated string from userspace.
1281  * @dst:   Destination address, in kernel space.  This buffer must be at
1282  *         least @count bytes long.
1283  * @src:   Source address, in user space.
1284  * @count: Maximum number of bytes to copy, including the trailing NUL.
1285  *
1286  * Copies a NUL-terminated string from userspace to kernel space.
1287  *
1288  * On success, returns the length of the string (not including the trailing
1289  * NUL).
1290  *
1291  * If access to userspace fails, returns -EFAULT (some data may have been
1292  * copied).
1293  *
1294  * If @count is smaller than the length of the string, copies @count bytes
1295  * and returns @count.
1296  */
1297 static inline long
1298 strncpy_from_user(char *__to, const char __user *__from, long __len)
1299 {
1300         long res;
1301
1302         if (segment_eq(get_fs(), get_ds())) {
1303                 __asm__ __volatile__(
1304                         "move\t$4, %1\n\t"
1305                         "move\t$5, %2\n\t"
1306                         "move\t$6, %3\n\t"
1307                         __MODULE_JAL(__strncpy_from_kernel_asm)
1308                         "move\t%0, $2"
1309                         : "=r" (res)
1310                         : "r" (__to), "r" (__from), "r" (__len)
1311                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1312         } else {
1313                 might_fault();
1314                 __asm__ __volatile__(
1315                         "move\t$4, %1\n\t"
1316                         "move\t$5, %2\n\t"
1317                         "move\t$6, %3\n\t"
1318                         __MODULE_JAL(__strncpy_from_user_asm)
1319                         "move\t%0, $2"
1320                         : "=r" (res)
1321                         : "r" (__to), "r" (__from), "r" (__len)
1322                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
1323         }
1324
1325         return res;
1326 }
1327
1328 /*
1329  * strlen_user: - Get the size of a string in user space.
1330  * @str: The string to measure.
1331  *
1332  * Context: User context only.  This function may sleep.
1333  *
1334  * Get the size of a NUL-terminated string in user space.
1335  *
1336  * Returns the size of the string INCLUDING the terminating NUL.
1337  * On exception, returns 0.
1338  *
1339  * If there is a limit on the length of a valid string, you may wish to
1340  * consider using strnlen_user() instead.
1341  */
1342 static inline long strlen_user(const char __user *s)
1343 {
1344         long res;
1345
1346         if (segment_eq(get_fs(), get_ds())) {
1347                 __asm__ __volatile__(
1348                         "move\t$4, %1\n\t"
1349                         __MODULE_JAL(__strlen_kernel_asm)
1350                         "move\t%0, $2"
1351                         : "=r" (res)
1352                         : "r" (s)
1353                         : "$2", "$4", __UA_t0, "$31");
1354         } else {
1355                 might_fault();
1356                 __asm__ __volatile__(
1357                         "move\t$4, %1\n\t"
1358                         __MODULE_JAL(__strlen_kernel_asm)
1359                         "move\t%0, $2"
1360                         : "=r" (res)
1361                         : "r" (s)
1362                         : "$2", "$4", __UA_t0, "$31");
1363         }
1364
1365         return res;
1366 }
1367
1368 /* Returns: 0 if bad, string length+1 (memory size) of string if ok */
1369 static inline long __strnlen_user(const char __user *s, long n)
1370 {
1371         long res;
1372
1373         if (segment_eq(get_fs(), get_ds())) {
1374                 __asm__ __volatile__(
1375                         "move\t$4, %1\n\t"
1376                         "move\t$5, %2\n\t"
1377                         __MODULE_JAL(__strnlen_kernel_nocheck_asm)
1378                         "move\t%0, $2"
1379                         : "=r" (res)
1380                         : "r" (s), "r" (n)
1381                         : "$2", "$4", "$5", __UA_t0, "$31");
1382         } else {
1383                 might_fault();
1384                 __asm__ __volatile__(
1385                         "move\t$4, %1\n\t"
1386                         "move\t$5, %2\n\t"
1387                         __MODULE_JAL(__strnlen_user_nocheck_asm)
1388                         "move\t%0, $2"
1389                         : "=r" (res)
1390                         : "r" (s), "r" (n)
1391                         : "$2", "$4", "$5", __UA_t0, "$31");
1392         }
1393
1394         return res;
1395 }
1396
1397 /*
1398  * strlen_user: - Get the size of a string in user space.
1399  * @str: The string to measure.
1400  *
1401  * Context: User context only.  This function may sleep.
1402  *
1403  * Get the size of a NUL-terminated string in user space.
1404  *
1405  * Returns the size of the string INCLUDING the terminating NUL.
1406  * On exception, returns 0.
1407  *
1408  * If there is a limit on the length of a valid string, you may wish to
1409  * consider using strnlen_user() instead.
1410  */
1411 static inline long strnlen_user(const char __user *s, long n)
1412 {
1413         long res;
1414
1415         might_fault();
1416         if (segment_eq(get_fs(), get_ds())) {
1417                 __asm__ __volatile__(
1418                         "move\t$4, %1\n\t"
1419                         "move\t$5, %2\n\t"
1420                         __MODULE_JAL(__strnlen_kernel_asm)
1421                         "move\t%0, $2"
1422                         : "=r" (res)
1423                         : "r" (s), "r" (n)
1424                         : "$2", "$4", "$5", __UA_t0, "$31");
1425         } else {
1426                 __asm__ __volatile__(
1427                         "move\t$4, %1\n\t"
1428                         "move\t$5, %2\n\t"
1429                         __MODULE_JAL(__strnlen_user_asm)
1430                         "move\t%0, $2"
1431                         : "=r" (res)
1432                         : "r" (s), "r" (n)
1433                         : "$2", "$4", "$5", __UA_t0, "$31");
1434         }
1435
1436         return res;
1437 }
1438
1439 struct exception_table_entry
1440 {
1441         unsigned long insn;
1442         unsigned long nextinsn;
1443 };
1444
1445 extern int fixup_exception(struct pt_regs *regs);
1446
1447 #endif /* _ASM_UACCESS_H */