arch/s390/include/asm/uaccess.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *  S390 version
   4  *    Copyright IBM Corp. 1999, 2000
   5  *    Author(s): Hartmut Penner ([email protected]),
   6  *               Martin Schwidefsky ([email protected])
   7  *
   8  *  Derived from "include/asm-i386/uaccess.h"
   9  */
  10 #ifndef __S390_UACCESS_H
  11 #define __S390_UACCESS_H
  12
  13 /*
  14  * User space memory access functions
  15  */
  16 #include <asm/asm-extable.h>
  17 #include <asm/processor.h>
  18 #include <asm/extable.h>
  19 #include <asm/facility.h>
  20 #include <asm-generic/access_ok.h>
  21 #include <linux/instrumented.h>
  22
  23 void debug_user_asce(int exit);
  24
  25 unsigned long __must_check
  26 raw_copy_from_user(void *to, const void __user *from, unsigned long n);
  27
  28 unsigned long __must_check
  29 raw_copy_to_user(void __user *to, const void *from, unsigned long n);
  30
  31 #ifndef CONFIG_KASAN
  32 #define INLINE_COPY_FROM_USER
  33 #define INLINE_COPY_TO_USER
  34 #endif
  35
  36 unsigned long __must_check
  37 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key);
  38
  39 static __always_inline unsigned long __must_check
  40 copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key)
  41 {
  42         if (check_copy_size(to, n, false))
  43                 n = _copy_from_user_key(to, from, n, key);
  44         return n;
  45 }
  46
  47 unsigned long __must_check
  48 _copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key);
  49
  50 static __always_inline unsigned long __must_check
  51 copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key)
  52 {
  53         if (check_copy_size(from, n, true))
  54                 n = _copy_to_user_key(to, from, n, key);
  55         return n;
  56 }
  57
  58 union oac {
  59         unsigned int val;
  60         struct {
  61                 struct {
  62                         unsigned short key : 4;
  63                         unsigned short     : 4;
  64                         unsigned short as  : 2;
  65                         unsigned short     : 4;
  66                         unsigned short k   : 1;
  67                         unsigned short a   : 1;
  68                 } oac1;
  69                 struct {
  70                         unsigned short key : 4;
  71                         unsigned short     : 4;
  72                         unsigned short as  : 2;
  73                         unsigned short     : 4;
  74                         unsigned short k   : 1;
  75                         unsigned short a   : 1;
  76                 } oac2;
  77         };
  78 };
  79
  80 int __noreturn __put_user_bad(void);
  81
  82 #ifdef CONFIG_KMSAN
  83 #define get_put_user_noinstr_attributes \
  84         noinline __maybe_unused __no_sanitize_memory
  85 #else
  86 #define get_put_user_noinstr_attributes __always_inline
  87 #endif
  88
  89 #define DEFINE_PUT_USER(type)                                           \
  90 static get_put_user_noinstr_attributes int                              \
  91 __put_user_##type##_noinstr(unsigned type __user *to,                   \
  92                             unsigned type *from,                        \
  93                             unsigned long size)                         \
  94 {                                                                       \
  95         union oac __oac_spec = {                                        \
  96                 .oac1.as = PSW_BITS_AS_SECONDARY,                       \
  97                 .oac1.a = 1,                                            \
  98         };                                                              \
  99         int rc;                                                         \
 100                                                                         \
 101         asm volatile(                                                   \
 102                 "       lr      0,%[spec]\n"                            \
 103                 "0:     mvcos   %[_to],%[_from],%[_size]\n"             \
 104                 "1:     xr      %[rc],%[rc]\n"                          \
 105                 "2:\n"                                                  \
 106                 EX_TABLE_UA_STORE(0b, 2b, %[rc])                        \
 107                 EX_TABLE_UA_STORE(1b, 2b, %[rc])                        \
 108                 : [rc] "=&d" (rc), [_to] "+Q" (*(to))                   \
 109                 : [_size] "d" (size), [_from] "Q" (*(from)),            \
 110                   [spec] "d" (__oac_spec.val)                           \
 111                 : "cc", "0");                                           \
 112         return rc;                                                      \
 113 }                                                                       \
 114                                                                         \
 115 static __always_inline int                                              \
 116 __put_user_##type(unsigned type __user *to, unsigned type *from,        \
 117                   unsigned long size)                                   \
 118 {                                                                       \
 119         int rc;                                                         \
 120                                                                         \
 121         rc = __put_user_##type##_noinstr(to, from, size);               \
 122         instrument_put_user(*from, to, size);                           \
 123         return rc;                                                      \
 124 }
 125
 126 DEFINE_PUT_USER(char);
 127 DEFINE_PUT_USER(short);
 128 DEFINE_PUT_USER(int);
 129 DEFINE_PUT_USER(long);
 130
 131 static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size)
 132 {
 133         int rc;
 134
 135         switch (size) {
 136         case 1:
 137                 rc = __put_user_char((unsigned char __user *)ptr,
 138                                      (unsigned char *)x,
 139                                      size);
 140                 break;
 141         case 2:
 142                 rc = __put_user_short((unsigned short __user *)ptr,
 143                                       (unsigned short *)x,
 144                                       size);
 145                 break;
 146         case 4:
 147                 rc = __put_user_int((unsigned int __user *)ptr,
 148                                     (unsigned int *)x,
 149                                     size);
 150                 break;
 151         case 8:
 152                 rc = __put_user_long((unsigned long __user *)ptr,
 153                                      (unsigned long *)x,
 154                                      size);
 155                 break;
 156         default:
 157                 __put_user_bad();
 158                 break;
 159         }
 160         return rc;
 161 }
 162
 163 int __noreturn __get_user_bad(void);
 164
 165 #define DEFINE_GET_USER(type)                                           \
 166 static get_put_user_noinstr_attributes int                              \
 167 __get_user_##type##_noinstr(unsigned type *to,                          \
 168                             unsigned type __user *from,                 \
 169                             unsigned long size)                         \
 170 {                                                                       \
 171         union oac __oac_spec = {                                        \
 172                 .oac2.as = PSW_BITS_AS_SECONDARY,                       \
 173                 .oac2.a = 1,                                            \
 174         };                                                              \
 175         int rc;                                                         \
 176                                                                         \
 177         asm volatile(                                                   \
 178                 "       lr      0,%[spec]\n"                            \
 179                 "0:     mvcos   0(%[_to]),%[_from],%[_size]\n"          \
 180                 "1:     xr      %[rc],%[rc]\n"                          \
 181                 "2:\n"                                                  \
 182                 EX_TABLE_UA_LOAD_MEM(0b, 2b, %[rc], %[_to], %[_ksize])  \
 183                 EX_TABLE_UA_LOAD_MEM(1b, 2b, %[rc], %[_to], %[_ksize])  \
 184                 : [rc] "=&d" (rc), "=Q" (*(to))                         \
 185                 : [_size] "d" (size), [_from] "Q" (*(from)),            \
 186                   [spec] "d" (__oac_spec.val), [_to] "a" (to),          \
 187                   [_ksize] "K" (size)                                   \
 188                 : "cc", "0");                                           \
 189         return rc;                                                      \
 190 }                                                                       \
 191                                                                         \
 192 static __always_inline int                                              \
 193 __get_user_##type(unsigned type *to, unsigned type __user *from,        \
 194                   unsigned long size)                                   \
 195 {                                                                       \
 196         int rc;                                                         \
 197                                                                         \
 198         rc = __get_user_##type##_noinstr(to, from, size);               \
 199         instrument_get_user(*to);                                       \
 200         return rc;                                                      \
 201 }
 202
 203 DEFINE_GET_USER(char);
 204 DEFINE_GET_USER(short);
 205 DEFINE_GET_USER(int);
 206 DEFINE_GET_USER(long);
 207
 208 static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size)
 209 {
 210         int rc;
 211
 212         switch (size) {
 213         case 1:
 214                 rc = __get_user_char((unsigned char *)x,
 215                                      (unsigned char __user *)ptr,
 216                                      size);
 217                 break;
 218         case 2:
 219                 rc = __get_user_short((unsigned short *)x,
 220                                       (unsigned short __user *)ptr,
 221                                       size);
 222                 break;
 223         case 4:
 224                 rc = __get_user_int((unsigned int *)x,
 225                                     (unsigned int __user *)ptr,
 226                                     size);
 227                 break;
 228         case 8:
 229                 rc = __get_user_long((unsigned long *)x,
 230                                      (unsigned long __user *)ptr,
 231                                      size);
 232                 break;
 233         default:
 234                 __get_user_bad();
 235                 break;
 236         }
 237         return rc;
 238 }
 239
 240 /*
 241  * These are the main single-value transfer routines.  They automatically
 242  * use the right size if we just have the right pointer type.
 243  */
 244 #define __put_user(x, ptr)                                              \
 245 ({                                                                      \
 246         __typeof__(*(ptr)) __x = (x);                                   \
 247         int __pu_err = -EFAULT;                                         \
 248                                                                         \
 249         __chk_user_ptr(ptr);                                            \
 250         switch (sizeof(*(ptr))) {                                       \
 251         case 1:                                                         \
 252         case 2:                                                         \
 253         case 4:                                                         \
 254         case 8:                                                         \
 255                 __pu_err = __put_user_fn(&__x, ptr, sizeof(*(ptr)));    \
 256                 break;                                                  \
 257         default:                                                        \
 258                 __put_user_bad();                                       \
 259                 break;                                                  \
 260         }                                                               \
 261         __builtin_expect(__pu_err, 0);                                  \
 262 })
 263
 264 #define put_user(x, ptr)                                                \
 265 ({                                                                      \
 266         might_fault();                                                  \
 267         __put_user(x, ptr);                                             \
 268 })
 269
 270 #define __get_user(x, ptr)                                              \
 271 ({                                                                      \
 272         int __gu_err = -EFAULT;                                         \
 273                                                                         \
 274         __chk_user_ptr(ptr);                                            \
 275         switch (sizeof(*(ptr))) {                                       \
 276         case 1: {                                                       \
 277                 unsigned char __x;                                      \
 278                                                                         \
 279                 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr)));    \
 280                 (x) = *(__force __typeof__(*(ptr)) *)&__x;              \
 281                 break;                                                  \
 282         };                                                              \
 283         case 2: {                                                       \
 284                 unsigned short __x;                                     \
 285                                                                         \
 286                 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr)));    \
 287                 (x) = *(__force __typeof__(*(ptr)) *)&__x;              \
 288                 break;                                                  \
 289         };                                                              \
 290         case 4: {                                                       \
 291                 unsigned int __x;                                       \
 292                                                                         \
 293                 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr)));    \
 294                 (x) = *(__force __typeof__(*(ptr)) *)&__x;              \
 295                 break;                                                  \
 296         };                                                              \
 297         case 8: {                                                       \
 298                 unsigned long __x;                                      \
 299                                                                         \
 300                 __gu_err = __get_user_fn(&__x, ptr, sizeof(*(ptr)));    \
 301                 (x) = *(__force __typeof__(*(ptr)) *)&__x;              \
 302                 break;                                                  \
 303         };                                                              \
 304         default:                                                        \
 305                 __get_user_bad();                                       \
 306                 break;                                                  \
 307         }                                                               \
 308         __builtin_expect(__gu_err, 0);                                  \
 309 })
 310
 311 #define get_user(x, ptr)                                                \
 312 ({                                                                      \
 313         might_fault();                                                  \
 314         __get_user(x, ptr);                                             \
 315 })
 316
 317 /*
 318  * Copy a null terminated string from userspace.
 319  */
 320 long __must_check strncpy_from_user(char *dst, const char __user *src, long count);
 321
 322 long __must_check strnlen_user(const char __user *src, long count);
 323
 324 /*
 325  * Zero Userspace
 326  */
 327 unsigned long __must_check __clear_user(void __user *to, unsigned long size);
 328
 329 static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
 330 {
 331         might_fault();
 332         return __clear_user(to, n);
 333 }
 334
 335 void *__s390_kernel_write(void *dst, const void *src, size_t size);
 336
 337 static inline void *s390_kernel_write(void *dst, const void *src, size_t size)
 338 {
 339         if (__is_defined(__DECOMPRESSOR))
 340                 return memcpy(dst, src, size);
 341         return __s390_kernel_write(dst, src, size);
 342 }
 343
 344 int __noreturn __put_kernel_bad(void);
 345
 346 #define __put_kernel_asm(val, to, insn)                                 \
 347 ({                                                                      \
 348         int __rc;                                                       \
 349                                                                         \
 350         asm volatile(                                                   \
 351                 "0:   " insn "  %[_val],%[_to]\n"                       \
 352                 "1:     xr      %[rc],%[rc]\n"                          \
 353                 "2:\n"                                                  \
 354                 EX_TABLE_UA_STORE(0b, 2b, %[rc])                        \
 355                 EX_TABLE_UA_STORE(1b, 2b, %[rc])                        \
 356                 : [rc] "=d" (__rc), [_to] "+Q" (*(to))                  \
 357                 : [_val] "d" (val)                                      \
 358                 : "cc");                                                \
 359         __rc;                                                           \
 360 })
 361
 362 #define __put_kernel_nofault(dst, src, type, err_label)                 \
 363 do {                                                                    \
 364         unsigned long __x = (unsigned long)(*((type *)(src)));          \
 365         int __pk_err;                                                   \
 366                                                                         \
 367         switch (sizeof(type)) {                                         \
 368         case 1:                                                         \
 369                 __pk_err = __put_kernel_asm(__x, (type *)(dst), "stc"); \
 370                 break;                                                  \
 371         case 2:                                                         \
 372                 __pk_err = __put_kernel_asm(__x, (type *)(dst), "sth"); \
 373                 break;                                                  \
 374         case 4:                                                         \
 375                 __pk_err = __put_kernel_asm(__x, (type *)(dst), "st");  \
 376                 break;                                                  \
 377         case 8:                                                         \
 378                 __pk_err = __put_kernel_asm(__x, (type *)(dst), "stg"); \
 379                 break;                                                  \
 380         default:                                                        \
 381                 __pk_err = __put_kernel_bad();                          \
 382                 break;                                                  \
 383         }                                                               \
 384         if (unlikely(__pk_err))                                         \
 385                 goto err_label;                                         \
 386 } while (0)
 387
 388 int __noreturn __get_kernel_bad(void);
 389
 390 #define __get_kernel_asm(val, from, insn)                               \
 391 ({                                                                      \
 392         int __rc;                                                       \
 393                                                                         \
 394         asm volatile(                                                   \
 395                 "0:   " insn "  %[_val],%[_from]\n"                     \
 396                 "1:     xr      %[rc],%[rc]\n"                          \
 397                 "2:\n"                                                  \
 398                 EX_TABLE_UA_LOAD_REG(0b, 2b, %[rc], %[_val])            \
 399                 EX_TABLE_UA_LOAD_REG(1b, 2b, %[rc], %[_val])            \
 400                 : [rc] "=d" (__rc), [_val] "=d" (val)                   \
 401                 : [_from] "Q" (*(from))                                 \
 402                 : "cc");                                                \
 403         __rc;                                                           \
 404 })
 405
 406 #define __get_kernel_nofault(dst, src, type, err_label)                 \
 407 do {                                                                    \
 408         int __gk_err;                                                   \
 409                                                                         \
 410         switch (sizeof(type)) {                                         \
 411         case 1: {                                                       \
 412                 unsigned char __x;                                      \
 413                                                                         \
 414                 __gk_err = __get_kernel_asm(__x, (type *)(src), "ic");  \
 415                 *((type *)(dst)) = (type)__x;                           \
 416                 break;                                                  \
 417         };                                                              \
 418         case 2: {                                                       \
 419                 unsigned short __x;                                     \
 420                                                                         \
 421                 __gk_err = __get_kernel_asm(__x, (type *)(src), "lh");  \
 422                 *((type *)(dst)) = (type)__x;                           \
 423                 break;                                                  \
 424         };                                                              \
 425         case 4: {                                                       \
 426                 unsigned int __x;                                       \
 427                                                                         \
 428                 __gk_err = __get_kernel_asm(__x, (type *)(src), "l");   \
 429                 *((type *)(dst)) = (type)__x;                           \
 430                 break;                                                  \
 431         };                                                              \
 432         case 8: {                                                       \
 433                 unsigned long __x;                                      \
 434                                                                         \
 435                 __gk_err = __get_kernel_asm(__x, (type *)(src), "lg");  \
 436                 *((type *)(dst)) = (type)__x;                           \
 437                 break;                                                  \
 438         };                                                              \
 439         default:                                                        \
 440                 __gk_err = __get_kernel_bad();                          \
 441                 break;                                                  \
 442         }                                                               \
 443         if (unlikely(__gk_err))                                         \
 444                 goto err_label;                                         \
 445 } while (0)
 446
 447 void __cmpxchg_user_key_called_with_bad_pointer(void);
 448
 449 #define CMPXCHG_USER_KEY_MAX_LOOPS 128
 450
 451 static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 452                                               __uint128_t old, __uint128_t new,
 453                                               unsigned long key, int size)
 454 {
 455         int rc = 0;
 456
 457         switch (size) {
 458         case 1: {
 459                 unsigned int prev, shift, mask, _old, _new;
 460                 unsigned long count;
 461
 462                 shift = (3 ^ (address & 3)) << 3;
 463                 address ^= address & 3;
 464                 _old = ((unsigned int)old & 0xff) << shift;
 465                 _new = ((unsigned int)new & 0xff) << shift;
 466                 mask = ~(0xff << shift);
 467                 asm volatile(
 468                         "       spka    0(%[key])\n"
 469                         "       sacf    256\n"
 470                         "       llill   %[count],%[max_loops]\n"
 471                         "0:     l       %[prev],%[address]\n"
 472                         "1:     nr      %[prev],%[mask]\n"
 473                         "       xilf    %[mask],0xffffffff\n"
 474                         "       or      %[new],%[prev]\n"
 475                         "       or      %[prev],%[tmp]\n"
 476                         "2:     lr      %[tmp],%[prev]\n"
 477                         "3:     cs      %[prev],%[new],%[address]\n"
 478                         "4:     jnl     5f\n"
 479                         "       xr      %[tmp],%[prev]\n"
 480                         "       xr      %[new],%[tmp]\n"
 481                         "       nr      %[tmp],%[mask]\n"
 482                         "       jnz     5f\n"
 483                         "       brct    %[count],2b\n"
 484                         "5:     sacf    768\n"
 485                         "       spka    %[default_key]\n"
 486                         EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
 487                         EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
 488                         EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
 489                         EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
 490                         : [rc] "+&d" (rc),
 491                           [prev] "=&d" (prev),
 492                           [address] "+Q" (*(int *)address),
 493                           [tmp] "+&d" (_old),
 494                           [new] "+&d" (_new),
 495                           [mask] "+&d" (mask),
 496                           [count] "=a" (count)
 497                         : [key] "%[count]" (key << 4),
 498                           [default_key] "J" (PAGE_DEFAULT_KEY),
 499                           [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
 500                         : "memory", "cc");
 501                 *(unsigned char *)uval = prev >> shift;
 502                 if (!count)
 503                         rc = -EAGAIN;
 504                 return rc;
 505         }
 506         case 2: {
 507                 unsigned int prev, shift, mask, _old, _new;
 508                 unsigned long count;
 509
 510                 shift = (2 ^ (address & 2)) << 3;
 511                 address ^= address & 2;
 512                 _old = ((unsigned int)old & 0xffff) << shift;
 513                 _new = ((unsigned int)new & 0xffff) << shift;
 514                 mask = ~(0xffff << shift);
 515                 asm volatile(
 516                         "       spka    0(%[key])\n"
 517                         "       sacf    256\n"
 518                         "       llill   %[count],%[max_loops]\n"
 519                         "0:     l       %[prev],%[address]\n"
 520                         "1:     nr      %[prev],%[mask]\n"
 521                         "       xilf    %[mask],0xffffffff\n"
 522                         "       or      %[new],%[prev]\n"
 523                         "       or      %[prev],%[tmp]\n"
 524                         "2:     lr      %[tmp],%[prev]\n"
 525                         "3:     cs      %[prev],%[new],%[address]\n"
 526                         "4:     jnl     5f\n"
 527                         "       xr      %[tmp],%[prev]\n"
 528                         "       xr      %[new],%[tmp]\n"
 529                         "       nr      %[tmp],%[mask]\n"
 530                         "       jnz     5f\n"
 531                         "       brct    %[count],2b\n"
 532                         "5:     sacf    768\n"
 533                         "       spka    %[default_key]\n"
 534                         EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
 535                         EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
 536                         EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
 537                         EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
 538                         : [rc] "+&d" (rc),
 539                           [prev] "=&d" (prev),
 540                           [address] "+Q" (*(int *)address),
 541                           [tmp] "+&d" (_old),
 542                           [new] "+&d" (_new),
 543                           [mask] "+&d" (mask),
 544                           [count] "=a" (count)
 545                         : [key] "%[count]" (key << 4),
 546                           [default_key] "J" (PAGE_DEFAULT_KEY),
 547                           [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
 548                         : "memory", "cc");
 549                 *(unsigned short *)uval = prev >> shift;
 550                 if (!count)
 551                         rc = -EAGAIN;
 552                 return rc;
 553         }
 554         case 4: {
 555                 unsigned int prev = old;
 556
 557                 asm volatile(
 558                         "       spka    0(%[key])\n"
 559                         "       sacf    256\n"
 560                         "0:     cs      %[prev],%[new],%[address]\n"
 561                         "1:     sacf    768\n"
 562                         "       spka    %[default_key]\n"
 563                         EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
 564                         EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
 565                         : [rc] "+&d" (rc),
 566                           [prev] "+&d" (prev),
 567                           [address] "+Q" (*(int *)address)
 568                         : [new] "d" ((unsigned int)new),
 569                           [key] "a" (key << 4),
 570                           [default_key] "J" (PAGE_DEFAULT_KEY)
 571                         : "memory", "cc");
 572                 *(unsigned int *)uval = prev;
 573                 return rc;
 574         }
 575         case 8: {
 576                 unsigned long prev = old;
 577
 578                 asm volatile(
 579                         "       spka    0(%[key])\n"
 580                         "       sacf    256\n"
 581                         "0:     csg     %[prev],%[new],%[address]\n"
 582                         "1:     sacf    768\n"
 583                         "       spka    %[default_key]\n"
 584                         EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
 585                         EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
 586                         : [rc] "+&d" (rc),
 587                           [prev] "+&d" (prev),
 588                           [address] "+QS" (*(long *)address)
 589                         : [new] "d" ((unsigned long)new),
 590                           [key] "a" (key << 4),
 591                           [default_key] "J" (PAGE_DEFAULT_KEY)
 592                         : "memory", "cc");
 593                 *(unsigned long *)uval = prev;
 594                 return rc;
 595         }
 596         case 16: {
 597                 __uint128_t prev = old;
 598
 599                 asm volatile(
 600                         "       spka    0(%[key])\n"
 601                         "       sacf    256\n"
 602                         "0:     cdsg    %[prev],%[new],%[address]\n"
 603                         "1:     sacf    768\n"
 604                         "       spka    %[default_key]\n"
 605                         EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev])
 606                         EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev])
 607                         : [rc] "+&d" (rc),
 608                           [prev] "+&d" (prev),
 609                           [address] "+QS" (*(__int128_t *)address)
 610                         : [new] "d" (new),
 611                           [key] "a" (key << 4),
 612                           [default_key] "J" (PAGE_DEFAULT_KEY)
 613                         : "memory", "cc");
 614                 *(__uint128_t *)uval = prev;
 615                 return rc;
 616         }
 617         }
 618         __cmpxchg_user_key_called_with_bad_pointer();
 619         return rc;
 620 }
 621
 622 /**
 623  * cmpxchg_user_key() - cmpxchg with user space target, honoring storage keys
 624  * @ptr: User space address of value to compare to @old and exchange with
 625  *       @new. Must be aligned to sizeof(*@ptr).
 626  * @uval: Address where the old value of *@ptr is written to.
 627  * @old: Old value. Compared to the content pointed to by @ptr in order to
 628  *       determine if the exchange occurs. The old value read from *@ptr is
 629  *       written to *@uval.
 630  * @new: New value to place at *@ptr.
 631  * @key: Access key to use for checking storage key protection.
 632  *
 633  * Perform a cmpxchg on a user space target, honoring storage key protection.
 634  * @key alone determines how key checking is performed, neither
 635  * storage-protection-override nor fetch-protection-override apply.
 636  * The caller must compare *@uval and @old to determine if values have been
 637  * exchanged. In case of an exception *@uval is set to zero.
 638  *
 639  * Return:     0: cmpxchg executed
 640  *             -EFAULT: an exception happened when trying to access *@ptr
 641  *             -EAGAIN: maxed out number of retries (byte and short only)
 642  */
 643 #define cmpxchg_user_key(ptr, uval, old, new, key)                      \
 644 ({                                                                      \
 645         __typeof__(ptr) __ptr = (ptr);                                  \
 646         __typeof__(uval) __uval = (uval);                               \
 647                                                                         \
 648         BUILD_BUG_ON(sizeof(*(__ptr)) != sizeof(*(__uval)));            \
 649         might_fault();                                                  \
 650         __chk_user_ptr(__ptr);                                          \
 651         __cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval),    \
 652                            (old), (new), (key), sizeof(*(__ptr)));      \
 653 })
 654
 655 #endif /* __S390_UACCESS_H */