tools/testing/selftests/mm/uffd-unit-tests.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Userfaultfd unit tests.
   4  *
   5  *  Copyright (C) 2015-2023  Red Hat, Inc.
   6  */
   7
   8 #include <asm-generic/unistd.h>
   9 #include "uffd-common.h"
  10
  11 #include "../../../../mm/gup_test.h"
  12
  13 /* The unit test doesn't need a large or random size, make it 32MB for now */
  14 #define  UFFD_TEST_MEM_SIZE               (32UL << 20)
  15
  16 #define  MEM_ANON                         BIT_ULL(0)
  17 #define  MEM_SHMEM                        BIT_ULL(1)
  18 #define  MEM_SHMEM_PRIVATE                BIT_ULL(2)
  19 #define  MEM_HUGETLB                      BIT_ULL(3)
  20 #define  MEM_HUGETLB_PRIVATE              BIT_ULL(4)
  21
  22 #define  MEM_ALL  (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
  23                    MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
  24
  25 #define ALIGN_UP(x, align_to) \
  26         ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
  27
  28 struct mem_type {
  29         const char *name;
  30         unsigned int mem_flag;
  31         uffd_test_ops_t *mem_ops;
  32         bool shared;
  33 };
  34 typedef struct mem_type mem_type_t;
  35
  36 mem_type_t mem_types[] = {
  37         {
  38                 .name = "anon",
  39                 .mem_flag = MEM_ANON,
  40                 .mem_ops = &anon_uffd_test_ops,
  41                 .shared = false,
  42         },
  43         {
  44                 .name = "shmem",
  45                 .mem_flag = MEM_SHMEM,
  46                 .mem_ops = &shmem_uffd_test_ops,
  47                 .shared = true,
  48         },
  49         {
  50                 .name = "shmem-private",
  51                 .mem_flag = MEM_SHMEM_PRIVATE,
  52                 .mem_ops = &shmem_uffd_test_ops,
  53                 .shared = false,
  54         },
  55         {
  56                 .name = "hugetlb",
  57                 .mem_flag = MEM_HUGETLB,
  58                 .mem_ops = &hugetlb_uffd_test_ops,
  59                 .shared = true,
  60         },
  61         {
  62                 .name = "hugetlb-private",
  63                 .mem_flag = MEM_HUGETLB_PRIVATE,
  64                 .mem_ops = &hugetlb_uffd_test_ops,
  65                 .shared = false,
  66         },
  67 };
  68
  69 /* Arguments to be passed over to each uffd unit test */
  70 struct uffd_test_args {
  71         mem_type_t *mem_type;
  72 };
  73 typedef struct uffd_test_args uffd_test_args_t;
  74
  75 /* Returns: UFFD_TEST_* */
  76 typedef void (*uffd_test_fn)(uffd_test_args_t *);
  77
  78 typedef struct {
  79         const char *name;
  80         uffd_test_fn uffd_fn;
  81         unsigned int mem_targets;
  82         uint64_t uffd_feature_required;
  83         uffd_test_case_ops_t *test_case_ops;
  84 } uffd_test_case_t;
  85
  86 static void uffd_test_report(void)
  87 {
  88         printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n",
  89                ksft_get_pass_cnt(),
  90                ksft_get_xskip_cnt(),
  91                ksft_get_fail_cnt(),
  92                ksft_test_num());
  93 }
  94
  95 static void uffd_test_pass(void)
  96 {
  97         printf("done\n");
  98         ksft_inc_pass_cnt();
  99 }
 100
 101 #define  uffd_test_start(...)  do {             \
 102                 printf("Testing ");             \
 103                 printf(__VA_ARGS__);            \
 104                 printf("... ");                 \
 105                 fflush(stdout);                 \
 106         } while (0)
 107
 108 #define  uffd_test_fail(...)  do {              \
 109                 printf("failed [reason: ");     \
 110                 printf(__VA_ARGS__);            \
 111                 printf("]\n");                  \
 112                 ksft_inc_fail_cnt();            \
 113         } while (0)
 114
 115 static void uffd_test_skip(const char *message)
 116 {
 117         printf("skipped [reason: %s]\n", message);
 118         ksft_inc_xskip_cnt();
 119 }
 120
 121 /*
 122  * Returns 1 if specific userfaultfd supported, 0 otherwise.  Note, we'll
 123  * return 1 even if some test failed as long as uffd supported, because in
 124  * that case we still want to proceed with the rest uffd unit tests.
 125  */
 126 static int test_uffd_api(bool use_dev)
 127 {
 128         struct uffdio_api uffdio_api;
 129         int uffd;
 130
 131         uffd_test_start("UFFDIO_API (with %s)",
 132                         use_dev ? "/dev/userfaultfd" : "syscall");
 133
 134         if (use_dev)
 135                 uffd = uffd_open_dev(UFFD_FLAGS);
 136         else
 137                 uffd = uffd_open_sys(UFFD_FLAGS);
 138         if (uffd < 0) {
 139                 uffd_test_skip("cannot open userfaultfd handle");
 140                 return 0;
 141         }
 142
 143         /* Test wrong UFFD_API */
 144         uffdio_api.api = 0xab;
 145         uffdio_api.features = 0;
 146         if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
 147                 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
 148                 goto out;
 149         }
 150
 151         /* Test wrong feature bit */
 152         uffdio_api.api = UFFD_API;
 153         uffdio_api.features = BIT_ULL(63);
 154         if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
 155                 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
 156                 goto out;
 157         }
 158
 159         /* Test normal UFFDIO_API */
 160         uffdio_api.api = UFFD_API;
 161         uffdio_api.features = 0;
 162         if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
 163                 uffd_test_fail("UFFDIO_API should succeed but failed");
 164                 goto out;
 165         }
 166
 167         /* Test double requests of UFFDIO_API with a random feature set */
 168         uffdio_api.features = BIT_ULL(0);
 169         if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
 170                 uffd_test_fail("UFFDIO_API should reject initialized uffd");
 171                 goto out;
 172         }
 173
 174         uffd_test_pass();
 175 out:
 176         close(uffd);
 177         /* We have a valid uffd handle */
 178         return 1;
 179 }
 180
 181 /*
 182  * This function initializes the global variables.  TODO: remove global
 183  * vars and then remove this.
 184  */
 185 static int
 186 uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test,
 187                        mem_type_t *mem_type, const char **errmsg)
 188 {
 189         map_shared = mem_type->shared;
 190         uffd_test_ops = mem_type->mem_ops;
 191         uffd_test_case_ops = test->test_case_ops;
 192
 193         if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))
 194                 page_size = default_huge_page_size();
 195         else
 196                 page_size = psize();
 197
 198         nr_pages = UFFD_TEST_MEM_SIZE / page_size;
 199         /* TODO: remove this global var.. it's so ugly */
 200         nr_cpus = 1;
 201
 202         /* Initialize test arguments */
 203         args->mem_type = mem_type;
 204
 205         return uffd_test_ctx_init(test->uffd_feature_required, errmsg);
 206 }
 207
 208 static bool uffd_feature_supported(uffd_test_case_t *test)
 209 {
 210         uint64_t features;
 211
 212         if (uffd_get_features(&features))
 213                 return false;
 214
 215         return (features & test->uffd_feature_required) ==
 216             test->uffd_feature_required;
 217 }
 218
 219 static int pagemap_open(void)
 220 {
 221         int fd = open("/proc/self/pagemap", O_RDONLY);
 222
 223         if (fd < 0)
 224                 err("open pagemap");
 225
 226         return fd;
 227 }
 228
 229 /* This macro let __LINE__ works in err() */
 230 #define  pagemap_check_wp(value, wp) do {                               \
 231                 if (!!(value & PM_UFFD_WP) != wp)                       \
 232                         err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
 233         } while (0)
 234
 235 typedef struct {
 236         int parent_uffd, child_uffd;
 237 } fork_event_args;
 238
 239 static void *fork_event_consumer(void *data)
 240 {
 241         fork_event_args *args = data;
 242         struct uffd_msg msg = { 0 };
 243
 244         ready_for_fork = true;
 245
 246         /* Read until a full msg received */
 247         while (uffd_read_msg(args->parent_uffd, &msg));
 248
 249         if (msg.event != UFFD_EVENT_FORK)
 250                 err("wrong message: %u\n", msg.event);
 251
 252         /* Just to be properly freed later */
 253         args->child_uffd = msg.arg.fork.ufd;
 254         return NULL;
 255 }
 256
 257 typedef struct {
 258         int gup_fd;
 259         bool pinned;
 260 } pin_args;
 261
 262 /*
 263  * Returns 0 if succeed, <0 for errors.  pin_pages() needs to be paired
 264  * with unpin_pages().  Currently it needs to be RO longterm pin to satisfy
 265  * all needs of the test cases (e.g., trigger unshare, trigger fork() early
 266  * CoW, etc.).
 267  */
 268 static int pin_pages(pin_args *args, void *buffer, size_t size)
 269 {
 270         struct pin_longterm_test test = {
 271                 .addr = (uintptr_t)buffer,
 272                 .size = size,
 273                 /* Read-only pins */
 274                 .flags = 0,
 275         };
 276
 277         if (args->pinned)
 278                 err("already pinned");
 279
 280         args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
 281         if (args->gup_fd < 0)
 282                 return -errno;
 283
 284         if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
 285                 /* Even if gup_test existed, can be an old gup_test / kernel */
 286                 close(args->gup_fd);
 287                 return -errno;
 288         }
 289         args->pinned = true;
 290         return 0;
 291 }
 292
 293 static void unpin_pages(pin_args *args)
 294 {
 295         if (!args->pinned)
 296                 err("unpin without pin first");
 297         if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
 298                 err("PIN_LONGTERM_TEST_STOP");
 299         close(args->gup_fd);
 300         args->pinned = false;
 301 }
 302
 303 static int pagemap_test_fork(int uffd, bool with_event, bool test_pin)
 304 {
 305         fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };
 306         pthread_t thread;
 307         pid_t child;
 308         uint64_t value;
 309         int fd, result;
 310
 311         /* Prepare a thread to resolve EVENT_FORK */
 312         if (with_event) {
 313                 ready_for_fork = false;
 314                 if (pthread_create(&thread, NULL, fork_event_consumer, &args))
 315                         err("pthread_create()");
 316                 while (!ready_for_fork)
 317                         ; /* Wait for the poll_thread to start executing before forking */
 318         }
 319
 320         child = fork();
 321         if (!child) {
 322                 /* Open the pagemap fd of the child itself */
 323                 pin_args args = {};
 324
 325                 fd = pagemap_open();
 326
 327                 if (test_pin && pin_pages(&args, area_dst, page_size))
 328                         /*
 329                          * Normally when reach here we have pinned in
 330                          * previous tests, so shouldn't fail anymore
 331                          */
 332                         err("pin page failed in child");
 333
 334                 value = pagemap_get_entry(fd, area_dst);
 335                 /*
 336                  * After fork(), we should handle uffd-wp bit differently:
 337                  *
 338                  * (1) when with EVENT_FORK, it should persist
 339                  * (2) when without EVENT_FORK, it should be dropped
 340                  */
 341                 pagemap_check_wp(value, with_event);
 342                 if (test_pin)
 343                         unpin_pages(&args);
 344                 /* Succeed */
 345                 exit(0);
 346         }
 347         waitpid(child, &result, 0);
 348
 349         if (with_event) {
 350                 if (pthread_join(thread, NULL))
 351                         err("pthread_join()");
 352                 if (args.child_uffd < 0)
 353                         err("Didn't receive child uffd");
 354                 close(args.child_uffd);
 355         }
 356
 357         return result;
 358 }
 359
 360 static void uffd_wp_unpopulated_test(uffd_test_args_t *args)
 361 {
 362         uint64_t value;
 363         int pagemap_fd;
 364
 365         if (uffd_register(uffd, area_dst, nr_pages * page_size,
 366                           false, true, false))
 367                 err("register failed");
 368
 369         pagemap_fd = pagemap_open();
 370
 371         /* Test applying pte marker to anon unpopulated */
 372         wp_range(uffd, (uint64_t)area_dst, page_size, true);
 373         value = pagemap_get_entry(pagemap_fd, area_dst);
 374         pagemap_check_wp(value, true);
 375
 376         /* Test unprotect on anon pte marker */
 377         wp_range(uffd, (uint64_t)area_dst, page_size, false);
 378         value = pagemap_get_entry(pagemap_fd, area_dst);
 379         pagemap_check_wp(value, false);
 380
 381         /* Test zap on anon marker */
 382         wp_range(uffd, (uint64_t)area_dst, page_size, true);
 383         if (madvise(area_dst, page_size, MADV_DONTNEED))
 384                 err("madvise(MADV_DONTNEED) failed");
 385         value = pagemap_get_entry(pagemap_fd, area_dst);
 386         pagemap_check_wp(value, false);
 387
 388         /* Test fault in after marker removed */
 389         *area_dst = 1;
 390         value = pagemap_get_entry(pagemap_fd, area_dst);
 391         pagemap_check_wp(value, false);
 392         /* Drop it to make pte none again */
 393         if (madvise(area_dst, page_size, MADV_DONTNEED))
 394                 err("madvise(MADV_DONTNEED) failed");
 395
 396         /* Test read-zero-page upon pte marker */
 397         wp_range(uffd, (uint64_t)area_dst, page_size, true);
 398         *(volatile char *)area_dst;
 399         /* Drop it to make pte none again */
 400         if (madvise(area_dst, page_size, MADV_DONTNEED))
 401                 err("madvise(MADV_DONTNEED) failed");
 402
 403         uffd_test_pass();
 404 }
 405
 406 static void uffd_wp_fork_test_common(uffd_test_args_t *args,
 407                                      bool with_event)
 408 {
 409         int pagemap_fd;
 410         uint64_t value;
 411
 412         if (uffd_register(uffd, area_dst, nr_pages * page_size,
 413                           false, true, false))
 414                 err("register failed");
 415
 416         pagemap_fd = pagemap_open();
 417
 418         /* Touch the page */
 419         *area_dst = 1;
 420         wp_range(uffd, (uint64_t)area_dst, page_size, true);
 421         value = pagemap_get_entry(pagemap_fd, area_dst);
 422         pagemap_check_wp(value, true);
 423         if (pagemap_test_fork(uffd, with_event, false)) {
 424                 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
 425                                with_event ? "missing" : "stall");
 426                 goto out;
 427         }
 428
 429         /*
 430          * This is an attempt for zapping the pgtable so as to test the
 431          * markers.
 432          *
 433          * For private mappings, PAGEOUT will only work on exclusive ptes
 434          * (PM_MMAP_EXCLUSIVE) which we should satisfy.
 435          *
 436          * For shared, PAGEOUT may not work.  Use DONTNEED instead which
 437          * plays a similar role of zapping (rather than freeing the page)
 438          * to expose pte markers.
 439          */
 440         if (args->mem_type->shared) {
 441                 if (madvise(area_dst, page_size, MADV_DONTNEED))
 442                         err("MADV_DONTNEED");
 443         } else {
 444                 /*
 445                  * NOTE: ignore retval because private-hugetlb doesn't yet
 446                  * support swapping, so it could fail.
 447                  */
 448                 madvise(area_dst, page_size, MADV_PAGEOUT);
 449         }
 450
 451         /* Uffd-wp should persist even swapped out */
 452         value = pagemap_get_entry(pagemap_fd, area_dst);
 453         pagemap_check_wp(value, true);
 454         if (pagemap_test_fork(uffd, with_event, false)) {
 455                 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
 456                                with_event ? "missing" : "stall");
 457                 goto out;
 458         }
 459
 460         /* Unprotect; this tests swap pte modifications */
 461         wp_range(uffd, (uint64_t)area_dst, page_size, false);
 462         value = pagemap_get_entry(pagemap_fd, area_dst);
 463         pagemap_check_wp(value, false);
 464
 465         /* Fault in the page from disk */
 466         *area_dst = 2;
 467         value = pagemap_get_entry(pagemap_fd, area_dst);
 468         pagemap_check_wp(value, false);
 469         uffd_test_pass();
 470 out:
 471         if (uffd_unregister(uffd, area_dst, nr_pages * page_size))
 472                 err("unregister failed");
 473         close(pagemap_fd);
 474 }
 475
 476 static void uffd_wp_fork_test(uffd_test_args_t *args)
 477 {
 478         uffd_wp_fork_test_common(args, false);
 479 }
 480
 481 static void uffd_wp_fork_with_event_test(uffd_test_args_t *args)
 482 {
 483         uffd_wp_fork_test_common(args, true);
 484 }
 485
 486 static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args,
 487                                          bool with_event)
 488 {
 489         int pagemap_fd;
 490         pin_args pin_args = {};
 491
 492         if (uffd_register(uffd, area_dst, page_size, false, true, false))
 493                 err("register failed");
 494
 495         pagemap_fd = pagemap_open();
 496
 497         /* Touch the page */
 498         *area_dst = 1;
 499         wp_range(uffd, (uint64_t)area_dst, page_size, true);
 500
 501         /*
 502          * 1. First pin, then fork().  This tests fork() special path when
 503          * doing early CoW if the page is private.
 504          */
 505         if (pin_pages(&pin_args, area_dst, page_size)) {
 506                 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
 507                                "or unprivileged");
 508                 close(pagemap_fd);
 509                 uffd_unregister(uffd, area_dst, page_size);
 510                 return;
 511         }
 512
 513         if (pagemap_test_fork(uffd, with_event, false)) {
 514                 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
 515                                with_event ? "missing" : "stall");
 516                 unpin_pages(&pin_args);
 517                 goto out;
 518         }
 519
 520         unpin_pages(&pin_args);
 521
 522         /*
 523          * 2. First fork(), then pin (in the child, where test_pin==true).
 524          * This tests COR, aka, page unsharing on private memories.
 525          */
 526         if (pagemap_test_fork(uffd, with_event, true)) {
 527                 uffd_test_fail("Detected %s uffd-wp bit when RO pin",
 528                                with_event ? "missing" : "stall");
 529                 goto out;
 530         }
 531         uffd_test_pass();
 532 out:
 533         if (uffd_unregister(uffd, area_dst, page_size))
 534                 err("register failed");
 535         close(pagemap_fd);
 536 }
 537
 538 static void uffd_wp_fork_pin_test(uffd_test_args_t *args)
 539 {
 540         uffd_wp_fork_pin_test_common(args, false);
 541 }
 542
 543 static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args)
 544 {
 545         uffd_wp_fork_pin_test_common(args, true);
 546 }
 547
 548 static void check_memory_contents(char *p)
 549 {
 550         unsigned long i, j;
 551         uint8_t expected_byte;
 552
 553         for (i = 0; i < nr_pages; ++i) {
 554                 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
 555                 for (j = 0; j < page_size; j++) {
 556                         uint8_t v = *(uint8_t *)(p + (i * page_size) + j);
 557                         if (v != expected_byte)
 558                                 err("unexpected page contents");
 559                 }
 560         }
 561 }
 562
 563 static void uffd_minor_test_common(bool test_collapse, bool test_wp)
 564 {
 565         unsigned long p;
 566         pthread_t uffd_mon;
 567         char c;
 568         struct uffd_args args = { 0 };
 569
 570         /*
 571          * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
 572          * both do not make much sense.
 573          */
 574         assert(!(test_collapse && test_wp));
 575
 576         if (uffd_register(uffd, area_dst_alias, nr_pages * page_size,
 577                           /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
 578                           false, test_wp, true))
 579                 err("register failure");
 580
 581         /*
 582          * After registering with UFFD, populate the non-UFFD-registered side of
 583          * the shared mapping. This should *not* trigger any UFFD minor faults.
 584          */
 585         for (p = 0; p < nr_pages; ++p)
 586                 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
 587                        page_size);
 588
 589         args.apply_wp = test_wp;
 590         if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
 591                 err("uffd_poll_thread create");
 592
 593         /*
 594          * Read each of the pages back using the UFFD-registered mapping. We
 595          * expect that the first time we touch a page, it will result in a minor
 596          * fault. uffd_poll_thread will resolve the fault by bit-flipping the
 597          * page's contents, and then issuing a CONTINUE ioctl.
 598          */
 599         check_memory_contents(area_dst_alias);
 600
 601         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
 602                 err("pipe write");
 603         if (pthread_join(uffd_mon, NULL))
 604                 err("join() failed");
 605
 606         if (test_collapse) {
 607                 if (madvise(area_dst_alias, nr_pages * page_size,
 608                             MADV_COLLAPSE)) {
 609                         /* It's fine to fail for this one... */
 610                         uffd_test_skip("MADV_COLLAPSE failed");
 611                         return;
 612                 }
 613
 614                 uffd_test_ops->check_pmd_mapping(area_dst,
 615                                                  nr_pages * page_size /
 616                                                  read_pmd_pagesize());
 617                 /*
 618                  * This won't cause uffd-fault - it purely just makes sure there
 619                  * was no corruption.
 620                  */
 621                 check_memory_contents(area_dst_alias);
 622         }
 623
 624         if (args.missing_faults != 0 || args.minor_faults != nr_pages)
 625                 uffd_test_fail("stats check error");
 626         else
 627                 uffd_test_pass();
 628 }
 629
 630 void uffd_minor_test(uffd_test_args_t *args)
 631 {
 632         uffd_minor_test_common(false, false);
 633 }
 634
 635 void uffd_minor_wp_test(uffd_test_args_t *args)
 636 {
 637         uffd_minor_test_common(false, true);
 638 }
 639
 640 void uffd_minor_collapse_test(uffd_test_args_t *args)
 641 {
 642         uffd_minor_test_common(true, false);
 643 }
 644
 645 static sigjmp_buf jbuf, *sigbuf;
 646
 647 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
 648 {
 649         if (sig == SIGBUS) {
 650                 if (sigbuf)
 651                         siglongjmp(*sigbuf, 1);
 652                 abort();
 653         }
 654 }
 655
 656 /*
 657  * For non-cooperative userfaultfd test we fork() a process that will
 658  * generate pagefaults, will mremap the area monitored by the
 659  * userfaultfd and at last this process will release the monitored
 660  * area.
 661  * For the anonymous and shared memory the area is divided into two
 662  * parts, the first part is accessed before mremap, and the second
 663  * part is accessed after mremap. Since hugetlbfs does not support
 664  * mremap, the entire monitored area is accessed in a single pass for
 665  * HUGETLB_TEST.
 666  * The release of the pages currently generates event for shmem and
 667  * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
 668  * for hugetlb.
 669  * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
 670  * monitored area, generate pagefaults and test that signal is delivered.
 671  * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
 672  * test robustness use case - we release monitored area, fork a process
 673  * that will generate pagefaults and verify signal is generated.
 674  * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
 675  * feature. Using monitor thread, verify no userfault events are generated.
 676  */
 677 static int faulting_process(int signal_test, bool wp)
 678 {
 679         unsigned long nr, i;
 680         unsigned long long count;
 681         unsigned long split_nr_pages;
 682         unsigned long lastnr;
 683         struct sigaction act;
 684         volatile unsigned long signalled = 0;
 685
 686         split_nr_pages = (nr_pages + 1) / 2;
 687
 688         if (signal_test) {
 689                 sigbuf = &jbuf;
 690                 memset(&act, 0, sizeof(act));
 691                 act.sa_sigaction = sighndl;
 692                 act.sa_flags = SA_SIGINFO;
 693                 if (sigaction(SIGBUS, &act, 0))
 694                         err("sigaction");
 695                 lastnr = (unsigned long)-1;
 696         }
 697
 698         for (nr = 0; nr < split_nr_pages; nr++) {
 699                 volatile int steps = 1;
 700                 unsigned long offset = nr * page_size;
 701
 702                 if (signal_test) {
 703                         if (sigsetjmp(*sigbuf, 1) != 0) {
 704                                 if (steps == 1 && nr == lastnr)
 705                                         err("Signal repeated");
 706
 707                                 lastnr = nr;
 708                                 if (signal_test == 1) {
 709                                         if (steps == 1) {
 710                                                 /* This is a MISSING request */
 711                                                 steps++;
 712                                                 if (copy_page(uffd, offset, wp))
 713                                                         signalled++;
 714                                         } else {
 715                                                 /* This is a WP request */
 716                                                 assert(steps == 2);
 717                                                 wp_range(uffd,
 718                                                          (__u64)area_dst +
 719                                                          offset,
 720                                                          page_size, false);
 721                                         }
 722                                 } else {
 723                                         signalled++;
 724                                         continue;
 725                                 }
 726                         }
 727                 }
 728
 729                 count = *area_count(area_dst, nr);
 730                 if (count != count_verify[nr])
 731                         err("nr %lu memory corruption %llu %llu\n",
 732                             nr, count, count_verify[nr]);
 733                 /*
 734                  * Trigger write protection if there is by writing
 735                  * the same value back.
 736                  */
 737                 *area_count(area_dst, nr) = count;
 738         }
 739
 740         if (signal_test)
 741                 return signalled != split_nr_pages;
 742
 743         area_dst = mremap(area_dst, nr_pages * page_size,  nr_pages * page_size,
 744                           MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
 745         if (area_dst == MAP_FAILED)
 746                 err("mremap");
 747         /* Reset area_src since we just clobbered it */
 748         area_src = NULL;
 749
 750         for (; nr < nr_pages; nr++) {
 751                 count = *area_count(area_dst, nr);
 752                 if (count != count_verify[nr]) {
 753                         err("nr %lu memory corruption %llu %llu\n",
 754                             nr, count, count_verify[nr]);
 755                 }
 756                 /*
 757                  * Trigger write protection if there is by writing
 758                  * the same value back.
 759                  */
 760                 *area_count(area_dst, nr) = count;
 761         }
 762
 763         uffd_test_ops->release_pages(area_dst);
 764
 765         for (nr = 0; nr < nr_pages; nr++)
 766                 for (i = 0; i < page_size; i++)
 767                         if (*(area_dst + nr * page_size + i) != 0)
 768                                 err("page %lu offset %lu is not zero", nr, i);
 769
 770         return 0;
 771 }
 772
 773 static void uffd_sigbus_test_common(bool wp)
 774 {
 775         unsigned long userfaults;
 776         pthread_t uffd_mon;
 777         pid_t pid;
 778         int err;
 779         char c;
 780         struct uffd_args args = { 0 };
 781
 782         ready_for_fork = false;
 783
 784         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
 785
 786         if (uffd_register(uffd, area_dst, nr_pages * page_size,
 787                           true, wp, false))
 788                 err("register failure");
 789
 790         if (faulting_process(1, wp))
 791                 err("faulting process failed");
 792
 793         uffd_test_ops->release_pages(area_dst);
 794
 795         args.apply_wp = wp;
 796         if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
 797                 err("uffd_poll_thread create");
 798
 799         while (!ready_for_fork)
 800                 ; /* Wait for the poll_thread to start executing before forking */
 801
 802         pid = fork();
 803         if (pid < 0)
 804                 err("fork");
 805
 806         if (!pid)
 807                 exit(faulting_process(2, wp));
 808
 809         waitpid(pid, &err, 0);
 810         if (err)
 811                 err("faulting process failed");
 812         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
 813                 err("pipe write");
 814         if (pthread_join(uffd_mon, (void **)&userfaults))
 815                 err("pthread_join()");
 816
 817         if (userfaults)
 818                 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
 819         else
 820                 uffd_test_pass();
 821 }
 822
 823 static void uffd_sigbus_test(uffd_test_args_t *args)
 824 {
 825         uffd_sigbus_test_common(false);
 826 }
 827
 828 static void uffd_sigbus_wp_test(uffd_test_args_t *args)
 829 {
 830         uffd_sigbus_test_common(true);
 831 }
 832
 833 static void uffd_events_test_common(bool wp)
 834 {
 835         pthread_t uffd_mon;
 836         pid_t pid;
 837         int err;
 838         char c;
 839         struct uffd_args args = { 0 };
 840
 841         ready_for_fork = false;
 842
 843         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
 844         if (uffd_register(uffd, area_dst, nr_pages * page_size,
 845                           true, wp, false))
 846                 err("register failure");
 847
 848         args.apply_wp = wp;
 849         if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
 850                 err("uffd_poll_thread create");
 851
 852         while (!ready_for_fork)
 853                 ; /* Wait for the poll_thread to start executing before forking */
 854
 855         pid = fork();
 856         if (pid < 0)
 857                 err("fork");
 858
 859         if (!pid)
 860                 exit(faulting_process(0, wp));
 861
 862         waitpid(pid, &err, 0);
 863         if (err)
 864                 err("faulting process failed");
 865         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
 866                 err("pipe write");
 867         if (pthread_join(uffd_mon, NULL))
 868                 err("pthread_join()");
 869
 870         if (args.missing_faults != nr_pages)
 871                 uffd_test_fail("Fault counts wrong");
 872         else
 873                 uffd_test_pass();
 874 }
 875
 876 static void uffd_events_test(uffd_test_args_t *args)
 877 {
 878         uffd_events_test_common(false);
 879 }
 880
 881 static void uffd_events_wp_test(uffd_test_args_t *args)
 882 {
 883         uffd_events_test_common(true);
 884 }
 885
 886 static void retry_uffdio_zeropage(int ufd,
 887                                   struct uffdio_zeropage *uffdio_zeropage)
 888 {
 889         uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
 890                                      uffdio_zeropage->range.len,
 891                                      0);
 892         if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
 893                 if (uffdio_zeropage->zeropage != -EEXIST)
 894                         err("UFFDIO_ZEROPAGE error: %"PRId64,
 895                             (int64_t)uffdio_zeropage->zeropage);
 896         } else {
 897                 err("UFFDIO_ZEROPAGE error: %"PRId64,
 898                     (int64_t)uffdio_zeropage->zeropage);
 899         }
 900 }
 901
 902 static bool do_uffdio_zeropage(int ufd, bool has_zeropage)
 903 {
 904         struct uffdio_zeropage uffdio_zeropage = { 0 };
 905         int ret;
 906         __s64 res;
 907
 908         uffdio_zeropage.range.start = (unsigned long) area_dst;
 909         uffdio_zeropage.range.len = page_size;
 910         uffdio_zeropage.mode = 0;
 911         ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
 912         res = uffdio_zeropage.zeropage;
 913         if (ret) {
 914                 /* real retval in ufdio_zeropage.zeropage */
 915                 if (has_zeropage)
 916                         err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
 917                 else if (res != -EINVAL)
 918                         err("UFFDIO_ZEROPAGE not -EINVAL");
 919         } else if (has_zeropage) {
 920                 if (res != page_size)
 921                         err("UFFDIO_ZEROPAGE unexpected size");
 922                 else
 923                         retry_uffdio_zeropage(ufd, &uffdio_zeropage);
 924                 return true;
 925         } else
 926                 err("UFFDIO_ZEROPAGE succeeded");
 927
 928         return false;
 929 }
 930
 931 /*
 932  * Registers a range with MISSING mode only for zeropage test.  Return true
 933  * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
 934  * because we want to detect .ioctls along the way.
 935  */
 936 static bool
 937 uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
 938 {
 939         uint64_t ioctls = 0;
 940
 941         if (uffd_register_with_ioctls(uffd, addr, len, true,
 942                                       false, false, &ioctls))
 943                 err("zeropage register fail");
 944
 945         return ioctls & (1 << _UFFDIO_ZEROPAGE);
 946 }
 947
 948 /* exercise UFFDIO_ZEROPAGE */
 949 static void uffd_zeropage_test(uffd_test_args_t *args)
 950 {
 951         bool has_zeropage;
 952         int i;
 953
 954         has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size);
 955         if (area_dst_alias)
 956                 /* Ignore the retval; we already have it */
 957                 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size);
 958
 959         if (do_uffdio_zeropage(uffd, has_zeropage))
 960                 for (i = 0; i < page_size; i++)
 961                         if (area_dst[i] != 0)
 962                                 err("data non-zero at offset %d\n", i);
 963
 964         if (uffd_unregister(uffd, area_dst, page_size))
 965                 err("unregister");
 966
 967         if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size))
 968                 err("unregister");
 969
 970         uffd_test_pass();
 971 }
 972
 973 static void uffd_register_poison(int uffd, void *addr, uint64_t len)
 974 {
 975         uint64_t ioctls = 0;
 976         uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
 977
 978         if (uffd_register_with_ioctls(uffd, addr, len, true,
 979                                       false, false, &ioctls))
 980                 err("poison register fail");
 981
 982         if ((ioctls & expected) != expected)
 983                 err("registered area doesn't support COPY and POISON ioctls");
 984 }
 985
 986 static void do_uffdio_poison(int uffd, unsigned long offset)
 987 {
 988         struct uffdio_poison uffdio_poison = { 0 };
 989         int ret;
 990         __s64 res;
 991
 992         uffdio_poison.range.start = (unsigned long) area_dst + offset;
 993         uffdio_poison.range.len = page_size;
 994         uffdio_poison.mode = 0;
 995         ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison);
 996         res = uffdio_poison.updated;
 997
 998         if (ret)
 999                 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
1000         else if (res != page_size)
1001                 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
1002 }
1003
1004 static void uffd_poison_handle_fault(
1005         struct uffd_msg *msg, struct uffd_args *args)
1006 {
1007         unsigned long offset;
1008
1009         if (msg->event != UFFD_EVENT_PAGEFAULT)
1010                 err("unexpected msg event %u", msg->event);
1011
1012         if (msg->arg.pagefault.flags &
1013             (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
1014                 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1015
1016         offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1017         offset &= ~(page_size-1);
1018
1019         /* Odd pages -> copy zeroed page; even pages -> poison. */
1020         if (offset & page_size)
1021                 copy_page(uffd, offset, false);
1022         else
1023                 do_uffdio_poison(uffd, offset);
1024 }
1025
1026 static void uffd_poison_test(uffd_test_args_t *targs)
1027 {
1028         pthread_t uffd_mon;
1029         char c;
1030         struct uffd_args args = { 0 };
1031         struct sigaction act = { 0 };
1032         unsigned long nr_sigbus = 0;
1033         unsigned long nr;
1034
1035         fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1036
1037         uffd_register_poison(uffd, area_dst, nr_pages * page_size);
1038         memset(area_src, 0, nr_pages * page_size);
1039
1040         args.handle_fault = uffd_poison_handle_fault;
1041         if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1042                 err("uffd_poll_thread create");
1043
1044         sigbuf = &jbuf;
1045         act.sa_sigaction = sighndl;
1046         act.sa_flags = SA_SIGINFO;
1047         if (sigaction(SIGBUS, &act, 0))
1048                 err("sigaction");
1049
1050         for (nr = 0; nr < nr_pages; ++nr) {
1051                 unsigned long offset = nr * page_size;
1052                 const char *bytes = (const char *) area_dst + offset;
1053                 const char *i;
1054
1055                 if (sigsetjmp(*sigbuf, 1)) {
1056                         /*
1057                          * Access below triggered a SIGBUS, which was caught by
1058                          * sighndl, which then jumped here. Count this SIGBUS,
1059                          * and move on to next page.
1060                          */
1061                         ++nr_sigbus;
1062                         continue;
1063                 }
1064
1065                 for (i = bytes; i < bytes + page_size; ++i) {
1066                         if (*i)
1067                                 err("nonzero byte in area_dst (%p) at %p: %u",
1068                                     area_dst, i, *i);
1069                 }
1070         }
1071
1072         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1073                 err("pipe write");
1074         if (pthread_join(uffd_mon, NULL))
1075                 err("pthread_join()");
1076
1077         if (nr_sigbus != nr_pages / 2)
1078                 err("expected to receive %lu SIGBUS, actually received %lu",
1079                     nr_pages / 2, nr_sigbus);
1080
1081         uffd_test_pass();
1082 }
1083
1084 static void
1085 uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args,
1086                               unsigned long len)
1087 {
1088         unsigned long offset;
1089
1090         if (msg->event != UFFD_EVENT_PAGEFAULT)
1091                 err("unexpected msg event %u", msg->event);
1092
1093         if (msg->arg.pagefault.flags &
1094             (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
1095                 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1096
1097         offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1098         offset &= ~(len-1);
1099
1100         if (move_page(uffd, offset, len))
1101                 args->missing_faults++;
1102 }
1103
1104 static void uffd_move_handle_fault(struct uffd_msg *msg,
1105                                    struct uffd_args *args)
1106 {
1107         uffd_move_handle_fault_common(msg, args, page_size);
1108 }
1109
1110 static void uffd_move_pmd_handle_fault(struct uffd_msg *msg,
1111                                        struct uffd_args *args)
1112 {
1113         uffd_move_handle_fault_common(msg, args, read_pmd_pagesize());
1114 }
1115
1116 static void
1117 uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size,
1118                       void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args))
1119 {
1120         unsigned long nr;
1121         pthread_t uffd_mon;
1122         char c;
1123         unsigned long long count;
1124         struct uffd_args args = { 0 };
1125         char *orig_area_src, *orig_area_dst;
1126         unsigned long step_size, step_count;
1127         unsigned long src_offs = 0;
1128         unsigned long dst_offs = 0;
1129
1130         /* Prevent source pages from being mapped more than once */
1131         if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK))
1132                 err("madvise(MADV_DONTFORK) failure");
1133
1134         if (uffd_register(uffd, area_dst, nr_pages * page_size,
1135                           true, false, false))
1136                 err("register failure");
1137
1138         args.handle_fault = handle_fault;
1139         if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1140                 err("uffd_poll_thread create");
1141
1142         step_size = chunk_size / page_size;
1143         step_count = nr_pages / step_size;
1144
1145         if (chunk_size > page_size) {
1146                 char *aligned_src = ALIGN_UP(area_src, chunk_size);
1147                 char *aligned_dst = ALIGN_UP(area_dst, chunk_size);
1148
1149                 if (aligned_src != area_src || aligned_dst != area_dst) {
1150                         src_offs = (aligned_src - area_src) / page_size;
1151                         dst_offs = (aligned_dst - area_dst) / page_size;
1152                         step_count--;
1153                 }
1154                 orig_area_src = area_src;
1155                 orig_area_dst = area_dst;
1156                 area_src = aligned_src;
1157                 area_dst = aligned_dst;
1158         }
1159
1160         /*
1161          * Read each of the pages back using the UFFD-registered mapping. We
1162          * expect that the first time we touch a page, it will result in a missing
1163          * fault. uffd_poll_thread will resolve the fault by moving source
1164          * page to destination.
1165          */
1166         for (nr = 0; nr < step_count * step_size; nr += step_size) {
1167                 unsigned long i;
1168
1169                 /* Check area_src content */
1170                 for (i = 0; i < step_size; i++) {
1171                         count = *area_count(area_src, nr + i);
1172                         if (count != count_verify[src_offs + nr + i])
1173                                 err("nr %lu source memory invalid %llu %llu\n",
1174                                     nr + i, count, count_verify[src_offs + nr + i]);
1175                 }
1176
1177                 /* Faulting into area_dst should move the page or the huge page */
1178                 for (i = 0; i < step_size; i++) {
1179                         count = *area_count(area_dst, nr + i);
1180                         if (count != count_verify[dst_offs + nr + i])
1181                                 err("nr %lu memory corruption %llu %llu\n",
1182                                     nr, count, count_verify[dst_offs + nr + i]);
1183                 }
1184
1185                 /* Re-check area_src content which should be empty */
1186                 for (i = 0; i < step_size; i++) {
1187                         count = *area_count(area_src, nr + i);
1188                         if (count != 0)
1189                                 err("nr %lu move failed %llu %llu\n",
1190                                     nr, count, count_verify[src_offs + nr + i]);
1191                 }
1192         }
1193         if (step_size > page_size) {
1194                 area_src = orig_area_src;
1195                 area_dst = orig_area_dst;
1196         }
1197
1198         if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1199                 err("pipe write");
1200         if (pthread_join(uffd_mon, NULL))
1201                 err("join() failed");
1202
1203         if (args.missing_faults != step_count || args.minor_faults != 0)
1204                 uffd_test_fail("stats check error");
1205         else
1206                 uffd_test_pass();
1207 }
1208
1209 static void uffd_move_test(uffd_test_args_t *targs)
1210 {
1211         uffd_move_test_common(targs, page_size, uffd_move_handle_fault);
1212 }
1213
1214 static void uffd_move_pmd_test(uffd_test_args_t *targs)
1215 {
1216         if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1217                 err("madvise(MADV_HUGEPAGE) failure");
1218         uffd_move_test_common(targs, read_pmd_pagesize(),
1219                               uffd_move_pmd_handle_fault);
1220 }
1221
1222 static void uffd_move_pmd_split_test(uffd_test_args_t *targs)
1223 {
1224         if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1225                 err("madvise(MADV_NOHUGEPAGE) failure");
1226         uffd_move_test_common(targs, read_pmd_pagesize(),
1227                               uffd_move_pmd_handle_fault);
1228 }
1229
1230 static int prevent_hugepages(const char **errmsg)
1231 {
1232         /* This should be done before source area is populated */
1233         if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) {
1234                 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
1235                 if (errno != EINVAL) {
1236                         if (errmsg)
1237                                 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";
1238                         return -errno;
1239                 }
1240         }
1241         return 0;
1242 }
1243
1244 static int request_hugepages(const char **errmsg)
1245 {
1246         /* This should be done before source area is populated */
1247         if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) {
1248                 if (errmsg) {
1249                         *errmsg = (errno == EINVAL) ?
1250                                 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :
1251                                 "madvise(MADV_HUGEPAGE) failed";
1252                 }
1253                 return -errno;
1254         }
1255         return 0;
1256 }
1257
1258 struct uffd_test_case_ops uffd_move_test_case_ops = {
1259         .post_alloc = prevent_hugepages,
1260 };
1261
1262 struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
1263         .post_alloc = request_hugepages,
1264 };
1265
1266 /*
1267  * Test the returned uffdio_register.ioctls with different register modes.
1268  * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
1269  */
1270 static void
1271 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)
1272 {
1273         uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
1274         mem_type_t *mem_type = args->mem_type;
1275         int ret;
1276
1277         ret = uffd_register_with_ioctls(uffd, area_dst, page_size,
1278                                         miss, wp, minor, &ioctls);
1279
1280         /*
1281          * Handle special cases of UFFDIO_REGISTER here where it should
1282          * just fail with -EINVAL first..
1283          *
1284          * Case 1: register MINOR on anon
1285          * Case 2: register with no mode selected
1286          */
1287         if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
1288             (!miss && !wp && !minor)) {
1289                 if (ret != -EINVAL)
1290                         err("register (miss=%d, wp=%d, minor=%d) failed "
1291                             "with wrong errno=%d", miss, wp, minor, ret);
1292                 return;
1293         }
1294
1295         /* UFFDIO_REGISTER should succeed, then check ioctls returned */
1296         if (miss)
1297                 expected |= BIT_ULL(_UFFDIO_COPY);
1298         if (wp)
1299                 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
1300         if (minor)
1301                 expected |= BIT_ULL(_UFFDIO_CONTINUE);
1302
1303         if ((ioctls & expected) != expected)
1304                 err("unexpected uffdio_register.ioctls "
1305                     "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
1306                     "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
1307
1308         if (uffd_unregister(uffd, area_dst, page_size))
1309                 err("unregister");
1310 }
1311
1312 static void uffd_register_ioctls_test(uffd_test_args_t *args)
1313 {
1314         int miss, wp, minor;
1315
1316         for (miss = 0; miss <= 1; miss++)
1317                 for (wp = 0; wp <= 1; wp++)
1318                         for (minor = 0; minor <= 1; minor++)
1319                                 do_register_ioctls_test(args, miss, wp, minor);
1320
1321         uffd_test_pass();
1322 }
1323
1324 uffd_test_case_t uffd_tests[] = {
1325         {
1326                 /* Test returned uffdio_register.ioctls. */
1327                 .name = "register-ioctls",
1328                 .uffd_fn = uffd_register_ioctls_test,
1329                 .mem_targets = MEM_ALL,
1330                 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
1331                 UFFD_FEATURE_MISSING_SHMEM |
1332                 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1333                 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1334                 UFFD_FEATURE_MINOR_HUGETLBFS |
1335                 UFFD_FEATURE_MINOR_SHMEM,
1336         },
1337         {
1338                 .name = "zeropage",
1339                 .uffd_fn = uffd_zeropage_test,
1340                 .mem_targets = MEM_ALL,
1341                 .uffd_feature_required = 0,
1342         },
1343         {
1344                 .name = "move",
1345                 .uffd_fn = uffd_move_test,
1346                 .mem_targets = MEM_ANON,
1347                 .uffd_feature_required = UFFD_FEATURE_MOVE,
1348                 .test_case_ops = &uffd_move_test_case_ops,
1349         },
1350         {
1351                 .name = "move-pmd",
1352                 .uffd_fn = uffd_move_pmd_test,
1353                 .mem_targets = MEM_ANON,
1354                 .uffd_feature_required = UFFD_FEATURE_MOVE,
1355                 .test_case_ops = &uffd_move_test_pmd_case_ops,
1356         },
1357         {
1358                 .name = "move-pmd-split",
1359                 .uffd_fn = uffd_move_pmd_split_test,
1360                 .mem_targets = MEM_ANON,
1361                 .uffd_feature_required = UFFD_FEATURE_MOVE,
1362                 .test_case_ops = &uffd_move_test_pmd_case_ops,
1363         },
1364         {
1365                 .name = "wp-fork",
1366                 .uffd_fn = uffd_wp_fork_test,
1367                 .mem_targets = MEM_ALL,
1368                 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1369                 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1370         },
1371         {
1372                 .name = "wp-fork-with-event",
1373                 .uffd_fn = uffd_wp_fork_with_event_test,
1374                 .mem_targets = MEM_ALL,
1375                 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1376                 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1377                 /* when set, child process should inherit uffd-wp bits */
1378                 UFFD_FEATURE_EVENT_FORK,
1379         },
1380         {
1381                 .name = "wp-fork-pin",
1382                 .uffd_fn = uffd_wp_fork_pin_test,
1383                 .mem_targets = MEM_ALL,
1384                 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1385                 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1386         },
1387         {
1388                 .name = "wp-fork-pin-with-event",
1389                 .uffd_fn = uffd_wp_fork_pin_with_event_test,
1390                 .mem_targets = MEM_ALL,
1391                 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1392                 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1393                 /* when set, child process should inherit uffd-wp bits */
1394                 UFFD_FEATURE_EVENT_FORK,
1395         },
1396         {
1397                 .name = "wp-unpopulated",
1398                 .uffd_fn = uffd_wp_unpopulated_test,
1399                 .mem_targets = MEM_ANON,
1400                 .uffd_feature_required =
1401                 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
1402         },
1403         {
1404                 .name = "minor",
1405                 .uffd_fn = uffd_minor_test,
1406                 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1407                 .uffd_feature_required =
1408                 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
1409         },
1410         {
1411                 .name = "minor-wp",
1412                 .uffd_fn = uffd_minor_wp_test,
1413                 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1414                 .uffd_feature_required =
1415                 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
1416                 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1417                 /*
1418                  * HACK: here we leveraged WP_UNPOPULATED to detect whether
1419                  * minor mode supports wr-protect.  There's no feature flag
1420                  * for it so this is the best we can test against.
1421                  */
1422                 UFFD_FEATURE_WP_UNPOPULATED,
1423         },
1424         {
1425                 .name = "minor-collapse",
1426                 .uffd_fn = uffd_minor_collapse_test,
1427                 /* MADV_COLLAPSE only works with shmem */
1428                 .mem_targets = MEM_SHMEM,
1429                 /* We can't test MADV_COLLAPSE, so try our luck */
1430                 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
1431         },
1432         {
1433                 .name = "sigbus",
1434                 .uffd_fn = uffd_sigbus_test,
1435                 .mem_targets = MEM_ALL,
1436                 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1437                 UFFD_FEATURE_EVENT_FORK,
1438         },
1439         {
1440                 .name = "sigbus-wp",
1441                 .uffd_fn = uffd_sigbus_wp_test,
1442                 .mem_targets = MEM_ALL,
1443                 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1444                 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1445                 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1446         },
1447         {
1448                 .name = "events",
1449                 .uffd_fn = uffd_events_test,
1450                 .mem_targets = MEM_ALL,
1451                 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1452                 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
1453         },
1454         {
1455                 .name = "events-wp",
1456                 .uffd_fn = uffd_events_wp_test,
1457                 .mem_targets = MEM_ALL,
1458                 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1459                 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
1460                 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1461                 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1462         },
1463         {
1464                 .name = "poison",
1465                 .uffd_fn = uffd_poison_test,
1466                 .mem_targets = MEM_ALL,
1467                 .uffd_feature_required = UFFD_FEATURE_POISON,
1468         },
1469 };
1470
1471 static void usage(const char *prog)
1472 {
1473         printf("usage: %s [-f TESTNAME]\n", prog);
1474         puts("");
1475         puts(" -f: test name to filter (e.g., event)");
1476         puts(" -h: show the help msg");
1477         puts(" -l: list tests only");
1478         puts("");
1479         exit(KSFT_FAIL);
1480 }
1481
1482 int main(int argc, char *argv[])
1483 {
1484         int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
1485         int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
1486         const char *test_filter = NULL;
1487         bool list_only = false;
1488         uffd_test_case_t *test;
1489         mem_type_t *mem_type;
1490         uffd_test_args_t args;
1491         const char *errmsg;
1492         int has_uffd, opt;
1493         int i, j;
1494
1495         while ((opt = getopt(argc, argv, "f:hl")) != -1) {
1496                 switch (opt) {
1497                 case 'f':
1498                         test_filter = optarg;
1499                         break;
1500                 case 'l':
1501                         list_only = true;
1502                         break;
1503                 case 'h':
1504                 default:
1505                         /* Unknown */
1506                         usage(argv[0]);
1507                         break;
1508                 }
1509         }
1510
1511         if (!test_filter && !list_only) {
1512                 has_uffd = test_uffd_api(false);
1513                 has_uffd |= test_uffd_api(true);
1514
1515                 if (!has_uffd) {
1516                         printf("Userfaultfd not supported or unprivileged, skip all tests\n");
1517                         exit(KSFT_SKIP);
1518                 }
1519         }
1520
1521         for (i = 0; i < n_tests; i++) {
1522                 test = &uffd_tests[i];
1523                 if (test_filter && !strstr(test->name, test_filter))
1524                         continue;
1525                 if (list_only) {
1526                         printf("%s\n", test->name);
1527                         continue;
1528                 }
1529                 for (j = 0; j < n_mems; j++) {
1530                         mem_type = &mem_types[j];
1531                         if (!(test->mem_targets & mem_type->mem_flag))
1532                                 continue;
1533
1534                         uffd_test_start("%s on %s", test->name, mem_type->name);
1535                         if ((mem_type->mem_flag == MEM_HUGETLB ||
1536                             mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&
1537                             (default_huge_page_size() == 0)) {
1538                                 uffd_test_skip("huge page size is 0, feature missing?");
1539                                 continue;
1540                         }
1541                         if (!uffd_feature_supported(test)) {
1542                                 uffd_test_skip("feature missing");
1543                                 continue;
1544                         }
1545                         if (uffd_setup_environment(&args, test, mem_type,
1546                                                    &errmsg)) {
1547                                 uffd_test_skip(errmsg);
1548                                 continue;
1549                         }
1550                         test->uffd_fn(&args);
1551                         uffd_test_ctx_clear();
1552                 }
1553         }
1554
1555         if (!list_only)
1556                 uffd_test_report();
1557
1558         return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;
1559 }
1560