tools/testing/selftests/mm/mremap_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2020 Google LLC
   4  */
   5 #define _GNU_SOURCE
   6
   7 #include <errno.h>
   8 #include <stdlib.h>
   9 #include <stdio.h>
  10 #include <string.h>
  11 #include <sys/mman.h>
  12 #include <time.h>
  13 #include <stdbool.h>
  14
  15 #include "../kselftest.h"
  16
  17 #define EXPECT_SUCCESS 0
  18 #define EXPECT_FAILURE 1
  19 #define NON_OVERLAPPING 0
  20 #define OVERLAPPING 1
  21 #define NS_PER_SEC 1000000000ULL
  22 #define VALIDATION_DEFAULT_THRESHOLD 4  /* 4MB */
  23 #define VALIDATION_NO_THRESHOLD 0       /* Verify the entire region */
  24
  25 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
  26 #define SIZE_MB(m) ((size_t)m * (1024 * 1024))
  27 #define SIZE_KB(k) ((size_t)k * 1024)
  28
  29 struct config {
  30         unsigned long long src_alignment;
  31         unsigned long long dest_alignment;
  32         unsigned long long region_size;
  33         int overlapping;
  34         int dest_preamble_size;
  35 };
  36
  37 struct test {
  38         const char *name;
  39         struct config config;
  40         int expect_failure;
  41 };
  42
  43 enum {
  44         _1KB = 1ULL << 10,      /* 1KB -> not page aligned */
  45         _4KB = 4ULL << 10,
  46         _8KB = 8ULL << 10,
  47         _1MB = 1ULL << 20,
  48         _2MB = 2ULL << 20,
  49         _4MB = 4ULL << 20,
  50         _5MB = 5ULL << 20,
  51         _1GB = 1ULL << 30,
  52         _2GB = 2ULL << 30,
  53         PMD = _2MB,
  54         PUD = _1GB,
  55 };
  56
  57 #define PTE page_size
  58
  59 #define MAKE_TEST(source_align, destination_align, size,        \
  60                   overlaps, should_fail, test_name)             \
  61 (struct test){                                                  \
  62         .name = test_name,                                      \
  63         .config = {                                             \
  64                 .src_alignment = source_align,                  \
  65                 .dest_alignment = destination_align,            \
  66                 .region_size = size,                            \
  67                 .overlapping = overlaps,                        \
  68         },                                                      \
  69         .expect_failure = should_fail                           \
  70 }
  71
  72 /*
  73  * Returns false if the requested remap region overlaps with an
  74  * existing mapping (e.g text, stack) else returns true.
  75  */
  76 static bool is_remap_region_valid(void *addr, unsigned long long size)
  77 {
  78         void *remap_addr = NULL;
  79         bool ret = true;
  80
  81         /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
  82         remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
  83                                          MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
  84                                          -1, 0);
  85
  86         if (remap_addr == MAP_FAILED) {
  87                 if (errno == EEXIST)
  88                         ret = false;
  89         } else {
  90                 munmap(remap_addr, size);
  91         }
  92
  93         return ret;
  94 }
  95
  96 /* Returns mmap_min_addr sysctl tunable from procfs */
  97 static unsigned long long get_mmap_min_addr(void)
  98 {
  99         FILE *fp;
 100         int n_matched;
 101         static unsigned long long addr;
 102
 103         if (addr)
 104                 return addr;
 105
 106         fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
 107         if (fp == NULL) {
 108                 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
 109                         strerror(errno));
 110                 exit(KSFT_SKIP);
 111         }
 112
 113         n_matched = fscanf(fp, "%llu", &addr);
 114         if (n_matched != 1) {
 115                 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
 116                         strerror(errno));
 117                 fclose(fp);
 118                 exit(KSFT_SKIP);
 119         }
 120
 121         fclose(fp);
 122         return addr;
 123 }
 124
 125 /*
 126  * Using /proc/self/maps, assert that the specified address range is contained
 127  * within a single mapping.
 128  */
 129 static bool is_range_mapped(FILE *maps_fp, void *start, void *end)
 130 {
 131         char *line = NULL;
 132         size_t len = 0;
 133         bool success = false;
 134
 135         rewind(maps_fp);
 136
 137         while (getline(&line, &len, maps_fp) != -1) {
 138                 char *first = strtok(line, "- ");
 139                 void *first_val = (void *)strtol(first, NULL, 16);
 140                 char *second = strtok(NULL, "- ");
 141                 void *second_val = (void *) strtol(second, NULL, 16);
 142
 143                 if (first_val <= start && second_val >= end) {
 144                         success = true;
 145                         break;
 146                 }
 147         }
 148
 149         return success;
 150 }
 151
 152 /*
 153  * Returns the start address of the mapping on success, else returns
 154  * NULL on failure.
 155  */
 156 static void *get_source_mapping(struct config c)
 157 {
 158         unsigned long long addr = 0ULL;
 159         void *src_addr = NULL;
 160         unsigned long long mmap_min_addr;
 161
 162         mmap_min_addr = get_mmap_min_addr();
 163         /*
 164          * For some tests, we need to not have any mappings below the
 165          * source mapping. Add some headroom to mmap_min_addr for this.
 166          */
 167         mmap_min_addr += 10 * _4MB;
 168
 169 retry:
 170         addr += c.src_alignment;
 171         if (addr < mmap_min_addr)
 172                 goto retry;
 173
 174         src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
 175                                         MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
 176                                         -1, 0);
 177         if (src_addr == MAP_FAILED) {
 178                 if (errno == EPERM || errno == EEXIST)
 179                         goto retry;
 180                 goto error;
 181         }
 182         /*
 183          * Check that the address is aligned to the specified alignment.
 184          * Addresses which have alignments that are multiples of that
 185          * specified are not considered valid. For instance, 1GB address is
 186          * 2MB-aligned, however it will not be considered valid for a
 187          * requested alignment of 2MB. This is done to reduce coincidental
 188          * alignment in the tests.
 189          */
 190         if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
 191                         !((unsigned long long) src_addr & c.src_alignment)) {
 192                 munmap(src_addr, c.region_size);
 193                 goto retry;
 194         }
 195
 196         if (!src_addr)
 197                 goto error;
 198
 199         return src_addr;
 200 error:
 201         ksft_print_msg("Failed to map source region: %s\n",
 202                         strerror(errno));
 203         return NULL;
 204 }
 205
 206 /*
 207  * This test validates that merge is called when expanding a mapping.
 208  * Mapping containing three pages is created, middle page is unmapped
 209  * and then the mapping containing the first page is expanded so that
 210  * it fills the created hole. The two parts should merge creating
 211  * single mapping with three pages.
 212  */
 213 static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size)
 214 {
 215         char *test_name = "mremap expand merge";
 216         bool success = false;
 217         char *remap, *start;
 218
 219         start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
 220                      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 221
 222         if (start == MAP_FAILED) {
 223                 ksft_print_msg("mmap failed: %s\n", strerror(errno));
 224                 goto out;
 225         }
 226
 227         munmap(start + page_size, page_size);
 228         remap = mremap(start, page_size, 2 * page_size, 0);
 229         if (remap == MAP_FAILED) {
 230                 ksft_print_msg("mremap failed: %s\n", strerror(errno));
 231                 munmap(start, page_size);
 232                 munmap(start + 2 * page_size, page_size);
 233                 goto out;
 234         }
 235
 236         success = is_range_mapped(maps_fp, start, start + 3 * page_size);
 237         munmap(start, 3 * page_size);
 238
 239 out:
 240         if (success)
 241                 ksft_test_result_pass("%s\n", test_name);
 242         else
 243                 ksft_test_result_fail("%s\n", test_name);
 244 }
 245
 246 /*
 247  * Similar to mremap_expand_merge() except instead of removing the middle page,
 248  * we remove the last then attempt to remap offset from the second page. This
 249  * should result in the mapping being restored to its former state.
 250  */
 251 static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size)
 252 {
 253
 254         char *test_name = "mremap expand merge offset";
 255         bool success = false;
 256         char *remap, *start;
 257
 258         start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
 259                      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 260
 261         if (start == MAP_FAILED) {
 262                 ksft_print_msg("mmap failed: %s\n", strerror(errno));
 263                 goto out;
 264         }
 265
 266         /* Unmap final page to ensure we have space to expand. */
 267         munmap(start + 2 * page_size, page_size);
 268         remap = mremap(start + page_size, page_size, 2 * page_size, 0);
 269         if (remap == MAP_FAILED) {
 270                 ksft_print_msg("mremap failed: %s\n", strerror(errno));
 271                 munmap(start, 2 * page_size);
 272                 goto out;
 273         }
 274
 275         success = is_range_mapped(maps_fp, start, start + 3 * page_size);
 276         munmap(start, 3 * page_size);
 277
 278 out:
 279         if (success)
 280                 ksft_test_result_pass("%s\n", test_name);
 281         else
 282                 ksft_test_result_fail("%s\n", test_name);
 283 }
 284
 285 /*
 286  * Verify that an mremap within a range does not cause corruption
 287  * of unrelated part of range.
 288  *
 289  * Consider the following range which is 2MB aligned and is
 290  * a part of a larger 20MB range which is not shown. Each
 291  * character is 256KB below making the source and destination
 292  * 2MB each. The lower case letters are moved (s to d) and the
 293  * upper case letters are not moved. The below test verifies
 294  * that the upper case S letters are not corrupted by the
 295  * adjacent mremap.
 296  *
 297  * |DDDDddddSSSSssss|
 298  */
 299 static void mremap_move_within_range(char pattern_seed)
 300 {
 301         char *test_name = "mremap mremap move within range";
 302         void *src, *dest;
 303         int i, success = 1;
 304
 305         size_t size = SIZE_MB(20);
 306         void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 307                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 308         if (ptr == MAP_FAILED) {
 309                 perror("mmap");
 310                 success = 0;
 311                 goto out;
 312         }
 313         memset(ptr, 0, size);
 314
 315         src = ptr + SIZE_MB(6);
 316         src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1));
 317
 318         /* Set byte pattern for source block. */
 319         srand(pattern_seed);
 320         for (i = 0; i < SIZE_MB(2); i++) {
 321                 ((char *)src)[i] = (char) rand();
 322         }
 323
 324         dest = src - SIZE_MB(2);
 325
 326         void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1),
 327                                                    MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1));
 328         if (new_ptr == MAP_FAILED) {
 329                 perror("mremap");
 330                 success = 0;
 331                 goto out;
 332         }
 333
 334         /* Verify byte pattern after remapping */
 335         srand(pattern_seed);
 336         for (i = 0; i < SIZE_MB(1); i++) {
 337                 char c = (char) rand();
 338
 339                 if (((char *)src)[i] != c) {
 340                         ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n",
 341                                        i);
 342                         ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
 343                                         ((char *) src)[i] & 0xff);
 344                         success = 0;
 345                 }
 346         }
 347
 348 out:
 349         if (munmap(ptr, size) == -1)
 350                 perror("munmap");
 351
 352         if (success)
 353                 ksft_test_result_pass("%s\n", test_name);
 354         else
 355                 ksft_test_result_fail("%s\n", test_name);
 356 }
 357
 358 /* Returns the time taken for the remap on success else returns -1. */
 359 static long long remap_region(struct config c, unsigned int threshold_mb,
 360                               char pattern_seed)
 361 {
 362         void *addr, *src_addr, *dest_addr, *dest_preamble_addr;
 363         unsigned long long i;
 364         struct timespec t_start = {0, 0}, t_end = {0, 0};
 365         long long  start_ns, end_ns, align_mask, ret, offset;
 366         unsigned long long threshold;
 367
 368         if (threshold_mb == VALIDATION_NO_THRESHOLD)
 369                 threshold = c.region_size;
 370         else
 371                 threshold = MIN(threshold_mb * _1MB, c.region_size);
 372
 373         src_addr = get_source_mapping(c);
 374         if (!src_addr) {
 375                 ret = -1;
 376                 goto out;
 377         }
 378
 379         /* Set byte pattern for source block. */
 380         srand(pattern_seed);
 381         for (i = 0; i < threshold; i++)
 382                 memset((char *) src_addr + i, (char) rand(), 1);
 383
 384         /* Mask to zero out lower bits of address for alignment */
 385         align_mask = ~(c.dest_alignment - 1);
 386         /* Offset of destination address from the end of the source region */
 387         offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
 388         addr = (void *) (((unsigned long long) src_addr + c.region_size
 389                           + offset) & align_mask);
 390
 391         /* Remap after the destination block preamble. */
 392         addr += c.dest_preamble_size;
 393
 394         /* See comment in get_source_mapping() */
 395         if (!((unsigned long long) addr & c.dest_alignment))
 396                 addr = (void *) ((unsigned long long) addr | c.dest_alignment);
 397
 398         /* Don't destroy existing mappings unless expected to overlap */
 399         while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
 400                 /* Check for unsigned overflow */
 401                 if (addr + c.dest_alignment < addr) {
 402                         ksft_print_msg("Couldn't find a valid region to remap to\n");
 403                         ret = -1;
 404                         goto clean_up_src;
 405                 }
 406                 addr += c.dest_alignment;
 407         }
 408
 409         if (c.dest_preamble_size) {
 410                 dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size,
 411                                           PROT_READ | PROT_WRITE,
 412                                           MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
 413                                                         -1, 0);
 414                 if (dest_preamble_addr == MAP_FAILED) {
 415                         ksft_print_msg("Failed to map dest preamble region: %s\n",
 416                                         strerror(errno));
 417                         ret = -1;
 418                         goto clean_up_src;
 419                 }
 420
 421                 /* Set byte pattern for the dest preamble block. */
 422                 srand(pattern_seed);
 423                 for (i = 0; i < c.dest_preamble_size; i++)
 424                         memset((char *) dest_preamble_addr + i, (char) rand(), 1);
 425         }
 426
 427         clock_gettime(CLOCK_MONOTONIC, &t_start);
 428         dest_addr = mremap(src_addr, c.region_size, c.region_size,
 429                                           MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
 430         clock_gettime(CLOCK_MONOTONIC, &t_end);
 431
 432         if (dest_addr == MAP_FAILED) {
 433                 ksft_print_msg("mremap failed: %s\n", strerror(errno));
 434                 ret = -1;
 435                 goto clean_up_dest_preamble;
 436         }
 437
 438         /* Verify byte pattern after remapping */
 439         srand(pattern_seed);
 440         for (i = 0; i < threshold; i++) {
 441                 char c = (char) rand();
 442
 443                 if (((char *) dest_addr)[i] != c) {
 444                         ksft_print_msg("Data after remap doesn't match at offset %llu\n",
 445                                        i);
 446                         ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
 447                                         ((char *) dest_addr)[i] & 0xff);
 448                         ret = -1;
 449                         goto clean_up_dest;
 450                 }
 451         }
 452
 453         /* Verify the dest preamble byte pattern after remapping */
 454         if (c.dest_preamble_size) {
 455                 srand(pattern_seed);
 456                 for (i = 0; i < c.dest_preamble_size; i++) {
 457                         char c = (char) rand();
 458
 459                         if (((char *) dest_preamble_addr)[i] != c) {
 460                                 ksft_print_msg("Preamble data after remap doesn't match at offset %d\n",
 461                                                i);
 462                                 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
 463                                                ((char *) dest_preamble_addr)[i] & 0xff);
 464                                 ret = -1;
 465                                 goto clean_up_dest;
 466                         }
 467                 }
 468         }
 469
 470         start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
 471         end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
 472         ret = end_ns - start_ns;
 473
 474 /*
 475  * Since the destination address is specified using MREMAP_FIXED, subsequent
 476  * mremap will unmap any previous mapping at the address range specified by
 477  * dest_addr and region_size. This significantly affects the remap time of
 478  * subsequent tests. So we clean up mappings after each test.
 479  */
 480 clean_up_dest:
 481         munmap(dest_addr, c.region_size);
 482 clean_up_dest_preamble:
 483         if (c.dest_preamble_size && dest_preamble_addr)
 484                 munmap(dest_preamble_addr, c.dest_preamble_size);
 485 clean_up_src:
 486         munmap(src_addr, c.region_size);
 487 out:
 488         return ret;
 489 }
 490
 491 /*
 492  * Verify that an mremap aligning down does not destroy
 493  * the beginning of the mapping just because the aligned
 494  * down address landed on a mapping that maybe does not exist.
 495  */
 496 static void mremap_move_1mb_from_start(char pattern_seed)
 497 {
 498         char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src";
 499         void *src = NULL, *dest = NULL;
 500         int i, success = 1;
 501
 502         /* Config to reuse get_source_mapping() to do an aligned mmap. */
 503         struct config c = {
 504                 .src_alignment = SIZE_MB(1) + SIZE_KB(256),
 505                 .region_size = SIZE_MB(6)
 506         };
 507
 508         src = get_source_mapping(c);
 509         if (!src) {
 510                 success = 0;
 511                 goto out;
 512         }
 513
 514         c.src_alignment = SIZE_MB(1) + SIZE_KB(256);
 515         dest = get_source_mapping(c);
 516         if (!dest) {
 517                 success = 0;
 518                 goto out;
 519         }
 520
 521         /* Set byte pattern for source block. */
 522         srand(pattern_seed);
 523         for (i = 0; i < SIZE_MB(2); i++) {
 524                 ((char *)src)[i] = (char) rand();
 525         }
 526
 527         /*
 528          * Unmap the beginning of dest so that the aligned address
 529          * falls on no mapping.
 530          */
 531         munmap(dest, SIZE_MB(1));
 532
 533         void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1),
 534                                                    MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1));
 535         if (new_ptr == MAP_FAILED) {
 536                 perror("mremap");
 537                 success = 0;
 538                 goto out;
 539         }
 540
 541         /* Verify byte pattern after remapping */
 542         srand(pattern_seed);
 543         for (i = 0; i < SIZE_MB(1); i++) {
 544                 char c = (char) rand();
 545
 546                 if (((char *)src)[i] != c) {
 547                         ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n",
 548                                        i);
 549                         ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
 550                                         ((char *) src)[i] & 0xff);
 551                         success = 0;
 552                 }
 553         }
 554
 555 out:
 556         if (src && munmap(src, c.region_size) == -1)
 557                 perror("munmap src");
 558
 559         if (dest && munmap(dest, c.region_size) == -1)
 560                 perror("munmap dest");
 561
 562         if (success)
 563                 ksft_test_result_pass("%s\n", test_name);
 564         else
 565                 ksft_test_result_fail("%s\n", test_name);
 566 }
 567
 568 static void run_mremap_test_case(struct test test_case, int *failures,
 569                                  unsigned int threshold_mb,
 570                                  unsigned int pattern_seed)
 571 {
 572         long long remap_time = remap_region(test_case.config, threshold_mb,
 573                                             pattern_seed);
 574
 575         if (remap_time < 0) {
 576                 if (test_case.expect_failure)
 577                         ksft_test_result_xfail("%s\n\tExpected mremap failure\n",
 578                                               test_case.name);
 579                 else {
 580                         ksft_test_result_fail("%s\n", test_case.name);
 581                         *failures += 1;
 582                 }
 583         } else {
 584                 /*
 585                  * Comparing mremap time is only applicable if entire region
 586                  * was faulted in.
 587                  */
 588                 if (threshold_mb == VALIDATION_NO_THRESHOLD ||
 589                     test_case.config.region_size <= threshold_mb * _1MB)
 590                         ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
 591                                               test_case.name, remap_time);
 592                 else
 593                         ksft_test_result_pass("%s\n", test_case.name);
 594         }
 595 }
 596
 597 static void usage(const char *cmd)
 598 {
 599         fprintf(stderr,
 600                 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
 601                 "-t\t only validate threshold_mb of the remapped region\n"
 602                 "  \t if 0 is supplied no threshold is used; all tests\n"
 603                 "  \t are run and remapped regions validated fully.\n"
 604                 "  \t The default threshold used is 4MB.\n"
 605                 "-p\t provide a seed to generate the random pattern for\n"
 606                 "  \t validating the remapped region.\n", cmd);
 607 }
 608
 609 static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
 610                       unsigned int *pattern_seed)
 611 {
 612         const char *optstr = "t:p:";
 613         int opt;
 614
 615         while ((opt = getopt(argc, argv, optstr)) != -1) {
 616                 switch (opt) {
 617                 case 't':
 618                         *threshold_mb = atoi(optarg);
 619                         break;
 620                 case 'p':
 621                         *pattern_seed = atoi(optarg);
 622                         break;
 623                 default:
 624                         usage(argv[0]);
 625                         return -1;
 626                 }
 627         }
 628
 629         if (optind < argc) {
 630                 usage(argv[0]);
 631                 return -1;
 632         }
 633
 634         return 0;
 635 }
 636
 637 #define MAX_TEST 15
 638 #define MAX_PERF_TEST 3
 639 int main(int argc, char **argv)
 640 {
 641         int failures = 0;
 642         int i, run_perf_tests;
 643         unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
 644         unsigned int pattern_seed;
 645         int num_expand_tests = 2;
 646         int num_misc_tests = 2;
 647         struct test test_cases[MAX_TEST] = {};
 648         struct test perf_test_cases[MAX_PERF_TEST];
 649         int page_size;
 650         time_t t;
 651         FILE *maps_fp;
 652
 653         pattern_seed = (unsigned int) time(&t);
 654
 655         if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
 656                 exit(EXIT_FAILURE);
 657
 658         ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
 659                        threshold_mb, pattern_seed);
 660
 661         page_size = sysconf(_SC_PAGESIZE);
 662
 663         /* Expected mremap failures */
 664         test_cases[0] = MAKE_TEST(page_size, page_size, page_size,
 665                                   OVERLAPPING, EXPECT_FAILURE,
 666                                   "mremap - Source and Destination Regions Overlapping");
 667
 668         test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
 669                                   NON_OVERLAPPING, EXPECT_FAILURE,
 670                                   "mremap - Destination Address Misaligned (1KB-aligned)");
 671         test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
 672                                   NON_OVERLAPPING, EXPECT_FAILURE,
 673                                   "mremap - Source Address Misaligned (1KB-aligned)");
 674
 675         /* Src addr PTE aligned */
 676         test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
 677                                   NON_OVERLAPPING, EXPECT_SUCCESS,
 678                                   "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
 679
 680         /* Src addr 1MB aligned */
 681         test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 682                                   "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
 683         test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 684                                   "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
 685
 686         /* Src addr PMD aligned */
 687         test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 688                                   "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
 689         test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 690                                   "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
 691         test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 692                                   "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
 693
 694         /* Src addr PUD aligned */
 695         test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 696                                   "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
 697         test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 698                                    "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
 699         test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 700                                    "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
 701         test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 702                                    "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
 703
 704         /* Src and Dest addr 1MB aligned. 5MB mremap. */
 705         test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 706                                   "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
 707
 708         /* Src and Dest addr 1MB aligned. 5MB mremap. */
 709         test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS,
 710                                   "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble");
 711         test_cases[14].config.dest_preamble_size = 10 * _4MB;
 712
 713         perf_test_cases[0] =  MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 714                                         "1GB mremap - Source PTE-aligned, Destination PTE-aligned");
 715         /*
 716          * mremap 1GB region - Page table level aligned time
 717          * comparison.
 718          */
 719         perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 720                                        "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
 721         perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
 722                                        "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
 723
 724         run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
 725                                 (threshold_mb * _1MB >= _1GB);
 726
 727         ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
 728                       ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests);
 729
 730         for (i = 0; i < ARRAY_SIZE(test_cases); i++)
 731                 run_mremap_test_case(test_cases[i], &failures, threshold_mb,
 732                                      pattern_seed);
 733
 734         maps_fp = fopen("/proc/self/maps", "r");
 735
 736         if (maps_fp == NULL) {
 737                 ksft_print_msg("Failed to read /proc/self/maps: %s\n", strerror(errno));
 738                 exit(KSFT_FAIL);
 739         }
 740
 741         mremap_expand_merge(maps_fp, page_size);
 742         mremap_expand_merge_offset(maps_fp, page_size);
 743
 744         fclose(maps_fp);
 745
 746         mremap_move_within_range(pattern_seed);
 747         mremap_move_1mb_from_start(pattern_seed);
 748
 749         if (run_perf_tests) {
 750                 ksft_print_msg("\n%s\n",
 751                  "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
 752                 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
 753                         run_mremap_test_case(perf_test_cases[i], &failures,
 754                                              threshold_mb, pattern_seed);
 755         }
 756
 757         if (failures > 0)
 758                 ksft_exit_fail();
 759         else
 760                 ksft_exit_pass();
 761 }