tools/testing/selftests/kvm/kvm_page_table_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KVM page table test
   4  *
   5  * Copyright (C) 2021, Huawei, Inc.
   6  *
   7  * Make sure that THP has been enabled or enough HUGETLB pages with specific
   8  * page size have been pre-allocated on your system, if you are planning to
   9  * use hugepages to back the guest memory for testing.
  10  */
  11 #include <stdio.h>
  12 #include <stdlib.h>
  13 #include <time.h>
  14 #include <pthread.h>
  15 #include <semaphore.h>
  16
  17 #include "test_util.h"
  18 #include "kvm_util.h"
  19 #include "processor.h"
  20 #include "guest_modes.h"
  21 #include "ucall_common.h"
  22
  23 #define TEST_MEM_SLOT_INDEX             1
  24
  25 /* Default size(1GB) of the memory for testing */
  26 #define DEFAULT_TEST_MEM_SIZE           (1 << 30)
  27
  28 /* Default guest test virtual memory offset */
  29 #define DEFAULT_GUEST_TEST_MEM          0xc0000000
  30
  31 /* Different guest memory accessing stages */
  32 enum test_stage {
  33         KVM_BEFORE_MAPPINGS,
  34         KVM_CREATE_MAPPINGS,
  35         KVM_UPDATE_MAPPINGS,
  36         KVM_ADJUST_MAPPINGS,
  37         NUM_TEST_STAGES,
  38 };
  39
  40 static const char * const test_stage_string[] = {
  41         "KVM_BEFORE_MAPPINGS",
  42         "KVM_CREATE_MAPPINGS",
  43         "KVM_UPDATE_MAPPINGS",
  44         "KVM_ADJUST_MAPPINGS",
  45 };
  46
  47 struct test_args {
  48         struct kvm_vm *vm;
  49         uint64_t guest_test_virt_mem;
  50         uint64_t host_page_size;
  51         uint64_t host_num_pages;
  52         uint64_t large_page_size;
  53         uint64_t large_num_pages;
  54         uint64_t host_pages_per_lpage;
  55         enum vm_mem_backing_src_type src_type;
  56         struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
  57 };
  58
  59 /*
  60  * Guest variables. Use addr_gva2hva() if these variables need
  61  * to be changed in host.
  62  */
  63 static enum test_stage guest_test_stage;
  64
  65 /* Host variables */
  66 static uint32_t nr_vcpus = 1;
  67 static struct test_args test_args;
  68 static enum test_stage *current_stage;
  69 static bool host_quit;
  70
  71 /* Whether the test stage is updated, or completed */
  72 static sem_t test_stage_updated;
  73 static sem_t test_stage_completed;
  74
  75 /*
  76  * Guest physical memory offset of the testing memory slot.
  77  * This will be set to the topmost valid physical address minus
  78  * the test memory size.
  79  */
  80 static uint64_t guest_test_phys_mem;
  81
  82 /*
  83  * Guest virtual memory offset of the testing memory slot.
  84  * Must not conflict with identity mapped test code.
  85  */
  86 static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
  87
  88 static void guest_code(bool do_write)
  89 {
  90         struct test_args *p = &test_args;
  91         enum test_stage *current_stage = &guest_test_stage;
  92         uint64_t addr;
  93         int i, j;
  94
  95         while (true) {
  96                 addr = p->guest_test_virt_mem;
  97
  98                 switch (READ_ONCE(*current_stage)) {
  99                 /*
 100                  * All vCPU threads will be started in this stage,
 101                  * where guest code of each vCPU will do nothing.
 102                  */
 103                 case KVM_BEFORE_MAPPINGS:
 104                         break;
 105
 106                 /*
 107                  * Before dirty logging, vCPUs concurrently access the first
 108                  * 8 bytes of each page (host page/large page) within the same
 109                  * memory region with different accessing types (read/write).
 110                  * Then KVM will create normal page mappings or huge block
 111                  * mappings for them.
 112                  */
 113                 case KVM_CREATE_MAPPINGS:
 114                         for (i = 0; i < p->large_num_pages; i++) {
 115                                 if (do_write)
 116                                         *(uint64_t *)addr = 0x0123456789ABCDEF;
 117                                 else
 118                                         READ_ONCE(*(uint64_t *)addr);
 119
 120                                 addr += p->large_page_size;
 121                         }
 122                         break;
 123
 124                 /*
 125                  * During dirty logging, KVM will only update attributes of the
 126                  * normal page mappings from RO to RW if memory backing src type
 127                  * is anonymous. In other cases, KVM will split the huge block
 128                  * mappings into normal page mappings if memory backing src type
 129                  * is THP or HUGETLB.
 130                  */
 131                 case KVM_UPDATE_MAPPINGS:
 132                         if (p->src_type == VM_MEM_SRC_ANONYMOUS) {
 133                                 for (i = 0; i < p->host_num_pages; i++) {
 134                                         *(uint64_t *)addr = 0x0123456789ABCDEF;
 135                                         addr += p->host_page_size;
 136                                 }
 137                                 break;
 138                         }
 139
 140                         for (i = 0; i < p->large_num_pages; i++) {
 141                                 /*
 142                                  * Write to the first host page in each large
 143                                  * page region, and triger break of large pages.
 144                                  */
 145                                 *(uint64_t *)addr = 0x0123456789ABCDEF;
 146
 147                                 /*
 148                                  * Access the middle host pages in each large
 149                                  * page region. Since dirty logging is enabled,
 150                                  * this will create new mappings at the smallest
 151                                  * granularity.
 152                                  */
 153                                 addr += p->large_page_size / 2;
 154                                 for (j = 0; j < p->host_pages_per_lpage / 2; j++) {
 155                                         READ_ONCE(*(uint64_t *)addr);
 156                                         addr += p->host_page_size;
 157                                 }
 158                         }
 159                         break;
 160
 161                 /*
 162                  * After dirty logging is stopped, vCPUs concurrently read
 163                  * from every single host page. Then KVM will coalesce the
 164                  * split page mappings back to block mappings. And a TLB
 165                  * conflict abort could occur here if TLB entries of the
 166                  * page mappings are not fully invalidated.
 167                  */
 168                 case KVM_ADJUST_MAPPINGS:
 169                         for (i = 0; i < p->host_num_pages; i++) {
 170                                 READ_ONCE(*(uint64_t *)addr);
 171                                 addr += p->host_page_size;
 172                         }
 173                         break;
 174
 175                 default:
 176                         GUEST_ASSERT(0);
 177                 }
 178
 179                 GUEST_SYNC(1);
 180         }
 181 }
 182
 183 static void *vcpu_worker(void *data)
 184 {
 185         struct kvm_vcpu *vcpu = data;
 186         bool do_write = !(vcpu->id % 2);
 187         struct timespec start;
 188         struct timespec ts_diff;
 189         enum test_stage stage;
 190         int ret;
 191
 192         vcpu_args_set(vcpu, 1, do_write);
 193
 194         while (!READ_ONCE(host_quit)) {
 195                 ret = sem_wait(&test_stage_updated);
 196                 TEST_ASSERT(ret == 0, "Error in sem_wait");
 197
 198                 if (READ_ONCE(host_quit))
 199                         return NULL;
 200
 201                 clock_gettime(CLOCK_MONOTONIC, &start);
 202                 ret = _vcpu_run(vcpu);
 203                 ts_diff = timespec_elapsed(start);
 204
 205                 TEST_ASSERT(ret == 0, "vcpu_run failed: %d", ret);
 206                 TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,
 207                             "Invalid guest sync status: exit_reason=%s",
 208                             exit_reason_str(vcpu->run->exit_reason));
 209
 210                 pr_debug("Got sync event from vCPU %d\n", vcpu->id);
 211                 stage = READ_ONCE(*current_stage);
 212
 213                 /*
 214                  * Here we can know the execution time of every
 215                  * single vcpu running in different test stages.
 216                  */
 217                 pr_debug("vCPU %d has completed stage %s\n"
 218                          "execution time is: %ld.%.9lds\n\n",
 219                          vcpu->id, test_stage_string[stage],
 220                          ts_diff.tv_sec, ts_diff.tv_nsec);
 221
 222                 ret = sem_post(&test_stage_completed);
 223                 TEST_ASSERT(ret == 0, "Error in sem_post");
 224         }
 225
 226         return NULL;
 227 }
 228
 229 struct test_params {
 230         uint64_t phys_offset;
 231         uint64_t test_mem_size;
 232         enum vm_mem_backing_src_type src_type;
 233 };
 234
 235 static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
 236 {
 237         int ret;
 238         struct test_params *p = arg;
 239         enum vm_mem_backing_src_type src_type = p->src_type;
 240         uint64_t large_page_size = get_backing_src_pagesz(src_type);
 241         uint64_t guest_page_size = vm_guest_mode_params[mode].page_size;
 242         uint64_t host_page_size = getpagesize();
 243         uint64_t test_mem_size = p->test_mem_size;
 244         uint64_t guest_num_pages;
 245         uint64_t alignment;
 246         void *host_test_mem;
 247         struct kvm_vm *vm;
 248
 249         /* Align up the test memory size */
 250         alignment = max(large_page_size, guest_page_size);
 251         test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1);
 252
 253         /* Create a VM with enough guest pages */
 254         guest_num_pages = test_mem_size / guest_page_size;
 255         vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages,
 256                                     guest_code, test_args.vcpus);
 257
 258         /* Align down GPA of the testing memslot */
 259         if (!p->phys_offset)
 260                 guest_test_phys_mem = (vm->max_gfn - guest_num_pages) *
 261                                        guest_page_size;
 262         else
 263                 guest_test_phys_mem = p->phys_offset;
 264 #ifdef __s390x__
 265         alignment = max(0x100000UL, alignment);
 266 #endif
 267         guest_test_phys_mem = align_down(guest_test_phys_mem, alignment);
 268
 269         /* Set up the shared data structure test_args */
 270         test_args.vm = vm;
 271         test_args.guest_test_virt_mem = guest_test_virt_mem;
 272         test_args.host_page_size = host_page_size;
 273         test_args.host_num_pages = test_mem_size / host_page_size;
 274         test_args.large_page_size = large_page_size;
 275         test_args.large_num_pages = test_mem_size / large_page_size;
 276         test_args.host_pages_per_lpage = large_page_size / host_page_size;
 277         test_args.src_type = src_type;
 278
 279         /* Add an extra memory slot with specified backing src type */
 280         vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem,
 281                                     TEST_MEM_SLOT_INDEX, guest_num_pages, 0);
 282
 283         /* Do mapping(GVA->GPA) for the testing memory slot */
 284         virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages);
 285
 286         /* Cache the HVA pointer of the region */
 287         host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);
 288
 289         /* Export shared structure test_args to guest */
 290         sync_global_to_guest(vm, test_args);
 291
 292         ret = sem_init(&test_stage_updated, 0, 0);
 293         TEST_ASSERT(ret == 0, "Error in sem_init");
 294
 295         ret = sem_init(&test_stage_completed, 0, 0);
 296         TEST_ASSERT(ret == 0, "Error in sem_init");
 297
 298         current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage));
 299         *current_stage = NUM_TEST_STAGES;
 300
 301         pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));
 302         pr_info("Testing memory backing src type: %s\n",
 303                 vm_mem_backing_src_alias(src_type)->name);
 304         pr_info("Testing memory backing src granularity: 0x%lx\n",
 305                 large_page_size);
 306         pr_info("Testing memory size(aligned): 0x%lx\n", test_mem_size);
 307         pr_info("Guest physical test memory offset: 0x%lx\n",
 308                 guest_test_phys_mem);
 309         pr_info("Host  virtual  test memory offset: 0x%lx\n",
 310                 (uint64_t)host_test_mem);
 311         pr_info("Number of testing vCPUs: %d\n", nr_vcpus);
 312
 313         return vm;
 314 }
 315
 316 static void vcpus_complete_new_stage(enum test_stage stage)
 317 {
 318         int ret;
 319         int vcpus;
 320
 321         /* Wake up all the vcpus to run new test stage */
 322         for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
 323                 ret = sem_post(&test_stage_updated);
 324                 TEST_ASSERT(ret == 0, "Error in sem_post");
 325         }
 326         pr_debug("All vcpus have been notified to continue\n");
 327
 328         /* Wait for all the vcpus to complete new test stage */
 329         for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
 330                 ret = sem_wait(&test_stage_completed);
 331                 TEST_ASSERT(ret == 0, "Error in sem_wait");
 332
 333                 pr_debug("%d vcpus have completed stage %s\n",
 334                          vcpus + 1, test_stage_string[stage]);
 335         }
 336
 337         pr_debug("All vcpus have completed stage %s\n",
 338                  test_stage_string[stage]);
 339 }
 340
 341 static void run_test(enum vm_guest_mode mode, void *arg)
 342 {
 343         pthread_t *vcpu_threads;
 344         struct kvm_vm *vm;
 345         struct timespec start;
 346         struct timespec ts_diff;
 347         int ret, i;
 348
 349         /* Create VM with vCPUs and make some pre-initialization */
 350         vm = pre_init_before_test(mode, arg);
 351
 352         vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
 353         TEST_ASSERT(vcpu_threads, "Memory allocation failed");
 354
 355         host_quit = false;
 356         *current_stage = KVM_BEFORE_MAPPINGS;
 357
 358         for (i = 0; i < nr_vcpus; i++)
 359                 pthread_create(&vcpu_threads[i], NULL, vcpu_worker,
 360                                test_args.vcpus[i]);
 361
 362         vcpus_complete_new_stage(*current_stage);
 363         pr_info("Started all vCPUs successfully\n");
 364
 365         /* Test the stage of KVM creating mappings */
 366         *current_stage = KVM_CREATE_MAPPINGS;
 367
 368         clock_gettime(CLOCK_MONOTONIC, &start);
 369         vcpus_complete_new_stage(*current_stage);
 370         ts_diff = timespec_elapsed(start);
 371
 372         pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
 373                 ts_diff.tv_sec, ts_diff.tv_nsec);
 374
 375         /* Test the stage of KVM updating mappings */
 376         vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX,
 377                                 KVM_MEM_LOG_DIRTY_PAGES);
 378
 379         *current_stage = KVM_UPDATE_MAPPINGS;
 380
 381         clock_gettime(CLOCK_MONOTONIC, &start);
 382         vcpus_complete_new_stage(*current_stage);
 383         ts_diff = timespec_elapsed(start);
 384
 385         pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
 386                 ts_diff.tv_sec, ts_diff.tv_nsec);
 387
 388         /* Test the stage of KVM adjusting mappings */
 389         vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0);
 390
 391         *current_stage = KVM_ADJUST_MAPPINGS;
 392
 393         clock_gettime(CLOCK_MONOTONIC, &start);
 394         vcpus_complete_new_stage(*current_stage);
 395         ts_diff = timespec_elapsed(start);
 396
 397         pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n",
 398                 ts_diff.tv_sec, ts_diff.tv_nsec);
 399
 400         /* Tell the vcpu thread to quit */
 401         host_quit = true;
 402         for (i = 0; i < nr_vcpus; i++) {
 403                 ret = sem_post(&test_stage_updated);
 404                 TEST_ASSERT(ret == 0, "Error in sem_post");
 405         }
 406
 407         for (i = 0; i < nr_vcpus; i++)
 408                 pthread_join(vcpu_threads[i], NULL);
 409
 410         ret = sem_destroy(&test_stage_updated);
 411         TEST_ASSERT(ret == 0, "Error in sem_destroy");
 412
 413         ret = sem_destroy(&test_stage_completed);
 414         TEST_ASSERT(ret == 0, "Error in sem_destroy");
 415
 416         free(vcpu_threads);
 417         kvm_vm_free(vm);
 418 }
 419
 420 static void help(char *name)
 421 {
 422         puts("");
 423         printf("usage: %s [-h] [-p offset] [-m mode] "
 424                "[-b mem-size] [-v vcpus] [-s mem-type]\n", name);
 425         puts("");
 426         printf(" -p: specify guest physical test memory offset\n"
 427                "     Warning: a low offset can conflict with the loaded test code.\n");
 428         guest_modes_help();
 429         printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n"
 430                "     (default: 1G)\n");
 431         printf(" -v: specify the number of vCPUs to run\n"
 432                "     (default: 1)\n");
 433         backing_src_help("-s");
 434         puts("");
 435 }
 436
 437 int main(int argc, char *argv[])
 438 {
 439         int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
 440         struct test_params p = {
 441                 .test_mem_size = DEFAULT_TEST_MEM_SIZE,
 442                 .src_type = DEFAULT_VM_MEM_SRC,
 443         };
 444         int opt;
 445
 446         guest_modes_append_default();
 447
 448         while ((opt = getopt(argc, argv, "hp:m:b:v:s:")) != -1) {
 449                 switch (opt) {
 450                 case 'p':
 451                         p.phys_offset = strtoull(optarg, NULL, 0);
 452                         break;
 453                 case 'm':
 454                         guest_modes_cmdline(optarg);
 455                         break;
 456                 case 'b':
 457                         p.test_mem_size = parse_size(optarg);
 458                         break;
 459                 case 'v':
 460                         nr_vcpus = atoi_positive("Number of vCPUs", optarg);
 461                         TEST_ASSERT(nr_vcpus <= max_vcpus,
 462                                     "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
 463                         break;
 464                 case 's':
 465                         p.src_type = parse_backing_src_type(optarg);
 466                         break;
 467                 case 'h':
 468                 default:
 469                         help(argv[0]);
 470                         exit(0);
 471                 }
 472         }
 473
 474         for_each_guest_mode(run_test, &p);
 475
 476         return 0;
 477 }