tools/testing/selftests/kvm/kvm_page_table_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KVM page table test
   4  *
   5  * Copyright (C) 2021, Huawei, Inc.
   6  *
   7  * Make sure that THP has been enabled or enough HUGETLB pages with specific
   8  * page size have been pre-allocated on your system, if you are planning to
   9  * use hugepages to back the guest memory for testing.
  10  */
  11
  12 #define _GNU_SOURCE /* for program_invocation_name */
  13
  14 #include <stdio.h>
  15 #include <stdlib.h>
  16 #include <time.h>
  17 #include <pthread.h>
  18 #include <semaphore.h>
  19
  20 #include "test_util.h"
  21 #include "kvm_util.h"
  22 #include "processor.h"
  23 #include "guest_modes.h"
  24
  25 #define TEST_MEM_SLOT_INDEX             1
  26
  27 /* Default size(1GB) of the memory for testing */
  28 #define DEFAULT_TEST_MEM_SIZE           (1 << 30)
  29
  30 /* Default guest test virtual memory offset */
  31 #define DEFAULT_GUEST_TEST_MEM          0xc0000000
  32
  33 /* Different guest memory accessing stages */
  34 enum test_stage {
  35         KVM_BEFORE_MAPPINGS,
  36         KVM_CREATE_MAPPINGS,
  37         KVM_UPDATE_MAPPINGS,
  38         KVM_ADJUST_MAPPINGS,
  39         NUM_TEST_STAGES,
  40 };
  41
  42 static const char * const test_stage_string[] = {
  43         "KVM_BEFORE_MAPPINGS",
  44         "KVM_CREATE_MAPPINGS",
  45         "KVM_UPDATE_MAPPINGS",
  46         "KVM_ADJUST_MAPPINGS",
  47 };
  48
  49 struct test_args {
  50         struct kvm_vm *vm;
  51         uint64_t guest_test_virt_mem;
  52         uint64_t host_page_size;
  53         uint64_t host_num_pages;
  54         uint64_t large_page_size;
  55         uint64_t large_num_pages;
  56         uint64_t host_pages_per_lpage;
  57         enum vm_mem_backing_src_type src_type;
  58         struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
  59 };
  60
  61 /*
  62  * Guest variables. Use addr_gva2hva() if these variables need
  63  * to be changed in host.
  64  */
  65 static enum test_stage guest_test_stage;
  66
  67 /* Host variables */
  68 static uint32_t nr_vcpus = 1;
  69 static struct test_args test_args;
  70 static enum test_stage *current_stage;
  71 static bool host_quit;
  72
  73 /* Whether the test stage is updated, or completed */
  74 static sem_t test_stage_updated;
  75 static sem_t test_stage_completed;
  76
  77 /*
  78  * Guest physical memory offset of the testing memory slot.
  79  * This will be set to the topmost valid physical address minus
  80  * the test memory size.
  81  */
  82 static uint64_t guest_test_phys_mem;
  83
  84 /*
  85  * Guest virtual memory offset of the testing memory slot.
  86  * Must not conflict with identity mapped test code.
  87  */
  88 static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
  89
  90 static void guest_code(bool do_write)
  91 {
  92         struct test_args *p = &test_args;
  93         enum test_stage *current_stage = &guest_test_stage;
  94         uint64_t addr;
  95         int i, j;
  96
  97         while (true) {
  98                 addr = p->guest_test_virt_mem;
  99
 100                 switch (READ_ONCE(*current_stage)) {
 101                 /*
 102                  * All vCPU threads will be started in this stage,
 103                  * where guest code of each vCPU will do nothing.
 104                  */
 105                 case KVM_BEFORE_MAPPINGS:
 106                         break;
 107
 108                 /*
 109                  * Before dirty logging, vCPUs concurrently access the first
 110                  * 8 bytes of each page (host page/large page) within the same
 111                  * memory region with different accessing types (read/write).
 112                  * Then KVM will create normal page mappings or huge block
 113                  * mappings for them.
 114                  */
 115                 case KVM_CREATE_MAPPINGS:
 116                         for (i = 0; i < p->large_num_pages; i++) {
 117                                 if (do_write)
 118                                         *(uint64_t *)addr = 0x0123456789ABCDEF;
 119                                 else
 120                                         READ_ONCE(*(uint64_t *)addr);
 121
 122                                 addr += p->large_page_size;
 123                         }
 124                         break;
 125
 126                 /*
 127                  * During dirty logging, KVM will only update attributes of the
 128                  * normal page mappings from RO to RW if memory backing src type
 129                  * is anonymous. In other cases, KVM will split the huge block
 130                  * mappings into normal page mappings if memory backing src type
 131                  * is THP or HUGETLB.
 132                  */
 133                 case KVM_UPDATE_MAPPINGS:
 134                         if (p->src_type == VM_MEM_SRC_ANONYMOUS) {
 135                                 for (i = 0; i < p->host_num_pages; i++) {
 136                                         *(uint64_t *)addr = 0x0123456789ABCDEF;
 137                                         addr += p->host_page_size;
 138                                 }
 139                                 break;
 140                         }
 141
 142                         for (i = 0; i < p->large_num_pages; i++) {
 143                                 /*
 144                                  * Write to the first host page in each large
 145                                  * page region, and triger break of large pages.
 146                                  */
 147                                 *(uint64_t *)addr = 0x0123456789ABCDEF;
 148
 149                                 /*
 150                                  * Access the middle host pages in each large
 151                                  * page region. Since dirty logging is enabled,
 152                                  * this will create new mappings at the smallest
 153                                  * granularity.
 154                                  */
 155                                 addr += p->large_page_size / 2;
 156                                 for (j = 0; j < p->host_pages_per_lpage / 2; j++) {
 157                                         READ_ONCE(*(uint64_t *)addr);
 158                                         addr += p->host_page_size;
 159                                 }
 160                         }
 161                         break;
 162
 163                 /*
 164                  * After dirty logging is stopped, vCPUs concurrently read
 165                  * from every single host page. Then KVM will coalesce the
 166                  * split page mappings back to block mappings. And a TLB
 167                  * conflict abort could occur here if TLB entries of the
 168                  * page mappings are not fully invalidated.
 169                  */
 170                 case KVM_ADJUST_MAPPINGS:
 171                         for (i = 0; i < p->host_num_pages; i++) {
 172                                 READ_ONCE(*(uint64_t *)addr);
 173                                 addr += p->host_page_size;
 174                         }
 175                         break;
 176
 177                 default:
 178                         GUEST_ASSERT(0);
 179                 }
 180
 181                 GUEST_SYNC(1);
 182         }
 183 }
 184
 185 static void *vcpu_worker(void *data)
 186 {
 187         struct kvm_vcpu *vcpu = data;
 188         bool do_write = !(vcpu->id % 2);
 189         struct timespec start;
 190         struct timespec ts_diff;
 191         enum test_stage stage;
 192         int ret;
 193
 194         vcpu_args_set(vcpu, 1, do_write);
 195
 196         while (!READ_ONCE(host_quit)) {
 197                 ret = sem_wait(&test_stage_updated);
 198                 TEST_ASSERT(ret == 0, "Error in sem_wait");
 199
 200                 if (READ_ONCE(host_quit))
 201                         return NULL;
 202
 203                 clock_gettime(CLOCK_MONOTONIC, &start);
 204                 ret = _vcpu_run(vcpu);
 205                 ts_diff = timespec_elapsed(start);
 206
 207                 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
 208                 TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,
 209                             "Invalid guest sync status: exit_reason=%s\n",
 210                             exit_reason_str(vcpu->run->exit_reason));
 211
 212                 pr_debug("Got sync event from vCPU %d\n", vcpu->id);
 213                 stage = READ_ONCE(*current_stage);
 214
 215                 /*
 216                  * Here we can know the execution time of every
 217                  * single vcpu running in different test stages.
 218                  */
 219                 pr_debug("vCPU %d has completed stage %s\n"
 220                          "execution time is: %ld.%.9lds\n\n",
 221                          vcpu->id, test_stage_string[stage],
 222                          ts_diff.tv_sec, ts_diff.tv_nsec);
 223
 224                 ret = sem_post(&test_stage_completed);
 225                 TEST_ASSERT(ret == 0, "Error in sem_post");
 226         }
 227
 228         return NULL;
 229 }
 230
 231 struct test_params {
 232         uint64_t phys_offset;
 233         uint64_t test_mem_size;
 234         enum vm_mem_backing_src_type src_type;
 235 };
 236
 237 static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
 238 {
 239         int ret;
 240         struct test_params *p = arg;
 241         enum vm_mem_backing_src_type src_type = p->src_type;
 242         uint64_t large_page_size = get_backing_src_pagesz(src_type);
 243         uint64_t guest_page_size = vm_guest_mode_params[mode].page_size;
 244         uint64_t host_page_size = getpagesize();
 245         uint64_t test_mem_size = p->test_mem_size;
 246         uint64_t guest_num_pages;
 247         uint64_t alignment;
 248         void *host_test_mem;
 249         struct kvm_vm *vm;
 250
 251         /* Align up the test memory size */
 252         alignment = max(large_page_size, guest_page_size);
 253         test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1);
 254
 255         /* Create a VM with enough guest pages */
 256         guest_num_pages = test_mem_size / guest_page_size;
 257         vm = __vm_create_with_vcpus(mode, nr_vcpus, guest_num_pages,
 258                                     guest_code, test_args.vcpus);
 259
 260         /* Align down GPA of the testing memslot */
 261         if (!p->phys_offset)
 262                 guest_test_phys_mem = (vm->max_gfn - guest_num_pages) *
 263                                        guest_page_size;
 264         else
 265                 guest_test_phys_mem = p->phys_offset;
 266 #ifdef __s390x__
 267         alignment = max(0x100000UL, alignment);
 268 #endif
 269         guest_test_phys_mem = align_down(guest_test_phys_mem, alignment);
 270
 271         /* Set up the shared data structure test_args */
 272         test_args.vm = vm;
 273         test_args.guest_test_virt_mem = guest_test_virt_mem;
 274         test_args.host_page_size = host_page_size;
 275         test_args.host_num_pages = test_mem_size / host_page_size;
 276         test_args.large_page_size = large_page_size;
 277         test_args.large_num_pages = test_mem_size / large_page_size;
 278         test_args.host_pages_per_lpage = large_page_size / host_page_size;
 279         test_args.src_type = src_type;
 280
 281         /* Add an extra memory slot with specified backing src type */
 282         vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem,
 283                                     TEST_MEM_SLOT_INDEX, guest_num_pages, 0);
 284
 285         /* Do mapping(GVA->GPA) for the testing memory slot */
 286         virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages);
 287
 288         /* Cache the HVA pointer of the region */
 289         host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);
 290
 291         /* Export shared structure test_args to guest */
 292         sync_global_to_guest(vm, test_args);
 293
 294         ret = sem_init(&test_stage_updated, 0, 0);
 295         TEST_ASSERT(ret == 0, "Error in sem_init");
 296
 297         ret = sem_init(&test_stage_completed, 0, 0);
 298         TEST_ASSERT(ret == 0, "Error in sem_init");
 299
 300         current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage));
 301         *current_stage = NUM_TEST_STAGES;
 302
 303         pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));
 304         pr_info("Testing memory backing src type: %s\n",
 305                 vm_mem_backing_src_alias(src_type)->name);
 306         pr_info("Testing memory backing src granularity: 0x%lx\n",
 307                 large_page_size);
 308         pr_info("Testing memory size(aligned): 0x%lx\n", test_mem_size);
 309         pr_info("Guest physical test memory offset: 0x%lx\n",
 310                 guest_test_phys_mem);
 311         pr_info("Host  virtual  test memory offset: 0x%lx\n",
 312                 (uint64_t)host_test_mem);
 313         pr_info("Number of testing vCPUs: %d\n", nr_vcpus);
 314
 315         return vm;
 316 }
 317
 318 static void vcpus_complete_new_stage(enum test_stage stage)
 319 {
 320         int ret;
 321         int vcpus;
 322
 323         /* Wake up all the vcpus to run new test stage */
 324         for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
 325                 ret = sem_post(&test_stage_updated);
 326                 TEST_ASSERT(ret == 0, "Error in sem_post");
 327         }
 328         pr_debug("All vcpus have been notified to continue\n");
 329
 330         /* Wait for all the vcpus to complete new test stage */
 331         for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
 332                 ret = sem_wait(&test_stage_completed);
 333                 TEST_ASSERT(ret == 0, "Error in sem_wait");
 334
 335                 pr_debug("%d vcpus have completed stage %s\n",
 336                          vcpus + 1, test_stage_string[stage]);
 337         }
 338
 339         pr_debug("All vcpus have completed stage %s\n",
 340                  test_stage_string[stage]);
 341 }
 342
 343 static void run_test(enum vm_guest_mode mode, void *arg)
 344 {
 345         pthread_t *vcpu_threads;
 346         struct kvm_vm *vm;
 347         struct timespec start;
 348         struct timespec ts_diff;
 349         int ret, i;
 350
 351         /* Create VM with vCPUs and make some pre-initialization */
 352         vm = pre_init_before_test(mode, arg);
 353
 354         vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
 355         TEST_ASSERT(vcpu_threads, "Memory allocation failed");
 356
 357         host_quit = false;
 358         *current_stage = KVM_BEFORE_MAPPINGS;
 359
 360         for (i = 0; i < nr_vcpus; i++)
 361                 pthread_create(&vcpu_threads[i], NULL, vcpu_worker,
 362                                test_args.vcpus[i]);
 363
 364         vcpus_complete_new_stage(*current_stage);
 365         pr_info("Started all vCPUs successfully\n");
 366
 367         /* Test the stage of KVM creating mappings */
 368         *current_stage = KVM_CREATE_MAPPINGS;
 369
 370         clock_gettime(CLOCK_MONOTONIC, &start);
 371         vcpus_complete_new_stage(*current_stage);
 372         ts_diff = timespec_elapsed(start);
 373
 374         pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
 375                 ts_diff.tv_sec, ts_diff.tv_nsec);
 376
 377         /* Test the stage of KVM updating mappings */
 378         vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX,
 379                                 KVM_MEM_LOG_DIRTY_PAGES);
 380
 381         *current_stage = KVM_UPDATE_MAPPINGS;
 382
 383         clock_gettime(CLOCK_MONOTONIC, &start);
 384         vcpus_complete_new_stage(*current_stage);
 385         ts_diff = timespec_elapsed(start);
 386
 387         pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
 388                 ts_diff.tv_sec, ts_diff.tv_nsec);
 389
 390         /* Test the stage of KVM adjusting mappings */
 391         vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0);
 392
 393         *current_stage = KVM_ADJUST_MAPPINGS;
 394
 395         clock_gettime(CLOCK_MONOTONIC, &start);
 396         vcpus_complete_new_stage(*current_stage);
 397         ts_diff = timespec_elapsed(start);
 398
 399         pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n",
 400                 ts_diff.tv_sec, ts_diff.tv_nsec);
 401
 402         /* Tell the vcpu thread to quit */
 403         host_quit = true;
 404         for (i = 0; i < nr_vcpus; i++) {
 405                 ret = sem_post(&test_stage_updated);
 406                 TEST_ASSERT(ret == 0, "Error in sem_post");
 407         }
 408
 409         for (i = 0; i < nr_vcpus; i++)
 410                 pthread_join(vcpu_threads[i], NULL);
 411
 412         ret = sem_destroy(&test_stage_updated);
 413         TEST_ASSERT(ret == 0, "Error in sem_destroy");
 414
 415         ret = sem_destroy(&test_stage_completed);
 416         TEST_ASSERT(ret == 0, "Error in sem_destroy");
 417
 418         free(vcpu_threads);
 419         kvm_vm_free(vm);
 420 }
 421
 422 static void help(char *name)
 423 {
 424         puts("");
 425         printf("usage: %s [-h] [-p offset] [-m mode] "
 426                "[-b mem-size] [-v vcpus] [-s mem-type]\n", name);
 427         puts("");
 428         printf(" -p: specify guest physical test memory offset\n"
 429                "     Warning: a low offset can conflict with the loaded test code.\n");
 430         guest_modes_help();
 431         printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n"
 432                "     (default: 1G)\n");
 433         printf(" -v: specify the number of vCPUs to run\n"
 434                "     (default: 1)\n");
 435         backing_src_help("-s");
 436         puts("");
 437 }
 438
 439 int main(int argc, char *argv[])
 440 {
 441         int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
 442         struct test_params p = {
 443                 .test_mem_size = DEFAULT_TEST_MEM_SIZE,
 444                 .src_type = DEFAULT_VM_MEM_SRC,
 445         };
 446         int opt;
 447
 448         guest_modes_append_default();
 449
 450         while ((opt = getopt(argc, argv, "hp:m:b:v:s:")) != -1) {
 451                 switch (opt) {
 452                 case 'p':
 453                         p.phys_offset = strtoull(optarg, NULL, 0);
 454                         break;
 455                 case 'm':
 456                         guest_modes_cmdline(optarg);
 457                         break;
 458                 case 'b':
 459                         p.test_mem_size = parse_size(optarg);
 460                         break;
 461                 case 'v':
 462                         nr_vcpus = atoi_positive("Number of vCPUs", optarg);
 463                         TEST_ASSERT(nr_vcpus <= max_vcpus,
 464                                     "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
 465                         break;
 466                 case 's':
 467                         p.src_type = parse_backing_src_type(optarg);
 468                         break;
 469                 case 'h':
 470                 default:
 471                         help(argv[0]);
 472                         exit(0);
 473                 }
 474         }
 475
 476         for_each_guest_mode(run_test, &p);
 477
 478         return 0;
 479 }