tools/testing/selftests/mm/vm_util.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <string.h>
   3 #include <fcntl.h>
   4 #include <dirent.h>
   5 #include <sys/ioctl.h>
   6 #include <linux/userfaultfd.h>
   7 #include <linux/fs.h>
   8 #include <sys/syscall.h>
   9 #include <unistd.h>
  10 #include "../kselftest.h"
  11 #include "vm_util.h"
  12
  13 #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
  14 #define SMAP_FILE_PATH "/proc/self/smaps"
  15 #define STATUS_FILE_PATH "/proc/self/status"
  16 #define MAX_LINE_LENGTH 500
  17
  18 unsigned int __page_size;
  19 unsigned int __page_shift;
  20
  21 uint64_t pagemap_get_entry(int fd, char *start)
  22 {
  23         const unsigned long pfn = (unsigned long)start / getpagesize();
  24         uint64_t entry;
  25         int ret;
  26
  27         ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
  28         if (ret != sizeof(entry))
  29                 ksft_exit_fail_msg("reading pagemap failed\n");
  30         return entry;
  31 }
  32
  33 static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
  34 {
  35         struct pm_scan_arg arg;
  36
  37         arg.start = (uintptr_t)start;
  38         arg.end = (uintptr_t)(start + psize());
  39         arg.vec = (uintptr_t)r;
  40         arg.vec_len = 1;
  41         arg.flags = 0;
  42         arg.size = sizeof(struct pm_scan_arg);
  43         arg.max_pages = 0;
  44         arg.category_inverted = 0;
  45         arg.category_mask = 0;
  46         arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
  47                                   PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
  48                                   PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
  49         arg.return_mask = arg.category_anyof_mask;
  50
  51         return ioctl(fd, PAGEMAP_SCAN, &arg);
  52 }
  53
  54 static uint64_t pagemap_scan_get_categories(int fd, char *start)
  55 {
  56         struct page_region r;
  57         long ret;
  58
  59         ret = __pagemap_scan_get_categories(fd, start, &r);
  60         if (ret < 0)
  61                 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
  62         if (ret == 0)
  63                 return 0;
  64         return r.categories;
  65 }
  66
  67 /* `start` is any valid address. */
  68 static bool pagemap_scan_supported(int fd, char *start)
  69 {
  70         static int supported = -1;
  71         int ret;
  72
  73         if (supported != -1)
  74                 return supported;
  75
  76         /* Provide an invalid address in order to trigger EFAULT. */
  77         ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
  78         if (ret == 0)
  79                 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
  80
  81         supported = errno == EFAULT;
  82
  83         return supported;
  84 }
  85
  86 static bool page_entry_is(int fd, char *start, char *desc,
  87                           uint64_t pagemap_flags, uint64_t pagescan_flags)
  88 {
  89         bool m = pagemap_get_entry(fd, start) & pagemap_flags;
  90
  91         if (pagemap_scan_supported(fd, start)) {
  92                 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
  93
  94                 if (m == s)
  95                         return m;
  96
  97                 ksft_exit_fail_msg(
  98                         "read and ioctl return unmatched results for %s: %d %d", desc, m, s);
  99         }
 100         return m;
 101 }
 102
 103 bool pagemap_is_softdirty(int fd, char *start)
 104 {
 105         return page_entry_is(fd, start, "soft-dirty",
 106                                 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
 107 }
 108
 109 bool pagemap_is_swapped(int fd, char *start)
 110 {
 111         return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
 112 }
 113
 114 bool pagemap_is_populated(int fd, char *start)
 115 {
 116         return page_entry_is(fd, start, "populated",
 117                                 PM_PRESENT | PM_SWAP,
 118                                 PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
 119 }
 120
 121 unsigned long pagemap_get_pfn(int fd, char *start)
 122 {
 123         uint64_t entry = pagemap_get_entry(fd, start);
 124
 125         /* If present (63th bit), PFN is at bit 0 -- 54. */
 126         if (entry & PM_PRESENT)
 127                 return entry & 0x007fffffffffffffull;
 128         return -1ul;
 129 }
 130
 131 void clear_softdirty(void)
 132 {
 133         int ret;
 134         const char *ctrl = "4";
 135         int fd = open("/proc/self/clear_refs", O_WRONLY);
 136
 137         if (fd < 0)
 138                 ksft_exit_fail_msg("opening clear_refs failed\n");
 139         ret = write(fd, ctrl, strlen(ctrl));
 140         close(fd);
 141         if (ret != strlen(ctrl))
 142                 ksft_exit_fail_msg("writing clear_refs failed\n");
 143 }
 144
 145 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
 146 {
 147         while (fgets(buf, len, fp)) {
 148                 if (!strncmp(buf, pattern, strlen(pattern)))
 149                         return true;
 150         }
 151         return false;
 152 }
 153
 154 uint64_t read_pmd_pagesize(void)
 155 {
 156         int fd;
 157         char buf[20];
 158         ssize_t num_read;
 159
 160         fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
 161         if (fd == -1)
 162                 return 0;
 163
 164         num_read = read(fd, buf, 19);
 165         if (num_read < 1) {
 166                 close(fd);
 167                 return 0;
 168         }
 169         buf[num_read] = '\0';
 170         close(fd);
 171
 172         return strtoul(buf, NULL, 10);
 173 }
 174
 175 unsigned long rss_anon(void)
 176 {
 177         unsigned long rss_anon = 0;
 178         FILE *fp;
 179         char buffer[MAX_LINE_LENGTH];
 180
 181         fp = fopen(STATUS_FILE_PATH, "r");
 182         if (!fp)
 183                 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);
 184
 185         if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))
 186                 goto err_out;
 187
 188         if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)
 189                 ksft_exit_fail_msg("Reading status error\n");
 190
 191 err_out:
 192         fclose(fp);
 193         return rss_anon;
 194 }
 195
 196 bool __check_huge(void *addr, char *pattern, int nr_hpages,
 197                   uint64_t hpage_size)
 198 {
 199         uint64_t thp = -1;
 200         int ret;
 201         FILE *fp;
 202         char buffer[MAX_LINE_LENGTH];
 203         char addr_pattern[MAX_LINE_LENGTH];
 204
 205         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
 206                        (unsigned long) addr);
 207         if (ret >= MAX_LINE_LENGTH)
 208                 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
 209
 210         fp = fopen(SMAP_FILE_PATH, "r");
 211         if (!fp)
 212                 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
 213
 214         if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
 215                 goto err_out;
 216
 217         /*
 218          * Fetch the pattern in the same block and check the number of
 219          * hugepages.
 220          */
 221         if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
 222                 goto err_out;
 223
 224         snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
 225
 226         if (sscanf(buffer, addr_pattern, &thp) != 1)
 227                 ksft_exit_fail_msg("Reading smap error\n");
 228
 229 err_out:
 230         fclose(fp);
 231         return thp == (nr_hpages * (hpage_size >> 10));
 232 }
 233
 234 bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
 235 {
 236         return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
 237 }
 238
 239 bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
 240 {
 241         return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
 242 }
 243
 244 bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
 245 {
 246         return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
 247 }
 248
 249 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
 250 {
 251         uint64_t ent[2];
 252
 253         /* drop pmd */
 254         if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
 255                  MAP_FIXED | MAP_ANONYMOUS |
 256                  MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
 257                 ksft_exit_fail_msg("mmap transhuge\n");
 258
 259         if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
 260                 ksft_exit_fail_msg("MADV_HUGEPAGE\n");
 261
 262         /* allocate transparent huge page */
 263         *(volatile void **)ptr = ptr;
 264
 265         if (pread(pagemap_fd, ent, sizeof(ent),
 266                   (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
 267                 ksft_exit_fail_msg("read pagemap\n");
 268
 269         if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
 270             PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
 271             !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
 272                 return PAGEMAP_PFN(ent[0]);
 273
 274         return -1;
 275 }
 276
 277 unsigned long default_huge_page_size(void)
 278 {
 279         unsigned long hps = 0;
 280         char *line = NULL;
 281         size_t linelen = 0;
 282         FILE *f = fopen("/proc/meminfo", "r");
 283
 284         if (!f)
 285                 return 0;
 286         while (getline(&line, &linelen, f) > 0) {
 287                 if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
 288                         hps <<= 10;
 289                         break;
 290                 }
 291         }
 292
 293         free(line);
 294         fclose(f);
 295         return hps;
 296 }
 297
 298 int detect_hugetlb_page_sizes(size_t sizes[], int max)
 299 {
 300         DIR *dir = opendir("/sys/kernel/mm/hugepages/");
 301         int count = 0;
 302
 303         if (!dir)
 304                 return 0;
 305
 306         while (count < max) {
 307                 struct dirent *entry = readdir(dir);
 308                 size_t kb;
 309
 310                 if (!entry)
 311                         break;
 312                 if (entry->d_type != DT_DIR)
 313                         continue;
 314                 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
 315                         continue;
 316                 sizes[count++] = kb * 1024;
 317                 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
 318                                kb);
 319         }
 320         closedir(dir);
 321         return count;
 322 }
 323
 324 /* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
 325 int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
 326                               bool miss, bool wp, bool minor, uint64_t *ioctls)
 327 {
 328         struct uffdio_register uffdio_register = { 0 };
 329         uint64_t mode = 0;
 330         int ret = 0;
 331
 332         if (miss)
 333                 mode |= UFFDIO_REGISTER_MODE_MISSING;
 334         if (wp)
 335                 mode |= UFFDIO_REGISTER_MODE_WP;
 336         if (minor)
 337                 mode |= UFFDIO_REGISTER_MODE_MINOR;
 338
 339         uffdio_register.range.start = (unsigned long)addr;
 340         uffdio_register.range.len = len;
 341         uffdio_register.mode = mode;
 342
 343         if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
 344                 ret = -errno;
 345         else if (ioctls)
 346                 *ioctls = uffdio_register.ioctls;
 347
 348         return ret;
 349 }
 350
 351 int uffd_register(int uffd, void *addr, uint64_t len,
 352                   bool miss, bool wp, bool minor)
 353 {
 354         return uffd_register_with_ioctls(uffd, addr, len,
 355                                          miss, wp, minor, NULL);
 356 }
 357
 358 int uffd_unregister(int uffd, void *addr, uint64_t len)
 359 {
 360         struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
 361         int ret = 0;
 362
 363         if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
 364                 ret = -errno;
 365
 366         return ret;
 367 }
 368
 369 unsigned long get_free_hugepages(void)
 370 {
 371         unsigned long fhp = 0;
 372         char *line = NULL;
 373         size_t linelen = 0;
 374         FILE *f = fopen("/proc/meminfo", "r");
 375
 376         if (!f)
 377                 return fhp;
 378         while (getline(&line, &linelen, f) > 0) {
 379                 if (sscanf(line, "HugePages_Free:      %lu", &fhp) == 1)
 380                         break;
 381         }
 382
 383         free(line);
 384         fclose(f);
 385         return fhp;
 386 }