mm/gup_test.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7 #include <linux/highmem.h>
   8 #include "gup_test.h"
   9
  10 static void put_back_pages(unsigned int cmd, struct page **pages,
  11                            unsigned long nr_pages, unsigned int gup_test_flags)
  12 {
  13         unsigned long i;
  14
  15         switch (cmd) {
  16         case GUP_FAST_BENCHMARK:
  17         case GUP_BASIC_TEST:
  18                 for (i = 0; i < nr_pages; i++)
  19                         put_page(pages[i]);
  20                 break;
  21
  22         case PIN_FAST_BENCHMARK:
  23         case PIN_BASIC_TEST:
  24         case PIN_LONGTERM_BENCHMARK:
  25                 unpin_user_pages(pages, nr_pages);
  26                 break;
  27         case DUMP_USER_PAGES_TEST:
  28                 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
  29                         unpin_user_pages(pages, nr_pages);
  30                 } else {
  31                         for (i = 0; i < nr_pages; i++)
  32                                 put_page(pages[i]);
  33
  34                 }
  35                 break;
  36         }
  37 }
  38
  39 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  40                               unsigned long nr_pages)
  41 {
  42         unsigned long i;
  43         struct page *page;
  44
  45         switch (cmd) {
  46         case PIN_FAST_BENCHMARK:
  47         case PIN_BASIC_TEST:
  48         case PIN_LONGTERM_BENCHMARK:
  49                 for (i = 0; i < nr_pages; i++) {
  50                         page = pages[i];
  51                         if (WARN(!page_maybe_dma_pinned(page),
  52                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  53
  54                                 dump_page(page, "gup_test failure");
  55                                 break;
  56                         } else if (cmd == PIN_LONGTERM_BENCHMARK &&
  57                                 WARN(!is_longterm_pinnable_page(page),
  58                                      "pages[%lu] is NOT pinnable but pinned\n",
  59                                      i)) {
  60                                 dump_page(page, "gup_test failure");
  61                                 break;
  62                         }
  63                 }
  64                 break;
  65         }
  66 }
  67
  68 static void dump_pages_test(struct gup_test *gup, struct page **pages,
  69                             unsigned long nr_pages)
  70 {
  71         unsigned int index_to_dump;
  72         unsigned int i;
  73
  74         /*
  75          * Zero out any user-supplied page index that is out of range. Remember:
  76          * .which_pages[] contains a 1-based set of page indices.
  77          */
  78         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  79                 if (gup->which_pages[i] > nr_pages) {
  80                         pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
  81                                 i, gup->which_pages[i]);
  82                         gup->which_pages[i] = 0;
  83                 }
  84         }
  85
  86         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  87                 index_to_dump = gup->which_pages[i];
  88
  89                 if (index_to_dump) {
  90                         index_to_dump--; // Decode from 1-based, to 0-based
  91                         pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
  92                                 index_to_dump, gup->addr);
  93                         dump_page(pages[index_to_dump],
  94                                   "gup_test: dump_pages() test");
  95                 }
  96         }
  97 }
  98
  99 static int __gup_test_ioctl(unsigned int cmd,
 100                 struct gup_test *gup)
 101 {
 102         ktime_t start_time, end_time;
 103         unsigned long i, nr_pages, addr, next;
 104         long nr;
 105         struct page **pages;
 106         int ret = 0;
 107         bool needs_mmap_lock =
 108                 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
 109
 110         if (gup->size > ULONG_MAX)
 111                 return -EINVAL;
 112
 113         nr_pages = gup->size / PAGE_SIZE;
 114         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
 115         if (!pages)
 116                 return -ENOMEM;
 117
 118         if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
 119                 ret = -EINTR;
 120                 goto free_pages;
 121         }
 122
 123         i = 0;
 124         nr = gup->nr_pages_per_call;
 125         start_time = ktime_get();
 126         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
 127                 if (nr != gup->nr_pages_per_call)
 128                         break;
 129
 130                 next = addr + nr * PAGE_SIZE;
 131                 if (next > gup->addr + gup->size) {
 132                         next = gup->addr + gup->size;
 133                         nr = (next - addr) / PAGE_SIZE;
 134                 }
 135
 136                 switch (cmd) {
 137                 case GUP_FAST_BENCHMARK:
 138                         nr = get_user_pages_fast(addr, nr, gup->gup_flags,
 139                                                  pages + i);
 140                         break;
 141                 case GUP_BASIC_TEST:
 142                         nr = get_user_pages(addr, nr, gup->gup_flags, pages + i);
 143                         break;
 144                 case PIN_FAST_BENCHMARK:
 145                         nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
 146                                                  pages + i);
 147                         break;
 148                 case PIN_BASIC_TEST:
 149                         nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i);
 150                         break;
 151                 case PIN_LONGTERM_BENCHMARK:
 152                         nr = pin_user_pages(addr, nr,
 153                                             gup->gup_flags | FOLL_LONGTERM,
 154                                             pages + i);
 155                         break;
 156                 case DUMP_USER_PAGES_TEST:
 157                         if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
 158                                 nr = pin_user_pages(addr, nr, gup->gup_flags,
 159                                                     pages + i);
 160                         else
 161                                 nr = get_user_pages(addr, nr, gup->gup_flags,
 162                                                     pages + i);
 163                         break;
 164                 default:
 165                         ret = -EINVAL;
 166                         goto unlock;
 167                 }
 168
 169                 if (nr <= 0)
 170                         break;
 171                 i += nr;
 172         }
 173         end_time = ktime_get();
 174
 175         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 176         nr_pages = i;
 177
 178         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 179         gup->size = addr - gup->addr;
 180
 181         /*
 182          * Take an un-benchmark-timed moment to verify DMA pinned
 183          * state: print a warning if any non-dma-pinned pages are found:
 184          */
 185         verify_dma_pinned(cmd, pages, nr_pages);
 186
 187         if (cmd == DUMP_USER_PAGES_TEST)
 188                 dump_pages_test(gup, pages, nr_pages);
 189
 190         start_time = ktime_get();
 191
 192         put_back_pages(cmd, pages, nr_pages, gup->test_flags);
 193
 194         end_time = ktime_get();
 195         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 196
 197 unlock:
 198         if (needs_mmap_lock)
 199                 mmap_read_unlock(current->mm);
 200 free_pages:
 201         kvfree(pages);
 202         return ret;
 203 }
 204
 205 static DEFINE_MUTEX(pin_longterm_test_mutex);
 206 static struct page **pin_longterm_test_pages;
 207 static unsigned long pin_longterm_test_nr_pages;
 208
 209 static inline void pin_longterm_test_stop(void)
 210 {
 211         if (pin_longterm_test_pages) {
 212                 if (pin_longterm_test_nr_pages)
 213                         unpin_user_pages(pin_longterm_test_pages,
 214                                          pin_longterm_test_nr_pages);
 215                 kvfree(pin_longterm_test_pages);
 216                 pin_longterm_test_pages = NULL;
 217                 pin_longterm_test_nr_pages = 0;
 218         }
 219 }
 220
 221 static inline int pin_longterm_test_start(unsigned long arg)
 222 {
 223         long nr_pages, cur_pages, addr, remaining_pages;
 224         int gup_flags = FOLL_LONGTERM;
 225         struct pin_longterm_test args;
 226         struct page **pages;
 227         int ret = 0;
 228         bool fast;
 229
 230         if (pin_longterm_test_pages)
 231                 return -EINVAL;
 232
 233         if (copy_from_user(&args, (void __user *)arg, sizeof(args)))
 234                 return -EFAULT;
 235
 236         if (args.flags &
 237             ~(PIN_LONGTERM_TEST_FLAG_USE_WRITE|PIN_LONGTERM_TEST_FLAG_USE_FAST))
 238                 return -EINVAL;
 239         if (!IS_ALIGNED(args.addr | args.size, PAGE_SIZE))
 240                 return -EINVAL;
 241         if (args.size > LONG_MAX)
 242                 return -EINVAL;
 243         nr_pages = args.size / PAGE_SIZE;
 244         if (!nr_pages)
 245                 return -EINVAL;
 246
 247         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
 248         if (!pages)
 249                 return -ENOMEM;
 250
 251         if (args.flags & PIN_LONGTERM_TEST_FLAG_USE_WRITE)
 252                 gup_flags |= FOLL_WRITE;
 253         fast = !!(args.flags & PIN_LONGTERM_TEST_FLAG_USE_FAST);
 254
 255         if (!fast && mmap_read_lock_killable(current->mm)) {
 256                 kvfree(pages);
 257                 return -EINTR;
 258         }
 259
 260         pin_longterm_test_pages = pages;
 261         pin_longterm_test_nr_pages = 0;
 262
 263         while (nr_pages - pin_longterm_test_nr_pages) {
 264                 remaining_pages = nr_pages - pin_longterm_test_nr_pages;
 265                 addr = args.addr + pin_longterm_test_nr_pages * PAGE_SIZE;
 266
 267                 if (fast)
 268                         cur_pages = pin_user_pages_fast(addr, remaining_pages,
 269                                                         gup_flags, pages);
 270                 else
 271                         cur_pages = pin_user_pages(addr, remaining_pages,
 272                                                    gup_flags, pages);
 273                 if (cur_pages < 0) {
 274                         pin_longterm_test_stop();
 275                         ret = cur_pages;
 276                         break;
 277                 }
 278                 pin_longterm_test_nr_pages += cur_pages;
 279                 pages += cur_pages;
 280         }
 281
 282         if (!fast)
 283                 mmap_read_unlock(current->mm);
 284         return ret;
 285 }
 286
 287 static inline int pin_longterm_test_read(unsigned long arg)
 288 {
 289         __u64 user_addr;
 290         unsigned long i;
 291
 292         if (!pin_longterm_test_pages)
 293                 return -EINVAL;
 294
 295         if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr)))
 296                 return -EFAULT;
 297
 298         for (i = 0; i < pin_longterm_test_nr_pages; i++) {
 299                 void *addr = kmap_local_page(pin_longterm_test_pages[i]);
 300                 unsigned long ret;
 301
 302                 ret = copy_to_user((void __user *)(unsigned long)user_addr, addr,
 303                                    PAGE_SIZE);
 304                 kunmap_local(addr);
 305                 if (ret)
 306                         return -EFAULT;
 307                 user_addr += PAGE_SIZE;
 308         }
 309         return 0;
 310 }
 311
 312 static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd,
 313                                     unsigned long arg)
 314 {
 315         int ret = -EINVAL;
 316
 317         if (mutex_lock_killable(&pin_longterm_test_mutex))
 318                 return -EINTR;
 319
 320         switch (cmd) {
 321         case PIN_LONGTERM_TEST_START:
 322                 ret = pin_longterm_test_start(arg);
 323                 break;
 324         case PIN_LONGTERM_TEST_STOP:
 325                 pin_longterm_test_stop();
 326                 ret = 0;
 327                 break;
 328         case PIN_LONGTERM_TEST_READ:
 329                 ret = pin_longterm_test_read(arg);
 330                 break;
 331         }
 332
 333         mutex_unlock(&pin_longterm_test_mutex);
 334         return ret;
 335 }
 336
 337 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
 338                 unsigned long arg)
 339 {
 340         struct gup_test gup;
 341         int ret;
 342
 343         switch (cmd) {
 344         case GUP_FAST_BENCHMARK:
 345         case PIN_FAST_BENCHMARK:
 346         case PIN_LONGTERM_BENCHMARK:
 347         case GUP_BASIC_TEST:
 348         case PIN_BASIC_TEST:
 349         case DUMP_USER_PAGES_TEST:
 350                 break;
 351         case PIN_LONGTERM_TEST_START:
 352         case PIN_LONGTERM_TEST_STOP:
 353         case PIN_LONGTERM_TEST_READ:
 354                 return pin_longterm_test_ioctl(filep, cmd, arg);
 355         default:
 356                 return -EINVAL;
 357         }
 358
 359         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 360                 return -EFAULT;
 361
 362         ret = __gup_test_ioctl(cmd, &gup);
 363         if (ret)
 364                 return ret;
 365
 366         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 367                 return -EFAULT;
 368
 369         return 0;
 370 }
 371
 372 static int gup_test_release(struct inode *inode, struct file *file)
 373 {
 374         pin_longterm_test_stop();
 375
 376         return 0;
 377 }
 378
 379 static const struct file_operations gup_test_fops = {
 380         .open = nonseekable_open,
 381         .unlocked_ioctl = gup_test_ioctl,
 382         .release = gup_test_release,
 383 };
 384
 385 static int __init gup_test_init(void)
 386 {
 387         debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
 388                                    &gup_test_fops);
 389
 390         return 0;
 391 }
 392
 393 late_initcall(gup_test_init);