mm/gup_test.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7 #include <linux/highmem.h>
   8 #include "gup_test.h"
   9
  10 static void put_back_pages(unsigned int cmd, struct page **pages,
  11                            unsigned long nr_pages, unsigned int gup_test_flags)
  12 {
  13         unsigned long i;
  14
  15         switch (cmd) {
  16         case GUP_FAST_BENCHMARK:
  17         case GUP_BASIC_TEST:
  18                 for (i = 0; i < nr_pages; i++)
  19                         put_page(pages[i]);
  20                 break;
  21
  22         case PIN_FAST_BENCHMARK:
  23         case PIN_BASIC_TEST:
  24         case PIN_LONGTERM_BENCHMARK:
  25                 unpin_user_pages(pages, nr_pages);
  26                 break;
  27         case DUMP_USER_PAGES_TEST:
  28                 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
  29                         unpin_user_pages(pages, nr_pages);
  30                 } else {
  31                         for (i = 0; i < nr_pages; i++)
  32                                 put_page(pages[i]);
  33
  34                 }
  35                 break;
  36         }
  37 }
  38
  39 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  40                               unsigned long nr_pages)
  41 {
  42         unsigned long i;
  43         struct page *page;
  44
  45         switch (cmd) {
  46         case PIN_FAST_BENCHMARK:
  47         case PIN_BASIC_TEST:
  48         case PIN_LONGTERM_BENCHMARK:
  49                 for (i = 0; i < nr_pages; i++) {
  50                         page = pages[i];
  51                         if (WARN(!page_maybe_dma_pinned(page),
  52                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  53
  54                                 dump_page(page, "gup_test failure");
  55                                 break;
  56                         } else if (cmd == PIN_LONGTERM_BENCHMARK &&
  57                                 WARN(!is_longterm_pinnable_page(page),
  58                                      "pages[%lu] is NOT pinnable but pinned\n",
  59                                      i)) {
  60                                 dump_page(page, "gup_test failure");
  61                                 break;
  62                         }
  63                 }
  64                 break;
  65         }
  66 }
  67
  68 static void dump_pages_test(struct gup_test *gup, struct page **pages,
  69                             unsigned long nr_pages)
  70 {
  71         unsigned int index_to_dump;
  72         unsigned int i;
  73
  74         /*
  75          * Zero out any user-supplied page index that is out of range. Remember:
  76          * .which_pages[] contains a 1-based set of page indices.
  77          */
  78         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  79                 if (gup->which_pages[i] > nr_pages) {
  80                         pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
  81                                 i, gup->which_pages[i]);
  82                         gup->which_pages[i] = 0;
  83                 }
  84         }
  85
  86         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  87                 index_to_dump = gup->which_pages[i];
  88
  89                 if (index_to_dump) {
  90                         index_to_dump--; // Decode from 1-based, to 0-based
  91                         pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
  92                                 index_to_dump, gup->addr);
  93                         dump_page(pages[index_to_dump],
  94                                   "gup_test: dump_pages() test");
  95                 }
  96         }
  97 }
  98
  99 static int __gup_test_ioctl(unsigned int cmd,
 100                 struct gup_test *gup)
 101 {
 102         ktime_t start_time, end_time;
 103         unsigned long i, nr_pages, addr, next;
 104         long nr;
 105         struct page **pages;
 106         int ret = 0;
 107         bool needs_mmap_lock =
 108                 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
 109
 110         if (gup->size > ULONG_MAX)
 111                 return -EINVAL;
 112
 113         nr_pages = gup->size / PAGE_SIZE;
 114         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
 115         if (!pages)
 116                 return -ENOMEM;
 117
 118         if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
 119                 ret = -EINTR;
 120                 goto free_pages;
 121         }
 122
 123         i = 0;
 124         nr = gup->nr_pages_per_call;
 125         start_time = ktime_get();
 126         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
 127                 if (nr != gup->nr_pages_per_call)
 128                         break;
 129
 130                 next = addr + nr * PAGE_SIZE;
 131                 if (next > gup->addr + gup->size) {
 132                         next = gup->addr + gup->size;
 133                         nr = (next - addr) / PAGE_SIZE;
 134                 }
 135
 136                 switch (cmd) {
 137                 case GUP_FAST_BENCHMARK:
 138                         nr = get_user_pages_fast(addr, nr, gup->gup_flags,
 139                                                  pages + i);
 140                         break;
 141                 case GUP_BASIC_TEST:
 142                         nr = get_user_pages(addr, nr, gup->gup_flags, pages + i,
 143                                             NULL);
 144                         break;
 145                 case PIN_FAST_BENCHMARK:
 146                         nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
 147                                                  pages + i);
 148                         break;
 149                 case PIN_BASIC_TEST:
 150                         nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i,
 151                                             NULL);
 152                         break;
 153                 case PIN_LONGTERM_BENCHMARK:
 154                         nr = pin_user_pages(addr, nr,
 155                                             gup->gup_flags | FOLL_LONGTERM,
 156                                             pages + i, NULL);
 157                         break;
 158                 case DUMP_USER_PAGES_TEST:
 159                         if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
 160                                 nr = pin_user_pages(addr, nr, gup->gup_flags,
 161                                                     pages + i, NULL);
 162                         else
 163                                 nr = get_user_pages(addr, nr, gup->gup_flags,
 164                                                     pages + i, NULL);
 165                         break;
 166                 default:
 167                         ret = -EINVAL;
 168                         goto unlock;
 169                 }
 170
 171                 if (nr <= 0)
 172                         break;
 173                 i += nr;
 174         }
 175         end_time = ktime_get();
 176
 177         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 178         nr_pages = i;
 179
 180         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 181         gup->size = addr - gup->addr;
 182
 183         /*
 184          * Take an un-benchmark-timed moment to verify DMA pinned
 185          * state: print a warning if any non-dma-pinned pages are found:
 186          */
 187         verify_dma_pinned(cmd, pages, nr_pages);
 188
 189         if (cmd == DUMP_USER_PAGES_TEST)
 190                 dump_pages_test(gup, pages, nr_pages);
 191
 192         start_time = ktime_get();
 193
 194         put_back_pages(cmd, pages, nr_pages, gup->test_flags);
 195
 196         end_time = ktime_get();
 197         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 198
 199 unlock:
 200         if (needs_mmap_lock)
 201                 mmap_read_unlock(current->mm);
 202 free_pages:
 203         kvfree(pages);
 204         return ret;
 205 }
 206
 207 static DEFINE_MUTEX(pin_longterm_test_mutex);
 208 static struct page **pin_longterm_test_pages;
 209 static unsigned long pin_longterm_test_nr_pages;
 210
 211 static inline void pin_longterm_test_stop(void)
 212 {
 213         if (pin_longterm_test_pages) {
 214                 if (pin_longterm_test_nr_pages)
 215                         unpin_user_pages(pin_longterm_test_pages,
 216                                          pin_longterm_test_nr_pages);
 217                 kvfree(pin_longterm_test_pages);
 218                 pin_longterm_test_pages = NULL;
 219                 pin_longterm_test_nr_pages = 0;
 220         }
 221 }
 222
 223 static inline int pin_longterm_test_start(unsigned long arg)
 224 {
 225         long nr_pages, cur_pages, addr, remaining_pages;
 226         int gup_flags = FOLL_LONGTERM;
 227         struct pin_longterm_test args;
 228         struct page **pages;
 229         int ret = 0;
 230         bool fast;
 231
 232         if (pin_longterm_test_pages)
 233                 return -EINVAL;
 234
 235         if (copy_from_user(&args, (void __user *)arg, sizeof(args)))
 236                 return -EFAULT;
 237
 238         if (args.flags &
 239             ~(PIN_LONGTERM_TEST_FLAG_USE_WRITE|PIN_LONGTERM_TEST_FLAG_USE_FAST))
 240                 return -EINVAL;
 241         if (!IS_ALIGNED(args.addr | args.size, PAGE_SIZE))
 242                 return -EINVAL;
 243         if (args.size > LONG_MAX)
 244                 return -EINVAL;
 245         nr_pages = args.size / PAGE_SIZE;
 246         if (!nr_pages)
 247                 return -EINVAL;
 248
 249         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
 250         if (!pages)
 251                 return -ENOMEM;
 252
 253         if (args.flags & PIN_LONGTERM_TEST_FLAG_USE_WRITE)
 254                 gup_flags |= FOLL_WRITE;
 255         fast = !!(args.flags & PIN_LONGTERM_TEST_FLAG_USE_FAST);
 256
 257         if (!fast && mmap_read_lock_killable(current->mm)) {
 258                 kvfree(pages);
 259                 return -EINTR;
 260         }
 261
 262         pin_longterm_test_pages = pages;
 263         pin_longterm_test_nr_pages = 0;
 264
 265         while (nr_pages - pin_longterm_test_nr_pages) {
 266                 remaining_pages = nr_pages - pin_longterm_test_nr_pages;
 267                 addr = args.addr + pin_longterm_test_nr_pages * PAGE_SIZE;
 268
 269                 if (fast)
 270                         cur_pages = pin_user_pages_fast(addr, remaining_pages,
 271                                                         gup_flags, pages);
 272                 else
 273                         cur_pages = pin_user_pages(addr, remaining_pages,
 274                                                    gup_flags, pages, NULL);
 275                 if (cur_pages < 0) {
 276                         pin_longterm_test_stop();
 277                         ret = cur_pages;
 278                         break;
 279                 }
 280                 pin_longterm_test_nr_pages += cur_pages;
 281                 pages += cur_pages;
 282         }
 283
 284         if (!fast)
 285                 mmap_read_unlock(current->mm);
 286         return ret;
 287 }
 288
 289 static inline int pin_longterm_test_read(unsigned long arg)
 290 {
 291         __u64 user_addr;
 292         unsigned long i;
 293
 294         if (!pin_longterm_test_pages)
 295                 return -EINVAL;
 296
 297         if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr)))
 298                 return -EFAULT;
 299
 300         for (i = 0; i < pin_longterm_test_nr_pages; i++) {
 301                 void *addr = kmap_local_page(pin_longterm_test_pages[i]);
 302                 unsigned long ret;
 303
 304                 ret = copy_to_user((void __user *)(unsigned long)user_addr, addr,
 305                                    PAGE_SIZE);
 306                 kunmap_local(addr);
 307                 if (ret)
 308                         return -EFAULT;
 309                 user_addr += PAGE_SIZE;
 310         }
 311         return 0;
 312 }
 313
 314 static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd,
 315                                     unsigned long arg)
 316 {
 317         int ret = -EINVAL;
 318
 319         if (mutex_lock_killable(&pin_longterm_test_mutex))
 320                 return -EINTR;
 321
 322         switch (cmd) {
 323         case PIN_LONGTERM_TEST_START:
 324                 ret = pin_longterm_test_start(arg);
 325                 break;
 326         case PIN_LONGTERM_TEST_STOP:
 327                 pin_longterm_test_stop();
 328                 ret = 0;
 329                 break;
 330         case PIN_LONGTERM_TEST_READ:
 331                 ret = pin_longterm_test_read(arg);
 332                 break;
 333         }
 334
 335         mutex_unlock(&pin_longterm_test_mutex);
 336         return ret;
 337 }
 338
 339 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
 340                 unsigned long arg)
 341 {
 342         struct gup_test gup;
 343         int ret;
 344
 345         switch (cmd) {
 346         case GUP_FAST_BENCHMARK:
 347         case PIN_FAST_BENCHMARK:
 348         case PIN_LONGTERM_BENCHMARK:
 349         case GUP_BASIC_TEST:
 350         case PIN_BASIC_TEST:
 351         case DUMP_USER_PAGES_TEST:
 352                 break;
 353         case PIN_LONGTERM_TEST_START:
 354         case PIN_LONGTERM_TEST_STOP:
 355         case PIN_LONGTERM_TEST_READ:
 356                 return pin_longterm_test_ioctl(filep, cmd, arg);
 357         default:
 358                 return -EINVAL;
 359         }
 360
 361         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 362                 return -EFAULT;
 363
 364         ret = __gup_test_ioctl(cmd, &gup);
 365         if (ret)
 366                 return ret;
 367
 368         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 369                 return -EFAULT;
 370
 371         return 0;
 372 }
 373
 374 static int gup_test_release(struct inode *inode, struct file *file)
 375 {
 376         pin_longterm_test_stop();
 377
 378         return 0;
 379 }
 380
 381 static const struct file_operations gup_test_fops = {
 382         .open = nonseekable_open,
 383         .unlocked_ioctl = gup_test_ioctl,
 384         .release = gup_test_release,
 385 };
 386
 387 static int __init gup_test_init(void)
 388 {
 389         debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
 390                                    &gup_test_fops);
 391
 392         return 0;
 393 }
 394
 395 late_initcall(gup_test_init);