mm/gup_test.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7 #include "gup_test.h"
   8
   9 static void put_back_pages(unsigned int cmd, struct page **pages,
  10                            unsigned long nr_pages, unsigned int gup_test_flags)
  11 {
  12         unsigned long i;
  13
  14         switch (cmd) {
  15         case GUP_FAST_BENCHMARK:
  16         case GUP_BASIC_TEST:
  17                 for (i = 0; i < nr_pages; i++)
  18                         put_page(pages[i]);
  19                 break;
  20
  21         case PIN_FAST_BENCHMARK:
  22         case PIN_BASIC_TEST:
  23         case PIN_LONGTERM_BENCHMARK:
  24                 unpin_user_pages(pages, nr_pages);
  25                 break;
  26         case DUMP_USER_PAGES_TEST:
  27                 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
  28                         unpin_user_pages(pages, nr_pages);
  29                 } else {
  30                         for (i = 0; i < nr_pages; i++)
  31                                 put_page(pages[i]);
  32
  33                 }
  34                 break;
  35         }
  36 }
  37
  38 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  39                               unsigned long nr_pages)
  40 {
  41         unsigned long i;
  42         struct page *page;
  43
  44         switch (cmd) {
  45         case PIN_FAST_BENCHMARK:
  46         case PIN_BASIC_TEST:
  47         case PIN_LONGTERM_BENCHMARK:
  48                 for (i = 0; i < nr_pages; i++) {
  49                         page = pages[i];
  50                         if (WARN(!page_maybe_dma_pinned(page),
  51                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  52
  53                                 dump_page(page, "gup_test failure");
  54                                 break;
  55                         }
  56                 }
  57                 break;
  58         }
  59 }
  60
  61 static void dump_pages_test(struct gup_test *gup, struct page **pages,
  62                             unsigned long nr_pages)
  63 {
  64         unsigned int index_to_dump;
  65         unsigned int i;
  66
  67         /*
  68          * Zero out any user-supplied page index that is out of range. Remember:
  69          * .which_pages[] contains a 1-based set of page indices.
  70          */
  71         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  72                 if (gup->which_pages[i] > nr_pages) {
  73                         pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
  74                                 i, gup->which_pages[i]);
  75                         gup->which_pages[i] = 0;
  76                 }
  77         }
  78
  79         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  80                 index_to_dump = gup->which_pages[i];
  81
  82                 if (index_to_dump) {
  83                         index_to_dump--; // Decode from 1-based, to 0-based
  84                         pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
  85                                 index_to_dump, gup->addr);
  86                         dump_page(pages[index_to_dump],
  87                                   "gup_test: dump_pages() test");
  88                 }
  89         }
  90 }
  91
  92 static int __gup_test_ioctl(unsigned int cmd,
  93                 struct gup_test *gup)
  94 {
  95         ktime_t start_time, end_time;
  96         unsigned long i, nr_pages, addr, next;
  97         int nr;
  98         struct page **pages;
  99         int ret = 0;
 100         bool needs_mmap_lock =
 101                 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
 102
 103         if (gup->size > ULONG_MAX)
 104                 return -EINVAL;
 105
 106         nr_pages = gup->size / PAGE_SIZE;
 107         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
 108         if (!pages)
 109                 return -ENOMEM;
 110
 111         if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
 112                 ret = -EINTR;
 113                 goto free_pages;
 114         }
 115
 116         i = 0;
 117         nr = gup->nr_pages_per_call;
 118         start_time = ktime_get();
 119         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
 120                 if (nr != gup->nr_pages_per_call)
 121                         break;
 122
 123                 next = addr + nr * PAGE_SIZE;
 124                 if (next > gup->addr + gup->size) {
 125                         next = gup->addr + gup->size;
 126                         nr = (next - addr) / PAGE_SIZE;
 127                 }
 128
 129                 /* Filter out most gup flags: only allow a tiny subset here: */
 130                 gup->flags &= FOLL_WRITE;
 131
 132                 switch (cmd) {
 133                 case GUP_FAST_BENCHMARK:
 134                         nr = get_user_pages_fast(addr, nr, gup->flags,
 135                                                  pages + i);
 136                         break;
 137                 case GUP_BASIC_TEST:
 138                         nr = get_user_pages(addr, nr, gup->flags, pages + i,
 139                                             NULL);
 140                         break;
 141                 case PIN_FAST_BENCHMARK:
 142                         nr = pin_user_pages_fast(addr, nr, gup->flags,
 143                                                  pages + i);
 144                         break;
 145                 case PIN_BASIC_TEST:
 146                         nr = pin_user_pages(addr, nr, gup->flags, pages + i,
 147                                             NULL);
 148                         break;
 149                 case PIN_LONGTERM_BENCHMARK:
 150                         nr = pin_user_pages(addr, nr,
 151                                             gup->flags | FOLL_LONGTERM,
 152                                             pages + i, NULL);
 153                         break;
 154                 case DUMP_USER_PAGES_TEST:
 155                         if (gup->flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
 156                                 nr = pin_user_pages(addr, nr, gup->flags,
 157                                                     pages + i, NULL);
 158                         else
 159                                 nr = get_user_pages(addr, nr, gup->flags,
 160                                                     pages + i, NULL);
 161                         break;
 162                 default:
 163                         ret = -EINVAL;
 164                         goto unlock;
 165                 }
 166
 167                 if (nr <= 0)
 168                         break;
 169                 i += nr;
 170         }
 171         end_time = ktime_get();
 172
 173         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 174         nr_pages = i;
 175
 176         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 177         gup->size = addr - gup->addr;
 178
 179         /*
 180          * Take an un-benchmark-timed moment to verify DMA pinned
 181          * state: print a warning if any non-dma-pinned pages are found:
 182          */
 183         verify_dma_pinned(cmd, pages, nr_pages);
 184
 185         if (cmd == DUMP_USER_PAGES_TEST)
 186                 dump_pages_test(gup, pages, nr_pages);
 187
 188         start_time = ktime_get();
 189
 190         put_back_pages(cmd, pages, nr_pages, gup->flags);
 191
 192         end_time = ktime_get();
 193         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 194
 195 unlock:
 196         if (needs_mmap_lock)
 197                 mmap_read_unlock(current->mm);
 198 free_pages:
 199         kvfree(pages);
 200         return ret;
 201 }
 202
 203 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
 204                 unsigned long arg)
 205 {
 206         struct gup_test gup;
 207         int ret;
 208
 209         switch (cmd) {
 210         case GUP_FAST_BENCHMARK:
 211         case PIN_FAST_BENCHMARK:
 212         case PIN_LONGTERM_BENCHMARK:
 213         case GUP_BASIC_TEST:
 214         case PIN_BASIC_TEST:
 215         case DUMP_USER_PAGES_TEST:
 216                 break;
 217         default:
 218                 return -EINVAL;
 219         }
 220
 221         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 222                 return -EFAULT;
 223
 224         ret = __gup_test_ioctl(cmd, &gup);
 225         if (ret)
 226                 return ret;
 227
 228         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 229                 return -EFAULT;
 230
 231         return 0;
 232 }
 233
 234 static const struct file_operations gup_test_fops = {
 235         .open = nonseekable_open,
 236         .unlocked_ioctl = gup_test_ioctl,
 237 };
 238
 239 static int __init gup_test_init(void)
 240 {
 241         debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
 242                                    &gup_test_fops);
 243
 244         return 0;
 245 }
 246
 247 late_initcall(gup_test_init);