mm/gup_test.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7 #include "gup_test.h"
   8
   9 static void put_back_pages(unsigned int cmd, struct page **pages,
  10                            unsigned long nr_pages, unsigned int gup_test_flags)
  11 {
  12         unsigned long i;
  13
  14         switch (cmd) {
  15         case GUP_FAST_BENCHMARK:
  16         case GUP_BASIC_TEST:
  17                 for (i = 0; i < nr_pages; i++)
  18                         put_page(pages[i]);
  19                 break;
  20
  21         case PIN_FAST_BENCHMARK:
  22         case PIN_BASIC_TEST:
  23         case PIN_LONGTERM_BENCHMARK:
  24                 unpin_user_pages(pages, nr_pages);
  25                 break;
  26         case DUMP_USER_PAGES_TEST:
  27                 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
  28                         unpin_user_pages(pages, nr_pages);
  29                 } else {
  30                         for (i = 0; i < nr_pages; i++)
  31                                 put_page(pages[i]);
  32
  33                 }
  34                 break;
  35         }
  36 }
  37
  38 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  39                               unsigned long nr_pages)
  40 {
  41         unsigned long i;
  42         struct page *page;
  43
  44         switch (cmd) {
  45         case PIN_FAST_BENCHMARK:
  46         case PIN_BASIC_TEST:
  47         case PIN_LONGTERM_BENCHMARK:
  48                 for (i = 0; i < nr_pages; i++) {
  49                         page = pages[i];
  50                         if (WARN(!page_maybe_dma_pinned(page),
  51                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  52
  53                                 dump_page(page, "gup_test failure");
  54                                 break;
  55                         }
  56                 }
  57                 break;
  58         }
  59 }
  60
  61 static void dump_pages_test(struct gup_test *gup, struct page **pages,
  62                             unsigned long nr_pages)
  63 {
  64         unsigned int index_to_dump;
  65         unsigned int i;
  66
  67         /*
  68          * Zero out any user-supplied page index that is out of range. Remember:
  69          * .which_pages[] contains a 1-based set of page indices.
  70          */
  71         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  72                 if (gup->which_pages[i] > nr_pages) {
  73                         pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
  74                                 i, gup->which_pages[i]);
  75                         gup->which_pages[i] = 0;
  76                 }
  77         }
  78
  79         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  80                 index_to_dump = gup->which_pages[i];
  81
  82                 if (index_to_dump) {
  83                         index_to_dump--; // Decode from 1-based, to 0-based
  84                         pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
  85                                 index_to_dump, gup->addr);
  86                         dump_page(pages[index_to_dump],
  87                                   "gup_test: dump_pages() test");
  88                 }
  89         }
  90 }
  91
  92 static int __gup_test_ioctl(unsigned int cmd,
  93                 struct gup_test *gup)
  94 {
  95         ktime_t start_time, end_time;
  96         unsigned long i, nr_pages, addr, next;
  97         long nr;
  98         struct page **pages;
  99         int ret = 0;
 100         bool needs_mmap_lock =
 101                 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
 102
 103         if (gup->size > ULONG_MAX)
 104                 return -EINVAL;
 105
 106         nr_pages = gup->size / PAGE_SIZE;
 107         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
 108         if (!pages)
 109                 return -ENOMEM;
 110
 111         if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
 112                 ret = -EINTR;
 113                 goto free_pages;
 114         }
 115
 116         i = 0;
 117         nr = gup->nr_pages_per_call;
 118         start_time = ktime_get();
 119         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
 120                 if (nr != gup->nr_pages_per_call)
 121                         break;
 122
 123                 next = addr + nr * PAGE_SIZE;
 124                 if (next > gup->addr + gup->size) {
 125                         next = gup->addr + gup->size;
 126                         nr = (next - addr) / PAGE_SIZE;
 127                 }
 128
 129                 switch (cmd) {
 130                 case GUP_FAST_BENCHMARK:
 131                         nr = get_user_pages_fast(addr, nr, gup->gup_flags,
 132                                                  pages + i);
 133                         break;
 134                 case GUP_BASIC_TEST:
 135                         nr = get_user_pages(addr, nr, gup->gup_flags, pages + i,
 136                                             NULL);
 137                         break;
 138                 case PIN_FAST_BENCHMARK:
 139                         nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
 140                                                  pages + i);
 141                         break;
 142                 case PIN_BASIC_TEST:
 143                         nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i,
 144                                             NULL);
 145                         break;
 146                 case PIN_LONGTERM_BENCHMARK:
 147                         nr = pin_user_pages(addr, nr,
 148                                             gup->gup_flags | FOLL_LONGTERM,
 149                                             pages + i, NULL);
 150                         break;
 151                 case DUMP_USER_PAGES_TEST:
 152                         if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
 153                                 nr = pin_user_pages(addr, nr, gup->gup_flags,
 154                                                     pages + i, NULL);
 155                         else
 156                                 nr = get_user_pages(addr, nr, gup->gup_flags,
 157                                                     pages + i, NULL);
 158                         break;
 159                 default:
 160                         ret = -EINVAL;
 161                         goto unlock;
 162                 }
 163
 164                 if (nr <= 0)
 165                         break;
 166                 i += nr;
 167         }
 168         end_time = ktime_get();
 169
 170         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 171         nr_pages = i;
 172
 173         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 174         gup->size = addr - gup->addr;
 175
 176         /*
 177          * Take an un-benchmark-timed moment to verify DMA pinned
 178          * state: print a warning if any non-dma-pinned pages are found:
 179          */
 180         verify_dma_pinned(cmd, pages, nr_pages);
 181
 182         if (cmd == DUMP_USER_PAGES_TEST)
 183                 dump_pages_test(gup, pages, nr_pages);
 184
 185         start_time = ktime_get();
 186
 187         put_back_pages(cmd, pages, nr_pages, gup->test_flags);
 188
 189         end_time = ktime_get();
 190         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 191
 192 unlock:
 193         if (needs_mmap_lock)
 194                 mmap_read_unlock(current->mm);
 195 free_pages:
 196         kvfree(pages);
 197         return ret;
 198 }
 199
 200 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
 201                 unsigned long arg)
 202 {
 203         struct gup_test gup;
 204         int ret;
 205
 206         switch (cmd) {
 207         case GUP_FAST_BENCHMARK:
 208         case PIN_FAST_BENCHMARK:
 209         case PIN_LONGTERM_BENCHMARK:
 210         case GUP_BASIC_TEST:
 211         case PIN_BASIC_TEST:
 212         case DUMP_USER_PAGES_TEST:
 213                 break;
 214         default:
 215                 return -EINVAL;
 216         }
 217
 218         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 219                 return -EFAULT;
 220
 221         ret = __gup_test_ioctl(cmd, &gup);
 222         if (ret)
 223                 return ret;
 224
 225         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 226                 return -EFAULT;
 227
 228         return 0;
 229 }
 230
 231 static const struct file_operations gup_test_fops = {
 232         .open = nonseekable_open,
 233         .unlocked_ioctl = gup_test_ioctl,
 234 };
 235
 236 static int __init gup_test_init(void)
 237 {
 238         debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
 239                                    &gup_test_fops);
 240
 241         return 0;
 242 }
 243
 244 late_initcall(gup_test_init);