mm/gup_benchmark.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7
   8 #define GUP_FAST_BENCHMARK      _IOWR('g', 1, struct gup_benchmark)
   9 #define GUP_BENCHMARK           _IOWR('g', 2, struct gup_benchmark)
  10 #define PIN_FAST_BENCHMARK      _IOWR('g', 3, struct gup_benchmark)
  11 #define PIN_BENCHMARK           _IOWR('g', 4, struct gup_benchmark)
  12 #define PIN_LONGTERM_BENCHMARK  _IOWR('g', 5, struct gup_benchmark)
  13
  14 struct gup_benchmark {
  15         __u64 get_delta_usec;
  16         __u64 put_delta_usec;
  17         __u64 addr;
  18         __u64 size;
  19         __u32 nr_pages_per_call;
  20         __u32 flags;
  21         __u64 expansion[10];    /* For future use */
  22 };
  23
  24 static void put_back_pages(unsigned int cmd, struct page **pages,
  25                            unsigned long nr_pages)
  26 {
  27         unsigned long i;
  28
  29         switch (cmd) {
  30         case GUP_FAST_BENCHMARK:
  31         case GUP_BENCHMARK:
  32                 for (i = 0; i < nr_pages; i++)
  33                         put_page(pages[i]);
  34                 break;
  35
  36         case PIN_FAST_BENCHMARK:
  37         case PIN_BENCHMARK:
  38         case PIN_LONGTERM_BENCHMARK:
  39                 unpin_user_pages(pages, nr_pages);
  40                 break;
  41         }
  42 }
  43
  44 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  45                               unsigned long nr_pages)
  46 {
  47         unsigned long i;
  48         struct page *page;
  49
  50         switch (cmd) {
  51         case PIN_FAST_BENCHMARK:
  52         case PIN_BENCHMARK:
  53         case PIN_LONGTERM_BENCHMARK:
  54                 for (i = 0; i < nr_pages; i++) {
  55                         page = pages[i];
  56                         if (WARN(!page_maybe_dma_pinned(page),
  57                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  58
  59                                 dump_page(page, "gup_benchmark failure");
  60                                 break;
  61                         }
  62                 }
  63                 break;
  64         }
  65 }
  66
  67 static int __gup_benchmark_ioctl(unsigned int cmd,
  68                 struct gup_benchmark *gup)
  69 {
  70         ktime_t start_time, end_time;
  71         unsigned long i, nr_pages, addr, next;
  72         int nr;
  73         struct page **pages;
  74         int ret = 0;
  75         bool needs_mmap_lock =
  76                 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
  77
  78         if (gup->size > ULONG_MAX)
  79                 return -EINVAL;
  80
  81         nr_pages = gup->size / PAGE_SIZE;
  82         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
  83         if (!pages)
  84                 return -ENOMEM;
  85
  86         if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
  87                 ret = -EINTR;
  88                 goto free_pages;
  89         }
  90
  91         i = 0;
  92         nr = gup->nr_pages_per_call;
  93         start_time = ktime_get();
  94         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
  95                 if (nr != gup->nr_pages_per_call)
  96                         break;
  97
  98                 next = addr + nr * PAGE_SIZE;
  99                 if (next > gup->addr + gup->size) {
 100                         next = gup->addr + gup->size;
 101                         nr = (next - addr) / PAGE_SIZE;
 102                 }
 103
 104                 /* Filter out most gup flags: only allow a tiny subset here: */
 105                 gup->flags &= FOLL_WRITE;
 106
 107                 switch (cmd) {
 108                 case GUP_FAST_BENCHMARK:
 109                         nr = get_user_pages_fast(addr, nr, gup->flags,
 110                                                  pages + i);
 111                         break;
 112                 case GUP_BENCHMARK:
 113                         nr = get_user_pages(addr, nr, gup->flags, pages + i,
 114                                             NULL);
 115                         break;
 116                 case PIN_FAST_BENCHMARK:
 117                         nr = pin_user_pages_fast(addr, nr, gup->flags,
 118                                                  pages + i);
 119                         break;
 120                 case PIN_BENCHMARK:
 121                         nr = pin_user_pages(addr, nr, gup->flags, pages + i,
 122                                             NULL);
 123                         break;
 124                 case PIN_LONGTERM_BENCHMARK:
 125                         nr = pin_user_pages(addr, nr,
 126                                             gup->flags | FOLL_LONGTERM,
 127                                             pages + i, NULL);
 128                         break;
 129                 default:
 130                         ret = -EINVAL;
 131                         goto unlock;
 132                 }
 133
 134                 if (nr <= 0)
 135                         break;
 136                 i += nr;
 137         }
 138         end_time = ktime_get();
 139
 140         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 141         nr_pages = i;
 142
 143         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 144         gup->size = addr - gup->addr;
 145
 146         /*
 147          * Take an un-benchmark-timed moment to verify DMA pinned
 148          * state: print a warning if any non-dma-pinned pages are found:
 149          */
 150         verify_dma_pinned(cmd, pages, nr_pages);
 151
 152         start_time = ktime_get();
 153
 154         put_back_pages(cmd, pages, nr_pages);
 155
 156         end_time = ktime_get();
 157         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 158
 159 unlock:
 160         if (needs_mmap_lock)
 161                 mmap_read_unlock(current->mm);
 162 free_pages:
 163         kvfree(pages);
 164         return ret;
 165 }
 166
 167 static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd,
 168                 unsigned long arg)
 169 {
 170         struct gup_benchmark gup;
 171         int ret;
 172
 173         switch (cmd) {
 174         case GUP_FAST_BENCHMARK:
 175         case GUP_BENCHMARK:
 176         case PIN_FAST_BENCHMARK:
 177         case PIN_BENCHMARK:
 178         case PIN_LONGTERM_BENCHMARK:
 179                 break;
 180         default:
 181                 return -EINVAL;
 182         }
 183
 184         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 185                 return -EFAULT;
 186
 187         ret = __gup_benchmark_ioctl(cmd, &gup);
 188         if (ret)
 189                 return ret;
 190
 191         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 192                 return -EFAULT;
 193
 194         return 0;
 195 }
 196
 197 static const struct file_operations gup_benchmark_fops = {
 198         .open = nonseekable_open,
 199         .unlocked_ioctl = gup_benchmark_ioctl,
 200 };
 201
 202 static int gup_benchmark_init(void)
 203 {
 204         debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL,
 205                                    &gup_benchmark_fops);
 206
 207         return 0;
 208 }
 209
 210 late_initcall(gup_benchmark_init);