Merge drm/drm-next into drm-intel-gt-next
[platform/kernel/linux-rpi.git] / mm / gup_test.c
1 #include <linux/kernel.h>
2 #include <linux/mm.h>
3 #include <linux/slab.h>
4 #include <linux/uaccess.h>
5 #include <linux/ktime.h>
6 #include <linux/debugfs.h>
7 #include "gup_test.h"
8
9 static void put_back_pages(unsigned int cmd, struct page **pages,
10                            unsigned long nr_pages, unsigned int gup_test_flags)
11 {
12         unsigned long i;
13
14         switch (cmd) {
15         case GUP_FAST_BENCHMARK:
16         case GUP_BASIC_TEST:
17                 for (i = 0; i < nr_pages; i++)
18                         put_page(pages[i]);
19                 break;
20
21         case PIN_FAST_BENCHMARK:
22         case PIN_BASIC_TEST:
23         case PIN_LONGTERM_BENCHMARK:
24                 unpin_user_pages(pages, nr_pages);
25                 break;
26         case DUMP_USER_PAGES_TEST:
27                 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
28                         unpin_user_pages(pages, nr_pages);
29                 } else {
30                         for (i = 0; i < nr_pages; i++)
31                                 put_page(pages[i]);
32
33                 }
34                 break;
35         }
36 }
37
38 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
39                               unsigned long nr_pages)
40 {
41         unsigned long i;
42         struct page *page;
43
44         switch (cmd) {
45         case PIN_FAST_BENCHMARK:
46         case PIN_BASIC_TEST:
47         case PIN_LONGTERM_BENCHMARK:
48                 for (i = 0; i < nr_pages; i++) {
49                         page = pages[i];
50                         if (WARN(!page_maybe_dma_pinned(page),
51                                  "pages[%lu] is NOT dma-pinned\n", i)) {
52
53                                 dump_page(page, "gup_test failure");
54                                 break;
55                         } else if (cmd == PIN_LONGTERM_BENCHMARK &&
56                                 WARN(!is_pinnable_page(page),
57                                      "pages[%lu] is NOT pinnable but pinned\n",
58                                      i)) {
59                                 dump_page(page, "gup_test failure");
60                                 break;
61                         }
62                 }
63                 break;
64         }
65 }
66
67 static void dump_pages_test(struct gup_test *gup, struct page **pages,
68                             unsigned long nr_pages)
69 {
70         unsigned int index_to_dump;
71         unsigned int i;
72
73         /*
74          * Zero out any user-supplied page index that is out of range. Remember:
75          * .which_pages[] contains a 1-based set of page indices.
76          */
77         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
78                 if (gup->which_pages[i] > nr_pages) {
79                         pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
80                                 i, gup->which_pages[i]);
81                         gup->which_pages[i] = 0;
82                 }
83         }
84
85         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
86                 index_to_dump = gup->which_pages[i];
87
88                 if (index_to_dump) {
89                         index_to_dump--; // Decode from 1-based, to 0-based
90                         pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
91                                 index_to_dump, gup->addr);
92                         dump_page(pages[index_to_dump],
93                                   "gup_test: dump_pages() test");
94                 }
95         }
96 }
97
98 static int __gup_test_ioctl(unsigned int cmd,
99                 struct gup_test *gup)
100 {
101         ktime_t start_time, end_time;
102         unsigned long i, nr_pages, addr, next;
103         long nr;
104         struct page **pages;
105         int ret = 0;
106         bool needs_mmap_lock =
107                 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
108
109         if (gup->size > ULONG_MAX)
110                 return -EINVAL;
111
112         nr_pages = gup->size / PAGE_SIZE;
113         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
114         if (!pages)
115                 return -ENOMEM;
116
117         if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
118                 ret = -EINTR;
119                 goto free_pages;
120         }
121
122         i = 0;
123         nr = gup->nr_pages_per_call;
124         start_time = ktime_get();
125         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
126                 if (nr != gup->nr_pages_per_call)
127                         break;
128
129                 next = addr + nr * PAGE_SIZE;
130                 if (next > gup->addr + gup->size) {
131                         next = gup->addr + gup->size;
132                         nr = (next - addr) / PAGE_SIZE;
133                 }
134
135                 switch (cmd) {
136                 case GUP_FAST_BENCHMARK:
137                         nr = get_user_pages_fast(addr, nr, gup->gup_flags,
138                                                  pages + i);
139                         break;
140                 case GUP_BASIC_TEST:
141                         nr = get_user_pages(addr, nr, gup->gup_flags, pages + i,
142                                             NULL);
143                         break;
144                 case PIN_FAST_BENCHMARK:
145                         nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
146                                                  pages + i);
147                         break;
148                 case PIN_BASIC_TEST:
149                         nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i,
150                                             NULL);
151                         break;
152                 case PIN_LONGTERM_BENCHMARK:
153                         nr = pin_user_pages(addr, nr,
154                                             gup->gup_flags | FOLL_LONGTERM,
155                                             pages + i, NULL);
156                         break;
157                 case DUMP_USER_PAGES_TEST:
158                         if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
159                                 nr = pin_user_pages(addr, nr, gup->gup_flags,
160                                                     pages + i, NULL);
161                         else
162                                 nr = get_user_pages(addr, nr, gup->gup_flags,
163                                                     pages + i, NULL);
164                         break;
165                 default:
166                         ret = -EINVAL;
167                         goto unlock;
168                 }
169
170                 if (nr <= 0)
171                         break;
172                 i += nr;
173         }
174         end_time = ktime_get();
175
176         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
177         nr_pages = i;
178
179         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
180         gup->size = addr - gup->addr;
181
182         /*
183          * Take an un-benchmark-timed moment to verify DMA pinned
184          * state: print a warning if any non-dma-pinned pages are found:
185          */
186         verify_dma_pinned(cmd, pages, nr_pages);
187
188         if (cmd == DUMP_USER_PAGES_TEST)
189                 dump_pages_test(gup, pages, nr_pages);
190
191         start_time = ktime_get();
192
193         put_back_pages(cmd, pages, nr_pages, gup->test_flags);
194
195         end_time = ktime_get();
196         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
197
198 unlock:
199         if (needs_mmap_lock)
200                 mmap_read_unlock(current->mm);
201 free_pages:
202         kvfree(pages);
203         return ret;
204 }
205
206 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
207                 unsigned long arg)
208 {
209         struct gup_test gup;
210         int ret;
211
212         switch (cmd) {
213         case GUP_FAST_BENCHMARK:
214         case PIN_FAST_BENCHMARK:
215         case PIN_LONGTERM_BENCHMARK:
216         case GUP_BASIC_TEST:
217         case PIN_BASIC_TEST:
218         case DUMP_USER_PAGES_TEST:
219                 break;
220         default:
221                 return -EINVAL;
222         }
223
224         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
225                 return -EFAULT;
226
227         ret = __gup_test_ioctl(cmd, &gup);
228         if (ret)
229                 return ret;
230
231         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
232                 return -EFAULT;
233
234         return 0;
235 }
236
237 static const struct file_operations gup_test_fops = {
238         .open = nonseekable_open,
239         .unlocked_ioctl = gup_test_ioctl,
240 };
241
242 static int __init gup_test_init(void)
243 {
244         debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
245                                    &gup_test_fops);
246
247         return 0;
248 }
249
250 late_initcall(gup_test_init);