Merge branch 'for-linus' into for-next
[platform/kernel/linux-starfive.git] / mm / hmm.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright 2013 Red Hat Inc.
4  *
5  * Authors: Jérôme Glisse <jglisse@redhat.com>
6  */
7 /*
8  * Refer to include/linux/hmm.h for information about heterogeneous memory
9  * management or HMM for short.
10  */
11 #include <linux/pagewalk.h>
12 #include <linux/hmm.h>
13 #include <linux/init.h>
14 #include <linux/rmap.h>
15 #include <linux/swap.h>
16 #include <linux/slab.h>
17 #include <linux/sched.h>
18 #include <linux/mmzone.h>
19 #include <linux/pagemap.h>
20 #include <linux/swapops.h>
21 #include <linux/hugetlb.h>
22 #include <linux/memremap.h>
23 #include <linux/sched/mm.h>
24 #include <linux/jump_label.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/mmu_notifier.h>
27 #include <linux/memory_hotplug.h>
28
29 struct hmm_vma_walk {
30         struct hmm_range        *range;
31         struct dev_pagemap      *pgmap;
32         unsigned long           last;
33         unsigned int            flags;
34 };
35
36 static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,
37                             bool write_fault, uint64_t *pfn)
38 {
39         unsigned int flags = FAULT_FLAG_REMOTE;
40         struct hmm_vma_walk *hmm_vma_walk = walk->private;
41         struct hmm_range *range = hmm_vma_walk->range;
42         struct vm_area_struct *vma = walk->vma;
43         vm_fault_t ret;
44
45         if (!vma)
46                 goto err;
47
48         if (hmm_vma_walk->flags & HMM_FAULT_ALLOW_RETRY)
49                 flags |= FAULT_FLAG_ALLOW_RETRY;
50         if (write_fault)
51                 flags |= FAULT_FLAG_WRITE;
52
53         ret = handle_mm_fault(vma, addr, flags);
54         if (ret & VM_FAULT_RETRY) {
55                 /* Note, handle_mm_fault did up_read(&mm->mmap_sem)) */
56                 return -EAGAIN;
57         }
58         if (ret & VM_FAULT_ERROR)
59                 goto err;
60
61         return -EBUSY;
62
63 err:
64         *pfn = range->values[HMM_PFN_ERROR];
65         return -EFAULT;
66 }
67
68 static int hmm_pfns_fill(unsigned long addr, unsigned long end,
69                 struct hmm_range *range, enum hmm_pfn_value_e value)
70 {
71         uint64_t *pfns = range->pfns;
72         unsigned long i;
73
74         i = (addr - range->start) >> PAGE_SHIFT;
75         for (; addr < end; addr += PAGE_SIZE, i++)
76                 pfns[i] = range->values[value];
77
78         return 0;
79 }
80
81 /*
82  * hmm_vma_walk_hole_() - handle a range lacking valid pmd or pte(s)
83  * @addr: range virtual start address (inclusive)
84  * @end: range virtual end address (exclusive)
85  * @fault: should we fault or not ?
86  * @write_fault: write fault ?
87  * @walk: mm_walk structure
88  * Return: 0 on success, -EBUSY after page fault, or page fault error
89  *
90  * This function will be called whenever pmd_none() or pte_none() returns true,
91  * or whenever there is no page directory covering the virtual address range.
92  */
93 static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end,
94                               bool fault, bool write_fault,
95                               struct mm_walk *walk)
96 {
97         struct hmm_vma_walk *hmm_vma_walk = walk->private;
98         struct hmm_range *range = hmm_vma_walk->range;
99         uint64_t *pfns = range->pfns;
100         unsigned long i;
101
102         hmm_vma_walk->last = addr;
103         i = (addr - range->start) >> PAGE_SHIFT;
104
105         if (write_fault && walk->vma && !(walk->vma->vm_flags & VM_WRITE))
106                 return -EPERM;
107
108         for (; addr < end; addr += PAGE_SIZE, i++) {
109                 pfns[i] = range->values[HMM_PFN_NONE];
110                 if (fault || write_fault) {
111                         int ret;
112
113                         ret = hmm_vma_do_fault(walk, addr, write_fault,
114                                                &pfns[i]);
115                         if (ret != -EBUSY)
116                                 return ret;
117                 }
118         }
119
120         return (fault || write_fault) ? -EBUSY : 0;
121 }
122
123 static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
124                                       uint64_t pfns, uint64_t cpu_flags,
125                                       bool *fault, bool *write_fault)
126 {
127         struct hmm_range *range = hmm_vma_walk->range;
128
129         if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT)
130                 return;
131
132         /*
133          * So we not only consider the individual per page request we also
134          * consider the default flags requested for the range. The API can
135          * be used 2 ways. The first one where the HMM user coalesces
136          * multiple page faults into one request and sets flags per pfn for
137          * those faults. The second one where the HMM user wants to pre-
138          * fault a range with specific flags. For the latter one it is a
139          * waste to have the user pre-fill the pfn arrays with a default
140          * flags value.
141          */
142         pfns = (pfns & range->pfn_flags_mask) | range->default_flags;
143
144         /* We aren't ask to do anything ... */
145         if (!(pfns & range->flags[HMM_PFN_VALID]))
146                 return;
147         /* If this is device memory then only fault if explicitly requested */
148         if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
149                 /* Do we fault on device memory ? */
150                 if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) {
151                         *write_fault = pfns & range->flags[HMM_PFN_WRITE];
152                         *fault = true;
153                 }
154                 return;
155         }
156
157         /* If CPU page table is not valid then we need to fault */
158         *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);
159         /* Need to write fault ? */
160         if ((pfns & range->flags[HMM_PFN_WRITE]) &&
161             !(cpu_flags & range->flags[HMM_PFN_WRITE])) {
162                 *write_fault = true;
163                 *fault = true;
164         }
165 }
166
167 static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
168                                  const uint64_t *pfns, unsigned long npages,
169                                  uint64_t cpu_flags, bool *fault,
170                                  bool *write_fault)
171 {
172         unsigned long i;
173
174         if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) {
175                 *fault = *write_fault = false;
176                 return;
177         }
178
179         *fault = *write_fault = false;
180         for (i = 0; i < npages; ++i) {
181                 hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags,
182                                    fault, write_fault);
183                 if ((*write_fault))
184                         return;
185         }
186 }
187
188 static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
189                              struct mm_walk *walk)
190 {
191         struct hmm_vma_walk *hmm_vma_walk = walk->private;
192         struct hmm_range *range = hmm_vma_walk->range;
193         bool fault, write_fault;
194         unsigned long i, npages;
195         uint64_t *pfns;
196
197         i = (addr - range->start) >> PAGE_SHIFT;
198         npages = (end - addr) >> PAGE_SHIFT;
199         pfns = &range->pfns[i];
200         hmm_range_need_fault(hmm_vma_walk, pfns, npages,
201                              0, &fault, &write_fault);
202         return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
203 }
204
205 static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)
206 {
207         if (pmd_protnone(pmd))
208                 return 0;
209         return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |
210                                 range->flags[HMM_PFN_WRITE] :
211                                 range->flags[HMM_PFN_VALID];
212 }
213
214 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
215 static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
216                 unsigned long end, uint64_t *pfns, pmd_t pmd)
217 {
218         struct hmm_vma_walk *hmm_vma_walk = walk->private;
219         struct hmm_range *range = hmm_vma_walk->range;
220         unsigned long pfn, npages, i;
221         bool fault, write_fault;
222         uint64_t cpu_flags;
223
224         npages = (end - addr) >> PAGE_SHIFT;
225         cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
226         hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags,
227                              &fault, &write_fault);
228
229         if (pmd_protnone(pmd) || fault || write_fault)
230                 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
231
232         pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
233         for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
234                 if (pmd_devmap(pmd)) {
235                         hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
236                                               hmm_vma_walk->pgmap);
237                         if (unlikely(!hmm_vma_walk->pgmap))
238                                 return -EBUSY;
239                 }
240                 pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags;
241         }
242         if (hmm_vma_walk->pgmap) {
243                 put_dev_pagemap(hmm_vma_walk->pgmap);
244                 hmm_vma_walk->pgmap = NULL;
245         }
246         hmm_vma_walk->last = end;
247         return 0;
248 }
249 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
250 /* stub to allow the code below to compile */
251 int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
252                 unsigned long end, uint64_t *pfns, pmd_t pmd);
253 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
254
255 static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)
256 {
257         if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte))
258                 return 0;
259         return pte_write(pte) ? range->flags[HMM_PFN_VALID] |
260                                 range->flags[HMM_PFN_WRITE] :
261                                 range->flags[HMM_PFN_VALID];
262 }
263
264 static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
265                               unsigned long end, pmd_t *pmdp, pte_t *ptep,
266                               uint64_t *pfn)
267 {
268         struct hmm_vma_walk *hmm_vma_walk = walk->private;
269         struct hmm_range *range = hmm_vma_walk->range;
270         bool fault, write_fault;
271         uint64_t cpu_flags;
272         pte_t pte = *ptep;
273         uint64_t orig_pfn = *pfn;
274
275         *pfn = range->values[HMM_PFN_NONE];
276         fault = write_fault = false;
277
278         if (pte_none(pte)) {
279                 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0,
280                                    &fault, &write_fault);
281                 if (fault || write_fault)
282                         goto fault;
283                 return 0;
284         }
285
286         if (!pte_present(pte)) {
287                 swp_entry_t entry = pte_to_swp_entry(pte);
288
289                 if (!non_swap_entry(entry)) {
290                         cpu_flags = pte_to_hmm_pfn_flags(range, pte);
291                         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
292                                            &fault, &write_fault);
293                         if (fault || write_fault)
294                                 goto fault;
295                         return 0;
296                 }
297
298                 /*
299                  * This is a special swap entry, ignore migration, use
300                  * device and report anything else as error.
301                  */
302                 if (is_device_private_entry(entry)) {
303                         cpu_flags = range->flags[HMM_PFN_VALID] |
304                                 range->flags[HMM_PFN_DEVICE_PRIVATE];
305                         cpu_flags |= is_write_device_private_entry(entry) ?
306                                 range->flags[HMM_PFN_WRITE] : 0;
307                         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
308                                            &fault, &write_fault);
309                         if (fault || write_fault)
310                                 goto fault;
311                         *pfn = hmm_device_entry_from_pfn(range,
312                                             swp_offset(entry));
313                         *pfn |= cpu_flags;
314                         return 0;
315                 }
316
317                 if (is_migration_entry(entry)) {
318                         if (fault || write_fault) {
319                                 pte_unmap(ptep);
320                                 hmm_vma_walk->last = addr;
321                                 migration_entry_wait(walk->mm, pmdp, addr);
322                                 return -EBUSY;
323                         }
324                         return 0;
325                 }
326
327                 /* Report error for everything else */
328                 *pfn = range->values[HMM_PFN_ERROR];
329                 return -EFAULT;
330         } else {
331                 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
332                 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
333                                    &fault, &write_fault);
334         }
335
336         if (fault || write_fault)
337                 goto fault;
338
339         if (pte_devmap(pte)) {
340                 hmm_vma_walk->pgmap = get_dev_pagemap(pte_pfn(pte),
341                                               hmm_vma_walk->pgmap);
342                 if (unlikely(!hmm_vma_walk->pgmap))
343                         return -EBUSY;
344         } else if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && pte_special(pte)) {
345                 if (!is_zero_pfn(pte_pfn(pte))) {
346                         *pfn = range->values[HMM_PFN_SPECIAL];
347                         return -EFAULT;
348                 }
349                 /*
350                  * Since each architecture defines a struct page for the zero
351                  * page, just fall through and treat it like a normal page.
352                  */
353         }
354
355         *pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags;
356         return 0;
357
358 fault:
359         if (hmm_vma_walk->pgmap) {
360                 put_dev_pagemap(hmm_vma_walk->pgmap);
361                 hmm_vma_walk->pgmap = NULL;
362         }
363         pte_unmap(ptep);
364         /* Fault any virtual address we were asked to fault */
365         return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
366 }
367
368 static int hmm_vma_walk_pmd(pmd_t *pmdp,
369                             unsigned long start,
370                             unsigned long end,
371                             struct mm_walk *walk)
372 {
373         struct hmm_vma_walk *hmm_vma_walk = walk->private;
374         struct hmm_range *range = hmm_vma_walk->range;
375         uint64_t *pfns = range->pfns;
376         unsigned long addr = start, i;
377         pte_t *ptep;
378         pmd_t pmd;
379
380 again:
381         pmd = READ_ONCE(*pmdp);
382         if (pmd_none(pmd))
383                 return hmm_vma_walk_hole(start, end, walk);
384
385         if (thp_migration_supported() && is_pmd_migration_entry(pmd)) {
386                 bool fault, write_fault;
387                 unsigned long npages;
388                 uint64_t *pfns;
389
390                 i = (addr - range->start) >> PAGE_SHIFT;
391                 npages = (end - addr) >> PAGE_SHIFT;
392                 pfns = &range->pfns[i];
393
394                 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
395                                      0, &fault, &write_fault);
396                 if (fault || write_fault) {
397                         hmm_vma_walk->last = addr;
398                         pmd_migration_entry_wait(walk->mm, pmdp);
399                         return -EBUSY;
400                 }
401                 return 0;
402         } else if (!pmd_present(pmd))
403                 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
404
405         if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) {
406                 /*
407                  * No need to take pmd_lock here, even if some other thread
408                  * is splitting the huge pmd we will get that event through
409                  * mmu_notifier callback.
410                  *
411                  * So just read pmd value and check again it's a transparent
412                  * huge or device mapping one and compute corresponding pfn
413                  * values.
414                  */
415                 pmd = pmd_read_atomic(pmdp);
416                 barrier();
417                 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
418                         goto again;
419
420                 i = (addr - range->start) >> PAGE_SHIFT;
421                 return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
422         }
423
424         /*
425          * We have handled all the valid cases above ie either none, migration,
426          * huge or transparent huge. At this point either it is a valid pmd
427          * entry pointing to pte directory or it is a bad pmd that will not
428          * recover.
429          */
430         if (pmd_bad(pmd))
431                 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
432
433         ptep = pte_offset_map(pmdp, addr);
434         i = (addr - range->start) >> PAGE_SHIFT;
435         for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
436                 int r;
437
438                 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
439                 if (r) {
440                         /* hmm_vma_handle_pte() did unmap pte directory */
441                         hmm_vma_walk->last = addr;
442                         return r;
443                 }
444         }
445         if (hmm_vma_walk->pgmap) {
446                 /*
447                  * We do put_dev_pagemap() here and not in hmm_vma_handle_pte()
448                  * so that we can leverage get_dev_pagemap() optimization which
449                  * will not re-take a reference on a pgmap if we already have
450                  * one.
451                  */
452                 put_dev_pagemap(hmm_vma_walk->pgmap);
453                 hmm_vma_walk->pgmap = NULL;
454         }
455         pte_unmap(ptep - 1);
456
457         hmm_vma_walk->last = addr;
458         return 0;
459 }
460
461 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \
462     defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
463 static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud)
464 {
465         if (!pud_present(pud))
466                 return 0;
467         return pud_write(pud) ? range->flags[HMM_PFN_VALID] |
468                                 range->flags[HMM_PFN_WRITE] :
469                                 range->flags[HMM_PFN_VALID];
470 }
471
472 static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
473                 struct mm_walk *walk)
474 {
475         struct hmm_vma_walk *hmm_vma_walk = walk->private;
476         struct hmm_range *range = hmm_vma_walk->range;
477         unsigned long addr = start, next;
478         pmd_t *pmdp;
479         pud_t pud;
480         int ret;
481
482 again:
483         pud = READ_ONCE(*pudp);
484         if (pud_none(pud))
485                 return hmm_vma_walk_hole(start, end, walk);
486
487         if (pud_huge(pud) && pud_devmap(pud)) {
488                 unsigned long i, npages, pfn;
489                 uint64_t *pfns, cpu_flags;
490                 bool fault, write_fault;
491
492                 if (!pud_present(pud))
493                         return hmm_vma_walk_hole(start, end, walk);
494
495                 i = (addr - range->start) >> PAGE_SHIFT;
496                 npages = (end - addr) >> PAGE_SHIFT;
497                 pfns = &range->pfns[i];
498
499                 cpu_flags = pud_to_hmm_pfn_flags(range, pud);
500                 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
501                                      cpu_flags, &fault, &write_fault);
502                 if (fault || write_fault)
503                         return hmm_vma_walk_hole_(addr, end, fault,
504                                                 write_fault, walk);
505
506                 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
507                 for (i = 0; i < npages; ++i, ++pfn) {
508                         hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
509                                               hmm_vma_walk->pgmap);
510                         if (unlikely(!hmm_vma_walk->pgmap))
511                                 return -EBUSY;
512                         pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
513                                   cpu_flags;
514                 }
515                 if (hmm_vma_walk->pgmap) {
516                         put_dev_pagemap(hmm_vma_walk->pgmap);
517                         hmm_vma_walk->pgmap = NULL;
518                 }
519                 hmm_vma_walk->last = end;
520                 return 0;
521         }
522
523         split_huge_pud(walk->vma, pudp, addr);
524         if (pud_none(*pudp))
525                 goto again;
526
527         pmdp = pmd_offset(pudp, addr);
528         do {
529                 next = pmd_addr_end(addr, end);
530                 ret = hmm_vma_walk_pmd(pmdp, addr, next, walk);
531                 if (ret)
532                         return ret;
533         } while (pmdp++, addr = next, addr != end);
534
535         return 0;
536 }
537 #else
538 #define hmm_vma_walk_pud        NULL
539 #endif
540
541 #ifdef CONFIG_HUGETLB_PAGE
542 static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
543                                       unsigned long start, unsigned long end,
544                                       struct mm_walk *walk)
545 {
546         unsigned long addr = start, i, pfn;
547         struct hmm_vma_walk *hmm_vma_walk = walk->private;
548         struct hmm_range *range = hmm_vma_walk->range;
549         struct vm_area_struct *vma = walk->vma;
550         uint64_t orig_pfn, cpu_flags;
551         bool fault, write_fault;
552         spinlock_t *ptl;
553         pte_t entry;
554         int ret = 0;
555
556         ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);
557         entry = huge_ptep_get(pte);
558
559         i = (start - range->start) >> PAGE_SHIFT;
560         orig_pfn = range->pfns[i];
561         range->pfns[i] = range->values[HMM_PFN_NONE];
562         cpu_flags = pte_to_hmm_pfn_flags(range, entry);
563         fault = write_fault = false;
564         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
565                            &fault, &write_fault);
566         if (fault || write_fault) {
567                 ret = -ENOENT;
568                 goto unlock;
569         }
570
571         pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
572         for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
573                 range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
574                                  cpu_flags;
575         hmm_vma_walk->last = end;
576
577 unlock:
578         spin_unlock(ptl);
579
580         if (ret == -ENOENT)
581                 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
582
583         return ret;
584 }
585 #else
586 #define hmm_vma_walk_hugetlb_entry NULL
587 #endif /* CONFIG_HUGETLB_PAGE */
588
589 static int hmm_vma_walk_test(unsigned long start, unsigned long end,
590                              struct mm_walk *walk)
591 {
592         struct hmm_vma_walk *hmm_vma_walk = walk->private;
593         struct hmm_range *range = hmm_vma_walk->range;
594         struct vm_area_struct *vma = walk->vma;
595
596         /*
597          * Skip vma ranges that don't have struct page backing them or
598          * map I/O devices directly.
599          */
600         if (vma->vm_flags & (VM_IO | VM_PFNMAP | VM_MIXEDMAP))
601                 return -EFAULT;
602
603         /*
604          * If the vma does not allow read access, then assume that it does not
605          * allow write access either. HMM does not support architectures
606          * that allow write without read.
607          */
608         if (!(vma->vm_flags & VM_READ)) {
609                 bool fault, write_fault;
610
611                 /*
612                  * Check to see if a fault is requested for any page in the
613                  * range.
614                  */
615                 hmm_range_need_fault(hmm_vma_walk, range->pfns +
616                                         ((start - range->start) >> PAGE_SHIFT),
617                                         (end - start) >> PAGE_SHIFT,
618                                         0, &fault, &write_fault);
619                 if (fault || write_fault)
620                         return -EFAULT;
621
622                 hmm_pfns_fill(start, end, range, HMM_PFN_NONE);
623                 hmm_vma_walk->last = end;
624
625                 /* Skip this vma and continue processing the next vma. */
626                 return 1;
627         }
628
629         return 0;
630 }
631
632 static const struct mm_walk_ops hmm_walk_ops = {
633         .pud_entry      = hmm_vma_walk_pud,
634         .pmd_entry      = hmm_vma_walk_pmd,
635         .pte_hole       = hmm_vma_walk_hole,
636         .hugetlb_entry  = hmm_vma_walk_hugetlb_entry,
637         .test_walk      = hmm_vma_walk_test,
638 };
639
640 /**
641  * hmm_range_fault - try to fault some address in a virtual address range
642  * @range:      range being faulted
643  * @flags:      HMM_FAULT_* flags
644  *
645  * Return: the number of valid pages in range->pfns[] (from range start
646  * address), which may be zero.  On error one of the following status codes
647  * can be returned:
648  *
649  * -EINVAL:     Invalid arguments or mm or virtual address is in an invalid vma
650  *              (e.g., device file vma).
651  * -ENOMEM:     Out of memory.
652  * -EPERM:      Invalid permission (e.g., asking for write and range is read
653  *              only).
654  * -EAGAIN:     A page fault needs to be retried and mmap_sem was dropped.
655  * -EBUSY:      The range has been invalidated and the caller needs to wait for
656  *              the invalidation to finish.
657  * -EFAULT:     Invalid (i.e., either no valid vma or it is illegal to access
658  *              that range) number of valid pages in range->pfns[] (from
659  *              range start address).
660  *
661  * This is similar to a regular CPU page fault except that it will not trigger
662  * any memory migration if the memory being faulted is not accessible by CPUs
663  * and caller does not ask for migration.
664  *
665  * On error, for one virtual address in the range, the function will mark the
666  * corresponding HMM pfn entry with an error flag.
667  */
668 long hmm_range_fault(struct hmm_range *range, unsigned int flags)
669 {
670         struct hmm_vma_walk hmm_vma_walk = {
671                 .range = range,
672                 .last = range->start,
673                 .flags = flags,
674         };
675         struct mm_struct *mm = range->notifier->mm;
676         int ret;
677
678         lockdep_assert_held(&mm->mmap_sem);
679
680         do {
681                 /* If range is no longer valid force retry. */
682                 if (mmu_interval_check_retry(range->notifier,
683                                              range->notifier_seq))
684                         return -EBUSY;
685                 ret = walk_page_range(mm, hmm_vma_walk.last, range->end,
686                                       &hmm_walk_ops, &hmm_vma_walk);
687         } while (ret == -EBUSY);
688
689         if (ret)
690                 return ret;
691         return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
692 }
693 EXPORT_SYMBOL(hmm_range_fault);