2 * Copyright 2013 Red Hat Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * Authors: Jérôme Glisse <jglisse@redhat.com>
17 * Refer to include/linux/hmm.h for information about heterogeneous memory
18 * management or HMM for short.
21 #include <linux/hmm.h>
22 #include <linux/init.h>
23 #include <linux/rmap.h>
24 #include <linux/swap.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <linux/mmzone.h>
28 #include <linux/pagemap.h>
29 #include <linux/swapops.h>
30 #include <linux/hugetlb.h>
31 #include <linux/memremap.h>
32 #include <linux/jump_label.h>
33 #include <linux/mmu_notifier.h>
34 #include <linux/memory_hotplug.h>
36 #define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
38 #if IS_ENABLED(CONFIG_HMM_MIRROR)
39 static const struct mmu_notifier_ops hmm_mmu_notifier_ops;
42 * struct hmm - HMM per mm struct
44 * @mm: mm struct this HMM struct is bound to
45 * @lock: lock protecting ranges list
46 * @sequence: we track updates to the CPU page table with a sequence number
47 * @ranges: list of range being snapshotted
48 * @mirrors: list of mirrors for this mm
49 * @mmu_notifier: mmu notifier to track updates to CPU page table
50 * @mirrors_sem: read/write semaphore protecting the mirrors list
56 struct list_head ranges;
57 struct list_head mirrors;
58 struct mmu_notifier mmu_notifier;
59 struct rw_semaphore mirrors_sem;
63 * hmm_register - register HMM against an mm (HMM internal)
65 * @mm: mm struct to attach to
67 * This is not intended to be used directly by device drivers. It allocates an
68 * HMM struct if mm does not have one, and initializes it.
70 static struct hmm *hmm_register(struct mm_struct *mm)
72 struct hmm *hmm = READ_ONCE(mm->hmm);
76 * The hmm struct can only be freed once the mm_struct goes away,
77 * hence we should always have pre-allocated an new hmm struct
83 hmm = kmalloc(sizeof(*hmm), GFP_KERNEL);
86 INIT_LIST_HEAD(&hmm->mirrors);
87 init_rwsem(&hmm->mirrors_sem);
88 atomic_set(&hmm->sequence, 0);
89 hmm->mmu_notifier.ops = NULL;
90 INIT_LIST_HEAD(&hmm->ranges);
91 spin_lock_init(&hmm->lock);
95 * We should only get here if hold the mmap_sem in write mode ie on
96 * registration of first mirror through hmm_mirror_register()
98 hmm->mmu_notifier.ops = &hmm_mmu_notifier_ops;
99 if (__mmu_notifier_register(&hmm->mmu_notifier, mm)) {
104 spin_lock(&mm->page_table_lock);
109 spin_unlock(&mm->page_table_lock);
112 mmu_notifier_unregister(&hmm->mmu_notifier, mm);
119 void hmm_mm_destroy(struct mm_struct *mm)
124 static void hmm_invalidate_range(struct hmm *hmm,
125 enum hmm_update_type action,
129 struct hmm_mirror *mirror;
130 struct hmm_range *range;
132 spin_lock(&hmm->lock);
133 list_for_each_entry(range, &hmm->ranges, list) {
134 unsigned long addr, idx, npages;
136 if (end < range->start || start >= range->end)
139 range->valid = false;
140 addr = max(start, range->start);
141 idx = (addr - range->start) >> PAGE_SHIFT;
142 npages = (min(range->end, end) - addr) >> PAGE_SHIFT;
143 memset(&range->pfns[idx], 0, sizeof(*range->pfns) * npages);
145 spin_unlock(&hmm->lock);
147 down_read(&hmm->mirrors_sem);
148 list_for_each_entry(mirror, &hmm->mirrors, list)
149 mirror->ops->sync_cpu_device_pagetables(mirror, action,
151 up_read(&hmm->mirrors_sem);
154 static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm)
156 struct hmm_mirror *mirror;
157 struct hmm *hmm = mm->hmm;
159 down_write(&hmm->mirrors_sem);
160 mirror = list_first_entry_or_null(&hmm->mirrors, struct hmm_mirror,
163 list_del_init(&mirror->list);
164 if (mirror->ops->release) {
166 * Drop mirrors_sem so callback can wait on any pending
167 * work that might itself trigger mmu_notifier callback
168 * and thus would deadlock with us.
170 up_write(&hmm->mirrors_sem);
171 mirror->ops->release(mirror);
172 down_write(&hmm->mirrors_sem);
174 mirror = list_first_entry_or_null(&hmm->mirrors,
175 struct hmm_mirror, list);
177 up_write(&hmm->mirrors_sem);
180 static int hmm_invalidate_range_start(struct mmu_notifier *mn,
181 struct mm_struct *mm,
186 struct hmm *hmm = mm->hmm;
190 atomic_inc(&hmm->sequence);
195 static void hmm_invalidate_range_end(struct mmu_notifier *mn,
196 struct mm_struct *mm,
200 struct hmm *hmm = mm->hmm;
204 hmm_invalidate_range(mm->hmm, HMM_UPDATE_INVALIDATE, start, end);
207 static const struct mmu_notifier_ops hmm_mmu_notifier_ops = {
208 .release = hmm_release,
209 .invalidate_range_start = hmm_invalidate_range_start,
210 .invalidate_range_end = hmm_invalidate_range_end,
214 * hmm_mirror_register() - register a mirror against an mm
216 * @mirror: new mirror struct to register
217 * @mm: mm to register against
219 * To start mirroring a process address space, the device driver must register
220 * an HMM mirror struct.
222 * THE mm->mmap_sem MUST BE HELD IN WRITE MODE !
224 int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm)
227 if (!mm || !mirror || !mirror->ops)
231 mirror->hmm = hmm_register(mm);
235 down_write(&mirror->hmm->mirrors_sem);
236 if (mirror->hmm->mm == NULL) {
238 * A racing hmm_mirror_unregister() is about to destroy the hmm
239 * struct. Try again to allocate a new one.
241 up_write(&mirror->hmm->mirrors_sem);
245 list_add(&mirror->list, &mirror->hmm->mirrors);
246 up_write(&mirror->hmm->mirrors_sem);
251 EXPORT_SYMBOL(hmm_mirror_register);
254 * hmm_mirror_unregister() - unregister a mirror
256 * @mirror: new mirror struct to register
258 * Stop mirroring a process address space, and cleanup.
260 void hmm_mirror_unregister(struct hmm_mirror *mirror)
262 bool should_unregister = false;
263 struct mm_struct *mm;
266 if (mirror->hmm == NULL)
270 down_write(&hmm->mirrors_sem);
271 list_del_init(&mirror->list);
272 should_unregister = list_empty(&hmm->mirrors);
276 up_write(&hmm->mirrors_sem);
278 if (!should_unregister || mm == NULL)
281 spin_lock(&mm->page_table_lock);
284 spin_unlock(&mm->page_table_lock);
286 mmu_notifier_unregister_no_release(&hmm->mmu_notifier, mm);
289 EXPORT_SYMBOL(hmm_mirror_unregister);
291 struct hmm_vma_walk {
292 struct hmm_range *range;
298 static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,
299 bool write_fault, uint64_t *pfn)
301 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_REMOTE;
302 struct hmm_vma_walk *hmm_vma_walk = walk->private;
303 struct hmm_range *range = hmm_vma_walk->range;
304 struct vm_area_struct *vma = walk->vma;
307 flags |= hmm_vma_walk->block ? 0 : FAULT_FLAG_ALLOW_RETRY;
308 flags |= write_fault ? FAULT_FLAG_WRITE : 0;
309 ret = handle_mm_fault(vma, addr, flags);
310 if (ret & VM_FAULT_RETRY)
312 if (ret & VM_FAULT_ERROR) {
313 *pfn = range->values[HMM_PFN_ERROR];
320 static int hmm_pfns_bad(unsigned long addr,
322 struct mm_walk *walk)
324 struct hmm_vma_walk *hmm_vma_walk = walk->private;
325 struct hmm_range *range = hmm_vma_walk->range;
326 uint64_t *pfns = range->pfns;
329 i = (addr - range->start) >> PAGE_SHIFT;
330 for (; addr < end; addr += PAGE_SIZE, i++)
331 pfns[i] = range->values[HMM_PFN_ERROR];
337 * hmm_vma_walk_hole() - handle a range lacking valid pmd or pte(s)
338 * @start: range virtual start address (inclusive)
339 * @end: range virtual end address (exclusive)
340 * @fault: should we fault or not ?
341 * @write_fault: write fault ?
342 * @walk: mm_walk structure
343 * Returns: 0 on success, -EAGAIN after page fault, or page fault error
345 * This function will be called whenever pmd_none() or pte_none() returns true,
346 * or whenever there is no page directory covering the virtual address range.
348 static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end,
349 bool fault, bool write_fault,
350 struct mm_walk *walk)
352 struct hmm_vma_walk *hmm_vma_walk = walk->private;
353 struct hmm_range *range = hmm_vma_walk->range;
354 uint64_t *pfns = range->pfns;
357 hmm_vma_walk->last = addr;
358 i = (addr - range->start) >> PAGE_SHIFT;
359 for (; addr < end; addr += PAGE_SIZE, i++) {
360 pfns[i] = range->values[HMM_PFN_NONE];
361 if (fault || write_fault) {
364 ret = hmm_vma_do_fault(walk, addr, write_fault,
371 return (fault || write_fault) ? -EAGAIN : 0;
374 static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
375 uint64_t pfns, uint64_t cpu_flags,
376 bool *fault, bool *write_fault)
378 struct hmm_range *range = hmm_vma_walk->range;
380 *fault = *write_fault = false;
381 if (!hmm_vma_walk->fault)
384 /* We aren't ask to do anything ... */
385 if (!(pfns & range->flags[HMM_PFN_VALID]))
387 /* If this is device memory than only fault if explicitly requested */
388 if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
389 /* Do we fault on device memory ? */
390 if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) {
391 *write_fault = pfns & range->flags[HMM_PFN_WRITE];
397 /* If CPU page table is not valid then we need to fault */
398 *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);
399 /* Need to write fault ? */
400 if ((pfns & range->flags[HMM_PFN_WRITE]) &&
401 !(cpu_flags & range->flags[HMM_PFN_WRITE])) {
407 static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
408 const uint64_t *pfns, unsigned long npages,
409 uint64_t cpu_flags, bool *fault,
414 if (!hmm_vma_walk->fault) {
415 *fault = *write_fault = false;
419 for (i = 0; i < npages; ++i) {
420 hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags,
422 if ((*fault) || (*write_fault))
427 static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
428 struct mm_walk *walk)
430 struct hmm_vma_walk *hmm_vma_walk = walk->private;
431 struct hmm_range *range = hmm_vma_walk->range;
432 bool fault, write_fault;
433 unsigned long i, npages;
436 i = (addr - range->start) >> PAGE_SHIFT;
437 npages = (end - addr) >> PAGE_SHIFT;
438 pfns = &range->pfns[i];
439 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
440 0, &fault, &write_fault);
441 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
444 static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)
446 if (pmd_protnone(pmd))
448 return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |
449 range->flags[HMM_PFN_WRITE] :
450 range->flags[HMM_PFN_VALID];
453 static int hmm_vma_handle_pmd(struct mm_walk *walk,
459 struct hmm_vma_walk *hmm_vma_walk = walk->private;
460 struct hmm_range *range = hmm_vma_walk->range;
461 unsigned long pfn, npages, i;
462 bool fault, write_fault;
465 npages = (end - addr) >> PAGE_SHIFT;
466 cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
467 hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags,
468 &fault, &write_fault);
470 if (pmd_protnone(pmd) || fault || write_fault)
471 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
473 pfn = pmd_pfn(pmd) + pte_index(addr);
474 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
475 pfns[i] = hmm_pfn_from_pfn(range, pfn) | cpu_flags;
476 hmm_vma_walk->last = end;
480 static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)
482 if (pte_none(pte) || !pte_present(pte))
484 return pte_write(pte) ? range->flags[HMM_PFN_VALID] |
485 range->flags[HMM_PFN_WRITE] :
486 range->flags[HMM_PFN_VALID];
489 static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
490 unsigned long end, pmd_t *pmdp, pte_t *ptep,
493 struct hmm_vma_walk *hmm_vma_walk = walk->private;
494 struct hmm_range *range = hmm_vma_walk->range;
495 struct vm_area_struct *vma = walk->vma;
496 bool fault, write_fault;
499 uint64_t orig_pfn = *pfn;
501 *pfn = range->values[HMM_PFN_NONE];
502 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
503 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
504 &fault, &write_fault);
507 if (fault || write_fault)
512 if (!pte_present(pte)) {
513 swp_entry_t entry = pte_to_swp_entry(pte);
515 if (!non_swap_entry(entry)) {
516 if (fault || write_fault)
522 * This is a special swap entry, ignore migration, use
523 * device and report anything else as error.
525 if (is_device_private_entry(entry)) {
526 cpu_flags = range->flags[HMM_PFN_VALID] |
527 range->flags[HMM_PFN_DEVICE_PRIVATE];
528 cpu_flags |= is_write_device_private_entry(entry) ?
529 range->flags[HMM_PFN_WRITE] : 0;
530 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
531 &fault, &write_fault);
532 if (fault || write_fault)
534 *pfn = hmm_pfn_from_pfn(range, swp_offset(entry));
539 if (is_migration_entry(entry)) {
540 if (fault || write_fault) {
542 hmm_vma_walk->last = addr;
543 migration_entry_wait(vma->vm_mm,
550 /* Report error for everything else */
551 *pfn = range->values[HMM_PFN_ERROR];
555 if (fault || write_fault)
558 *pfn = hmm_pfn_from_pfn(range, pte_pfn(pte)) | cpu_flags;
563 /* Fault any virtual address we were asked to fault */
564 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
567 static int hmm_vma_walk_pmd(pmd_t *pmdp,
570 struct mm_walk *walk)
572 struct hmm_vma_walk *hmm_vma_walk = walk->private;
573 struct hmm_range *range = hmm_vma_walk->range;
574 uint64_t *pfns = range->pfns;
575 unsigned long addr = start, i;
578 i = (addr - range->start) >> PAGE_SHIFT;
582 return hmm_vma_walk_hole(start, end, walk);
584 if (pmd_huge(*pmdp) && (range->vma->vm_flags & VM_HUGETLB))
585 return hmm_pfns_bad(start, end, walk);
587 if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
591 * No need to take pmd_lock here, even if some other threads
592 * is splitting the huge pmd we will get that event through
593 * mmu_notifier callback.
595 * So just read pmd value and check again its a transparent
596 * huge or device mapping one and compute corresponding pfn
599 pmd = pmd_read_atomic(pmdp);
601 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
604 return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
608 return hmm_pfns_bad(start, end, walk);
610 ptep = pte_offset_map(pmdp, addr);
611 for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
614 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
616 /* hmm_vma_handle_pte() did unmap pte directory */
617 hmm_vma_walk->last = addr;
623 hmm_vma_walk->last = addr;
627 static void hmm_pfns_clear(struct hmm_range *range,
632 for (; addr < end; addr += PAGE_SIZE, pfns++)
633 *pfns = range->values[HMM_PFN_NONE];
636 static void hmm_pfns_special(struct hmm_range *range)
638 unsigned long addr = range->start, i = 0;
640 for (; addr < range->end; addr += PAGE_SIZE, i++)
641 range->pfns[i] = range->values[HMM_PFN_SPECIAL];
645 * hmm_vma_get_pfns() - snapshot CPU page table for a range of virtual addresses
646 * @range: range being snapshotted
647 * Returns: -EINVAL if invalid argument, -ENOMEM out of memory, -EPERM invalid
648 * vma permission, 0 success
650 * This snapshots the CPU page table for a range of virtual addresses. Snapshot
651 * validity is tracked by range struct. See hmm_vma_range_done() for further
654 * The range struct is initialized here. It tracks the CPU page table, but only
655 * if the function returns success (0), in which case the caller must then call
656 * hmm_vma_range_done() to stop CPU page table update tracking on this range.
658 * NOT CALLING hmm_vma_range_done() IF FUNCTION RETURNS 0 WILL LEAD TO SERIOUS
659 * MEMORY CORRUPTION ! YOU HAVE BEEN WARNED !
661 int hmm_vma_get_pfns(struct hmm_range *range)
663 struct vm_area_struct *vma = range->vma;
664 struct hmm_vma_walk hmm_vma_walk;
665 struct mm_walk mm_walk;
668 /* Sanity check, this really should not happen ! */
669 if (range->start < vma->vm_start || range->start >= vma->vm_end)
671 if (range->end < vma->vm_start || range->end > vma->vm_end)
674 hmm = hmm_register(vma->vm_mm);
677 /* Caller must have registered a mirror, via hmm_mirror_register() ! */
678 if (!hmm->mmu_notifier.ops)
681 /* FIXME support hugetlb fs */
682 if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL) ||
684 hmm_pfns_special(range);
688 if (!(vma->vm_flags & VM_READ)) {
690 * If vma do not allow read access, then assume that it does
691 * not allow write access, either. Architecture that allow
692 * write without read access are not supported by HMM, because
693 * operations such has atomic access would not work.
695 hmm_pfns_clear(range, range->pfns, range->start, range->end);
699 /* Initialize range to track CPU page table update */
700 spin_lock(&hmm->lock);
702 list_add_rcu(&range->list, &hmm->ranges);
703 spin_unlock(&hmm->lock);
705 hmm_vma_walk.fault = false;
706 hmm_vma_walk.range = range;
707 mm_walk.private = &hmm_vma_walk;
710 mm_walk.mm = vma->vm_mm;
711 mm_walk.pte_entry = NULL;
712 mm_walk.test_walk = NULL;
713 mm_walk.hugetlb_entry = NULL;
714 mm_walk.pmd_entry = hmm_vma_walk_pmd;
715 mm_walk.pte_hole = hmm_vma_walk_hole;
717 walk_page_range(range->start, range->end, &mm_walk);
720 EXPORT_SYMBOL(hmm_vma_get_pfns);
723 * hmm_vma_range_done() - stop tracking change to CPU page table over a range
724 * @range: range being tracked
725 * Returns: false if range data has been invalidated, true otherwise
727 * Range struct is used to track updates to the CPU page table after a call to
728 * either hmm_vma_get_pfns() or hmm_vma_fault(). Once the device driver is done
729 * using the data, or wants to lock updates to the data it got from those
730 * functions, it must call the hmm_vma_range_done() function, which will then
731 * stop tracking CPU page table updates.
733 * Note that device driver must still implement general CPU page table update
734 * tracking either by using hmm_mirror (see hmm_mirror_register()) or by using
735 * the mmu_notifier API directly.
737 * CPU page table update tracking done through hmm_range is only temporary and
738 * to be used while trying to duplicate CPU page table contents for a range of
741 * There are two ways to use this :
743 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
744 * trans = device_build_page_table_update_transaction(pfns);
745 * device_page_table_lock();
746 * if (!hmm_vma_range_done(range)) {
747 * device_page_table_unlock();
750 * device_commit_transaction(trans);
751 * device_page_table_unlock();
754 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
755 * device_page_table_lock();
756 * hmm_vma_range_done(range);
757 * device_update_page_table(range->pfns);
758 * device_page_table_unlock();
760 bool hmm_vma_range_done(struct hmm_range *range)
762 unsigned long npages = (range->end - range->start) >> PAGE_SHIFT;
765 if (range->end <= range->start) {
770 hmm = hmm_register(range->vma->vm_mm);
772 memset(range->pfns, 0, sizeof(*range->pfns) * npages);
776 spin_lock(&hmm->lock);
777 list_del_rcu(&range->list);
778 spin_unlock(&hmm->lock);
782 EXPORT_SYMBOL(hmm_vma_range_done);
785 * hmm_vma_fault() - try to fault some address in a virtual address range
786 * @range: range being faulted
787 * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem)
788 * Returns: 0 success, error otherwise (-EAGAIN means mmap_sem have been drop)
790 * This is similar to a regular CPU page fault except that it will not trigger
791 * any memory migration if the memory being faulted is not accessible by CPUs.
793 * On error, for one virtual address in the range, the function will mark the
794 * corresponding HMM pfn entry with an error flag.
796 * Expected use pattern:
798 * down_read(&mm->mmap_sem);
799 * // Find vma and address device wants to fault, initialize hmm_pfn_t
800 * // array accordingly
801 * ret = hmm_vma_fault(range, write, block);
804 * hmm_vma_range_done(range);
805 * // You might want to rate limit or yield to play nicely, you may
806 * // also commit any valid pfn in the array assuming that you are
807 * // getting true from hmm_vma_range_monitor_end()
816 * up_read(&mm->mmap_sem)
819 * // Take device driver lock that serialize device page table update
820 * driver_lock_device_page_table_update();
821 * hmm_vma_range_done(range);
822 * // Commit pfns we got from hmm_vma_fault()
823 * driver_unlock_device_page_table_update();
824 * up_read(&mm->mmap_sem)
826 * YOU MUST CALL hmm_vma_range_done() AFTER THIS FUNCTION RETURN SUCCESS (0)
827 * BEFORE FREEING THE range struct OR YOU WILL HAVE SERIOUS MEMORY CORRUPTION !
829 * YOU HAVE BEEN WARNED !
831 int hmm_vma_fault(struct hmm_range *range, bool block)
833 struct vm_area_struct *vma = range->vma;
834 unsigned long start = range->start;
835 struct hmm_vma_walk hmm_vma_walk;
836 struct mm_walk mm_walk;
840 /* Sanity check, this really should not happen ! */
841 if (range->start < vma->vm_start || range->start >= vma->vm_end)
843 if (range->end < vma->vm_start || range->end > vma->vm_end)
846 hmm = hmm_register(vma->vm_mm);
848 hmm_pfns_clear(range, range->pfns, range->start, range->end);
851 /* Caller must have registered a mirror using hmm_mirror_register() */
852 if (!hmm->mmu_notifier.ops)
855 /* FIXME support hugetlb fs */
856 if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL) ||
858 hmm_pfns_special(range);
862 if (!(vma->vm_flags & VM_READ)) {
864 * If vma do not allow read access, then assume that it does
865 * not allow write access, either. Architecture that allow
866 * write without read access are not supported by HMM, because
867 * operations such has atomic access would not work.
869 hmm_pfns_clear(range, range->pfns, range->start, range->end);
873 /* Initialize range to track CPU page table update */
874 spin_lock(&hmm->lock);
876 list_add_rcu(&range->list, &hmm->ranges);
877 spin_unlock(&hmm->lock);
879 hmm_vma_walk.fault = true;
880 hmm_vma_walk.block = block;
881 hmm_vma_walk.range = range;
882 mm_walk.private = &hmm_vma_walk;
883 hmm_vma_walk.last = range->start;
886 mm_walk.mm = vma->vm_mm;
887 mm_walk.pte_entry = NULL;
888 mm_walk.test_walk = NULL;
889 mm_walk.hugetlb_entry = NULL;
890 mm_walk.pmd_entry = hmm_vma_walk_pmd;
891 mm_walk.pte_hole = hmm_vma_walk_hole;
894 ret = walk_page_range(start, range->end, &mm_walk);
895 start = hmm_vma_walk.last;
896 } while (ret == -EAGAIN);
901 i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
902 hmm_pfns_clear(range, &range->pfns[i], hmm_vma_walk.last,
904 hmm_vma_range_done(range);
908 EXPORT_SYMBOL(hmm_vma_fault);
909 #endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
912 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
913 struct page *hmm_vma_alloc_locked_page(struct vm_area_struct *vma,
918 page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
924 EXPORT_SYMBOL(hmm_vma_alloc_locked_page);
927 static void hmm_devmem_ref_release(struct percpu_ref *ref)
929 struct hmm_devmem *devmem;
931 devmem = container_of(ref, struct hmm_devmem, ref);
932 complete(&devmem->completion);
935 static void hmm_devmem_ref_exit(void *data)
937 struct percpu_ref *ref = data;
938 struct hmm_devmem *devmem;
940 devmem = container_of(ref, struct hmm_devmem, ref);
941 percpu_ref_exit(ref);
942 devm_remove_action(devmem->device, &hmm_devmem_ref_exit, data);
945 static void hmm_devmem_ref_kill(void *data)
947 struct percpu_ref *ref = data;
948 struct hmm_devmem *devmem;
950 devmem = container_of(ref, struct hmm_devmem, ref);
951 percpu_ref_kill(ref);
952 wait_for_completion(&devmem->completion);
953 devm_remove_action(devmem->device, &hmm_devmem_ref_kill, data);
956 static int hmm_devmem_fault(struct vm_area_struct *vma,
958 const struct page *page,
962 struct hmm_devmem *devmem = page->pgmap->data;
964 return devmem->ops->fault(devmem, vma, addr, page, flags, pmdp);
967 static void hmm_devmem_free(struct page *page, void *data)
969 struct hmm_devmem *devmem = data;
971 page->mapping = NULL;
973 devmem->ops->free(devmem, page);
976 static DEFINE_MUTEX(hmm_devmem_lock);
977 static RADIX_TREE(hmm_devmem_radix, GFP_KERNEL);
979 static void hmm_devmem_radix_release(struct resource *resource)
983 mutex_lock(&hmm_devmem_lock);
984 for (key = resource->start;
985 key <= resource->end;
986 key += PA_SECTION_SIZE)
987 radix_tree_delete(&hmm_devmem_radix, key >> PA_SECTION_SHIFT);
988 mutex_unlock(&hmm_devmem_lock);
991 static void hmm_devmem_release(struct device *dev, void *data)
993 struct hmm_devmem *devmem = data;
994 struct resource *resource = devmem->resource;
995 unsigned long start_pfn, npages;
999 if (percpu_ref_tryget_live(&devmem->ref)) {
1000 dev_WARN(dev, "%s: page mapping is still live!\n", __func__);
1001 percpu_ref_put(&devmem->ref);
1004 /* pages are dead and unused, undo the arch mapping */
1005 start_pfn = (resource->start & ~(PA_SECTION_SIZE - 1)) >> PAGE_SHIFT;
1006 npages = ALIGN(resource_size(resource), PA_SECTION_SIZE) >> PAGE_SHIFT;
1008 page = pfn_to_page(start_pfn);
1009 zone = page_zone(page);
1011 mem_hotplug_begin();
1012 if (resource->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY)
1013 __remove_pages(zone, start_pfn, npages, NULL);
1015 arch_remove_memory(start_pfn << PAGE_SHIFT,
1016 npages << PAGE_SHIFT, NULL);
1019 hmm_devmem_radix_release(resource);
1022 static int hmm_devmem_pages_create(struct hmm_devmem *devmem)
1024 resource_size_t key, align_start, align_size, align_end;
1025 struct device *device = devmem->device;
1026 int ret, nid, is_ram;
1029 align_start = devmem->resource->start & ~(PA_SECTION_SIZE - 1);
1030 align_size = ALIGN(devmem->resource->start +
1031 resource_size(devmem->resource),
1032 PA_SECTION_SIZE) - align_start;
1034 is_ram = region_intersects(align_start, align_size,
1035 IORESOURCE_SYSTEM_RAM,
1037 if (is_ram == REGION_MIXED) {
1038 WARN_ONCE(1, "%s attempted on mixed region %pr\n",
1039 __func__, devmem->resource);
1042 if (is_ram == REGION_INTERSECTS)
1045 if (devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY)
1046 devmem->pagemap.type = MEMORY_DEVICE_PUBLIC;
1048 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
1050 devmem->pagemap.res = *devmem->resource;
1051 devmem->pagemap.page_fault = hmm_devmem_fault;
1052 devmem->pagemap.page_free = hmm_devmem_free;
1053 devmem->pagemap.dev = devmem->device;
1054 devmem->pagemap.ref = &devmem->ref;
1055 devmem->pagemap.data = devmem;
1057 mutex_lock(&hmm_devmem_lock);
1058 align_end = align_start + align_size - 1;
1059 for (key = align_start; key <= align_end; key += PA_SECTION_SIZE) {
1060 struct hmm_devmem *dup;
1062 dup = radix_tree_lookup(&hmm_devmem_radix,
1063 key >> PA_SECTION_SHIFT);
1065 dev_err(device, "%s: collides with mapping for %s\n",
1066 __func__, dev_name(dup->device));
1067 mutex_unlock(&hmm_devmem_lock);
1071 ret = radix_tree_insert(&hmm_devmem_radix,
1072 key >> PA_SECTION_SHIFT,
1075 dev_err(device, "%s: failed: %d\n", __func__, ret);
1076 mutex_unlock(&hmm_devmem_lock);
1080 mutex_unlock(&hmm_devmem_lock);
1082 nid = dev_to_node(device);
1084 nid = numa_mem_id();
1086 mem_hotplug_begin();
1088 * For device private memory we call add_pages() as we only need to
1089 * allocate and initialize struct page for the device memory. More-
1090 * over the device memory is un-accessible thus we do not want to
1091 * create a linear mapping for the memory like arch_add_memory()
1094 * For device public memory, which is accesible by the CPU, we do
1095 * want the linear mapping and thus use arch_add_memory().
1097 if (devmem->pagemap.type == MEMORY_DEVICE_PUBLIC)
1098 ret = arch_add_memory(nid, align_start, align_size, NULL,
1101 ret = add_pages(nid, align_start >> PAGE_SHIFT,
1102 align_size >> PAGE_SHIFT, NULL, false);
1105 goto error_add_memory;
1107 move_pfn_range_to_zone(&NODE_DATA(nid)->node_zones[ZONE_DEVICE],
1108 align_start >> PAGE_SHIFT,
1109 align_size >> PAGE_SHIFT, NULL);
1112 for (pfn = devmem->pfn_first; pfn < devmem->pfn_last; pfn++) {
1113 struct page *page = pfn_to_page(pfn);
1115 page->pgmap = &devmem->pagemap;
1120 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
1122 hmm_devmem_radix_release(devmem->resource);
1127 static int hmm_devmem_match(struct device *dev, void *data, void *match_data)
1129 struct hmm_devmem *devmem = data;
1131 return devmem->resource == match_data;
1134 static void hmm_devmem_pages_remove(struct hmm_devmem *devmem)
1136 devres_release(devmem->device, &hmm_devmem_release,
1137 &hmm_devmem_match, devmem->resource);
1141 * hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory
1143 * @ops: memory event device driver callback (see struct hmm_devmem_ops)
1144 * @device: device struct to bind the resource too
1145 * @size: size in bytes of the device memory to add
1146 * Returns: pointer to new hmm_devmem struct ERR_PTR otherwise
1148 * This function first finds an empty range of physical address big enough to
1149 * contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which
1150 * in turn allocates struct pages. It does not do anything beyond that; all
1151 * events affecting the memory will go through the various callbacks provided
1152 * by hmm_devmem_ops struct.
1154 * Device driver should call this function during device initialization and
1155 * is then responsible of memory management. HMM only provides helpers.
1157 struct hmm_devmem *hmm_devmem_add(const struct hmm_devmem_ops *ops,
1158 struct device *device,
1161 struct hmm_devmem *devmem;
1162 resource_size_t addr;
1165 dev_pagemap_get_ops();
1167 devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1168 GFP_KERNEL, dev_to_node(device));
1170 return ERR_PTR(-ENOMEM);
1172 init_completion(&devmem->completion);
1173 devmem->pfn_first = -1UL;
1174 devmem->pfn_last = -1UL;
1175 devmem->resource = NULL;
1176 devmem->device = device;
1179 ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1182 goto error_percpu_ref;
1184 ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1186 goto error_devm_add_action;
1188 size = ALIGN(size, PA_SECTION_SIZE);
1189 addr = min((unsigned long)iomem_resource.end,
1190 (1UL << MAX_PHYSMEM_BITS) - 1);
1191 addr = addr - size + 1UL;
1194 * FIXME add a new helper to quickly walk resource tree and find free
1197 * FIXME what about ioport_resource resource ?
1199 for (; addr > size && addr >= iomem_resource.start; addr -= size) {
1200 ret = region_intersects(addr, size, 0, IORES_DESC_NONE);
1201 if (ret != REGION_DISJOINT)
1204 devmem->resource = devm_request_mem_region(device, addr, size,
1206 if (!devmem->resource) {
1208 goto error_no_resource;
1212 if (!devmem->resource) {
1214 goto error_no_resource;
1217 devmem->resource->desc = IORES_DESC_DEVICE_PRIVATE_MEMORY;
1218 devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1219 devmem->pfn_last = devmem->pfn_first +
1220 (resource_size(devmem->resource) >> PAGE_SHIFT);
1222 ret = hmm_devmem_pages_create(devmem);
1226 devres_add(device, devmem);
1228 ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1230 hmm_devmem_remove(devmem);
1231 return ERR_PTR(ret);
1237 devm_release_mem_region(device, devmem->resource->start,
1238 resource_size(devmem->resource));
1240 error_devm_add_action:
1241 hmm_devmem_ref_kill(&devmem->ref);
1242 hmm_devmem_ref_exit(&devmem->ref);
1244 devres_free(devmem);
1245 return ERR_PTR(ret);
1247 EXPORT_SYMBOL(hmm_devmem_add);
1249 struct hmm_devmem *hmm_devmem_add_resource(const struct hmm_devmem_ops *ops,
1250 struct device *device,
1251 struct resource *res)
1253 struct hmm_devmem *devmem;
1256 if (res->desc != IORES_DESC_DEVICE_PUBLIC_MEMORY)
1257 return ERR_PTR(-EINVAL);
1259 dev_pagemap_get_ops();
1261 devmem = devres_alloc_node(&hmm_devmem_release, sizeof(*devmem),
1262 GFP_KERNEL, dev_to_node(device));
1264 return ERR_PTR(-ENOMEM);
1266 init_completion(&devmem->completion);
1267 devmem->pfn_first = -1UL;
1268 devmem->pfn_last = -1UL;
1269 devmem->resource = res;
1270 devmem->device = device;
1273 ret = percpu_ref_init(&devmem->ref, &hmm_devmem_ref_release,
1276 goto error_percpu_ref;
1278 ret = devm_add_action(device, hmm_devmem_ref_exit, &devmem->ref);
1280 goto error_devm_add_action;
1283 devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
1284 devmem->pfn_last = devmem->pfn_first +
1285 (resource_size(devmem->resource) >> PAGE_SHIFT);
1287 ret = hmm_devmem_pages_create(devmem);
1289 goto error_devm_add_action;
1291 devres_add(device, devmem);
1293 ret = devm_add_action(device, hmm_devmem_ref_kill, &devmem->ref);
1295 hmm_devmem_remove(devmem);
1296 return ERR_PTR(ret);
1301 error_devm_add_action:
1302 hmm_devmem_ref_kill(&devmem->ref);
1303 hmm_devmem_ref_exit(&devmem->ref);
1305 devres_free(devmem);
1306 return ERR_PTR(ret);
1308 EXPORT_SYMBOL(hmm_devmem_add_resource);
1311 * hmm_devmem_remove() - remove device memory (kill and free ZONE_DEVICE)
1313 * @devmem: hmm_devmem struct use to track and manage the ZONE_DEVICE memory
1315 * This will hot-unplug memory that was hotplugged by hmm_devmem_add on behalf
1316 * of the device driver. It will free struct page and remove the resource that
1317 * reserved the physical address range for this device memory.
1319 void hmm_devmem_remove(struct hmm_devmem *devmem)
1321 resource_size_t start, size;
1322 struct device *device;
1328 device = devmem->device;
1329 start = devmem->resource->start;
1330 size = resource_size(devmem->resource);
1332 cdm = devmem->resource->desc == IORES_DESC_DEVICE_PUBLIC_MEMORY;
1333 hmm_devmem_ref_kill(&devmem->ref);
1334 hmm_devmem_ref_exit(&devmem->ref);
1335 hmm_devmem_pages_remove(devmem);
1338 devm_release_mem_region(device, start, size);
1340 EXPORT_SYMBOL(hmm_devmem_remove);
1343 * A device driver that wants to handle multiple devices memory through a
1344 * single fake device can use hmm_device to do so. This is purely a helper
1345 * and it is not needed to make use of any HMM functionality.
1347 #define HMM_DEVICE_MAX 256
1349 static DECLARE_BITMAP(hmm_device_mask, HMM_DEVICE_MAX);
1350 static DEFINE_SPINLOCK(hmm_device_lock);
1351 static struct class *hmm_device_class;
1352 static dev_t hmm_device_devt;
1354 static void hmm_device_release(struct device *device)
1356 struct hmm_device *hmm_device;
1358 hmm_device = container_of(device, struct hmm_device, device);
1359 spin_lock(&hmm_device_lock);
1360 clear_bit(hmm_device->minor, hmm_device_mask);
1361 spin_unlock(&hmm_device_lock);
1366 struct hmm_device *hmm_device_new(void *drvdata)
1368 struct hmm_device *hmm_device;
1370 hmm_device = kzalloc(sizeof(*hmm_device), GFP_KERNEL);
1372 return ERR_PTR(-ENOMEM);
1374 spin_lock(&hmm_device_lock);
1375 hmm_device->minor = find_first_zero_bit(hmm_device_mask, HMM_DEVICE_MAX);
1376 if (hmm_device->minor >= HMM_DEVICE_MAX) {
1377 spin_unlock(&hmm_device_lock);
1379 return ERR_PTR(-EBUSY);
1381 set_bit(hmm_device->minor, hmm_device_mask);
1382 spin_unlock(&hmm_device_lock);
1384 dev_set_name(&hmm_device->device, "hmm_device%d", hmm_device->minor);
1385 hmm_device->device.devt = MKDEV(MAJOR(hmm_device_devt),
1387 hmm_device->device.release = hmm_device_release;
1388 dev_set_drvdata(&hmm_device->device, drvdata);
1389 hmm_device->device.class = hmm_device_class;
1390 device_initialize(&hmm_device->device);
1394 EXPORT_SYMBOL(hmm_device_new);
1396 void hmm_device_put(struct hmm_device *hmm_device)
1398 put_device(&hmm_device->device);
1400 EXPORT_SYMBOL(hmm_device_put);
1402 static int __init hmm_init(void)
1406 ret = alloc_chrdev_region(&hmm_device_devt, 0,
1412 hmm_device_class = class_create(THIS_MODULE, "hmm_device");
1413 if (IS_ERR(hmm_device_class)) {
1414 unregister_chrdev_region(hmm_device_devt, HMM_DEVICE_MAX);
1415 return PTR_ERR(hmm_device_class);
1420 device_initcall(hmm_init);
1421 #endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */