1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2009 Red Hat, Inc.
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 #include <linux/sched.h>
10 #include <linux/sched/mm.h>
11 #include <linux/sched/coredump.h>
12 #include <linux/sched/numa_balancing.h>
13 #include <linux/highmem.h>
14 #include <linux/hugetlb.h>
15 #include <linux/mmu_notifier.h>
16 #include <linux/rmap.h>
17 #include <linux/swap.h>
18 #include <linux/shrinker.h>
19 #include <linux/mm_inline.h>
20 #include <linux/swapops.h>
21 #include <linux/backing-dev.h>
22 #include <linux/dax.h>
23 #include <linux/khugepaged.h>
24 #include <linux/freezer.h>
25 #include <linux/pfn_t.h>
26 #include <linux/mman.h>
27 #include <linux/memremap.h>
28 #include <linux/pagemap.h>
29 #include <linux/debugfs.h>
30 #include <linux/migrate.h>
31 #include <linux/hashtable.h>
32 #include <linux/userfaultfd_k.h>
33 #include <linux/page_idle.h>
34 #include <linux/shmem_fs.h>
35 #include <linux/oom.h>
36 #include <linux/numa.h>
37 #include <linux/page_owner.h>
38 #include <linux/sched/sysctl.h>
39 #include <linux/memory-tiers.h>
42 #include <asm/pgalloc.h>
46 #define CREATE_TRACE_POINTS
47 #include <trace/events/thp.h>
50 * By default, transparent hugepage support is disabled in order to avoid
51 * risking an increased memory footprint for applications that are not
52 * guaranteed to benefit from it. When transparent hugepage support is
53 * enabled, it is for all mappings, and khugepaged scans all mappings.
54 * Defrag is invoked by khugepaged hugepage allocations and by page faults
55 * for all hugepage allocations.
57 unsigned long transparent_hugepage_flags __read_mostly =
58 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
59 (1<<TRANSPARENT_HUGEPAGE_FLAG)|
61 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
62 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
64 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
65 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
66 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
68 static struct shrinker deferred_split_shrinker;
70 static atomic_t huge_zero_refcount;
71 struct page *huge_zero_page __read_mostly;
72 unsigned long huge_zero_pfn __read_mostly = ~0UL;
74 bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags,
75 bool smaps, bool in_pf, bool enforce_sysfs)
77 if (!vma->vm_mm) /* vdso */
81 * Explicitly disabled through madvise or prctl, or some
82 * architectures may disable THP for some mappings, for
85 if ((vm_flags & VM_NOHUGEPAGE) ||
86 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
89 * If the hardware/firmware marked hugepage support disabled.
91 if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_NEVER_DAX))
94 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */
99 * Special VMA and hugetlb VMA.
100 * Must be checked after dax since some dax mappings may have
103 if (vm_flags & VM_NO_KHUGEPAGED)
107 * Check alignment for file vma and size for both file and anon vma.
109 * Skip the check for page fault. Huge fault does the check in fault
110 * handlers. And this check is not suitable for huge PUD fault.
113 !transhuge_vma_suitable(vma, (vma->vm_end - HPAGE_PMD_SIZE)))
117 * Enabled via shmem mount options or sysfs settings.
118 * Must be done before hugepage flags check since shmem has its
121 if (!in_pf && shmem_file(vma->vm_file))
122 return shmem_is_huge(file_inode(vma->vm_file), vma->vm_pgoff,
123 !enforce_sysfs, vma->vm_mm, vm_flags);
125 /* Enforce sysfs THP requirements as necessary */
127 (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
128 !hugepage_flags_always())))
131 /* Only regular file is valid */
132 if (!in_pf && file_thp_enabled(vma))
135 if (!vma_is_anonymous(vma))
138 if (vma_is_temporary_stack(vma))
142 * THPeligible bit of smaps should show 1 for proper VMAs even
143 * though anon_vma is not initialized yet.
145 * Allow page fault since anon_vma may be not initialized until
146 * the first page fault.
149 return (smaps || in_pf);
154 static bool get_huge_zero_page(void)
156 struct page *zero_page;
158 if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
161 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
164 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
168 if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
170 __free_pages(zero_page, compound_order(zero_page));
173 WRITE_ONCE(huge_zero_pfn, page_to_pfn(zero_page));
175 /* We take additional reference here. It will be put back by shrinker */
176 atomic_set(&huge_zero_refcount, 2);
178 count_vm_event(THP_ZERO_PAGE_ALLOC);
182 static void put_huge_zero_page(void)
185 * Counter should never go to zero here. Only shrinker can put
188 BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
191 struct page *mm_get_huge_zero_page(struct mm_struct *mm)
193 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
194 return READ_ONCE(huge_zero_page);
196 if (!get_huge_zero_page())
199 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
200 put_huge_zero_page();
202 return READ_ONCE(huge_zero_page);
205 void mm_put_huge_zero_page(struct mm_struct *mm)
207 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
208 put_huge_zero_page();
211 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
212 struct shrink_control *sc)
214 /* we can free zero page only if last reference remains */
215 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
218 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
219 struct shrink_control *sc)
221 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
222 struct page *zero_page = xchg(&huge_zero_page, NULL);
223 BUG_ON(zero_page == NULL);
224 WRITE_ONCE(huge_zero_pfn, ~0UL);
225 __free_pages(zero_page, compound_order(zero_page));
232 static struct shrinker huge_zero_page_shrinker = {
233 .count_objects = shrink_huge_zero_page_count,
234 .scan_objects = shrink_huge_zero_page_scan,
235 .seeks = DEFAULT_SEEKS,
239 static ssize_t enabled_show(struct kobject *kobj,
240 struct kobj_attribute *attr, char *buf)
244 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
245 output = "[always] madvise never";
246 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
247 &transparent_hugepage_flags))
248 output = "always [madvise] never";
250 output = "always madvise [never]";
252 return sysfs_emit(buf, "%s\n", output);
255 static ssize_t enabled_store(struct kobject *kobj,
256 struct kobj_attribute *attr,
257 const char *buf, size_t count)
261 if (sysfs_streq(buf, "always")) {
262 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
263 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
264 } else if (sysfs_streq(buf, "madvise")) {
265 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
266 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
267 } else if (sysfs_streq(buf, "never")) {
268 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
269 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
274 int err = start_stop_khugepaged();
281 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled);
283 ssize_t single_hugepage_flag_show(struct kobject *kobj,
284 struct kobj_attribute *attr, char *buf,
285 enum transparent_hugepage_flag flag)
287 return sysfs_emit(buf, "%d\n",
288 !!test_bit(flag, &transparent_hugepage_flags));
291 ssize_t single_hugepage_flag_store(struct kobject *kobj,
292 struct kobj_attribute *attr,
293 const char *buf, size_t count,
294 enum transparent_hugepage_flag flag)
299 ret = kstrtoul(buf, 10, &value);
306 set_bit(flag, &transparent_hugepage_flags);
308 clear_bit(flag, &transparent_hugepage_flags);
313 static ssize_t defrag_show(struct kobject *kobj,
314 struct kobj_attribute *attr, char *buf)
318 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
319 &transparent_hugepage_flags))
320 output = "[always] defer defer+madvise madvise never";
321 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
322 &transparent_hugepage_flags))
323 output = "always [defer] defer+madvise madvise never";
324 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
325 &transparent_hugepage_flags))
326 output = "always defer [defer+madvise] madvise never";
327 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
328 &transparent_hugepage_flags))
329 output = "always defer defer+madvise [madvise] never";
331 output = "always defer defer+madvise madvise [never]";
333 return sysfs_emit(buf, "%s\n", output);
336 static ssize_t defrag_store(struct kobject *kobj,
337 struct kobj_attribute *attr,
338 const char *buf, size_t count)
340 if (sysfs_streq(buf, "always")) {
341 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
342 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
343 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
344 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
345 } else if (sysfs_streq(buf, "defer+madvise")) {
346 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
347 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
348 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
349 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
350 } else if (sysfs_streq(buf, "defer")) {
351 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
352 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
353 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
354 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
355 } else if (sysfs_streq(buf, "madvise")) {
356 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
357 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
358 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
359 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
360 } else if (sysfs_streq(buf, "never")) {
361 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
362 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
363 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
364 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
370 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag);
372 static ssize_t use_zero_page_show(struct kobject *kobj,
373 struct kobj_attribute *attr, char *buf)
375 return single_hugepage_flag_show(kobj, attr, buf,
376 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
378 static ssize_t use_zero_page_store(struct kobject *kobj,
379 struct kobj_attribute *attr, const char *buf, size_t count)
381 return single_hugepage_flag_store(kobj, attr, buf, count,
382 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
384 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page);
386 static ssize_t hpage_pmd_size_show(struct kobject *kobj,
387 struct kobj_attribute *attr, char *buf)
389 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE);
391 static struct kobj_attribute hpage_pmd_size_attr =
392 __ATTR_RO(hpage_pmd_size);
394 static struct attribute *hugepage_attr[] = {
397 &use_zero_page_attr.attr,
398 &hpage_pmd_size_attr.attr,
400 &shmem_enabled_attr.attr,
405 static const struct attribute_group hugepage_attr_group = {
406 .attrs = hugepage_attr,
409 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
413 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
414 if (unlikely(!*hugepage_kobj)) {
415 pr_err("failed to create transparent hugepage kobject\n");
419 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
421 pr_err("failed to register transparent hugepage group\n");
425 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
427 pr_err("failed to register transparent hugepage group\n");
428 goto remove_hp_group;
434 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
436 kobject_put(*hugepage_kobj);
440 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
442 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
443 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
444 kobject_put(hugepage_kobj);
447 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
452 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
455 #endif /* CONFIG_SYSFS */
457 static int __init hugepage_init(void)
460 struct kobject *hugepage_kobj;
462 if (!has_transparent_hugepage()) {
464 * Hardware doesn't support hugepages, hence disable
467 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_NEVER_DAX;
472 * hugepages can't be allocated by the buddy allocator
474 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER);
476 * we use page->mapping and page->index in second tail page
477 * as list_head: assuming THP order >= 2
479 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
481 err = hugepage_init_sysfs(&hugepage_kobj);
485 err = khugepaged_init();
489 err = register_shrinker(&huge_zero_page_shrinker, "thp-zero");
491 goto err_hzp_shrinker;
492 err = register_shrinker(&deferred_split_shrinker, "thp-deferred_split");
494 goto err_split_shrinker;
497 * By default disable transparent hugepages on smaller systems,
498 * where the extra memory used could hurt more than TLB overhead
499 * is likely to save. The admin can still enable it through /sys.
501 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) {
502 transparent_hugepage_flags = 0;
506 err = start_stop_khugepaged();
512 unregister_shrinker(&deferred_split_shrinker);
514 unregister_shrinker(&huge_zero_page_shrinker);
516 khugepaged_destroy();
518 hugepage_exit_sysfs(hugepage_kobj);
522 subsys_initcall(hugepage_init);
524 static int __init setup_transparent_hugepage(char *str)
529 if (!strcmp(str, "always")) {
530 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
531 &transparent_hugepage_flags);
532 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
533 &transparent_hugepage_flags);
535 } else if (!strcmp(str, "madvise")) {
536 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
537 &transparent_hugepage_flags);
538 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
539 &transparent_hugepage_flags);
541 } else if (!strcmp(str, "never")) {
542 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
543 &transparent_hugepage_flags);
544 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
545 &transparent_hugepage_flags);
550 pr_warn("transparent_hugepage= cannot parse, ignored\n");
553 __setup("transparent_hugepage=", setup_transparent_hugepage);
555 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
557 if (likely(vma->vm_flags & VM_WRITE))
558 pmd = pmd_mkwrite(pmd);
564 struct deferred_split *get_deferred_split_queue(struct folio *folio)
566 struct mem_cgroup *memcg = folio_memcg(folio);
567 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));
570 return &memcg->deferred_split_queue;
572 return &pgdat->deferred_split_queue;
576 struct deferred_split *get_deferred_split_queue(struct folio *folio)
578 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));
580 return &pgdat->deferred_split_queue;
584 void prep_transhuge_page(struct page *page)
586 struct folio *folio = (struct folio *)page;
588 VM_BUG_ON_FOLIO(folio_order(folio) < 2, folio);
589 INIT_LIST_HEAD(&folio->_deferred_list);
590 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
593 static inline bool is_transparent_hugepage(struct page *page)
597 if (!PageCompound(page))
600 folio = page_folio(page);
601 return is_huge_zero_page(&folio->page) ||
602 folio->_folio_dtor == TRANSHUGE_PAGE_DTOR;
605 static unsigned long __thp_get_unmapped_area(struct file *filp,
606 unsigned long addr, unsigned long len,
607 loff_t off, unsigned long flags, unsigned long size)
609 loff_t off_end = off + len;
610 loff_t off_align = round_up(off, size);
611 unsigned long len_pad, ret;
613 if (off_end <= off_align || (off_end - off_align) < size)
616 len_pad = len + size;
617 if (len_pad < len || (off + len_pad) < off)
620 ret = current->mm->get_unmapped_area(filp, addr, len_pad,
621 off >> PAGE_SHIFT, flags);
624 * The failure might be due to length padding. The caller will retry
625 * without the padding.
627 if (IS_ERR_VALUE(ret))
631 * Do not try to align to THP boundary if allocation at the address
637 ret += (off - ret) & (size - 1);
641 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
642 unsigned long len, unsigned long pgoff, unsigned long flags)
645 loff_t off = (loff_t)pgoff << PAGE_SHIFT;
647 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);
651 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
653 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
655 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,
656 struct page *page, gfp_t gfp)
658 struct vm_area_struct *vma = vmf->vma;
660 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
663 VM_BUG_ON_PAGE(!PageCompound(page), page);
665 if (mem_cgroup_charge(page_folio(page), vma->vm_mm, gfp)) {
667 count_vm_event(THP_FAULT_FALLBACK);
668 count_vm_event(THP_FAULT_FALLBACK_CHARGE);
669 return VM_FAULT_FALLBACK;
671 cgroup_throttle_swaprate(page, gfp);
673 pgtable = pte_alloc_one(vma->vm_mm);
674 if (unlikely(!pgtable)) {
679 clear_huge_page(page, vmf->address, HPAGE_PMD_NR);
681 * The memory barrier inside __SetPageUptodate makes sure that
682 * clear_huge_page writes become visible before the set_pmd_at()
685 __SetPageUptodate(page);
687 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
688 if (unlikely(!pmd_none(*vmf->pmd))) {
693 ret = check_stable_address_space(vma->vm_mm);
697 /* Deliver the page fault to userland */
698 if (userfaultfd_missing(vma)) {
699 spin_unlock(vmf->ptl);
701 pte_free(vma->vm_mm, pgtable);
702 ret = handle_userfault(vmf, VM_UFFD_MISSING);
703 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
707 entry = mk_huge_pmd(page, vma->vm_page_prot);
708 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
709 page_add_new_anon_rmap(page, vma, haddr);
710 lru_cache_add_inactive_or_unevictable(page, vma);
711 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
712 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
713 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
714 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
715 mm_inc_nr_ptes(vma->vm_mm);
716 spin_unlock(vmf->ptl);
717 count_vm_event(THP_FAULT_ALLOC);
718 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);
723 spin_unlock(vmf->ptl);
726 pte_free(vma->vm_mm, pgtable);
733 * always: directly stall for all thp allocations
734 * defer: wake kswapd and fail if not immediately available
735 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
736 * fail if not immediately available
737 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
739 * never: never stall for any thp allocation
741 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma)
743 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE);
745 /* Always do synchronous compaction */
746 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
747 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
749 /* Kick kcompactd and fail quickly */
750 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
751 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
753 /* Synchronous compaction if madvised, otherwise kick kcompactd */
754 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
755 return GFP_TRANSHUGE_LIGHT |
756 (vma_madvised ? __GFP_DIRECT_RECLAIM :
757 __GFP_KSWAPD_RECLAIM);
759 /* Only do synchronous compaction if madvised */
760 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
761 return GFP_TRANSHUGE_LIGHT |
762 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);
764 return GFP_TRANSHUGE_LIGHT;
767 /* Caller must hold page table lock. */
768 static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
769 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
770 struct page *zero_page)
775 entry = mk_pmd(zero_page, vma->vm_page_prot);
776 entry = pmd_mkhuge(entry);
777 pgtable_trans_huge_deposit(mm, pmd, pgtable);
778 set_pmd_at(mm, haddr, pmd, entry);
782 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
784 struct vm_area_struct *vma = vmf->vma;
787 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
789 if (!transhuge_vma_suitable(vma, haddr))
790 return VM_FAULT_FALLBACK;
791 if (unlikely(anon_vma_prepare(vma)))
793 khugepaged_enter_vma(vma, vma->vm_flags);
795 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
796 !mm_forbids_zeropage(vma->vm_mm) &&
797 transparent_hugepage_use_zero_page()) {
799 struct page *zero_page;
801 pgtable = pte_alloc_one(vma->vm_mm);
802 if (unlikely(!pgtable))
804 zero_page = mm_get_huge_zero_page(vma->vm_mm);
805 if (unlikely(!zero_page)) {
806 pte_free(vma->vm_mm, pgtable);
807 count_vm_event(THP_FAULT_FALLBACK);
808 return VM_FAULT_FALLBACK;
810 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
812 if (pmd_none(*vmf->pmd)) {
813 ret = check_stable_address_space(vma->vm_mm);
815 spin_unlock(vmf->ptl);
816 pte_free(vma->vm_mm, pgtable);
817 } else if (userfaultfd_missing(vma)) {
818 spin_unlock(vmf->ptl);
819 pte_free(vma->vm_mm, pgtable);
820 ret = handle_userfault(vmf, VM_UFFD_MISSING);
821 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
823 set_huge_zero_page(pgtable, vma->vm_mm, vma,
824 haddr, vmf->pmd, zero_page);
825 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
826 spin_unlock(vmf->ptl);
829 spin_unlock(vmf->ptl);
830 pte_free(vma->vm_mm, pgtable);
834 gfp = vma_thp_gfp_mask(vma);
835 folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, vma, haddr, true);
836 if (unlikely(!folio)) {
837 count_vm_event(THP_FAULT_FALLBACK);
838 return VM_FAULT_FALLBACK;
840 return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp);
843 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
844 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,
847 struct mm_struct *mm = vma->vm_mm;
851 ptl = pmd_lock(mm, pmd);
852 if (!pmd_none(*pmd)) {
854 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {
855 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));
858 entry = pmd_mkyoung(*pmd);
859 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
860 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))
861 update_mmu_cache_pmd(vma, addr, pmd);
867 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
868 if (pfn_t_devmap(pfn))
869 entry = pmd_mkdevmap(entry);
871 entry = pmd_mkyoung(pmd_mkdirty(entry));
872 entry = maybe_pmd_mkwrite(entry, vma);
876 pgtable_trans_huge_deposit(mm, pmd, pgtable);
881 set_pmd_at(mm, addr, pmd, entry);
882 update_mmu_cache_pmd(vma, addr, pmd);
887 pte_free(mm, pgtable);
891 * vmf_insert_pfn_pmd_prot - insert a pmd size pfn
892 * @vmf: Structure describing the fault
893 * @pfn: pfn to insert
894 * @pgprot: page protection to use
895 * @write: whether it's a write fault
897 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info and
898 * also consult the vmf_insert_mixed_prot() documentation when
899 * @pgprot != @vmf->vma->vm_page_prot.
901 * Return: vm_fault_t value.
903 vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn,
904 pgprot_t pgprot, bool write)
906 unsigned long addr = vmf->address & PMD_MASK;
907 struct vm_area_struct *vma = vmf->vma;
908 pgtable_t pgtable = NULL;
911 * If we had pmd_special, we could avoid all these restrictions,
912 * but we need to be consistent with PTEs and architectures that
913 * can't support a 'special' bit.
915 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
917 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
918 (VM_PFNMAP|VM_MIXEDMAP));
919 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
921 if (addr < vma->vm_start || addr >= vma->vm_end)
922 return VM_FAULT_SIGBUS;
924 if (arch_needs_pgtable_deposit()) {
925 pgtable = pte_alloc_one(vma->vm_mm);
930 track_pfn_insert(vma, &pgprot, pfn);
932 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);
933 return VM_FAULT_NOPAGE;
935 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd_prot);
937 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
938 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
940 if (likely(vma->vm_flags & VM_WRITE))
941 pud = pud_mkwrite(pud);
945 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
946 pud_t *pud, pfn_t pfn, pgprot_t prot, bool write)
948 struct mm_struct *mm = vma->vm_mm;
952 ptl = pud_lock(mm, pud);
953 if (!pud_none(*pud)) {
955 if (pud_pfn(*pud) != pfn_t_to_pfn(pfn)) {
956 WARN_ON_ONCE(!is_huge_zero_pud(*pud));
959 entry = pud_mkyoung(*pud);
960 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);
961 if (pudp_set_access_flags(vma, addr, pud, entry, 1))
962 update_mmu_cache_pud(vma, addr, pud);
967 entry = pud_mkhuge(pfn_t_pud(pfn, prot));
968 if (pfn_t_devmap(pfn))
969 entry = pud_mkdevmap(entry);
971 entry = pud_mkyoung(pud_mkdirty(entry));
972 entry = maybe_pud_mkwrite(entry, vma);
974 set_pud_at(mm, addr, pud, entry);
975 update_mmu_cache_pud(vma, addr, pud);
982 * vmf_insert_pfn_pud_prot - insert a pud size pfn
983 * @vmf: Structure describing the fault
984 * @pfn: pfn to insert
985 * @pgprot: page protection to use
986 * @write: whether it's a write fault
988 * Insert a pud size pfn. See vmf_insert_pfn() for additional info and
989 * also consult the vmf_insert_mixed_prot() documentation when
990 * @pgprot != @vmf->vma->vm_page_prot.
992 * Return: vm_fault_t value.
994 vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn,
995 pgprot_t pgprot, bool write)
997 unsigned long addr = vmf->address & PUD_MASK;
998 struct vm_area_struct *vma = vmf->vma;
1001 * If we had pud_special, we could avoid all these restrictions,
1002 * but we need to be consistent with PTEs and architectures that
1003 * can't support a 'special' bit.
1005 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
1006 !pfn_t_devmap(pfn));
1007 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1008 (VM_PFNMAP|VM_MIXEDMAP));
1009 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
1011 if (addr < vma->vm_start || addr >= vma->vm_end)
1012 return VM_FAULT_SIGBUS;
1014 track_pfn_insert(vma, &pgprot, pfn);
1016 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);
1017 return VM_FAULT_NOPAGE;
1019 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud_prot);
1020 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1022 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
1023 pmd_t *pmd, bool write)
1027 _pmd = pmd_mkyoung(*pmd);
1029 _pmd = pmd_mkdirty(_pmd);
1030 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
1032 update_mmu_cache_pmd(vma, addr, pmd);
1035 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
1036 pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
1038 unsigned long pfn = pmd_pfn(*pmd);
1039 struct mm_struct *mm = vma->vm_mm;
1043 assert_spin_locked(pmd_lockptr(mm, pmd));
1045 if (flags & FOLL_WRITE && !pmd_write(*pmd))
1048 if (pmd_present(*pmd) && pmd_devmap(*pmd))
1053 if (flags & FOLL_TOUCH)
1054 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);
1057 * device mapped pages can only be returned if the
1058 * caller will manage the page reference count.
1060 if (!(flags & (FOLL_GET | FOLL_PIN)))
1061 return ERR_PTR(-EEXIST);
1063 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
1064 *pgmap = get_dev_pagemap(pfn, *pgmap);
1066 return ERR_PTR(-EFAULT);
1067 page = pfn_to_page(pfn);
1068 ret = try_grab_page(page, flags);
1070 page = ERR_PTR(ret);
1075 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1076 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1077 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1079 spinlock_t *dst_ptl, *src_ptl;
1080 struct page *src_page;
1082 pgtable_t pgtable = NULL;
1085 /* Skip if can be re-fill on fault */
1086 if (!vma_is_anonymous(dst_vma))
1089 pgtable = pte_alloc_one(dst_mm);
1090 if (unlikely(!pgtable))
1093 dst_ptl = pmd_lock(dst_mm, dst_pmd);
1094 src_ptl = pmd_lockptr(src_mm, src_pmd);
1095 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1100 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1101 if (unlikely(is_swap_pmd(pmd))) {
1102 swp_entry_t entry = pmd_to_swp_entry(pmd);
1104 VM_BUG_ON(!is_pmd_migration_entry(pmd));
1105 if (!is_readable_migration_entry(entry)) {
1106 entry = make_readable_migration_entry(
1108 pmd = swp_entry_to_pmd(entry);
1109 if (pmd_swp_soft_dirty(*src_pmd))
1110 pmd = pmd_swp_mksoft_dirty(pmd);
1111 if (pmd_swp_uffd_wp(*src_pmd))
1112 pmd = pmd_swp_mkuffd_wp(pmd);
1113 set_pmd_at(src_mm, addr, src_pmd, pmd);
1115 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1116 mm_inc_nr_ptes(dst_mm);
1117 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
1118 if (!userfaultfd_wp(dst_vma))
1119 pmd = pmd_swp_clear_uffd_wp(pmd);
1120 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
1126 if (unlikely(!pmd_trans_huge(pmd))) {
1127 pte_free(dst_mm, pgtable);
1131 * When page table lock is held, the huge zero pmd should not be
1132 * under splitting since we don't split the page itself, only pmd to
1135 if (is_huge_zero_pmd(pmd)) {
1137 * get_huge_zero_page() will never allocate a new page here,
1138 * since we already have a zero page to copy. It just takes a
1141 mm_get_huge_zero_page(dst_mm);
1145 src_page = pmd_page(pmd);
1146 VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
1149 if (unlikely(page_try_dup_anon_rmap(src_page, true, src_vma))) {
1150 /* Page maybe pinned: split and retry the fault on PTEs. */
1152 pte_free(dst_mm, pgtable);
1153 spin_unlock(src_ptl);
1154 spin_unlock(dst_ptl);
1155 __split_huge_pmd(src_vma, src_pmd, addr, false, NULL);
1158 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1160 mm_inc_nr_ptes(dst_mm);
1161 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
1162 pmdp_set_wrprotect(src_mm, addr, src_pmd);
1163 if (!userfaultfd_wp(dst_vma))
1164 pmd = pmd_clear_uffd_wp(pmd);
1165 pmd = pmd_mkold(pmd_wrprotect(pmd));
1166 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
1170 spin_unlock(src_ptl);
1171 spin_unlock(dst_ptl);
1176 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1177 static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
1178 pud_t *pud, bool write)
1182 _pud = pud_mkyoung(*pud);
1184 _pud = pud_mkdirty(_pud);
1185 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
1187 update_mmu_cache_pud(vma, addr, pud);
1190 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
1191 pud_t *pud, int flags, struct dev_pagemap **pgmap)
1193 unsigned long pfn = pud_pfn(*pud);
1194 struct mm_struct *mm = vma->vm_mm;
1198 assert_spin_locked(pud_lockptr(mm, pud));
1200 if (flags & FOLL_WRITE && !pud_write(*pud))
1203 if (pud_present(*pud) && pud_devmap(*pud))
1208 if (flags & FOLL_TOUCH)
1209 touch_pud(vma, addr, pud, flags & FOLL_WRITE);
1212 * device mapped pages can only be returned if the
1213 * caller will manage the page reference count.
1215 * At least one of FOLL_GET | FOLL_PIN must be set, so assert that here:
1217 if (!(flags & (FOLL_GET | FOLL_PIN)))
1218 return ERR_PTR(-EEXIST);
1220 pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
1221 *pgmap = get_dev_pagemap(pfn, *pgmap);
1223 return ERR_PTR(-EFAULT);
1224 page = pfn_to_page(pfn);
1226 ret = try_grab_page(page, flags);
1228 page = ERR_PTR(ret);
1233 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1234 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1235 struct vm_area_struct *vma)
1237 spinlock_t *dst_ptl, *src_ptl;
1241 dst_ptl = pud_lock(dst_mm, dst_pud);
1242 src_ptl = pud_lockptr(src_mm, src_pud);
1243 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1247 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud)))
1251 * When page table lock is held, the huge zero pud should not be
1252 * under splitting since we don't split the page itself, only pud to
1255 if (is_huge_zero_pud(pud)) {
1256 /* No huge zero pud yet */
1260 * TODO: once we support anonymous pages, use page_try_dup_anon_rmap()
1261 * and split if duplicating fails.
1263 pudp_set_wrprotect(src_mm, addr, src_pud);
1264 pud = pud_mkold(pud_wrprotect(pud));
1265 set_pud_at(dst_mm, addr, dst_pud, pud);
1269 spin_unlock(src_ptl);
1270 spin_unlock(dst_ptl);
1274 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
1276 bool write = vmf->flags & FAULT_FLAG_WRITE;
1278 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
1279 if (unlikely(!pud_same(*vmf->pud, orig_pud)))
1282 touch_pud(vmf->vma, vmf->address, vmf->pud, write);
1284 spin_unlock(vmf->ptl);
1286 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1288 void huge_pmd_set_accessed(struct vm_fault *vmf)
1290 bool write = vmf->flags & FAULT_FLAG_WRITE;
1292 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1293 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd)))
1296 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write);
1299 spin_unlock(vmf->ptl);
1302 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
1304 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
1305 struct vm_area_struct *vma = vmf->vma;
1306 struct folio *folio;
1308 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1309 pmd_t orig_pmd = vmf->orig_pmd;
1311 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
1312 VM_BUG_ON_VMA(!vma->anon_vma, vma);
1314 if (is_huge_zero_pmd(orig_pmd))
1317 spin_lock(vmf->ptl);
1319 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
1320 spin_unlock(vmf->ptl);
1324 page = pmd_page(orig_pmd);
1325 folio = page_folio(page);
1326 VM_BUG_ON_PAGE(!PageHead(page), page);
1328 /* Early check when only holding the PT lock. */
1329 if (PageAnonExclusive(page))
1332 if (!folio_trylock(folio)) {
1334 spin_unlock(vmf->ptl);
1336 spin_lock(vmf->ptl);
1337 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
1338 spin_unlock(vmf->ptl);
1339 folio_unlock(folio);
1346 /* Recheck after temporarily dropping the PT lock. */
1347 if (PageAnonExclusive(page)) {
1348 folio_unlock(folio);
1353 * See do_wp_page(): we can only reuse the folio exclusively if
1354 * there are no additional references. Note that we always drain
1355 * the LRU pagevecs immediately after adding a THP.
1357 if (folio_ref_count(folio) >
1358 1 + folio_test_swapcache(folio) * folio_nr_pages(folio))
1359 goto unlock_fallback;
1360 if (folio_test_swapcache(folio))
1361 folio_free_swap(folio);
1362 if (folio_ref_count(folio) == 1) {
1365 page_move_anon_rmap(page, vma);
1366 folio_unlock(folio);
1368 if (unlikely(unshare)) {
1369 spin_unlock(vmf->ptl);
1372 entry = pmd_mkyoung(orig_pmd);
1373 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1374 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))
1375 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1376 spin_unlock(vmf->ptl);
1381 folio_unlock(folio);
1382 spin_unlock(vmf->ptl);
1384 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
1385 return VM_FAULT_FALLBACK;
1388 static inline bool can_change_pmd_writable(struct vm_area_struct *vma,
1389 unsigned long addr, pmd_t pmd)
1393 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
1396 /* Don't touch entries that are not even readable (NUMA hinting). */
1397 if (pmd_protnone(pmd))
1400 /* Do we need write faults for softdirty tracking? */
1401 if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd))
1404 /* Do we need write faults for uffd-wp tracking? */
1405 if (userfaultfd_huge_pmd_wp(vma, pmd))
1408 if (!(vma->vm_flags & VM_SHARED)) {
1409 /* See can_change_pte_writable(). */
1410 page = vm_normal_page_pmd(vma, addr, pmd);
1411 return page && PageAnon(page) && PageAnonExclusive(page);
1414 /* See can_change_pte_writable(). */
1415 return pmd_dirty(pmd);
1418 /* FOLL_FORCE can write to even unwritable PMDs in COW mappings. */
1419 static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
1420 struct vm_area_struct *vma,
1423 /* If the pmd is writable, we can write to the page. */
1427 /* Maybe FOLL_FORCE is set to override it? */
1428 if (!(flags & FOLL_FORCE))
1431 /* But FOLL_FORCE has no effect on shared mappings */
1432 if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED))
1435 /* ... or read-only private ones */
1436 if (!(vma->vm_flags & VM_MAYWRITE))
1439 /* ... or already writable ones that just need to take a write fault */
1440 if (vma->vm_flags & VM_WRITE)
1444 * See can_change_pte_writable(): we broke COW and could map the page
1445 * writable if we have an exclusive anonymous page ...
1447 if (!page || !PageAnon(page) || !PageAnonExclusive(page))
1450 /* ... and a write-fault isn't required for other reasons. */
1451 if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd))
1453 return !userfaultfd_huge_pmd_wp(vma, pmd);
1456 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1461 struct mm_struct *mm = vma->vm_mm;
1465 assert_spin_locked(pmd_lockptr(mm, pmd));
1467 page = pmd_page(*pmd);
1468 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1470 if ((flags & FOLL_WRITE) &&
1471 !can_follow_write_pmd(*pmd, page, vma, flags))
1474 /* Avoid dumping huge zero page */
1475 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1476 return ERR_PTR(-EFAULT);
1478 /* Full NUMA hinting faults to serialise migration in fault paths */
1479 if (pmd_protnone(*pmd) && !gup_can_follow_protnone(flags))
1482 if (!pmd_write(*pmd) && gup_must_unshare(vma, flags, page))
1483 return ERR_PTR(-EMLINK);
1485 VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
1486 !PageAnonExclusive(page), page);
1488 ret = try_grab_page(page, flags);
1490 return ERR_PTR(ret);
1492 if (flags & FOLL_TOUCH)
1493 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE);
1495 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1496 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1501 /* NUMA hinting page fault entry point for trans huge pmds */
1502 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
1504 struct vm_area_struct *vma = vmf->vma;
1505 pmd_t oldpmd = vmf->orig_pmd;
1508 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1509 int page_nid = NUMA_NO_NODE;
1510 int target_nid, last_cpupid = (-1 & LAST_CPUPID_MASK);
1511 bool migrated = false, writable = false;
1514 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1515 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {
1516 spin_unlock(vmf->ptl);
1520 pmd = pmd_modify(oldpmd, vma->vm_page_prot);
1523 * Detect now whether the PMD could be writable; this information
1524 * is only valid while holding the PT lock.
1526 writable = pmd_write(pmd);
1527 if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
1528 can_change_pmd_writable(vma, vmf->address, pmd))
1531 page = vm_normal_page_pmd(vma, haddr, pmd);
1535 /* See similar comment in do_numa_page for explanation */
1537 flags |= TNF_NO_GROUP;
1539 page_nid = page_to_nid(page);
1541 * For memory tiering mode, cpupid of slow memory page is used
1542 * to record page access time. So use default value.
1544 if (node_is_toptier(page_nid))
1545 last_cpupid = page_cpupid_last(page);
1546 target_nid = numa_migrate_prep(page, vma, haddr, page_nid,
1549 if (target_nid == NUMA_NO_NODE) {
1554 spin_unlock(vmf->ptl);
1557 migrated = migrate_misplaced_page(page, vma, target_nid);
1559 flags |= TNF_MIGRATED;
1560 page_nid = target_nid;
1562 flags |= TNF_MIGRATE_FAIL;
1563 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1564 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) {
1565 spin_unlock(vmf->ptl);
1572 if (page_nid != NUMA_NO_NODE)
1573 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR,
1579 /* Restore the PMD */
1580 pmd = pmd_modify(oldpmd, vma->vm_page_prot);
1581 pmd = pmd_mkyoung(pmd);
1583 pmd = pmd_mkwrite(pmd);
1584 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
1585 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1586 spin_unlock(vmf->ptl);
1591 * Return true if we do MADV_FREE successfully on entire pmd page.
1592 * Otherwise, return false.
1594 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1595 pmd_t *pmd, unsigned long addr, unsigned long next)
1599 struct folio *folio;
1600 struct mm_struct *mm = tlb->mm;
1603 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1605 ptl = pmd_trans_huge_lock(pmd, vma);
1610 if (is_huge_zero_pmd(orig_pmd))
1613 if (unlikely(!pmd_present(orig_pmd))) {
1614 VM_BUG_ON(thp_migration_supported() &&
1615 !is_pmd_migration_entry(orig_pmd));
1619 folio = pfn_folio(pmd_pfn(orig_pmd));
1621 * If other processes are mapping this folio, we couldn't discard
1622 * the folio unless they all do MADV_FREE so let's skip the folio.
1624 if (folio_mapcount(folio) != 1)
1627 if (!folio_trylock(folio))
1631 * If user want to discard part-pages of THP, split it so MADV_FREE
1632 * will deactivate only them.
1634 if (next - addr != HPAGE_PMD_SIZE) {
1638 folio_unlock(folio);
1643 if (folio_test_dirty(folio))
1644 folio_clear_dirty(folio);
1645 folio_unlock(folio);
1647 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1648 pmdp_invalidate(vma, addr, pmd);
1649 orig_pmd = pmd_mkold(orig_pmd);
1650 orig_pmd = pmd_mkclean(orig_pmd);
1652 set_pmd_at(mm, addr, pmd, orig_pmd);
1653 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1656 folio_mark_lazyfree(folio);
1664 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
1668 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1669 pte_free(mm, pgtable);
1673 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1674 pmd_t *pmd, unsigned long addr)
1679 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1681 ptl = __pmd_trans_huge_lock(pmd, vma);
1685 * For architectures like ppc64 we look at deposited pgtable
1686 * when calling pmdp_huge_get_and_clear. So do the
1687 * pgtable_trans_huge_withdraw after finishing pmdp related
1690 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,
1692 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1693 if (vma_is_special_huge(vma)) {
1694 if (arch_needs_pgtable_deposit())
1695 zap_deposited_table(tlb->mm, pmd);
1697 } else if (is_huge_zero_pmd(orig_pmd)) {
1698 zap_deposited_table(tlb->mm, pmd);
1701 struct page *page = NULL;
1702 int flush_needed = 1;
1704 if (pmd_present(orig_pmd)) {
1705 page = pmd_page(orig_pmd);
1706 page_remove_rmap(page, vma, true);
1707 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1708 VM_BUG_ON_PAGE(!PageHead(page), page);
1709 } else if (thp_migration_supported()) {
1712 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
1713 entry = pmd_to_swp_entry(orig_pmd);
1714 page = pfn_swap_entry_to_page(entry);
1717 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
1719 if (PageAnon(page)) {
1720 zap_deposited_table(tlb->mm, pmd);
1721 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1723 if (arch_needs_pgtable_deposit())
1724 zap_deposited_table(tlb->mm, pmd);
1725 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR);
1730 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
1735 #ifndef pmd_move_must_withdraw
1736 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
1737 spinlock_t *old_pmd_ptl,
1738 struct vm_area_struct *vma)
1741 * With split pmd lock we also need to move preallocated
1742 * PTE page table if new_pmd is on different PMD page table.
1744 * We also don't deposit and withdraw tables for file pages.
1746 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
1750 static pmd_t move_soft_dirty_pmd(pmd_t pmd)
1752 #ifdef CONFIG_MEM_SOFT_DIRTY
1753 if (unlikely(is_pmd_migration_entry(pmd)))
1754 pmd = pmd_swp_mksoft_dirty(pmd);
1755 else if (pmd_present(pmd))
1756 pmd = pmd_mksoft_dirty(pmd);
1761 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
1762 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
1764 spinlock_t *old_ptl, *new_ptl;
1766 struct mm_struct *mm = vma->vm_mm;
1767 bool force_flush = false;
1770 * The destination pmd shouldn't be established, free_pgtables()
1771 * should have release it.
1773 if (WARN_ON(!pmd_none(*new_pmd))) {
1774 VM_BUG_ON(pmd_trans_huge(*new_pmd));
1779 * We don't have to worry about the ordering of src and dst
1780 * ptlocks because exclusive mmap_lock prevents deadlock.
1782 old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
1784 new_ptl = pmd_lockptr(mm, new_pmd);
1785 if (new_ptl != old_ptl)
1786 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1787 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
1788 if (pmd_present(pmd))
1790 VM_BUG_ON(!pmd_none(*new_pmd));
1792 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
1794 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1795 pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
1797 pmd = move_soft_dirty_pmd(pmd);
1798 set_pmd_at(mm, new_addr, new_pmd, pmd);
1800 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
1801 if (new_ptl != old_ptl)
1802 spin_unlock(new_ptl);
1803 spin_unlock(old_ptl);
1811 * - 0 if PMD could not be locked
1812 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary
1813 * or if prot_numa but THP migration is not supported
1814 * - HPAGE_PMD_NR if protections changed and TLB flush necessary
1816 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1817 pmd_t *pmd, unsigned long addr, pgprot_t newprot,
1818 unsigned long cp_flags)
1820 struct mm_struct *mm = vma->vm_mm;
1822 pmd_t oldpmd, entry;
1823 bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
1824 bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
1825 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
1828 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1830 if (prot_numa && !thp_migration_supported())
1833 ptl = __pmd_trans_huge_lock(pmd, vma);
1837 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1838 if (is_swap_pmd(*pmd)) {
1839 swp_entry_t entry = pmd_to_swp_entry(*pmd);
1840 struct page *page = pfn_swap_entry_to_page(entry);
1843 VM_BUG_ON(!is_pmd_migration_entry(*pmd));
1844 if (is_writable_migration_entry(entry)) {
1846 * A protection check is difficult so
1847 * just be safe and disable write
1850 entry = make_readable_exclusive_migration_entry(swp_offset(entry));
1852 entry = make_readable_migration_entry(swp_offset(entry));
1853 newpmd = swp_entry_to_pmd(entry);
1854 if (pmd_swp_soft_dirty(*pmd))
1855 newpmd = pmd_swp_mksoft_dirty(newpmd);
1856 if (pmd_swp_uffd_wp(*pmd))
1857 newpmd = pmd_swp_mkuffd_wp(newpmd);
1863 newpmd = pmd_swp_mkuffd_wp(newpmd);
1864 else if (uffd_wp_resolve)
1865 newpmd = pmd_swp_clear_uffd_wp(newpmd);
1866 if (!pmd_same(*pmd, newpmd))
1867 set_pmd_at(mm, addr, pmd, newpmd);
1876 * Avoid trapping faults against the zero page. The read-only
1877 * data is likely to be read-cached on the local CPU and
1878 * local/remote hits to the zero page are not interesting.
1880 if (is_huge_zero_pmd(*pmd))
1883 if (pmd_protnone(*pmd))
1886 page = pmd_page(*pmd);
1887 toptier = node_is_toptier(page_to_nid(page));
1889 * Skip scanning top tier node if normal numa
1890 * balancing is disabled
1892 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
1896 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
1898 xchg_page_access_time(page, jiffies_to_msecs(jiffies));
1901 * In case prot_numa, we are under mmap_read_lock(mm). It's critical
1902 * to not clear pmd intermittently to avoid race with MADV_DONTNEED
1903 * which is also under mmap_read_lock(mm):
1906 * change_huge_pmd(prot_numa=1)
1907 * pmdp_huge_get_and_clear_notify()
1908 * madvise_dontneed()
1910 * pmd_trans_huge(*pmd) == 0 (without ptl)
1913 * // pmd is re-established
1915 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it
1916 * which may break userspace.
1918 * pmdp_invalidate_ad() is required to make sure we don't miss
1919 * dirty/young flags set by hardware.
1921 oldpmd = pmdp_invalidate_ad(vma, addr, pmd);
1923 entry = pmd_modify(oldpmd, newprot);
1925 entry = pmd_mkuffd_wp(entry);
1926 else if (uffd_wp_resolve)
1928 * Leave the write bit to be handled by PF interrupt
1929 * handler, then things like COW could be properly
1932 entry = pmd_clear_uffd_wp(entry);
1934 /* See change_pte_range(). */
1935 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
1936 can_change_pmd_writable(vma, addr, entry))
1937 entry = pmd_mkwrite(entry);
1940 set_pmd_at(mm, addr, pmd, entry);
1942 if (huge_pmd_needs_flush(oldpmd, entry))
1943 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE);
1950 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1952 * Note that if it returns page table lock pointer, this routine returns without
1953 * unlocking page table lock. So callers must unlock it.
1955 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
1958 ptl = pmd_lock(vma->vm_mm, pmd);
1959 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||
1967 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise.
1969 * Note that if it returns page table lock pointer, this routine returns without
1970 * unlocking page table lock. So callers must unlock it.
1972 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
1976 ptl = pud_lock(vma->vm_mm, pud);
1977 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud)))
1983 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1984 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
1985 pud_t *pud, unsigned long addr)
1989 ptl = __pud_trans_huge_lock(pud, vma);
1993 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm);
1994 tlb_remove_pud_tlb_entry(tlb, pud, addr);
1995 if (vma_is_special_huge(vma)) {
1997 /* No zero page support yet */
1999 /* No support for anonymous PUD pages yet */
2005 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
2006 unsigned long haddr)
2008 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
2009 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2010 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
2011 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));
2013 count_vm_event(THP_SPLIT_PUD);
2015 pudp_huge_clear_flush_notify(vma, haddr, pud);
2018 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
2019 unsigned long address)
2022 struct mmu_notifier_range range;
2024 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
2025 address & HPAGE_PUD_MASK,
2026 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
2027 mmu_notifier_invalidate_range_start(&range);
2028 ptl = pud_lock(vma->vm_mm, pud);
2029 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
2031 __split_huge_pud_locked(vma, pud, range.start);
2036 * No need to double call mmu_notifier->invalidate_range() callback as
2037 * the above pudp_huge_clear_flush_notify() did already call it.
2039 mmu_notifier_invalidate_range_only_end(&range);
2041 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
2043 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2044 unsigned long haddr, pmd_t *pmd)
2046 struct mm_struct *mm = vma->vm_mm;
2048 pmd_t _pmd, old_pmd;
2052 * Leave pmd empty until pte is filled note that it is fine to delay
2053 * notification until mmu_notifier_invalidate_range_end() as we are
2054 * replacing a zero pmd write protected page with a zero pte write
2057 * See Documentation/mm/mmu_notifier.rst
2059 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);
2061 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2062 pmd_populate(mm, &_pmd, pgtable);
2064 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2066 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2067 entry = pte_mkspecial(entry);
2068 if (pmd_uffd_wp(old_pmd))
2069 entry = pte_mkuffd_wp(entry);
2070 pte = pte_offset_map(&_pmd, haddr);
2071 VM_BUG_ON(!pte_none(*pte));
2072 set_pte_at(mm, haddr, pte, entry);
2075 smp_wmb(); /* make pte visible before pmd */
2076 pmd_populate(mm, pmd, pgtable);
2079 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
2080 unsigned long haddr, bool freeze)
2082 struct mm_struct *mm = vma->vm_mm;
2085 pmd_t old_pmd, _pmd;
2086 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false;
2087 bool anon_exclusive = false, dirty = false;
2091 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
2092 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2093 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
2094 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)
2095 && !pmd_devmap(*pmd));
2097 count_vm_event(THP_SPLIT_PMD);
2099 if (!vma_is_anonymous(vma)) {
2100 old_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2102 * We are going to unmap this huge page. So
2103 * just go ahead and zap it
2105 if (arch_needs_pgtable_deposit())
2106 zap_deposited_table(mm, pmd);
2107 if (vma_is_special_huge(vma))
2109 if (unlikely(is_pmd_migration_entry(old_pmd))) {
2112 entry = pmd_to_swp_entry(old_pmd);
2113 page = pfn_swap_entry_to_page(entry);
2115 page = pmd_page(old_pmd);
2116 if (!PageDirty(page) && pmd_dirty(old_pmd))
2117 set_page_dirty(page);
2118 if (!PageReferenced(page) && pmd_young(old_pmd))
2119 SetPageReferenced(page);
2120 page_remove_rmap(page, vma, true);
2123 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
2127 if (is_huge_zero_pmd(*pmd)) {
2129 * FIXME: Do we want to invalidate secondary mmu by calling
2130 * mmu_notifier_invalidate_range() see comments below inside
2131 * __split_huge_pmd() ?
2133 * We are going from a zero huge page write protected to zero
2134 * small page also write protected so it does not seems useful
2135 * to invalidate secondary mmu at this time.
2137 return __split_huge_zero_page_pmd(vma, haddr, pmd);
2141 * Up to this point the pmd is present and huge and userland has the
2142 * whole access to the hugepage during the split (which happens in
2143 * place). If we overwrite the pmd with the not-huge version pointing
2144 * to the pte here (which of course we could if all CPUs were bug
2145 * free), userland could trigger a small page size TLB miss on the
2146 * small sized TLB while the hugepage TLB entry is still established in
2147 * the huge TLB. Some CPU doesn't like that.
2148 * See http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum
2149 * 383 on page 105. Intel should be safe but is also warns that it's
2150 * only safe if the permission and cache attributes of the two entries
2151 * loaded in the two TLB is identical (which should be the case here).
2152 * But it is generally safer to never allow small and huge TLB entries
2153 * for the same virtual address to be loaded simultaneously. So instead
2154 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
2155 * current pmd notpresent (atomically because here the pmd_trans_huge
2156 * must remain set at all times on the pmd until the split is complete
2157 * for this pmd), then we flush the SMP TLB and finally we write the
2158 * non-huge version of the pmd entry with pmd_populate.
2160 old_pmd = pmdp_invalidate(vma, haddr, pmd);
2162 pmd_migration = is_pmd_migration_entry(old_pmd);
2163 if (unlikely(pmd_migration)) {
2166 entry = pmd_to_swp_entry(old_pmd);
2167 page = pfn_swap_entry_to_page(entry);
2168 write = is_writable_migration_entry(entry);
2170 anon_exclusive = is_readable_exclusive_migration_entry(entry);
2171 young = is_migration_entry_young(entry);
2172 dirty = is_migration_entry_dirty(entry);
2173 soft_dirty = pmd_swp_soft_dirty(old_pmd);
2174 uffd_wp = pmd_swp_uffd_wp(old_pmd);
2176 page = pmd_page(old_pmd);
2177 if (pmd_dirty(old_pmd)) {
2181 write = pmd_write(old_pmd);
2182 young = pmd_young(old_pmd);
2183 soft_dirty = pmd_soft_dirty(old_pmd);
2184 uffd_wp = pmd_uffd_wp(old_pmd);
2186 VM_BUG_ON_PAGE(!page_count(page), page);
2189 * Without "freeze", we'll simply split the PMD, propagating the
2190 * PageAnonExclusive() flag for each PTE by setting it for
2191 * each subpage -- no need to (temporarily) clear.
2193 * With "freeze" we want to replace mapped pages by
2194 * migration entries right away. This is only possible if we
2195 * managed to clear PageAnonExclusive() -- see
2196 * set_pmd_migration_entry().
2198 * In case we cannot clear PageAnonExclusive(), split the PMD
2199 * only and let try_to_migrate_one() fail later.
2201 * See page_try_share_anon_rmap(): invalidate PMD first.
2203 anon_exclusive = PageAnon(page) && PageAnonExclusive(page);
2204 if (freeze && anon_exclusive && page_try_share_anon_rmap(page))
2207 page_ref_add(page, HPAGE_PMD_NR - 1);
2211 * Withdraw the table only after we mark the pmd entry invalid.
2212 * This's critical for some architectures (Power).
2214 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2215 pmd_populate(mm, &_pmd, pgtable);
2217 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
2220 * Note that NUMA hinting access restrictions are not
2221 * transferred to avoid any possibility of altering
2222 * permissions across VMAs.
2224 if (freeze || pmd_migration) {
2225 swp_entry_t swp_entry;
2227 swp_entry = make_writable_migration_entry(
2228 page_to_pfn(page + i));
2229 else if (anon_exclusive)
2230 swp_entry = make_readable_exclusive_migration_entry(
2231 page_to_pfn(page + i));
2233 swp_entry = make_readable_migration_entry(
2234 page_to_pfn(page + i));
2236 swp_entry = make_migration_entry_young(swp_entry);
2238 swp_entry = make_migration_entry_dirty(swp_entry);
2239 entry = swp_entry_to_pte(swp_entry);
2241 entry = pte_swp_mksoft_dirty(entry);
2243 entry = pte_swp_mkuffd_wp(entry);
2245 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
2246 entry = maybe_mkwrite(entry, vma);
2248 SetPageAnonExclusive(page + i);
2250 entry = pte_mkold(entry);
2251 /* NOTE: this may set soft-dirty too on some archs */
2253 entry = pte_mkdirty(entry);
2255 * NOTE: this needs to happen after pte_mkdirty,
2256 * because some archs (sparc64, loongarch) could
2257 * set hw write bit when mkdirty.
2260 entry = pte_wrprotect(entry);
2262 entry = pte_mksoft_dirty(entry);
2264 entry = pte_mkuffd_wp(entry);
2265 page_add_anon_rmap(page + i, vma, addr, false);
2267 pte = pte_offset_map(&_pmd, addr);
2268 BUG_ON(!pte_none(*pte));
2269 set_pte_at(mm, addr, pte, entry);
2274 page_remove_rmap(page, vma, true);
2278 smp_wmb(); /* make pte visible before pmd */
2279 pmd_populate(mm, pmd, pgtable);
2282 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2283 unsigned long address, bool freeze, struct folio *folio)
2286 struct mmu_notifier_range range;
2288 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
2289 address & HPAGE_PMD_MASK,
2290 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
2291 mmu_notifier_invalidate_range_start(&range);
2292 ptl = pmd_lock(vma->vm_mm, pmd);
2295 * If caller asks to setup a migration entry, we need a folio to check
2296 * pmd against. Otherwise we can end up replacing wrong folio.
2298 VM_BUG_ON(freeze && !folio);
2299 VM_WARN_ON_ONCE(folio && !folio_test_locked(folio));
2301 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd) ||
2302 is_pmd_migration_entry(*pmd)) {
2304 * It's safe to call pmd_page when folio is set because it's
2305 * guaranteed that pmd is present.
2307 if (folio && folio != page_folio(pmd_page(*pmd)))
2309 __split_huge_pmd_locked(vma, pmd, range.start, freeze);
2315 * No need to double call mmu_notifier->invalidate_range() callback.
2316 * They are 3 cases to consider inside __split_huge_pmd_locked():
2317 * 1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious
2318 * 2) __split_huge_zero_page_pmd() read only zero page and any write
2319 * fault will trigger a flush_notify before pointing to a new page
2320 * (it is fine if the secondary mmu keeps pointing to the old zero
2321 * page in the meantime)
2322 * 3) Split a huge pmd into pte pointing to the same page. No need
2323 * to invalidate secondary tlb entry they are all still valid.
2324 * any further changes to individual pte will notify. So no need
2325 * to call mmu_notifier->invalidate_range()
2327 mmu_notifier_invalidate_range_only_end(&range);
2330 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
2331 bool freeze, struct folio *folio)
2333 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address);
2338 __split_huge_pmd(vma, pmd, address, freeze, folio);
2341 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
2344 * If the new address isn't hpage aligned and it could previously
2345 * contain an hugepage: check if we need to split an huge pmd.
2347 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) &&
2348 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),
2349 ALIGN(address, HPAGE_PMD_SIZE)))
2350 split_huge_pmd_address(vma, address, false, NULL);
2353 void vma_adjust_trans_huge(struct vm_area_struct *vma,
2354 unsigned long start,
2358 /* Check if we need to split start first. */
2359 split_huge_pmd_if_needed(vma, start);
2361 /* Check if we need to split end next. */
2362 split_huge_pmd_if_needed(vma, end);
2365 * If we're also updating the next vma vm_start,
2366 * check if we need to split it.
2368 if (adjust_next > 0) {
2369 struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end);
2370 unsigned long nstart = next->vm_start;
2371 nstart += adjust_next;
2372 split_huge_pmd_if_needed(next, nstart);
2376 static void unmap_folio(struct folio *folio)
2378 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
2381 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
2384 * Anon pages need migration entries to preserve them, but file
2385 * pages can simply be left unmapped, then faulted back on demand.
2386 * If that is ever changed (perhaps for mlock), update remap_page().
2388 if (folio_test_anon(folio))
2389 try_to_migrate(folio, ttu_flags);
2391 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
2394 static void remap_page(struct folio *folio, unsigned long nr)
2398 /* If unmap_folio() uses try_to_migrate() on file, remove this check */
2399 if (!folio_test_anon(folio))
2402 remove_migration_ptes(folio, folio, true);
2403 i += folio_nr_pages(folio);
2406 folio = folio_next(folio);
2410 static void lru_add_page_tail(struct page *head, struct page *tail,
2411 struct lruvec *lruvec, struct list_head *list)
2413 VM_BUG_ON_PAGE(!PageHead(head), head);
2414 VM_BUG_ON_PAGE(PageCompound(tail), head);
2415 VM_BUG_ON_PAGE(PageLRU(tail), head);
2416 lockdep_assert_held(&lruvec->lru_lock);
2419 /* page reclaim is reclaiming a huge page */
2420 VM_WARN_ON(PageLRU(head));
2422 list_add_tail(&tail->lru, list);
2424 /* head is still on lru (and we have it frozen) */
2425 VM_WARN_ON(!PageLRU(head));
2426 if (PageUnevictable(tail))
2427 tail->mlock_count = 0;
2429 list_add_tail(&tail->lru, &head->lru);
2434 static void __split_huge_page_tail(struct page *head, int tail,
2435 struct lruvec *lruvec, struct list_head *list)
2437 struct page *page_tail = head + tail;
2439 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
2442 * Clone page flags before unfreezing refcount.
2444 * After successful get_page_unless_zero() might follow flags change,
2445 * for example lock_page() which set PG_waiters.
2447 * Note that for mapped sub-pages of an anonymous THP,
2448 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in
2449 * the migration entry instead from where remap_page() will restore it.
2450 * We can still have PG_anon_exclusive set on effectively unmapped and
2451 * unreferenced sub-pages of an anonymous THP: we can simply drop
2452 * PG_anon_exclusive (-> PG_mappedtodisk) for these here.
2454 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
2455 page_tail->flags |= (head->flags &
2456 ((1L << PG_referenced) |
2457 (1L << PG_swapbacked) |
2458 (1L << PG_swapcache) |
2459 (1L << PG_mlocked) |
2460 (1L << PG_uptodate) |
2462 (1L << PG_workingset) |
2464 (1L << PG_unevictable) |
2465 #ifdef CONFIG_ARCH_USES_PG_ARCH_X
2470 LRU_GEN_MASK | LRU_REFS_MASK));
2472 /* ->mapping in first and second tail page is replaced by other uses */
2473 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
2475 page_tail->mapping = head->mapping;
2476 page_tail->index = head->index + tail;
2479 * page->private should not be set in tail pages with the exception
2480 * of swap cache pages that store the swp_entry_t in tail pages.
2481 * Fix up and warn once if private is unexpectedly set.
2483 * What of 32-bit systems, on which folio->_pincount overlays
2484 * head[1].private? No problem: THP_SWAP is not enabled on 32-bit, and
2485 * pincount must be 0 for folio_ref_freeze() to have succeeded.
2487 if (!folio_test_swapcache(page_folio(head))) {
2488 VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
2489 page_tail->private = 0;
2492 /* Page flags must be visible before we make the page non-compound. */
2496 * Clear PageTail before unfreezing page refcount.
2498 * After successful get_page_unless_zero() might follow put_page()
2499 * which needs correct compound_head().
2501 clear_compound_head(page_tail);
2503 /* Finally unfreeze refcount. Additional reference from page cache. */
2504 page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) ||
2505 PageSwapCache(head)));
2507 if (page_is_young(head))
2508 set_page_young(page_tail);
2509 if (page_is_idle(head))
2510 set_page_idle(page_tail);
2512 page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
2515 * always add to the tail because some iterators expect new
2516 * pages to show after the currently processed elements - e.g.
2519 lru_add_page_tail(head, page_tail, lruvec, list);
2522 static void __split_huge_page(struct page *page, struct list_head *list,
2525 struct folio *folio = page_folio(page);
2526 struct page *head = &folio->page;
2527 struct lruvec *lruvec;
2528 struct address_space *swap_cache = NULL;
2529 unsigned long offset = 0;
2530 unsigned int nr = thp_nr_pages(head);
2533 /* complete memcg works before add pages to LRU */
2534 split_page_memcg(head, nr);
2536 if (PageAnon(head) && PageSwapCache(head)) {
2537 swp_entry_t entry = { .val = page_private(head) };
2539 offset = swp_offset(entry);
2540 swap_cache = swap_address_space(entry);
2541 xa_lock(&swap_cache->i_pages);
2544 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
2545 lruvec = folio_lruvec_lock(folio);
2547 ClearPageHasHWPoisoned(head);
2549 for (i = nr - 1; i >= 1; i--) {
2550 __split_huge_page_tail(head, i, lruvec, list);
2551 /* Some pages can be beyond EOF: drop them from page cache */
2552 if (head[i].index >= end) {
2553 struct folio *tail = page_folio(head + i);
2555 if (shmem_mapping(head->mapping))
2556 shmem_uncharge(head->mapping->host, 1);
2557 else if (folio_test_clear_dirty(tail))
2558 folio_account_cleaned(tail,
2559 inode_to_wb(folio->mapping->host));
2560 __filemap_remove_folio(tail, NULL);
2562 } else if (!PageAnon(page)) {
2563 __xa_store(&head->mapping->i_pages, head[i].index,
2565 } else if (swap_cache) {
2566 __xa_store(&swap_cache->i_pages, offset + i,
2571 ClearPageCompound(head);
2572 unlock_page_lruvec(lruvec);
2573 /* Caller disabled irqs, so they are still disabled here */
2575 split_page_owner(head, nr);
2577 /* See comment in __split_huge_page_tail() */
2578 if (PageAnon(head)) {
2579 /* Additional pin to swap cache */
2580 if (PageSwapCache(head)) {
2581 page_ref_add(head, 2);
2582 xa_unlock(&swap_cache->i_pages);
2587 /* Additional pin to page cache */
2588 page_ref_add(head, 2);
2589 xa_unlock(&head->mapping->i_pages);
2593 remap_page(folio, nr);
2595 if (PageSwapCache(head)) {
2596 swp_entry_t entry = { .val = page_private(head) };
2598 split_swap_cluster(entry);
2601 for (i = 0; i < nr; i++) {
2602 struct page *subpage = head + i;
2603 if (subpage == page)
2605 unlock_page(subpage);
2608 * Subpages may be freed if there wasn't any mapping
2609 * like if add_to_swap() is running on a lru page that
2610 * had its mapping zapped. And freeing these pages
2611 * requires taking the lru_lock so we do the put_page
2612 * of the tail pages after the split is complete.
2614 free_page_and_swap_cache(subpage);
2618 /* Racy check whether the huge page can be split */
2619 bool can_split_folio(struct folio *folio, int *pextra_pins)
2623 /* Additional pins from page cache */
2624 if (folio_test_anon(folio))
2625 extra_pins = folio_test_swapcache(folio) ?
2626 folio_nr_pages(folio) : 0;
2628 extra_pins = folio_nr_pages(folio);
2630 *pextra_pins = extra_pins;
2631 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 1;
2635 * This function splits huge page into normal pages. @page can point to any
2636 * subpage of huge page to split. Split doesn't change the position of @page.
2638 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2639 * The huge page must be locked.
2641 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
2643 * Both head page and tail pages will inherit mapping, flags, and so on from
2646 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
2647 * they are not mapped.
2649 * Returns 0 if the hugepage is split successfully.
2650 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2653 int split_huge_page_to_list(struct page *page, struct list_head *list)
2655 struct folio *folio = page_folio(page);
2656 struct deferred_split *ds_queue = get_deferred_split_queue(folio);
2657 XA_STATE(xas, &folio->mapping->i_pages, folio->index);
2658 struct anon_vma *anon_vma = NULL;
2659 struct address_space *mapping = NULL;
2660 int extra_pins, ret;
2664 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2665 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
2667 is_hzp = is_huge_zero_page(&folio->page);
2669 pr_warn_ratelimited("Called split_huge_page for huge zero page\n");
2673 if (folio_test_writeback(folio))
2676 if (folio_test_anon(folio)) {
2678 * The caller does not necessarily hold an mmap_lock that would
2679 * prevent the anon_vma disappearing so we first we take a
2680 * reference to it and then lock the anon_vma for write. This
2681 * is similar to folio_lock_anon_vma_read except the write lock
2682 * is taken to serialise against parallel split or collapse
2685 anon_vma = folio_get_anon_vma(folio);
2692 anon_vma_lock_write(anon_vma);
2696 mapping = folio->mapping;
2704 gfp = current_gfp_context(mapping_gfp_mask(mapping) &
2707 if (folio_test_private(folio) &&
2708 !filemap_release_folio(folio, gfp)) {
2713 xas_split_alloc(&xas, folio, folio_order(folio), gfp);
2714 if (xas_error(&xas)) {
2715 ret = xas_error(&xas);
2720 i_mmap_lock_read(mapping);
2723 *__split_huge_page() may need to trim off pages beyond EOF:
2724 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock,
2725 * which cannot be nested inside the page tree lock. So note
2726 * end now: i_size itself may be changed at any moment, but
2727 * folio lock is good enough to serialize the trimming.
2729 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
2730 if (shmem_mapping(mapping))
2731 end = shmem_fallocend(mapping->host, end);
2735 * Racy check if we can split the page, before unmap_folio() will
2738 if (!can_split_folio(folio, &extra_pins)) {
2745 /* block interrupt reentry in xa_lock and spinlock */
2746 local_irq_disable();
2749 * Check if the folio is present in page cache.
2750 * We assume all tail are present too, if folio is there.
2754 if (xas_load(&xas) != folio)
2758 /* Prevent deferred_split_scan() touching ->_refcount */
2759 spin_lock(&ds_queue->split_queue_lock);
2760 if (folio_ref_freeze(folio, 1 + extra_pins)) {
2761 if (!list_empty(&folio->_deferred_list)) {
2762 ds_queue->split_queue_len--;
2763 list_del(&folio->_deferred_list);
2765 spin_unlock(&ds_queue->split_queue_lock);
2767 int nr = folio_nr_pages(folio);
2769 xas_split(&xas, folio, folio_order(folio));
2770 if (folio_test_swapbacked(folio)) {
2771 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS,
2774 __lruvec_stat_mod_folio(folio, NR_FILE_THPS,
2776 filemap_nr_thps_dec(mapping);
2780 __split_huge_page(page, list, end);
2783 spin_unlock(&ds_queue->split_queue_lock);
2788 remap_page(folio, folio_nr_pages(folio));
2794 anon_vma_unlock_write(anon_vma);
2795 put_anon_vma(anon_vma);
2798 i_mmap_unlock_read(mapping);
2801 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
2805 void free_transhuge_page(struct page *page)
2807 struct folio *folio = (struct folio *)page;
2808 struct deferred_split *ds_queue = get_deferred_split_queue(folio);
2809 unsigned long flags;
2811 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
2812 if (!list_empty(&folio->_deferred_list)) {
2813 ds_queue->split_queue_len--;
2814 list_del(&folio->_deferred_list);
2816 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
2817 free_compound_page(page);
2820 void deferred_split_folio(struct folio *folio)
2822 struct deferred_split *ds_queue = get_deferred_split_queue(folio);
2824 struct mem_cgroup *memcg = folio_memcg(folio);
2826 unsigned long flags;
2828 VM_BUG_ON_FOLIO(folio_order(folio) < 2, folio);
2831 * The try_to_unmap() in page reclaim path might reach here too,
2832 * this may cause a race condition to corrupt deferred split queue.
2833 * And, if page reclaim is already handling the same folio, it is
2834 * unnecessary to handle it again in shrinker.
2836 * Check the swapcache flag to determine if the folio is being
2837 * handled by page reclaim since THP swap would add the folio into
2838 * swap cache before calling try_to_unmap().
2840 if (folio_test_swapcache(folio))
2843 if (!list_empty(&folio->_deferred_list))
2846 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
2847 if (list_empty(&folio->_deferred_list)) {
2848 count_vm_event(THP_DEFERRED_SPLIT_PAGE);
2849 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
2850 ds_queue->split_queue_len++;
2853 set_shrinker_bit(memcg, folio_nid(folio),
2854 deferred_split_shrinker.id);
2857 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
2860 static unsigned long deferred_split_count(struct shrinker *shrink,
2861 struct shrink_control *sc)
2863 struct pglist_data *pgdata = NODE_DATA(sc->nid);
2864 struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
2868 ds_queue = &sc->memcg->deferred_split_queue;
2870 return READ_ONCE(ds_queue->split_queue_len);
2873 static unsigned long deferred_split_scan(struct shrinker *shrink,
2874 struct shrink_control *sc)
2876 struct pglist_data *pgdata = NODE_DATA(sc->nid);
2877 struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
2878 unsigned long flags;
2880 struct folio *folio, *next;
2885 ds_queue = &sc->memcg->deferred_split_queue;
2888 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
2889 /* Take pin on all head pages to avoid freeing them under us */
2890 list_for_each_entry_safe(folio, next, &ds_queue->split_queue,
2892 if (folio_try_get(folio)) {
2893 list_move(&folio->_deferred_list, &list);
2895 /* We lost race with folio_put() */
2896 list_del_init(&folio->_deferred_list);
2897 ds_queue->split_queue_len--;
2899 if (!--sc->nr_to_scan)
2902 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
2904 list_for_each_entry_safe(folio, next, &list, _deferred_list) {
2905 if (!folio_trylock(folio))
2907 /* split_huge_page() removes page from list on success */
2908 if (!split_folio(folio))
2910 folio_unlock(folio);
2915 spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
2916 list_splice_tail(&list, &ds_queue->split_queue);
2917 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
2920 * Stop shrinker if we didn't split any page, but the queue is empty.
2921 * This can happen if pages were freed under us.
2923 if (!split && list_empty(&ds_queue->split_queue))
2928 static struct shrinker deferred_split_shrinker = {
2929 .count_objects = deferred_split_count,
2930 .scan_objects = deferred_split_scan,
2931 .seeks = DEFAULT_SEEKS,
2932 .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE |
2936 #ifdef CONFIG_DEBUG_FS
2937 static void split_huge_pages_all(void)
2941 struct folio *folio;
2942 unsigned long pfn, max_zone_pfn;
2943 unsigned long total = 0, split = 0;
2945 pr_debug("Split all THPs\n");
2946 for_each_zone(zone) {
2947 if (!managed_zone(zone))
2949 max_zone_pfn = zone_end_pfn(zone);
2950 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
2953 page = pfn_to_online_page(pfn);
2954 if (!page || PageTail(page))
2956 folio = page_folio(page);
2957 if (!folio_try_get(folio))
2960 if (unlikely(page_folio(page) != folio))
2963 if (zone != folio_zone(folio))
2966 if (!folio_test_large(folio)
2967 || folio_test_hugetlb(folio)
2968 || !folio_test_lru(folio))
2973 nr_pages = folio_nr_pages(folio);
2974 if (!split_folio(folio))
2976 pfn += nr_pages - 1;
2977 folio_unlock(folio);
2984 pr_debug("%lu of %lu THP split\n", split, total);
2987 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
2989 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||
2990 is_vm_hugetlb_page(vma);
2993 static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
2994 unsigned long vaddr_end)
2997 struct task_struct *task;
2998 struct mm_struct *mm;
2999 unsigned long total = 0, split = 0;
3002 vaddr_start &= PAGE_MASK;
3003 vaddr_end &= PAGE_MASK;
3005 /* Find the task_struct from pid */
3007 task = find_task_by_vpid(pid);
3013 get_task_struct(task);
3016 /* Find the mm_struct */
3017 mm = get_task_mm(task);
3018 put_task_struct(task);
3025 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n",
3026 pid, vaddr_start, vaddr_end);
3030 * always increase addr by PAGE_SIZE, since we could have a PTE page
3031 * table filled with PTE-mapped THPs, each of which is distinct.
3033 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) {
3034 struct vm_area_struct *vma = vma_lookup(mm, addr);
3040 /* skip special VMA and hugetlb VMA */
3041 if (vma_not_suitable_for_thp_split(vma)) {
3046 /* FOLL_DUMP to ignore special (like zero) pages */
3047 page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);
3049 if (IS_ERR_OR_NULL(page))
3052 if (!is_transparent_hugepage(page))
3056 if (!can_split_folio(page_folio(page), NULL))
3059 if (!trylock_page(page))
3062 if (!split_huge_page(page))
3070 mmap_read_unlock(mm);
3073 pr_debug("%lu of %lu THP split\n", split, total);
3079 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start,
3082 struct filename *file;
3083 struct file *candidate;
3084 struct address_space *mapping;
3088 unsigned long total = 0, split = 0;
3090 file = getname_kernel(file_path);
3094 candidate = file_open_name(file, O_RDONLY, 0);
3095 if (IS_ERR(candidate))
3098 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n",
3099 file_path, off_start, off_end);
3101 mapping = candidate->f_mapping;
3103 for (index = off_start; index < off_end; index += nr_pages) {
3104 struct folio *folio = __filemap_get_folio(mapping, index,
3108 if (xa_is_value(folio) || !folio)
3111 if (!folio_test_large(folio))
3115 nr_pages = folio_nr_pages(folio);
3117 if (!folio_trylock(folio))
3120 if (!split_folio(folio))
3123 folio_unlock(folio);
3129 filp_close(candidate, NULL);
3132 pr_debug("%lu of %lu file-backed THP split\n", split, total);
3138 #define MAX_INPUT_BUF_SZ 255
3140 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf,
3141 size_t count, loff_t *ppops)
3143 static DEFINE_MUTEX(split_debug_mutex);
3145 /* hold pid, start_vaddr, end_vaddr or file_path, off_start, off_end */
3146 char input_buf[MAX_INPUT_BUF_SZ];
3148 unsigned long vaddr_start, vaddr_end;
3150 ret = mutex_lock_interruptible(&split_debug_mutex);
3156 memset(input_buf, 0, MAX_INPUT_BUF_SZ);
3157 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ)))
3160 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0';
3162 if (input_buf[0] == '/') {
3164 char *buf = input_buf;
3165 char file_path[MAX_INPUT_BUF_SZ];
3166 pgoff_t off_start = 0, off_end = 0;
3167 size_t input_len = strlen(input_buf);
3169 tok = strsep(&buf, ",");
3171 strcpy(file_path, tok);
3177 ret = sscanf(buf, "0x%lx,0x%lx", &off_start, &off_end);
3182 ret = split_huge_pages_in_file(file_path, off_start, off_end);
3189 ret = sscanf(input_buf, "%d,0x%lx,0x%lx", &pid, &vaddr_start, &vaddr_end);
3190 if (ret == 1 && pid == 1) {
3191 split_huge_pages_all();
3192 ret = strlen(input_buf);
3194 } else if (ret != 3) {
3199 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end);
3201 ret = strlen(input_buf);
3203 mutex_unlock(&split_debug_mutex);
3208 static const struct file_operations split_huge_pages_fops = {
3209 .owner = THIS_MODULE,
3210 .write = split_huge_pages_write,
3211 .llseek = no_llseek,
3214 static int __init split_huge_pages_debugfs(void)
3216 debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
3217 &split_huge_pages_fops);
3220 late_initcall(split_huge_pages_debugfs);
3223 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
3224 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
3227 struct vm_area_struct *vma = pvmw->vma;
3228 struct mm_struct *mm = vma->vm_mm;
3229 unsigned long address = pvmw->address;
3230 bool anon_exclusive;
3235 if (!(pvmw->pmd && !pvmw->pte))
3238 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
3239 pmdval = pmdp_invalidate(vma, address, pvmw->pmd);
3241 /* See page_try_share_anon_rmap(): invalidate PMD first. */
3242 anon_exclusive = PageAnon(page) && PageAnonExclusive(page);
3243 if (anon_exclusive && page_try_share_anon_rmap(page)) {
3244 set_pmd_at(mm, address, pvmw->pmd, pmdval);
3248 if (pmd_dirty(pmdval))
3249 set_page_dirty(page);
3250 if (pmd_write(pmdval))
3251 entry = make_writable_migration_entry(page_to_pfn(page));
3252 else if (anon_exclusive)
3253 entry = make_readable_exclusive_migration_entry(page_to_pfn(page));
3255 entry = make_readable_migration_entry(page_to_pfn(page));
3256 if (pmd_young(pmdval))
3257 entry = make_migration_entry_young(entry);
3258 if (pmd_dirty(pmdval))
3259 entry = make_migration_entry_dirty(entry);
3260 pmdswp = swp_entry_to_pmd(entry);
3261 if (pmd_soft_dirty(pmdval))
3262 pmdswp = pmd_swp_mksoft_dirty(pmdswp);
3263 if (pmd_uffd_wp(pmdval))
3264 pmdswp = pmd_swp_mkuffd_wp(pmdswp);
3265 set_pmd_at(mm, address, pvmw->pmd, pmdswp);
3266 page_remove_rmap(page, vma, true);
3268 trace_set_migration_pmd(address, pmd_val(pmdswp));
3273 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
3275 struct vm_area_struct *vma = pvmw->vma;
3276 struct mm_struct *mm = vma->vm_mm;
3277 unsigned long address = pvmw->address;
3278 unsigned long haddr = address & HPAGE_PMD_MASK;
3282 if (!(pvmw->pmd && !pvmw->pte))
3285 entry = pmd_to_swp_entry(*pvmw->pmd);
3287 pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot));
3288 if (pmd_swp_soft_dirty(*pvmw->pmd))
3289 pmde = pmd_mksoft_dirty(pmde);
3290 if (pmd_swp_uffd_wp(*pvmw->pmd))
3291 pmde = pmd_mkuffd_wp(pmde);
3292 if (!is_migration_entry_young(entry))
3293 pmde = pmd_mkold(pmde);
3294 /* NOTE: this may contain setting soft-dirty on some archs */
3295 if (PageDirty(new) && is_migration_entry_dirty(entry))
3296 pmde = pmd_mkdirty(pmde);
3297 if (is_writable_migration_entry(entry))
3298 pmde = maybe_pmd_mkwrite(pmde, vma);
3300 pmde = pmd_wrprotect(pmde);
3302 if (PageAnon(new)) {
3303 rmap_t rmap_flags = RMAP_COMPOUND;
3305 if (!is_readable_migration_entry(entry))
3306 rmap_flags |= RMAP_EXCLUSIVE;
3308 page_add_anon_rmap(new, vma, haddr, rmap_flags);
3310 page_add_file_rmap(new, vma, true);
3312 VM_BUG_ON(pmd_write(pmde) && PageAnon(new) && !PageAnonExclusive(new));
3313 set_pmd_at(mm, haddr, pvmw->pmd, pmde);
3315 /* No need to invalidate - it was non-present before */
3316 update_mmu_cache_pmd(vma, address, pvmw->pmd);
3317 trace_remove_migration_pmd(address, pmd_val(pmde));