Merge tag 'for-v6.6-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux...

[platform/kernel/linux-starfive.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ba6d39b..52d2607 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4980,7 +4980,7 @@ static bool is_hugetlb_entry_hwpoisoned(pte_t pte)
 
 static void
 hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
-                     struct folio *new_folio, pte_t old)
+                     struct folio *new_folio, pte_t old, unsigned long sz)
 {
        pte_t newpte = make_huge_pte(vma, &new_folio->page, 1);
 
@@ -4988,7 +4988,7 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add
        hugepage_add_new_anon_rmap(new_folio, vma, addr);
        if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old))
                newpte = huge_pte_mkuffd_wp(newpte);
-       set_huge_pte_at(vma->vm_mm, addr, ptep, newpte);
+       set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
        hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
        folio_set_hugetlb_migratable(new_folio);
 }
@@ -5065,7 +5065,7 @@ again:
                } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
                        if (!userfaultfd_wp(dst_vma))
                                entry = huge_pte_clear_uffd_wp(entry);
-                       set_huge_pte_at(dst, addr, dst_pte, entry);
+                       set_huge_pte_at(dst, addr, dst_pte, entry, sz);
                } else if (unlikely(is_hugetlb_entry_migration(entry))) {
                        swp_entry_t swp_entry = pte_to_swp_entry(entry);
                        bool uffd_wp = pte_swp_uffd_wp(entry);
@@ -5080,18 +5080,18 @@ again:
                                entry = swp_entry_to_pte(swp_entry);
                                if (userfaultfd_wp(src_vma) && uffd_wp)
                                        entry = pte_swp_mkuffd_wp(entry);
-                               set_huge_pte_at(src, addr, src_pte, entry);
+                               set_huge_pte_at(src, addr, src_pte, entry, sz);
                        }
                        if (!userfaultfd_wp(dst_vma))
                                entry = huge_pte_clear_uffd_wp(entry);
-                       set_huge_pte_at(dst, addr, dst_pte, entry);
+                       set_huge_pte_at(dst, addr, dst_pte, entry, sz);
                } else if (unlikely(is_pte_marker(entry))) {
                        pte_marker marker = copy_pte_marker(
                                pte_to_swp_entry(entry), dst_vma);
 
                        if (marker)
                                set_huge_pte_at(dst, addr, dst_pte,
-                                               make_pte_marker(marker));
+                                               make_pte_marker(marker), sz);
                } else {
                        entry = huge_ptep_get(src_pte);
                        pte_folio = page_folio(pte_page(entry));
@@ -5145,7 +5145,7 @@ again:
                                        goto again;
                                }
                                hugetlb_install_folio(dst_vma, dst_pte, addr,
-                                                     new_folio, src_pte_old);
+                                                     new_folio, src_pte_old, sz);
                                spin_unlock(src_ptl);
                                spin_unlock(dst_ptl);
                                continue;
@@ -5166,7 +5166,7 @@ again:
                        if (!userfaultfd_wp(dst_vma))
                                entry = huge_pte_clear_uffd_wp(entry);
 
-                       set_huge_pte_at(dst, addr, dst_pte, entry);
+                       set_huge_pte_at(dst, addr, dst_pte, entry, sz);
                        hugetlb_count_add(npages, dst);
                }
                spin_unlock(src_ptl);
@@ -5184,7 +5184,8 @@ again:
 }
 
 static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
-                         unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte)
+                         unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte,
+                         unsigned long sz)
 {
        struct hstate *h = hstate_vma(vma);
        struct mm_struct *mm = vma->vm_mm;
@@ -5202,7 +5203,7 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
                spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
 
        pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
-       set_huge_pte_at(mm, new_addr, dst_pte, pte);
+       set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
 
        if (src_ptl != dst_ptl)
                spin_unlock(src_ptl);
@@ -5259,7 +5260,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
                if (!dst_pte)
                        break;
 
-               move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte);
+               move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz);
        }
 
        if (shared_pmd)
@@ -5337,7 +5338,8 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct
                        if (pte_swp_uffd_wp_any(pte) &&
                            !(zap_flags & ZAP_FLAG_DROP_MARKER))
                                set_huge_pte_at(mm, address, ptep,
-                                               make_pte_marker(PTE_MARKER_UFFD_WP));
+                                               make_pte_marker(PTE_MARKER_UFFD_WP),
+                                               sz);
                        else
                                huge_pte_clear(mm, address, ptep, sz);
                        spin_unlock(ptl);
@@ -5371,7 +5373,8 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct
                if (huge_pte_uffd_wp(pte) &&
                    !(zap_flags & ZAP_FLAG_DROP_MARKER))
                        set_huge_pte_at(mm, address, ptep,
-                                       make_pte_marker(PTE_MARKER_UFFD_WP));
+                                       make_pte_marker(PTE_MARKER_UFFD_WP),
+                                       sz);
                hugetlb_count_sub(pages_per_huge_page(h), mm);
                page_remove_rmap(page, vma, true);
 
@@ -5676,7 +5679,7 @@ retry_avoidcopy:
                hugepage_add_new_anon_rmap(new_folio, vma, haddr);
                if (huge_pte_uffd_wp(pte))
                        newpte = huge_pte_mkuffd_wp(newpte);
-               set_huge_pte_at(mm, haddr, ptep, newpte);
+               set_huge_pte_at(mm, haddr, ptep, newpte, huge_page_size(h));
                folio_set_hugetlb_migratable(new_folio);
                /* Make the old page be freed below */
                new_folio = old_folio;
@@ -5972,7 +5975,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
         */
        if (unlikely(pte_marker_uffd_wp(old_pte)))
                new_pte = huge_pte_mkuffd_wp(new_pte);
-       set_huge_pte_at(mm, haddr, ptep, new_pte);
+       set_huge_pte_at(mm, haddr, ptep, new_pte, huge_page_size(h));
 
        hugetlb_count_add(pages_per_huge_page(h), mm);
        if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
@@ -6261,7 +6264,8 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
                }
 
                _dst_pte = make_pte_marker(PTE_MARKER_POISONED);
-               set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
+               set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte,
+                               huge_page_size(h));
 
                /* No need to invalidate - it was non-present before */
                update_mmu_cache(dst_vma, dst_addr, dst_pte);
@@ -6412,7 +6416,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
        if (wp_enabled)
                _dst_pte = huge_pte_mkuffd_wp(_dst_pte);
 
-       set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
+       set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, huge_page_size(h));
 
        hugetlb_count_add(pages_per_huge_page(h), dst_mm);
 
@@ -6598,7 +6602,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
                        else if (uffd_wp_resolve)
                                newpte = pte_swp_clear_uffd_wp(newpte);
                        if (!pte_same(pte, newpte))
-                               set_huge_pte_at(mm, address, ptep, newpte);
+                               set_huge_pte_at(mm, address, ptep, newpte, psize);
                } else if (unlikely(is_pte_marker(pte))) {
                        /* No other markers apply for now. */
                        WARN_ON_ONCE(!pte_marker_uffd_wp(pte));
@@ -6623,7 +6627,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
                        if (unlikely(uffd_wp))
                                /* Safe to modify directly (none->non-present). */
                                set_huge_pte_at(mm, address, ptep,
-                                               make_pte_marker(PTE_MARKER_UFFD_WP));
+                                               make_pte_marker(PTE_MARKER_UFFD_WP),
+                                               psize);
                }
                spin_unlock(ptl);
        }