From 78df7c9b0b1662288349db6cd2de55d76e56929a Mon Sep 17 00:00:00 2001 From: Sung-hun Kim Date: Thu, 16 Sep 2021 13:44:25 +0900 Subject: [PATCH] mm, thp, migrate: handling migration of 64KB hugepages When a 64KB hugepage is migrated, it should be properly handled since it is different from other normal page mappings. The kernel should handle a set of sequential 16 page mappings at once. If not, the kernel can mishandle map counts of a compound page (that is, a set of pages). It can be a source of kernel bugs and the bug is easily reproduced on low-memory devices. This patch deals with the migration of 64KB hugepages. Fixes: 7d5372737d34 ('mm: THP: introducing a fine-grained transparent hugepage technique for ARM64 architecture') Change-Id: I50a5d4e9a263e7dcbded15c982f57c15a3a48f39 Signed-off-by: Sung-hun Kim --- arch/arm64/mm/huge_memory.c | 75 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/swapops.h | 21 +++++++++++++ mm/migrate.c | 17 +++++----- mm/rmap.c | 8 +++++ 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/arch/arm64/mm/huge_memory.c b/arch/arm64/mm/huge_memory.c index 2ef1a21..1073fde 100644 --- a/arch/arm64/mm/huge_memory.c +++ b/arch/arm64/mm/huge_memory.c @@ -1087,4 +1087,79 @@ void split_huge_pte_address(struct vm_area_struct *vma, unsigned long address, __split_huge_pte(vma, pmd, pte, haddr, freeze, page); } + +void set_huge_pte_migration_entry( + struct page_vma_mapped_walk *pvmw, + struct page *page) +{ + int i; + struct vm_area_struct *vma = pvmw->vma; + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address; + pte_t pteval, *pte; + swp_entry_t entry; + pte_t pteswp; + struct page *_page = page; + + if (!(pvmw->pmd && pvmw->pte)) + return; + + flush_cache_range(vma, address, address + HPAGE_CONT_PTE_SIZE); + pte = pvmw->pte; + + //arch_set_huge_pte_at(mm, address, pvmw->pte, ptee); + for (i = 0, pte = pvmw->pte; i < HPAGE_CONT_PTE_NR; i++, pte++) { + pteval = ptep_invalidate(vma, address, pte); + if (pte_dirty(pteval)) + set_page_dirty(_page); + entry = make_migration_entry(page, pte_write(pteval)); + pteswp = swp_entry_to_pte(entry); + if (pte_soft_dirty(pteval)) + pteswp = pte_swp_mksoft_dirty(pteswp); + set_pte_at(mm, address, pte, pteswp); + _page++; + address += PAGE_SIZE; + } + + pvmw->pte = pte; + pvmw->address = address; + + page_remove_rmap(page, true); + put_page(page); +} + +void remove_migration_huge_pte( + struct page_vma_mapped_walk *pvmw, struct page *new) +{ + struct vm_area_struct *vma = pvmw->vma; + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address; + unsigned long mmun_start = address & HPAGE_CONT_PTE_MASK; + pte_t ptee; + swp_entry_t entry; + + if (!(pvmw->pmd && !pvmw->pte)) + return; + + entry = pmd_to_swp_entry(*pvmw->pmd); + get_page(new); + ptee = pte_mkold(arch_make_huge_pte(new, vma)); + if (pte_swp_soft_dirty(*pvmw->pte)) + ptee = pte_mksoft_dirty(ptee); + if (is_write_migration_entry(entry)) + ptee = maybe_mkwrite(ptee, vma); + + flush_cache_range(vma, mmun_start, mmun_start + HPAGE_CONT_PTE_SIZE); + if (PageAnon(new)) + page_add_anon_rmap(new, vma, mmun_start, true); + else + page_add_file_rmap(new, true); + + arch_set_huge_pte_at(mm, mmun_start, pvmw->pte, ptee, 0); + if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) + mlock_vma_page(new); + pvmw->address = address + HPAGE_CONT_PTE_SIZE; + pvmw->pte = pvmw->pte + HPAGE_CONT_PTE_NR; + update_mmu_cache_pmd(vma, address, pvmw->pmd); +} #endif /* CONFIG_FINEGRAINED_THP */ diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 71aa4b7..bdfbc8e 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -250,6 +250,14 @@ extern void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new); +#ifdef CONFIG_FINEGRAINED_THP +extern void set_huge_pte_migration_entry(struct page_vma_mapped_walk *pvmw, + struct page *page); + +extern void remove_migration_huge_pte(struct page_vma_mapped_walk *pvmw, + struct page *new); +#endif + extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) @@ -292,6 +300,19 @@ static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, { BUILD_BUG(); } +#ifdef CONFIG_FINEGRAINED_THP +static inline void set_huge_pte_migration_entry(struct page_vma_mapped_walk *pvmw, + struct page *page) +{ + BUILD_BUG(); +} + +static inline void remove_migration_huge_pte(struct page_vma_mapped_walk *pvmw, + struct page *new) +{ + BUILD_BUG(); +} +#endif static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { } diff --git a/mm/migrate.c b/mm/migrate.c index b16e340..de299c3 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -230,6 +230,13 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, remove_migration_pmd(&pvmw, new); continue; } +#ifdef CONFIG_FINEGRAINED_THP + if (PageTransHuge(page) && pte_cont(*pvmw.pte)) { + VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); + remove_migration_huge_pte(&pvmw, new); + continue; + } +#endif /* CONFIG_FINEGRAINED_THP */ #endif get_page(new); @@ -266,16 +273,6 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma, page_dup_rmap(new, true); } else #endif -#ifdef CONFIG_FINEGRAINED_THP - if (PageTransHuge(new)) { - pte = pte_mkcont(pte_mkhuge(pte)); - arch_set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte, 0); - if (PageAnon(new)) - page_add_anon_rmap(new, vma, pvmw.address, true); - else - page_dup_rmap(new, true); - } else -#endif /* CONFIG_FINEGRAINED_THP */ { set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); diff --git a/mm/rmap.c b/mm/rmap.c index 64de8c1..0eca948 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1480,6 +1480,14 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, set_pmd_migration_entry(&pvmw, page); continue; } +#ifdef CONFIG_FINEGRAINED_THP + if (pvmw.pte && pte_cont(*pvmw.pte) && (flags & TTU_MIGRATION)) { + VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page); + + set_huge_pte_migration_entry(&pvmw, page); + continue; + } +#endif /* CONFIG_FINEGRAINED_THP */ #endif /* -- 2.7.4