mm, thp, migrate: handling migration of 64KB hugepages

When a 64KB hugepage is migrated, it should be properly
handled since it is different from other normal page
mappings. The kernel should handle a set of sequential
16 page mappings at once. If not, the kernel can mishandle
map counts of a compound page (that is, a set of pages).
It can be a source of kernel bugs and the bug is easily
reproduced on low-memory devices.

This patch deals with the migration of 64KB hugepages.

Fixes: 90eb23660fef ('mm: THP: introducing a fine-grained transparent hugepage technique for ARM64 architecture')
Change-Id: I50a5d4e9a263e7dcbded15c982f57c15a3a48f39
Signed-off-by: Sung-hun Kim <sfoon.kim@samsung.com>

diff --git a/arch/arm64/mm/huge_memory.c b/arch/arm64/mm/huge_memory.c
index 2ef1a21..1073fde 100644 (file)
--- a/arch/arm64/mm/huge_memory.c
+++ b/arch/arm64/mm/huge_memory.c
@@ -1087,4 +1087,79 @@ void split_huge_pte_address(struct vm_area_struct *vma, unsigned long address,
 
        __split_huge_pte(vma, pmd, pte, haddr, freeze, page);
 }
+
+void set_huge_pte_migration_entry(
+               struct page_vma_mapped_walk *pvmw,
+               struct page *page)
+{
+       int i;
+       struct vm_area_struct *vma = pvmw->vma;
+       struct mm_struct *mm = vma->vm_mm;
+       unsigned long address = pvmw->address;
+       pte_t pteval, *pte;
+       swp_entry_t entry;
+       pte_t pteswp;
+       struct page *_page = page;
+
+       if (!(pvmw->pmd && pvmw->pte))
+               return;
+
+       flush_cache_range(vma, address, address + HPAGE_CONT_PTE_SIZE);
+       pte = pvmw->pte;
+
+       //arch_set_huge_pte_at(mm, address, pvmw->pte, ptee);
+       for (i = 0, pte = pvmw->pte; i < HPAGE_CONT_PTE_NR; i++, pte++) {
+               pteval = ptep_invalidate(vma, address, pte);
+               if (pte_dirty(pteval))
+                       set_page_dirty(_page);
+               entry = make_migration_entry(page, pte_write(pteval));
+               pteswp = swp_entry_to_pte(entry);
+               if (pte_soft_dirty(pteval))
+                       pteswp = pte_swp_mksoft_dirty(pteswp);
+               set_pte_at(mm, address, pte, pteswp);
+               _page++;
+               address += PAGE_SIZE;
+       }
+
+       pvmw->pte = pte;
+       pvmw->address = address;
+
+       page_remove_rmap(page, true);
+       put_page(page);
+}
+
+void remove_migration_huge_pte(
+               struct page_vma_mapped_walk *pvmw, struct page *new)
+{
+       struct vm_area_struct *vma = pvmw->vma;
+       struct mm_struct *mm = vma->vm_mm;
+       unsigned long address = pvmw->address;
+       unsigned long mmun_start = address & HPAGE_CONT_PTE_MASK;
+       pte_t ptee;
+       swp_entry_t entry;
+
+       if (!(pvmw->pmd && !pvmw->pte))
+               return;
+
+       entry = pmd_to_swp_entry(*pvmw->pmd);
+       get_page(new);
+       ptee = pte_mkold(arch_make_huge_pte(new, vma));
+       if (pte_swp_soft_dirty(*pvmw->pte))
+               ptee = pte_mksoft_dirty(ptee);
+       if (is_write_migration_entry(entry))
+               ptee = maybe_mkwrite(ptee, vma);
+
+       flush_cache_range(vma, mmun_start, mmun_start + HPAGE_CONT_PTE_SIZE);
+       if (PageAnon(new))
+               page_add_anon_rmap(new, vma, mmun_start, true);
+       else
+               page_add_file_rmap(new, true);
+
+       arch_set_huge_pte_at(mm, mmun_start, pvmw->pte, ptee, 0);
+       if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
+               mlock_vma_page(new);
+       pvmw->address = address + HPAGE_CONT_PTE_SIZE;
+       pvmw->pte = pvmw->pte + HPAGE_CONT_PTE_NR;
+       update_mmu_cache_pmd(vma, address, pvmw->pmd);
+}
 #endif /* CONFIG_FINEGRAINED_THP */

diff --git a/include/linux/swapops.h b/include/linux/swapops.h
index fb0cc92..27f8d48 100644 (file)
--- a/include/linux/swapops.h
+++ b/include/linux/swapops.h
@@ -257,6 +257,14 @@ extern void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
                struct page *new);
 
+#ifdef CONFIG_FINEGRAINED_THP
+extern void set_huge_pte_migration_entry(struct page_vma_mapped_walk *pvmw,
+               struct page *page);
+
+extern void remove_migration_huge_pte(struct page_vma_mapped_walk *pvmw,
+               struct page *new);
+#endif
+
 extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd);
 
 static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
@@ -301,6 +309,19 @@ static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
 {
        BUILD_BUG();
 }
+#ifdef CONFIG_FINEGRAINED_THP
+static inline void set_huge_pte_migration_entry(struct page_vma_mapped_walk *pvmw,
+               struct page *page)
+{
+       BUILD_BUG();
+}
+
+static inline void remove_migration_huge_pte(struct page_vma_mapped_walk *pvmw,
+               struct page *new)
+{
+       BUILD_BUG();
+}
+#endif
 
 static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { }
 

diff --git a/mm/migrate.c b/mm/migrate.c
index b366006..56f63d9 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -230,6 +230,13 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
                        remove_migration_pmd(&pvmw, new);
                        continue;
                }
+#ifdef CONFIG_FINEGRAINED_THP
+               if (PageTransHuge(page) && pte_cont(*pvmw.pte)) {
+                       VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
+                       remove_migration_huge_pte(&pvmw, new);
+                       continue;
+               }
+#endif /* CONFIG_FINEGRAINED_THP */
 #endif
 
                get_page(new);
@@ -266,16 +273,6 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
                                page_dup_rmap(new, true);
                } else
 #endif
-#ifdef CONFIG_FINEGRAINED_THP
-               if (PageTransHuge(new)) {
-                       pte = pte_mkcont(pte_mkhuge(pte));
-                       arch_set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte, 0);
-                       if (PageAnon(new))
-                               page_add_anon_rmap(new, vma, pvmw.address, true);
-                       else
-                               page_dup_rmap(new, true);
-               } else
-#endif /* CONFIG_FINEGRAINED_THP */
                {
                        set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
 

diff --git a/mm/rmap.c b/mm/rmap.c
index a49943e..82e2aa9 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1487,6 +1487,14 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                        set_pmd_migration_entry(&pvmw, page);
                        continue;
                }
+#ifdef CONFIG_FINEGRAINED_THP
+               if (pvmw.pte && pte_cont(*pvmw.pte) && (flags & TTU_MIGRATION)) {
+                       VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
+
+                       set_huge_pte_migration_entry(&pvmw, page);
+                       continue;
+               }
+#endif /* CONFIG_FINEGRAINED_THP */
 #endif
 
                /*