mm/damon/ops-common: atomically test and clear young on ptes and pmds

commit c11d34fa139e4b0fb4249a30f37b178353533fa1 upstream.

It is racy to non-atomically read a pte, then clear the young bit, then
write it back as this could discard dirty information.  Further, it is bad
practice to directly set a pte entry within a table.  Instead clearing
young must go through the arch-provided helper,
ptep_test_and_clear_young() to ensure it is modified atomically and to
give the arch code visibility and allow it to check (and potentially
modify) the operation.

Link: https://lkml.kernel.org/r/20230602092949.545577-3-ryan.roberts@arm.com
Fixes: 3f49584b262c ("mm/damon: implement primitives for the virtual memory address spaces").
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: SeongJae Park <sj@kernel.org>
Reviewed-by: Mike Rapoport (IBM) <rppt@kernel.org>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index 7540960..13b9997 100644 (file)
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -33,7 +33,7 @@ struct page *damon_get_page(unsigned long pfn)
        return page;
 }
 
-void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
+void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)
 {
        bool referenced = false;
        struct page *page = damon_get_page(pte_pfn(*pte));
@@ -41,13 +41,11 @@ void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
        if (!page)
                return;
 
-       if (pte_young(*pte)) {
+       if (ptep_test_and_clear_young(vma, addr, pte))
                referenced = true;
-               *pte = pte_mkold(*pte);
-       }
 
 #ifdef CONFIG_MMU_NOTIFIER
-       if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE))
+       if (mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE))
                referenced = true;
 #endif /* CONFIG_MMU_NOTIFIER */
 
@@ -58,7 +56,7 @@ void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
        put_page(page);
 }
 
-void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
+void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr)
 {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        bool referenced = false;
@@ -67,13 +65,11 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
        if (!page)
                return;
 
-       if (pmd_young(*pmd)) {
+       if (pmdp_test_and_clear_young(vma, addr, pmd))
                referenced = true;
-               *pmd = pmd_mkold(*pmd);
-       }
 
 #ifdef CONFIG_MMU_NOTIFIER
-       if (mmu_notifier_clear_young(mm, addr, addr + HPAGE_PMD_SIZE))
+       if (mmu_notifier_clear_young(vma->vm_mm, addr, addr + HPAGE_PMD_SIZE))
                referenced = true;
 #endif /* CONFIG_MMU_NOTIFIER */
 

diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h
index 8d82d37..e062a88 100644 (file)
--- a/mm/damon/ops-common.h
+++ b/mm/damon/ops-common.h
@@ -9,8 +9,8 @@
 
 struct page *damon_get_page(unsigned long pfn);
 
-void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr);
-void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr);
+void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr);
+void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr);
 
 int damon_cold_score(struct damon_ctx *c, struct damon_region *r,
                        struct damos *s);

diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index 402d30b..7bc8d79 100644 (file)
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -24,9 +24,9 @@ static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
        while (page_vma_mapped_walk(&pvmw)) {
                addr = pvmw.address;
                if (pvmw.pte)
-                       damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr);
+                       damon_ptep_mkold(pvmw.pte, vma, addr);
                else
-                       damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr);
+                       damon_pmdp_mkold(pvmw.pmd, vma, addr);
        }
        return true;
 }

diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index 15f03df..26d561a 100644 (file)
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -311,7 +311,7 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
                }
 
                if (pmd_trans_huge(*pmd)) {
-                       damon_pmdp_mkold(pmd, walk->mm, addr);
+                       damon_pmdp_mkold(pmd, walk->vma, addr);
                        spin_unlock(ptl);
                        return 0;
                }
@@ -323,7 +323,7 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
        pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
        if (!pte_present(*pte))
                goto out;
-       damon_ptep_mkold(pte, walk->mm, addr);
+       damon_ptep_mkold(pte, walk->vma, addr);
 out:
        pte_unmap_unlock(pte, ptl);
        return 0;