mm: pagewalk: add 'depth' parameter to pte_hole

The pte_hole() callback is called at multiple levels of the page tables.
Code dumping the kernel page tables needs to know what at what depth the
missing entry is.  Add this is an extra parameter to pte_hole().  When the
depth isn't know (e.g.  processing a vma) then -1 is passed.

The depth that is reported is the actual level where the entry is missing
(ignoring any folding that is in place), i.e.  any levels where
PTRS_PER_P?D is set to 1 are ignored.

Note that depth starts at 0 for a PGD so that PUD/PMD/PTE retain their
natural numbers as levels 2/3/4.

Link: http://lkml.kernel.org/r/20191218162402.45610-16-steven.price@arm.com
Signed-off-by: Steven Price <steven.price@arm.com>
Tested-by: Zong Li <zong.li@sifive.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: James Hogan <jhogan@kernel.org>
Cc: James Morse <james.morse@arm.com>
Cc: Jerome Glisse <jglisse@redhat.com>
Cc: "Liang, Kan" <kan.liang@linux.intel.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Paul Burton <paul.burton@mips.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vineet Gupta <vgupta@synopsys.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 9442631fd4afcb16e3a09d6a661ebdc630fcbe87..3ba9ae83bff5a54a98d831c9cee416e06fa41a08 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -505,7 +505,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
 
 #ifdef CONFIG_SHMEM
 static int smaps_pte_hole(unsigned long addr, unsigned long end,
-               struct mm_walk *walk)
+                         __always_unused int depth, struct mm_walk *walk)
 {
        struct mem_size_stats *mss = walk->private;
 
@@ -1282,7 +1282,7 @@ static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
 }
 
 static int pagemap_pte_hole(unsigned long start, unsigned long end,
-                               struct mm_walk *walk)
+                           __always_unused int depth, struct mm_walk *walk)
 {
        struct pagemapread *pm = walk->private;
        unsigned long addr = start;

diff --git a/include/linux/pagewalk.h b/include/linux/pagewalk.h
index d5d07f7a9c1407c5fc9b359e7f89667b8dc62f6f..745a654c6ea7fd38d20514f94c8d99e6d247f363 100644 (file)
--- a/include/linux/pagewalk.h
+++ b/include/linux/pagewalk.h
@@ -17,7 +17,10 @@ struct mm_walk;
  *                     split_huge_page() instead of handling it explicitly.
  * @pte_entry:         if set, called for each non-empty PTE (lowest-level)
  *                     entry
- * @pte_hole:          if set, called for each hole at all levels
+ * @pte_hole:          if set, called for each hole at all levels,
+ *                     depth is -1 if not known, 0:PGD, 1:P4D, 2:PUD, 3:PMD
+ *                     4:PTE. Any folded depths (where PTRS_PER_P?D is equal
+ *                     to 1) are skipped.
  * @hugetlb_entry:     if set, called for each hugetlb entry
  * @test_walk:         caller specific callback function to determine whether
  *                     we walk over the current vma or not. Returning 0 means
@@ -43,7 +46,7 @@ struct mm_walk_ops {
        int (*pte_entry)(pte_t *pte, unsigned long addr,
                         unsigned long next, struct mm_walk *walk);
        int (*pte_hole)(unsigned long addr, unsigned long next,
-                       struct mm_walk *walk);
+                       int depth, struct mm_walk *walk);
        int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
                             unsigned long addr, unsigned long next,
                             struct mm_walk *walk);

diff --git a/mm/hmm.c b/mm/hmm.c
index c49e9dfce5b472faa9bac507a7298e0b26d1ab1a..72e5a6d9a41756c93697cb3160634dbcaed25f5f 100644 (file)
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -186,7 +186,7 @@ static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
 }
 
 static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
-                            struct mm_walk *walk)
+                            __always_unused int depth, struct mm_walk *walk)
 {
        struct hmm_vma_walk *hmm_vma_walk = walk->private;
        struct hmm_range *range = hmm_vma_walk->range;
@@ -380,7 +380,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
 again:
        pmd = READ_ONCE(*pmdp);
        if (pmd_none(pmd))
-               return hmm_vma_walk_hole(start, end, walk);
+               return hmm_vma_walk_hole(start, end, -1, walk);
 
        if (thp_migration_supported() && is_pmd_migration_entry(pmd)) {
                bool fault, write_fault;
@@ -487,7 +487,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
 
        pud = READ_ONCE(*pudp);
        if (pud_none(pud)) {
-               ret = hmm_vma_walk_hole(start, end, walk);
+               ret = hmm_vma_walk_hole(start, end, -1, walk);
                goto out_unlock;
        }
 
@@ -497,7 +497,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
                bool fault, write_fault;
 
                if (!pud_present(pud)) {
-                       ret = hmm_vma_walk_hole(start, end, walk);
+                       ret = hmm_vma_walk_hole(start, end, -1, walk);
                        goto out_unlock;
                }
 

diff --git a/mm/migrate.c b/mm/migrate.c
index edf42ed9003038dd064052065cd5322015b6c4b2..b1092876e537cdce83387ab2f6c0c002f4abc1bc 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2151,6 +2151,7 @@ out_unlock:
 #ifdef CONFIG_DEVICE_PRIVATE
 static int migrate_vma_collect_hole(unsigned long start,
                                    unsigned long end,
+                                   __always_unused int depth,
                                    struct mm_walk *walk)
 {
        struct migrate_vma *migrate = walk->private;
@@ -2195,7 +2196,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 
 again:
        if (pmd_none(*pmdp))
-               return migrate_vma_collect_hole(start, end, walk);
+               return migrate_vma_collect_hole(start, end, -1, walk);
 
        if (pmd_trans_huge(*pmdp)) {
                struct page *page;
@@ -2228,7 +2229,7 @@ again:
                                return migrate_vma_collect_skip(start, end,
                                                                walk);
                        if (pmd_none(*pmdp))
-                               return migrate_vma_collect_hole(start, end,
+                               return migrate_vma_collect_hole(start, end, -1,
                                                                walk);
                }
        }

diff --git a/mm/mincore.c b/mm/mincore.c
index 49b6fa2f6aa1d6ef84eddfddbbf9754ae7858893..0e6dd9948f1ae922cf81cbb4419a9bf0afcc70e6 100644 (file)
--- a/mm/mincore.c
+++ b/mm/mincore.c
@@ -112,6 +112,7 @@ static int __mincore_unmapped_range(unsigned long addr, unsigned long end,
 }
 
 static int mincore_unmapped_range(unsigned long addr, unsigned long end,
+                                  __always_unused int depth,
                                   struct mm_walk *walk)
 {
        walk->private += __mincore_unmapped_range(addr, end,

diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 6732fc7ac4c8cbb38a4ba2971e9355b39c6eb73a..5895ce4f1a8563eeffb71aa67f86addf58a9b74f 100644 (file)
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -4,6 +4,22 @@
 #include <linux/sched.h>
 #include <linux/hugetlb.h>
 
+/*
+ * We want to know the real level where a entry is located ignoring any
+ * folding of levels which may be happening. For example if p4d is folded then
+ * a missing entry found at level 1 (p4d) is actually at level 0 (pgd).
+ */
+static int real_depth(int depth)
+{
+       if (depth == 3 && PTRS_PER_PMD == 1)
+               depth = 2;
+       if (depth == 2 && PTRS_PER_PUD == 1)
+               depth = 1;
+       if (depth == 1 && PTRS_PER_P4D == 1)
+               depth = 0;
+       return depth;
+}
+
 static int walk_pte_range_inner(pte_t *pte, unsigned long addr,
                                unsigned long end, struct mm_walk *walk)
 {
@@ -49,6 +65,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
        unsigned long next;
        const struct mm_walk_ops *ops = walk->ops;
        int err = 0;
+       int depth = real_depth(3);
 
        pmd = pmd_offset(pud, addr);
        do {
@@ -56,7 +73,7 @@ again:
                next = pmd_addr_end(addr, end);
                if (pmd_none(*pmd) || (!walk->vma && !walk->no_vma)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, depth, walk);
                        if (err)
                                break;
                        continue;
@@ -106,6 +123,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
        unsigned long next;
        const struct mm_walk_ops *ops = walk->ops;
        int err = 0;
+       int depth = real_depth(2);
 
        pud = pud_offset(p4d, addr);
        do {
@@ -113,7 +131,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
                next = pud_addr_end(addr, end);
                if (pud_none(*pud) || (!walk->vma && !walk->no_vma)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, depth, walk);
                        if (err)
                                break;
                        continue;
@@ -154,13 +172,14 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
        unsigned long next;
        const struct mm_walk_ops *ops = walk->ops;
        int err = 0;
+       int depth = real_depth(1);
 
        p4d = p4d_offset(pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(p4d)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, depth, walk);
                        if (err)
                                break;
                        continue;
@@ -192,7 +211,7 @@ static int walk_pgd_range(unsigned long addr, unsigned long end,
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd)) {
                        if (ops->pte_hole)
-                               err = ops->pte_hole(addr, next, walk);
+                               err = ops->pte_hole(addr, next, 0, walk);
                        if (err)
                                break;
                        continue;
@@ -239,7 +258,7 @@ static int walk_hugetlb_range(unsigned long addr, unsigned long end,
                if (pte)
                        err = ops->hugetlb_entry(pte, hmask, addr, next, walk);
                else if (ops->pte_hole)
-                       err = ops->pte_hole(addr, next, walk);
+                       err = ops->pte_hole(addr, next, -1, walk);
 
                if (err)
                        break;
@@ -283,7 +302,7 @@ static int walk_page_test(unsigned long start, unsigned long end,
        if (vma->vm_flags & VM_PFNMAP) {
                int err = 1;
                if (ops->pte_hole)
-                       err = ops->pte_hole(start, end, walk);
+                       err = ops->pte_hole(start, end, -1, walk);
                return err ? err : 1;
        }
        return 0;