mm: pagewalk: add p4d_entry() and pgd_entry()

[platform/kernel/linux-rpi.git] / mm / pagewalk.c

diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index d48c2a9..b36795d 100644 (file)
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -45,6 +45,9 @@ again:
                                break;
                        continue;
                }
+
+               walk->action = ACTION_SUBTREE;
+
                /*
                 * This implies that each ->pmd_entry() handler
                 * needs to know about pmd_trans_huge() pmds
@@ -54,16 +57,21 @@ again:
                if (err)
                        break;
 
+               if (walk->action == ACTION_AGAIN)
+                       goto again;
+
                /*
                 * Check this here so we only break down trans_huge
                 * pages when we _need_ to
                 */
-               if (!ops->pte_entry)
+               if (walk->action == ACTION_CONTINUE ||
+                   !(ops->pte_entry))
                        continue;
 
                split_huge_pmd(walk->vma, pmd, addr);
                if (pmd_trans_unstable(pmd))
                        goto again;
+
                err = walk_pte_range(pmd, addr, next, walk);
                if (err)
                        break;
@@ -92,24 +100,25 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
                        continue;
                }
 
-               if (ops->pud_entry) {
-                       spinlock_t *ptl = pud_trans_huge_lock(pud, walk->vma);
+               walk->action = ACTION_SUBTREE;
 
-                       if (ptl) {
-                               err = ops->pud_entry(pud, addr, next, walk);
-                               spin_unlock(ptl);
-                               if (err)
-                                       break;
-                               continue;
-                       }
-               }
+               if (ops->pud_entry)
+                       err = ops->pud_entry(pud, addr, next, walk);
+               if (err)
+                       break;
+
+               if (walk->action == ACTION_AGAIN)
+                       goto again;
+
+               if (walk->action == ACTION_CONTINUE ||
+                   !(ops->pmd_entry || ops->pte_entry))
+                       continue;
 
                split_huge_pud(walk->vma, pud, addr);
                if (pud_none(*pud))
                        goto again;
 
-               if (ops->pmd_entry || ops->pte_entry)
-                       err = walk_pmd_range(pud, addr, next, walk);
+               err = walk_pmd_range(pud, addr, next, walk);
                if (err)
                        break;
        } while (pud++, addr = next, addr != end);
@@ -135,7 +144,12 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
                                break;
                        continue;
                }
-               if (ops->pmd_entry || ops->pte_entry)
+               if (ops->p4d_entry) {
+                       err = ops->p4d_entry(p4d, addr, next, walk);
+                       if (err)
+                               break;
+               }
+               if (ops->pud_entry || ops->pmd_entry || ops->pte_entry)
                        err = walk_pud_range(p4d, addr, next, walk);
                if (err)
                        break;
@@ -162,7 +176,13 @@ static int walk_pgd_range(unsigned long addr, unsigned long end,
                                break;
                        continue;
                }
-               if (ops->pmd_entry || ops->pte_entry)
+               if (ops->pgd_entry) {
+                       err = ops->pgd_entry(pgd, addr, next, walk);
+                       if (err)
+                               break;
+               }
+               if (ops->p4d_entry || ops->pud_entry || ops->pmd_entry ||
+                   ops->pte_entry)
                        err = walk_p4d_range(pgd, addr, next, walk);
                if (err)
                        break;