/*
* A default 3 level paging testing setup in software walked MMU
* MMUv4 (8K page): <4> : <7> : <8> : <13>
+ * A default 4 level paging testing setup in software walked MMU
+ * MMUv4 (8K page): <4> : <3> : <4> : <8> : <13>
*/
#define PGDIR_SHIFT 28
+#if CONFIG_PGTABLE_LEVELS > 3
+#define PUD_SHIFT 25
+#endif
#if CONFIG_PGTABLE_LEVELS > 2
#define PMD_SHIFT 21
#endif
#define PGDIR_MASK (~(PGDIR_SIZE - 1))
#define PTRS_PER_PGD BIT(32 - PGDIR_SHIFT)
+#if CONFIG_PGTABLE_LEVELS > 3
+#define PUD_SIZE BIT(PUD_SHIFT)
+#define PUD_MASK (~(PUD_SIZE - 1))
+#define PTRS_PER_PUD BIT(PGDIR_SHIFT - PUD_SHIFT)
+#endif
+
#if CONFIG_PGTABLE_LEVELS > 2
#define PMD_SIZE BIT(PMD_SHIFT)
#define PMD_MASK (~(PMD_SIZE - 1))
-#define PTRS_PER_PMD BIT(PGDIR_SHIFT - PMD_SHIFT)
+#define PTRS_PER_PMD BIT(PUD_SHIFT - PMD_SHIFT)
#endif
#define PTRS_PER_PTE BIT(PMD_SHIFT - PAGE_SHIFT)
#ifndef __ASSEMBLY__
-#if CONFIG_PGTABLE_LEVELS > 2
+#if CONFIG_PGTABLE_LEVELS > 3
+#include <asm-generic/pgtable-nop4d.h>
+#elif CONFIG_PGTABLE_LEVELS > 2
#include <asm-generic/pgtable-nopud.h>
#else
#include <asm-generic/pgtable-nopmd.h>
#define pgd_ERROR(e) \
pr_crit("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
+#if CONFIG_PGTABLE_LEVELS > 3
+
+/* In 4 level paging, p4d_* macros work on pgd */
+#define p4d_none(x) (!p4d_val(x))
+#define p4d_bad(x) ((p4d_val(x) & ~PAGE_MASK))
+#define p4d_present(x) (p4d_val(x))
+#define p4d_clear(xp) do { p4d_val(*(xp)) = 0; } while (0)
+#define p4d_pgtable(p4d) ((pud_t *)(p4d_val(p4d) & PAGE_MASK))
+#define p4d_page(p4d) virt_to_page(p4d_pgtable(p4d))
+#define set_p4d(p4dp, p4d) (*(p4dp) = p4d)
+
+/*
+ * 2nd level paging: pud
+ */
+#define pud_ERROR(e) \
+ pr_crit("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
+
+#endif
+
#if CONFIG_PGTABLE_LEVELS > 2
-/* In 3 level paging, pud_* macros work on pgd */
+/*
+ * In 3 level paging, pud_* macros work on pgd
+ * In 4 level paging, pud_* macros work on pud
+ */
#define pud_none(x) (!pud_val(x))
#define pud_bad(x) ((pud_val(x) & ~PAGE_MASK))
#define pud_present(x) (pud_val(x))
#define set_pud(pudp, pud) (*(pudp) = pud)
/*
- * 2nd level paging: pmd
+ * 3rd level paging: pmd
*/
#define pmd_ERROR(e) \
pr_crit("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
#define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd))
/*
- * 3rd level paging: pte
+ * 4th level paging: pte
*/
#define pte_ERROR(e) \
pr_crit("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
tst r3, r3
bz do_slow_path_pf ; if no Page Table, do page fault
+#if CONFIG_PGTABLE_LEVELS > 3
+ lsr r0, r2, PUD_SHIFT ; Bits for indexing into PUD
+ and r0, r0, (PTRS_PER_PUD - 1)
+ ld.as r1, [r3, r0] ; PMD entry
+ tst r1, r1
+ bz do_slow_path_pf
+ mov r3, r1
+#endif
+
#if CONFIG_PGTABLE_LEVELS > 2
lsr r0, r2, PMD_SHIFT ; Bits for indexing into PMD
and r0, r0, (PTRS_PER_PMD - 1)