Merge tag 'pci-v5.13-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/helgaa...
[platform/kernel/linux-starfive.git] / mm / debug_vm_pgtable.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * This kernel test validates architecture page table helpers and
4  * accessors and helps in verifying their continued compliance with
5  * expected generic MM semantics.
6  *
7  * Copyright (C) 2019 ARM Ltd.
8  *
9  * Author: Anshuman Khandual <anshuman.khandual@arm.com>
10  */
11 #define pr_fmt(fmt) "debug_vm_pgtable: [%-25s]: " fmt, __func__
12
13 #include <linux/gfp.h>
14 #include <linux/highmem.h>
15 #include <linux/hugetlb.h>
16 #include <linux/kernel.h>
17 #include <linux/kconfig.h>
18 #include <linux/mm.h>
19 #include <linux/mman.h>
20 #include <linux/mm_types.h>
21 #include <linux/module.h>
22 #include <linux/pfn_t.h>
23 #include <linux/printk.h>
24 #include <linux/pgtable.h>
25 #include <linux/random.h>
26 #include <linux/spinlock.h>
27 #include <linux/swap.h>
28 #include <linux/swapops.h>
29 #include <linux/start_kernel.h>
30 #include <linux/sched/mm.h>
31 #include <linux/io.h>
32 #include <asm/pgalloc.h>
33 #include <asm/tlbflush.h>
34
35 /*
36  * Please refer Documentation/vm/arch_pgtable_helpers.rst for the semantics
37  * expectations that are being validated here. All future changes in here
38  * or the documentation need to be in sync.
39  */
40
41 #define VMFLAGS (VM_READ|VM_WRITE|VM_EXEC)
42
43 /*
44  * On s390 platform, the lower 4 bits are used to identify given page table
45  * entry type. But these bits might affect the ability to clear entries with
46  * pxx_clear() because of how dynamic page table folding works on s390. So
47  * while loading up the entries do not change the lower 4 bits. It does not
48  * have affect any other platform. Also avoid the 62nd bit on ppc64 that is
49  * used to mark a pte entry.
50  */
51 #define S390_SKIP_MASK          GENMASK(3, 0)
52 #if __BITS_PER_LONG == 64
53 #define PPC64_SKIP_MASK         GENMASK(62, 62)
54 #else
55 #define PPC64_SKIP_MASK         0x0
56 #endif
57 #define ARCH_SKIP_MASK (S390_SKIP_MASK | PPC64_SKIP_MASK)
58 #define RANDOM_ORVALUE (GENMASK(BITS_PER_LONG - 1, 0) & ~ARCH_SKIP_MASK)
59 #define RANDOM_NZVALUE  GENMASK(7, 0)
60
61 static void __init pte_basic_tests(unsigned long pfn, int idx)
62 {
63         pgprot_t prot = protection_map[idx];
64         pte_t pte = pfn_pte(pfn, prot);
65         unsigned long val = idx, *ptr = &val;
66
67         pr_debug("Validating PTE basic (%pGv)\n", ptr);
68
69         /*
70          * This test needs to be executed after the given page table entry
71          * is created with pfn_pte() to make sure that protection_map[idx]
72          * does not have the dirty bit enabled from the beginning. This is
73          * important for platforms like arm64 where (!PTE_RDONLY) indicate
74          * dirty bit being set.
75          */
76         WARN_ON(pte_dirty(pte_wrprotect(pte)));
77
78         WARN_ON(!pte_same(pte, pte));
79         WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte))));
80         WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte))));
81         WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte))));
82         WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte))));
83         WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte))));
84         WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte))));
85         WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte))));
86         WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte))));
87 }
88
89 static void __init pte_advanced_tests(struct mm_struct *mm,
90                                       struct vm_area_struct *vma, pte_t *ptep,
91                                       unsigned long pfn, unsigned long vaddr,
92                                       pgprot_t prot)
93 {
94         pte_t pte = pfn_pte(pfn, prot);
95
96         /*
97          * Architectures optimize set_pte_at by avoiding TLB flush.
98          * This requires set_pte_at to be not used to update an
99          * existing pte entry. Clear pte before we do set_pte_at
100          */
101
102         pr_debug("Validating PTE advanced\n");
103         pte = pfn_pte(pfn, prot);
104         set_pte_at(mm, vaddr, ptep, pte);
105         ptep_set_wrprotect(mm, vaddr, ptep);
106         pte = ptep_get(ptep);
107         WARN_ON(pte_write(pte));
108         ptep_get_and_clear(mm, vaddr, ptep);
109         pte = ptep_get(ptep);
110         WARN_ON(!pte_none(pte));
111
112         pte = pfn_pte(pfn, prot);
113         pte = pte_wrprotect(pte);
114         pte = pte_mkclean(pte);
115         set_pte_at(mm, vaddr, ptep, pte);
116         pte = pte_mkwrite(pte);
117         pte = pte_mkdirty(pte);
118         ptep_set_access_flags(vma, vaddr, ptep, pte, 1);
119         pte = ptep_get(ptep);
120         WARN_ON(!(pte_write(pte) && pte_dirty(pte)));
121         ptep_get_and_clear_full(mm, vaddr, ptep, 1);
122         pte = ptep_get(ptep);
123         WARN_ON(!pte_none(pte));
124
125         pte = pfn_pte(pfn, prot);
126         pte = pte_mkyoung(pte);
127         set_pte_at(mm, vaddr, ptep, pte);
128         ptep_test_and_clear_young(vma, vaddr, ptep);
129         pte = ptep_get(ptep);
130         WARN_ON(pte_young(pte));
131 }
132
133 static void __init pte_savedwrite_tests(unsigned long pfn, pgprot_t prot)
134 {
135         pte_t pte = pfn_pte(pfn, prot);
136
137         if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
138                 return;
139
140         pr_debug("Validating PTE saved write\n");
141         WARN_ON(!pte_savedwrite(pte_mk_savedwrite(pte_clear_savedwrite(pte))));
142         WARN_ON(pte_savedwrite(pte_clear_savedwrite(pte_mk_savedwrite(pte))));
143 }
144
145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
146 static void __init pmd_basic_tests(unsigned long pfn, int idx)
147 {
148         pgprot_t prot = protection_map[idx];
149         pmd_t pmd = pfn_pmd(pfn, prot);
150         unsigned long val = idx, *ptr = &val;
151
152         if (!has_transparent_hugepage())
153                 return;
154
155         pr_debug("Validating PMD basic (%pGv)\n", ptr);
156
157         /*
158          * This test needs to be executed after the given page table entry
159          * is created with pfn_pmd() to make sure that protection_map[idx]
160          * does not have the dirty bit enabled from the beginning. This is
161          * important for platforms like arm64 where (!PTE_RDONLY) indicate
162          * dirty bit being set.
163          */
164         WARN_ON(pmd_dirty(pmd_wrprotect(pmd)));
165
166
167         WARN_ON(!pmd_same(pmd, pmd));
168         WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd))));
169         WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd))));
170         WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd))));
171         WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd))));
172         WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd))));
173         WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd))));
174         WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd))));
175         WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd))));
176         /*
177          * A huge page does not point to next level page table
178          * entry. Hence this must qualify as pmd_bad().
179          */
180         WARN_ON(!pmd_bad(pmd_mkhuge(pmd)));
181 }
182
183 static void __init pmd_advanced_tests(struct mm_struct *mm,
184                                       struct vm_area_struct *vma, pmd_t *pmdp,
185                                       unsigned long pfn, unsigned long vaddr,
186                                       pgprot_t prot, pgtable_t pgtable)
187 {
188         pmd_t pmd = pfn_pmd(pfn, prot);
189
190         if (!has_transparent_hugepage())
191                 return;
192
193         pr_debug("Validating PMD advanced\n");
194         /* Align the address wrt HPAGE_PMD_SIZE */
195         vaddr = (vaddr & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE;
196
197         pgtable_trans_huge_deposit(mm, pmdp, pgtable);
198
199         pmd = pfn_pmd(pfn, prot);
200         set_pmd_at(mm, vaddr, pmdp, pmd);
201         pmdp_set_wrprotect(mm, vaddr, pmdp);
202         pmd = READ_ONCE(*pmdp);
203         WARN_ON(pmd_write(pmd));
204         pmdp_huge_get_and_clear(mm, vaddr, pmdp);
205         pmd = READ_ONCE(*pmdp);
206         WARN_ON(!pmd_none(pmd));
207
208         pmd = pfn_pmd(pfn, prot);
209         pmd = pmd_wrprotect(pmd);
210         pmd = pmd_mkclean(pmd);
211         set_pmd_at(mm, vaddr, pmdp, pmd);
212         pmd = pmd_mkwrite(pmd);
213         pmd = pmd_mkdirty(pmd);
214         pmdp_set_access_flags(vma, vaddr, pmdp, pmd, 1);
215         pmd = READ_ONCE(*pmdp);
216         WARN_ON(!(pmd_write(pmd) && pmd_dirty(pmd)));
217         pmdp_huge_get_and_clear_full(vma, vaddr, pmdp, 1);
218         pmd = READ_ONCE(*pmdp);
219         WARN_ON(!pmd_none(pmd));
220
221         pmd = pmd_mkhuge(pfn_pmd(pfn, prot));
222         pmd = pmd_mkyoung(pmd);
223         set_pmd_at(mm, vaddr, pmdp, pmd);
224         pmdp_test_and_clear_young(vma, vaddr, pmdp);
225         pmd = READ_ONCE(*pmdp);
226         WARN_ON(pmd_young(pmd));
227
228         /*  Clear the pte entries  */
229         pmdp_huge_get_and_clear(mm, vaddr, pmdp);
230         pgtable = pgtable_trans_huge_withdraw(mm, pmdp);
231 }
232
233 static void __init pmd_leaf_tests(unsigned long pfn, pgprot_t prot)
234 {
235         pmd_t pmd = pfn_pmd(pfn, prot);
236
237         pr_debug("Validating PMD leaf\n");
238         /*
239          * PMD based THP is a leaf entry.
240          */
241         pmd = pmd_mkhuge(pmd);
242         WARN_ON(!pmd_leaf(pmd));
243 }
244
245 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
246 static void __init pmd_huge_tests(pmd_t *pmdp, unsigned long pfn, pgprot_t prot)
247 {
248         pmd_t pmd;
249
250         if (!arch_vmap_pmd_supported(prot))
251                 return;
252
253         pr_debug("Validating PMD huge\n");
254         /*
255          * X86 defined pmd_set_huge() verifies that the given
256          * PMD is not a populated non-leaf entry.
257          */
258         WRITE_ONCE(*pmdp, __pmd(0));
259         WARN_ON(!pmd_set_huge(pmdp, __pfn_to_phys(pfn), prot));
260         WARN_ON(!pmd_clear_huge(pmdp));
261         pmd = READ_ONCE(*pmdp);
262         WARN_ON(!pmd_none(pmd));
263 }
264 #else /* CONFIG_HAVE_ARCH_HUGE_VMAP */
265 static void __init pmd_huge_tests(pmd_t *pmdp, unsigned long pfn, pgprot_t prot) { }
266 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
267
268 static void __init pmd_savedwrite_tests(unsigned long pfn, pgprot_t prot)
269 {
270         pmd_t pmd = pfn_pmd(pfn, prot);
271
272         if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
273                 return;
274
275         pr_debug("Validating PMD saved write\n");
276         WARN_ON(!pmd_savedwrite(pmd_mk_savedwrite(pmd_clear_savedwrite(pmd))));
277         WARN_ON(pmd_savedwrite(pmd_clear_savedwrite(pmd_mk_savedwrite(pmd))));
278 }
279
280 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
281 static void __init pud_basic_tests(struct mm_struct *mm, unsigned long pfn, int idx)
282 {
283         pgprot_t prot = protection_map[idx];
284         pud_t pud = pfn_pud(pfn, prot);
285         unsigned long val = idx, *ptr = &val;
286
287         if (!has_transparent_hugepage())
288                 return;
289
290         pr_debug("Validating PUD basic (%pGv)\n", ptr);
291
292         /*
293          * This test needs to be executed after the given page table entry
294          * is created with pfn_pud() to make sure that protection_map[idx]
295          * does not have the dirty bit enabled from the beginning. This is
296          * important for platforms like arm64 where (!PTE_RDONLY) indicate
297          * dirty bit being set.
298          */
299         WARN_ON(pud_dirty(pud_wrprotect(pud)));
300
301         WARN_ON(!pud_same(pud, pud));
302         WARN_ON(!pud_young(pud_mkyoung(pud_mkold(pud))));
303         WARN_ON(!pud_dirty(pud_mkdirty(pud_mkclean(pud))));
304         WARN_ON(pud_dirty(pud_mkclean(pud_mkdirty(pud))));
305         WARN_ON(!pud_write(pud_mkwrite(pud_wrprotect(pud))));
306         WARN_ON(pud_write(pud_wrprotect(pud_mkwrite(pud))));
307         WARN_ON(pud_young(pud_mkold(pud_mkyoung(pud))));
308         WARN_ON(pud_dirty(pud_wrprotect(pud_mkclean(pud))));
309         WARN_ON(!pud_dirty(pud_wrprotect(pud_mkdirty(pud))));
310
311         if (mm_pmd_folded(mm))
312                 return;
313
314         /*
315          * A huge page does not point to next level page table
316          * entry. Hence this must qualify as pud_bad().
317          */
318         WARN_ON(!pud_bad(pud_mkhuge(pud)));
319 }
320
321 static void __init pud_advanced_tests(struct mm_struct *mm,
322                                       struct vm_area_struct *vma, pud_t *pudp,
323                                       unsigned long pfn, unsigned long vaddr,
324                                       pgprot_t prot)
325 {
326         pud_t pud = pfn_pud(pfn, prot);
327
328         if (!has_transparent_hugepage())
329                 return;
330
331         pr_debug("Validating PUD advanced\n");
332         /* Align the address wrt HPAGE_PUD_SIZE */
333         vaddr = (vaddr & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE;
334
335         set_pud_at(mm, vaddr, pudp, pud);
336         pudp_set_wrprotect(mm, vaddr, pudp);
337         pud = READ_ONCE(*pudp);
338         WARN_ON(pud_write(pud));
339
340 #ifndef __PAGETABLE_PMD_FOLDED
341         pudp_huge_get_and_clear(mm, vaddr, pudp);
342         pud = READ_ONCE(*pudp);
343         WARN_ON(!pud_none(pud));
344 #endif /* __PAGETABLE_PMD_FOLDED */
345         pud = pfn_pud(pfn, prot);
346         pud = pud_wrprotect(pud);
347         pud = pud_mkclean(pud);
348         set_pud_at(mm, vaddr, pudp, pud);
349         pud = pud_mkwrite(pud);
350         pud = pud_mkdirty(pud);
351         pudp_set_access_flags(vma, vaddr, pudp, pud, 1);
352         pud = READ_ONCE(*pudp);
353         WARN_ON(!(pud_write(pud) && pud_dirty(pud)));
354
355 #ifndef __PAGETABLE_PMD_FOLDED
356         pudp_huge_get_and_clear_full(mm, vaddr, pudp, 1);
357         pud = READ_ONCE(*pudp);
358         WARN_ON(!pud_none(pud));
359 #endif /* __PAGETABLE_PMD_FOLDED */
360
361         pud = pfn_pud(pfn, prot);
362         pud = pud_mkyoung(pud);
363         set_pud_at(mm, vaddr, pudp, pud);
364         pudp_test_and_clear_young(vma, vaddr, pudp);
365         pud = READ_ONCE(*pudp);
366         WARN_ON(pud_young(pud));
367
368         pudp_huge_get_and_clear(mm, vaddr, pudp);
369 }
370
371 static void __init pud_leaf_tests(unsigned long pfn, pgprot_t prot)
372 {
373         pud_t pud = pfn_pud(pfn, prot);
374
375         pr_debug("Validating PUD leaf\n");
376         /*
377          * PUD based THP is a leaf entry.
378          */
379         pud = pud_mkhuge(pud);
380         WARN_ON(!pud_leaf(pud));
381 }
382
383 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
384 static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot)
385 {
386         pud_t pud;
387
388         if (!arch_vmap_pud_supported(prot))
389                 return;
390
391         pr_debug("Validating PUD huge\n");
392         /*
393          * X86 defined pud_set_huge() verifies that the given
394          * PUD is not a populated non-leaf entry.
395          */
396         WRITE_ONCE(*pudp, __pud(0));
397         WARN_ON(!pud_set_huge(pudp, __pfn_to_phys(pfn), prot));
398         WARN_ON(!pud_clear_huge(pudp));
399         pud = READ_ONCE(*pudp);
400         WARN_ON(!pud_none(pud));
401 }
402 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
403 static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot) { }
404 #endif /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
405
406 #else  /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
407 static void __init pud_basic_tests(struct mm_struct *mm, unsigned long pfn, int idx) { }
408 static void __init pud_advanced_tests(struct mm_struct *mm,
409                                       struct vm_area_struct *vma, pud_t *pudp,
410                                       unsigned long pfn, unsigned long vaddr,
411                                       pgprot_t prot)
412 {
413 }
414 static void __init pud_leaf_tests(unsigned long pfn, pgprot_t prot) { }
415 static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot)
416 {
417 }
418 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
419 #else  /* !CONFIG_TRANSPARENT_HUGEPAGE */
420 static void __init pmd_basic_tests(unsigned long pfn, int idx) { }
421 static void __init pud_basic_tests(struct mm_struct *mm, unsigned long pfn, int idx) { }
422 static void __init pmd_advanced_tests(struct mm_struct *mm,
423                                       struct vm_area_struct *vma, pmd_t *pmdp,
424                                       unsigned long pfn, unsigned long vaddr,
425                                       pgprot_t prot, pgtable_t pgtable)
426 {
427 }
428 static void __init pud_advanced_tests(struct mm_struct *mm,
429                                       struct vm_area_struct *vma, pud_t *pudp,
430                                       unsigned long pfn, unsigned long vaddr,
431                                       pgprot_t prot)
432 {
433 }
434 static void __init pmd_leaf_tests(unsigned long pfn, pgprot_t prot) { }
435 static void __init pud_leaf_tests(unsigned long pfn, pgprot_t prot) { }
436 static void __init pmd_huge_tests(pmd_t *pmdp, unsigned long pfn, pgprot_t prot)
437 {
438 }
439 static void __init pud_huge_tests(pud_t *pudp, unsigned long pfn, pgprot_t prot)
440 {
441 }
442 static void __init pmd_savedwrite_tests(unsigned long pfn, pgprot_t prot) { }
443 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
444
445 static void __init p4d_basic_tests(unsigned long pfn, pgprot_t prot)
446 {
447         p4d_t p4d;
448
449         pr_debug("Validating P4D basic\n");
450         memset(&p4d, RANDOM_NZVALUE, sizeof(p4d_t));
451         WARN_ON(!p4d_same(p4d, p4d));
452 }
453
454 static void __init pgd_basic_tests(unsigned long pfn, pgprot_t prot)
455 {
456         pgd_t pgd;
457
458         pr_debug("Validating PGD basic\n");
459         memset(&pgd, RANDOM_NZVALUE, sizeof(pgd_t));
460         WARN_ON(!pgd_same(pgd, pgd));
461 }
462
463 #ifndef __PAGETABLE_PUD_FOLDED
464 static void __init pud_clear_tests(struct mm_struct *mm, pud_t *pudp)
465 {
466         pud_t pud = READ_ONCE(*pudp);
467
468         if (mm_pmd_folded(mm))
469                 return;
470
471         pr_debug("Validating PUD clear\n");
472         pud = __pud(pud_val(pud) | RANDOM_ORVALUE);
473         WRITE_ONCE(*pudp, pud);
474         pud_clear(pudp);
475         pud = READ_ONCE(*pudp);
476         WARN_ON(!pud_none(pud));
477 }
478
479 static void __init pud_populate_tests(struct mm_struct *mm, pud_t *pudp,
480                                       pmd_t *pmdp)
481 {
482         pud_t pud;
483
484         if (mm_pmd_folded(mm))
485                 return;
486
487         pr_debug("Validating PUD populate\n");
488         /*
489          * This entry points to next level page table page.
490          * Hence this must not qualify as pud_bad().
491          */
492         pud_populate(mm, pudp, pmdp);
493         pud = READ_ONCE(*pudp);
494         WARN_ON(pud_bad(pud));
495 }
496 #else  /* !__PAGETABLE_PUD_FOLDED */
497 static void __init pud_clear_tests(struct mm_struct *mm, pud_t *pudp) { }
498 static void __init pud_populate_tests(struct mm_struct *mm, pud_t *pudp,
499                                       pmd_t *pmdp)
500 {
501 }
502 #endif /* PAGETABLE_PUD_FOLDED */
503
504 #ifndef __PAGETABLE_P4D_FOLDED
505 static void __init p4d_clear_tests(struct mm_struct *mm, p4d_t *p4dp)
506 {
507         p4d_t p4d = READ_ONCE(*p4dp);
508
509         if (mm_pud_folded(mm))
510                 return;
511
512         pr_debug("Validating P4D clear\n");
513         p4d = __p4d(p4d_val(p4d) | RANDOM_ORVALUE);
514         WRITE_ONCE(*p4dp, p4d);
515         p4d_clear(p4dp);
516         p4d = READ_ONCE(*p4dp);
517         WARN_ON(!p4d_none(p4d));
518 }
519
520 static void __init p4d_populate_tests(struct mm_struct *mm, p4d_t *p4dp,
521                                       pud_t *pudp)
522 {
523         p4d_t p4d;
524
525         if (mm_pud_folded(mm))
526                 return;
527
528         pr_debug("Validating P4D populate\n");
529         /*
530          * This entry points to next level page table page.
531          * Hence this must not qualify as p4d_bad().
532          */
533         pud_clear(pudp);
534         p4d_clear(p4dp);
535         p4d_populate(mm, p4dp, pudp);
536         p4d = READ_ONCE(*p4dp);
537         WARN_ON(p4d_bad(p4d));
538 }
539
540 static void __init pgd_clear_tests(struct mm_struct *mm, pgd_t *pgdp)
541 {
542         pgd_t pgd = READ_ONCE(*pgdp);
543
544         if (mm_p4d_folded(mm))
545                 return;
546
547         pr_debug("Validating PGD clear\n");
548         pgd = __pgd(pgd_val(pgd) | RANDOM_ORVALUE);
549         WRITE_ONCE(*pgdp, pgd);
550         pgd_clear(pgdp);
551         pgd = READ_ONCE(*pgdp);
552         WARN_ON(!pgd_none(pgd));
553 }
554
555 static void __init pgd_populate_tests(struct mm_struct *mm, pgd_t *pgdp,
556                                       p4d_t *p4dp)
557 {
558         pgd_t pgd;
559
560         if (mm_p4d_folded(mm))
561                 return;
562
563         pr_debug("Validating PGD populate\n");
564         /*
565          * This entry points to next level page table page.
566          * Hence this must not qualify as pgd_bad().
567          */
568         p4d_clear(p4dp);
569         pgd_clear(pgdp);
570         pgd_populate(mm, pgdp, p4dp);
571         pgd = READ_ONCE(*pgdp);
572         WARN_ON(pgd_bad(pgd));
573 }
574 #else  /* !__PAGETABLE_P4D_FOLDED */
575 static void __init p4d_clear_tests(struct mm_struct *mm, p4d_t *p4dp) { }
576 static void __init pgd_clear_tests(struct mm_struct *mm, pgd_t *pgdp) { }
577 static void __init p4d_populate_tests(struct mm_struct *mm, p4d_t *p4dp,
578                                       pud_t *pudp)
579 {
580 }
581 static void __init pgd_populate_tests(struct mm_struct *mm, pgd_t *pgdp,
582                                       p4d_t *p4dp)
583 {
584 }
585 #endif /* PAGETABLE_P4D_FOLDED */
586
587 static void __init pte_clear_tests(struct mm_struct *mm, pte_t *ptep,
588                                    unsigned long pfn, unsigned long vaddr,
589                                    pgprot_t prot)
590 {
591         pte_t pte = pfn_pte(pfn, prot);
592
593         pr_debug("Validating PTE clear\n");
594 #ifndef CONFIG_RISCV
595         pte = __pte(pte_val(pte) | RANDOM_ORVALUE);
596 #endif
597         set_pte_at(mm, vaddr, ptep, pte);
598         barrier();
599         pte_clear(mm, vaddr, ptep);
600         pte = ptep_get(ptep);
601         WARN_ON(!pte_none(pte));
602 }
603
604 static void __init pmd_clear_tests(struct mm_struct *mm, pmd_t *pmdp)
605 {
606         pmd_t pmd = READ_ONCE(*pmdp);
607
608         pr_debug("Validating PMD clear\n");
609         pmd = __pmd(pmd_val(pmd) | RANDOM_ORVALUE);
610         WRITE_ONCE(*pmdp, pmd);
611         pmd_clear(pmdp);
612         pmd = READ_ONCE(*pmdp);
613         WARN_ON(!pmd_none(pmd));
614 }
615
616 static void __init pmd_populate_tests(struct mm_struct *mm, pmd_t *pmdp,
617                                       pgtable_t pgtable)
618 {
619         pmd_t pmd;
620
621         pr_debug("Validating PMD populate\n");
622         /*
623          * This entry points to next level page table page.
624          * Hence this must not qualify as pmd_bad().
625          */
626         pmd_populate(mm, pmdp, pgtable);
627         pmd = READ_ONCE(*pmdp);
628         WARN_ON(pmd_bad(pmd));
629 }
630
631 static void __init pte_special_tests(unsigned long pfn, pgprot_t prot)
632 {
633         pte_t pte = pfn_pte(pfn, prot);
634
635         if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL))
636                 return;
637
638         pr_debug("Validating PTE special\n");
639         WARN_ON(!pte_special(pte_mkspecial(pte)));
640 }
641
642 static void __init pte_protnone_tests(unsigned long pfn, pgprot_t prot)
643 {
644         pte_t pte = pfn_pte(pfn, prot);
645
646         if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
647                 return;
648
649         pr_debug("Validating PTE protnone\n");
650         WARN_ON(!pte_protnone(pte));
651         WARN_ON(!pte_present(pte));
652 }
653
654 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
655 static void __init pmd_protnone_tests(unsigned long pfn, pgprot_t prot)
656 {
657         pmd_t pmd = pmd_mkhuge(pfn_pmd(pfn, prot));
658
659         if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
660                 return;
661
662         pr_debug("Validating PMD protnone\n");
663         WARN_ON(!pmd_protnone(pmd));
664         WARN_ON(!pmd_present(pmd));
665 }
666 #else  /* !CONFIG_TRANSPARENT_HUGEPAGE */
667 static void __init pmd_protnone_tests(unsigned long pfn, pgprot_t prot) { }
668 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
669
670 #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP
671 static void __init pte_devmap_tests(unsigned long pfn, pgprot_t prot)
672 {
673         pte_t pte = pfn_pte(pfn, prot);
674
675         pr_debug("Validating PTE devmap\n");
676         WARN_ON(!pte_devmap(pte_mkdevmap(pte)));
677 }
678
679 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
680 static void __init pmd_devmap_tests(unsigned long pfn, pgprot_t prot)
681 {
682         pmd_t pmd = pfn_pmd(pfn, prot);
683
684         pr_debug("Validating PMD devmap\n");
685         WARN_ON(!pmd_devmap(pmd_mkdevmap(pmd)));
686 }
687
688 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
689 static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot)
690 {
691         pud_t pud = pfn_pud(pfn, prot);
692
693         pr_debug("Validating PUD devmap\n");
694         WARN_ON(!pud_devmap(pud_mkdevmap(pud)));
695 }
696 #else  /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
697 static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot) { }
698 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
699 #else  /* CONFIG_TRANSPARENT_HUGEPAGE */
700 static void __init pmd_devmap_tests(unsigned long pfn, pgprot_t prot) { }
701 static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot) { }
702 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
703 #else
704 static void __init pte_devmap_tests(unsigned long pfn, pgprot_t prot) { }
705 static void __init pmd_devmap_tests(unsigned long pfn, pgprot_t prot) { }
706 static void __init pud_devmap_tests(unsigned long pfn, pgprot_t prot) { }
707 #endif /* CONFIG_ARCH_HAS_PTE_DEVMAP */
708
709 static void __init pte_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
710 {
711         pte_t pte = pfn_pte(pfn, prot);
712
713         if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
714                 return;
715
716         pr_debug("Validating PTE soft dirty\n");
717         WARN_ON(!pte_soft_dirty(pte_mksoft_dirty(pte)));
718         WARN_ON(pte_soft_dirty(pte_clear_soft_dirty(pte)));
719 }
720
721 static void __init pte_swap_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
722 {
723         pte_t pte = pfn_pte(pfn, prot);
724
725         if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
726                 return;
727
728         pr_debug("Validating PTE swap soft dirty\n");
729         WARN_ON(!pte_swp_soft_dirty(pte_swp_mksoft_dirty(pte)));
730         WARN_ON(pte_swp_soft_dirty(pte_swp_clear_soft_dirty(pte)));
731 }
732
733 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
734 static void __init pmd_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
735 {
736         pmd_t pmd = pfn_pmd(pfn, prot);
737
738         if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
739                 return;
740
741         pr_debug("Validating PMD soft dirty\n");
742         WARN_ON(!pmd_soft_dirty(pmd_mksoft_dirty(pmd)));
743         WARN_ON(pmd_soft_dirty(pmd_clear_soft_dirty(pmd)));
744 }
745
746 static void __init pmd_swap_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
747 {
748         pmd_t pmd = pfn_pmd(pfn, prot);
749
750         if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) ||
751                 !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION))
752                 return;
753
754         pr_debug("Validating PMD swap soft dirty\n");
755         WARN_ON(!pmd_swp_soft_dirty(pmd_swp_mksoft_dirty(pmd)));
756         WARN_ON(pmd_swp_soft_dirty(pmd_swp_clear_soft_dirty(pmd)));
757 }
758 #else  /* !CONFIG_ARCH_HAS_PTE_DEVMAP */
759 static void __init pmd_soft_dirty_tests(unsigned long pfn, pgprot_t prot) { }
760 static void __init pmd_swap_soft_dirty_tests(unsigned long pfn, pgprot_t prot)
761 {
762 }
763 #endif /* CONFIG_ARCH_HAS_PTE_DEVMAP */
764
765 static void __init pte_swap_tests(unsigned long pfn, pgprot_t prot)
766 {
767         swp_entry_t swp;
768         pte_t pte;
769
770         pr_debug("Validating PTE swap\n");
771         pte = pfn_pte(pfn, prot);
772         swp = __pte_to_swp_entry(pte);
773         pte = __swp_entry_to_pte(swp);
774         WARN_ON(pfn != pte_pfn(pte));
775 }
776
777 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
778 static void __init pmd_swap_tests(unsigned long pfn, pgprot_t prot)
779 {
780         swp_entry_t swp;
781         pmd_t pmd;
782
783         pr_debug("Validating PMD swap\n");
784         pmd = pfn_pmd(pfn, prot);
785         swp = __pmd_to_swp_entry(pmd);
786         pmd = __swp_entry_to_pmd(swp);
787         WARN_ON(pfn != pmd_pfn(pmd));
788 }
789 #else  /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */
790 static void __init pmd_swap_tests(unsigned long pfn, pgprot_t prot) { }
791 #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
792
793 static void __init swap_migration_tests(void)
794 {
795         struct page *page;
796         swp_entry_t swp;
797
798         if (!IS_ENABLED(CONFIG_MIGRATION))
799                 return;
800
801         pr_debug("Validating swap migration\n");
802         /*
803          * swap_migration_tests() requires a dedicated page as it needs to
804          * be locked before creating a migration entry from it. Locking the
805          * page that actually maps kernel text ('start_kernel') can be real
806          * problematic. Lets allocate a dedicated page explicitly for this
807          * purpose that will be freed subsequently.
808          */
809         page = alloc_page(GFP_KERNEL);
810         if (!page) {
811                 pr_err("page allocation failed\n");
812                 return;
813         }
814
815         /*
816          * make_migration_entry() expects given page to be
817          * locked, otherwise it stumbles upon a BUG_ON().
818          */
819         __SetPageLocked(page);
820         swp = make_migration_entry(page, 1);
821         WARN_ON(!is_migration_entry(swp));
822         WARN_ON(!is_write_migration_entry(swp));
823
824         make_migration_entry_read(&swp);
825         WARN_ON(!is_migration_entry(swp));
826         WARN_ON(is_write_migration_entry(swp));
827
828         swp = make_migration_entry(page, 0);
829         WARN_ON(!is_migration_entry(swp));
830         WARN_ON(is_write_migration_entry(swp));
831         __ClearPageLocked(page);
832         __free_page(page);
833 }
834
835 #ifdef CONFIG_HUGETLB_PAGE
836 static void __init hugetlb_basic_tests(unsigned long pfn, pgprot_t prot)
837 {
838         struct page *page;
839         pte_t pte;
840
841         pr_debug("Validating HugeTLB basic\n");
842         /*
843          * Accessing the page associated with the pfn is safe here,
844          * as it was previously derived from a real kernel symbol.
845          */
846         page = pfn_to_page(pfn);
847         pte = mk_huge_pte(page, prot);
848
849         WARN_ON(!huge_pte_dirty(huge_pte_mkdirty(pte)));
850         WARN_ON(!huge_pte_write(huge_pte_mkwrite(huge_pte_wrprotect(pte))));
851         WARN_ON(huge_pte_write(huge_pte_wrprotect(huge_pte_mkwrite(pte))));
852
853 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
854         pte = pfn_pte(pfn, prot);
855
856         WARN_ON(!pte_huge(pte_mkhuge(pte)));
857 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
858 }
859 #else  /* !CONFIG_HUGETLB_PAGE */
860 static void __init hugetlb_basic_tests(unsigned long pfn, pgprot_t prot) { }
861 #endif /* CONFIG_HUGETLB_PAGE */
862
863 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
864 static void __init pmd_thp_tests(unsigned long pfn, pgprot_t prot)
865 {
866         pmd_t pmd;
867
868         if (!has_transparent_hugepage())
869                 return;
870
871         pr_debug("Validating PMD based THP\n");
872         /*
873          * pmd_trans_huge() and pmd_present() must return positive after
874          * MMU invalidation with pmd_mkinvalid(). This behavior is an
875          * optimization for transparent huge page. pmd_trans_huge() must
876          * be true if pmd_page() returns a valid THP to avoid taking the
877          * pmd_lock when others walk over non transhuge pmds (i.e. there
878          * are no THP allocated). Especially when splitting a THP and
879          * removing the present bit from the pmd, pmd_trans_huge() still
880          * needs to return true. pmd_present() should be true whenever
881          * pmd_trans_huge() returns true.
882          */
883         pmd = pfn_pmd(pfn, prot);
884         WARN_ON(!pmd_trans_huge(pmd_mkhuge(pmd)));
885
886 #ifndef __HAVE_ARCH_PMDP_INVALIDATE
887         WARN_ON(!pmd_trans_huge(pmd_mkinvalid(pmd_mkhuge(pmd))));
888         WARN_ON(!pmd_present(pmd_mkinvalid(pmd_mkhuge(pmd))));
889 #endif /* __HAVE_ARCH_PMDP_INVALIDATE */
890 }
891
892 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
893 static void __init pud_thp_tests(unsigned long pfn, pgprot_t prot)
894 {
895         pud_t pud;
896
897         if (!has_transparent_hugepage())
898                 return;
899
900         pr_debug("Validating PUD based THP\n");
901         pud = pfn_pud(pfn, prot);
902         WARN_ON(!pud_trans_huge(pud_mkhuge(pud)));
903
904         /*
905          * pud_mkinvalid() has been dropped for now. Enable back
906          * these tests when it comes back with a modified pud_present().
907          *
908          * WARN_ON(!pud_trans_huge(pud_mkinvalid(pud_mkhuge(pud))));
909          * WARN_ON(!pud_present(pud_mkinvalid(pud_mkhuge(pud))));
910          */
911 }
912 #else  /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
913 static void __init pud_thp_tests(unsigned long pfn, pgprot_t prot) { }
914 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
915 #else  /* !CONFIG_TRANSPARENT_HUGEPAGE */
916 static void __init pmd_thp_tests(unsigned long pfn, pgprot_t prot) { }
917 static void __init pud_thp_tests(unsigned long pfn, pgprot_t prot) { }
918 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
919
920 static unsigned long __init get_random_vaddr(void)
921 {
922         unsigned long random_vaddr, random_pages, total_user_pages;
923
924         total_user_pages = (TASK_SIZE - FIRST_USER_ADDRESS) / PAGE_SIZE;
925
926         random_pages = get_random_long() % total_user_pages;
927         random_vaddr = FIRST_USER_ADDRESS + random_pages * PAGE_SIZE;
928
929         return random_vaddr;
930 }
931
932 static int __init debug_vm_pgtable(void)
933 {
934         struct vm_area_struct *vma;
935         struct mm_struct *mm;
936         pgd_t *pgdp;
937         p4d_t *p4dp, *saved_p4dp;
938         pud_t *pudp, *saved_pudp;
939         pmd_t *pmdp, *saved_pmdp, pmd;
940         pte_t *ptep;
941         pgtable_t saved_ptep;
942         pgprot_t prot, protnone;
943         phys_addr_t paddr;
944         unsigned long vaddr, pte_aligned, pmd_aligned;
945         unsigned long pud_aligned, p4d_aligned, pgd_aligned;
946         spinlock_t *ptl = NULL;
947         int idx;
948
949         pr_info("Validating architecture page table helpers\n");
950         prot = vm_get_page_prot(VMFLAGS);
951         vaddr = get_random_vaddr();
952         mm = mm_alloc();
953         if (!mm) {
954                 pr_err("mm_struct allocation failed\n");
955                 return 1;
956         }
957
958         /*
959          * __P000 (or even __S000) will help create page table entries with
960          * PROT_NONE permission as required for pxx_protnone_tests().
961          */
962         protnone = __P000;
963
964         vma = vm_area_alloc(mm);
965         if (!vma) {
966                 pr_err("vma allocation failed\n");
967                 return 1;
968         }
969
970         /*
971          * PFN for mapping at PTE level is determined from a standard kernel
972          * text symbol. But pfns for higher page table levels are derived by
973          * masking lower bits of this real pfn. These derived pfns might not
974          * exist on the platform but that does not really matter as pfn_pxx()
975          * helpers will still create appropriate entries for the test. This
976          * helps avoid large memory block allocations to be used for mapping
977          * at higher page table levels.
978          */
979         paddr = __pa_symbol(&start_kernel);
980
981         pte_aligned = (paddr & PAGE_MASK) >> PAGE_SHIFT;
982         pmd_aligned = (paddr & PMD_MASK) >> PAGE_SHIFT;
983         pud_aligned = (paddr & PUD_MASK) >> PAGE_SHIFT;
984         p4d_aligned = (paddr & P4D_MASK) >> PAGE_SHIFT;
985         pgd_aligned = (paddr & PGDIR_MASK) >> PAGE_SHIFT;
986         WARN_ON(!pfn_valid(pte_aligned));
987
988         pgdp = pgd_offset(mm, vaddr);
989         p4dp = p4d_alloc(mm, pgdp, vaddr);
990         pudp = pud_alloc(mm, p4dp, vaddr);
991         pmdp = pmd_alloc(mm, pudp, vaddr);
992         /*
993          * Allocate pgtable_t
994          */
995         if (pte_alloc(mm, pmdp)) {
996                 pr_err("pgtable allocation failed\n");
997                 return 1;
998         }
999
1000         /*
1001          * Save all the page table page addresses as the page table
1002          * entries will be used for testing with random or garbage
1003          * values. These saved addresses will be used for freeing
1004          * page table pages.
1005          */
1006         pmd = READ_ONCE(*pmdp);
1007         saved_p4dp = p4d_offset(pgdp, 0UL);
1008         saved_pudp = pud_offset(p4dp, 0UL);
1009         saved_pmdp = pmd_offset(pudp, 0UL);
1010         saved_ptep = pmd_pgtable(pmd);
1011
1012         /*
1013          * Iterate over the protection_map[] to make sure that all
1014          * the basic page table transformation validations just hold
1015          * true irrespective of the starting protection value for a
1016          * given page table entry.
1017          */
1018         for (idx = 0; idx < ARRAY_SIZE(protection_map); idx++) {
1019                 pte_basic_tests(pte_aligned, idx);
1020                 pmd_basic_tests(pmd_aligned, idx);
1021                 pud_basic_tests(mm, pud_aligned, idx);
1022         }
1023
1024         /*
1025          * Both P4D and PGD level tests are very basic which do not
1026          * involve creating page table entries from the protection
1027          * value and the given pfn. Hence just keep them out from
1028          * the above iteration for now to save some test execution
1029          * time.
1030          */
1031         p4d_basic_tests(p4d_aligned, prot);
1032         pgd_basic_tests(pgd_aligned, prot);
1033
1034         pmd_leaf_tests(pmd_aligned, prot);
1035         pud_leaf_tests(pud_aligned, prot);
1036
1037         pte_savedwrite_tests(pte_aligned, protnone);
1038         pmd_savedwrite_tests(pmd_aligned, protnone);
1039
1040         pte_special_tests(pte_aligned, prot);
1041         pte_protnone_tests(pte_aligned, protnone);
1042         pmd_protnone_tests(pmd_aligned, protnone);
1043
1044         pte_devmap_tests(pte_aligned, prot);
1045         pmd_devmap_tests(pmd_aligned, prot);
1046         pud_devmap_tests(pud_aligned, prot);
1047
1048         pte_soft_dirty_tests(pte_aligned, prot);
1049         pmd_soft_dirty_tests(pmd_aligned, prot);
1050         pte_swap_soft_dirty_tests(pte_aligned, prot);
1051         pmd_swap_soft_dirty_tests(pmd_aligned, prot);
1052
1053         pte_swap_tests(pte_aligned, prot);
1054         pmd_swap_tests(pmd_aligned, prot);
1055
1056         swap_migration_tests();
1057
1058         pmd_thp_tests(pmd_aligned, prot);
1059         pud_thp_tests(pud_aligned, prot);
1060
1061         hugetlb_basic_tests(pte_aligned, prot);
1062
1063         /*
1064          * Page table modifying tests. They need to hold
1065          * proper page table lock.
1066          */
1067
1068         ptep = pte_offset_map_lock(mm, pmdp, vaddr, &ptl);
1069         pte_clear_tests(mm, ptep, pte_aligned, vaddr, prot);
1070         pte_advanced_tests(mm, vma, ptep, pte_aligned, vaddr, prot);
1071         pte_unmap_unlock(ptep, ptl);
1072
1073         ptl = pmd_lock(mm, pmdp);
1074         pmd_clear_tests(mm, pmdp);
1075         pmd_advanced_tests(mm, vma, pmdp, pmd_aligned, vaddr, prot, saved_ptep);
1076         pmd_huge_tests(pmdp, pmd_aligned, prot);
1077         pmd_populate_tests(mm, pmdp, saved_ptep);
1078         spin_unlock(ptl);
1079
1080         ptl = pud_lock(mm, pudp);
1081         pud_clear_tests(mm, pudp);
1082         pud_advanced_tests(mm, vma, pudp, pud_aligned, vaddr, prot);
1083         pud_huge_tests(pudp, pud_aligned, prot);
1084         pud_populate_tests(mm, pudp, saved_pmdp);
1085         spin_unlock(ptl);
1086
1087         spin_lock(&mm->page_table_lock);
1088         p4d_clear_tests(mm, p4dp);
1089         pgd_clear_tests(mm, pgdp);
1090         p4d_populate_tests(mm, p4dp, saved_pudp);
1091         pgd_populate_tests(mm, pgdp, saved_p4dp);
1092         spin_unlock(&mm->page_table_lock);
1093
1094         p4d_free(mm, saved_p4dp);
1095         pud_free(mm, saved_pudp);
1096         pmd_free(mm, saved_pmdp);
1097         pte_free(mm, saved_ptep);
1098
1099         vm_area_free(vma);
1100         mm_dec_nr_puds(mm);
1101         mm_dec_nr_pmds(mm);
1102         mm_dec_nr_ptes(mm);
1103         mmdrop(mm);
1104         return 0;
1105 }
1106 late_initcall(debug_vm_pgtable);