1 // SPDX-License-Identifier: GPL-2.0-only
3 * Based on arch/arm/mm/mmu.c
5 * Copyright (C) 1995-2005 Russell King
6 * Copyright (C) 2012 ARM Ltd.
9 #include <linux/cache.h>
10 #include <linux/export.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/init.h>
14 #include <linux/ioport.h>
15 #include <linux/kexec.h>
16 #include <linux/libfdt.h>
17 #include <linux/mman.h>
18 #include <linux/nodemask.h>
19 #include <linux/memblock.h>
20 #include <linux/memremap.h>
21 #include <linux/memory.h>
25 #include <linux/vmalloc.h>
26 #include <linux/set_memory.h>
27 #include <linux/kfence.h>
29 #include <asm/barrier.h>
30 #include <asm/cputype.h>
31 #include <asm/fixmap.h>
32 #include <asm/kasan.h>
33 #include <asm/kernel-pgtable.h>
34 #include <asm/sections.h>
35 #include <asm/setup.h>
36 #include <linux/sizes.h>
38 #include <asm/mmu_context.h>
39 #include <asm/ptdump.h>
40 #include <asm/tlbflush.h>
41 #include <asm/pgalloc.h>
42 #include <asm/kfence.h>
44 #define NO_BLOCK_MAPPINGS BIT(0)
45 #define NO_CONT_MAPPINGS BIT(1)
46 #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */
48 int idmap_t0sz __ro_after_init;
51 u64 vabits_actual __ro_after_init = VA_BITS_MIN;
52 EXPORT_SYMBOL(vabits_actual);
55 u64 kimage_vaddr __ro_after_init = (u64)&_text;
56 EXPORT_SYMBOL(kimage_vaddr);
58 u64 kimage_voffset __ro_after_init;
59 EXPORT_SYMBOL(kimage_voffset);
61 u32 __boot_cpu_mode[] = { BOOT_CPU_MODE_EL2, BOOT_CPU_MODE_EL1 };
64 * The booting CPU updates the failed status @__early_cpu_boot_status,
65 * with MMU turned off.
67 long __section(".mmuoff.data.write") __early_cpu_boot_status;
70 * Empty_zero_page is a special page that is used for zero-initialized data
73 unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
74 EXPORT_SYMBOL(empty_zero_page);
76 static DEFINE_SPINLOCK(swapper_pgdir_lock);
77 static DEFINE_MUTEX(fixmap_lock);
79 void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
83 spin_lock(&swapper_pgdir_lock);
84 fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp));
85 WRITE_ONCE(*fixmap_pgdp, pgd);
87 * We need dsb(ishst) here to ensure the page-table-walker sees
88 * our new entry before set_p?d() returns. The fixmap's
89 * flush_tlb_kernel_range() via clear_fixmap() does this for us.
92 spin_unlock(&swapper_pgdir_lock);
95 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
96 unsigned long size, pgprot_t vma_prot)
98 if (!pfn_is_map_memory(pfn))
99 return pgprot_noncached(vma_prot);
100 else if (file->f_flags & O_SYNC)
101 return pgprot_writecombine(vma_prot);
104 EXPORT_SYMBOL(phys_mem_access_prot);
106 static phys_addr_t __init early_pgtable_alloc(int shift)
111 phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0,
112 MEMBLOCK_ALLOC_NOLEAKTRACE);
114 panic("Failed to allocate page table page\n");
117 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
118 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
119 * any level of table.
121 ptr = pte_set_fixmap(phys);
123 memset(ptr, 0, PAGE_SIZE);
126 * Implicit barriers also ensure the zeroed page is visible to the page
134 bool pgattr_change_is_safe(u64 old, u64 new)
137 * The following mapping attributes may be updated in live
138 * kernel mappings without the need for break-before-make.
140 pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG;
142 /* creating or taking down mappings is always safe */
143 if (!pte_valid(__pte(old)) || !pte_valid(__pte(new)))
146 /* A live entry's pfn should not change */
147 if (pte_pfn(__pte(old)) != pte_pfn(__pte(new)))
150 /* live contiguous mappings may not be manipulated at all */
151 if ((old | new) & PTE_CONT)
154 /* Transitioning from Non-Global to Global is unsafe */
155 if (old & ~new & PTE_NG)
159 * Changing the memory type between Normal and Normal-Tagged is safe
160 * since Tagged is considered a permission attribute from the
161 * mismatched attribute aliases perspective.
163 if (((old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
164 (old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)) &&
165 ((new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
166 (new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)))
167 mask |= PTE_ATTRINDX_MASK;
169 return ((old ^ new) & ~mask) == 0;
172 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
173 phys_addr_t phys, pgprot_t prot)
177 ptep = pte_set_fixmap_offset(pmdp, addr);
179 pte_t old_pte = READ_ONCE(*ptep);
181 set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot));
184 * After the PTE entry has been populated once, we
185 * only allow updates to the permission attributes.
187 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte),
188 READ_ONCE(pte_val(*ptep))));
191 } while (ptep++, addr += PAGE_SIZE, addr != end);
196 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
197 unsigned long end, phys_addr_t phys,
199 phys_addr_t (*pgtable_alloc)(int),
203 pmd_t pmd = READ_ONCE(*pmdp);
205 BUG_ON(pmd_sect(pmd));
207 pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN;
208 phys_addr_t pte_phys;
210 if (flags & NO_EXEC_MAPPINGS)
211 pmdval |= PMD_TABLE_PXN;
212 BUG_ON(!pgtable_alloc);
213 pte_phys = pgtable_alloc(PAGE_SHIFT);
214 __pmd_populate(pmdp, pte_phys, pmdval);
215 pmd = READ_ONCE(*pmdp);
217 BUG_ON(pmd_bad(pmd));
220 pgprot_t __prot = prot;
222 next = pte_cont_addr_end(addr, end);
224 /* use a contiguous mapping if the range is suitably aligned */
225 if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
226 (flags & NO_CONT_MAPPINGS) == 0)
227 __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
229 init_pte(pmdp, addr, next, phys, __prot);
232 } while (addr = next, addr != end);
235 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
236 phys_addr_t phys, pgprot_t prot,
237 phys_addr_t (*pgtable_alloc)(int), int flags)
242 pmdp = pmd_set_fixmap_offset(pudp, addr);
244 pmd_t old_pmd = READ_ONCE(*pmdp);
246 next = pmd_addr_end(addr, end);
248 /* try section mapping first */
249 if (((addr | next | phys) & ~PMD_MASK) == 0 &&
250 (flags & NO_BLOCK_MAPPINGS) == 0) {
251 pmd_set_huge(pmdp, phys, prot);
254 * After the PMD entry has been populated once, we
255 * only allow updates to the permission attributes.
257 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
258 READ_ONCE(pmd_val(*pmdp))));
260 alloc_init_cont_pte(pmdp, addr, next, phys, prot,
261 pgtable_alloc, flags);
263 BUG_ON(pmd_val(old_pmd) != 0 &&
264 pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
267 } while (pmdp++, addr = next, addr != end);
272 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
273 unsigned long end, phys_addr_t phys,
275 phys_addr_t (*pgtable_alloc)(int), int flags)
278 pud_t pud = READ_ONCE(*pudp);
281 * Check for initial section mappings in the pgd/pud.
283 BUG_ON(pud_sect(pud));
285 pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN;
286 phys_addr_t pmd_phys;
288 if (flags & NO_EXEC_MAPPINGS)
289 pudval |= PUD_TABLE_PXN;
290 BUG_ON(!pgtable_alloc);
291 pmd_phys = pgtable_alloc(PMD_SHIFT);
292 __pud_populate(pudp, pmd_phys, pudval);
293 pud = READ_ONCE(*pudp);
295 BUG_ON(pud_bad(pud));
298 pgprot_t __prot = prot;
300 next = pmd_cont_addr_end(addr, end);
302 /* use a contiguous mapping if the range is suitably aligned */
303 if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
304 (flags & NO_CONT_MAPPINGS) == 0)
305 __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
307 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags);
310 } while (addr = next, addr != end);
313 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
314 phys_addr_t phys, pgprot_t prot,
315 phys_addr_t (*pgtable_alloc)(int),
320 p4d_t *p4dp = p4d_offset(pgdp, addr);
321 p4d_t p4d = READ_ONCE(*p4dp);
324 p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN;
325 phys_addr_t pud_phys;
327 if (flags & NO_EXEC_MAPPINGS)
328 p4dval |= P4D_TABLE_PXN;
329 BUG_ON(!pgtable_alloc);
330 pud_phys = pgtable_alloc(PUD_SHIFT);
331 __p4d_populate(p4dp, pud_phys, p4dval);
332 p4d = READ_ONCE(*p4dp);
334 BUG_ON(p4d_bad(p4d));
336 pudp = pud_set_fixmap_offset(p4dp, addr);
338 pud_t old_pud = READ_ONCE(*pudp);
340 next = pud_addr_end(addr, end);
343 * For 4K granule only, attempt to put down a 1GB block
345 if (pud_sect_supported() &&
346 ((addr | next | phys) & ~PUD_MASK) == 0 &&
347 (flags & NO_BLOCK_MAPPINGS) == 0) {
348 pud_set_huge(pudp, phys, prot);
351 * After the PUD entry has been populated once, we
352 * only allow updates to the permission attributes.
354 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
355 READ_ONCE(pud_val(*pudp))));
357 alloc_init_cont_pmd(pudp, addr, next, phys, prot,
358 pgtable_alloc, flags);
360 BUG_ON(pud_val(old_pud) != 0 &&
361 pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
364 } while (pudp++, addr = next, addr != end);
369 static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys,
370 unsigned long virt, phys_addr_t size,
372 phys_addr_t (*pgtable_alloc)(int),
375 unsigned long addr, end, next;
376 pgd_t *pgdp = pgd_offset_pgd(pgdir, virt);
379 * If the virtual and physical address don't have the same offset
380 * within a page, we cannot map the region as the caller expects.
382 if (WARN_ON((phys ^ virt) & ~PAGE_MASK))
386 addr = virt & PAGE_MASK;
387 end = PAGE_ALIGN(virt + size);
390 next = pgd_addr_end(addr, end);
391 alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,
394 } while (pgdp++, addr = next, addr != end);
397 static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
398 unsigned long virt, phys_addr_t size,
400 phys_addr_t (*pgtable_alloc)(int),
403 mutex_lock(&fixmap_lock);
404 __create_pgd_mapping_locked(pgdir, phys, virt, size, prot,
405 pgtable_alloc, flags);
406 mutex_unlock(&fixmap_lock);
409 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
410 extern __alias(__create_pgd_mapping_locked)
411 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
412 phys_addr_t size, pgprot_t prot,
413 phys_addr_t (*pgtable_alloc)(int), int flags);
416 static phys_addr_t __pgd_pgtable_alloc(int shift)
418 void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL);
421 /* Ensure the zeroed page is visible to the page table walker */
426 static phys_addr_t pgd_pgtable_alloc(int shift)
428 phys_addr_t pa = __pgd_pgtable_alloc(shift);
431 * Call proper page table ctor in case later we need to
432 * call core mm functions like apply_to_page_range() on
433 * this pre-allocated page table.
435 * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
436 * folded, and if so pgtable_pmd_page_ctor() becomes nop.
438 if (shift == PAGE_SHIFT)
439 BUG_ON(!pgtable_pte_page_ctor(phys_to_page(pa)));
440 else if (shift == PMD_SHIFT)
441 BUG_ON(!pgtable_pmd_page_ctor(phys_to_page(pa)));
447 * This function can only be used to modify existing table entries,
448 * without allocating new levels of table. Note that this permits the
449 * creation of new section or page entries.
451 void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
452 phys_addr_t size, pgprot_t prot)
454 if ((virt >= PAGE_END) && (virt < VMALLOC_START)) {
455 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
459 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
463 void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
464 unsigned long virt, phys_addr_t size,
465 pgprot_t prot, bool page_mappings_only)
469 BUG_ON(mm == &init_mm);
471 if (page_mappings_only)
472 flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
474 __create_pgd_mapping(mm->pgd, phys, virt, size, prot,
475 pgd_pgtable_alloc, flags);
478 static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
479 phys_addr_t size, pgprot_t prot)
481 if ((virt >= PAGE_END) && (virt < VMALLOC_START)) {
482 pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
487 __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL,
490 /* flush the TLBs after updating live kernel mappings */
491 flush_tlb_kernel_range(virt, virt + size);
494 static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start,
495 phys_addr_t end, pgprot_t prot, int flags)
497 __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
498 prot, early_pgtable_alloc, flags);
501 void __init mark_linear_text_alias_ro(void)
504 * Remove the write permissions from the linear alias of .text/.rodata
506 update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext),
507 (unsigned long)__init_begin - (unsigned long)_stext,
513 bool __ro_after_init kfence_early_init = !!CONFIG_KFENCE_SAMPLE_INTERVAL;
515 /* early_param() will be parsed before map_mem() below. */
516 static int __init parse_kfence_early_init(char *arg)
520 if (get_option(&arg, &val))
521 kfence_early_init = !!val;
524 early_param("kfence.sample_interval", parse_kfence_early_init);
526 static phys_addr_t __init arm64_kfence_alloc_pool(void)
528 phys_addr_t kfence_pool;
530 if (!kfence_early_init)
533 kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE);
535 pr_err("failed to allocate kfence pool\n");
536 kfence_early_init = false;
540 /* Temporarily mark as NOMAP. */
541 memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE);
546 static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp)
551 /* KFENCE pool needs page-level mapping. */
552 __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE,
553 pgprot_tagged(PAGE_KERNEL),
554 NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
555 memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE);
556 __kfence_pool = phys_to_virt(kfence_pool);
558 #else /* CONFIG_KFENCE */
560 static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; }
561 static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { }
563 #endif /* CONFIG_KFENCE */
565 static void __init map_mem(pgd_t *pgdp)
567 static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN);
568 phys_addr_t kernel_start = __pa_symbol(_stext);
569 phys_addr_t kernel_end = __pa_symbol(__init_begin);
570 phys_addr_t start, end;
571 phys_addr_t early_kfence_pool;
572 int flags = NO_EXEC_MAPPINGS;
576 * Setting hierarchical PXNTable attributes on table entries covering
577 * the linear region is only possible if it is guaranteed that no table
578 * entries at any level are being shared between the linear region and
579 * the vmalloc region. Check whether this is true for the PGD level, in
580 * which case it is guaranteed to be true for all other levels as well.
582 BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end));
584 early_kfence_pool = arm64_kfence_alloc_pool();
586 if (can_set_direct_map())
587 flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
590 * Take care not to create a writable alias for the
591 * read-only text and rodata sections of the kernel image.
592 * So temporarily mark them as NOMAP to skip mappings in
593 * the following for-loop
595 memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
597 /* map all the memory banks */
598 for_each_mem_range(i, &start, &end) {
602 * The linear map must allow allocation tags reading/writing
603 * if MTE is present. Otherwise, it has the same attributes as
606 __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL),
611 * Map the linear alias of the [_stext, __init_begin) interval
612 * as non-executable now, and remove the write permission in
613 * mark_linear_text_alias_ro() below (which will be called after
614 * alternative patching has completed). This makes the contents
615 * of the region accessible to subsystems such as hibernate,
616 * but protects it from inadvertent modification or execution.
617 * Note that contiguous mappings cannot be remapped in this way,
618 * so we should avoid them here.
620 __map_memblock(pgdp, kernel_start, kernel_end,
621 PAGE_KERNEL, NO_CONT_MAPPINGS);
622 memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
623 arm64_kfence_map_pool(early_kfence_pool, pgdp);
626 void mark_rodata_ro(void)
628 unsigned long section_size;
631 * mark .rodata as read only. Use __init_begin rather than __end_rodata
632 * to cover NOTES and EXCEPTION_TABLE.
634 section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
635 update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
636 section_size, PAGE_KERNEL_RO);
641 static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end,
642 pgprot_t prot, struct vm_struct *vma,
643 int flags, unsigned long vm_flags)
645 phys_addr_t pa_start = __pa_symbol(va_start);
646 unsigned long size = va_end - va_start;
648 BUG_ON(!PAGE_ALIGNED(pa_start));
649 BUG_ON(!PAGE_ALIGNED(size));
651 __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot,
652 early_pgtable_alloc, flags);
654 if (!(vm_flags & VM_NO_GUARD))
657 vma->addr = va_start;
658 vma->phys_addr = pa_start;
660 vma->flags = VM_MAP | vm_flags;
661 vma->caller = __builtin_return_address(0);
663 vm_area_add_early(vma);
666 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
667 static int __init map_entry_trampoline(void)
671 pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
672 phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
674 /* The trampoline is always mapped and can therefore be global */
675 pgprot_val(prot) &= ~PTE_NG;
677 /* Map only the text into the trampoline page table */
678 memset(tramp_pg_dir, 0, PGD_SIZE);
679 __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
680 entry_tramp_text_size(), prot,
681 __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
683 /* Map both the text and data into the kernel page table */
684 for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
685 __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
686 pa_start + i * PAGE_SIZE, prot);
688 if (IS_ENABLED(CONFIG_RELOCATABLE))
689 __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
690 pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO);
694 core_initcall(map_entry_trampoline);
698 * Open coded check for BTI, only for use to determine configuration
699 * for early mappings for before the cpufeature code has run.
701 static bool arm64_early_this_cpu_has_bti(void)
705 if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
708 pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1);
709 return cpuid_feature_extract_unsigned_field(pfr1,
710 ID_AA64PFR1_EL1_BT_SHIFT);
714 * Create fine-grained mappings for the kernel.
716 static void __init map_kernel(pgd_t *pgdp)
718 static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext,
719 vmlinux_initdata, vmlinux_data;
722 * External debuggers may need to write directly to the text
723 * mapping to install SW breakpoints. Allow this (only) when
724 * explicitly requested with rodata=off.
726 pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
729 * If we have a CPU that supports BTI and a kernel built for
730 * BTI then mark the kernel executable text as guarded pages
731 * now so we don't have to rewrite the page tables later.
733 if (arm64_early_this_cpu_has_bti())
734 text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP);
737 * Only rodata will be remapped with different permissions later on,
738 * all other segments are allowed to use contiguous mappings.
740 map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0,
742 map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL,
743 &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD);
744 map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot,
745 &vmlinux_inittext, 0, VM_NO_GUARD);
746 map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL,
747 &vmlinux_initdata, 0, VM_NO_GUARD);
748 map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0);
751 kasan_copy_shadow(pgdp);
754 static void __init create_idmap(void)
756 u64 start = __pa_symbol(__idmap_text_start);
757 u64 size = __pa_symbol(__idmap_text_end) - start;
758 pgd_t *pgd = idmap_pg_dir;
761 /* check if we need an additional level of translation */
762 if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) {
763 pgd_phys = early_pgtable_alloc(PAGE_SHIFT);
764 set_pgd(&idmap_pg_dir[start >> VA_BITS],
765 __pgd(pgd_phys | P4D_TYPE_TABLE));
766 pgd = __va(pgd_phys);
768 __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX,
769 early_pgtable_alloc, 0);
771 if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
772 extern u32 __idmap_kpti_flag;
773 u64 pa = __pa_symbol(&__idmap_kpti_flag);
776 * The KPTI G-to-nG conversion code needs a read-write mapping
777 * of its synchronization flag in the ID map.
779 __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL,
780 early_pgtable_alloc, 0);
784 void __init paging_init(void)
786 pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir));
787 extern pgd_t init_idmap_pg_dir[];
789 idmap_t0sz = 63UL - __fls(__pa_symbol(_end) | GENMASK(VA_BITS_MIN - 1, 0));
796 cpu_replace_ttbr1(lm_alias(swapper_pg_dir), init_idmap_pg_dir);
797 init_mm.pgd = swapper_pg_dir;
799 memblock_phys_free(__pa_symbol(init_pg_dir),
800 __pa_symbol(init_pg_end) - __pa_symbol(init_pg_dir));
802 memblock_allow_resize();
807 #ifdef CONFIG_MEMORY_HOTPLUG
808 static void free_hotplug_page_range(struct page *page, size_t size,
809 struct vmem_altmap *altmap)
812 vmem_altmap_free(altmap, size >> PAGE_SHIFT);
814 WARN_ON(PageReserved(page));
815 free_pages((unsigned long)page_address(page), get_order(size));
819 static void free_hotplug_pgtable_page(struct page *page)
821 free_hotplug_page_range(page, PAGE_SIZE, NULL);
824 static bool pgtable_range_aligned(unsigned long start, unsigned long end,
825 unsigned long floor, unsigned long ceiling,
838 if (end - 1 > ceiling - 1)
843 static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr,
844 unsigned long end, bool free_mapped,
845 struct vmem_altmap *altmap)
850 ptep = pte_offset_kernel(pmdp, addr);
851 pte = READ_ONCE(*ptep);
855 WARN_ON(!pte_present(pte));
856 pte_clear(&init_mm, addr, ptep);
857 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
859 free_hotplug_page_range(pte_page(pte),
861 } while (addr += PAGE_SIZE, addr < end);
864 static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr,
865 unsigned long end, bool free_mapped,
866 struct vmem_altmap *altmap)
872 next = pmd_addr_end(addr, end);
873 pmdp = pmd_offset(pudp, addr);
874 pmd = READ_ONCE(*pmdp);
878 WARN_ON(!pmd_present(pmd));
883 * One TLBI should be sufficient here as the PMD_SIZE
884 * range is mapped with a single block entry.
886 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
888 free_hotplug_page_range(pmd_page(pmd),
892 WARN_ON(!pmd_table(pmd));
893 unmap_hotplug_pte_range(pmdp, addr, next, free_mapped, altmap);
894 } while (addr = next, addr < end);
897 static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr,
898 unsigned long end, bool free_mapped,
899 struct vmem_altmap *altmap)
905 next = pud_addr_end(addr, end);
906 pudp = pud_offset(p4dp, addr);
907 pud = READ_ONCE(*pudp);
911 WARN_ON(!pud_present(pud));
916 * One TLBI should be sufficient here as the PUD_SIZE
917 * range is mapped with a single block entry.
919 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
921 free_hotplug_page_range(pud_page(pud),
925 WARN_ON(!pud_table(pud));
926 unmap_hotplug_pmd_range(pudp, addr, next, free_mapped, altmap);
927 } while (addr = next, addr < end);
930 static void unmap_hotplug_p4d_range(pgd_t *pgdp, unsigned long addr,
931 unsigned long end, bool free_mapped,
932 struct vmem_altmap *altmap)
938 next = p4d_addr_end(addr, end);
939 p4dp = p4d_offset(pgdp, addr);
940 p4d = READ_ONCE(*p4dp);
944 WARN_ON(!p4d_present(p4d));
945 unmap_hotplug_pud_range(p4dp, addr, next, free_mapped, altmap);
946 } while (addr = next, addr < end);
949 static void unmap_hotplug_range(unsigned long addr, unsigned long end,
950 bool free_mapped, struct vmem_altmap *altmap)
956 * altmap can only be used as vmemmap mapping backing memory.
957 * In case the backing memory itself is not being freed, then
958 * altmap is irrelevant. Warn about this inconsistency when
961 WARN_ON(!free_mapped && altmap);
964 next = pgd_addr_end(addr, end);
965 pgdp = pgd_offset_k(addr);
966 pgd = READ_ONCE(*pgdp);
970 WARN_ON(!pgd_present(pgd));
971 unmap_hotplug_p4d_range(pgdp, addr, next, free_mapped, altmap);
972 } while (addr = next, addr < end);
975 static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
976 unsigned long end, unsigned long floor,
977 unsigned long ceiling)
980 unsigned long i, start = addr;
983 ptep = pte_offset_kernel(pmdp, addr);
984 pte = READ_ONCE(*ptep);
987 * This is just a sanity check here which verifies that
988 * pte clearing has been done by earlier unmap loops.
990 WARN_ON(!pte_none(pte));
991 } while (addr += PAGE_SIZE, addr < end);
993 if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK))
997 * Check whether we can free the pte page if the rest of the
998 * entries are empty. Overlap with other regions have been
999 * handled by the floor/ceiling check.
1001 ptep = pte_offset_kernel(pmdp, 0UL);
1002 for (i = 0; i < PTRS_PER_PTE; i++) {
1003 if (!pte_none(READ_ONCE(ptep[i])))
1008 __flush_tlb_kernel_pgtable(start);
1009 free_hotplug_pgtable_page(virt_to_page(ptep));
1012 static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
1013 unsigned long end, unsigned long floor,
1014 unsigned long ceiling)
1017 unsigned long i, next, start = addr;
1020 next = pmd_addr_end(addr, end);
1021 pmdp = pmd_offset(pudp, addr);
1022 pmd = READ_ONCE(*pmdp);
1026 WARN_ON(!pmd_present(pmd) || !pmd_table(pmd) || pmd_sect(pmd));
1027 free_empty_pte_table(pmdp, addr, next, floor, ceiling);
1028 } while (addr = next, addr < end);
1030 if (CONFIG_PGTABLE_LEVELS <= 2)
1033 if (!pgtable_range_aligned(start, end, floor, ceiling, PUD_MASK))
1037 * Check whether we can free the pmd page if the rest of the
1038 * entries are empty. Overlap with other regions have been
1039 * handled by the floor/ceiling check.
1041 pmdp = pmd_offset(pudp, 0UL);
1042 for (i = 0; i < PTRS_PER_PMD; i++) {
1043 if (!pmd_none(READ_ONCE(pmdp[i])))
1048 __flush_tlb_kernel_pgtable(start);
1049 free_hotplug_pgtable_page(virt_to_page(pmdp));
1052 static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
1053 unsigned long end, unsigned long floor,
1054 unsigned long ceiling)
1057 unsigned long i, next, start = addr;
1060 next = pud_addr_end(addr, end);
1061 pudp = pud_offset(p4dp, addr);
1062 pud = READ_ONCE(*pudp);
1066 WARN_ON(!pud_present(pud) || !pud_table(pud) || pud_sect(pud));
1067 free_empty_pmd_table(pudp, addr, next, floor, ceiling);
1068 } while (addr = next, addr < end);
1070 if (CONFIG_PGTABLE_LEVELS <= 3)
1073 if (!pgtable_range_aligned(start, end, floor, ceiling, PGDIR_MASK))
1077 * Check whether we can free the pud page if the rest of the
1078 * entries are empty. Overlap with other regions have been
1079 * handled by the floor/ceiling check.
1081 pudp = pud_offset(p4dp, 0UL);
1082 for (i = 0; i < PTRS_PER_PUD; i++) {
1083 if (!pud_none(READ_ONCE(pudp[i])))
1088 __flush_tlb_kernel_pgtable(start);
1089 free_hotplug_pgtable_page(virt_to_page(pudp));
1092 static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
1093 unsigned long end, unsigned long floor,
1094 unsigned long ceiling)
1100 next = p4d_addr_end(addr, end);
1101 p4dp = p4d_offset(pgdp, addr);
1102 p4d = READ_ONCE(*p4dp);
1106 WARN_ON(!p4d_present(p4d));
1107 free_empty_pud_table(p4dp, addr, next, floor, ceiling);
1108 } while (addr = next, addr < end);
1111 static void free_empty_tables(unsigned long addr, unsigned long end,
1112 unsigned long floor, unsigned long ceiling)
1118 next = pgd_addr_end(addr, end);
1119 pgdp = pgd_offset_k(addr);
1120 pgd = READ_ONCE(*pgdp);
1124 WARN_ON(!pgd_present(pgd));
1125 free_empty_p4d_table(pgdp, addr, next, floor, ceiling);
1126 } while (addr = next, addr < end);
1130 void __meminit vmemmap_set_pmd(pmd_t *pmdp, void *p, int node,
1131 unsigned long addr, unsigned long next)
1133 pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));
1136 int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
1137 unsigned long addr, unsigned long next)
1139 vmemmap_verify((pte_t *)pmdp, node, addr, next);
1143 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
1144 struct vmem_altmap *altmap)
1146 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
1148 if (!IS_ENABLED(CONFIG_ARM64_4K_PAGES))
1149 return vmemmap_populate_basepages(start, end, node, altmap);
1151 return vmemmap_populate_hugepages(start, end, node, altmap);
1154 #ifdef CONFIG_MEMORY_HOTPLUG
1155 void vmemmap_free(unsigned long start, unsigned long end,
1156 struct vmem_altmap *altmap)
1158 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
1160 unmap_hotplug_range(start, end, true, altmap);
1161 free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END);
1163 #endif /* CONFIG_MEMORY_HOTPLUG */
1165 int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
1167 pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot));
1169 /* Only allow permission changes for now */
1170 if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)),
1174 VM_BUG_ON(phys & ~PUD_MASK);
1175 set_pud(pudp, new_pud);
1179 int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
1181 pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot));
1183 /* Only allow permission changes for now */
1184 if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)),
1188 VM_BUG_ON(phys & ~PMD_MASK);
1189 set_pmd(pmdp, new_pmd);
1193 int pud_clear_huge(pud_t *pudp)
1195 if (!pud_sect(READ_ONCE(*pudp)))
1201 int pmd_clear_huge(pmd_t *pmdp)
1203 if (!pmd_sect(READ_ONCE(*pmdp)))
1209 int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
1214 pmd = READ_ONCE(*pmdp);
1216 if (!pmd_table(pmd)) {
1221 table = pte_offset_kernel(pmdp, addr);
1223 __flush_tlb_kernel_pgtable(addr);
1224 pte_free_kernel(NULL, table);
1228 int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
1233 unsigned long next, end;
1235 pud = READ_ONCE(*pudp);
1237 if (!pud_table(pud)) {
1242 table = pmd_offset(pudp, addr);
1245 end = addr + PUD_SIZE;
1247 pmd_free_pte_page(pmdp, next);
1248 } while (pmdp++, next += PMD_SIZE, next != end);
1251 __flush_tlb_kernel_pgtable(addr);
1252 pmd_free(NULL, table);
1256 #ifdef CONFIG_MEMORY_HOTPLUG
1257 static void __remove_pgd_mapping(pgd_t *pgdir, unsigned long start, u64 size)
1259 unsigned long end = start + size;
1261 WARN_ON(pgdir != init_mm.pgd);
1262 WARN_ON((start < PAGE_OFFSET) || (end > PAGE_END));
1264 unmap_hotplug_range(start, end, false, NULL);
1265 free_empty_tables(start, end, PAGE_OFFSET, PAGE_END);
1268 struct range arch_get_mappable_range(void)
1270 struct range mhp_range;
1271 u64 start_linear_pa = __pa(_PAGE_OFFSET(vabits_actual));
1272 u64 end_linear_pa = __pa(PAGE_END - 1);
1274 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
1276 * Check for a wrap, it is possible because of randomized linear
1277 * mapping the start physical address is actually bigger than
1278 * the end physical address. In this case set start to zero
1279 * because [0, end_linear_pa] range must still be able to cover
1280 * all addressable physical addresses.
1282 if (start_linear_pa > end_linear_pa)
1283 start_linear_pa = 0;
1286 WARN_ON(start_linear_pa > end_linear_pa);
1289 * Linear mapping region is the range [PAGE_OFFSET..(PAGE_END - 1)]
1290 * accommodating both its ends but excluding PAGE_END. Max physical
1291 * range which can be mapped inside this linear mapping range, must
1292 * also be derived from its end points.
1294 mhp_range.start = start_linear_pa;
1295 mhp_range.end = end_linear_pa;
1300 int arch_add_memory(int nid, u64 start, u64 size,
1301 struct mhp_params *params)
1303 int ret, flags = NO_EXEC_MAPPINGS;
1305 VM_BUG_ON(!mhp_range_allowed(start, size, true));
1307 if (can_set_direct_map())
1308 flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
1310 __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
1311 size, params->pgprot, __pgd_pgtable_alloc,
1314 memblock_clear_nomap(start, size);
1316 ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
1319 __remove_pgd_mapping(swapper_pg_dir,
1320 __phys_to_virt(start), size);
1322 max_pfn = PFN_UP(start + size);
1323 max_low_pfn = max_pfn;
1329 void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
1331 unsigned long start_pfn = start >> PAGE_SHIFT;
1332 unsigned long nr_pages = size >> PAGE_SHIFT;
1334 __remove_pages(start_pfn, nr_pages, altmap);
1335 __remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
1339 * This memory hotplug notifier helps prevent boot memory from being
1340 * inadvertently removed as it blocks pfn range offlining process in
1341 * __offline_pages(). Hence this prevents both offlining as well as
1342 * removal process for boot memory which is initially always online.
1343 * In future if and when boot memory could be removed, this notifier
1344 * should be dropped and free_hotplug_page_range() should handle any
1345 * reserved pages allocated during boot.
1347 static int prevent_bootmem_remove_notifier(struct notifier_block *nb,
1348 unsigned long action, void *data)
1350 struct mem_section *ms;
1351 struct memory_notify *arg = data;
1352 unsigned long end_pfn = arg->start_pfn + arg->nr_pages;
1353 unsigned long pfn = arg->start_pfn;
1355 if ((action != MEM_GOING_OFFLINE) && (action != MEM_OFFLINE))
1358 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1359 unsigned long start = PFN_PHYS(pfn);
1360 unsigned long end = start + (1UL << PA_SECTION_SHIFT);
1362 ms = __pfn_to_section(pfn);
1363 if (!early_section(ms))
1366 if (action == MEM_GOING_OFFLINE) {
1368 * Boot memory removal is not supported. Prevent
1369 * it via blocking any attempted offline request
1370 * for the boot memory and just report it.
1372 pr_warn("Boot memory [%lx %lx] offlining attempted\n", start, end);
1374 } else if (action == MEM_OFFLINE) {
1376 * This should have never happened. Boot memory
1377 * offlining should have been prevented by this
1378 * very notifier. Probably some memory removal
1379 * procedure might have changed which would then
1380 * require further debug.
1382 pr_err("Boot memory [%lx %lx] offlined\n", start, end);
1385 * Core memory hotplug does not process a return
1386 * code from the notifier for MEM_OFFLINE events.
1387 * The error condition has been reported. Return
1388 * from here as if ignored.
1396 static struct notifier_block prevent_bootmem_remove_nb = {
1397 .notifier_call = prevent_bootmem_remove_notifier,
1401 * This ensures that boot memory sections on the platform are online
1402 * from early boot. Memory sections could not be prevented from being
1403 * offlined, unless for some reason they are not online to begin with.
1404 * This helps validate the basic assumption on which the above memory
1405 * event notifier works to prevent boot memory section offlining and
1406 * its possible removal.
1408 static void validate_bootmem_online(void)
1410 phys_addr_t start, end, addr;
1411 struct mem_section *ms;
1415 * Scanning across all memblock might be expensive
1416 * on some big memory systems. Hence enable this
1417 * validation only with DEBUG_VM.
1419 if (!IS_ENABLED(CONFIG_DEBUG_VM))
1422 for_each_mem_range(i, &start, &end) {
1423 for (addr = start; addr < end; addr += (1UL << PA_SECTION_SHIFT)) {
1424 ms = __pfn_to_section(PHYS_PFN(addr));
1427 * All memory ranges in the system at this point
1428 * should have been marked as early sections.
1430 WARN_ON(!early_section(ms));
1433 * Memory notifier mechanism here to prevent boot
1434 * memory offlining depends on the fact that each
1435 * early section memory on the system is initially
1436 * online. Otherwise a given memory section which
1437 * is already offline will be overlooked and can
1438 * be removed completely. Call out such sections.
1440 if (!online_section(ms))
1441 pr_err("Boot memory [%llx %llx] is offline, can be removed\n",
1442 addr, addr + (1UL << PA_SECTION_SHIFT));
1447 static int __init prevent_bootmem_remove_init(void)
1451 if (!IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
1454 validate_bootmem_online();
1455 ret = register_memory_notifier(&prevent_bootmem_remove_nb);
1457 pr_err("%s: Notifier registration failed %d\n", __func__, ret);
1461 early_initcall(prevent_bootmem_remove_init);
1464 pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
1466 if (IS_ENABLED(CONFIG_ARM64_ERRATUM_2645198) &&
1467 cpus_have_const_cap(ARM64_WORKAROUND_2645198)) {
1469 * Break-before-make (BBM) is required for all user space mappings
1470 * when the permission changes from executable to non-executable
1471 * in cases where cpu is affected with errata #2645198.
1473 if (pte_user_exec(READ_ONCE(*ptep)))
1474 return ptep_clear_flush(vma, addr, ptep);
1476 return ptep_get_and_clear(vma->vm_mm, addr, ptep);
1479 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
1480 pte_t old_pte, pte_t pte)
1482 set_pte_at(vma->vm_mm, addr, ptep, pte);