arm64: kexec: install a copy of the linear-map

To perform the kexec relocation with the MMU enabled, we need a copy
of the linear map.

Create one, and install it from the relocation code. This has to be done
from the assembly code as it will be idmapped with TTBR0. The kernel
runs in TTRB1, so can't use the break-before-make sequence on the mapping
it is executing from.

The makes no difference yet as the relocation code runs with the MMU
disabled.

Suggested-by: James Morse <james.morse@arm.com>
Signed-off-by: Pasha Tatashin <pasha.tatashin@soleen.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20210930143113.1502553-12-pasha.tatashin@soleen.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index d5281f75a58d76f622a348660451c9f0fc2ef300..5da176dd37a93b52218b9b7dbb1234a6eb7b5764 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -483,6 +483,25 @@ alternative_endif
        _cond_extable .Licache_op\@, \fixup
        .endm
 
+/*
+ * To prevent the possibility of old and new partial table walks being visible
+ * in the tlb, switch the ttbr to a zero page when we invalidate the old
+ * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i
+ * Even switching to our copied tables will cause a changed output address at
+ * each stage of the walk.
+ */
+       .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2
+       phys_to_ttbr \tmp, \zero_page
+       msr     ttbr1_el1, \tmp
+       isb
+       tlbi    vmalle1
+       dsb     nsh
+       phys_to_ttbr \tmp, \page_table
+       offset_ttbr1 \tmp, \tmp2
+       msr     ttbr1_el1, \tmp
+       isb
+       .endm
+
 /*
  * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
  */

diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h
index 753a1c39889869ce3927f0a2768b075579336554..d678f0ceb7ee77a8e56523355ad6b33895ecfafd 100644 (file)
--- a/arch/arm64/include/asm/kexec.h
+++ b/arch/arm64/include/asm/kexec.h
@@ -97,6 +97,8 @@ struct kimage_arch {
        phys_addr_t dtb_mem;
        phys_addr_t kern_reloc;
        phys_addr_t el2_vectors;
+       phys_addr_t ttbr1;
+       phys_addr_t zero_page;
 };
 
 #ifdef CONFIG_KEXEC_FILE

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 6a2b8b1a4872bc8f7d10da90094cb5aedc09bd7c..1f565224dafd21f22044b9b81fcdb3c826a11266 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -175,6 +175,8 @@ int main(void)
 #ifdef CONFIG_KEXEC_CORE
   DEFINE(KIMAGE_ARCH_DTB_MEM,          offsetof(struct kimage, arch.dtb_mem));
   DEFINE(KIMAGE_ARCH_EL2_VECTORS,      offsetof(struct kimage, arch.el2_vectors));
+  DEFINE(KIMAGE_ARCH_ZERO_PAGE,                offsetof(struct kimage, arch.zero_page));
+  DEFINE(KIMAGE_ARCH_TTBR1,            offsetof(struct kimage, arch.ttbr1));
   DEFINE(KIMAGE_HEAD,                  offsetof(struct kimage, head));
   DEFINE(KIMAGE_START,                 offsetof(struct kimage, start));
   BLANK();

diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S
index a30a2c3f905ecd6fd62e2a3047c44f7e2d016551..0e1d9c3c6a9339a222f54a950e3229ec4f04dc0b 100644 (file)
--- a/arch/arm64/kernel/hibernate-asm.S
+++ b/arch/arm64/kernel/hibernate-asm.S
@@ -15,26 +15,6 @@
 #include <asm/page.h>
 #include <asm/virt.h>
 
-/*
- * To prevent the possibility of old and new partial table walks being visible
- * in the tlb, switch the ttbr to a zero page when we invalidate the old
- * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i
- * Even switching to our copied tables will cause a changed output address at
- * each stage of the walk.
- */
-.macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2
-       phys_to_ttbr \tmp, \zero_page
-       msr     ttbr1_el1, \tmp
-       isb
-       tlbi    vmalle1
-       dsb     nsh
-       phys_to_ttbr \tmp, \page_table
-       offset_ttbr1 \tmp, \tmp2
-       msr     ttbr1_el1, \tmp
-       isb
-.endm
-
-
 /*
  * Resume from hibernate
  *

diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index 320442d35811a537544ced50f18e290f23fc106f..fbff545565f175bdf78911fc4c1816f6990c012c 100644 (file)
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -159,6 +159,8 @@ static void *kexec_page_alloc(void *arg)
 
 int machine_kexec_post_load(struct kimage *kimage)
 {
+       int rc;
+       pgd_t *trans_pgd;
        void *reloc_code = page_to_virt(kimage->control_code_page);
        long reloc_size;
        struct trans_pgd_info info = {
@@ -175,12 +177,22 @@ int machine_kexec_post_load(struct kimage *kimage)
 
        kimage->arch.el2_vectors = 0;
        if (is_hyp_nvhe()) {
-               int rc = trans_pgd_copy_el2_vectors(&info,
-                                                   &kimage->arch.el2_vectors);
+               rc = trans_pgd_copy_el2_vectors(&info,
+                                               &kimage->arch.el2_vectors);
                if (rc)
                        return rc;
        }
 
+       /* Create a copy of the linear map */
+       trans_pgd = kexec_page_alloc(kimage);
+       if (!trans_pgd)
+               return -ENOMEM;
+       rc = trans_pgd_create_copy(&info, &trans_pgd, PAGE_OFFSET, PAGE_END);
+       if (rc)
+               return rc;
+       kimage->arch.ttbr1 = __pa(trans_pgd);
+       kimage->arch.zero_page = __pa(empty_zero_page);
+
        reloc_size = __relocate_new_kernel_end - __relocate_new_kernel_start;
        memcpy(reloc_code, __relocate_new_kernel_start, reloc_size);
        kimage->arch.kern_reloc = __pa(reloc_code);

diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index 2227741b96fa91a7018bfd301ecad583e00eb5d6..2b80232246f7f7e36493083d7085b3ffad39d8e4 100644 (file)
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -29,10 +29,13 @@
  */
 SYM_CODE_START(arm64_relocate_new_kernel)
        /* Setup the list loop variables. */
+       ldr     x18, [x0, #KIMAGE_ARCH_ZERO_PAGE] /* x18 = zero page for BBM */
+       ldr     x17, [x0, #KIMAGE_ARCH_TTBR1]   /* x17 = linear map copy */
        ldr     x16, [x0, #KIMAGE_HEAD]         /* x16 = kimage_head */
        mov     x14, xzr                        /* x14 = entry ptr */
        mov     x13, xzr                        /* x13 = copy dest */
        raw_dcache_line_size x15, x1            /* x15 = dcache line size */
+       break_before_make_ttbr_switch   x18, x17, x1, x2 /* set linear map */
 .Lloop:
        and     x12, x16, PAGE_MASK             /* x12 = addr */