arm64: alternatives: use tpidr_el2 on VHE hosts

When upstream 6d99b68933fb was back-ported to upstream v4.9.x stable as
eea59020a7f2, the edits to arch/arm64/mm/proc.S were dropped because
proc.S didn't save/restore tpidr_el1 at all. Separately, in android-4.9,
0ec37136b90e ("UPSTREAM: arm64: move sp_el0 and tpidr_el1 into
cpu_suspend_ctx") modified proc.S to save/restore tpidir_el1. These two
paths were later merged together in android-4.9. The missing edits to
proc.S should have been added in during the merge, but were not. This
change restores those edits. The original upstream change description
of 6d99b68933fb follows; this is where the missing code appeared
originally.

Commit 6d99b68933fbcf51f84fcbba49246ce1209ec193 upstream.

Now that KVM uses tpidr_el2 in the same way as Linux's cpu_offset in
tpidr_el1, merge the two. This saves KVM from save/restoring tpidr_el1
on VHE hosts, and allows future code to blindly access per-cpu variables
without triggering world-switch.

Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Christoffer Dall <cdall@linaro.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Fixes: eea59020a7f2 ("arm64: alternatives: use tpidr_el2 on VHE hosts")
Fixes: 0ec37136b90e ("UPSTREAM: arm64: move sp_el0 and tpidr_el1 into cpu_suspend_ctx")
Fixes: 4a5211fa1474 ("Merge 4.9.114 into android-4.9-p")
Signed-off-by: Stephen Warren <swarren@nvidia.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
[sw0312.kim: cherry-pick android-4.9-q commit c337caddb549 to fix booting issue
- also fully applying linux-4.9.y commit eea59020a7f2 ("arm64: alternatives: use tpidr_el2 on VHE hosts") skipped from khadas' revert]
Signed-off-by: Seung-Woo Kim <sw0312.kim@samsung.com>
Change-Id: Iad39e6f1fb662b95967e8ebd07a36844bf46cd2d

diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h
index d8b00a0..b7205c2 100644 (file)
--- a/arch/arm64/include/asm/alternative.h
+++ b/arch/arm64/include/asm/alternative.h
@@ -13,6 +13,8 @@
 #include <linux/stddef.h>
 #include <linux/stringify.h>
 
+extern int alternatives_applied;
+
 struct alt_instr {
        s32 orig_offset;        /* offset to original instruction */
        s32 alt_offset;         /* offset to replacement instruction */

diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 6c858ba..1abad23 100644 (file)
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -21,7 +21,10 @@
 
 static inline void set_my_cpu_offset(unsigned long off)
 {
-       asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
+       asm volatile(ALTERNATIVE("msr tpidr_el1, %0",
+                                "msr tpidr_el2, %0",
+                                ARM64_HAS_VIRT_HOST_EXTN)
+                       :: "r" (off) : "memory");
 }
 
 static inline unsigned long __my_cpu_offset(void)
@@ -32,7 +35,10 @@ static inline unsigned long __my_cpu_offset(void)
         * We want to allow caching the value, so avoid using volatile and
         * instead use a fake stack read to hazard against barrier().
         */
-       asm("mrs %0, tpidr_el1" : "=r" (off) :
+       asm(ALTERNATIVE("mrs %0, tpidr_el1",
+                       "mrs %0, tpidr_el2",
+                       ARM64_HAS_VIRT_HOST_EXTN)
+               : "=r" (off) :
                "Q" (*(const unsigned long *)current_stack_pointer));
 
        return off;

diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 81ea12b..0917480 100644 (file)
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -32,6 +32,8 @@
 #define ALT_ORIG_PTR(a)                __ALT_PTR(a, orig_offset)
 #define ALT_REPL_PTR(a)                __ALT_PTR(a, alt_offset)
 
+int alternatives_applied;
+
 struct alt_region {
        struct alt_instr *begin;
        struct alt_instr *end;
@@ -163,7 +165,6 @@ static void __apply_alternatives(void *alt_region)
  */
 static int __apply_alternatives_multi_stop(void *unused)
 {
-       static int patched = 0;
        struct alt_region region = {
                .begin  = (struct alt_instr *)__alt_instructions,
                .end    = (struct alt_instr *)__alt_instructions_end,
@@ -171,14 +172,14 @@ static int __apply_alternatives_multi_stop(void *unused)
 
        /* We always have a CPU 0 at this point (__init) */
        if (smp_processor_id()) {
-               while (!READ_ONCE(patched))
+               while (!READ_ONCE(alternatives_applied))
                        cpu_relax();
                isb();
        } else {
-               BUG_ON(patched);
+               BUG_ON(alternatives_applied);
                __apply_alternatives(&region);
                /* Barriers provided by the cache flushing */
-               WRITE_ONCE(patched, 1);
+               WRITE_ONCE(alternatives_applied, 1);
        }
 
        return 0;

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 7480160..0a185ab 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -846,6 +846,22 @@ static int __init parse_kpti(char *str)
 early_param("kpti", parse_kpti);
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 
+static int cpu_copy_el2regs(void *__unused)
+{
+       /*
+        * Copy register values that aren't redirected by hardware.
+        *
+        * Before code patching, we only set tpidr_el1, all CPUs need to copy
+        * this value to tpidr_el2 before we patch the code. Once we've done
+        * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
+        * do anything here.
+        */
+       if (!alternatives_applied)
+               write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
+
+       return 0;
+}
+
 static const struct arm64_cpu_capabilities arm64_features[] = {
        {
                .desc = "GIC system register CPU interface",
@@ -912,6 +928,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .capability = ARM64_HAS_VIRT_HOST_EXTN,
                .def_scope = SCOPE_SYSTEM,
                .matches = runs_at_el2,
+               .enable = cpu_copy_el2regs,
        },
        {
                .desc = "32-bit EL0 Support",

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 65e19ec..7df7082 100644 (file)
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -70,7 +70,11 @@ ENTRY(cpu_do_suspend)
        mrs     x8, mdscr_el1
        mrs     x9, oslsr_el1
        mrs     x10, sctlr_el1
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
        mrs     x11, tpidr_el1
+alternative_else
+       mrs     x11, tpidr_el2
+alternative_endif
        mrs     x12, sp_el0
        stp     x2, x3, [x0]
        stp     x4, xzr, [x0, #16]
@@ -116,7 +120,11 @@ ENTRY(cpu_do_resume)
        msr     mdscr_el1, x10
 
        msr     sctlr_el1, x12
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
        msr     tpidr_el1, x13
+alternative_else
+       msr     tpidr_el2, x13
+alternative_endif
        msr     sp_el0, x14
        /*
         * Restore oslsr_el1 by writing oslar_el1