s390: add options to change branch prediction behaviour for the kernel

[ Upstream commit d768bd892fc8f066cd3aa000eb1867bcf32db0ee ]

Add the PPA instruction to the system entry and exit path to switch
the kernel to a different branch prediction behaviour. The instructions
are added via CPU alternatives and can be disabled with the "nospec"
or the "nobp=0" kernel parameter. If the default behaviour selected
with CONFIG_KERNEL_NOBP is set to "n" then the "nobp=1" parameter can be
used to enable the changed kernel branch prediction.

Acked-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index f5a154d..02eb7a9 100644 (file)
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -704,6 +704,23 @@ config SECCOMP
 
          If unsure, say Y.
 
+config KERNEL_NOBP
+       def_bool n
+       prompt "Enable modified branch prediction for the kernel by default"
+       help
+         If this option is selected the kernel will switch to a modified
+         branch prediction mode if the firmware interface is available.
+         The modified branch prediction mode improves the behaviour in
+         regard to speculative execution.
+
+         With the option enabled the kernel parameter "nobp=0" or "nospec"
+         can be used to run the kernel in the normal branch prediction mode.
+
+         With the option disabled the modified branch prediction mode is
+         enabled with the "nobp=1" kernel parameter.
+
+         If unsure, say N.
+
 endmenu
 
 menu "Power Management"

diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 6bcbbec..44de5d0 100644 (file)
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -84,6 +84,7 @@ void cpu_detect_mhz_feature(void);
 extern const struct seq_operations cpuinfo_op;
 extern int sysctl_ieee_emulation_warnings;
 extern void execve_tail(void);
+extern void __bpon(void);
 
 /*
  * User space process size: 2GB for 31 bit, 4TB or 8PT for 64 bit.

diff --git a/arch/s390/kernel/alternative.c b/arch/s390/kernel/alternative.c
index f5060af..ed21de9 100644 (file)
--- a/arch/s390/kernel/alternative.c
+++ b/arch/s390/kernel/alternative.c
@@ -14,6 +14,29 @@ static int __init disable_alternative_instructions(char *str)
 
 early_param("noaltinstr", disable_alternative_instructions);
 
+static int __init nobp_setup_early(char *str)
+{
+       bool enabled;
+       int rc;
+
+       rc = kstrtobool(str, &enabled);
+       if (rc)
+               return rc;
+       if (enabled && test_facility(82))
+               __set_facility(82, S390_lowcore.alt_stfle_fac_list);
+       else
+               __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
+       return 0;
+}
+early_param("nobp", nobp_setup_early);
+
+static int __init nospec_setup_early(char *str)
+{
+       __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
+       return 0;
+}
+early_param("nospec", nospec_setup_early);
+
 struct brcl_insn {
        u16 opc;
        s32 disp;

diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index 171aaa2..0c7a7d5 100644 (file)
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -302,6 +302,8 @@ static noinline __init void setup_facility_list(void)
        memcpy(S390_lowcore.alt_stfle_fac_list,
               S390_lowcore.stfle_fac_list,
               sizeof(S390_lowcore.alt_stfle_fac_list));
+       if (!IS_ENABLED(CONFIG_KERNEL_NOBP))
+               __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
 }
 
 static __init void detect_diag9c(void)

diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 04bff4d..bfcf04d 100644 (file)
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -163,6 +163,34 @@ _PIF_WORK  = (_PIF_PER_TRAP)
                tm      off+\addr, \mask
        .endm
 
+       .macro BPOFF
+       .pushsection .altinstr_replacement, "ax"
+660:   .long   0xb2e8c000
+       .popsection
+661:   .long   0x47000000
+       .pushsection .altinstructions, "a"
+       .long 661b - .
+       .long 660b - .
+       .word 82
+       .byte 4
+       .byte 4
+       .popsection
+       .endm
+
+       .macro BPON
+       .pushsection .altinstr_replacement, "ax"
+662:   .long   0xb2e8d000
+       .popsection
+663:   .long   0x47000000
+       .pushsection .altinstructions, "a"
+       .long 663b - .
+       .long 662b - .
+       .word 82
+       .byte 4
+       .byte 4
+       .popsection
+       .endm
+
        .section .kprobes.text, "ax"
 .Ldummy:
        /*
@@ -175,6 +203,11 @@ _PIF_WORK  = (_PIF_PER_TRAP)
         */
        nop     0
 
+ENTRY(__bpon)
+       .globl __bpon
+       BPON
+       br      %r14
+
 /*
  * Scheduler resume function, called by switch_to
  *  gpr2 = (task_struct *) prev
@@ -234,7 +267,10 @@ ENTRY(sie64a)
        jnz     .Lsie_skip
        TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
        jo      .Lsie_skip                      # exit if fp/vx regs changed
+       BPON
        sie     0(%r14)
+.Lsie_exit:
+       BPOFF
 .Lsie_skip:
        ni      __SIE_PROG0C+3(%r14),0xfe       # no longer in SIE
        lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
@@ -286,6 +322,7 @@ ENTRY(system_call)
        stpt    __LC_SYNC_ENTER_TIMER
 .Lsysc_stmg:
        stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
+       BPOFF
        lg      %r10,__LC_LAST_BREAK
        lg      %r12,__LC_THREAD_INFO
        lghi    %r14,_PIF_SYSCALL
@@ -332,6 +369,7 @@ ENTRY(system_call)
        jnz     .Lsysc_work                     # check for work
        TSTMSK  __LC_CPU_FLAGS,_CIF_WORK
        jnz     .Lsysc_work
+       BPON
 .Lsysc_restore:
        lg      %r14,__LC_VDSO_PER_CPU
        lmg     %r0,%r10,__PT_R0(%r11)
@@ -492,6 +530,7 @@ ENTRY(kernel_thread_starter)
 
 ENTRY(pgm_check_handler)
        stpt    __LC_SYNC_ENTER_TIMER
+       BPOFF
        stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
        lg      %r10,__LC_LAST_BREAK
        lg      %r12,__LC_THREAD_INFO
@@ -590,6 +629,7 @@ ENTRY(pgm_check_handler)
 ENTRY(io_int_handler)
        STCK    __LC_INT_CLOCK
        stpt    __LC_ASYNC_ENTER_TIMER
+       BPOFF
        stmg    %r8,%r15,__LC_SAVE_AREA_ASYNC
        lg      %r10,__LC_LAST_BREAK
        lg      %r12,__LC_THREAD_INFO
@@ -641,9 +681,13 @@ ENTRY(io_int_handler)
        lg      %r14,__LC_VDSO_PER_CPU
        lmg     %r0,%r10,__PT_R0(%r11)
        mvc     __LC_RETURN_PSW(16),__PT_PSW(%r11)
+       tm      __PT_PSW+1(%r11),0x01   # returning to user ?
+       jno     .Lio_exit_kernel
+       BPON
 .Lio_exit_timer:
        stpt    __LC_EXIT_TIMER
        mvc     __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
+.Lio_exit_kernel:
        lmg     %r11,%r15,__PT_R11(%r11)
        lpswe   __LC_RETURN_PSW
 .Lio_done:
@@ -775,6 +819,7 @@ ENTRY(io_int_handler)
 ENTRY(ext_int_handler)
        STCK    __LC_INT_CLOCK
        stpt    __LC_ASYNC_ENTER_TIMER
+       BPOFF
        stmg    %r8,%r15,__LC_SAVE_AREA_ASYNC
        lg      %r10,__LC_LAST_BREAK
        lg      %r12,__LC_THREAD_INFO
@@ -824,6 +869,7 @@ ENTRY(psw_idle)
 .Lpsw_idle_stcctm:
 #endif
        oi      __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
+       BPON
        STCK    __CLOCK_IDLE_ENTER(%r2)
        stpt    __TIMER_IDLE_ENTER(%r2)
 .Lpsw_idle_lpsw:
@@ -931,6 +977,7 @@ load_fpu_regs:
  */
 ENTRY(mcck_int_handler)
        STCK    __LC_MCCK_CLOCK
+       BPOFF
        la      %r1,4095                # revalidate r1
        spt     __LC_CPU_TIMER_SAVE_AREA-4095(%r1)      # revalidate cpu timer
        lmg     %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
@@ -997,6 +1044,7 @@ ENTRY(mcck_int_handler)
        mvc     __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
        tm      __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
        jno     0f
+       BPON
        stpt    __LC_EXIT_TIMER
        mvc     __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
 0:     lmg     %r11,%r15,__PT_R11(%r11)

diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 39127b6..df49f2a 100644 (file)
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -563,6 +563,7 @@ static struct kset *ipl_kset;
 
 static void __ipl_run(void *unused)
 {
+       __bpon();
        diag308(DIAG308_LOAD_CLEAR, NULL);
        if (MACHINE_IS_VM)
                __cpcmd("IPL", NULL, 0, NULL);

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 63c8a84..b34ff07 100644 (file)
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -304,6 +304,7 @@ static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
        mem_assign_absolute(lc->restart_fn, (unsigned long) func);
        mem_assign_absolute(lc->restart_data, (unsigned long) data);
        mem_assign_absolute(lc->restart_source, source_cpu);
+       __bpon();
        asm volatile(
                "0:     sigp    0,%0,%2 # sigp restart to target cpu\n"
                "       brc     2,0b    # busy, try again\n"
@@ -877,6 +878,7 @@ void __cpu_die(unsigned int cpu)
 void __noreturn cpu_die(void)
 {
        idle_task_exit();
+       __bpon();
        pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
        for (;;) ;
 }