x86/entry/64: Move SWAPGS into the common IRET-to-usermode path

All of the code paths that ended up doing IRET to usermode did
SWAPGS immediately beforehand.  Move the SWAPGS into the common
code.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bpetkov@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/27fd6f45b7cd640de38fb9066fd0349bcd11f8e1.1509609304.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 3eeb169..d6ffdc9 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -249,12 +249,14 @@ return_from_SYSCALL_64:
 
        /*
         * Try to use SYSRET instead of IRET if we're returning to
-        * a completely clean 64-bit userspace context.
+        * a completely clean 64-bit userspace context.  If we're not,
+        * go to the slow exit path.
         */
        movq    RCX(%rsp), %rcx
        movq    RIP(%rsp), %r11
-       cmpq    %rcx, %r11                      /* RCX == RIP */
-       jne     opportunistic_sysret_failed
+
+       cmpq    %rcx, %r11      /* SYSRET requires RCX == RIP */
+       jne     swapgs_restore_regs_and_return_to_usermode
 
        /*
         * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
@@ -272,14 +274,14 @@ return_from_SYSCALL_64:
 
        /* If this changed %rcx, it was not canonical */
        cmpq    %rcx, %r11
-       jne     opportunistic_sysret_failed
+       jne     swapgs_restore_regs_and_return_to_usermode
 
        cmpq    $__USER_CS, CS(%rsp)            /* CS must match SYSRET */
-       jne     opportunistic_sysret_failed
+       jne     swapgs_restore_regs_and_return_to_usermode
 
        movq    R11(%rsp), %r11
        cmpq    %r11, EFLAGS(%rsp)              /* R11 == RFLAGS */
-       jne     opportunistic_sysret_failed
+       jne     swapgs_restore_regs_and_return_to_usermode
 
        /*
         * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
@@ -300,12 +302,12 @@ return_from_SYSCALL_64:
         * would never get past 'stuck_here'.
         */
        testq   $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
-       jnz     opportunistic_sysret_failed
+       jnz     swapgs_restore_regs_and_return_to_usermode
 
        /* nothing to check for RSP */
 
        cmpq    $__USER_DS, SS(%rsp)            /* SS must match SYSRET */
-       jne     opportunistic_sysret_failed
+       jne     swapgs_restore_regs_and_return_to_usermode
 
        /*
         * We win! This label is here just for ease of understanding
@@ -318,10 +320,6 @@ syscall_return_via_sysret:
        movq    RSP(%rsp), %rsp
        UNWIND_HINT_EMPTY
        USERGS_SYSRET64
-
-opportunistic_sysret_failed:
-       SWAPGS
-       jmp     restore_regs_and_return_to_usermode
 END(entry_SYSCALL_64)
 
 ENTRY(stub_ptregs_64)
@@ -422,8 +420,7 @@ ENTRY(ret_from_fork)
        movq    %rsp, %rdi
        call    syscall_return_slowpath /* returns with IRQs disabled */
        TRACE_IRQS_ON                   /* user mode is traced as IRQS on */
-       SWAPGS
-       jmp     restore_regs_and_return_to_usermode
+       jmp     swapgs_restore_regs_and_return_to_usermode
 
 1:
        /* kernel thread */
@@ -611,9 +608,8 @@ GLOBAL(retint_user)
        mov     %rsp,%rdi
        call    prepare_exit_to_usermode
        TRACE_IRQS_IRETQ
-       SWAPGS
 
-GLOBAL(restore_regs_and_return_to_usermode)
+GLOBAL(swapgs_restore_regs_and_return_to_usermode)
 #ifdef CONFIG_DEBUG_ENTRY
        /* Assert that pt_regs indicates user mode. */
        testl   $3, CS(%rsp)
@@ -621,6 +617,7 @@ GLOBAL(restore_regs_and_return_to_usermode)
        ud2
 1:
 #endif
+       SWAPGS
        RESTORE_EXTRA_REGS
        RESTORE_C_REGS
        REMOVE_PT_GPREGS_FROM_STACK 8
@@ -1342,8 +1339,7 @@ ENTRY(nmi)
         * Return back to user mode.  We must *not* do the normal exit
         * work, because we don't want to enable interrupts.
         */
-       SWAPGS
-       jmp     restore_regs_and_return_to_usermode
+       jmp     swapgs_restore_regs_and_return_to_usermode
 
 .Lnmi_from_kernel:
        /*

diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 9ca014a..932b96c 100644 (file)
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -336,8 +336,7 @@ ENTRY(entry_INT80_compat)
 
        /* Go back to user mode. */
        TRACE_IRQS_ON
-       SWAPGS
-       jmp     restore_regs_and_return_to_usermode
+       jmp     swapgs_restore_regs_and_return_to_usermode
 END(entry_INT80_compat)
 
 ENTRY(stub32_clone)