s390: lowcore stack pointer offsets

Store the stack pointers in the lowcore for the kernel stack, the async
stack and the panic stack with the offset required for the first user.
This avoids an unnecessary add instruction on the system call path.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 94feff7..17d5cc0 100644 (file)
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -45,6 +45,7 @@ _TIF_TRACE    = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
 
 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
 STACK_SIZE  = 1 << STACK_SHIFT
+STACK_INIT  = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
 
 #define BASED(name) name-system_call(%r13)
 
@@ -97,10 +98,10 @@ STACK_SIZE  = 1 << STACK_SHIFT
        sra     %r14,\shift
        jnz     1f
        CHECK_STACK 1<<\shift,\savearea
+       ahi     %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        j       2f
 1:     l       %r15,\stack             # load target stack
-2:     ahi     %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
-       la      %r11,STACK_FRAME_OVERHEAD(%r15)
+2:     la      %r11,STACK_FRAME_OVERHEAD(%r15)
        .endm
 
        .macro  ADD64 high,low,timer
@@ -150,7 +151,7 @@ ENTRY(__switch_to)
        l       %r4,__THREAD_info(%r2)          # get thread_info of prev
        l       %r5,__THREAD_info(%r3)          # get thread_info of next
        lr      %r15,%r5
-       ahi     %r15,STACK_SIZE                 # end of kernel stack of next
+       ahi     %r15,STACK_INIT                 # end of kernel stack of next
        st      %r3,__LC_CURRENT                # store task struct of next
        st      %r5,__LC_THREAD_INFO            # store thread info of next
        st      %r15,__LC_KERNEL_STACK          # store end of kernel stack
@@ -178,7 +179,6 @@ sysc_stm:
        l       %r13,__LC_SVC_NEW_PSW+4
 sysc_per:
        l       %r15,__LC_KERNEL_STACK
-       ahi     %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        la      %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
 sysc_vtime:
        UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
@@ -359,11 +359,11 @@ ENTRY(pgm_check_handler)
        tm      __LC_PGM_ILC+3,0x80     # check for per exception
        jnz     pgm_svcper              # -> single stepped svc
 0:     CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
+       ahi     %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        j       2f
 1:     UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
        l       %r15,__LC_KERNEL_STACK
-2:     ahi     %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
-       la      %r11,STACK_FRAME_OVERHEAD(%r15)
+2:     la      %r11,STACK_FRAME_OVERHEAD(%r15)
        stm     %r0,%r7,__PT_R0(%r11)
        mvc     __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
        stm     %r8,%r9,__PT_PSW(%r11)
@@ -485,7 +485,6 @@ io_work:
 #
 io_work_user:
        l       %r1,__LC_KERNEL_STACK
-       ahi     %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        mvc     STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
        xc      __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
        la      %r11,STACK_FRAME_OVERHEAD(%r1)
@@ -646,7 +645,6 @@ mcck_skip:
        tm      __PT_PSW+1(%r11),0x01   # returning to user ?
        jno     mcck_return
        l       %r1,__LC_KERNEL_STACK   # switch to kernel stack
-       ahi     %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        mvc     STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
        xc      __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
        la      %r11,STACK_FRAME_OVERHEAD(%r15)
@@ -674,6 +672,7 @@ mcck_panic:
        sra     %r14,PAGE_SHIFT
        jz      0f
        l       %r15,__LC_PANIC_STACK
+       j       mcck_skip
 0:     ahi     %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        j       mcck_skip
 
@@ -714,12 +713,10 @@ ENTRY(restart_int_handler)
  */
 stack_overflow:
        l       %r15,__LC_PANIC_STACK   # change to panic stack
-       ahi     %r15,-__PT_SIZE         # create pt_regs
-       stm     %r0,%r7,__PT_R0(%r15)
-       stm     %r8,%r9,__PT_PSW(%r15)
+       la      %r11,STACK_FRAME_OVERHEAD(%r15)
+       stm     %r0,%r7,__PT_R0(%r11)
+       stm     %r8,%r9,__PT_PSW(%r11)
        mvc     __PT_R8(32,%r11),0(%r14)
-       lr      %r15,%r11
-       ahi     %r15,-STACK_FRAME_OVERHEAD
        l       %r1,BASED(1f)
        xc      __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
        lr      %r2,%r11                # pass pointer to pt_regs
@@ -799,15 +796,14 @@ cleanup_system_call:
        mvc     __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
        # set up saved register 11
        l       %r15,__LC_KERNEL_STACK
-       ahi     %r15,-__PT_SIZE
-       st      %r15,12(%r11)           # r11 pt_regs pointer
+       la      %r9,STACK_FRAME_OVERHEAD(%r15)
+       st      %r9,12(%r11)            # r11 pt_regs pointer
        # fill pt_regs
-       mvc     __PT_R8(32,%r15),__LC_SAVE_AREA_SYNC
-       stm     %r0,%r7,__PT_R0(%r15)
-       mvc     __PT_PSW(8,%r15),__LC_SVC_OLD_PSW
-       mvc     __PT_INT_CODE(4,%r15),__LC_SVC_ILC
+       mvc     __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC
+       stm     %r0,%r7,__PT_R0(%r9)
+       mvc     __PT_PSW(8,%r9),__LC_SVC_OLD_PSW
+       mvc     __PT_INT_CODE(4,%r9),__LC_SVC_ILC
        # setup saved register 15
-       ahi     %r15,-STACK_FRAME_OVERHEAD
        st      %r15,28(%r11)           # r15 stack pointer
        # set new psw address and exit
        l       %r9,BASED(cleanup_table+4)      # sysc_do_svc + 0x80000000

diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 2e6d60c..72f230b 100644 (file)
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -39,6 +39,7 @@ __PT_R15     =        __PT_GPRS + 120
 
 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
 STACK_SIZE  = 1 << STACK_SHIFT
+STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
 
 _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
                 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
@@ -124,10 +125,10 @@ _TIF_EXIT_SIE = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING)
        srag    %r14,%r14,\shift
        jnz     1f
        CHECK_STACK 1<<\shift,\savearea
+       aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        j       2f
 1:     lg      %r15,\stack             # load target stack
-2:     aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
-       la      %r11,STACK_FRAME_OVERHEAD(%r15)
+2:     la      %r11,STACK_FRAME_OVERHEAD(%r15)
        .endm
 
        .macro UPDATE_VTIME scratch,enter_timer
@@ -177,7 +178,7 @@ ENTRY(__switch_to)
        lg      %r4,__THREAD_info(%r2)          # get thread_info of prev
        lg      %r5,__THREAD_info(%r3)          # get thread_info of next
        lgr     %r15,%r5
-       aghi    %r15,STACK_SIZE                 # end of kernel stack of next
+       aghi    %r15,STACK_INIT                 # end of kernel stack of next
        stg     %r3,__LC_CURRENT                # store task struct of next
        stg     %r5,__LC_THREAD_INFO            # store thread info of next
        stg     %r15,__LC_KERNEL_STACK          # store end of kernel stack
@@ -203,10 +204,8 @@ sysc_stmg:
        stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
        lg      %r10,__LC_LAST_BREAK
        lg      %r12,__LC_THREAD_INFO
-       larl    %r13,system_call
 sysc_per:
        lg      %r15,__LC_KERNEL_STACK
-       aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        la      %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
 sysc_vtime:
        UPDATE_VTIME %r13,__LC_SYNC_ENTER_TIMER
@@ -389,6 +388,7 @@ ENTRY(pgm_check_handler)
        tm      __LC_PGM_ILC+3,0x80     # check for per exception
        jnz     pgm_svcper              # -> single stepped svc
 0:     CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
+       aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        j       2f
 1:     UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER
        LAST_BREAK %r14
@@ -398,8 +398,7 @@ ENTRY(pgm_check_handler)
        tm      __LC_PGM_ILC+2,0x02     # check for transaction abort
        jz      2f
        mvc     __THREAD_trap_tdb(256,%r14),0(%r13)
-2:     aghi    %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
-       la      %r11,STACK_FRAME_OVERHEAD(%r15)
+2:     la      %r11,STACK_FRAME_OVERHEAD(%r15)
        stmg    %r0,%r7,__PT_R0(%r11)
        mvc     __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
        stmg    %r8,%r9,__PT_PSW(%r11)
@@ -526,7 +525,6 @@ io_work:
 #
 io_work_user:
        lg      %r1,__LC_KERNEL_STACK
-       aghi    %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        mvc     STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
        xc      __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
        la      %r11,STACK_FRAME_OVERHEAD(%r1)
@@ -688,7 +686,6 @@ mcck_skip:
        tm      __PT_PSW+1(%r11),0x01   # returning to user ?
        jno     mcck_return
        lg      %r1,__LC_KERNEL_STACK   # switch to kernel stack
-       aghi    %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
        mvc     STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
        xc      __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
        la      %r11,STACK_FRAME_OVERHEAD(%r1)
@@ -755,14 +752,12 @@ ENTRY(restart_int_handler)
  * Setup a pt_regs so that show_trace can provide a good call trace.
  */
 stack_overflow:
-       lg      %r11,__LC_PANIC_STACK   # change to panic stack
-       aghi    %r11,-__PT_SIZE         # create pt_regs
+       lg      %r15,__LC_PANIC_STACK   # change to panic stack
+       la      %r11,STACK_FRAME_OVERHEAD(%r15)
        stmg    %r0,%r7,__PT_R0(%r11)
        stmg    %r8,%r9,__PT_PSW(%r11)
        mvc     __PT_R8(64,%r11),0(%r14)
        stg     %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
-       lgr     %r15,%r11
-       aghi    %r15,-STACK_FRAME_OVERHEAD
        xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
        lgr     %r2,%r11                # pass pointer to pt_regs
        jg      kernel_stack_overflow
@@ -846,15 +841,14 @@ cleanup_system_call:
        mvc     __TI_last_break(8,%r12),16(%r11)
 0:     # set up saved register r11
        lg      %r15,__LC_KERNEL_STACK
-       aghi    %r15,-__PT_SIZE
-       stg     %r15,24(%r11)           # r11 pt_regs pointer
+       la      %r9,STACK_FRAME_OVERHEAD(%r15)
+       stg     %r9,24(%r11)            # r11 pt_regs pointer
        # fill pt_regs
-       mvc     __PT_R8(64,%r15),__LC_SAVE_AREA_SYNC
-       stmg    %r0,%r7,__PT_R0(%r15)
-       mvc     __PT_PSW(16,%r15),__LC_SVC_OLD_PSW
-       mvc     __PT_INT_CODE(4,%r15),__LC_SVC_ILC
+       mvc     __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC
+       stmg    %r0,%r7,__PT_R0(%r9)
+       mvc     __PT_PSW(16,%r9),__LC_SVC_OLD_PSW
+       mvc     __PT_INT_CODE(4,%r9),__LC_SVC_ILC
        # setup saved register r15
-       aghi    %r15,-STACK_FRAME_OVERHEAD
        stg     %r15,56(%r11)           # r15 stack pointer
        # set new psw address and exit
        larl    %r9,sysc_do_svc

diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 2926885..0f419c5 100644 (file)
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -377,11 +377,14 @@ static void __init setup_lowcore(void)
                PSW_MASK_DAT | PSW_MASK_MCHECK;
        lc->io_new_psw.addr = PSW_ADDR_AMODE | (unsigned long) io_int_handler;
        lc->clock_comparator = -1ULL;
-       lc->kernel_stack = ((unsigned long) &init_thread_union) + THREAD_SIZE;
+       lc->kernel_stack = ((unsigned long) &init_thread_union)
+               + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
        lc->async_stack = (unsigned long)
-               __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0) + ASYNC_SIZE;
+               __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0)
+               + ASYNC_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
        lc->panic_stack = (unsigned long)
-               __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0) + PAGE_SIZE;
+               __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0)
+               + PAGE_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
        lc->current_task = (unsigned long) init_thread_union.thread_info.task;
        lc->thread_info = (unsigned long) &init_thread_union;
        lc->machine_flags = S390_lowcore.machine_flags;

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 549c9d1..8bde89e 100644 (file)
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -181,8 +181,10 @@ static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
        lc = pcpu->lowcore;
        memcpy(lc, &S390_lowcore, 512);
        memset((char *) lc + 512, 0, sizeof(*lc) - 512);
-       lc->async_stack = pcpu->async_stack + ASYNC_SIZE;
-       lc->panic_stack = pcpu->panic_stack + PAGE_SIZE;
+       lc->async_stack = pcpu->async_stack + ASYNC_SIZE
+               - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
+       lc->panic_stack = pcpu->panic_stack + PAGE_SIZE
+               - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
        lc->cpu_nr = cpu;
 #ifndef CONFIG_64BIT
        if (MACHINE_HAS_IEEE) {
@@ -253,7 +255,8 @@ static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
        struct _lowcore *lc = pcpu->lowcore;
        struct thread_info *ti = task_thread_info(tsk);
 
-       lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE;
+       lc->kernel_stack = (unsigned long) task_stack_page(tsk)
+               + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
        lc->thread_info = (unsigned long) task_thread_info(tsk);
        lc->current_task = (unsigned long) tsk;
        lc->user_timer = ti->user_timer;
@@ -810,8 +813,10 @@ void __init smp_prepare_boot_cpu(void)
        pcpu->state = CPU_STATE_CONFIGURED;
        pcpu->address = boot_cpu_address;
        pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();
-       pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE;
-       pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE;
+       pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE
+               + STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
+       pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE
+               + STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
        S390_lowcore.percpu_offset = __per_cpu_offset[0];
        smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
        set_cpu_present(0, true);