s390/vdso: optimize getcpu system call

Add the CPU number to the per-cpu vdso data page and add the
__kernel_getcpu function to the vdso object to retrieve the
CPU number in user space.

Suggested-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Reviewed-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h
index f9b02d3..d0a2dbf 100644 (file)
--- a/arch/s390/include/asm/vdso.h
+++ b/arch/s390/include/asm/vdso.h
@@ -38,6 +38,8 @@ struct vdso_data {
 struct vdso_per_cpu_data {
        __u64 ectg_timer_base;
        __u64 ectg_user_time;
+       __u32 cpu_nr;
+       __u32 node_id;
 };
 
 extern struct vdso_data *vdso_data;

diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index d5916ef..53bbc9e 100644 (file)
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -80,6 +80,8 @@ int main(void)
        OFFSET(__VDSO_TK_SHIFT, vdso_data, tk_shift);
        OFFSET(__VDSO_ECTG_BASE, vdso_per_cpu_data, ectg_timer_base);
        OFFSET(__VDSO_ECTG_USER, vdso_per_cpu_data, ectg_user_time);
+       OFFSET(__VDSO_CPU_NR, vdso_per_cpu_data, cpu_nr);
+       OFFSET(__VDSO_NODE_ID, vdso_per_cpu_data, node_id);
        BLANK();
        /* constants used by the vdso */
        DEFINE(__CLOCK_REALTIME, CLOCK_REALTIME);

diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c
index 604f4a6..94495ca 100644 (file)
--- a/arch/s390/kernel/vdso.c
+++ b/arch/s390/kernel/vdso.c
@@ -80,7 +80,7 @@ struct vdso_data *vdso_data = &vdso_data_store.data;
 /*
  * Setup vdso data page.
  */
-static void vdso_init_data(struct vdso_data *vd)
+static void __init vdso_init_data(struct vdso_data *vd)
 {
        vd->ectg_available = test_facility(31);
 }
@@ -93,6 +93,7 @@ static void vdso_init_data(struct vdso_data *vd)
 int vdso_alloc_per_cpu(struct lowcore *lowcore)
 {
        unsigned long segment_table, page_table, page_frame;
+       struct vdso_per_cpu_data *vd;
        u32 *psal, *aste;
        int i;
 
@@ -107,6 +108,12 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore)
        if (!segment_table || !page_table || !page_frame)
                goto out;
 
+       /* Initialize per-cpu vdso data page */
+       vd = (struct vdso_per_cpu_data *) page_frame;
+       vd->cpu_nr = lowcore->cpu_nr;
+       vd->node_id = cpu_to_node(vd->cpu_nr);
+
+       /* Set up access register mode page table */
        clear_table((unsigned long *) segment_table, _SEGMENT_ENTRY_EMPTY,
                    PAGE_SIZE << SEGMENT_ORDER);
        clear_table((unsigned long *) page_table, _PAGE_INVALID,

diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile
index ee8a18e..f9c4595 100644 (file)
--- a/arch/s390/kernel/vdso32/Makefile
+++ b/arch/s390/kernel/vdso32/Makefile
@@ -1,6 +1,6 @@
 # List of files in the vdso, has to be asm only for now
 
-obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o
+obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o
 
 # Build rules
 

diff --git a/arch/s390/kernel/vdso32/getcpu.S b/arch/s390/kernel/vdso32/getcpu.S
new file mode 100644 (file)
index 0000000..c1ed0b7
--- /dev/null
+++ b/arch/s390/kernel/vdso32/getcpu.S
@@ -0,0 +1,43 @@
+/*
+ * Userland implementation of getcpu() for 32 bits processes in a
+ * s390 kernel for use in the vDSO
+ *
+ *  Copyright IBM Corp. 2016
+ *  Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+#include <asm/vdso.h>
+#include <asm/asm-offsets.h>
+
+       .text
+       .align 4
+       .globl __kernel_getcpu
+       .type  __kernel_getcpu,@function
+__kernel_getcpu:
+       .cfi_startproc
+       ear     %r1,%a4
+       lhi     %r4,1
+       sll     %r4,24
+       sar     %a4,%r4
+       la      %r4,0
+       epsw    %r0,0
+       sacf    512
+       l       %r5,__VDSO_CPU_NR(%r4)
+       l       %r4,__VDSO_NODE_ID(%r4)
+       tml     %r0,0x4000
+       jo      1f
+       tml     %r0,0x8000
+       jno     0f
+       sacf    256
+       j       1f
+0:     sacf    0
+1:     sar     %a4,%r1
+       ltr     %r2,%r2
+       jz      2f
+       st      %r5,0(%r2)
+2:     ltr     %r3,%r3
+       jz      3f
+       st      %r4,0(%r3)
+3:     lhi     %r2,0
+       br      %r14
+       .cfi_endproc
+       .size   __kernel_getcpu,.-__kernel_getcpu

diff --git a/arch/s390/kernel/vdso32/vdso32.lds.S b/arch/s390/kernel/vdso32/vdso32.lds.S
index a8c379f..8f048c2 100644 (file)
--- a/arch/s390/kernel/vdso32/vdso32.lds.S
+++ b/arch/s390/kernel/vdso32/vdso32.lds.S
@@ -132,6 +132,7 @@ VERSION
                __kernel_gettimeofday;
                __kernel_clock_gettime;
                __kernel_clock_getres;
+               __kernel_getcpu;
 
        local: *;
        };

diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile
index c4b03f9..058659c 100644 (file)
--- a/arch/s390/kernel/vdso64/Makefile
+++ b/arch/s390/kernel/vdso64/Makefile
@@ -1,6 +1,6 @@
 # List of files in the vdso, has to be asm only for now
 
-obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o
+obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o
 
 # Build rules
 

diff --git a/arch/s390/kernel/vdso64/getcpu.S b/arch/s390/kernel/vdso64/getcpu.S
new file mode 100644 (file)
index 0000000..4cbe982
--- /dev/null
+++ b/arch/s390/kernel/vdso64/getcpu.S
@@ -0,0 +1,42 @@
+/*
+ * Userland implementation of getcpu() for 64 bits processes in a
+ * s390 kernel for use in the vDSO
+ *
+ *  Copyright IBM Corp. 2016
+ *  Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+#include <asm/vdso.h>
+#include <asm/asm-offsets.h>
+
+       .text
+       .align 4
+       .globl __kernel_getcpu
+       .type  __kernel_getcpu,@function
+__kernel_getcpu:
+       .cfi_startproc
+       ear     %r1,%a4
+       llilh   %r4,0x0100
+       sar     %a4,%r4
+       la      %r4,0
+       epsw    %r0,0
+       sacf    512
+       l       %r5,__VDSO_CPU_NR(%r4)
+       l       %r4,__VDSO_NODE_ID(%r4)
+       tml     %r0,0x4000
+       jo      1f
+       tml     %r0,0x8000
+       jno     0f
+       sacf    256
+       j       1f
+0:     sacf    0
+1:     sar     %a4,%r1
+       ltgr    %r2,%r2
+       jz      2f
+       st      %r5,0(%r2)
+2:     ltgr    %r3,%r3
+       jz      3f
+       st      %r4,0(%r3)
+3:     lghi    %r2,0
+       br      %r14
+       .cfi_endproc
+       .size   __kernel_getcpu,.-__kernel_getcpu

diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S
index 9f5979d..f35455d 100644 (file)
--- a/arch/s390/kernel/vdso64/vdso64.lds.S
+++ b/arch/s390/kernel/vdso64/vdso64.lds.S
@@ -132,6 +132,7 @@ VERSION
                __kernel_gettimeofday;
                __kernel_clock_gettime;
                __kernel_clock_getres;
+               __kernel_getcpu;
 
        local: *;
        };