From 6e7be06e072cddc5d34617f28f60a32484b9e910 Mon Sep 17 00:00:00 2001 From: Zhang Xianyi Date: Fri, 19 Feb 2016 17:56:07 -0500 Subject: [PATCH] Refs JuliaLang/julia#5728. Fix gemv performance bug on Haswell Mac OSX. On Mac OS X, it should use .align 4 (equal to .align 16 on Linux). I didn't get the performance benefit from .align. Thus, I deleted it. --- kernel/x86_64/cgemv_n_microk_haswell-4.c | 8 ++-- kernel/x86_64/cgemv_t_microk_haswell-4.c | 6 +-- kernel/x86_64/dgemv_n_4.c | 6 +-- kernel/x86_64/dgemv_n_microk_haswell-4.c | 77 +++++++++++++++++++++++++++++++- kernel/x86_64/dgemv_t_4.c | 6 +-- kernel/x86_64/dgemv_t_microk_haswell-4.c | 2 +- kernel/x86_64/sgemv_n_4.c | 6 +-- kernel/x86_64/sgemv_n_microk_haswell-4.c | 4 +- kernel/x86_64/sgemv_t_4.c | 6 +-- kernel/x86_64/sgemv_t_microk_haswell-4.c | 2 +- kernel/x86_64/zgemv_n_microk_haswell-4.c | 8 ++-- kernel/x86_64/zgemv_t_microk_haswell-4.c | 6 +-- 12 files changed, 106 insertions(+), 31 deletions(-) diff --git a/kernel/x86_64/cgemv_n_microk_haswell-4.c b/kernel/x86_64/cgemv_n_microk_haswell-4.c index 2b9b1f2..3603450 100644 --- a/kernel/x86_64/cgemv_n_microk_haswell-4.c +++ b/kernel/x86_64/cgemv_n_microk_haswell-4.c @@ -51,7 +51,7 @@ static void cgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "cmpq $0 , %1 \n\t" "je 2f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 320(%4,%0,4) \n\t" "vmovups (%4,%0,4), %%ymm8 \n\t" // 4 complex values form a0 @@ -202,7 +202,7 @@ static void cgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "cmpq $0 , %1 \n\t" "je 2f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 320(%4,%0,4) \n\t" "vmovups (%4,%0,4), %%ymm8 \n\t" // 4 complex values form a0 @@ -322,7 +322,7 @@ static void cgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) "cmpq $0 , %1 \n\t" "je 2f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 320(%4,%0,4) \n\t" "vmovups (%4,%0,4), %%ymm8 \n\t" // 4 complex values form a0 @@ -454,7 +454,7 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT a "cmpq $0 , %1 \n\t" "je 2f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "vmovups (%2,%0,4), %%ymm8 \n\t" // 4 complex values from src "vmovups 32(%2,%0,4), %%ymm9 \n\t" diff --git a/kernel/x86_64/cgemv_t_microk_haswell-4.c b/kernel/x86_64/cgemv_t_microk_haswell-4.c index 5e48650..1c20751 100644 --- a/kernel/x86_64/cgemv_t_microk_haswell-4.c +++ b/kernel/x86_64/cgemv_t_microk_haswell-4.c @@ -76,7 +76,7 @@ static void cgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%4,%0,4) \n\t" "vmovups (%4,%0,4), %%ymm4 \n\t" // 4 complex values from a0 @@ -292,7 +292,7 @@ static void cgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%4,%0,4) \n\t" "vmovups (%4,%0,4), %%ymm4 \n\t" // 4 complex values from a0 @@ -446,7 +446,7 @@ static void cgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT * "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%4,%0,4) \n\t" "vmovups (%4,%0,4), %%ymm4 \n\t" // 4 complex values from a0 diff --git a/kernel/x86_64/dgemv_n_4.c b/kernel/x86_64/dgemv_n_4.c index 62016fc..485b234 100644 --- a/kernel/x86_64/dgemv_n_4.c +++ b/kernel/x86_64/dgemv_n_4.c @@ -82,7 +82,7 @@ static void dgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "shufpd $0, %%xmm12, %%xmm12 \n\t" "shufpd $0, %%xmm13, %%xmm13 \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%3,%0,8), %%xmm4 \n\t" // 2 * y "movups 16(%3,%0,8), %%xmm5 \n\t" // 2 * y @@ -129,7 +129,7 @@ static void dgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT #endif -#ifndef HAVE_KERNEL_4x2 +#ifndef HAVE_KERNEL_4x1 static void dgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline)); @@ -144,7 +144,7 @@ static void dgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *a "mulsd (%5), %%xmm12 \n\t" // alpha "shufpd $0, %%xmm12, %%xmm12 \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%4,%0,8), %%xmm8 \n\t" // 2 * a "movups 16(%4,%0,8), %%xmm9 \n\t" // 2 * a diff --git a/kernel/x86_64/dgemv_n_microk_haswell-4.c b/kernel/x86_64/dgemv_n_microk_haswell-4.c index b9f6440..7b36ffe 100644 --- a/kernel/x86_64/dgemv_n_microk_haswell-4.c +++ b/kernel/x86_64/dgemv_n_microk_haswell-4.c @@ -52,7 +52,7 @@ static void dgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "subq $4 , %1 \n\t" "jz 2f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "vmulpd %%ymm0 , %%ymm12, %%ymm4 \n\t" @@ -114,3 +114,78 @@ static void dgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT } +#define HAVE_KERNEL_4x2 + +static void dgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline)); + +static void dgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) +{ + + BLASLONG register i = 0; + + __asm__ __volatile__ + ( + "vbroadcastsd (%2), %%ymm12 \n\t" // x0 + "vbroadcastsd 8(%2), %%ymm13 \n\t" // x1 + + "vmovups (%4,%0,8), %%ymm0 \n\t" + "vmovups (%5,%0,8), %%ymm1 \n\t" + + "vbroadcastsd (%6), %%ymm6 \n\t" // alpha + + "addq $4 , %0 \n\t" + "subq $4 , %1 \n\t" + "jz 2f \n\t" + + "1: \n\t" + + "vmulpd %%ymm0 , %%ymm12, %%ymm4 \n\t" + "vmulpd %%ymm1 , %%ymm13, %%ymm5 \n\t" + "vmovups (%4,%0,8), %%ymm0 \n\t" + "vmovups (%5,%0,8), %%ymm1 \n\t" + + "vmovups -32(%3,%0,8), %%ymm8 \n\t" // 4 * y + "vaddpd %%ymm4 , %%ymm5 , %%ymm4 \n\t" + "vfmadd231pd %%ymm6 , %%ymm4 , %%ymm8 \n\t" + + "vmovups %%ymm8, -32(%3,%0,8) \n\t" // 4 * y + + "addq $4 , %0 \n\t" + "subq $4 , %1 \n\t" + "jnz 1b \n\t" + + + "2: \n\t" + + "vmulpd %%ymm0 , %%ymm12, %%ymm4 \n\t" + "vmulpd %%ymm1 , %%ymm13, %%ymm5 \n\t" + + + "vmovups -32(%3,%0,8), %%ymm8 \n\t" // 4 * y + "vaddpd %%ymm4 , %%ymm5 , %%ymm4 \n\t" + "vfmadd231pd %%ymm6 , %%ymm4 , %%ymm8 \n\t" + + "vmovups %%ymm8, -32(%3,%0,8) \n\t" // 4 * y + + + "vzeroupper \n\t" + + + : + : + "r" (i), // 0 + "r" (n), // 1 + "r" (x), // 2 + "r" (y), // 3 + "r" (ap[0]), // 4 + "r" (ap[1]), // 5 + "r" (alpha) // 6 + : "cc", + "%xmm0", "%xmm1", + "%xmm4", "%xmm5", + "%xmm6", + "%xmm8", + "%xmm12", "%xmm13", + "memory" + ); +} diff --git a/kernel/x86_64/dgemv_t_4.c b/kernel/x86_64/dgemv_t_4.c index 7c550a7..8ed821d 100644 --- a/kernel/x86_64/dgemv_t_4.c +++ b/kernel/x86_64/dgemv_t_4.c @@ -95,7 +95,7 @@ static void dgemv_kernel_4x2(BLASLONG n, FLOAT *ap0, FLOAT *ap1, FLOAT *x, FLOAT "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%5,%0,8) , %%xmm14 \n\t" // x @@ -171,7 +171,7 @@ static void dgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%3,%0,8) , %%xmm12 \n\t" @@ -245,7 +245,7 @@ static void add_y(BLASLONG n, FLOAT da , FLOAT *src, FLOAT *dest, BLASLONG inc_d "movsd (%2) , %%xmm10 \n\t" "shufpd $0 , %%xmm10 , %%xmm10 \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%3,%0,8) , %%xmm12 \n\t" diff --git a/kernel/x86_64/dgemv_t_microk_haswell-4.c b/kernel/x86_64/dgemv_t_microk_haswell-4.c index 1e76a57..07fca85 100644 --- a/kernel/x86_64/dgemv_t_microk_haswell-4.c +++ b/kernel/x86_64/dgemv_t_microk_haswell-4.c @@ -59,7 +59,7 @@ static void dgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" // "prefetcht0 384(%2,%0,8) \n\t" "vmovups (%2,%0,8), %%ymm12 \n\t" // 4 * x diff --git a/kernel/x86_64/sgemv_n_4.c b/kernel/x86_64/sgemv_n_4.c index dc88ea0..c7b4516 100644 --- a/kernel/x86_64/sgemv_n_4.c +++ b/kernel/x86_64/sgemv_n_4.c @@ -131,7 +131,7 @@ static void sgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "shufps $0, %%xmm12, %%xmm12 \n\t" "shufps $0, %%xmm13, %%xmm13 \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%3,%0,4), %%xmm4 \n\t" // 4 * y @@ -189,7 +189,7 @@ static void sgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *a "cmpq $0, %1 \n\t" "je 2f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%3,%0,4), %%xmm4 \n\t" // 4 * y "movups 16(%3,%0,4), %%xmm5 \n\t" // 4 * y @@ -264,7 +264,7 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest) __asm__ __volatile__ ( - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%2,%0,4) , %%xmm12 \n\t" diff --git a/kernel/x86_64/sgemv_n_microk_haswell-4.c b/kernel/x86_64/sgemv_n_microk_haswell-4.c index 79054f6..b4b88ed 100644 --- a/kernel/x86_64/sgemv_n_microk_haswell-4.c +++ b/kernel/x86_64/sgemv_n_microk_haswell-4.c @@ -112,7 +112,7 @@ static void sgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLO "je 4f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "vxorps %%ymm4 , %%ymm4, %%ymm4 \n\t" @@ -246,7 +246,7 @@ static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "je 4f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "vxorps %%ymm4 , %%ymm4, %%ymm4 \n\t" "vxorps %%ymm5 , %%ymm5, %%ymm5 \n\t" diff --git a/kernel/x86_64/sgemv_t_4.c b/kernel/x86_64/sgemv_t_4.c index da91be9..5c7d1a5 100644 --- a/kernel/x86_64/sgemv_t_4.c +++ b/kernel/x86_64/sgemv_t_4.c @@ -105,7 +105,7 @@ static void sgemv_kernel_4x2(BLASLONG n, FLOAT *ap0, FLOAT *ap1, FLOAT *x, FLOAT "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%5,%0,4) , %%xmm14 \n\t" // x @@ -183,7 +183,7 @@ static void sgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) "cmpq $0, %1 \n\t" "je 3f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%3,%0,4) , %%xmm12 \n\t" @@ -258,7 +258,7 @@ static void add_y(BLASLONG n, FLOAT da , FLOAT *src, FLOAT *dest, BLASLONG inc_d "movss (%2) , %%xmm10 \n\t" "shufps $0 , %%xmm10 , %%xmm10 \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "movups (%3,%0,4) , %%xmm12 \n\t" diff --git a/kernel/x86_64/sgemv_t_microk_haswell-4.c b/kernel/x86_64/sgemv_t_microk_haswell-4.c index 14fe1ec..eca8586 100644 --- a/kernel/x86_64/sgemv_t_microk_haswell-4.c +++ b/kernel/x86_64/sgemv_t_microk_haswell-4.c @@ -75,7 +75,7 @@ static void sgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "je 4f \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 384(%2,%0,4) \n\t" "vmovups (%2,%0,4), %%ymm12 \n\t" // 8 * x diff --git a/kernel/x86_64/zgemv_n_microk_haswell-4.c b/kernel/x86_64/zgemv_n_microk_haswell-4.c index b38cc57..559ed5b 100644 --- a/kernel/x86_64/zgemv_n_microk_haswell-4.c +++ b/kernel/x86_64/zgemv_n_microk_haswell-4.c @@ -47,7 +47,7 @@ static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "vbroadcastsd 56(%2), %%ymm7 \n\t" // imag part x3 - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%4,%0,8) \n\t" "vmovups (%4,%0,8), %%ymm8 \n\t" // 2 complex values form a0 @@ -152,7 +152,7 @@ static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) "vbroadcastsd 24(%2), %%ymm3 \n\t" // imag part x1 - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%4,%0,8) \n\t" "vmovups (%4,%0,8), %%ymm8 \n\t" // 2 complex values form a0 @@ -236,7 +236,7 @@ static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) "vbroadcastsd (%2), %%ymm0 \n\t" // real part x0 "vbroadcastsd 8(%2), %%ymm1 \n\t" // imag part x0 - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%4,%0,8) \n\t" "vmovups (%4,%0,8), %%ymm8 \n\t" // 2 complex values form a0 @@ -338,7 +338,7 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT a "vbroadcastsd (%4), %%ymm0 \n\t" // alpha_r "vbroadcastsd (%5), %%ymm1 \n\t" // alpha_i - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%2,%0,8) \n\t" "vmovups (%2,%0,8), %%ymm8 \n\t" // 2 complex values from src diff --git a/kernel/x86_64/zgemv_t_microk_haswell-4.c b/kernel/x86_64/zgemv_t_microk_haswell-4.c index 8a851a5..e391012 100644 --- a/kernel/x86_64/zgemv_t_microk_haswell-4.c +++ b/kernel/x86_64/zgemv_t_microk_haswell-4.c @@ -46,7 +46,7 @@ static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "vxorpd %%ymm14, %%ymm14, %%ymm14 \n\t" "vxorpd %%ymm15, %%ymm15, %%ymm15 \n\t" - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%2,%0,8) \n\t" @@ -219,7 +219,7 @@ static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT "vxorpd %%ymm10, %%ymm10, %%ymm10 \n\t" // temp "vxorpd %%ymm11, %%ymm11, %%ymm11 \n\t" // temp - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%2,%0,8) \n\t" @@ -341,7 +341,7 @@ static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT * "vxorpd %%ymm8 , %%ymm8 , %%ymm8 \n\t" // temp "vxorpd %%ymm9 , %%ymm9 , %%ymm9 \n\t" // temp - ".align 16 \n\t" + // ".align 16 \n\t" "1: \n\t" "prefetcht0 192(%2,%0,8) \n\t" -- 2.7.4