Merged some improvements into dgemm_kernel_4x4_bulldozer.S.

Changed the copy functions to generic to solve prefetch conflicts

diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER
index d596685..2ac035f 100644 (file)
--- a/kernel/x86_64/KERNEL.BULLDOZER
+++ b/kernel/x86_64/KERNEL.BULLDOZER
@@ -13,8 +13,8 @@ SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
 DGEMMKERNEL    =  dgemm_kernel_4x4_bulldozer.S
 DGEMMINCOPY    =
 DGEMMITCOPY    =
-DGEMMONCOPY    =  gemm_ncopy_4_opteron.S
-DGEMMOTCOPY    =  gemm_tcopy_4_opteron.S
+DGEMMONCOPY    =  ../generic/gemm_ncopy_4.c
+DGEMMOTCOPY    =  ../generic/gemm_tcopy_4.c
 DGEMMINCOPYOBJ =
 DGEMMITCOPYOBJ =
 DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)

diff --git a/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S b/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S
index ef98d9f..05bad59 100644 (file)
--- a/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S
+++ b/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S
@@ -37,14 +37,26 @@
 /*********************************************************************/\r
 \r
 /*********************************************************************\r
+* Changelog:\r
+*\r
+* 2013/04/15 Saar\r
+* Prefetch for A and B\r
+* unroll of inner Loop\r
+* using generic versions for ncopy and tcopy\r
+* moved vmovddup        ALPHA, %xmm7 down\r
+* define A_PR1 192\r
+* define B_PR1 512\r
+**********************************************************************/\r
+\r
+/*********************************************************************\r
 * 2013/04/12   Saar\r
 * Performance:\r
 * 3584x3584    89 GFLOPS with 8 threads on 4 modules\r
-*              72 GFLOPS with 4 threads on 4 modules\r
-*              52 GFLOPS with 4 threads on 2 modules\r
-*              42 GFLOPS with 2 threads on 2 modules\r
+*              76 GFLOPS with 4 threads on 4 modules\r
+*              53 GFLOPS with 4 threads on 2 modules\r
+*              46 GFLOPS with 2 threads on 2 modules\r
 *              28 GFLOPS with 2 threads on 1 module\r
-*              22 GFLOPS with 1 thread  on 1 module\r
+*            23,6 GFLOPS with 1 thread  on 1 module\r
 *********************************************************************/\r
 \r
 #define ASSEMBLER\r
@@ -98,27 +110,30 @@
 #define movapd movaps\r
 #define movupd movups\r
 \r
+#define A_PR1 192\r
+#define B_PR1 512\r
+\r
 #define KERNEL1(xx) \\r
        vmovddup        -15 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups         -14 * SIZE(AO, %rax, 4),%xmm2 ;\\r
        vfmaddpd %xmm8,%xmm6,%xmm7,%xmm8 ;\\r
        vfmaddpd %xmm12,%xmm2,%xmm7,%xmm12 ;\\r
        vmovddup        -14 * SIZE(BO, %rax, 4), %xmm5 ;\\r
+       vmovddup        -13 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vfmaddpd %xmm9,%xmm6,%xmm3,%xmm9 ;\\r
        vfmaddpd %xmm13,%xmm2,%xmm3,%xmm13 ;\\r
-       vmovddup        -13 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vmovddup         -8 * SIZE(BO, %rax, 4), %xmm7 ;\\r
        vfmaddpd %xmm10,%xmm6,%xmm5,%xmm10 ;\\r
-       vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
        vfmaddpd %xmm11,%xmm6,%xmm4,%xmm11 ;\\r
+       vmovddup        -12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
        vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
+       vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
        vmovups         -8 * SIZE(AO, %rax, 4),%xmm6 ;\\r
 \r
 #define KERNEL2(xx) \\r
-       vmovddup        -12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vmovddup        -11 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups         -12 * SIZE(AO, %rax, 4), %xmm2 ;\\r
        vmovups         -10 * SIZE(AO, %rax, 4),%xmm0 ;\\r
+       vmovddup        -11 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vfmaddpd %xmm8, %xmm2, %xmm1,%xmm8 ;\\r
        vfmaddpd %xmm12,%xmm0, %xmm1,%xmm12 ;\\r
        vmovddup        -10 * SIZE(BO, %rax, 4), %xmm5 ;\\r
@@ -131,26 +146,26 @@
        vfmaddpd %xmm15,%xmm0, %xmm4,%xmm15 ;\\r
 \r
 #define KERNEL3(xx) \\r
-       vmovddup         -7 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups         -6 * SIZE(AO, %rax, 4),%xmm2 ;\\r
+       vmovddup         -7 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vfmaddpd %xmm8, %xmm6, %xmm7, %xmm8 ;\\r
        vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
        vmovddup         -6 * SIZE(BO, %rax, 4), %xmm5 ;\\r
+       vmovddup         -5 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vfmaddpd %xmm9, %xmm6, %xmm3,%xmm9 ;\\r
        vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         -5 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vmovddup                  (BO, %rax, 4), %xmm7 ;\\r
        vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
        vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
+       vmovddup         -4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
        vfmaddpd %xmm11,%xmm6, %xmm4, %xmm11 ;\\r
        vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
        vmovups                   (AO, %rax, 4), %xmm6 ;\\r
 \r
 #define KERNEL4(xx) \\r
-       vmovddup         -4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vmovddup         -3 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups          -4 * SIZE(AO, %rax, 4), %xmm2 ;\\r
        vmovups          -2 * SIZE(AO, %rax, 4),%xmm0 ;\\r
+       vmovddup         -3 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vfmaddpd %xmm8,%xmm2, %xmm1,%xmm8 ;\\r
        vfmaddpd %xmm12,%xmm0, %xmm1 ,%xmm12;\\r
        vmovddup         -2 * SIZE(BO, %rax, 4), %xmm5 ;\\r
@@ -163,26 +178,26 @@
        vfmaddpd %xmm15,%xmm0, %xmm4,%xmm15 ;\\r
 \r
 #define KERNEL5(xx) \\r
-       vmovddup          1 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups           2 * SIZE(AO, %rax, 4),%xmm2 ;\\r
+       vmovddup          1 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vfmaddpd %xmm8,%xmm6, %xmm7,%xmm8 ;\\r
        vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
        vmovddup          2 * SIZE(BO, %rax, 4), %xmm5 ;\\r
+       vmovddup          3 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vfmaddpd %xmm9,%xmm6, %xmm3,%xmm9 ;\\r
        vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup          3 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vmovddup          8 * SIZE(BO, %rax, 4), %xmm7 ;\\r
        vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
        vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
+       vmovddup        4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
        vfmaddpd %xmm11,%xmm6, %xmm4,%xmm11 ;\\r
        vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
        vmovups           8 * SIZE(AO, %rax, 4), %xmm6 ;\\r
 \r
 #define KERNEL6(xx) \\r
-       vmovddup        4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vmovddup        5 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups         4 * SIZE(AO, %rax, 4), %xmm2 ;\\r
        vmovups         6 * SIZE(AO, %rax, 4),%xmm0 ;\\r
+       vmovddup        5 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vfmaddpd %xmm8,%xmm2, %xmm1,%xmm8 ;\\r
        vfmaddpd %xmm12,%xmm0, %xmm1,%xmm12 ;\\r
        vmovddup          6 * SIZE(BO, %rax, 4), %xmm5 ;\\r
@@ -195,26 +210,26 @@
        vfmaddpd %xmm15,%xmm0, %xmm4,%xmm15 ;\\r
 \r
 #define KERNEL7(xx) \\r
-       vmovddup          9 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups          10 * SIZE(AO, %rax, 4),%xmm2 ;\\r
+       vmovddup          9 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vfmaddpd %xmm8,%xmm6, %xmm7,%xmm8 ;\\r
        vfmaddpd %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
        vmovddup         10 * SIZE(BO, %rax, 4), %xmm5 ;\\r
+       vmovddup         11 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vfmaddpd %xmm9,%xmm6, %xmm3,%xmm9 ;\\r
        vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         11 * SIZE(BO, %rax, 4), %xmm4 ;\\r
        vmovddup         16 * SIZE(BO, %rax, 4), %xmm7 ;\\r
        vfmaddpd %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
        vfmaddpd %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
+       vmovddup         12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
        vfmaddpd %xmm11,%xmm6, %xmm4,%xmm11 ;\\r
        vfmaddpd %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
        vmovups          16 * SIZE(AO, %rax, 4),%xmm6 ;\\r
 \r
 #define KERNEL8(xx) \\r
-       vmovddup         12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vmovddup         13 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vmovups          12 * SIZE(AO, %rax, 4), %xmm2 ;\\r
        vmovups          14 * SIZE(AO, %rax, 4), %xmm0 ;\\r
+       vmovddup         13 * SIZE(BO, %rax, 4), %xmm3 ;\\r
        vfmaddpd %xmm8,%xmm2, %xmm1,%xmm8 ;\\r
        vfmaddpd %xmm12,%xmm0, %xmm1,%xmm12 ;\\r
        vmovddup         14 * SIZE(BO, %rax, 4), %xmm5 ;\\r
@@ -409,12 +424,12 @@
        vxorpd  %xmm13, %xmm13,%xmm13\r
        vxorpd  %xmm14, %xmm14,%xmm14\r
        vxorpd  %xmm15, %xmm15,%xmm15\r
-/*\r
-       prefetcht0       (CO1)\r
-       prefetcht0       (CO1,LDC)\r
-       prefetcht0       (CO2)\r
-       prefetcht0       (CO2,LDC)\r
-*/\r
+\r
+       // prefetchw       (CO1)\r
+       // prefetchw       (CO1,LDC)\r
+       // prefetchw       (CO2)\r
+       // prefetchw       (CO2,LDC)\r
+\r
 #ifndef TRMMKERNEL\r
        movq    K, %rax\r
 #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
@@ -436,44 +451,145 @@
        leaq    (AO, %rax, 4), AO\r
        leaq    (BO, %rax, 4), BO\r
        negq    %rax\r
-       NOBRANCH\r
        je      .L15\r
        // ALIGN_4\r
 \r
        vmovups         -16 * SIZE(AO, %rax, 4),%xmm6 \r
        vmovddup        -16 * SIZE(BO, %rax, 4), %xmm7 \r
-       .align 16\r
 \r
-#define A_PR1 512\r
-#define A_PR2 576\r
-#define B_PR1 256\r
-#define B_PR2 576\r
+       .align 16\r
 \r
 .L12:\r
 \r
-       //prefetcht0 A_PR1(AO,%rax,4)\r
-       // prefetcht0 B_PR1(BO,%rax,4)\r
-       KERNEL1(16 *  0)\r
-       KERNEL2(16 *  0)\r
-       //prefetcht0 A_PR1+64(AO,%rax,4)\r
-       // prefetcht0 B_PR1+64(BO,%rax,4)\r
-       KERNEL3(16 *  0)\r
-       KERNEL4(16 *  0)\r
-       //prefetcht0 A_PR1+128(AO,%rax,4)\r
-       // prefetcht0 B_PR1+128(BO,%rax,4)\r
-       KERNEL5(16 *  0)\r
-       KERNEL6(16 *  0)\r
-       //prefetcht0 A_PR1+192(AO,%rax,4)\r
-       // prefetcht0 B_PR1+192(BO,%rax,4)\r
-       KERNEL7(16 *  0)\r
-       KERNEL8(16 *  0)\r
+#ifndef SMP \r
+\r
+       prefetcht0 A_PR1(AO,%rax,4)\r
+        prefetcht0 B_PR1(BO,%rax,4)\r
+        KERNEL1(16 *  0)\r
+        KERNEL2(16 *  0)\r
+        prefetcht0 A_PR1+64(AO,%rax,4)\r
+        prefetcht0 B_PR1+64(BO,%rax,4)\r
+        KERNEL3(16 *  0)\r
+        KERNEL4(16 *  0)\r
+        prefetcht0 A_PR1+128(AO,%rax,4)\r
+        prefetcht0 B_PR1+128(BO,%rax,4)\r
+        KERNEL5(16 *  0)\r
+        KERNEL6(16 *  0)\r
+        prefetcht0 A_PR1+192(AO,%rax,4)\r
+        prefetcht0 B_PR1+192(BO,%rax,4)\r
+        KERNEL7(16 *  0)\r
+        KERNEL8(16 *  0)\r
+\r
+       addq    $8 * SIZE, %rax \r
+       je      .L15\r
+\r
+       prefetcht0 A_PR1(AO,%rax,4)\r
+        prefetcht0 B_PR1(BO,%rax,4)\r
+        KERNEL1(16 *  0)\r
+        KERNEL2(16 *  0)\r
+        prefetcht0 A_PR1+64(AO,%rax,4)\r
+        prefetcht0 B_PR1+64(BO,%rax,4)\r
+        KERNEL3(16 *  0)\r
+        KERNEL4(16 *  0)\r
+        prefetcht0 A_PR1+128(AO,%rax,4)\r
+        prefetcht0 B_PR1+128(BO,%rax,4)\r
+        KERNEL5(16 *  0)\r
+        KERNEL6(16 *  0)\r
+        prefetcht0 A_PR1+192(AO,%rax,4)\r
+        prefetcht0 B_PR1+192(BO,%rax,4)\r
+        KERNEL7(16 *  0)\r
+        KERNEL8(16 *  0)\r
+\r
+       addq    $8 * SIZE, %rax \r
+       je      .L15\r
+\r
+\r
+       prefetcht0 A_PR1(AO,%rax,4)\r
+        prefetcht0 B_PR1(BO,%rax,4)\r
+        KERNEL1(16 *  0)\r
+        KERNEL2(16 *  0)\r
+        prefetcht0 A_PR1+64(AO,%rax,4)\r
+        prefetcht0 B_PR1+64(BO,%rax,4)\r
+        KERNEL3(16 *  0)\r
+        KERNEL4(16 *  0)\r
+        prefetcht0 A_PR1+128(AO,%rax,4)\r
+        prefetcht0 B_PR1+128(BO,%rax,4)\r
+        KERNEL5(16 *  0)\r
+        KERNEL6(16 *  0)\r
+        prefetcht0 A_PR1+192(AO,%rax,4)\r
+        prefetcht0 B_PR1+192(BO,%rax,4)\r
+        KERNEL7(16 *  0)\r
+        KERNEL8(16 *  0)\r
+\r
+       addq    $8 * SIZE, %rax \r
+       je      .L15\r
+\r
+       prefetcht0 A_PR1(AO,%rax,4)\r
+        prefetcht0 B_PR1(BO,%rax,4)\r
+        KERNEL1(16 *  0)\r
+        KERNEL2(16 *  0)\r
+        prefetcht0 A_PR1+64(AO,%rax,4)\r
+        prefetcht0 B_PR1+64(BO,%rax,4)\r
+        KERNEL3(16 *  0)\r
+        KERNEL4(16 *  0)\r
+        prefetcht0 A_PR1+128(AO,%rax,4)\r
+        prefetcht0 B_PR1+128(BO,%rax,4)\r
+        KERNEL5(16 *  0)\r
+        KERNEL6(16 *  0)\r
+        prefetcht0 A_PR1+192(AO,%rax,4)\r
+        prefetcht0 B_PR1+192(BO,%rax,4)\r
+        KERNEL7(16 *  0)\r
+        KERNEL8(16 *  0)\r
+\r
        addq    $8 * SIZE, %rax \r
        jnz     .L12\r
 \r
-       ALIGN_4\r
+       .align 16\r
+#else\r
+#ifdef OPTMODULE\r
+\r
+       prefetcht0 A_PR1(AO,%rax,4)\r
+        prefetcht0 B_PR1(BO,%rax,4)\r
+        KERNEL1(16 *  0)\r
+        KERNEL2(16 *  0)\r
+        prefetcht0 A_PR1+64(AO,%rax,4)\r
+        prefetcht0 B_PR1+64(BO,%rax,4)\r
+        KERNEL3(16 *  0)\r
+        KERNEL4(16 *  0)\r
+        prefetcht0 A_PR1+128(AO,%rax,4)\r
+        prefetcht0 B_PR1+128(BO,%rax,4)\r
+        KERNEL5(16 *  0)\r
+        KERNEL6(16 *  0)\r
+        prefetcht0 A_PR1+192(AO,%rax,4)\r
+        prefetcht0 B_PR1+192(BO,%rax,4)\r
+        KERNEL7(16 *  0)\r
+        KERNEL8(16 *  0)\r
+\r
+       addq    $8 * SIZE, %rax \r
+       je      .L15\r
+       jmp     .L12\r
+       .align 16\r
+\r
+#else\r
+        KERNEL1(16 *  0)\r
+        KERNEL2(16 *  0)\r
+        KERNEL3(16 *  0)\r
+        KERNEL4(16 *  0)\r
+        KERNEL5(16 *  0)\r
+        KERNEL6(16 *  0)\r
+        KERNEL7(16 *  0)\r
+        KERNEL8(16 *  0)\r
+\r
+       addq    $8 * SIZE, %rax \r
+       je      .L15\r
+       jmp     .L12\r
+       .align 16\r
+\r
+#endif\r
+#endif\r
+\r
 \r
 .L15:\r
-       vmovddup        ALPHA, %xmm7\r
 \r
 #ifndef TRMMKERNEL\r
        movq    K, %rax\r
@@ -499,6 +615,7 @@
 #else\r
        movq    KKK, %rax\r
 #endif\r
+       vmovddup        ALPHA, %xmm7\r
        andq    $3, %rax                # if (k & 1)\r
        je .L19\r
 \r