USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
**********************************************************************************/\r
\r
+/*********************************************************************\r
+* 2014/07/29 Saar\r
+* BLASTEST : OK\r
+* CTEST : OK\r
+* TEST : OK\r
+*\r
+* 2013/10/28 Saar\r
+* Parameter:\r
+* CGEMM_DEFAULT_UNROLL_N 2\r
+* CGEMM_DEFAULT_UNROLL_M 8\r
+* CGEMM_DEFAULT_P 768\r
+* CGEMM_DEFAULT_Q 512\r
+* A_PR1 512\r
+* B_PR1 512\r
+*\r
+* 2014/07/29 Saar\r
+* Performance at 6192x6192x6192:\r
+* 1 thread: 49 GFLOPS (MKL: 52)\r
+* 2 threads: 99 GFLOPS (MKL: 102)\r
+* 3 threads: 148 GFLOPS (MKL: 150)\r
+* 4 threads: 195 GFLOPS (MKL: 194)\r
+* 8 threads: 354 GFLOPS (MKL: 317)\r
+*\r
+*\r
+*********************************************************************/\r
+\r
\r
#define ASSEMBLER\r
#include "common.h"\r
\r
/***************************************************************************************************************************/\r
\r
-.macro KERNEL8x2_SUB\r
+.macro KERNEL8x2_1\r
\r
vmovups -16 * SIZE(AO, %rax, SIZE), %ymm0\r
vbroadcastss -8 * SIZE(BO, BI, SIZE), %ymm4\r
- VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )\r
vmovups -8 * SIZE(AO, %rax, SIZE), %ymm1\r
- VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )\r
vbroadcastss -7 * SIZE(BO, BI, SIZE), %ymm5\r
+ prefetcht0 A_PR1(AO, %rax, SIZE)\r
+\r
+ VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )\r
+ vbroadcastss -6 * SIZE(BO, BI, SIZE), %ymm6\r
VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )\r
+ vbroadcastss -5 * SIZE(BO, BI, SIZE), %ymm7\r
+ VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )\r
VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )\r
- vbroadcastss -6 * SIZE(BO, BI, SIZE), %ymm6\r
+\r
+\r
+ VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )\r
+ vbroadcastss -4 * SIZE(BO, BI, SIZE), %ymm4\r
+ VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )\r
+ vbroadcastss -3 * SIZE(BO, BI, SIZE), %ymm5\r
+ VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )\r
+ vmovups 0 * SIZE(AO, %rax, SIZE), %ymm0\r
+ VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )\r
+\r
+ vmovups 8 * SIZE(AO, %rax, SIZE), %ymm1\r
+ prefetcht0 A_PR1+64(AO, %rax, SIZE)\r
+\r
+ VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )\r
+ vbroadcastss -2 * SIZE(BO, BI, SIZE), %ymm6\r
+ VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )\r
+ vbroadcastss -1 * SIZE(BO, BI, SIZE), %ymm7\r
+ VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )\r
+ VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )\r
+\r
+\r
+ VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )\r
+ vbroadcastss 0 * SIZE(BO, BI, SIZE), %ymm4\r
+ VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )\r
+ vbroadcastss 1 * SIZE(BO, BI, SIZE), %ymm5\r
+ VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )\r
+ vmovups 16 * SIZE(AO, %rax, SIZE), %ymm0\r
+ VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )\r
+\r
+ vmovups 24 * SIZE(AO, %rax, SIZE), %ymm1\r
+ prefetcht0 A_PR1+128(AO, %rax, SIZE)\r
+\r
+ VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )\r
+ vbroadcastss 2 * SIZE(BO, BI, SIZE), %ymm6\r
+ VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )\r
+ vbroadcastss 3 * SIZE(BO, BI, SIZE), %ymm7\r
+ VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )\r
+ VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )\r
+\r
+\r
VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )\r
+ vbroadcastss 4 * SIZE(BO, BI, SIZE), %ymm4\r
+ VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )\r
+ vbroadcastss 5 * SIZE(BO, BI, SIZE), %ymm5\r
VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )\r
+ vmovups 32 * SIZE(AO, %rax, SIZE), %ymm0\r
+ VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )\r
+\r
+ vmovups 40 * SIZE(AO, %rax, SIZE), %ymm1\r
+ prefetcht0 A_PR1+192(AO, %rax, SIZE)\r
+\r
+ VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )\r
+ vbroadcastss 6 * SIZE(BO, BI, SIZE), %ymm6\r
+ VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )\r
+ vbroadcastss 7 * SIZE(BO, BI, SIZE), %ymm7\r
+ VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )\r
+ VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )\r
+\r
+ VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )\r
+ VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )\r
+ addq $ 16, BI \r
+ VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )\r
+ VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )\r
+\r
+ addq $ 64, %rax \r
+.endm\r
+\r
+\r
+.macro KERNEL8x2_SUB\r
+\r
+ vmovups -16 * SIZE(AO, %rax, SIZE), %ymm0\r
+ vmovups -8 * SIZE(AO, %rax, SIZE), %ymm1\r
+ vbroadcastss -8 * SIZE(BO, BI, SIZE), %ymm4\r
+ vbroadcastss -7 * SIZE(BO, BI, SIZE), %ymm5\r
+\r
+ VFMADDPS_YR( %ymm8,%ymm4,%ymm0 )\r
+ vbroadcastss -6 * SIZE(BO, BI, SIZE), %ymm6\r
+ VFMADDPS_YI( %ymm9,%ymm5,%ymm0 )\r
vbroadcastss -5 * SIZE(BO, BI, SIZE), %ymm7\r
+ VFMADDPS_YR( %ymm12,%ymm4,%ymm1 )\r
+ VFMADDPS_YI( %ymm13,%ymm5,%ymm1 )\r
+\r
+\r
+ VFMADDPS_YR( %ymm10,%ymm6,%ymm0 )\r
VFMADDPS_YI( %ymm11,%ymm7,%ymm0 )\r
+ VFMADDPS_YR( %ymm14,%ymm6,%ymm1 )\r
VFMADDPS_YI( %ymm15,%ymm7,%ymm1 )\r
+\r
addq $ 4 , BI \r
addq $ 16, %rax \r
.endm\r
\r
.L2_8_12:\r
\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
prefetcht0 B_PR1(BO,BI,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
+ KERNEL8x2_1\r
\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
prefetcht0 B_PR1(BO,BI,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
+ KERNEL8x2_1\r
\r
je .L2_8_16\r
\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
prefetcht0 B_PR1(BO,BI,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
+ KERNEL8x2_1\r
\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
prefetcht0 B_PR1(BO,BI,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
- prefetcht0 A_PR1(AO,%rax,SIZE)\r
- KERNEL8x2_SUB\r
+ KERNEL8x2_1\r
\r
je .L2_8_16\r
\r
.L2_4_12:\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL4x2_SUB\r
KERNEL4x2_SUB\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
KERNEL4x2_SUB\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL4x2_SUB\r
KERNEL4x2_SUB\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
je .L2_4_16\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL4x2_SUB\r
KERNEL4x2_SUB\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
KERNEL4x2_SUB\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL4x2_SUB\r
KERNEL4x2_SUB\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
.L2_4_22:\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
je .L2_4_26\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
KERNEL2x2_SUB\r
.L2_4_42:\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
je .L2_4_46\r
\r
prefetcht0 A_PR1(AO,%rax,SIZE)\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
\r
- prefetcht0 B_PR1(BO,BI,SIZE)\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r
KERNEL1x2_SUB\r