Fixed #395. Enable optimized cgemm for Sandybridge. Added optimized sdot kernel.

Fixed c/zgemm, zgemv computational error of haswell, piledriver, bullldozer, and
barcelona on Windows.

Merge branch 'develop' of https://github.com/wernsaar/OpenBLAS into wernsaar-develop

Conflicts:
	kernel/Makefile.L1
	kernel/x86_64/KERNEL
	param.h

diff --git a/Makefile b/Makefile
index 5b0ca0d..397836d 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ endif
 SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench
 
 .PHONY : all libs netlib test ctest shared install
-.NOTPARALLEL : all libs prof lapack-test install
+.NOTPARALLEL : all libs prof lapack-test install blas-test
 
 all :: libs netlib tests shared
        @echo
@@ -282,6 +282,11 @@ lapack-test :
        make -j 1 -C $(NETLIB_LAPACK_DIR)/TESTING xeigtstc  xeigtstd  xeigtsts  xeigtstz  xlintstc  xlintstd  xlintstds  xlintstrfd  xlintstrfz  xlintsts  xlintstz  xlintstzc xlintstrfs xlintstrfc
        (cd $(NETLIB_LAPACK_DIR); ./lapack_testing.py -r )
 
+blas-test:
+       (cd $(NETLIB_LAPACK_DIR)/BLAS && rm -f x* *.out)
+       make -j 1 -C $(NETLIB_LAPACK_DIR) blas_testing
+       (cd $(NETLIB_LAPACK_DIR)/BLAS && cat *.out)
+
 
 dummy :
 

diff --git a/kernel/Makefile.L1 b/kernel/Makefile.L1
index bd31503..7c7cb27 100644 (file)
--- a/kernel/Makefile.L1
+++ b/kernel/Makefile.L1
@@ -687,15 +687,27 @@ $(KDIR)ddot_k$(TSUFFIX).$(SUFFIX) $(KDIR)ddot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNEL
 $(KDIR)qdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)qdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(QDOTKERNEL)
        $(CC) -c $(CFLAGS) -UCOMPLEX -DXDOUBLE $< -o $@
 
-$(KDIR)dsdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)dsdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SDOTKERNEL)
-       $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE -DDSDOT $< -o $@
-
 $(KDIR)sdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)sdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SDOTKERNEL)
        $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE $< -o $@
 
+ifdef DSDOTKERNEL
+
+$(KDIR)dsdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)dsdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DSDOTKERNEL)
+       $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE -DDSDOT $< -o $@
+
+$(KDIR)sdsdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)sdsdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(DSDOTKERNEL)
+       $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE -DDSDOT $< -o $@
+
+else
+
+$(KDIR)dsdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)dsdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SDOTKERNEL)
+       $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE -DDSDOT $< -o $@
+
 $(KDIR)sdsdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)sdsdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SDOTKERNEL)
        $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE -DDSDOT $< -o $@
 
+endif
+
 $(KDIR)zdotu_k$(TSUFFIX).$(SUFFIX) $(KDIR)zdotu_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(ZDOTKERNEL)
        $(CC) -c $(CFLAGS) -DCOMPLEX -DDOUBLE -UCONJ $< -o $@
 

diff --git a/kernel/x86_64/KERNEL b/kernel/x86_64/KERNEL
index d348628..fa6282c 100644 (file)
--- a/kernel/x86_64/KERNEL
+++ b/kernel/x86_64/KERNEL
@@ -119,9 +119,15 @@ XCOPYKERNEL = zcopy.S
 endif
 
 ifndef SDOTKERNEL
-SDOTKERNEL = ../arm/dot.c
+SDOTKERNEL = dot_sse.S
 endif
 
+
+ifndef DSDOTKERNEL
+DSDOTKERNEL = ../arm/dot.c 
+endif
+
+
 ifndef DDOTKERNEL
 DDOTKERNEL =  dot_sse2.S
 endif

diff --git a/kernel/x86_64/KERNEL.BARCELONA b/kernel/x86_64/KERNEL.BARCELONA
index b1e099e..7c73595 100644 (file)
--- a/kernel/x86_64/KERNEL.BARCELONA
+++ b/kernel/x86_64/KERNEL.BARCELONA
@@ -2,7 +2,7 @@ SGEMVNKERNEL = sgemv_n.S
 SGEMVTKERNEL = sgemv_t.S
 
 ZGEMVNKERNEL = zgemv_n_dup.S
-ZGEMVTKERNEL = zgemv_t_dup.S
+ZGEMVTKERNEL = zgemv_t.S
 
 SGEMMKERNEL    =  gemm_kernel_8x4_barcelona.S
 SGEMMINCOPY    =  ../generic/gemm_ncopy_8.c

diff --git a/kernel/x86_64/KERNEL.BOBCAT b/kernel/x86_64/KERNEL.BOBCAT
index 2b6b2fe..313c62d 100644 (file)
--- a/kernel/x86_64/KERNEL.BOBCAT
+++ b/kernel/x86_64/KERNEL.BOBCAT
@@ -1,5 +1,5 @@
 ZGEMVNKERNEL = zgemv_n_dup.S
-ZGEMVTKERNEL = zgemv_t_dup.S
+ZGEMVTKERNEL = zgemv_t.S
 
 SGEMMKERNEL    =  gemm_kernel_8x4_barcelona.S
 SGEMMINCOPY    =  ../generic/gemm_ncopy_8.c

diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER
index f8ac3db..d9b9f84 100644 (file)
--- a/kernel/x86_64/KERNEL.BULLDOZER
+++ b/kernel/x86_64/KERNEL.BULLDOZER
@@ -2,7 +2,7 @@ SGEMVNKERNEL = sgemv_n.S
 SGEMVTKERNEL = sgemv_t.S
 
 ZGEMVNKERNEL = zgemv_n_dup.S
-ZGEMVTKERNEL = zgemv_t_dup.S
+ZGEMVTKERNEL = zgemv_t.S
 
 DGEMVNKERNEL = dgemv_n_bulldozer.S
 DGEMVTKERNEL = dgemv_t_bulldozer.S

diff --git a/kernel/x86_64/KERNEL.OPTERON_SSE3 b/kernel/x86_64/KERNEL.OPTERON_SSE3
index 9367bd6..7218ad6 100644 (file)
--- a/kernel/x86_64/KERNEL.OPTERON_SSE3
+++ b/kernel/x86_64/KERNEL.OPTERON_SSE3
@@ -1,5 +1,5 @@
 ZGEMVNKERNEL = zgemv_n_dup.S
-ZGEMVTKERNEL = zgemv_t_dup.S
+ZGEMVTKERNEL = zgemv_t.S
 
 SGEMMKERNEL    =  gemm_kernel_8x4_sse.S
 SGEMMINCOPY    =  ../generic/gemm_ncopy_8.c

diff --git a/kernel/x86_64/KERNEL.PILEDRIVER b/kernel/x86_64/KERNEL.PILEDRIVER
index a06a04c..b083b5d 100644 (file)
--- a/kernel/x86_64/KERNEL.PILEDRIVER
+++ b/kernel/x86_64/KERNEL.PILEDRIVER
@@ -2,7 +2,7 @@ SGEMVNKERNEL = sgemv_n.S
 SGEMVTKERNEL = sgemv_t.S
 
 ZGEMVNKERNEL = zgemv_n_dup.S
-ZGEMVTKERNEL = zgemv_t_dup.S
+ZGEMVTKERNEL = zgemv_t.S
 
 DGEMVNKERNEL = dgemv_n_bulldozer.S
 DGEMVTKERNEL = dgemv_t_bulldozer.S

diff --git a/kernel/x86_64/KERNEL.PRESCOTT b/kernel/x86_64/KERNEL.PRESCOTT
index 03d6664..9b3e514 100644 (file)
--- a/kernel/x86_64/KERNEL.PRESCOTT
+++ b/kernel/x86_64/KERNEL.PRESCOTT
@@ -1,5 +1,5 @@
 ZGEMVNKERNEL = zgemv_n_dup.S
-ZGEMVTKERNEL = zgemv_t_dup.S
+ZGEMVTKERNEL = zgemv_t.S
 
 SGEMMKERNEL    =  gemm_kernel_8x4_sse3.S
 SGEMMINCOPY    =  ../generic/gemm_ncopy_8.c

diff --git a/kernel/x86_64/KERNEL.SANDYBRIDGE b/kernel/x86_64/KERNEL.SANDYBRIDGE
index 1b4228c..b84a5f3 100644 (file)
--- a/kernel/x86_64/KERNEL.SANDYBRIDGE
+++ b/kernel/x86_64/KERNEL.SANDYBRIDGE
@@ -21,11 +21,11 @@ DGEMMITCOPYOBJ =  dgemm_itcopy$(TSUFFIX).$(SUFFIX)
 DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)
 DGEMMOTCOPYOBJ =  dgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
-CGEMMKERNEL    =  zgemm_kernel_2x4_nehalem.S
-CGEMMINCOPY    =  zgemm_ncopy_2.S
-CGEMMITCOPY    =  zgemm_tcopy_2.S
-CGEMMONCOPY    =  ../generic/zgemm_ncopy_4.c
-CGEMMOTCOPY    =  ../generic/zgemm_tcopy_4.c
+CGEMMKERNEL    =  cgemm_kernel_8x2_sandy.S
+CGEMMINCOPY    =  ../generic/zgemm_ncopy_8.c
+CGEMMITCOPY    =  ../generic/zgemm_tcopy_8.c
+CGEMMONCOPY    =  ../generic/zgemm_ncopy_2.c
+CGEMMOTCOPY    =  ../generic/zgemm_tcopy_2.c
 CGEMMINCOPYOBJ =  cgemm_incopy$(TSUFFIX).$(SUFFIX)
 CGEMMITCOPYOBJ =  cgemm_itcopy$(TSUFFIX).$(SUFFIX)
 CGEMMONCOPYOBJ =  cgemm_oncopy$(TSUFFIX).$(SUFFIX)

diff --git a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S
index 33d3d29..97958a8 100644 (file)
--- a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S
+++ b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S
@@ -522,16 +522,16 @@
 #ifdef WINDOWS_ABI\r
        movq    %rdi,    48(%rsp)\r
        movq    %rsi,    56(%rsp)\r
-       movups  %xmm6,   64(%rsp)\r
-       movups  %xmm7,   80(%rsp)\r
-       movups  %xmm8,   96(%rsp)\r
-       movups  %xmm9,  112(%rsp)\r
-       movups  %xmm10, 128(%rsp)\r
-       movups  %xmm11, 144(%rsp)\r
-       movups  %xmm12, 160(%rsp)\r
-       movups  %xmm13, 176(%rsp)\r
-       movups  %xmm14, 192(%rsp)\r
-       movups  %xmm15, 208(%rsp)\r
+       vmovups %xmm6,   64(%rsp)\r
+       vmovups %xmm7,   80(%rsp)\r
+       vmovups %xmm8,   96(%rsp)\r
+       vmovups %xmm9,  112(%rsp)\r
+       vmovups %xmm10, 128(%rsp)\r
+       vmovups %xmm11, 144(%rsp)\r
+       vmovups %xmm12, 160(%rsp)\r
+       vmovups %xmm13, 176(%rsp)\r
+       vmovups %xmm14, 192(%rsp)\r
+       vmovups %xmm15, 208(%rsp)\r
 \r
        movq    ARG1,      OLD_M\r
        movq    ARG2,      OLD_N\r
@@ -541,14 +541,15 @@
        movq    OLD_C,     C\r
        movq    OLD_LDC,   LDC\r
 #ifdef TRMMKERNEL\r
-       movsd   OLD_OFFSET, %xmm12\r
+       vmovsd  OLD_OFFSET, %xmm12\r
 #endif\r
        vmovaps %xmm3, %xmm0\r
+       vmovsd   OLD_ALPHA_I, %xmm1\r
 \r
 #else\r
        movq    STACKSIZE +  8(%rsp), LDC\r
 #ifdef TRMMKERNEL\r
-       movsd   STACKSIZE + 16(%rsp), %xmm12\r
+       vmovsd  STACKSIZE + 16(%rsp), %xmm12\r
 #endif\r
 \r
 #endif\r
@@ -1865,6 +1866,8 @@
 \r
 \r
 .L999:\r
+       vzeroupper\r
+\r
        movq            SP, %rsp\r
        movq       (%rsp), %rbx\r
        movq      8(%rsp), %rbp\r
@@ -1876,16 +1879,16 @@
 #ifdef WINDOWS_ABI\r
        movq     48(%rsp), %rdi\r
        movq     56(%rsp), %rsi\r
-       movups   64(%rsp), %xmm6\r
-       movups   80(%rsp), %xmm7\r
-       movups   96(%rsp), %xmm8\r
-       movups  112(%rsp), %xmm9\r
-       movups  128(%rsp), %xmm10\r
-       movups  144(%rsp), %xmm11\r
-       movups  160(%rsp), %xmm12\r
-       movups  176(%rsp), %xmm13\r
-       movups  192(%rsp), %xmm14\r
-       movups  208(%rsp), %xmm15\r
+       vmovups  64(%rsp), %xmm6\r
+       vmovups  80(%rsp), %xmm7\r
+       vmovups  96(%rsp), %xmm8\r
+       vmovups 112(%rsp), %xmm9\r
+       vmovups 128(%rsp), %xmm10\r
+       vmovups 144(%rsp), %xmm11\r
+       vmovups 160(%rsp), %xmm12\r
+       vmovups 176(%rsp), %xmm13\r
+       vmovups 192(%rsp), %xmm14\r
+       vmovups 208(%rsp), %xmm15\r
 #endif\r
 \r
        addq    $STACKSIZE, %rsp\r

diff --git a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S
index 76d6810..72deee1 100644 (file)
--- a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S
+++ b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S
@@ -26,7 +26,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *****************************************************************************/\r
 /*********************************************************************\r
 *\r
-* 2013/10/31 Saar\r
+* 2014/06/28 Saar\r
 *        BLASTEST               : OK\r
 *        CTEST                  : OK\r
 *        TEST                   : OK\r
@@ -546,16 +546,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq    %rdi,    48(%rsp)\r
        movq    %rsi,    56(%rsp)\r
-       movups  %xmm6,   64(%rsp)\r
-       movups  %xmm7,   80(%rsp)\r
-       movups  %xmm8,   96(%rsp)\r
-       movups  %xmm9,  112(%rsp)\r
-       movups  %xmm10, 128(%rsp)\r
-       movups  %xmm11, 144(%rsp)\r
-       movups  %xmm12, 160(%rsp)\r
-       movups  %xmm13, 176(%rsp)\r
-       movups  %xmm14, 192(%rsp)\r
-       movups  %xmm15, 208(%rsp)\r
+       vmovups %xmm6,   64(%rsp)\r
+       vmovups %xmm7,   80(%rsp)\r
+       vmovups %xmm8,   96(%rsp)\r
+       vmovups %xmm9,  112(%rsp)\r
+       vmovups %xmm10, 128(%rsp)\r
+       vmovups %xmm11, 144(%rsp)\r
+       vmovups %xmm12, 160(%rsp)\r
+       vmovups %xmm13, 176(%rsp)\r
+       vmovups %xmm14, 192(%rsp)\r
+       vmovups %xmm15, 208(%rsp)\r
 \r
        movq    ARG1,      OLD_M\r
        movq    ARG2,      OLD_N\r
@@ -568,6 +568,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        movsd   OLD_OFFSET, %xmm12\r
 #endif\r
        vmovaps %xmm3, %xmm0\r
+       vmovsd   OLD_ALPHA_I, %xmm1\r
 \r
 #else\r
        movq    STACKSIZE +  8(%rsp), LDC\r
@@ -1889,6 +1890,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 \r
 .L999:\r
+       vzeroupper\r
+\r
        movq            SP, %rsp\r
        movq       (%rsp), %rbx\r
        movq      8(%rsp), %rbp\r
@@ -1900,16 +1903,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq     48(%rsp), %rdi\r
        movq     56(%rsp), %rsi\r
-       movups   64(%rsp), %xmm6\r
-       movups   80(%rsp), %xmm7\r
-       movups   96(%rsp), %xmm8\r
-       movups  112(%rsp), %xmm9\r
-       movups  128(%rsp), %xmm10\r
-       movups  144(%rsp), %xmm11\r
-       movups  160(%rsp), %xmm12\r
-       movups  176(%rsp), %xmm13\r
-       movups  192(%rsp), %xmm14\r
-       movups  208(%rsp), %xmm15\r
+       vmovups  64(%rsp), %xmm6\r
+       vmovups  80(%rsp), %xmm7\r
+       vmovups  96(%rsp), %xmm8\r
+       vmovups 112(%rsp), %xmm9\r
+       vmovups 128(%rsp), %xmm10\r
+       vmovups 144(%rsp), %xmm11\r
+       vmovups 160(%rsp), %xmm12\r
+       vmovups 176(%rsp), %xmm13\r
+       vmovups 192(%rsp), %xmm14\r
+       vmovups 208(%rsp), %xmm15\r
 #endif\r
 \r
        addq    $STACKSIZE, %rsp\r

diff --git a/kernel/x86_64/cgemm_kernel_8x2_haswell.S b/kernel/x86_64/cgemm_kernel_8x2_haswell.S
index 285a9f0..baee3cd 100644 (file)
--- a/kernel/x86_64/cgemm_kernel_8x2_haswell.S
+++ b/kernel/x86_64/cgemm_kernel_8x2_haswell.S
@@ -26,7 +26,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 **********************************************************************************/\r
 \r
 /*********************************************************************\r
-* 2013/11/13 Saar\r
+* 2014/06/28 Saar\r
 *        BLASTEST               : OK\r
 *        CTEST                  : OK\r
 *        TEST                   : OK\r
@@ -816,16 +816,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq    %rdi,    48(%rsp)\r
        movq    %rsi,    56(%rsp)\r
-       movups  %xmm6,   64(%rsp)\r
-       movups  %xmm7,   80(%rsp)\r
-       movups  %xmm8,   96(%rsp)\r
-       movups  %xmm9,  112(%rsp)\r
-       movups  %xmm10, 128(%rsp)\r
-       movups  %xmm11, 144(%rsp)\r
-       movups  %xmm12, 160(%rsp)\r
-       movups  %xmm13, 176(%rsp)\r
-       movups  %xmm14, 192(%rsp)\r
-       movups  %xmm15, 208(%rsp)\r
+       vmovups %xmm6,   64(%rsp)\r
+       vmovups %xmm7,   80(%rsp)\r
+       vmovups %xmm8,   96(%rsp)\r
+       vmovups %xmm9,  112(%rsp)\r
+       vmovups %xmm10, 128(%rsp)\r
+       vmovups %xmm11, 144(%rsp)\r
+       vmovups %xmm12, 160(%rsp)\r
+       vmovups %xmm13, 176(%rsp)\r
+       vmovups %xmm14, 192(%rsp)\r
+       vmovups %xmm15, 208(%rsp)\r
 \r
        movq    ARG1,      OLD_M\r
        movq    ARG2,      OLD_N\r
@@ -838,6 +838,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        movsd   OLD_OFFSET, %xmm12\r
 #endif\r
        vmovaps %xmm3, %xmm0\r
+       vmovsd   OLD_ALPHA_I, %xmm1\r
 \r
 #else\r
        movq    STACKSIZE +  8(%rsp), LDC\r
@@ -2253,6 +2254,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 \r
 .L999:\r
+       vzeroupper\r
+\r
        movq            SP, %rsp\r
        movq       (%rsp), %rbx\r
        movq      8(%rsp), %rbp\r
@@ -2264,16 +2267,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq     48(%rsp), %rdi\r
        movq     56(%rsp), %rsi\r
-       movups   64(%rsp), %xmm6\r
-       movups   80(%rsp), %xmm7\r
-       movups   96(%rsp), %xmm8\r
-       movups  112(%rsp), %xmm9\r
-       movups  128(%rsp), %xmm10\r
-       movups  144(%rsp), %xmm11\r
-       movups  160(%rsp), %xmm12\r
-       movups  176(%rsp), %xmm13\r
-       movups  192(%rsp), %xmm14\r
-       movups  208(%rsp), %xmm15\r
+       vmovups  64(%rsp), %xmm6\r
+       vmovups  80(%rsp), %xmm7\r
+       vmovups  96(%rsp), %xmm8\r
+       vmovups 112(%rsp), %xmm9\r
+       vmovups 128(%rsp), %xmm10\r
+       vmovups 144(%rsp), %xmm11\r
+       vmovups 160(%rsp), %xmm12\r
+       vmovups 176(%rsp), %xmm13\r
+       vmovups 192(%rsp), %xmm14\r
+       vmovups 208(%rsp), %xmm15\r
 #endif\r
 \r
        addq    $ STACKSIZE, %rsp\r

diff --git a/kernel/x86_64/cgemm_kernel_8x2_sandy.S b/kernel/x86_64/cgemm_kernel_8x2_sandy.S
new file mode 100644 (file)
index 0000000..564b733
--- /dev/null
+++ b/kernel/x86_64/cgemm_kernel_8x2_sandy.S
@@ -0,0 +1,2281 @@
+/*********************************************************************************\r
+Copyright (c) 2013, The OpenBLAS Project\r
+All rights reserved.\r
+Redistribution and use in source and binary forms, with or without\r
+modification, are permitted provided that the following conditions are\r
+met:\r
+1. Redistributions of source code must retain the above copyright\r
+notice, this list of conditions and the following disclaimer.\r
+2. Redistributions in binary form must reproduce the above copyright\r
+notice, this list of conditions and the following disclaimer in\r
+the documentation and/or other materials provided with the\r
+distribution.\r
+3. Neither the name of the OpenBLAS project nor the names of\r
+its contributors may be used to endorse or promote products\r
+derived from this software without specific prior written permission.\r
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"\r
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\r
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE\r
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\r
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+**********************************************************************************/\r
+\r
+\r
+#define ASSEMBLER\r
+#include "common.h"\r
+ \r
+#define OLD_M  %rdi\r
+#define OLD_N  %rsi\r
+#define M      %r13\r
+#define J      %r14\r
+#define OLD_K  %rdx\r
+\r
+#define A      %rcx\r
+#define B      %r8\r
+#define C      %r9\r
+#define LDC    %r10\r
+       \r
+#define I      %r11\r
+#define AO     %rdi\r
+#define BO     %rsi\r
+#define        CO1     %r15\r
+#define K      %r12\r
+#define BI     %rbp\r
+#define        SP      %rbx\r
+\r
+#define BO1    %rdi\r
+#define BO2    %r15\r
+\r
+#ifndef WINDOWS_ABI\r
+\r
+#define STACKSIZE 96\r
+\r
+#else\r
+\r
+#define STACKSIZE 320\r
+\r
+#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)\r
+#define OLD_A           48 + STACKSIZE(%rsp)\r
+#define OLD_B           56 + STACKSIZE(%rsp)\r
+#define OLD_C           64 + STACKSIZE(%rsp)\r
+#define OLD_LDC         72 + STACKSIZE(%rsp)\r
+#define OLD_OFFSET      80 + STACKSIZE(%rsp)\r
+\r
+#endif\r
+\r
+#define L_BUFFER_SIZE 8192\r
+\r
+#define Ndiv6   24(%rsp)\r
+#define Nmod6   32(%rsp)\r
+#define N       40(%rsp)\r
+#define ALPHA_R  48(%rsp)\r
+#define ALPHA_I  56(%rsp)\r
+#define OFFSET   64(%rsp)\r
+#define KK       72(%rsp)\r
+#define KKK      80(%rsp)\r
+#define BUFFER1                   128(%rsp)\r
+\r
+#if defined(OS_WINDOWS)\r
+#if   L_BUFFER_SIZE > 16384\r
+#define STACK_TOUCH \\r
+        movl    $ 0,  4096 * 4(%rsp);\\r
+        movl    $ 0,  4096 * 3(%rsp);\\r
+        movl    $ 0,  4096 * 2(%rsp);\\r
+        movl    $ 0,  4096 * 1(%rsp);\r
+#elif L_BUFFER_SIZE > 12288\r
+#define STACK_TOUCH \\r
+        movl    $ 0,  4096 * 3(%rsp);\\r
+        movl    $ 0,  4096 * 2(%rsp);\\r
+        movl    $ 0,  4096 * 1(%rsp);\r
+#elif L_BUFFER_SIZE > 8192\r
+#define STACK_TOUCH \\r
+        movl    $ 0,  4096 * 2(%rsp);\\r
+        movl    $ 0,  4096 * 1(%rsp);\r
+#elif L_BUFFER_SIZE > 4096\r
+#define STACK_TOUCH \\r
+        movl    $ 0,  4096 * 1(%rsp);\r
+#else\r
+#define STACK_TOUCH\r
+#endif\r
+#else\r
+#define STACK_TOUCH\r
+#endif\r
+\r
+\r
+\r
+#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)\r
+\r
+#define        VFMADDPS_YR( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm2;\\r
+                               vaddps y0,%ymm2,y0\r
+\r
+#define        VFMADDPS_YI( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm3;\\r
+                               vaddps y0,%ymm3,y0\r
+\r
+#define        VFMADDPS_R( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm2;\\r
+                               vaddps y0,%xmm2,y0\r
+\r
+#define        VFMADDPS_I( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm3;\\r
+                               vaddps y0,%xmm3,y0\r
+\r
+\r
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)\r
+\r
+#define        VFMADDPS_YR( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm2;\\r
+                               vsubps %ymm2,y0,y0\r
+\r
+#define        VFMADDPS_YI( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm3;\\r
+                               vaddps y0,%ymm3,y0\r
+\r
+#define        VFMADDPS_R( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm2;\\r
+                               vsubps %xmm2,y0,y0\r
+\r
+#define        VFMADDPS_I( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm3;\\r
+                               vaddps y0,%xmm3,y0\r
+\r
+\r
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+#define        VFMADDPS_YR( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm2;\\r
+                               vaddps y0,%ymm2,y0\r
+\r
+#define        VFMADDPS_YI( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm3;\\r
+                               vsubps %ymm3,y0,y0\r
+\r
+#define        VFMADDPS_R( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm2;\\r
+                               vaddps y0,%xmm2,y0\r
+\r
+#define        VFMADDPS_I( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm3;\\r
+                               vsubps %xmm3,y0,y0\r
+\r
+\r
+#else\r
+\r
+#define        VFMADDPS_YR( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm2;\\r
+                               vsubps %ymm2,y0,y0\r
+\r
+#define        VFMADDPS_YI( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%ymm3;\\r
+                               vsubps %ymm3,y0,y0\r
+\r
+#define        VFMADDPS_R( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm2;\\r
+                               vsubps %xmm2,y0,y0\r
+\r
+#define        VFMADDPS_I( y0,y1,y2 ) \\r
+                               vmulps y1,y2,%xmm3;\\r
+                               vsubps %xmm3,y0,y0\r
+\r
+\r
+#endif\r
+\r
+\r
+#define        A_PR1   512\r
+#define        B_PR1   512\r
+\r
+/***************************************************************************************************************************/\r
+\r
+.macro KERNEL8x2_SUB\r
+\r
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0\r
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %ymm4\r
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )\r
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1\r
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )\r
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %ymm5\r
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )\r
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )\r
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %ymm6\r
+        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )\r
+        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )\r
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %ymm7\r
+        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )\r
+        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )\r
+        addq    $ 4 , BI                           \r
+        addq    $ 16, %rax                         \r
+.endm\r
+\r
+.macro SAVE8x2\r
+\r
+       vbroadcastss    ALPHA_R, %ymm0\r
+       vbroadcastss    ALPHA_I, %ymm1\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9\r
+        vshufps $ 0xb1, %ymm11, %ymm11, %ymm11\r
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13\r
+        vshufps $ 0xb1, %ymm15, %ymm15, %ymm15\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %ymm9, %ymm8 , %ymm8\r
+        vaddsubps %ymm11,%ymm10, %ymm10\r
+        vaddsubps %ymm13,%ymm12, %ymm12\r
+        vaddsubps %ymm15,%ymm14, %ymm14\r
+\r
+        vshufps $ 0xb1, %ymm8 , %ymm8, %ymm9\r
+        vshufps $ 0xb1, %ymm10, %ymm10, %ymm11\r
+        vshufps $ 0xb1, %ymm12, %ymm12, %ymm13\r
+        vshufps $ 0xb1, %ymm14, %ymm14, %ymm15\r
+\r
+#else\r
+        vaddsubps %ymm8,  %ymm9 ,%ymm9\r
+        vaddsubps %ymm10, %ymm11,%ymm11\r
+        vaddsubps %ymm12, %ymm13,%ymm13\r
+        vaddsubps %ymm14, %ymm15,%ymm15\r
+\r
+        vmovaps   %ymm9,  %ymm8\r
+        vmovaps   %ymm11, %ymm10\r
+        vmovaps   %ymm13, %ymm12\r
+        vmovaps   %ymm15, %ymm14\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9\r
+        vshufps $ 0xb1, %ymm11, %ymm11, %ymm11\r
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13\r
+        vshufps $ 0xb1, %ymm15, %ymm15, %ymm15\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %ymm8 , %ymm0, %ymm8\r
+        vmulps  %ymm10, %ymm0, %ymm10\r
+        vmulps  %ymm12, %ymm0, %ymm12\r
+        vmulps  %ymm14, %ymm0, %ymm14\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %ymm9 , %ymm1, %ymm9\r
+        vmulps  %ymm11, %ymm1, %ymm11\r
+        vmulps  %ymm13, %ymm1, %ymm13\r
+        vmulps  %ymm15, %ymm1, %ymm15\r
+\r
+       vaddsubps %ymm9, %ymm8 , %ymm8\r
+        vaddsubps %ymm11,%ymm10, %ymm10\r
+        vaddsubps %ymm13,%ymm12, %ymm12\r
+        vaddsubps %ymm15,%ymm14, %ymm14\r
+\r
+\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vaddps          (CO1), %ymm8 , %ymm8\r
+       vaddps  8 * SIZE(CO1), %ymm12, %ymm12\r
+\r
+       vaddps          (CO1, LDC), %ymm10, %ymm10\r
+       vaddps  8 * SIZE(CO1, LDC), %ymm14, %ymm14\r
+\r
+#endif\r
+\r
+       vmovups %ymm8 ,         (CO1)\r
+       vmovups %ymm12 , 8 * SIZE(CO1)\r
+\r
+       vmovups %ymm10 ,        (CO1, LDC)\r
+       vmovups %ymm14 , 8 * SIZE(CO1, LDC)\r
+\r
+       prefetcht0      64(CO1)\r
+       prefetcht0      64(CO1, LDC)\r
+\r
+.endm\r
+\r
+/***************************************************************************************************************************/\r
+\r
+.macro KERNEL4x2_SUB\r
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4\r
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )\r
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1\r
+        VFMADDPS_R(        %xmm12,%xmm4,%xmm1 )\r
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5\r
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )\r
+        VFMADDPS_I(        %xmm13,%xmm5,%xmm1 )\r
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6\r
+        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )\r
+        VFMADDPS_R(        %xmm14,%xmm6,%xmm1 )\r
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7\r
+        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )\r
+        VFMADDPS_I(        %xmm15,%xmm7,%xmm1 )\r
+        addq    $ 4, BI                           \r
+        addq    $ 8, %rax                         \r
+.endm\r
+\r
+.macro SAVE4x2\r
+\r
+       vbroadcastss    ALPHA_R, %xmm0\r
+       vbroadcastss    ALPHA_I, %xmm1\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13\r
+        vshufps $ 0xb1, %xmm15, %xmm15, %xmm15\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+        vaddsubps %xmm13,%xmm12, %xmm12\r
+        vaddsubps %xmm15,%xmm14, %xmm14\r
+\r
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9\r
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11\r
+        vshufps $ 0xb1, %xmm12, %xmm12, %xmm13\r
+        vshufps $ 0xb1, %xmm14, %xmm14, %xmm15\r
+\r
+#else\r
+        vaddsubps %xmm8,  %xmm9 ,%xmm9\r
+        vaddsubps %xmm10, %xmm11,%xmm11\r
+        vaddsubps %xmm12, %xmm13,%xmm13\r
+        vaddsubps %xmm14, %xmm15,%xmm15\r
+\r
+        vmovaps   %xmm9,  %xmm8\r
+        vmovaps   %xmm11, %xmm10\r
+        vmovaps   %xmm13, %xmm12\r
+        vmovaps   %xmm15, %xmm14\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13\r
+        vshufps $ 0xb1, %xmm15, %xmm15, %xmm15\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %xmm8 , %xmm0, %xmm8\r
+        vmulps  %xmm10, %xmm0, %xmm10\r
+        vmulps  %xmm12, %xmm0, %xmm12\r
+        vmulps  %xmm14, %xmm0, %xmm14\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %xmm9 , %xmm1, %xmm9\r
+        vmulps  %xmm11, %xmm1, %xmm11\r
+        vmulps  %xmm13, %xmm1, %xmm13\r
+        vmulps  %xmm15, %xmm1, %xmm15\r
+\r
+       vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+        vaddsubps %xmm13,%xmm12, %xmm12\r
+        vaddsubps %xmm15,%xmm14, %xmm14\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vaddps          (CO1), %xmm8 , %xmm8\r
+       vaddps  4 * SIZE(CO1), %xmm12, %xmm12\r
+\r
+       vaddps          (CO1, LDC), %xmm10, %xmm10\r
+       vaddps  4 * SIZE(CO1, LDC), %xmm14, %xmm14\r
+\r
+#endif\r
+\r
+       vmovups %xmm8 ,         (CO1)\r
+       vmovups %xmm12 , 4 * SIZE(CO1)\r
+\r
+       vmovups %xmm10 ,        (CO1, LDC)\r
+       vmovups %xmm14 , 4 * SIZE(CO1, LDC)\r
+\r
+.endm\r
+\r
+/************************************************************************************************/\r
+\r
+.macro KERNEL2x2_SUB\r
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4\r
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )\r
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5\r
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )\r
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6\r
+        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )\r
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7\r
+        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )\r
+        addq    $ 4, BI                           \r
+        addq    $ 4, %rax                         \r
+.endm\r
+\r
+.macro SAVE2x2\r
+\r
+       vbroadcastss    ALPHA_R, %xmm0\r
+       vbroadcastss    ALPHA_I, %xmm1\r
+\r
+       // swap high and low 4 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+\r
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9\r
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11\r
+\r
+#else\r
+        vaddsubps %xmm8,  %xmm9 ,%xmm9\r
+        vaddsubps %xmm10, %xmm11,%xmm11\r
+\r
+        vmovaps   %xmm9,  %xmm8\r
+        vmovaps   %xmm11, %xmm10\r
+\r
+       // swap high and low 4 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %xmm8 , %xmm0, %xmm8\r
+        vmulps  %xmm10, %xmm0, %xmm10\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %xmm9 , %xmm1, %xmm9\r
+        vmulps  %xmm11, %xmm1, %xmm11\r
+\r
+       vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vaddps          (CO1), %xmm8 , %xmm8\r
+\r
+       vaddps          (CO1, LDC), %xmm10, %xmm10\r
+\r
+#endif\r
+\r
+       vmovups %xmm8 ,         (CO1)\r
+\r
+       vmovups %xmm10 ,        (CO1, LDC)\r
+\r
+.endm\r
+\r
+/************************************************************************************************/\r
+\r
+.macro KERNEL1x2_SUB\r
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4\r
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )\r
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5\r
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )\r
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6\r
+        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )\r
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7\r
+        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )\r
+        addq    $ 4, BI                           \r
+        addq    $ 2, %rax                         \r
+.endm\r
+\r
+.macro SAVE1x2\r
+\r
+       vbroadcastss    ALPHA_R, %xmm0\r
+       vbroadcastss    ALPHA_I, %xmm1\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+\r
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9\r
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11\r
+\r
+#else\r
+        vaddsubps %xmm8,  %xmm9 ,%xmm9\r
+        vaddsubps %xmm10, %xmm11,%xmm11\r
+\r
+        vmovaps   %xmm9,  %xmm8\r
+        vmovaps   %xmm11, %xmm10\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %xmm8 , %xmm0, %xmm8\r
+        vmulps  %xmm10, %xmm0, %xmm10\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %xmm9 , %xmm1, %xmm9\r
+        vmulps  %xmm11, %xmm1, %xmm11\r
+\r
+       vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vmovsd          (CO1), %xmm14\r
+       vaddps          %xmm14, %xmm8 , %xmm8\r
+\r
+       vmovsd          (CO1, LDC), %xmm15\r
+       vaddps          %xmm15, %xmm10, %xmm10\r
+\r
+#endif\r
+\r
+       vmovsd  %xmm8 ,         (CO1)\r
+       vmovsd  %xmm10 ,        (CO1, LDC)\r
+\r
+.endm\r
+\r
+/************************************************************************************************/\r
+\r
+.macro KERNEL8x1_SUB\r
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0\r
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1\r
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %ymm4\r
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )\r
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )\r
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %ymm5\r
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )\r
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )\r
+        addq    $ 2 , BI                           \r
+        addq    $ 16, %rax                         \r
+.endm\r
+\r
+.macro SAVE8x1\r
+\r
+       vbroadcastss    ALPHA_R, %ymm0\r
+       vbroadcastss    ALPHA_I, %ymm1\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9\r
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %ymm9, %ymm8 , %ymm8\r
+        vaddsubps %ymm13,%ymm12, %ymm12\r
+\r
+        vshufps $ 0xb1, %ymm8 , %ymm8, %ymm9\r
+        vshufps $ 0xb1, %ymm12, %ymm12, %ymm13\r
+\r
+#else\r
+        vaddsubps %ymm8,  %ymm9 ,%ymm9\r
+        vaddsubps %ymm12, %ymm13,%ymm13\r
+\r
+        vmovaps   %ymm9,  %ymm8\r
+        vmovaps   %ymm13, %ymm12\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9\r
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %ymm8 , %ymm0, %ymm8\r
+        vmulps  %ymm12, %ymm0, %ymm12\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %ymm9 , %ymm1, %ymm9\r
+        vmulps  %ymm13, %ymm1, %ymm13\r
+\r
+       vaddsubps %ymm9, %ymm8 , %ymm8\r
+        vaddsubps %ymm13,%ymm12, %ymm12\r
+\r
+\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vaddps          (CO1), %ymm8 , %ymm8\r
+       vaddps  8 * SIZE(CO1), %ymm12, %ymm12\r
+\r
+#endif\r
+\r
+       vmovups %ymm8 ,         (CO1)\r
+       vmovups %ymm12 , 8 * SIZE(CO1)\r
+\r
+.endm\r
+\r
+\r
+/************************************************************************************************/\r
+\r
+.macro KERNEL4x1_SUB\r
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4\r
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )\r
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1\r
+        VFMADDPS_R(        %xmm12,%xmm4,%xmm1 )\r
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5\r
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )\r
+        VFMADDPS_I(        %xmm13,%xmm5,%xmm1 )\r
+        addq    $ 2, BI                           \r
+        addq    $ 8, %rax                         \r
+.endm\r
+\r
+.macro SAVE4x1\r
+\r
+       vbroadcastss    ALPHA_R, %xmm0\r
+       vbroadcastss    ALPHA_I, %xmm1\r
+\r
+       // swap high and low 4 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm13,%xmm12, %xmm12\r
+\r
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9\r
+        vshufps $ 0xb1, %xmm12, %xmm12, %xmm13\r
+\r
+#else\r
+        vaddsubps %xmm8,  %xmm9 ,%xmm9\r
+        vaddsubps %xmm12, %xmm13,%xmm13\r
+\r
+        vmovaps   %xmm9,  %xmm8\r
+        vmovaps   %xmm13, %xmm12\r
+\r
+       // swap high and low 4 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %xmm8 , %xmm0, %xmm8\r
+        vmulps  %xmm12, %xmm0, %xmm12\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %xmm9 , %xmm1, %xmm9\r
+        vmulps  %xmm13, %xmm1, %xmm13\r
+\r
+       vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm13,%xmm12, %xmm12\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vaddps          (CO1), %xmm8 , %xmm8\r
+       vaddps  4 * SIZE(CO1), %xmm12, %xmm12\r
+\r
+#endif\r
+\r
+       vmovups %xmm8 ,         (CO1)\r
+       vmovups %xmm12 , 4 * SIZE(CO1)\r
+\r
+.endm\r
+\r
+/************************************************************************************************/\r
+\r
+.macro KERNEL2x1_SUB\r
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4\r
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )\r
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5\r
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )\r
+        addq    $ 2, BI                           \r
+        addq    $ 4, %rax                         \r
+.endm\r
+\r
+.macro SAVE2x1\r
+\r
+       vbroadcastss    ALPHA_R, %xmm0\r
+       vbroadcastss    ALPHA_I, %xmm1\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %xmm9, %xmm8 , %xmm8\r
+\r
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9\r
+\r
+#else\r
+        vaddsubps %xmm8,  %xmm9 ,%xmm9\r
+\r
+        vmovaps   %xmm9,  %xmm8\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %xmm8 , %xmm0, %xmm8\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %xmm9 , %xmm1, %xmm9\r
+\r
+       vaddsubps %xmm9, %xmm8 , %xmm8\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vaddps          (CO1), %xmm8 , %xmm8\r
+\r
+#endif\r
+\r
+       vmovups %xmm8 ,         (CO1)\r
+\r
+.endm\r
+\r
+/************************************************************************************************/\r
+\r
+.macro KERNEL1x1_SUB\r
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4\r
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0 )\r
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5\r
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0 )\r
+        addq    $ 2, BI                           \r
+        addq    $ 2, %rax                         \r
+.endm\r
+\r
+.macro SAVE1x1\r
+\r
+       vbroadcastss    ALPHA_R, %xmm0\r
+       vbroadcastss    ALPHA_I, %xmm1\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %xmm9, %xmm8 , %xmm8\r
+\r
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9\r
+\r
+#else\r
+        vaddsubps %xmm8,  %xmm9 ,%xmm9\r
+\r
+        vmovaps   %xmm9,  %xmm8\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %xmm8 , %xmm0, %xmm8\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %xmm9 , %xmm1, %xmm9\r
+\r
+       vaddsubps %xmm9, %xmm8 , %xmm8\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vmovsd          (CO1), %xmm14\r
+       vaddps          %xmm14, %xmm8 , %xmm8\r
+\r
+#endif\r
+\r
+       vmovsd  %xmm8 ,         (CO1)\r
+\r
+.endm\r
+\r
+/************************************************************************************************/\r
+\r
+\r
+\r
+\r
+       PROLOGUE\r
+       PROFCODE\r
+       \r
+       subq    $ STACKSIZE, %rsp\r
+       movq    %rbx,   (%rsp)\r
+       movq    %rbp,  8(%rsp)\r
+       movq    %r12, 16(%rsp)\r
+       movq    %r13, 24(%rsp)\r
+       movq    %r14, 32(%rsp)\r
+       movq    %r15, 40(%rsp)\r
+\r
+       vzeroupper\r
+\r
+#ifdef WINDOWS_ABI\r
+       movq    %rdi,    48(%rsp)\r
+       movq    %rsi,    56(%rsp)\r
+       vmovups %xmm6,   64(%rsp)\r
+       vmovups %xmm7,   80(%rsp)\r
+       vmovups %xmm8,   96(%rsp)\r
+       vmovups %xmm9,  112(%rsp)\r
+       vmovups %xmm10, 128(%rsp)\r
+       vmovups %xmm11, 144(%rsp)\r
+       vmovups %xmm12, 160(%rsp)\r
+       vmovups %xmm13, 176(%rsp)\r
+       vmovups %xmm14, 192(%rsp)\r
+       vmovups %xmm15, 208(%rsp)\r
+\r
+       movq    ARG1,      OLD_M\r
+       movq    ARG2,      OLD_N\r
+       movq    ARG3,      OLD_K\r
+       movq    OLD_A,     A\r
+       movq    OLD_B,     B\r
+       movq    OLD_C,     C\r
+       movq    OLD_LDC,   LDC\r
+#ifdef TRMMKERNEL\r
+       movsd   OLD_OFFSET, %xmm12\r
+#endif\r
+       vmovaps %xmm3, %xmm0\r
+       vmovsd   OLD_ALPHA_I, %xmm1\r
+\r
+#else\r
+       movq    STACKSIZE +  8(%rsp), LDC\r
+#ifdef TRMMKERNEL\r
+       movsd   STACKSIZE + 16(%rsp), %xmm12\r
+#endif\r
+\r
+#endif\r
+\r
+       movq    %rsp, SP      # save old stack\r
+        subq    $ 128 + L_BUFFER_SIZE, %rsp\r
+        andq    $ -4096, %rsp    # align stack\r
+\r
+        STACK_TOUCH\r
+\r
+       cmpq    $ 0, OLD_M\r
+       je      .L999\r
+\r
+       cmpq    $ 0, OLD_N\r
+       je      .L999\r
+\r
+       cmpq    $ 0, OLD_K\r
+       je      .L999\r
+\r
+       movq    OLD_M, M\r
+       movq    OLD_N, N\r
+       movq    OLD_K, K\r
+\r
+       vmovss   %xmm0, ALPHA_R\r
+       vmovss   %xmm1, ALPHA_I\r
+\r
+       salq    $ ZBASE_SHIFT, LDC\r
+\r
+       movq    N, %rax\r
+        xorq    %rdx, %rdx\r
+        movq    $ 2,  %rdi\r
+        divq    %rdi                    //    N / 2\r
+        movq    %rax, Ndiv6             //    N / 2\r
+        movq    %rdx, Nmod6             //    N % 2\r
+\r
+       \r
+\r
+#ifdef TRMMKERNEL\r
+       vmovsd  %xmm12, OFFSET\r
+       vmovsd  %xmm12, KK\r
+#ifndef LEFT\r
+       negq    KK\r
+#endif \r
+#endif\r
+\r
+.L2_0:\r
+\r
+       movq    Ndiv6,  J\r
+       cmpq    $ 0, J\r
+       je      .L1_0\r
+       ALIGN_4\r
+\r
+\r
+\r
+.L2_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       ALIGN_4\r
+\r
+.L2_02b:\r
+\r
+       vmovups (BO1), %xmm0\r
+       vmovups %xmm0,       (BO)\r
+       addq    $ 4*SIZE,BO1\r
+       addq    $ 4*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L2_02b\r
+\r
+.L2_02c:\r
+\r
+       movq    BO1, B                  // next offset of B\r
+\r
+.L2_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 2), C          // c += 2 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+       \r
+       movq    A, AO                   // aoffset = a\r
+       addq    $ 16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $ 3, I                  // i = (m >> 3)\r
+       je      .L2_4_10\r
+\r
+       ALIGN_4\r
+/**********************************************************************************************************/\r
+\r
+.L2_8_11:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 8, %rax        // number of values in AO\r
+#else\r
+        addq    $ 2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L2_8_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_8_12:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+\r
+       je      .L2_8_16\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x2_SUB\r
+\r
+       je      .L2_8_16\r
+\r
+       jmp     .L2_8_12\r
+       ALIGN_4\r
+\r
+.L2_8_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L2_8_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_8_17:\r
+\r
+       KERNEL8x2_SUB\r
+\r
+       jl      .L2_8_17\r
+       ALIGN_4\r
+\r
+\r
+.L2_8_19:\r
+\r
+       SAVE8x2\r
+\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 8, KK\r
+#endif\r
+\r
+       addq    $ 16 * SIZE, CO1                # coffset += 16\r
+       decq    I                       # i --\r
+       jg      .L2_8_11\r
+       ALIGN_4 \r
+\r
+\r
+/**********************************************************************************************************/\r
+\r
+\r
+\r
+\r
+.L2_4_10:\r
+       testq   $ 7, M          \r
+       jz      .L2_4_60                // to next 2 lines of N\r
+\r
+       testq   $ 4, M          \r
+       jz      .L2_4_20\r
+       ALIGN_4\r
+\r
+\r
+.L2_4_11:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 4, %rax        // number of values in AO\r
+#else\r
+        addq    $ 2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L2_4_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_4_12:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       je      .L2_4_16\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       je      .L2_4_16\r
+\r
+       jmp     .L2_4_12\r
+       ALIGN_4\r
+\r
+.L2_4_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L2_4_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_4_17:\r
+\r
+       KERNEL4x2_SUB\r
+\r
+       jl      .L2_4_17\r
+       ALIGN_4\r
+\r
+\r
+.L2_4_19:\r
+\r
+       SAVE4x2\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 4, KK\r
+#endif\r
+\r
+       addq    $ 8 * SIZE, CO1         # coffset += 8\r
+       ALIGN_4 \r
+\r
+\r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+\r
+.L2_4_20:\r
+\r
+       testq   $ 2, M          \r
+       jz      .L2_4_40\r
+       ALIGN_4\r
+\r
+.L2_4_21:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 2, %rax        // number of values in AO\r
+#else\r
+        addq    $ 2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L2_4_26\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_4_22:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       je      .L2_4_26\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       je      .L2_4_26\r
+\r
+       jmp     .L2_4_22\r
+       ALIGN_4\r
+\r
+.L2_4_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L2_4_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_4_27:\r
+\r
+       KERNEL2x2_SUB\r
+\r
+       jl      .L2_4_27\r
+       ALIGN_4\r
+\r
+\r
+.L2_4_29:\r
+\r
+       vbroadcastss    ALPHA_R, %xmm0\r
+       vbroadcastss    ALPHA_I, %xmm1\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+\r
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \\r
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)\r
+\r
+        vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+\r
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9\r
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11\r
+\r
+#else\r
+        vaddsubps %xmm8,  %xmm9 ,%xmm9\r
+        vaddsubps %xmm10, %xmm11,%xmm11\r
+\r
+        vmovaps   %xmm9,  %xmm8\r
+        vmovaps   %xmm11, %xmm10\r
+\r
+       // swap high and low 64 bytes\r
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9\r
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11\r
+\r
+#endif\r
+\r
+       // multiply with ALPHA_R\r
+        vmulps  %xmm8 , %xmm0, %xmm8\r
+        vmulps  %xmm10, %xmm0, %xmm10\r
+\r
+       // multiply with ALPHA_I\r
+        vmulps  %xmm9 , %xmm1, %xmm9\r
+        vmulps  %xmm11, %xmm1, %xmm11\r
+\r
+       vaddsubps %xmm9, %xmm8 , %xmm8\r
+        vaddsubps %xmm11,%xmm10, %xmm10\r
+\r
+\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vaddps          (CO1), %xmm8 , %xmm8\r
+\r
+       vaddps          (CO1, LDC), %xmm10, %xmm10\r
+\r
+#endif\r
+\r
+       vmovups %xmm8 ,         (CO1)\r
+\r
+       vmovups %xmm10 ,        (CO1, LDC)\r
+\r
+\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 2, KK\r
+#endif\r
+\r
+       addq    $ 4 * SIZE, CO1         # coffset += 4\r
+       decq    I                       # i --\r
+       jg      .L2_4_21\r
+       ALIGN_4 \r
+\r
+\r
+\r
+/**************************************************************************/\r
+.L2_4_40:\r
+       testq   $ 1, M          \r
+       jz      .L2_4_60                // to next 2 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L2_4_41:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 8 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 1, %rax        // number of values in AO\r
+#else\r
+        addq    $ 2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L2_4_46\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_4_42:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       je      .L2_4_46\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       je      .L2_4_46\r
+\r
+       jmp     .L2_4_42\r
+       ALIGN_4\r
+\r
+.L2_4_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L2_4_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_4_47:\r
+\r
+       KERNEL1x2_SUB\r
+\r
+       jl      .L2_4_47\r
+       ALIGN_4\r
+\r
+\r
+.L2_4_49:\r
+\r
+       SAVE1x2\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 1, KK\r
+#endif\r
+\r
+       addq    $ 2 * SIZE, CO1         # coffset += 2\r
+       decq    I                       # i --\r
+       jg      .L2_4_41\r
+       ALIGN_4 \r
+\r
+\r
+\r
+       \r
+.L2_4_60:\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+        addq    $ 2, KK\r
+#endif\r
+\r
+       decq    J                       // j --\r
+       jg      .L2_01                  // next 2 lines of N\r
+\r
+\r
+\r
+.L1_0:\r
+\r
+/************************************************************************************************\r
+* Loop for Nmod6 % 2 > 0\r
+*************************************************************************************************/\r
+\r
+       movq    Nmod6, J                \r
+       andq    $ 1, J                  // j % 2\r
+       je      .L999\r
+       ALIGN_4\r
+\r
+.L1_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       ALIGN_4\r
+\r
+.L1_02b:\r
+\r
+       vmovsd          (BO1), %xmm0\r
+       vmovsd  %xmm0,       (BO)\r
+       addq    $ 2*SIZE,BO1\r
+       addq    $ 2*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L1_02b\r
+\r
+.L1_02c:\r
+\r
+       movq    BO1, B                  // next offset of B\r
+\r
+.L1_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 1), C          // c += 1 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+       \r
+       movq    A, AO                   // aoffset = a\r
+       addq    $ 16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $ 3, I                  // i = (m >> 3)\r
+       je      .L1_4_10\r
+\r
+       ALIGN_4\r
+\r
+/**************************************************************************************************/\r
+\r
+.L1_8_11:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 8, %rax        // number of values in AO\r
+#else\r
+        addq    $ 1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L1_8_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_8_12:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+\r
+       je      .L1_8_16\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL8x1_SUB\r
+\r
+       je      .L1_8_16\r
+\r
+       jmp     .L1_8_12\r
+       ALIGN_4\r
+\r
+.L1_8_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L1_8_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_8_17:\r
+\r
+       KERNEL8x1_SUB\r
+\r
+       jl      .L1_8_17\r
+       ALIGN_4\r
+\r
+\r
+.L1_8_19:\r
+\r
+       SAVE8x1\r
+\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 4, %rax                       // rax = rax *16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 8, KK\r
+#endif\r
+\r
+       addq    $ 16 * SIZE, CO1                # coffset += 16\r
+       decq    I                       # i --\r
+       jg      .L1_8_11\r
+       ALIGN_4 \r
+\r
+\r
+\r
+/**************************************************************************************************/\r
+.L1_4_10:\r
+\r
+       testq   $ 7, M          \r
+       jz      .L999\r
+\r
+       testq   $ 4, M          \r
+       jz      .L1_4_20\r
+\r
+\r
+.L1_4_11:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 4, %rax        // number of values in AO\r
+#else\r
+        addq    $ 1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L1_4_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_4_12:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       je      .L1_4_16\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       je      .L1_4_16\r
+\r
+       jmp     .L1_4_12\r
+       ALIGN_4\r
+\r
+.L1_4_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L1_4_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_4_17:\r
+\r
+       KERNEL4x1_SUB\r
+\r
+       jl      .L1_4_17\r
+       ALIGN_4\r
+\r
+\r
+.L1_4_19:\r
+\r
+       SAVE4x1\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 3, %rax                       // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 4, KK\r
+#endif\r
+\r
+       addq    $ 8 * SIZE, CO1         # coffset += 8\r
+       ALIGN_4 \r
+\r
+\r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+\r
+.L1_4_20:\r
+\r
+       testq   $ 2, M          \r
+       jz      .L1_4_40\r
+       ALIGN_4\r
+\r
+.L1_4_21:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 2, %rax        // number of values in AO\r
+#else\r
+        addq    $ 1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L1_4_26\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_4_22:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+\r
+       je      .L1_4_26\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+\r
+       je      .L1_4_26\r
+\r
+       jmp     .L1_4_22\r
+       ALIGN_4\r
+\r
+.L1_4_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L1_4_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2; number of values\r
+\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_4_27:\r
+\r
+       KERNEL2x1_SUB\r
+\r
+       jl      .L1_4_27\r
+       ALIGN_4\r
+\r
+\r
+.L1_4_29:\r
+\r
+       SAVE2x1\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 2, %rax                       // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 2, KK\r
+#endif\r
+\r
+       addq    $ 4 * SIZE, CO1         # coffset += 4\r
+       ALIGN_4 \r
+\r
+\r
+\r
+/**************************************************************************/\r
+.L1_4_40:\r
+       testq   $ 1, M          \r
+       jz      .L999           // to next 2 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L1_4_41:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO                     // first buffer to BO\r
+       addq    $ 4 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $ 1, %rax        // number of values in AO\r
+#else\r
+        addq    $ 1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $ -8, %rax                      //  K = K - ( K % 8 )\r
+       je      .L1_4_46\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_4_42:\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+\r
+       je      .L1_4_46\r
+\r
+       prefetcht0      A_PR1(AO,%rax,SIZE)\r
+       prefetcht0      B_PR1(BO,BI,SIZE)\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+\r
+       je      .L1_4_46\r
+\r
+       jmp     .L1_4_42\r
+       ALIGN_4\r
+\r
+.L1_4_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $ 7, %rax               # if (k & 1)\r
+       je .L1_4_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_4_47:\r
+\r
+       KERNEL1x1_SUB\r
+\r
+       jl      .L1_4_47\r
+       ALIGN_4\r
+\r
+\r
+.L1_4_49:\r
+\r
+       SAVE1x1\r
+\r
+\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+       salq    $ 1, %rax                       // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $ 1, KK\r
+#endif\r
+\r
+       addq    $ 2 * SIZE, CO1         # coffset += 2\r
+       ALIGN_4 \r
+\r
+\r
+.L999:\r
+       vzeroupper\r
+\r
+       movq            SP, %rsp\r
+       movq       (%rsp), %rbx\r
+       movq      8(%rsp), %rbp\r
+       movq     16(%rsp), %r12\r
+       movq     24(%rsp), %r13\r
+       movq     32(%rsp), %r14\r
+       movq     40(%rsp), %r15\r
+\r
+#ifdef WINDOWS_ABI\r
+       movq     48(%rsp), %rdi\r
+       movq     56(%rsp), %rsi\r
+       vmovups  64(%rsp), %xmm6\r
+       vmovups  80(%rsp), %xmm7\r
+       vmovups  96(%rsp), %xmm8\r
+       vmovups 112(%rsp), %xmm9\r
+       vmovups 128(%rsp), %xmm10\r
+       vmovups 144(%rsp), %xmm11\r
+       vmovups 160(%rsp), %xmm12\r
+       vmovups 176(%rsp), %xmm13\r
+       vmovups 192(%rsp), %xmm14\r
+       vmovups 208(%rsp), %xmm15\r
+#endif\r
+\r
+       addq    $ STACKSIZE, %rsp\r
+       ret\r
+\r
+       EPILOGUE\r

diff --git a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S
index 7cf1fda..94e2f61 100644 (file)
--- a/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S
+++ b/kernel/x86_64/zgemm_kernel_2x2_bulldozer.S
@@ -412,16 +412,16 @@
 #ifdef WINDOWS_ABI\r
        movq    %rdi,    48(%rsp)\r
        movq    %rsi,    56(%rsp)\r
-       movups  %xmm6,   64(%rsp)\r
-       movups  %xmm7,   80(%rsp)\r
-       movups  %xmm8,   96(%rsp)\r
-       movups  %xmm9,  112(%rsp)\r
-       movups  %xmm10, 128(%rsp)\r
-       movups  %xmm11, 144(%rsp)\r
-       movups  %xmm12, 160(%rsp)\r
-       movups  %xmm13, 176(%rsp)\r
-       movups  %xmm14, 192(%rsp)\r
-       movups  %xmm15, 208(%rsp)\r
+       vmovups %xmm6,   64(%rsp)\r
+       vmovups %xmm7,   80(%rsp)\r
+       vmovups %xmm8,   96(%rsp)\r
+       vmovups %xmm9,  112(%rsp)\r
+       vmovups %xmm10, 128(%rsp)\r
+       vmovups %xmm11, 144(%rsp)\r
+       vmovups %xmm12, 160(%rsp)\r
+       vmovups %xmm13, 176(%rsp)\r
+       vmovups %xmm14, 192(%rsp)\r
+       vmovups %xmm15, 208(%rsp)\r
 \r
        movq    ARG1,      OLD_M\r
        movq    ARG2,      OLD_N\r
@@ -431,14 +431,15 @@
        movq    OLD_C,     C\r
        movq    OLD_LDC,   LDC\r
 #ifdef TRMMKERNEL\r
-       movsd   OLD_OFFSET, %xmm12\r
+       vmovsd  OLD_OFFSET, %xmm12\r
 #endif\r
        vmovaps %xmm3, %xmm0\r
+       vmovsd   OLD_ALPHA_I, %xmm1\r
 \r
 #else\r
        movq    STACKSIZE +  8(%rsp), LDC\r
 #ifdef TRMMKERNEL\r
-       movsd   STACKSIZE + 16(%rsp), %xmm12\r
+       vmovsd  STACKSIZE + 16(%rsp), %xmm12\r
 #endif\r
 \r
 #endif\r
@@ -1372,6 +1373,8 @@
 \r
 \r
 .L999:\r
+       vzeroupper\r
+\r
        movq            SP, %rsp\r
        movq       (%rsp), %rbx\r
        movq      8(%rsp), %rbp\r
@@ -1383,16 +1386,16 @@
 #ifdef WINDOWS_ABI\r
        movq     48(%rsp), %rdi\r
        movq     56(%rsp), %rsi\r
-       movups   64(%rsp), %xmm6\r
-       movups   80(%rsp), %xmm7\r
-       movups   96(%rsp), %xmm8\r
-       movups  112(%rsp), %xmm9\r
-       movups  128(%rsp), %xmm10\r
-       movups  144(%rsp), %xmm11\r
-       movups  160(%rsp), %xmm12\r
-       movups  176(%rsp), %xmm13\r
-       movups  192(%rsp), %xmm14\r
-       movups  208(%rsp), %xmm15\r
+       vmovups  64(%rsp), %xmm6\r
+       vmovups  80(%rsp), %xmm7\r
+       vmovups  96(%rsp), %xmm8\r
+       vmovups 112(%rsp), %xmm9\r
+       vmovups 128(%rsp), %xmm10\r
+       vmovups 144(%rsp), %xmm11\r
+       vmovups 160(%rsp), %xmm12\r
+       vmovups 176(%rsp), %xmm13\r
+       vmovups 192(%rsp), %xmm14\r
+       vmovups 208(%rsp), %xmm15\r
 #endif\r
 \r
        addq    $STACKSIZE, %rsp\r

diff --git a/kernel/x86_64/zgemm_kernel_2x2_piledriver.S b/kernel/x86_64/zgemm_kernel_2x2_piledriver.S
index b4c416b..848b6f2 100644 (file)
--- a/kernel/x86_64/zgemm_kernel_2x2_piledriver.S
+++ b/kernel/x86_64/zgemm_kernel_2x2_piledriver.S
@@ -27,7 +27,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 /*********************************************************************\r
 *\r
-* 2013/10/30 Saar\r
+* 2014/06/28 Saar\r
 *        BLASTEST               : OK\r
 *        CTEST                  : OK\r
 *        TEST                   : OK\r
@@ -437,16 +437,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq    %rdi,    48(%rsp)\r
        movq    %rsi,    56(%rsp)\r
-       movups  %xmm6,   64(%rsp)\r
-       movups  %xmm7,   80(%rsp)\r
-       movups  %xmm8,   96(%rsp)\r
-       movups  %xmm9,  112(%rsp)\r
-       movups  %xmm10, 128(%rsp)\r
-       movups  %xmm11, 144(%rsp)\r
-       movups  %xmm12, 160(%rsp)\r
-       movups  %xmm13, 176(%rsp)\r
-       movups  %xmm14, 192(%rsp)\r
-       movups  %xmm15, 208(%rsp)\r
+       vmovups %xmm6,   64(%rsp)\r
+       vmovups %xmm7,   80(%rsp)\r
+       vmovups %xmm8,   96(%rsp)\r
+       vmovups %xmm9,  112(%rsp)\r
+       vmovups %xmm10, 128(%rsp)\r
+       vmovups %xmm11, 144(%rsp)\r
+       vmovups %xmm12, 160(%rsp)\r
+       vmovups %xmm13, 176(%rsp)\r
+       vmovups %xmm14, 192(%rsp)\r
+       vmovups %xmm15, 208(%rsp)\r
 \r
        movq    ARG1,      OLD_M\r
        movq    ARG2,      OLD_N\r
@@ -456,14 +456,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        movq    OLD_C,     C\r
        movq    OLD_LDC,   LDC\r
 #ifdef TRMMKERNEL\r
-       movsd   OLD_OFFSET, %xmm12\r
+       vmovsd  OLD_OFFSET, %xmm12\r
 #endif\r
        vmovaps %xmm3, %xmm0\r
+       vmovsd   OLD_ALPHA_I, %xmm1\r
 \r
 #else\r
        movq    STACKSIZE +  8(%rsp), LDC\r
 #ifdef TRMMKERNEL\r
-       movsd   STACKSIZE + 16(%rsp), %xmm12\r
+       vmovsd  STACKSIZE + 16(%rsp), %xmm12\r
 #endif\r
 \r
 #endif\r
@@ -1397,6 +1398,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 \r
 .L999:\r
+       vzeroupper\r
+\r
        movq            SP, %rsp\r
        movq       (%rsp), %rbx\r
        movq      8(%rsp), %rbp\r
@@ -1408,16 +1411,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq     48(%rsp), %rdi\r
        movq     56(%rsp), %rsi\r
-       movups   64(%rsp), %xmm6\r
-       movups   80(%rsp), %xmm7\r
-       movups   96(%rsp), %xmm8\r
-       movups  112(%rsp), %xmm9\r
-       movups  128(%rsp), %xmm10\r
-       movups  144(%rsp), %xmm11\r
-       movups  160(%rsp), %xmm12\r
-       movups  176(%rsp), %xmm13\r
-       movups  192(%rsp), %xmm14\r
-       movups  208(%rsp), %xmm15\r
+       vmovups  64(%rsp), %xmm6\r
+       vmovups  80(%rsp), %xmm7\r
+       vmovups  96(%rsp), %xmm8\r
+       vmovups 112(%rsp), %xmm9\r
+       vmovups 128(%rsp), %xmm10\r
+       vmovups 144(%rsp), %xmm11\r
+       vmovups 160(%rsp), %xmm12\r
+       vmovups 176(%rsp), %xmm13\r
+       vmovups 192(%rsp), %xmm14\r
+       vmovups 208(%rsp), %xmm15\r
 #endif\r
 \r
        addq    $STACKSIZE, %rsp\r

diff --git a/kernel/x86_64/zgemm_kernel_4x2_haswell.S b/kernel/x86_64/zgemm_kernel_4x2_haswell.S
index 289fbbf..a71fff7 100644 (file)
--- a/kernel/x86_64/zgemm_kernel_4x2_haswell.S
+++ b/kernel/x86_64/zgemm_kernel_4x2_haswell.S
@@ -26,7 +26,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 **********************************************************************************/\r
 \r
 /********************************************************************************\r
-* 2013/11/13 Saar\r
+* 2014/06/28 Saar\r
 *        BLASTEST               : OK\r
 *        CTEST                  : OK\r
 *        TEST                   : OK\r
@@ -693,16 +693,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq    %rdi,    48(%rsp)\r
        movq    %rsi,    56(%rsp)\r
-       movups  %xmm6,   64(%rsp)\r
-       movups  %xmm7,   80(%rsp)\r
-       movups  %xmm8,   96(%rsp)\r
-       movups  %xmm9,  112(%rsp)\r
-       movups  %xmm10, 128(%rsp)\r
-       movups  %xmm11, 144(%rsp)\r
-       movups  %xmm12, 160(%rsp)\r
-       movups  %xmm13, 176(%rsp)\r
-       movups  %xmm14, 192(%rsp)\r
-       movups  %xmm15, 208(%rsp)\r
+       vmovups %xmm6,   64(%rsp)\r
+       vmovups %xmm7,   80(%rsp)\r
+       vmovups %xmm8,   96(%rsp)\r
+       vmovups %xmm9,  112(%rsp)\r
+       vmovups %xmm10, 128(%rsp)\r
+       vmovups %xmm11, 144(%rsp)\r
+       vmovups %xmm12, 160(%rsp)\r
+       vmovups %xmm13, 176(%rsp)\r
+       vmovups %xmm14, 192(%rsp)\r
+       vmovups %xmm15, 208(%rsp)\r
 \r
        movq    ARG1,      OLD_M\r
        movq    ARG2,      OLD_N\r
@@ -715,6 +715,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        movsd   OLD_OFFSET, %xmm12\r
 #endif\r
        vmovaps %xmm3, %xmm0\r
+       vmovsd   OLD_ALPHA_I, %xmm1\r
 \r
 #else\r
        movq    STACKSIZE +  8(%rsp), LDC\r
@@ -1781,6 +1782,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 \r
 .L999:\r
+       vzeroupper\r
+\r
        movq            SP, %rsp\r
        movq       (%rsp), %rbx\r
        movq      8(%rsp), %rbp\r
@@ -1792,16 +1795,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef WINDOWS_ABI\r
        movq     48(%rsp), %rdi\r
        movq     56(%rsp), %rsi\r
-       movups   64(%rsp), %xmm6\r
-       movups   80(%rsp), %xmm7\r
-       movups   96(%rsp), %xmm8\r
-       movups  112(%rsp), %xmm9\r
-       movups  128(%rsp), %xmm10\r
-       movups  144(%rsp), %xmm11\r
-       movups  160(%rsp), %xmm12\r
-       movups  176(%rsp), %xmm13\r
-       movups  192(%rsp), %xmm14\r
-       movups  208(%rsp), %xmm15\r
+       vmovups  64(%rsp), %xmm6\r
+       vmovups  80(%rsp), %xmm7\r
+       vmovups  96(%rsp), %xmm8\r
+       vmovups 112(%rsp), %xmm9\r
+       vmovups 128(%rsp), %xmm10\r
+       vmovups 144(%rsp), %xmm11\r
+       vmovups 160(%rsp), %xmm12\r
+       vmovups 176(%rsp), %xmm13\r
+       vmovups 192(%rsp), %xmm14\r
+       vmovups 208(%rsp), %xmm15\r
 #endif\r
 \r
        addq    $ STACKSIZE, %rsp\r

diff --git a/make.inc b/make.inc
index affae3a..da430b7 100644 (file)
--- a/make.inc
+++ b/make.inc
@@ -1,7 +1,7 @@
 SHELL = /bin/sh
 PLAT = _LINUX
 DRVOPTS  = $(OPTS)
-LOADER   = $(FORTRAN) -pthread
+LOADER   = $(FORTRAN)
 ARCHFLAGS= -ru
 #RANLIB   = ranlib
 

diff --git a/param.h b/param.h
index 71af5c6..880219b 100644 (file)
--- a/param.h
+++ b/param.h
@@ -1111,14 +1111,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define SGEMM_DEFAULT_UNROLL_M 16
 #define DGEMM_DEFAULT_UNROLL_M 8
 #define QGEMM_DEFAULT_UNROLL_M 2
-#define CGEMM_DEFAULT_UNROLL_M 2
+#define CGEMM_DEFAULT_UNROLL_M 8
 #define ZGEMM_DEFAULT_UNROLL_M 4
 #define XGEMM_DEFAULT_UNROLL_M 1
 
 #define SGEMM_DEFAULT_UNROLL_N 4
 #define DGEMM_DEFAULT_UNROLL_N 4
 #define QGEMM_DEFAULT_UNROLL_N 2
-#define CGEMM_DEFAULT_UNROLL_N 4
+#define CGEMM_DEFAULT_UNROLL_N 2
 #define ZGEMM_DEFAULT_UNROLL_N 4
 #define XGEMM_DEFAULT_UNROLL_N 1
 #endif
@@ -1134,7 +1134,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define QGEMM_DEFAULT_P 504
 #define QGEMM_DEFAULT_R qgemm_r
 
-#define CGEMM_DEFAULT_P 128
+#define CGEMM_DEFAULT_P 384
 //#define CGEMM_DEFAULT_R cgemm_r
 #define CGEMM_DEFAULT_R 1024
 
@@ -1148,7 +1148,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define SGEMM_DEFAULT_Q 384
 #define DGEMM_DEFAULT_Q 256
 #define QGEMM_DEFAULT_Q 128
-#define CGEMM_DEFAULT_Q 256
+#define CGEMM_DEFAULT_Q 192
 #define ZGEMM_DEFAULT_Q 192
 #define XGEMM_DEFAULT_Q 128