optimized sgemm kernel for haswell

diff --git a/kernel/x86_64/sgemm_kernel_16x4_haswell.S b/kernel/x86_64/sgemm_kernel_16x4_haswell.S
index 1f9f88657ed22f9167b1e7c35d22d231b100293d..d88add02bc2dacf8e084ae4c0b346dc554ed337a 100644 (file)
--- a/kernel/x86_64/sgemm_kernel_16x4_haswell.S
+++ b/kernel/x86_64/sgemm_kernel_16x4_haswell.S
@@ -26,7 +26,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 **********************************************************************************/\r
 \r
 /*********************************************************************\r
-* 2013/11/13 Saar\r
+* 2014/07/28 Saar\r
 *        BLASTEST               : OK\r
 *        CTEST                  : OK\r
 *        TEST                   : OK\r
@@ -40,12 +40,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *      A_PR1                   512\r
 *      B_PR1                   512\r
 *      \r
-*\r
+* \r
+* 2014/07/28 Saar\r
 * Performance at 9216x9216x9216:\r
-*       1 thread:       86 GFLOPS       (SANDYBRIDGE:  59)      (MKL:   83)\r
-*       2 threads:     157 GFLOPS       (SANDYBRIDGE: 116)      (MKL:  155)\r
-*       3 threads:     235 GFLOPS       (SANDYBRIDGE: 165)      (MKL:  230)\r
-*       4 threads:     288 GFLOPS       (SANDYBRIDGE: 223)      (MKL:  267)\r
+*       1 thread:      102 GFLOPS       (SANDYBRIDGE:  59)      (MKL:   83)\r
+*       2 threads:     195 GFLOPS       (SANDYBRIDGE: 116)      (MKL:  155)\r
+*       3 threads:     281 GFLOPS       (SANDYBRIDGE: 165)      (MKL:  230)\r
+*       4 threads:     366 GFLOPS       (SANDYBRIDGE: 223)      (MKL:  267)\r
 *\r
 *********************************************************************/\r
 \r
@@ -69,6 +70,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define        CO1     %r15\r
 #define K      %r12\r
 #define BI     %rbp\r
+#define BO2    %rbp\r
 #define        SP      %rbx\r
 \r
 #define BO1    %rdi\r
@@ -90,7 +92,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 #endif\r
 \r
+#if defined(OS_WINDOWS)\r
 #define L_BUFFER_SIZE 8192\r
+#else\r
+#define L_BUFFER_SIZE 12288\r
+#endif\r
 \r
 #define Ndiv6   24(%rsp)\r
 #define Nmod6   32(%rsp)\r
@@ -145,6 +151,373 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define        A_PR1   512\r
 #define        B_PR1   512\r
 \r
+/*******************************************************************************************\r
+* 6 lines of N\r
+*******************************************************************************************/\r
+\r
+.macro KERNEL16x6_SUB\r
+       vmovups         -16 * SIZE(AO), %ymm0\r
+       vmovups          -8 * SIZE(AO), %ymm1\r
+       vbroadcastss     -4 * SIZE(BO), %ymm2\r
+       vbroadcastss     -3 * SIZE(BO), %ymm3\r
+       prefetcht0      A_PR1(AO)\r
+\r
+       VFMADD231PS_(   %ymm4,%ymm2,%ymm0 )\r
+       VFMADD231PS_(   %ymm5,%ymm2,%ymm1 )\r
+       VFMADD231PS_(   %ymm6,%ymm3,%ymm0 )\r
+       VFMADD231PS_(   %ymm7,%ymm3,%ymm1 )\r
+\r
+       vbroadcastss     -2 * SIZE(BO), %ymm2\r
+       vbroadcastss     -1 * SIZE(BO), %ymm3\r
+       VFMADD231PS_(   %ymm8,%ymm2,%ymm0  )\r
+       VFMADD231PS_(   %ymm9,%ymm2,%ymm1  )\r
+       VFMADD231PS_(   %ymm10,%ymm3,%ymm0 )\r
+       VFMADD231PS_(   %ymm11,%ymm3,%ymm1 )\r
+\r
+       vbroadcastss      0 * SIZE(BO), %ymm2\r
+       vbroadcastss      1 * SIZE(BO), %ymm3\r
+       VFMADD231PS_(   %ymm12,%ymm2,%ymm0  )\r
+       VFMADD231PS_(   %ymm13,%ymm2,%ymm1  )\r
+       VFMADD231PS_(   %ymm14,%ymm3,%ymm0 )\r
+       VFMADD231PS_(   %ymm15,%ymm3,%ymm1 )\r
+\r
+       addq    $6*SIZE, BO \r
+       addq    $16*SIZE, AO \r
+       decq    %rax \r
+.endm\r
+\r
+.macro SAVE16x6\r
+\r
+       vbroadcastss    ALPHA, %ymm0\r
+\r
+       vmulps  %ymm0 , %ymm4 , %ymm4\r
+       vmulps  %ymm0 , %ymm5 , %ymm5\r
+       vmulps  %ymm0 , %ymm6 , %ymm6\r
+       vmulps  %ymm0 , %ymm7 , %ymm7\r
+       vmulps  %ymm0 , %ymm8 , %ymm8\r
+       vmulps  %ymm0 , %ymm9 , %ymm9\r
+       vmulps  %ymm0 , %ymm10, %ymm10\r
+       vmulps  %ymm0 , %ymm11, %ymm11\r
+       vmulps  %ymm0 , %ymm12, %ymm12\r
+       vmulps  %ymm0 , %ymm13, %ymm13\r
+       vmulps  %ymm0 , %ymm14, %ymm14\r
+       vmulps  %ymm0 , %ymm15, %ymm15\r
+\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddps          (CO1), %ymm4,%ymm4\r
+       vaddps  8 * SIZE(CO1), %ymm5,%ymm5\r
+\r
+       vaddps          (CO1, LDC), %ymm6,%ymm6\r
+       vaddps  8 * SIZE(CO1, LDC), %ymm7,%ymm7\r
+\r
+       vaddps          (CO1, LDC,2), %ymm8,%ymm8\r
+       vaddps  8 * SIZE(CO1, LDC,2), %ymm9,%ymm9\r
+\r
+       vaddps          (CO2), %ymm10,%ymm10\r
+       vaddps  8 * SIZE(CO2), %ymm11,%ymm11\r
+\r
+       vaddps          (CO2, LDC), %ymm12,%ymm12\r
+       vaddps  8 * SIZE(CO2, LDC), %ymm13,%ymm13\r
+\r
+       vaddps          (CO2, LDC,2), %ymm14,%ymm14\r
+       vaddps  8 * SIZE(CO2, LDC,2), %ymm15,%ymm15\r
+\r
+#endif\r
+\r
+       vmovups %ymm4 ,         (CO1)\r
+       vmovups %ymm5 , 8 * SIZE(CO1)\r
+\r
+       vmovups %ymm6 ,         (CO1, LDC)\r
+       vmovups %ymm7 , 8 * SIZE(CO1, LDC)\r
+\r
+       vmovups %ymm8 ,         (CO1, LDC,2)\r
+       vmovups %ymm9 , 8 * SIZE(CO1, LDC,2)\r
+\r
+       vmovups %ymm10,         (CO2)\r
+       vmovups %ymm11, 8 * SIZE(CO2)\r
+\r
+       vmovups %ymm12,         (CO2, LDC)\r
+       vmovups %ymm13, 8 * SIZE(CO2, LDC)\r
+\r
+       vmovups %ymm14,         (CO2, LDC,2)\r
+       vmovups %ymm15, 8 * SIZE(CO2, LDC,2)\r
+\r
+.endm\r
+\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL8x6_SUB\r
+       vmovups         -16 * SIZE(AO), %ymm0\r
+       vbroadcastss     -4 * SIZE(BO), %ymm2\r
+       vbroadcastss     -3 * SIZE(BO), %ymm3\r
+\r
+       VFMADD231PS_(   %ymm4,%ymm2,%ymm0 )\r
+       VFMADD231PS_(   %ymm6,%ymm3,%ymm0 )\r
+\r
+       vbroadcastss     -2 * SIZE(BO), %ymm2\r
+       vbroadcastss     -1 * SIZE(BO), %ymm3\r
+       VFMADD231PS_(   %ymm8,%ymm2,%ymm0  )\r
+       VFMADD231PS_(   %ymm10,%ymm3,%ymm0 )\r
+\r
+       vbroadcastss      0 * SIZE(BO), %ymm2\r
+       vbroadcastss      1 * SIZE(BO), %ymm3\r
+       VFMADD231PS_(   %ymm12,%ymm2,%ymm0  )\r
+       VFMADD231PS_(   %ymm14,%ymm3,%ymm0 )\r
+\r
+       addq    $6*SIZE, BO \r
+       addq    $8*SIZE, AO \r
+       decq    %rax \r
+.endm\r
+\r
+.macro SAVE8x6\r
+\r
+       vbroadcastss    ALPHA, %ymm0\r
+\r
+       vmulps  %ymm0 , %ymm4 , %ymm4\r
+       vmulps  %ymm0 , %ymm6 , %ymm6\r
+       vmulps  %ymm0 , %ymm8 , %ymm8\r
+       vmulps  %ymm0 , %ymm10, %ymm10\r
+       vmulps  %ymm0 , %ymm12, %ymm12\r
+       vmulps  %ymm0 , %ymm14, %ymm14\r
+\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddps          (CO1), %ymm4,%ymm4\r
+       vaddps          (CO1, LDC), %ymm6,%ymm6\r
+       vaddps          (CO1, LDC,2), %ymm8,%ymm8\r
+       vaddps          (CO2), %ymm10,%ymm10\r
+       vaddps          (CO2, LDC), %ymm12,%ymm12\r
+       vaddps          (CO2, LDC,2), %ymm14,%ymm14\r
+\r
+#endif\r
+\r
+       vmovups %ymm4 ,         (CO1)\r
+       vmovups %ymm6 ,         (CO1, LDC)\r
+       vmovups %ymm8 ,         (CO1, LDC,2)\r
+       vmovups %ymm10,         (CO2)\r
+       vmovups %ymm12,         (CO2, LDC)\r
+       vmovups %ymm14,         (CO2, LDC,2)\r
+\r
+.endm\r
+\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL4x6_SUB\r
+       vmovups         -16 * SIZE(AO), %xmm0\r
+       vbroadcastss     -4 * SIZE(BO), %xmm2\r
+       vbroadcastss     -3 * SIZE(BO), %xmm3\r
+\r
+       VFMADD231PS_(   %xmm4,%xmm2,%xmm0 )\r
+       VFMADD231PS_(   %xmm6,%xmm3,%xmm0 )\r
+\r
+       vbroadcastss     -2 * SIZE(BO), %xmm2\r
+       vbroadcastss     -1 * SIZE(BO), %xmm3\r
+       VFMADD231PS_(   %xmm8,%xmm2,%xmm0  )\r
+       VFMADD231PS_(   %xmm10,%xmm3,%xmm0 )\r
+\r
+       vbroadcastss      0 * SIZE(BO), %xmm2\r
+       vbroadcastss      1 * SIZE(BO), %xmm3\r
+       VFMADD231PS_(   %xmm12,%xmm2,%xmm0  )\r
+       VFMADD231PS_(   %xmm14,%xmm3,%xmm0 )\r
+\r
+       addq    $6*SIZE, BO \r
+       addq    $4*SIZE, AO \r
+       decq    %rax \r
+.endm\r
+\r
+.macro SAVE4x6\r
+\r
+       vbroadcastss    ALPHA, %xmm0\r
+\r
+       vmulps  %xmm0 , %xmm4 , %xmm4\r
+       vmulps  %xmm0 , %xmm6 , %xmm6\r
+       vmulps  %xmm0 , %xmm8 , %xmm8\r
+       vmulps  %xmm0 , %xmm10, %xmm10\r
+       vmulps  %xmm0 , %xmm12, %xmm12\r
+       vmulps  %xmm0 , %xmm14, %xmm14\r
+\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddps          (CO1), %xmm4,%xmm4\r
+       vaddps          (CO1, LDC), %xmm6,%xmm6\r
+       vaddps          (CO1, LDC,2), %xmm8,%xmm8\r
+       vaddps          (CO2), %xmm10,%xmm10\r
+       vaddps          (CO2, LDC), %xmm12,%xmm12\r
+       vaddps          (CO2, LDC,2), %xmm14,%xmm14\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm6 ,         (CO1, LDC)\r
+       vmovups %xmm8 ,         (CO1, LDC,2)\r
+       vmovups %xmm10,         (CO2)\r
+       vmovups %xmm12,         (CO2, LDC)\r
+       vmovups %xmm14,         (CO2, LDC,2)\r
+\r
+.endm\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL2x6_SUB\r
+       vmovss  -16 * SIZE(AO), %xmm0\r
+       vmovss  -15 * SIZE(AO), %xmm1\r
+       vmovss   -4 * SIZE(BO), %xmm2\r
+       vmovss   -3 * SIZE(BO), %xmm3\r
+\r
+       VFMADD231SS_(   %xmm4,%xmm2,%xmm0 )\r
+       VFMADD231SS_(   %xmm5,%xmm2,%xmm1 )\r
+       VFMADD231SS_(   %xmm6,%xmm3,%xmm0 )\r
+       VFMADD231SS_(   %xmm7,%xmm3,%xmm1 )\r
+\r
+       vmovss   -2 * SIZE(BO), %xmm2\r
+       vmovss   -1 * SIZE(BO), %xmm3\r
+       VFMADD231SS_(   %xmm8,%xmm2,%xmm0  )\r
+       VFMADD231SS_(   %xmm9,%xmm2,%xmm1  )\r
+       VFMADD231SS_(   %xmm10,%xmm3,%xmm0 )\r
+       VFMADD231SS_(   %xmm11,%xmm3,%xmm1 )\r
+\r
+       vmovss    0 * SIZE(BO), %xmm2\r
+       vmovss    1 * SIZE(BO), %xmm3\r
+       VFMADD231SS_(   %xmm12,%xmm2,%xmm0  )\r
+       VFMADD231SS_(   %xmm13,%xmm2,%xmm1  )\r
+       VFMADD231SS_(   %xmm14,%xmm3,%xmm0 )\r
+       VFMADD231SS_(   %xmm15,%xmm3,%xmm1 )\r
+\r
+       addq    $6*SIZE, BO \r
+       addq    $2*SIZE, AO \r
+       decq    %rax \r
+.endm\r
+\r
+.macro SAVE2x6\r
+\r
+       vmovss  ALPHA, %xmm0\r
+\r
+       vmulss  %xmm0 , %xmm4 , %xmm4\r
+       vmulss  %xmm0 , %xmm5 , %xmm5\r
+       vmulss  %xmm0 , %xmm6 , %xmm6\r
+       vmulss  %xmm0 , %xmm7 , %xmm7\r
+       vmulss  %xmm0 , %xmm8 , %xmm8\r
+       vmulss  %xmm0 , %xmm9 , %xmm9\r
+       vmulss  %xmm0 , %xmm10, %xmm10\r
+       vmulss  %xmm0 , %xmm11, %xmm11\r
+       vmulss  %xmm0 , %xmm12, %xmm12\r
+       vmulss  %xmm0 , %xmm13, %xmm13\r
+       vmulss  %xmm0 , %xmm14, %xmm14\r
+       vmulss  %xmm0 , %xmm15, %xmm15\r
+\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+       vaddss  1 * SIZE(CO1), %xmm5,%xmm5\r
+\r
+       vaddss          (CO1, LDC), %xmm6,%xmm6\r
+       vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7\r
+\r
+       vaddss          (CO1, LDC,2), %xmm8,%xmm8\r
+       vaddss  1 * SIZE(CO1, LDC,2), %xmm9,%xmm9\r
+\r
+       vaddss          (CO2), %xmm10,%xmm10\r
+       vaddss  1 * SIZE(CO2), %xmm11,%xmm11\r
+\r
+       vaddss          (CO2, LDC), %xmm12,%xmm12\r
+       vaddss  1 * SIZE(CO2, LDC), %xmm13,%xmm13\r
+\r
+       vaddss          (CO2, LDC,2), %xmm14,%xmm14\r
+       vaddss  1 * SIZE(CO2, LDC,2), %xmm15,%xmm15\r
+\r
+#endif\r
+\r
+       vmovss  %xmm4 ,         (CO1)\r
+       vmovss  %xmm5 , 1 * SIZE(CO1)\r
+\r
+       vmovss  %xmm6 ,         (CO1, LDC)\r
+       vmovss  %xmm7 , 1 * SIZE(CO1, LDC)\r
+\r
+       vmovss  %xmm8 ,         (CO1, LDC,2)\r
+       vmovss  %xmm9 , 1 * SIZE(CO1, LDC,2)\r
+\r
+       vmovss  %xmm10,         (CO2)\r
+       vmovss  %xmm11, 1 * SIZE(CO2)\r
+\r
+       vmovss  %xmm12,         (CO2, LDC)\r
+       vmovss  %xmm13, 1 * SIZE(CO2, LDC)\r
+\r
+       vmovss  %xmm14,         (CO2, LDC,2)\r
+       vmovss  %xmm15, 1 * SIZE(CO2, LDC,2)\r
+\r
+.endm\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL1x6_SUB\r
+       vmovss  -16 * SIZE(AO), %xmm0\r
+       vmovss   -4 * SIZE(BO), %xmm2\r
+       vmovss   -3 * SIZE(BO), %xmm3\r
+\r
+       VFMADD231SS_(   %xmm4,%xmm2,%xmm0 )\r
+       VFMADD231SS_(   %xmm6,%xmm3,%xmm0 )\r
+\r
+       vmovss   -2 * SIZE(BO), %xmm2\r
+       vmovss   -1 * SIZE(BO), %xmm3\r
+       VFMADD231SS_(   %xmm8,%xmm2,%xmm0  )\r
+       VFMADD231SS_(   %xmm10,%xmm3,%xmm0 )\r
+\r
+       vmovss    0 * SIZE(BO), %xmm2\r
+       vmovss    1 * SIZE(BO), %xmm3\r
+       VFMADD231SS_(   %xmm12,%xmm2,%xmm0  )\r
+       VFMADD231SS_(   %xmm14,%xmm3,%xmm0 )\r
+\r
+       addq    $6*SIZE, BO \r
+       addq    $1*SIZE, AO \r
+       decq    %rax \r
+.endm\r
+\r
+.macro SAVE1x6\r
+\r
+       vmovss  ALPHA, %xmm0\r
+\r
+       vmulss  %xmm0 , %xmm4 , %xmm4\r
+       vmulss  %xmm0 , %xmm6 , %xmm6\r
+       vmulss  %xmm0 , %xmm8 , %xmm8\r
+       vmulss  %xmm0 , %xmm10, %xmm10\r
+       vmulss  %xmm0 , %xmm12, %xmm12\r
+       vmulss  %xmm0 , %xmm14, %xmm14\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+       vaddss          (CO1, LDC), %xmm6,%xmm6\r
+       vaddss          (CO1, LDC,2), %xmm8,%xmm8\r
+       vaddss          (CO2), %xmm10,%xmm10\r
+       vaddss          (CO2, LDC), %xmm12,%xmm12\r
+       vaddss          (CO2, LDC,2), %xmm14,%xmm14\r
+\r
+#endif\r
+\r
+       vmovss  %xmm4 ,         (CO1)\r
+       vmovss  %xmm6 ,         (CO1, LDC)\r
+       vmovss  %xmm8 ,         (CO1, LDC,2)\r
+       vmovss  %xmm10,         (CO2)\r
+       vmovss  %xmm12,         (CO2, LDC)\r
+       vmovss  %xmm14,         (CO2, LDC,2)\r
+\r
+.endm\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+\r
 /*******************************************************************************************\r
 * 4 lines of N\r
 *******************************************************************************************/\r
@@ -343,17 +716,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 #if !defined(TRMMKERNEL)\r
 \r
-       vaddps          (CO1), %xmm4,%xmm4\r
-       vaddps  1 * SIZE(CO1), %xmm5,%xmm5\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+       vaddss  1 * SIZE(CO1), %xmm5,%xmm5\r
 \r
-       vaddps          (CO1, LDC), %xmm6,%xmm6\r
-       vaddps  1 * SIZE(CO1, LDC), %xmm7,%xmm7\r
+       vaddss          (CO1, LDC), %xmm6,%xmm6\r
+       vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7\r
 \r
-       vaddps          (CO2), %xmm8,%xmm8\r
-       vaddps  1 * SIZE(CO2), %xmm9,%xmm9\r
+       vaddss          (CO2), %xmm8,%xmm8\r
+       vaddss  1 * SIZE(CO2), %xmm9,%xmm9\r
 \r
-       vaddps          (CO2, LDC), %xmm10,%xmm10\r
-       vaddps  1 * SIZE(CO2, LDC), %xmm11,%xmm11\r
+       vaddss          (CO2, LDC), %xmm10,%xmm10\r
+       vaddss  1 * SIZE(CO2, LDC), %xmm11,%xmm11\r
 \r
 #endif\r
 \r
@@ -400,10 +773,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 #if !defined(TRMMKERNEL)\r
 \r
-       vaddps          (CO1), %xmm4,%xmm4\r
-       vaddps          (CO1, LDC), %xmm6,%xmm6\r
-       vaddps          (CO2), %xmm8,%xmm8\r
-       vaddps          (CO2, LDC), %xmm10,%xmm10\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+       vaddss          (CO1, LDC), %xmm6,%xmm6\r
+       vaddss          (CO2), %xmm8,%xmm8\r
+       vaddss          (CO2, LDC), %xmm10,%xmm10\r
 \r
 #endif\r
 \r
@@ -556,11 +929,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 #if !defined(TRMMKERNEL)\r
 \r
-       vaddps          (CO1), %xmm4,%xmm4\r
-       vaddps  1 * SIZE(CO1), %xmm5,%xmm5\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+       vaddss  1 * SIZE(CO1), %xmm5,%xmm5\r
 \r
-       vaddps          (CO1, LDC), %xmm6,%xmm6\r
-       vaddps  1 * SIZE(CO1, LDC), %xmm7,%xmm7\r
+       vaddss          (CO1, LDC), %xmm6,%xmm6\r
+       vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7\r
 \r
 #endif\r
 \r
@@ -594,8 +967,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 #if !defined(TRMMKERNEL)\r
 \r
-       vaddps          (CO1), %xmm4,%xmm4\r
-       vaddps          (CO1, LDC), %xmm6,%xmm6\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+       vaddss          (CO1, LDC), %xmm6,%xmm6\r
 \r
 #endif\r
 \r
@@ -621,141 +994,3418 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        addq    $ 16, %rax \r
 .endm\r
 \r
-.macro SAVE16x1\r
+.macro SAVE16x1\r
+\r
+       vbroadcastss    ALPHA, %ymm0\r
+\r
+       vmulps  %ymm0 , %ymm4 , %ymm4\r
+       vmulps  %ymm0 , %ymm5 , %ymm5\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddps          (CO1), %ymm4,%ymm4\r
+       vaddps  8 * SIZE(CO1), %ymm5,%ymm5\r
+\r
+#endif\r
+\r
+       vmovups %ymm4 ,         (CO1)\r
+       vmovups %ymm5 , 8 * SIZE(CO1)\r
+\r
+.endm\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL8x1_SUB\r
+       vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0\r
+       vbroadcastss     -4 * SIZE(BO, BI, SIZE), %ymm2\r
+       VFMADD231PS_(   %ymm4,%ymm2,%ymm0 )\r
+       addq    $ 1 , BI        \r
+       addq    $ 8 , %rax \r
+.endm\r
+\r
+.macro SAVE8x1\r
+\r
+       vbroadcastss    ALPHA, %ymm0\r
+\r
+       vmulps  %ymm0 , %ymm4 , %ymm4\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddps          (CO1), %ymm4,%ymm4\r
+\r
+#endif\r
+\r
+       vmovups %ymm4 ,         (CO1)\r
+\r
+.endm\r
+\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL4x1_SUB\r
+       vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+       vbroadcastss     -4 * SIZE(BO, BI, SIZE), %xmm2\r
+       VFMADD231PS_(   %xmm4,%xmm2,%xmm0 )\r
+       addq    $ 1 , BI        \r
+       addq    $ 4 , %rax \r
+.endm\r
+\r
+.macro SAVE4x1\r
+\r
+       vbroadcastss    ALPHA, %xmm0\r
+\r
+       vmulps  %xmm0 , %xmm4 , %xmm4\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddps          (CO1), %xmm4,%xmm4\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+\r
+.endm\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL2x1_SUB\r
+       vmovss  -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+       vmovss  -15 * SIZE(AO, %rax, SIZE), %xmm1\r
+       vmovss   -4 * SIZE(BO, BI, SIZE), %xmm2\r
+       VFMADD231SS_(   %xmm4,%xmm2,%xmm0 )\r
+       VFMADD231SS_(   %xmm5,%xmm2,%xmm1 )\r
+       addq    $ 1 , BI        \r
+       addq    $ 2 , %rax \r
+.endm\r
+\r
+.macro SAVE2x1\r
+\r
+       vmovss  ALPHA, %xmm0\r
+\r
+       vmulss  %xmm0 , %xmm4 , %xmm4\r
+       vmulss  %xmm0 , %xmm5 , %xmm5\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+       vaddss  1 * SIZE(CO1), %xmm5,%xmm5\r
+\r
+#endif\r
+\r
+       vmovss  %xmm4 ,         (CO1)\r
+       vmovss  %xmm5 , 1 * SIZE(CO1)\r
+\r
+.endm\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.macro KERNEL1x1_SUB\r
+       vmovss  -16 * SIZE(AO, %rax, SIZE), %xmm0\r
+       vmovss   -4 * SIZE(BO, BI, SIZE), %xmm2\r
+       VFMADD231SS_(   %xmm4,%xmm2,%xmm0 )\r
+       addq    $ 1 , BI        \r
+       addq    $ 1 , %rax \r
+.endm\r
+\r
+.macro SAVE1x1\r
+\r
+       vmovss  ALPHA, %xmm0\r
+\r
+       vmulss  %xmm0 , %xmm4 , %xmm4\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+       vaddss          (CO1), %xmm4,%xmm4\r
+\r
+#endif\r
+\r
+       vmovss  %xmm4 ,         (CO1)\r
+\r
+.endm\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+#if !defined(TRMMKERNEL)\r
+\r
+/*************************************************************************************\r
+* GEMM Kernel\r
+*************************************************************************************/\r
+\r
+\r
+       PROLOGUE\r
+       PROFCODE\r
+       \r
+       subq    $STACKSIZE, %rsp\r
+       movq    %rbx,   (%rsp)\r
+       movq    %rbp,  8(%rsp)\r
+       movq    %r12, 16(%rsp)\r
+       movq    %r13, 24(%rsp)\r
+       movq    %r14, 32(%rsp)\r
+       movq    %r15, 40(%rsp)\r
+\r
+       vzeroupper\r
+\r
+#ifdef WINDOWS_ABI\r
+       movq    %rdi,    48(%rsp)\r
+       movq    %rsi,    56(%rsp)\r
+       movups  %xmm6,   64(%rsp)\r
+       movups  %xmm7,   80(%rsp)\r
+       movups  %xmm8,   96(%rsp)\r
+       movups  %xmm9,  112(%rsp)\r
+       movups  %xmm10, 128(%rsp)\r
+       movups  %xmm11, 144(%rsp)\r
+       movups  %xmm12, 160(%rsp)\r
+       movups  %xmm13, 176(%rsp)\r
+       movups  %xmm14, 192(%rsp)\r
+       movups  %xmm15, 208(%rsp)\r
+\r
+       movq    ARG1,      OLD_M\r
+       movq    ARG2,      OLD_N\r
+       movq    ARG3,      OLD_K\r
+       movq    OLD_A,     A\r
+       movq    OLD_B,     B\r
+       movq    OLD_C,     C\r
+       movq    OLD_LDC,   LDC\r
+#ifdef TRMMKERNEL\r
+       vmovsd  OLD_OFFSET, %xmm12\r
+#endif\r
+       vmovaps %xmm3, %xmm0\r
+\r
+#else\r
+       movq    STACKSIZE +  8(%rsp), LDC\r
+#ifdef TRMMKERNEL\r
+       movsd   STACKSIZE + 16(%rsp), %xmm12\r
+#endif\r
+\r
+#endif\r
+\r
+       movq    %rsp, SP      # save old stack\r
+        subq    $128 + L_BUFFER_SIZE, %rsp\r
+        andq    $-4096, %rsp    # align stack\r
+\r
+        STACK_TOUCH\r
+\r
+       cmpq    $0, OLD_M\r
+       je      .L999\r
+\r
+       cmpq    $0, OLD_N\r
+       je      .L999\r
+\r
+       cmpq    $0, OLD_K\r
+       je      .L999\r
+\r
+       movq    OLD_M, M\r
+       movq    OLD_N, N\r
+       movq    OLD_K, K\r
+\r
+       vmovss   %xmm0, ALPHA\r
+\r
+       salq    $BASE_SHIFT, LDC\r
+\r
+       movq    N, %rax\r
+        xorq    %rdx, %rdx\r
+        movq    $12,  %rdi\r
+        divq    %rdi                    //    N / 12\r
+        movq    %rax, Ndiv6             //    N / 12\r
+        movq    %rdx, Nmod6             //    N % 12\r
+\r
+       movq    Ndiv6,  J\r
+       cmpq    $0, J\r
+       je      .L4_00\r
+       ALIGN_4\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+.L6_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       salq    $2, %rax                // 4 values of B\r
+        leaq    (B, %rax,4), BO2\r
+        movq    BO2, B                  // next offset of B\r
+        movq    K, %rax\r
+\r
+       ALIGN_4\r
+\r
+\r
+.L6_02c:\r
+\r
+       vmovups (BO1), %xmm0\r
+       vmovsd  (BO2), %xmm1\r
+       vmovups %xmm0, (BO)\r
+       vmovsd  %xmm1, 4*SIZE(BO)\r
+       addq    $ 4*SIZE,BO1\r
+       addq    $ 4*SIZE,BO2\r
+       addq    $ 6*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L6_02c\r
+\r
+\r
+.L6_10:\r
+       movq     C, CO1\r
+       leaq    (C,   LDC, 2), CO2      \r
+       leaq    (CO2, LDC, 1), CO2              // co2 = c + 3 * ldc\r
+       leaq    (C,   LDC, 4), C        \r
+       leaq    (C,   LDC, 2), C                // c = c + 6 * ldc\r
+\r
+       movq    A, AO                   // aoffset = a\r
+       addq    $ 16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $4, I                   // i = (m >> 4)\r
+       je      .L6_20\r
+\r
+       ALIGN_4\r
+\r
+.L6_11:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L6_16\r
+\r
+       ALIGN_4\r
+\r
+.L6_12:\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       je      .L6_16\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       je      .L6_16\r
+\r
+       jmp     .L6_12\r
+       ALIGN_4\r
+\r
+.L6_16:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_19\r
+\r
+       ALIGN_4\r
+\r
+.L6_17:\r
+\r
+       KERNEL16x6_SUB\r
+\r
+       jnz     .L6_17\r
+       ALIGN_4\r
+\r
+\r
+.L6_19:\r
+\r
+       SAVE16x6\r
+\r
+       addq    $16 * SIZE, CO1         # coffset += 16\r
+       addq    $16 * SIZE, CO2         # coffset += 16\r
+       decq    I                       # i --\r
+       jg      .L6_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L6_20:\r
+       // Test rest of M\r
+\r
+       testq   $15, M\r
+       jz      .L6_60          // to next 6 lines of N\r
+\r
+       testq   $8, M           \r
+       jz      .L6_21pre\r
+       ALIGN_4\r
+\r
+/**************************************************************************/\r
+\r
+.L6_20_1:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L6_20_6\r
+\r
+       ALIGN_4\r
+\r
+.L6_20_2:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       je      .L6_20_6\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       je      .L6_20_6\r
+\r
+       jmp     .L6_20_2\r
+       ALIGN_4\r
+\r
+.L6_20_6:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_20_9\r
+\r
+       ALIGN_4\r
+\r
+.L6_20_7:\r
+\r
+       KERNEL8x6_SUB\r
+\r
+       jnz     .L6_20_7\r
+       ALIGN_4\r
+\r
+\r
+.L6_20_9:\r
+\r
+       SAVE8x6\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       addq    $8 * SIZE, CO2          # coffset += 8\r
+       ALIGN_4\r
+       \r
+\r
+\r
+/**************************************************************************/\r
+\r
+.L6_21pre:\r
+\r
+       testq   $4, M           \r
+       jz      .L6_30\r
+       ALIGN_4\r
+\r
+.L6_21:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L6_26\r
+\r
+       ALIGN_4\r
+\r
+.L6_22:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       je      .L6_26\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       je      .L6_26\r
+\r
+       jmp     .L6_22\r
+       ALIGN_4\r
+\r
+.L6_26:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_29\r
+\r
+       ALIGN_4\r
+\r
+.L6_27:\r
+\r
+       KERNEL4x6_SUB\r
+\r
+       jnz     .L6_27\r
+       ALIGN_4\r
+\r
+\r
+.L6_29:\r
+\r
+       SAVE4x6\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       addq    $4 * SIZE, CO2          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L6_30:\r
+       testq   $2, M           \r
+       jz      .L6_40\r
+\r
+       ALIGN_4\r
+\r
+.L6_31:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L6_36\r
+\r
+       ALIGN_4\r
+\r
+.L6_32:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       je      .L6_36\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       je      .L6_36\r
+\r
+       jmp     .L6_32\r
+       ALIGN_4\r
+\r
+.L6_36:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_39\r
+\r
+       ALIGN_4\r
+\r
+.L6_37:\r
+\r
+       KERNEL2x6_SUB\r
+\r
+       jnz     .L6_37\r
+       ALIGN_4\r
+\r
+\r
+.L6_39:\r
+\r
+       SAVE2x6\r
+\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       addq    $2 * SIZE, CO2          # coffset += 2\r
+       ALIGN_4\r
+\r
+.L6_40:\r
+       testq   $1, M           \r
+       jz      .L6_60          // to next 4 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L6_41:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L6_46\r
+\r
+       ALIGN_4\r
+\r
+.L6_42:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       je      .L6_46\r
+\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       je      .L6_46\r
+\r
+       jmp     .L6_42\r
+       ALIGN_4\r
+\r
+.L6_46:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_49\r
+\r
+       ALIGN_4\r
+\r
+.L6_47:\r
+\r
+       KERNEL1x6_SUB\r
+\r
+       jnz     .L6_47\r
+       ALIGN_4\r
+\r
+\r
+.L6_49:\r
+\r
+       SAVE1x6\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       addq    $1 * SIZE, CO2          # coffset += 1\r
+       ALIGN_4\r
+       \r
+\r
+\r
+\r
+       \r
+.L6_60:\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+\r
+.L7_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       salq    $2, %rax                // 4 values of B\r
+        leaq    (B, %rax,4), BO2\r
+        movq    K, %rax\r
+\r
+       ALIGN_4\r
+\r
+\r
+.L7_02c:\r
+\r
+       vmovsd  2*SIZE(BO1), %xmm0\r
+       vmovups       (BO2), %xmm1\r
+       vmovsd  %xmm0, (BO)\r
+       vmovups %xmm1, 2*SIZE(BO)\r
+       addq    $ 4*SIZE,BO1\r
+       addq    $ 4*SIZE,BO2\r
+       addq    $ 6*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L7_02c\r
+\r
+        movq    BO2, B                  // next offset of B\r
+\r
+.L7_10:\r
+       movq     C, CO1\r
+       leaq    (C,   LDC, 2), CO2      \r
+       leaq    (CO2, LDC, 1), CO2              // co2 = c + 3 * ldc\r
+       leaq    (C,   LDC, 4), C        \r
+       leaq    (C,   LDC, 2), C                // c = c + 6 * ldc\r
+\r
+       movq    A, AO                   // aoffset = a\r
+       addq    $ 16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $4, I                   // i = (m >> 4)\r
+       je      .L7_20\r
+\r
+       ALIGN_4\r
+\r
+.L7_11:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L7_16\r
+\r
+       ALIGN_4\r
+\r
+.L7_12:\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       je      .L7_16\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+       KERNEL16x6_SUB\r
+\r
+       je      .L7_16\r
+\r
+       jmp     .L7_12\r
+       ALIGN_4\r
+\r
+.L7_16:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_19\r
+\r
+       ALIGN_4\r
+\r
+.L7_17:\r
+\r
+       KERNEL16x6_SUB\r
+\r
+       jnz     .L7_17\r
+       ALIGN_4\r
+\r
+\r
+.L7_19:\r
+\r
+       SAVE16x6\r
+\r
+       addq    $16 * SIZE, CO1         # coffset += 16\r
+       addq    $16 * SIZE, CO2         # coffset += 16\r
+       decq    I                       # i --\r
+       jg      .L7_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L7_20:\r
+       // Test rest of M\r
+\r
+       testq   $15, M\r
+       jz      .L7_60          // to next 6 lines of N\r
+\r
+       testq   $8, M           \r
+       jz      .L7_21pre\r
+       ALIGN_4\r
+\r
+/**************************************************************************/\r
+\r
+.L7_20_1:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_20_6\r
+\r
+       ALIGN_4\r
+\r
+.L7_20_2:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       je      .L7_20_6\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL8x6_SUB\r
+       KERNEL8x6_SUB\r
+\r
+       je      .L7_20_6\r
+\r
+       jmp     .L7_20_2\r
+       ALIGN_4\r
+\r
+.L7_20_6:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_20_9\r
+\r
+       ALIGN_4\r
+\r
+.L7_20_7:\r
+\r
+       KERNEL8x6_SUB\r
+\r
+       jnz     .L7_20_7\r
+       ALIGN_4\r
+\r
+\r
+.L7_20_9:\r
+\r
+       SAVE8x6\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       addq    $8 * SIZE, CO2          # coffset += 8\r
+       ALIGN_4\r
+       \r
+\r
+\r
+/**************************************************************************/\r
+\r
+.L7_21pre:\r
+\r
+       testq   $4, M           \r
+       jz      .L7_30\r
+       ALIGN_4\r
+\r
+.L7_21:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_26\r
+\r
+       ALIGN_4\r
+\r
+.L7_22:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       je      .L7_26\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+       KERNEL4x6_SUB\r
+\r
+       je      .L7_26\r
+\r
+       jmp     .L7_22\r
+       ALIGN_4\r
+\r
+.L7_26:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_29\r
+\r
+       ALIGN_4\r
+\r
+.L7_27:\r
+\r
+       KERNEL4x6_SUB\r
+\r
+       jnz     .L7_27\r
+       ALIGN_4\r
+\r
+\r
+.L7_29:\r
+\r
+       SAVE4x6\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       addq    $4 * SIZE, CO2          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L7_30:\r
+       testq   $2, M           \r
+       jz      .L7_40\r
+\r
+       ALIGN_4\r
+\r
+.L7_31:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_36\r
+\r
+       ALIGN_4\r
+\r
+.L7_32:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       je      .L7_36\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+       KERNEL2x6_SUB\r
+\r
+       je      .L7_36\r
+\r
+       jmp     .L7_32\r
+       ALIGN_4\r
+\r
+.L7_36:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_39\r
+\r
+       ALIGN_4\r
+\r
+.L7_37:\r
+\r
+       KERNEL2x6_SUB\r
+\r
+       jnz     .L7_37\r
+       ALIGN_4\r
+\r
+\r
+.L7_39:\r
+\r
+       SAVE2x6\r
+\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       addq    $2 * SIZE, CO2          # coffset += 2\r
+       ALIGN_4\r
+\r
+.L7_40:\r
+       testq   $1, M           \r
+       jz      .L7_60          // to next 4 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L7_41:\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+\r
+       vzeroall\r
+\r
+        movq    K, %rax\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_46\r
+\r
+       ALIGN_4\r
+\r
+.L7_42:\r
+\r
+       prefetcht0      A_PR1(AO)\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       je      .L7_46\r
+\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+       KERNEL1x6_SUB\r
+\r
+       je      .L7_46\r
+\r
+       jmp     .L7_42\r
+       ALIGN_4\r
+\r
+.L7_46:\r
+        movq    K, %rax\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_49\r
+\r
+       ALIGN_4\r
+\r
+.L7_47:\r
+\r
+       KERNEL1x6_SUB\r
+\r
+       jnz     .L7_47\r
+       ALIGN_4\r
+\r
+\r
+.L7_49:\r
+\r
+       SAVE1x6\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       addq    $1 * SIZE, CO2          # coffset += 1\r
+       ALIGN_4\r
+       \r
+\r
+\r
+\r
+       \r
+.L7_60:\r
+\r
+       decq    J                       // j --\r
+       jg      .L6_01                  // next 12 lines of N\r
+\r
+\r
+\r
+\r
+/*******************************************************************************************/\r
+.L4_00:\r
+\r
+       movq    Nmod6,  J\r
+        sarq    $2, J           // j = j / 4\r
+        cmpq    $ 0, J\r
+        je      .L2_00\r
+        ALIGN_4\r
+\r
+\r
+.L4_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       sarq    $2, %rax                // K / 4\r
+       jz      .L4_01b\r
+       ALIGN_4\r
+\r
+\r
+.L4_01a:\r
+        prefetcht0 512(BO1)\r
+        prefetchw  512(BO)\r
+\r
+       vmovups        (BO1), %xmm0\r
+       vmovups  4*SIZE(BO1), %xmm1\r
+       vmovups  8*SIZE(BO1), %xmm2\r
+       vmovups 12*SIZE(BO1), %xmm3\r
+\r
+       vmovups %xmm0,       (BO)\r
+       vmovups %xmm1, 4*SIZE(BO)\r
+       vmovups %xmm2, 8*SIZE(BO)\r
+       vmovups %xmm3,12*SIZE(BO)\r
+\r
+       addq    $ 16*SIZE,BO1\r
+       addq    $ 16*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L4_01a\r
+\r
+\r
+.L4_01b:\r
+\r
+        movq    K, %rax\r
+        andq    $3, %rax                // K % 4\r
+        jz      .L4_02d\r
+        ALIGN_4\r
+\r
+.L4_02c:\r
+\r
+       vmovups (BO1), %xmm0\r
+       vmovups %xmm0, (BO)\r
+       addq    $ 4*SIZE,BO1\r
+       addq    $ 4*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L4_02c\r
+\r
+.L4_02d:\r
+\r
+       movq    BO1, B                  // next offset of B\r
+\r
+.L4_10:\r
+       movq     C, CO1\r
+       leaq    (C, LDC, 2), CO2        \r
+       leaq    (C, LDC, 4), C          // c += 4 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+       \r
+       movq    A, AO                   // aoffset = a\r
+       addq    $ 16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $4, I                   // i = (m >> 4)\r
+       je      .L4_20\r
+\r
+       ALIGN_4\r
+\r
+.L4_11:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO                    // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $16, %rax      // number of values in AO\r
+#else\r
+        addq    $4, %rax       // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L4_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4) , BI                           //  BI = BI * 4 ; number of values\r
+\r
+       salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_12:\r
+\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       prefetcht0      B_PR1(BO, BI  , SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       prefetcht0      B_PR1(BO, BI  , SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+\r
+       je      .L4_16\r
+\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       prefetcht0      B_PR1(BO, BI  , SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       prefetcht0      B_PR1(BO, BI  , SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+       prefetcht0      A_PR1(AO, %rax, SIZE)\r
+       KERNEL16x4_SUB\r
+\r
+       je      .L4_16\r
+\r
+       jmp     .L4_12\r
+       ALIGN_4\r
+\r
+.L4_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L4_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_17:\r
+\r
+       KERNEL16x4_SUB\r
+\r
+       jl      .L4_17\r
+       ALIGN_4\r
+\r
+\r
+.L4_19:\r
+\r
+       SAVE16x4\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $16, KK                                \r
+#endif\r
+\r
+       addq    $16 * SIZE, CO1         # coffset += 16\r
+       addq    $16 * SIZE, CO2         # coffset += 16\r
+       decq    I                       # i --\r
+       jg      .L4_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L4_20:\r
+       // Test rest of M\r
+\r
+       testq   $15, M\r
+       jz      .L4_60          // to next 3 lines of N\r
+\r
+       testq   $8, M           \r
+       jz      .L4_21pre\r
+       ALIGN_4\r
+\r
+/**************************************************************************/\r
+\r
+.L4_20_1:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $8, %rax        // number of values in A\r
+#else\r
+        addq    $4, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L4_20_6\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_20_2:\r
+\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+\r
+       je      .L4_20_6\r
+\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+       KERNEL8x4_SUB\r
+\r
+       je      .L4_20_6\r
+\r
+       jmp     .L4_20_2\r
+       ALIGN_4\r
+\r
+.L4_20_6:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L4_20_9\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_20_7:\r
+\r
+       KERNEL8x4_SUB\r
+\r
+       jl      .L4_20_7\r
+       ALIGN_4\r
+\r
+\r
+.L4_20_9:\r
+\r
+       SAVE8x4\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $8, KK\r
+#endif\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       addq    $8 * SIZE, CO2          # coffset += 8\r
+       ALIGN_4\r
+       \r
+\r
+\r
+/**************************************************************************/\r
+\r
+.L4_21pre:\r
+\r
+       testq   $4, M           \r
+       jz      .L4_30\r
+       ALIGN_4\r
+\r
+.L4_21:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $4, %rax        // number of values in A\r
+#else\r
+        addq    $4, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L4_26\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_22:\r
+\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+\r
+       je      .L4_26\r
+\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+       KERNEL4x4_SUB\r
+\r
+       je      .L4_26\r
+\r
+       jmp     .L4_22\r
+       ALIGN_4\r
+\r
+.L4_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L4_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_27:\r
+\r
+       KERNEL4x4_SUB\r
+\r
+       jl      .L4_27\r
+       ALIGN_4\r
+\r
+\r
+.L4_29:\r
+\r
+       SAVE4x4\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       addq    $4 * SIZE, CO2          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L4_30:\r
+       testq   $2, M           \r
+       jz      .L4_40\r
+\r
+       ALIGN_4\r
+\r
+.L4_31:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $2, %rax        // number of values in AO\r
+#else\r
+        addq    $4, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L4_36\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_32:\r
+\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+\r
+       je      .L4_36\r
+\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+       KERNEL2x4_SUB\r
+\r
+       je      .L4_36\r
+\r
+       jmp     .L4_32\r
+       ALIGN_4\r
+\r
+.L4_36:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L4_39\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+       \r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_37:\r
+\r
+       KERNEL2x4_SUB\r
+\r
+       jl      .L4_37\r
+       ALIGN_4\r
+\r
+\r
+.L4_39:\r
+\r
+       SAVE2x4\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $2, KK\r
+#endif\r
+\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       addq    $2 * SIZE, CO2          # coffset += 2\r
+       ALIGN_4\r
+\r
+.L4_40:\r
+       testq   $1, M           \r
+       jz      .L4_60          // to next 4 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L4_41:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $1, %rax        // number of values in AO\r
+#else\r
+        addq    $4, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax\r
+       je      .L4_46\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_42:\r
+\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+\r
+       je      .L4_46\r
+\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+       KERNEL1x4_SUB\r
+\r
+       je      .L4_46\r
+\r
+       jmp     .L4_42\r
+       ALIGN_4\r
+\r
+.L4_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L4_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (,BI,4), BI                    //  BI = BI * 4 ; number of values\r
+\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L4_47:\r
+\r
+       KERNEL1x4_SUB\r
+\r
+       jl      .L4_47\r
+       ALIGN_4\r
+\r
+\r
+.L4_49:\r
+\r
+       SAVE1x4\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+       leaq    (,BI, 4), BI                    // BI = BI * 4 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $1, KK\r
+#endif\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       addq    $1 * SIZE, CO2          # coffset += 1\r
+       ALIGN_4\r
+       \r
+\r
+\r
+\r
+       \r
+.L4_60:\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       decq    J                       // j --\r
+       jg      .L4_01                  // next 4 lines of N\r
+\r
+\r
+\r
+/*******************************************************************************************/\r
+.L2_00:\r
+\r
+       movq    Nmod6, J                \r
+       andq    $3, J                   // j % 4\r
+       je      .L999\r
+\r
+       movq    Nmod6, J                \r
+       andq    $2, J                   // j % 4\r
+       je      .L1_0\r
+\r
+.L2_01:\r
+\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       sarq    $2, %rax                // K / 4\r
+       jz      .L2_01b\r
+       ALIGN_4\r
+\r
+.L2_01a:\r
+\r
+       vmovsd        (BO1), %xmm0\r
+       vmovsd  2*SIZE(BO1), %xmm1\r
+       vmovsd  4*SIZE(BO1), %xmm2\r
+       vmovsd  6*SIZE(BO1), %xmm3\r
+\r
+       vmovsd  %xmm0,       (BO)\r
+       vmovsd  %xmm1, 2*SIZE(BO)\r
+       vmovsd  %xmm2, 4*SIZE(BO)\r
+       vmovsd  %xmm3, 6*SIZE(BO)\r
+\r
+       addq    $8*SIZE,BO1\r
+       addq    $8*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L2_01a\r
+\r
+\r
+.L2_01b:\r
+\r
+        movq    K, %rax\r
+        andq    $3, %rax                // K % 4\r
+        jz      .L2_02d\r
+        ALIGN_4\r
+\r
+.L2_02c:\r
+\r
+       vmovsd  (BO1), %xmm0\r
+       vmovsd  %xmm0, (BO)\r
+       addq    $2*SIZE,BO1\r
+       addq    $2*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L2_02c\r
+\r
+.L2_02d:\r
+\r
+       movq    BO1, B                  // next offset of B\r
+\r
+.L2_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 2), C          // c += 2 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+       \r
+       movq    A, AO                   // aoffset = a\r
+       addq    $16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $4, I                   // i = (m >> 4)\r
+       je      .L2_20\r
+\r
+       ALIGN_4\r
+\r
+.L2_11:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $16, %rax      // number of values in AO\r
+#else\r
+        addq    $2, %rax       // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L2_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_12:\r
+\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+\r
+       je      .L2_16\r
+\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+       KERNEL16x2_SUB\r
+\r
+       je      .L2_16\r
+\r
+       jmp     .L2_12\r
+       ALIGN_4\r
+\r
+.L2_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_17:\r
+\r
+       KERNEL16x2_SUB\r
+\r
+       jl      .L2_17\r
+       ALIGN_4\r
+\r
+\r
+.L2_19:\r
+\r
+       SAVE16x2\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $16, KK                                \r
+#endif\r
+\r
+       addq    $16 * SIZE, CO1         # coffset += 16\r
+       decq    I                       # i --\r
+       jg      .L2_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L2_20:\r
+       // Test rest of M\r
+\r
+       testq   $15, M\r
+       jz      .L2_60          // to next 2 lines of N\r
+\r
+       testq   $8, M           \r
+       jz      .L2_21pre\r
+       ALIGN_4\r
+\r
+/**************************************************************************/\r
+\r
+.L2_20_1:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $8, %rax        // number of values in A\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L2_20_6\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_20_2:\r
+\r
+\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+\r
+       je      .L2_20_6\r
+\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+       KERNEL8x2_SUB\r
+\r
+       je      .L2_20_6\r
+\r
+       jmp     .L2_20_2\r
+       ALIGN_4\r
+\r
+.L2_20_6:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_20_9\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_20_7:\r
+\r
+       KERNEL8x2_SUB\r
+\r
+       jl      .L2_20_7\r
+       ALIGN_4\r
+\r
+\r
+.L2_20_9:\r
+\r
+       SAVE8x2\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $8, KK\r
+#endif\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       ALIGN_4\r
+       \r
+\r
+\r
+/**************************************************************************/\r
+\r
+.L2_21pre:\r
+\r
+       testq   $4, M           \r
+       jz      .L2_30\r
+       ALIGN_4\r
+\r
+.L2_21:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $4, %rax        // number of values in A\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L2_26\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_22:\r
+\r
+\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       je      .L2_26\r
+\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+       KERNEL4x2_SUB\r
+\r
+       je      .L2_26\r
+\r
+       jmp     .L2_22\r
+       ALIGN_4\r
+\r
+.L2_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_27:\r
+\r
+       KERNEL4x2_SUB\r
+\r
+       jl      .L2_27\r
+       ALIGN_4\r
+\r
+\r
+.L2_29:\r
+\r
+       SAVE4x2\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L2_30:\r
+       testq   $2, M           \r
+       jz      .L2_40\r
+\r
+       ALIGN_4\r
+\r
+.L2_31:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $2, %rax        // number of values in AO\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L2_36\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_32:\r
+\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       je      .L2_36\r
+\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+       KERNEL2x2_SUB\r
+\r
+       je      .L2_36\r
+\r
+       jmp     .L2_32\r
+       ALIGN_4\r
+\r
+.L2_36:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_39\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+       \r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_37:\r
+\r
+       KERNEL2x2_SUB\r
+\r
+       jl      .L2_37\r
+       ALIGN_4\r
+\r
+\r
+.L2_39:\r
+\r
+       SAVE2x2\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $2, KK\r
+#endif\r
+\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       ALIGN_4\r
+\r
+.L2_40:\r
+       testq   $1, M           \r
+       jz      .L2_60          // to next 2 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L2_41:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $1, %rax        // number of values in AO\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax\r
+       je      .L2_46\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_42:\r
+\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       je      .L2_46\r
+\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+       KERNEL1x2_SUB\r
+\r
+       je      .L2_46\r
+\r
+       jmp     .L2_42\r
+       ALIGN_4\r
+\r
+.L2_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_47:\r
+\r
+       KERNEL1x2_SUB\r
+\r
+       jl      .L2_47\r
+       ALIGN_4\r
+\r
+\r
+.L2_49:\r
+\r
+       SAVE1x2\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, SIZE), BO         \r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $1, KK\r
+#endif\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       ALIGN_4\r
+       \r
+\r
+\r
+\r
+       \r
+.L2_60:\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+        addq    $2, KK\r
+#endif\r
+\r
+\r
+\r
+\r
+.L1_0:\r
+\r
+/************************************************************************************************\r
+* Loop for Nmod6 % 2 > 0\r
+*************************************************************************************************/\r
+\r
+       movq    Nmod6, J                \r
+       andq    $1, J                   // j % 2\r
+       je      .L999\r
+       ALIGN_4\r
+\r
+.L1_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       ALIGN_4\r
+\r
+.L1_02b:\r
+\r
+       vmovss  (BO1), %xmm0\r
+       vmovss  %xmm0,       (BO)\r
+       addq    $1*SIZE,BO1\r
+       addq    $1*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L1_02b\r
+\r
+.L1_02c:\r
+\r
+       movq    BO1, B                  // next offset of B\r
+\r
+.L1_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 1), C          // c += 1 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+       \r
+       movq    A, AO                   // aoffset = a\r
+       addq    $16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $4, I                   // i = (m >> 4)\r
+       je      .L1_20\r
+\r
+       ALIGN_4\r
+\r
+.L1_11:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $16, %rax      // number of values in AO\r
+#else\r
+        addq    $1, %rax       // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L1_16\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_12:\r
+\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+\r
+       je      .L1_16\r
+\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+       KERNEL16x1_SUB\r
+\r
+       je      .L1_16\r
+\r
+       jmp     .L1_12\r
+       ALIGN_4\r
+\r
+.L1_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_17:\r
+\r
+       KERNEL16x1_SUB\r
+\r
+       jl      .L1_17\r
+       ALIGN_4\r
+\r
+\r
+.L1_19:\r
+\r
+       SAVE16x1\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $4, %rax                        // rax = rax * 16 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $16, KK                                \r
+#endif\r
+\r
+       addq    $16 * SIZE, CO1         # coffset += 16\r
+       decq    I                       # i --\r
+       jg      .L1_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L1_20:\r
+       // Test rest of M\r
+\r
+       testq   $15, M\r
+       jz      .L999\r
+\r
+       testq   $8, M           \r
+       jz      .L1_21pre\r
+       ALIGN_4\r
+\r
+/**************************************************************************/\r
+\r
+.L1_20_1:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $8, %rax        // number of values in A\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L1_20_6\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_20_2:\r
+\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+\r
+       je      .L1_20_6\r
+\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+       KERNEL8x1_SUB\r
+\r
+       je      .L1_20_6\r
+\r
+       jmp     .L1_20_2\r
+       ALIGN_4\r
+\r
+.L1_20_6:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_20_9\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_20_7:\r
+\r
+       KERNEL8x1_SUB\r
+\r
+       jl      .L1_20_7\r
+       ALIGN_4\r
+\r
+\r
+.L1_20_9:\r
+\r
+       SAVE8x1\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $8, KK\r
+#endif\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       ALIGN_4\r
+       \r
+\r
+\r
+/**************************************************************************/\r
+\r
+.L1_21pre:\r
+\r
+       testq   $4, M           \r
+       jz      .L1_30\r
+       ALIGN_4\r
+\r
+.L1_21:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $4, %rax        // number of values in A\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L1_26\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_22:\r
+\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       je      .L1_26\r
+\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+       KERNEL4x1_SUB\r
+\r
+       je      .L1_26\r
+\r
+       jmp     .L1_22\r
+       ALIGN_4\r
+\r
+.L1_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_27:\r
+\r
+       KERNEL4x1_SUB\r
+\r
+       jl      .L1_27\r
+       ALIGN_4\r
+\r
+\r
+.L1_29:\r
+\r
+       SAVE4x1\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L1_30:\r
+       testq   $2, M           \r
+       jz      .L1_40\r
+\r
+       ALIGN_4\r
+\r
+.L1_31:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $2, %rax        // number of values in AO\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L1_36\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_32:\r
+\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
 \r
-       vbroadcastss    ALPHA, %ymm0\r
+       je      .L1_36\r
 \r
-       vmulps  %ymm0 , %ymm4 , %ymm4\r
-       vmulps  %ymm0 , %ymm5 , %ymm5\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
 \r
-#if !defined(TRMMKERNEL)\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
+       KERNEL2x1_SUB\r
 \r
-       vaddps          (CO1), %ymm4,%ymm4\r
-       vaddps  8 * SIZE(CO1), %ymm5,%ymm5\r
+       je      .L1_36\r
+\r
+       jmp     .L1_32\r
+       ALIGN_4\r
 \r
+.L1_36:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
 #endif\r
 \r
-       vmovups %ymm4 ,         (CO1)\r
-       vmovups %ymm5 , 8 * SIZE(CO1)\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_39\r
 \r
-.endm\r
+       movq    %rax, BI                        //  Index for BO\r
+       \r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
 \r
+.L1_37:\r
 \r
-/*******************************************************************************************/\r
+       KERNEL2x1_SUB\r
 \r
-.macro KERNEL8x1_SUB\r
-       vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0\r
-       vbroadcastss     -4 * SIZE(BO, BI, SIZE), %ymm2\r
-       VFMADD231PS_(   %ymm4,%ymm2,%ymm0 )\r
-       addq    $ 1 , BI        \r
-       addq    $ 8 , %rax \r
-.endm\r
+       jl      .L1_37\r
+       ALIGN_4\r
 \r
-.macro SAVE8x1\r
 \r
-       vbroadcastss    ALPHA, %ymm0\r
+.L1_39:\r
 \r
-       vmulps  %ymm0 , %ymm4 , %ymm4\r
+       SAVE2x1\r
 \r
-#if !defined(TRMMKERNEL)\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO         \r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
 \r
-       vaddps          (CO1), %ymm4,%ymm4\r
 \r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $2, KK\r
 #endif\r
 \r
-       vmovups %ymm4 ,         (CO1)\r
-\r
-.endm\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       ALIGN_4\r
 \r
+.L1_40:\r
+       testq   $1, M           \r
+       jz      .L999\r
 \r
+       ALIGN_4\r
 \r
-/*******************************************************************************************/\r
+.L1_41:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO\r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif\r
 \r
-.macro KERNEL4x1_SUB\r
-       vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0\r
-       vbroadcastss     -4 * SIZE(BO, BI, SIZE), %xmm2\r
-       VFMADD231PS_(   %xmm4,%xmm2,%xmm0 )\r
-       addq    $ 1 , BI        \r
-       addq    $ 4 , %rax \r
-.endm\r
 \r
-.macro SAVE4x1\r
+       vzeroall\r
 \r
-       vbroadcastss    ALPHA, %xmm0\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $1, %rax        // number of values in AO\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
 \r
-       vmulps  %xmm0 , %xmm4 , %xmm4\r
+       andq    $-8, %rax\r
+       je      .L1_46\r
+       movq    %rax, BI                        //  Index for BO\r
 \r
-#if !defined(TRMMKERNEL)\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
 \r
-       vaddps          (CO1), %xmm4,%xmm4\r
+.L1_42:\r
 \r
-#endif\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
 \r
-       vmovups %xmm4 ,         (CO1)\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
 \r
-.endm\r
+       je      .L1_46\r
 \r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
 \r
-/*******************************************************************************************/\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
+       KERNEL1x1_SUB\r
 \r
-.macro KERNEL2x1_SUB\r
-       vmovss  -16 * SIZE(AO, %rax, SIZE), %xmm0\r
-       vmovss  -15 * SIZE(AO, %rax, SIZE), %xmm1\r
-       vmovss   -4 * SIZE(BO, BI, SIZE), %xmm2\r
-       VFMADD231SS_(   %xmm4,%xmm2,%xmm0 )\r
-       VFMADD231SS_(   %xmm5,%xmm2,%xmm1 )\r
-       addq    $ 1 , BI        \r
-       addq    $ 2 , %rax \r
-.endm\r
+       je      .L1_46\r
 \r
-.macro SAVE2x1\r
+       jmp     .L1_42\r
+       ALIGN_4\r
 \r
-       vmovss  ALPHA, %xmm0\r
+.L1_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
 \r
-       vmulss  %xmm0 , %xmm4 , %xmm4\r
-       vmulss  %xmm0 , %xmm5 , %xmm5\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_49\r
 \r
-#if !defined(TRMMKERNEL)\r
+       movq    %rax, BI                        //  Index for BO\r
 \r
-       vaddps          (CO1), %xmm4,%xmm4\r
-       vaddps  1 * SIZE(CO1), %xmm5,%xmm5\r
+       leaq    (AO, %rax, SIZE), AO\r
+       leaq    (BO, BI, SIZE), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
 \r
-#endif\r
+.L1_47:\r
 \r
-       vmovss  %xmm4 ,         (CO1)\r
-       vmovss  %xmm5 , 1 * SIZE(CO1)\r
+       KERNEL1x1_SUB\r
 \r
-.endm\r
+       jl      .L1_47\r
+       ALIGN_4\r
 \r
 \r
-/*******************************************************************************************/\r
+.L1_49:\r
 \r
-.macro KERNEL1x1_SUB\r
-       vmovss  -16 * SIZE(AO, %rax, SIZE), %xmm0\r
-       vmovss   -4 * SIZE(BO, BI, SIZE), %xmm2\r
-       VFMADD231SS_(   %xmm4,%xmm2,%xmm0 )\r
-       addq    $ 1 , BI        \r
-       addq    $ 1 , %rax \r
-.endm\r
+       SAVE1x1\r
 \r
-.macro SAVE1x1\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, SIZE), BO         \r
+        leaq    (AO, %rax, SIZE), AO\r
+#endif  \r
 \r
-       vmovss  ALPHA, %xmm0\r
 \r
-       vmulss  %xmm0 , %xmm4 , %xmm4\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $1, KK\r
+#endif\r
 \r
-#if !defined(TRMMKERNEL)\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       ALIGN_4\r
+       \r
 \r
-       vaddps          (CO1), %xmm4,%xmm4\r
+.L999:\r
+       movq            SP, %rsp\r
+       movq       (%rsp), %rbx\r
+       movq      8(%rsp), %rbp\r
+       movq     16(%rsp), %r12\r
+       movq     24(%rsp), %r13\r
+       movq     32(%rsp), %r14\r
+       movq     40(%rsp), %r15\r
 \r
+#ifdef WINDOWS_ABI\r
+       movq     48(%rsp), %rdi\r
+       movq     56(%rsp), %rsi\r
+       movups   64(%rsp), %xmm6\r
+       movups   80(%rsp), %xmm7\r
+       movups   96(%rsp), %xmm8\r
+       movups  112(%rsp), %xmm9\r
+       movups  128(%rsp), %xmm10\r
+       movups  144(%rsp), %xmm11\r
+       movups  160(%rsp), %xmm12\r
+       movups  176(%rsp), %xmm13\r
+       movups  192(%rsp), %xmm14\r
+       movups  208(%rsp), %xmm15\r
 #endif\r
 \r
-       vmovss  %xmm4 ,         (CO1)\r
+       addq    $STACKSIZE, %rsp\r
+       ret\r
 \r
-.endm\r
+       EPILOGUE\r
 \r
 \r
-/*******************************************************************************************/\r
+\r
+#else\r
 \r
 /*************************************************************************************\r
 * TRMM Kernel\r
@@ -3152,6 +6802,5 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        EPILOGUE\r
 \r
 \r
-\r
-\r
+#endif\r
 \r

diff --git a/param.h b/param.h
index 863e83c32d40692bde580805251996579a1fd6cf..c545d21a87a931fbc3207e0902166c4650e0598f 100644 (file)
--- a/param.h
+++ b/param.h
@@ -1237,10 +1237,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CGEMM_DEFAULT_P 384
 #define ZGEMM_DEFAULT_P 256
 
-#define SGEMM_DEFAULT_Q 384
 #ifdef WINDOWS_ABI
+#define SGEMM_DEFAULT_Q 320
 #define DGEMM_DEFAULT_Q 128
 #else
+#define SGEMM_DEFAULT_Q 384
 #define DGEMM_DEFAULT_Q 256
 #endif
 #define CGEMM_DEFAULT_Q 192