New dgemm kernel for BULLDOZER: dgemm_kernel_8x2_bulldozer.S

diff --git a/driver/level3/level3.c b/driver/level3/level3.c
index 20e811c..27e503b 100644 (file)
--- a/driver/level3/level3.c
+++ b/driver/level3/level3.c
@@ -332,7 +332,20 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 #else
       for(jjs = js; jjs < js + min_j; jjs += min_jj){
        min_jj = min_j + js - jjs;
-       if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
+
+#if defined(BULLDOZER) && defined(ARCH_X86_64) && defined(DOUBLE) && !defined(COMPLEX)
+        if (min_jj >= 12*GEMM_UNROLL_N) min_jj = 12*GEMM_UNROLL_N;
+        else
+                if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+                else
+                        if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
+                        else
+                                if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
+#else
+
+        if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
+#endif
+
        
        START_RPCC();
        

diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c
index 000d423..87a3289 100644 (file)
--- a/driver/level3/level3_thread.c
+++ b/driver/level3/level3_thread.c
@@ -360,8 +360,20 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 
       for(jjs = xxx; jjs < MIN(n_to, xxx + div_n); jjs += min_jj){
        min_jj = MIN(n_to, xxx + div_n) - jjs;
+
+#if defined(BULLDOZER) && defined(ARCH_X86_64) && defined(DOUBLE) && !defined(COMPLEX)
+       if (min_jj >= 12*GEMM_UNROLL_N) min_jj = 12*GEMM_UNROLL_N;
+       else
+               if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+               else
+                       if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
+                       else
+                               if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
+#else
+
        if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-       
+#endif 
+
        START_RPCC();
        
        OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, 
@@ -634,7 +646,7 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
       
       num_cpu_n ++;
     }
-    
+ 
     for (j = 0; j < num_cpu_m; j++) {
       for (i = 0; i < num_cpu_m; i++) {
        for (k = 0; k < DIVIDE_RATE; k++) {

diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER
index 2ac035f..70ae51f 100644 (file)
--- a/kernel/x86_64/KERNEL.BULLDOZER
+++ b/kernel/x86_64/KERNEL.BULLDOZER
@@ -10,13 +10,13 @@ SGEMMINCOPYOBJ =  sgemm_incopy$(TSUFFIX).$(SUFFIX)
 SGEMMITCOPYOBJ =  sgemm_itcopy$(TSUFFIX).$(SUFFIX) 
 SGEMMONCOPYOBJ =  sgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
-DGEMMKERNEL    =  dgemm_kernel_4x4_bulldozer.S
-DGEMMINCOPY    =
-DGEMMITCOPY    =
-DGEMMONCOPY    =  ../generic/gemm_ncopy_4.c
-DGEMMOTCOPY    =  ../generic/gemm_tcopy_4.c
-DGEMMINCOPYOBJ =
-DGEMMITCOPYOBJ =
+DGEMMKERNEL    =  dgemm_kernel_8x2_bulldozer.S
+DGEMMINCOPY    =  ../generic/gemm_ncopy_8.c
+DGEMMITCOPY    =  ../generic/gemm_tcopy_8.c
+DGEMMONCOPY    =  ../generic/gemm_ncopy_2.c
+DGEMMOTCOPY    =  ../generic/gemm_tcopy_2.c
+DGEMMINCOPYOBJ =  dgemm_incopy$(TSUFFIX).$(SUFFIX)
+DGEMMITCOPYOBJ =  dgemm_itcopy$(TSUFFIX).$(SUFFIX)
 DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)
 DGEMMOTCOPYOBJ =  dgemm_otcopy$(TSUFFIX).$(SUFFIX)
 CGEMMKERNEL    =  zgemm_kernel_4x2_barcelona.S
@@ -38,25 +38,27 @@ ZGEMMITCOPYOBJ =
 ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
 ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
-STRSMKERNEL_LN =  trsm_kernel_LN_8x4_sse.S
-STRSMKERNEL_LT =  trsm_kernel_LT_8x4_sse.S
-STRSMKERNEL_RN =  trsm_kernel_LT_8x4_sse.S
-STRSMKERNEL_RT =  trsm_kernel_RT_8x4_sse.S
+CGEMM3MKERNEL    =  zgemm3m_kernel_8x4_barcelona.S
+ZGEMM3MKERNEL    =  zgemm3m_kernel_4x4_barcelona.S
 
-DTRSMKERNEL_LN =  trsm_kernel_LN_4x4_barcelona.S
-DTRSMKERNEL_LT =  trsm_kernel_LT_4x4_barcelona.S
-DTRSMKERNEL_RN =  trsm_kernel_LT_4x4_barcelona.S
-DTRSMKERNEL_RT =  trsm_kernel_RT_4x4_barcelona.S
+STRSMKERNEL_LN  =  ../generic/trsm_kernel_LN.c
+STRSMKERNEL_LT  =  ../generic/trsm_kernel_LT.c
+STRSMKERNEL_RN  =  ../generic/trsm_kernel_RN.c
+STRSMKERNEL_RT  =  ../generic/trsm_kernel_RT.c
 
-CTRSMKERNEL_LN =  ztrsm_kernel_LN_4x2_sse.S
-CTRSMKERNEL_LT =  ztrsm_kernel_LT_4x2_sse.S
-CTRSMKERNEL_RN =  ztrsm_kernel_LT_4x2_sse.S
-CTRSMKERNEL_RT =  ztrsm_kernel_RT_4x2_sse.S
+DTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
+DTRSMKERNEL_LT  = ../generic/trsm_kernel_LT.c
+DTRSMKERNEL_RN  = ../generic/trsm_kernel_RN.c
+DTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
+
+CTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
+CTRSMKERNEL_LT  = ../generic/trsm_kernel_LT.c
+CTRSMKERNEL_RN  = ../generic/trsm_kernel_RN.c
+CTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
+
+ZTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
+ZTRSMKERNEL_LT  = ../generic/trsm_kernel_LT.c
+ZTRSMKERNEL_RN  = ../generic/trsm_kernel_RN.c
+ZTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
 
-ZTRSMKERNEL_LN =  ztrsm_kernel_LN_2x2_sse2.S
-ZTRSMKERNEL_LT =  ztrsm_kernel_LT_2x2_sse2.S
-ZTRSMKERNEL_RN =  ztrsm_kernel_LT_2x2_sse2.S
-ZTRSMKERNEL_RT =  ztrsm_kernel_RT_2x2_sse2.S
 
-CGEMM3MKERNEL    =  zgemm3m_kernel_8x4_barcelona.S
-ZGEMM3MKERNEL    =  zgemm3m_kernel_4x4_barcelona.S

diff --git a/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S b/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S
deleted file mode 100644 (file)
index 91cd492..0000000
--- a/kernel/x86_64/dgemm_kernel_4x4_bulldozer.S
+++ /dev/null
@@ -1,1959 +0,0 @@
-/*********************************************************************/\r
-/* Copyright 2009, 2010 The University of Texas at Austin.           */\r
-/* All rights reserved.                                              */\r
-/*                                                                   */\r
-/* Redistribution and use in source and binary forms, with or        */\r
-/* without modification, are permitted provided that the following   */\r
-/* conditions are met:                                               */\r
-/*                                                                   */\r
-/*   1. Redistributions of source code must retain the above         */\r
-/*      copyright notice, this list of conditions and the following  */\r
-/*      disclaimer.                                                  */\r
-/*                                                                   */\r
-/*   2. Redistributions in binary form must reproduce the above      */\r
-/*      copyright notice, this list of conditions and the following  */\r
-/*      disclaimer in the documentation and/or other materials       */\r
-/*      provided with the distribution.                              */\r
-/*                                                                   */\r
-/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */\r
-/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */\r
-/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */\r
-/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */\r
-/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */\r
-/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */\r
-/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */\r
-/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */\r
-/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */\r
-/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */\r
-/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */\r
-/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */\r
-/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */\r
-/*    POSSIBILITY OF SUCH DAMAGE.                                    */\r
-/*                                                                   */\r
-/* The views and conclusions contained in the software and           */\r
-/* documentation are those of the authors and should not be          */\r
-/* interpreted as representing official policies, either expressed   */\r
-/* or implied, of The University of Texas at Austin.                 */\r
-/*********************************************************************/\r
-\r
-/*********************************************************************\r
-* Changelog:\r
-*\r
-* 2013/04/15 Saar\r
-* Prefetch for A and B\r
-* unroll of inner Loop\r
-* using generic versions for ncopy and tcopy\r
-* moved vmovddup        ALPHA, %xmm7 down\r
-* define A_PR1 192\r
-* define B_PR1 512\r
-*\r
-* 2013/04/27 Saar\r
-* define A_PR1 224\r
-* define B_PR1 224\r
-* created 2 different Kernels\r
-**********************************************************************/\r
-\r
-/*********************************************************************\r
-* 2013/04/12   Saar\r
-* Performance:\r
-* 3584x3584    89 GFLOPS with 8 threads on 4 modules\r
-*              76 GFLOPS with 4 threads on 4 modules\r
-*              53 GFLOPS with 4 threads on 2 modules\r
-*              46 GFLOPS with 2 threads on 2 modules\r
-*              28 GFLOPS with 2 threads on 1 module\r
-*            23,1 GFLOPS with 1 thread  on 1 module\r
-*********************************************************************/\r
-\r
-#define ASSEMBLER\r
-#include "common.h"\r
- \r
-#define OLD_M  %rdi\r
-#define OLD_N  %rsi\r
-#define M      %r13\r
-#define N      %r14\r
-#define K      %rdx\r
-\r
-#define A      %rcx\r
-#define B      %r8\r
-#define C      %r9\r
-#define LDC    %r10\r
-       \r
-#define I      %r11\r
-#define AO     %rdi\r
-#define BO     %rsi\r
-#define        CO1     %r15\r
-#define CO2    %r12\r
-#define BB     %rbp\r
-#define        J       %rbx\r
-\r
-#ifndef WINDOWS_ABI\r
-\r
-#define STACKSIZE 96\r
-\r
-#define ALPHA   48(%rsp)\r
-#define OFFSET  56(%rsp)\r
-#define KK      64(%rsp)\r
-#define KKK     72(%rsp)\r
-\r
-#else\r
-\r
-#define STACKSIZE 256\r
-\r
-#define OLD_A          40 + STACKSIZE(%rsp)\r
-#define OLD_B          48 + STACKSIZE(%rsp)\r
-#define OLD_C          56 + STACKSIZE(%rsp)\r
-#define OLD_LDC                64 + STACKSIZE(%rsp)\r
-#define OLD_OFFSET     72 + STACKSIZE(%rsp)\r
-\r
-#define ALPHA  224(%rsp)\r
-#define OFFSET 232(%rsp)\r
-#define KK     240(%rsp)\r
-#define KKK    248(%rsp)\r
-\r
-#endif\r
-\r
-#define movapd movaps\r
-#define movupd movups\r
-\r
-#define A_PR1 224\r
-#define B_PR1 224\r
-\r
-\r
-#if  defined(OPTBYMODULE) || !defined(SMP)\r
-\r
-#define KERNEL1(xx) \\r
-       vmovups         -14 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-       vfmaddpd                %xmm8,%xmm6,%xmm7,%xmm8 ;\\r
-       vmovddup        -14 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm12,%xmm2,%xmm7,%xmm12 ;\\r
-       vmovddup        -13 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm9,%xmm6,%xmm3,%xmm9 ;\\r
-       vmovddup        -12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vfmaddpd                %xmm13,%xmm2,%xmm3,%xmm13 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup        -11 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-       vmovups         -12 * SIZE(AO, %rax, 4),%xmm0 ;\\r
-       vfmaddpd                %xmm10,%xmm6,%xmm5,%xmm10 ;\\r
-       vfmaddpd                %xmm11,%xmm6,%xmm4,%xmm11 ;\\r
-\r
-#define KERNEL2(xx) \\r
-       vmovups         -10 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-       vfmaddpd                %xmm8, %xmm0, %xmm1,%xmm8 ;\\r
-       vmovups          -8 * SIZE(AO, %rax, 4),%xmm6 ;\\r
-       vfmaddpd                %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-       vmovddup        -10 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm9, %xmm0, %xmm3,%xmm9 ;\\r
-       vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         -9 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-       vmovddup         -8 * SIZE(BO, %rax, 4), %xmm7 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup         -7 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL3(xx) \\r
-       vmovups          -6 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-       vfmaddpd                %xmm8, %xmm6, %xmm7, %xmm8 ;\\r
-       vmovddup         -6 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
-       vmovddup         -5 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm9, %xmm6, %xmm3,%xmm9 ;\\r
-       vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         -4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup         -3 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-       vmovups          -4 * SIZE(AO, %rax, 4),%xmm0 ;\\r
-       vfmaddpd                %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
-       vfmaddpd                %xmm11,%xmm6, %xmm4, %xmm11 ;\\r
-\r
-#define KERNEL4(xx) \\r
-       vmovups          -2 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-       vfmaddpd                %xmm8,%xmm0, %xmm1,%xmm8 ;\\r
-       vmovups                   (AO, %rax, 4), %xmm6 ;\\r
-       vfmaddpd                %xmm12,%xmm2, %xmm1 ,%xmm12;\\r
-       vmovddup         -2 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm9,%xmm0, %xmm3,%xmm9 ;\\r
-       vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         -1 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-       vmovddup                  (BO, %rax, 4), %xmm7 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup          1 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL5(xx) \\r
-       vmovups           2 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-       vfmaddpd                %xmm8,%xmm6, %xmm7,%xmm8 ;\\r
-       vmovddup          2 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
-       vmovddup          3 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm9,%xmm6, %xmm3,%xmm9 ;\\r
-       vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup          4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup          5 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-       vmovups           4 * SIZE(AO, %rax, 4),%xmm0 ;\\r
-       vfmaddpd                %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
-       vfmaddpd                %xmm11,%xmm6, %xmm4,%xmm11 ;\\r
-\r
-#define KERNEL6(xx) \\r
-       vmovups           6 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-       vfmaddpd                %xmm8,%xmm0, %xmm1,%xmm8 ;\\r
-       vmovups           8 * SIZE(AO, %rax, 4), %xmm6 ;\\r
-       vfmaddpd                %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-       vmovddup          6 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm9,%xmm0, %xmm3,%xmm9 ;\\r
-       vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup          7 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-       vmovddup          8 * SIZE(BO, %rax, 4), %xmm7 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup          9 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL7(xx) \\r
-       vmovups          10 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-       vfmaddpd                %xmm8,%xmm6, %xmm7,%xmm8 ;\\r
-       vmovddup         10 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
-       vmovddup         11 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm9,%xmm6, %xmm3,%xmm9 ;\\r
-       vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup         13 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-       vmovups          12 * SIZE(AO, %rax, 4), %xmm0 ;\\r
-       vfmaddpd                %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
-       vfmaddpd                %xmm11,%xmm6, %xmm4,%xmm11 ;\\r
-\r
-#define KERNEL8(xx) \\r
-       vmovups          14 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-       vfmaddpd                %xmm8,%xmm0, %xmm1,%xmm8 ;\\r
-       vmovups          16 * SIZE(AO, %rax, 4),%xmm6 ;\\r
-       vfmaddpd                %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-       vmovddup         14 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-       vfmaddpd                %xmm9,%xmm0, %xmm3,%xmm9 ;\\r
-       vfmaddpd                %xmm13, %xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         15 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-       vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-       vmovddup         16 * SIZE(BO, %rax, 4), %xmm7 ;\\r
-       vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-       vmovddup         17 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-       vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-       vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#else\r
-\r
-#define KERNEL1(xx) \\r
-        vmovups         -14 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-        vfmaddpd                %xmm8,%xmm6,%xmm7,%xmm8 ;\\r
-        vmovddup        -14 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm12,%xmm2,%xmm7,%xmm12 ;\\r
-        vmovddup        -13 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm9,%xmm6,%xmm3,%xmm9 ;\\r
-        vmovddup        -12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-        vfmaddpd                %xmm13,%xmm2,%xmm3,%xmm13 ;\\r
-        vmovddup         -8 * SIZE(BO, %rax, 4), %xmm7 ;\\r
-        vfmaddpd                %xmm10,%xmm6,%xmm5,%xmm10 ;\\r
-        vmovddup        -11 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm6,%xmm4,%xmm11 ;\\r
-        vmovups         -12 * SIZE(AO, %rax, 4),%xmm0 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-        vmovups          -8 * SIZE(AO, %rax, 4),%xmm6 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-\r
-#define KERNEL2(xx) \\r
-        vmovups         -10 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-        vfmaddpd                %xmm8, %xmm0, %xmm1,%xmm8 ;\\r
-        vfmaddpd                %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-        vmovddup        -10 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm9, %xmm0, %xmm3,%xmm9 ;\\r
-        vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup         -9 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-        vmovddup         -7 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL3(xx) \\r
-        vmovups          -6 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-        vfmaddpd                %xmm8, %xmm6, %xmm7, %xmm8 ;\\r
-        vmovddup         -6 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
-        vmovddup         -5 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm9, %xmm6, %xmm3,%xmm9 ;\\r
-        vmovddup                  (BO, %rax, 4), %xmm7 ;\\r
-        vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup         -4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-        vfmaddpd                %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
-        vmovddup         -3 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm6, %xmm4, %xmm11 ;\\r
-        vmovups          -4 * SIZE(AO, %rax, 4),%xmm0 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-        vmovups                   (AO, %rax, 4), %xmm6 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL4(xx) \\r
-        vmovups          -2 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-        vfmaddpd                %xmm8,%xmm0, %xmm1,%xmm8 ;\\r
-        vfmaddpd                %xmm12,%xmm2, %xmm1 ,%xmm12;\\r
-        vmovddup         -2 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm9,%xmm0, %xmm3,%xmm9 ;\\r
-        vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup         -1 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-        vmovddup          1 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL5(xx) \\r
-        vmovups           2 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-        vfmaddpd                %xmm8,%xmm6, %xmm7,%xmm8 ;\\r
-        vmovddup          2 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
-        vmovddup          3 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm9,%xmm6, %xmm3,%xmm9 ;\\r
-        vmovddup          8 * SIZE(BO, %rax, 4), %xmm7 ;\\r
-        vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup          4 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-        vfmaddpd                %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
-        vmovddup          5 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm6, %xmm4,%xmm11 ;\\r
-        vmovups           4 * SIZE(AO, %rax, 4),%xmm0 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-        vmovups           8 * SIZE(AO, %rax, 4), %xmm6 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL6(xx) \\r
-        vmovups           6 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-        vfmaddpd                %xmm8,%xmm0, %xmm1,%xmm8 ;\\r
-        vfmaddpd                %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-        vmovddup          6 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm9,%xmm0, %xmm3,%xmm9 ;\\r
-        vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup          7 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-        vmovddup          9 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL7(xx) \\r
-        vmovups          10 * SIZE(AO, %rax, 4),%xmm2 ;\\r
-        vfmaddpd                %xmm8,%xmm6, %xmm7,%xmm8 ;\\r
-        vmovddup         10 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm12,%xmm2, %xmm7,%xmm12 ;\\r
-        vmovddup         11 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm9,%xmm6, %xmm3,%xmm9 ;\\r
-        vmovddup         16 * SIZE(BO, %rax, 4), %xmm7 ;\\r
-        vfmaddpd                %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup         12 * SIZE(BO, %rax, 4), %xmm1 ;\\r
-        vfmaddpd                %xmm10,%xmm6, %xmm5,%xmm10 ;\\r
-        vmovddup         13 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm6, %xmm4,%xmm11 ;\\r
-        vmovups          12 * SIZE(AO, %rax, 4), %xmm0 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-        vmovups          16 * SIZE(AO, %rax, 4),%xmm6 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#define KERNEL8(xx) \\r
-        vmovups          14 * SIZE(AO, %rax, 4), %xmm2 ;\\r
-        vfmaddpd                %xmm8,%xmm0, %xmm1,%xmm8 ;\\r
-        vfmaddpd                %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-        vmovddup         14 * SIZE(BO, %rax, 4), %xmm5 ;\\r
-        vfmaddpd                %xmm9,%xmm0, %xmm3,%xmm9 ;\\r
-        vfmaddpd                %xmm13, %xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup         15 * SIZE(BO, %rax, 4), %xmm4 ;\\r
-        vfmaddpd                %xmm10,%xmm0, %xmm5,%xmm10 ;\\r
-        vfmaddpd                %xmm14,%xmm2, %xmm5,%xmm14 ;\\r
-        vmovddup         17 * SIZE(BO, %rax, 4), %xmm3 ;\\r
-        vfmaddpd                %xmm11,%xmm0, %xmm4,%xmm11 ;\\r
-        vfmaddpd                %xmm15,%xmm2, %xmm4,%xmm15 ;\\r
-\r
-#endif\r
-\r
-#define KERNEL_SUB1(xx) \\r
-       vmovups         -16 * SIZE(AO),%xmm0 ;\\r
-       vmovups         -14 * SIZE(AO),%xmm2 ;\\r
-        vmovddup       -16 * SIZE(BO), %xmm1 ;\\r
-        vmovddup       -15 * SIZE(BO), %xmm3 ;\\r
-        vfmaddpd   %xmm8, %xmm0, %xmm1,%xmm8 ;\\r
-        vfmaddpd   %xmm9, %xmm0, %xmm3,%xmm9 ;\\r
-        vfmaddpd   %xmm12, %xmm2, %xmm1,%xmm12 ;\\r
-        vfmaddpd   %xmm13, %xmm2, %xmm3,%xmm13 ;\\r
-        vmovddup       -14 * SIZE(BO), %xmm1 ;\\r
-        vmovddup       -13 * SIZE(BO), %xmm3 ;\\r
-        vfmaddpd   %xmm10, %xmm0, %xmm1,%xmm10 ;\\r
-        vfmaddpd   %xmm11, %xmm0, %xmm3,%xmm11 ;\\r
-        vfmaddpd   %xmm14, %xmm2, %xmm1,%xmm14 ;\\r
-        vfmaddpd   %xmm15, %xmm2, %xmm3,%xmm15 ;\\r
-\r
-\r
-#define KERNEL_SUB2(xx) \\r
-        vmovups        -12 * SIZE(AO), %xmm0 ;\\r
-       vmovups         -10 * SIZE(AO), %xmm2 ;\\r
-        vmovddup       -12 * SIZE(BO), %xmm1 ;\\r
-        vmovddup       -11 * SIZE(BO), %xmm3 ;\\r
-       vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\\r
-       vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\\r
-       vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-       vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup        -10 * SIZE(BO), %xmm1 ;\\r
-       vmovddup         -9 * SIZE(BO), %xmm3 ;\\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 ;\\r
-       vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 ;\\r
-       vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14 ;\\r
-       vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15 ;\\r
-\r
-#define KERNEL_SUB3(xx) \\r
-       vmovups          -8 * SIZE(AO),%xmm0 ;\\r
-       vmovups          -6 * SIZE(AO),%xmm2 ;\\r
-       vmovddup         -8 * SIZE(BO), %xmm1 ;\\r
-       vmovddup         -7 * SIZE(BO), %xmm3 ;\\r
-       vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\\r
-       vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\\r
-       vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-       vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         -6 * SIZE(BO), %xmm1 ;\\r
-       vmovddup         -5 * SIZE(BO), %xmm3 ;\\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 ;\\r
-       vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 ;\\r
-       vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14 ;\\r
-       vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15 ;\\r
-\r
-#define KERNEL_SUB4(xx) \\r
-       vmovups          -4 * SIZE(AO), %xmm0 ;\\r
-       vmovups          -2 * SIZE(AO), %xmm2 ;\\r
-       vmovddup         -4 * SIZE(BO), %xmm1 ;\\r
-       vmovddup         -3 * SIZE(BO), %xmm3 ;\\r
-       vfmaddpd %xmm8, %xmm0, %xmm1,%xmm8 ;\\r
-       vfmaddpd %xmm9, %xmm0, %xmm3,%xmm9 ;\\r
-       vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12 ;\\r
-       vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13 ;\\r
-       vmovddup         -2 * SIZE(BO), %xmm1 ;\\r
-       vmovddup         -1 * SIZE(BO), %xmm3 ;\\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10 ;\\r
-       vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11 ;\\r
-       vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14 ;\\r
-       vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15 ;\\r
-       vmovups                   (AO), %xmm0 ;\\r
-       vmovddup                  (BO), %xmm1 ;\\r
-       vmovddup          1 * SIZE(BO), %xmm3 ;\\r
-       vmovaps %xmm0, %xmm2\r
-\r
-       PROLOGUE\r
-       PROFCODE\r
-       \r
-       subq    $STACKSIZE, %rsp\r
-       movq    %rbx,   (%rsp)\r
-       movq    %rbp,  8(%rsp)\r
-       movq    %r12, 16(%rsp)\r
-       movq    %r13, 24(%rsp)\r
-       movq    %r14, 32(%rsp)\r
-       movq    %r15, 40(%rsp)\r
-\r
-       vzeroupper\r
-\r
-#ifdef WINDOWS_ABI\r
-       movq    %rdi,    48(%rsp)\r
-       movq    %rsi,    56(%rsp)\r
-       movups  %xmm6,   64(%rsp)\r
-       movups  %xmm7,   80(%rsp)\r
-       movups  %xmm8,   96(%rsp)\r
-       movups  %xmm9,  112(%rsp)\r
-       movups  %xmm10, 128(%rsp)\r
-       movups  %xmm11, 144(%rsp)\r
-       movups  %xmm12, 160(%rsp)\r
-       movups  %xmm13, 176(%rsp)\r
-       movups  %xmm14, 192(%rsp)\r
-       movups  %xmm15, 208(%rsp)\r
-\r
-       movq    ARG1,      OLD_M\r
-       movq    ARG2,      OLD_N\r
-       movq    ARG3,      K\r
-       movq    OLD_A,     A\r
-       movq    OLD_B,     B\r
-       movq    OLD_C,     C\r
-       movq    OLD_LDC,   LDC\r
-#ifdef TRMMKERNEL\r
-       movsd   OLD_OFFSET, %xmm12\r
-#endif\r
-       vmovaps %xmm3, %xmm0\r
-\r
-#else\r
-       movq    STACKSIZE +  8(%rsp), LDC\r
-#ifdef TRMMKERNEL\r
-       movsd   STACKSIZE + 16(%rsp), %xmm12\r
-#endif\r
-\r
-#endif\r
-\r
-       movq    OLD_M, M\r
-       movq    OLD_N, N\r
-\r
-       subq    $-16 * SIZE, A\r
-       subq    $-16 * SIZE, B\r
-\r
-       vmovsd   %xmm0, ALPHA\r
-\r
-       salq    $BASE_SHIFT, LDC        # LDC << 3 # LDC * 8\r
-\r
-#ifdef TRMMKERNEL\r
-       vmovsd  %xmm12, OFFSET\r
-       vmovsd  %xmm12, KK\r
-#ifndef LEFT\r
-       negq    KK\r
-#endif \r
-#endif\r
-       movq    N,  J\r
-       sarq    $2, J           # j = (n >> 2) # j = n / 4\r
-       jle     .L40\r
-       ALIGN_4\r
-\r
-.L01:\r
-       movq    C, CO1                  # coffset1 = c\r
-       leaq    (C, LDC, 2), CO2        # coffset2 = c + ldc\r
-       \r
-       leaq    (C, LDC, 4), C          # c += 4 * ldc\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       movq    OFFSET, %rax\r
-       movq    %rax, KK\r
-#endif \r
-\r
-       movq    A, AO           # aoffset = a\r
-\r
-       movq    K, %rax\r
-       salq    $BASE_SHIFT + 2, %rax    # k << 5 # K * 32\r
-       leaq     (B, %rax), BB\r
-\r
-       movq    M,  I\r
-       sarq    $2, I   # i = (m >> 2)\r
-       jle     .L20\r
-       ALIGN_4\r
-\r
-       .align 16\r
-.L11:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (B,  %rax, 4), BO\r
-#endif \r
-\r
-       \r
-\r
-       vxorpd  %xmm8, %xmm8,%xmm8\r
-       vxorpd  %xmm9, %xmm9,%xmm9\r
-       vxorpd  %xmm10, %xmm10,%xmm10\r
-       vxorpd  %xmm11, %xmm11,%xmm11\r
-       vxorpd  %xmm12, %xmm12,%xmm12\r
-       vxorpd  %xmm13, %xmm13,%xmm13\r
-       vxorpd  %xmm14, %xmm14,%xmm14\r
-       vxorpd  %xmm15, %xmm15,%xmm15\r
-\r
-       prefetchw       (CO1)\r
-       // prefetchw       (CO1,LDC)\r
-       // prefetchw       (CO2)\r
-       // prefetchw       (CO2,LDC)\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $4, %rax\r
-#else\r
-       addq    $4, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-8, %rax\r
-       salq    $BASE_SHIFT, %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 4), BO\r
-       negq    %rax\r
-       je      .L15\r
-       // ALIGN_4\r
-\r
-       vmovups         -16 * SIZE(AO, %rax, 4),%xmm6 \r
-       vmovddup        -16 * SIZE(BO, %rax, 4), %xmm7 \r
-       vmovddup        -15 * SIZE(BO, %rax, 4), %xmm3 \r
-\r
-       .align 32\r
-\r
-.L12:\r
-\r
-#if  defined(OPTBYMODULE) || !defined(SMP)\r
-\r
-       prefetcht0 A_PR1(AO,%rax,4)\r
-        prefetcht0 B_PR1(BO,%rax,4)\r
-        KERNEL1(16 *  0)\r
-        KERNEL2(16 *  0)\r
-        prefetcht0 A_PR1+64(AO,%rax,4)\r
-        prefetcht0 B_PR1+64(BO,%rax,4)\r
-        KERNEL3(16 *  0)\r
-       KERNEL4(16 *  0)\r
-       prefetcht0 A_PR1+128(AO,%rax,4)\r
-        prefetcht0 B_PR1+128(BO,%rax,4)\r
-        KERNEL5(16 *  0)\r
-        KERNEL6(16 *  0)\r
-        prefetcht0 A_PR1+192(AO,%rax,4)\r
-        prefetcht0 B_PR1+192(BO,%rax,4)\r
-        KERNEL7(16 *  0)\r
-        KERNEL8(16 *  0)\r
-\r
-       addq    $8 * SIZE, %rax \r
-       je      .L15\r
-       jmp     .L12\r
-       .align 16\r
-\r
-#else\r
-        KERNEL1(16 *  0)\r
-        KERNEL2(16 *  0)\r
-        KERNEL3(16 *  0)\r
-        KERNEL4(16 *  0)\r
-        KERNEL5(16 *  0)\r
-        KERNEL6(16 *  0)\r
-        KERNEL7(16 *  0)\r
-        KERNEL8(16 *  0)\r
-\r
-       addq    $8 * SIZE, %rax \r
-       je      .L15\r
-       jmp     .L12\r
-       .align 16\r
-\r
-#endif\r
-\r
-\r
-.L15:\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       testq   $4, %rax\r
-       je .L16\r
-       ALIGN_4\r
-\r
-       KERNEL_SUB1(16 *  0)\r
-       KERNEL_SUB2(16 *  0)\r
-       KERNEL_SUB3(16 *  0)\r
-       KERNEL_SUB4(16 *  0)\r
-\r
-       subq    $-16 * SIZE, BO\r
-       subq    $-16 * SIZE, AO\r
-       ALIGN_4\r
-\r
-.L16:\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       vmovddup        ALPHA, %xmm7\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L19\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 4), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L17:\r
-       vmovups         -16 * SIZE(AO, %rax, 4), %xmm0\r
-       vmovups         -14 * SIZE(AO, %rax, 4), %xmm2\r
-       vmovddup        -16 * SIZE(BO, %rax, 4), %xmm1\r
-       vmovddup        -15 * SIZE(BO, %rax, 4), %xmm3\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9\r
-       vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12\r
-       vfmaddpd %xmm13,%xmm2, %xmm3,%xmm13\r
-       vmovddup        -14 * SIZE(BO, %rax, 4), %xmm1\r
-       vmovddup        -13 * SIZE(BO, %rax, 4), %xmm3\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10\r
-       vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11\r
-       vfmaddpd %xmm14,%xmm2, %xmm1,%xmm14\r
-       vfmaddpd %xmm15,%xmm2, %xmm3,%xmm15\r
-/*\r
-       vmovups         -12 * SIZE(AO, %rax, 4), %xmm0\r
-       vmovddup        -12 * SIZE(BO, %rax, 4), %xmm1\r
-       vmovddup        -11 * SIZE(BO, %rax, 4), %xmm3\r
-       vmovaps %xmm0, %xmm2\r
-*/\r
-       addq    $SIZE, %rax\r
-       jl      .L17\r
-       ALIGN_4\r
-\r
-.L19:\r
-       // prefetch        -8 * SIZE(BB)\r
-       subq             $-16 * SIZE, BB\r
-\r
-#ifndef TRMMKERNEL\r
-\r
-       vfmaddpd (CO1),%xmm7, %xmm8,%xmm8\r
-       vfmaddpd 2 * SIZE(CO1),%xmm7, %xmm12,%xmm12\r
-       vfmaddpd (CO1, LDC),%xmm7, %xmm9,%xmm9\r
-       vfmaddpd 2 * SIZE(CO1, LDC),%xmm7, %xmm13,%xmm13\r
-       vfmaddpd (CO2),%xmm7, %xmm10,%xmm10\r
-       vfmaddpd 2 * SIZE(CO2),%xmm7, %xmm14,%xmm14\r
-       vfmaddpd (CO2, LDC),%xmm7, %xmm11,%xmm11\r
-       vfmaddpd 2 * SIZE(CO2, LDC),%xmm7, %xmm15,%xmm15\r
-\r
-#else\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-       vmulpd  %xmm7, %xmm12,%xmm12\r
-       vmulpd  %xmm7, %xmm9,%xmm9\r
-       vmulpd  %xmm7, %xmm13,%xmm13\r
-       vmulpd  %xmm7, %xmm10,%xmm10\r
-       vmulpd  %xmm7, %xmm14,%xmm14\r
-       vmulpd  %xmm7, %xmm11,%xmm11\r
-       vmulpd  %xmm7, %xmm15,%xmm15\r
-\r
-#endif\r
-\r
-       vmovups %xmm8,  (CO1)\r
-       vmovups %xmm12, 2 * SIZE(CO1)\r
-       vmovups %xmm9,  (CO1, LDC)\r
-       vmovups %xmm13, 2 * SIZE(CO1, LDC)\r
-       vmovups %xmm10, (CO2)\r
-       vmovups %xmm14, 2 * SIZE(CO2)\r
-       vmovups %xmm11, (CO2, LDC)\r
-       vmovups %xmm15, 2 * SIZE(CO2, LDC)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 4), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $4, KK\r
-#endif\r
-\r
-       addq    $4 * SIZE, CO1          # coffset += 4\r
-       addq    $4 * SIZE, CO2          # coffset += 4\r
-       decq    I                       # i --\r
-       BRANCH\r
-       jg      .L11\r
-       ALIGN_4 \r
-\r
-.L20:\r
-       testq   $3, M\r
-       je      .L39\r
-\r
-       testq   $2, M\r
-       je      .L30\r
-       ALIGN_4\r
-\r
-.L21:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (B,  %rax, 4), BO\r
-#endif \r
-\r
-       vmovups -16 * SIZE(AO), %xmm0\r
-       vxorps  %xmm8, %xmm8, %xmm8\r
-       vmovups -12 * SIZE(AO), %xmm2\r
-       vxorps  %xmm9, %xmm9 ,%xmm9\r
-       vmovddup        -16 * SIZE(BO), %xmm1\r
-       vxorps  %xmm10, %xmm10, %xmm10\r
-       vmovddup        -15 * SIZE(BO), %xmm5\r
-       vxorps  %xmm11, %xmm11, %xmm11\r
-       vmovddup         -8 * SIZE(BO), %xmm3\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $2, %rax\r
-#else\r
-       addq    $4, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 4), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L26\r
-       ALIGN_4\r
-\r
-.L22:\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vfmaddpd %xmm9,%xmm0, %xmm5,%xmm9\r
-       vmovddup        -14 * SIZE(BO, %rax, 4), %xmm1\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10\r
-       vmovddup        -13 * SIZE(BO, %rax, 4), %xmm5\r
-       vfmaddpd %xmm11,%xmm0, %xmm5,%xmm11\r
-       vmovups -14 * SIZE(AO, %rax, 2), %xmm0\r
-       vmovddup        -12 * SIZE(BO, %rax, 4), %xmm1\r
-       vmovddup        -11 * SIZE(BO, %rax, 4), %xmm5\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vfmaddpd %xmm9,%xmm0, %xmm5,%xmm9\r
-       vmovddup        -10 * SIZE(BO, %rax, 4), %xmm1\r
-       vmovddup         -9 * SIZE(BO, %rax, 4), %xmm5\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10\r
-       vfmaddpd %xmm11,%xmm0, %xmm5,%xmm11\r
-       vmovddup          (BO, %rax, 4), %xmm1\r
-       vmovddup         -7 * SIZE(BO, %rax, 4), %xmm5\r
-       vmovups  -8 * SIZE(AO, %rax, 2), %xmm0\r
-       vfmaddpd %xmm8,%xmm2, %xmm3,%xmm8\r
-       vfmaddpd %xmm9,%xmm2, %xmm5,%xmm9\r
-       vmovddup         -6 * SIZE(BO, %rax, 4), %xmm3\r
-       vmovddup         -5 * SIZE(BO, %rax, 4), %xmm5\r
-       vfmaddpd %xmm10,%xmm2, %xmm3,%xmm10\r
-       vfmaddpd %xmm11,%xmm2, %xmm5,%xmm11\r
-       vmovups -10 * SIZE(AO, %rax, 2), %xmm2\r
-       vmovddup         -4 * SIZE(BO, %rax, 4), %xmm3\r
-       vmovddup         -3 * SIZE(BO, %rax, 4), %xmm5\r
-       vfmaddpd %xmm8,%xmm2, %xmm3,%xmm8\r
-       vfmaddpd %xmm9,%xmm2, %xmm5,%xmm9\r
-       vmovddup         -2 * SIZE(BO, %rax, 4), %xmm3\r
-       vmovddup         -1 * SIZE(BO, %rax, 4), %xmm5\r
-       vfmaddpd %xmm10,%xmm2, %xmm3,%xmm10\r
-       vfmaddpd %xmm11,%xmm2, %xmm5,%xmm11\r
-       vmovddup          8 * SIZE(BO, %rax, 4), %xmm3\r
-       vmovups  -4 * SIZE(AO, %rax, 2), %xmm2\r
-       vmovddup          1 * SIZE(BO, %rax, 4), %xmm5\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L22\r
-       ALIGN_4\r
-\r
-.L26:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L29\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 4), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L27:\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vmovddup        -14 * SIZE(BO, %rax, 4), %xmm1\r
-       vfmaddpd %xmm9,%xmm0, %xmm5,%xmm9\r
-       vmovddup        -13 * SIZE(BO, %rax, 4), %xmm5\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10\r
-       vmovddup        -12 * SIZE(BO, %rax, 4), %xmm1\r
-       vfmaddpd %xmm11,%xmm0, %xmm5,%xmm11\r
-       vmovups -14 * SIZE(AO, %rax, 2), %xmm0\r
-       vmovddup        -11 * SIZE(BO, %rax, 4), %xmm5\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L27\r
-       ALIGN_4\r
-\r
-.L29:\r
-#ifndef TRMMKERNEL\r
-\r
-       vfmaddpd (CO1),%xmm7, %xmm8,%xmm8\r
-       vfmaddpd (CO1, LDC),%xmm7, %xmm9,%xmm9\r
-       vfmaddpd (CO2),%xmm7, %xmm10,%xmm10\r
-       vfmaddpd (CO2, LDC),%xmm7, %xmm11,%xmm11\r
-\r
-#else\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-       vmulpd  %xmm7, %xmm9,%xmm9\r
-       vmulpd  %xmm7, %xmm10,%xmm10\r
-       vmulpd  %xmm7, %xmm11,%xmm11\r
-\r
-#endif\r
-\r
-       vmovups %xmm8,  (CO1)\r
-       vmovups %xmm9,  (CO1, LDC)\r
-\r
-       vmovups %xmm10, (CO2)\r
-       vmovups %xmm11, (CO2, LDC)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 4), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $2, KK\r
-#endif\r
-\r
-       addq    $2 * SIZE, CO1\r
-       addq    $2 * SIZE, CO2\r
-       ALIGN_4 \r
-\r
-.L30:\r
-       testq   $1, M\r
-       je      .L39\r
-\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (B,  %rax, 4), BO\r
-#endif \r
-\r
-       vmovddup        -16 * SIZE(AO), %xmm0\r
-       vxorps  %xmm8, %xmm8, %xmm8\r
-       vmovddup        -14 * SIZE(AO), %xmm2\r
-       vxorps  %xmm9, %xmm9, %xmm9\r
-       vmovddup        -15 * SIZE(AO), %xmm4\r
-       vxorps  %xmm10, %xmm10,%xmm10\r
-       vmovups -16 * SIZE(BO), %xmm1\r
-       vxorps  %xmm11, %xmm11,%xmm11\r
-       vmovups  -8 * SIZE(BO), %xmm3\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $1, %rax\r
-#else\r
-       addq    $4, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 4), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L36\r
-       ALIGN_4\r
-\r
-.L32:\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 4), %xmm0,%xmm9\r
-       vmovups -12 * SIZE(BO, %rax, 4), %xmm1\r
-       vmovddup -12 * SIZE(AO, %rax, 1), %xmm0\r
-       vfmaddpd %xmm10,%xmm4, %xmm1,%xmm10\r
-       vfmaddpd %xmm11,-10 * SIZE(BO, %rax, 4), %xmm4,%xmm11\r
-       vmovups   (BO, %rax, 4), %xmm1\r
-       vmovddup        -11 * SIZE(AO, %rax, 1), %xmm4\r
-       vfmaddpd %xmm8,%xmm2, %xmm3,%xmm8\r
-       vfmaddpd %xmm9,-6 * SIZE(BO, %rax, 4), %xmm2,%xmm9\r
-       vmovups  -4 * SIZE(BO, %rax, 4), %xmm3\r
-       vmovddup        -13 * SIZE(AO, %rax, 1), %xmm2\r
-       vfmaddpd %xmm10,%xmm2, %xmm3,%xmm10\r
-       vfmaddpd %xmm11,-2 * SIZE(BO, %rax, 4), %xmm2,%xmm11\r
-       vmovups   8 * SIZE(BO, %rax, 4), %xmm3\r
-       vmovddup        -10 * SIZE(AO, %rax, 1), %xmm2\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L32\r
-       ALIGN_4\r
-\r
-.L36:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L38\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 4), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L37:\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 4), %xmm0,%xmm9\r
-       vmovups -12 * SIZE(BO, %rax, 4), %xmm1\r
-       vmovddup -15 * SIZE(AO, %rax, 1), %xmm0\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L37\r
-       ALIGN_4\r
-\r
-.L38:\r
-       vaddpd  %xmm10, %xmm8,%xmm8\r
-       vaddpd  %xmm11, %xmm9,%xmm9\r
-\r
-#ifndef TRMMKERNEL\r
-       vmovsd  (CO1), %xmm0\r
-       vmovhpd (CO1, LDC), %xmm0,%xmm0\r
-       vmovsd  (CO2), %xmm1\r
-       vmovhpd (CO2, LDC), %xmm1,%xmm1\r
-\r
-\r
-       vfmaddpd        %xmm0, %xmm7,%xmm8,%xmm8\r
-       vfmaddpd        %xmm1, %xmm7,%xmm9,%xmm9\r
-#else\r
-\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-       vmulpd  %xmm7, %xmm9,%xmm9\r
-\r
-#endif\r
-\r
-       vmovsd  %xmm8,  (CO1)\r
-       vmovhpd %xmm8,  (CO1, LDC)\r
-       vmovsd  %xmm9,  (CO2)\r
-       vmovhpd %xmm9,  (CO2, LDC)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 4), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $1, KK\r
-#endif\r
-       ALIGN_4 \r
-       \r
-.L39:\r
-#if defined(TRMMKERNEL) && !defined(LEFT)\r
-       addq    $4, KK\r
-#endif\r
-       \r
-       movq    BO, B\r
-\r
-       decq    J                       # j --\r
-       jg      .L01\r
-       ALIGN_4\r
-\r
-.L40:   # N % 4\r
-       testq   $3, N                   # N % 4 == 3\r
-       je      .L999                   # Jump to end if N % 4 == 0\r
-\r
-       testq   $2, N                   # N % 4 == 2\r
-       je      .L80\r
-       ALIGN_4\r
-\r
-.L41:                                  # N % 4 > 1\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       movq    OFFSET, %rax\r
-       movq    %rax, KK\r
-#endif \r
-\r
-       movq    C, CO1                  # coffset1 = c\r
-       leaq    (C, LDC, 1), CO2        # coffset2 = c + ldc\r
-       movq    A, AO                   # aoffset = a\r
-\r
-       movq    K, %rax\r
-       salq    $BASE_SHIFT + 1, %rax   # k << 4\r
-       leaq    (B, %rax), BB\r
-\r
-       movq    M,  I\r
-       sarq    $2, I                   # i = (m >> 2)\r
-       jle     .L60\r
-       ALIGN_4\r
-\r
-.L51:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (B,  %rax, 2), BO\r
-#endif \r
-\r
-       vmovddup        -16 * SIZE(BO), %xmm1\r
-       vmovddup        -15 * SIZE(BO), %xmm5\r
-       vmovddup        -12 * SIZE(BO), %xmm3\r
-       vxorps  %xmm8, %xmm8,%xmm8\r
-       vxorps  %xmm9, %xmm9,%xmm9\r
-       vxorps  %xmm12, %xmm12,%xmm12\r
-       vxorps  %xmm13, %xmm13,%xmm13\r
-       vmovups -16 * SIZE(AO), %xmm0\r
-       vmovups  -8 * SIZE(AO), %xmm4\r
-       vmovups %xmm0, %xmm2\r
-       subq             $-8 * SIZE, BB\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $4, %rax\r
-#else\r
-       addq    $2, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 2), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L56\r
-       ALIGN_4\r
-\r
-.L52:                                                          # Loop for (N % 4) == 2\r
-       vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8\r
-       vfmaddpd %xmm9,%xmm5, %xmm2,%xmm9\r
-       vmovups         -14 * SIZE(AO, %rax, 4),%xmm2\r
-       vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12\r
-       vmovups         -12 * SIZE(AO, %rax, 4), %xmm0\r
-       vmovddup        -14 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13\r
-       vmovddup        -13 * SIZE(BO, %rax, 2), %xmm5\r
-       vmovups         -10 * SIZE(AO, %rax, 4), %xmm2\r
-       vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8\r
-       vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12\r
-       vfmaddpd %xmm9,%xmm5, %xmm0,%xmm9\r
-       vmovups                  (AO, %rax, 4), %xmm0\r
-       vmovddup                -8 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13\r
-       vmovddup        -11 * SIZE(BO, %rax, 2), %xmm5\r
-       vmovups         -6 * SIZE(AO, %rax, 4), %xmm2\r
-       vfmaddpd %xmm8,%xmm3, %xmm4,%xmm8\r
-       vfmaddpd %xmm12,%xmm2, %xmm3,%xmm12\r
-       vfmaddpd %xmm9,%xmm5, %xmm4,%xmm9\r
-       vmovups         -4 * SIZE(AO, %rax, 4), %xmm4\r
-       vmovddup        -10 * SIZE(BO, %rax, 2), %xmm3\r
-       vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13\r
-       vmovddup         -9 * SIZE(BO, %rax, 2), %xmm5\r
-       vmovups         -2 * SIZE(AO, %rax, 4), %xmm2\r
-       vfmaddpd %xmm8,%xmm3, %xmm4,%xmm8\r
-       vfmaddpd %xmm12,%xmm2, %xmm3,%xmm12\r
-       vfmaddpd %xmm9,%xmm5, %xmm4,%xmm9\r
-       vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13\r
-       vmovups           8 * SIZE(AO, %rax, 4), %xmm4\r
-       vmovddup         -4 * SIZE(BO, %rax, 2), %xmm3\r
-       vmovddup         -7 * SIZE(BO, %rax, 2), %xmm5\r
-       vmovaps %xmm0, %xmm2\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L52\r
-       ALIGN_4\r
-\r
-.L56:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L59\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 2), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L57:\r
-       vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8\r
-       vfmaddpd %xmm9,%xmm5, %xmm2,%xmm9\r
-       vmovups -14 * SIZE(AO, %rax, 4),%xmm2\r
-       vfmaddpd %xmm12,%xmm2, %xmm1,%xmm12\r
-       vmovups -12 * SIZE(AO, %rax, 4), %xmm0\r
-       vmovddup        -14 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm13,%xmm2, %xmm5,%xmm13\r
-       vmovddup        -13 * SIZE(BO, %rax, 2), %xmm5\r
-       vmovaps %xmm0, %xmm2\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L57\r
-       ALIGN_4\r
-\r
-.L59:\r
-#ifndef TRMMKERNEL\r
-       vfmaddpd        (CO1),%xmm7, %xmm8, %xmm8\r
-       vfmaddpd        2 * SIZE(CO1),%xmm7, %xmm12, %xmm12\r
-       vfmaddpd        (CO2),%xmm7, %xmm9, %xmm9\r
-       vfmaddpd        2 * SIZE(CO2),%xmm7, %xmm13, %xmm13\r
-\r
-#else\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-       vmulpd  %xmm7, %xmm9,%xmm9\r
-       vmulpd  %xmm7, %xmm12,%xmm12\r
-       vmulpd  %xmm7, %xmm13,%xmm13\r
-\r
-#endif\r
-\r
-       vmovups %xmm8,  (CO1)\r
-       vmovups %xmm12, 2 * SIZE(CO1)\r
-\r
-       vmovups %xmm9,  (CO2)\r
-       vmovups %xmm13, 2 * SIZE(CO2)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 2), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $4, KK\r
-#endif\r
-\r
-       addq    $4 * SIZE, CO1          # coffset += 4\r
-       addq    $4 * SIZE, CO2          # coffset += 4\r
-       decq    I                       # i --\r
-       jg      .L51\r
-       ALIGN_4 \r
-\r
-.L60:\r
-       testq   $2, M\r
-       je      .L70\r
-       ALIGN_4\r
-\r
-.L61:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (B,  %rax, 2), BO\r
-#endif \r
-\r
-       vmovups -16 * SIZE(AO), %xmm0\r
-       vxorps  %xmm8, %xmm8,%xmm8\r
-       vmovups -12 * SIZE(AO), %xmm2\r
-       vxorps  %xmm9, %xmm9,%xmm9\r
-       vmovddup        -16 * SIZE(BO), %xmm1\r
-       vxorps  %xmm10, %xmm10,%xmm10\r
-       vmovddup        -15 * SIZE(BO), %xmm3\r
-       vxorps  %xmm11, %xmm11,%xmm11\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $2, %rax\r
-#else\r
-       addq    $2, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 2), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L66\r
-       ALIGN_4\r
-\r
-.L62:\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vmovddup        -14 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9\r
-       vmovups -14 * SIZE(AO, %rax, 2), %xmm0\r
-       vmovddup        -13 * SIZE(BO, %rax, 2), %xmm3\r
-       vfmaddpd %xmm10,%xmm0, %xmm1,%xmm10\r
-       vmovddup        -12 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm11,%xmm0, %xmm3,%xmm11\r
-       vmovups  -8 * SIZE(AO, %rax, 2), %xmm0\r
-       vmovddup        -11 * SIZE(BO, %rax, 2), %xmm3\r
-       vfmaddpd %xmm8,%xmm2, %xmm1,%xmm8\r
-       vmovddup        -10 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm9,%xmm2, %xmm3,%xmm9\r
-       vmovups -10 * SIZE(AO, %rax, 2), %xmm2\r
-       vmovddup         -9 * SIZE(BO, %rax, 2), %xmm3\r
-       vfmaddpd %xmm10,%xmm2, %xmm1,%xmm10\r
-       vmovddup         -8 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm11,%xmm2, %xmm3,%xmm11\r
-       vmovups  -4 * SIZE(AO, %rax, 2), %xmm2\r
-       vmovddup         -7 * SIZE(BO, %rax, 2), %xmm3\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L62\r
-       ALIGN_4\r
-\r
-.L66:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L69\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 2), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L67:\r
-       vfmaddpd %xmm8,%xmm0, %xmm1,%xmm8\r
-       vmovddup        -14 * SIZE(BO, %rax, 2), %xmm1\r
-       vfmaddpd %xmm9,%xmm0, %xmm3,%xmm9\r
-       vmovups -14 * SIZE(AO, %rax, 2), %xmm0\r
-       vmovddup        -13 * SIZE(BO, %rax, 2), %xmm3\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L67\r
-       ALIGN_4\r
-\r
-.L69:\r
-       vaddpd  %xmm10, %xmm8,%xmm8\r
-       vaddpd  %xmm11, %xmm9,%xmm9\r
-       \r
-#ifndef TRMMKERNEL\r
-\r
-       vfmaddpd        (CO1),%xmm7, %xmm8,%xmm8\r
-       vfmaddpd        (CO2),%xmm7, %xmm9,%xmm9\r
-\r
-#else\r
-\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-       vmulpd  %xmm7, %xmm9,%xmm9\r
-\r
-#endif\r
-\r
-       vmovups %xmm8,  (CO1)\r
-       vmovups %xmm9,  (CO2)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 2), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $2, KK\r
-#endif\r
-\r
-       addq    $2 * SIZE, CO1          # coffset += 4\r
-       addq    $2 * SIZE, CO2          # coffset += 4\r
-       ALIGN_4 \r
-\r
-.L70:\r
-       testq   $1, M\r
-       je      .L79\r
-       ALIGN_4\r
-\r
-.L71:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (B,  %rax, 2), BO\r
-#endif \r
-\r
-       vmovddup        -16 * SIZE(AO), %xmm0\r
-       vxorps  %xmm8, %xmm8,%xmm8\r
-       vmovddup        -15 * SIZE(AO), %xmm1\r
-       vxorps  %xmm9, %xmm9,%xmm9\r
-       vmovddup        -14 * SIZE(AO), %xmm2\r
-       vxorps  %xmm10, %xmm10,%xmm10\r
-       vmovddup        -13 * SIZE(AO), %xmm3\r
-       vxorps  %xmm11, %xmm11,%xmm11\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $1, %rax\r
-#else\r
-       addq    $2, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 2), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L76\r
-       ALIGN_4\r
-\r
-.L72:\r
-       vfmaddpd %xmm8,-16 * SIZE(BO, %rax, 2), %xmm0,%xmm8\r
-       vmovddup        -12 * SIZE(AO, %rax, 1), %xmm0\r
-\r
-       vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 2), %xmm1,%xmm9\r
-       vmovddup        -11 * SIZE(AO, %rax, 1), %xmm1\r
-\r
-       vfmaddpd %xmm10,-12 * SIZE(BO, %rax, 2), %xmm2,%xmm10\r
-       vmovddup        -10 * SIZE(AO, %rax, 1), %xmm2\r
-\r
-       vfmaddpd %xmm11,-10 * SIZE(BO, %rax, 2), %xmm3,%xmm11\r
-       vmovddup         -9 * SIZE(AO, %rax, 1), %xmm3\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L72\r
-       ALIGN_4\r
-\r
-.L76:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L78\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 2), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L77:\r
-       vfmaddpd %xmm8,-16 * SIZE(BO, %rax, 2), %xmm0,%xmm8\r
-       vmovddup        -15 * SIZE(AO, %rax, 1), %xmm0\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L77\r
-       ALIGN_4\r
-\r
-.L78:\r
-       vaddpd  %xmm9,  %xmm8,%xmm8\r
-       vaddpd  %xmm11, %xmm10,%xmm10\r
-       vaddpd  %xmm10, %xmm8,%xmm8\r
-\r
-#ifndef TRMMKERNEL\r
-       vmovsd  (CO1), %xmm0\r
-       vmovhpd (CO2), %xmm0,%xmm0\r
-#endif\r
-\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-\r
-#ifndef TRMMKERNEL\r
-       vaddpd  %xmm0, %xmm8,%xmm8\r
-#endif\r
-\r
-       vmovsd  %xmm8,  (CO1)\r
-       vmovhpd %xmm8,  (CO2)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 2), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $1, KK\r
-#endif\r
-       ALIGN_4 \r
-       \r
-.L79:\r
-#if defined(TRMMKERNEL) && !defined(LEFT)\r
-       addq    $2, KK\r
-#endif\r
-\r
-       movq    BO, B\r
-\r
-       leaq    (C, LDC, 2), C\r
-       ALIGN_4\r
-\r
-.L80:\r
-       testq   $1, N                   # N % 4 == 1\r
-       je      .L999                   # Jump to end if N % 4 == 0\r
-       ALIGN_4\r
-\r
-.L81:\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       movq    OFFSET, %rax\r
-       movq    %rax, KK\r
-#endif \r
-\r
-       movq    C, CO1                  # coffset1 = c\r
-       movq    A, AO           # aoffset = a\r
-\r
-       movq    M,  I\r
-       sarq    $2, I   # i = (m >> 2)\r
-       jle     .L100\r
-       ALIGN_4\r
-\r
-.L91:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (B,  %rax, 1), BO\r
-#endif \r
-\r
-       vmovups  -8 * SIZE(AO), %xmm2\r
-       vxorps  %xmm8, %xmm8,%xmm8\r
-       vmovups -16 * SIZE(AO), %xmm0\r
-       vxorps  %xmm9, %xmm9,%xmm9\r
-       vmovddup        -16 * SIZE(BO), %xmm1\r
-       vxorps  %xmm12, %xmm12,%xmm12\r
-       vmovddup        -14 * SIZE(BO), %xmm3\r
-       vxorps  %xmm13, %xmm13,%xmm13\r
-       vmovddup        -15 * SIZE(BO), %xmm5\r
-\r
-       // prefetchw      3 * SIZE(CO1)\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $4, %rax\r
-#else\r
-       addq    $1, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 1), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L96\r
-       ALIGN_4\r
-\r
-.L92:\r
-       vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8\r
-       vfmaddpd %xmm12,-14 * SIZE(AO, %rax, 4), %xmm1,%xmm12\r
-       vmovapd -12 * SIZE(AO, %rax, 4), %xmm0\r
-       vmovddup        -12 * SIZE(BO, %rax, 1), %xmm1\r
-       vfmaddpd %xmm9,%xmm5, %xmm0,%xmm9\r
-       vfmaddpd %xmm13,-10 * SIZE(AO, %rax, 4), %xmm5,%xmm13\r
-       vmovapd   (AO, %rax, 4), %xmm0\r
-       vmovddup        -13 * SIZE(BO, %rax, 1), %xmm5\r
-       vfmaddpd %xmm8,%xmm3, %xmm2,%xmm8\r
-       vfmaddpd %xmm12,-6 * SIZE(AO, %rax, 4), %xmm3,%xmm12\r
-       vmovapd  -4 * SIZE(AO, %rax, 4), %xmm2\r
-       vmovddup        -10 * SIZE(BO, %rax, 1), %xmm3\r
-       vfmaddpd %xmm9,%xmm5, %xmm2,%xmm9\r
-       vfmaddpd %xmm13,-2 * SIZE(AO, %rax, 4), %xmm5,%xmm13\r
-       vmovapd   8 * SIZE(AO, %rax, 4), %xmm2\r
-       vmovddup        -11 * SIZE(BO, %rax, 1), %xmm5\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L92\r
-       ALIGN_4\r
-\r
-.L96:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L99\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 1), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L97:\r
-       vfmaddpd %xmm8,%xmm1, %xmm0,%xmm8\r
-       vfmaddpd %xmm12,-14 * SIZE(AO, %rax, 4), %xmm1,%xmm12\r
-       vmovups -12 * SIZE(AO, %rax, 4), %xmm0\r
-       vmovddup        -15 * SIZE(BO, %rax, 1), %xmm1\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L97\r
-       ALIGN_4\r
-\r
-.L99:\r
-       vaddpd  %xmm9,  %xmm8,%xmm8\r
-       vaddpd  %xmm13, %xmm12,%xmm12\r
-\r
-#ifndef TRMMKERNEL\r
-\r
-       vfmaddpd                (CO1),%xmm7, %xmm8,%xmm8\r
-       vfmaddpd        2 * SIZE(CO1),%xmm7,%xmm12,%xmm12\r
-\r
-#else\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-       vmulpd  %xmm7, %xmm12,%xmm12\r
-\r
-#endif\r
-\r
-       vmovups %xmm8,  (CO1)\r
-       vmovups %xmm12, 2 * SIZE(CO1)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 4), AO\r
-       leaq    (BO, %rax, 1), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $4, KK\r
-#endif\r
-\r
-       addq    $4 * SIZE, CO1          # coffset += 4\r
-       decq    I                       # i --\r
-       jg      .L91\r
-       ALIGN_4 \r
-\r
-.L100:\r
-       testq   $2, M\r
-       je      .L110\r
-       ALIGN_4\r
-\r
-.L101:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (B,  %rax, 1), BO\r
-#endif \r
-\r
-       vmovddup        -16 * SIZE(BO), %xmm0\r
-       vxorps  %xmm8, %xmm8,%xmm8\r
-       vmovddup        -15 * SIZE(BO), %xmm1\r
-       vxorps  %xmm9, %xmm9,%xmm9\r
-       vmovddup        -14 * SIZE(BO), %xmm2\r
-       vxorps  %xmm10, %xmm10,%xmm10\r
-       vmovddup        -13 * SIZE(BO), %xmm3\r
-       vxorps  %xmm11, %xmm11,%xmm11\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $2, %rax\r
-#else\r
-       addq    $1, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 1), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L106\r
-       ALIGN_4\r
-\r
-.L102:\r
-       vfmaddpd %xmm8,-16 * SIZE(AO, %rax, 2), %xmm0,%xmm8\r
-       vmovddup        -12 * SIZE(BO, %rax, 1), %xmm0\r
-\r
-       vfmaddpd %xmm9,-14 * SIZE(AO, %rax, 2), %xmm1,%xmm9\r
-       vmovddup        -11 * SIZE(BO, %rax, 1), %xmm1\r
-\r
-       vfmaddpd %xmm10,-12 * SIZE(AO, %rax, 2), %xmm2,%xmm10\r
-       vmovddup        -10 * SIZE(BO, %rax, 1), %xmm2\r
-\r
-       vfmaddpd %xmm11,-10 * SIZE(AO, %rax, 2), %xmm3,%xmm11\r
-       vmovddup         -9 * SIZE(BO, %rax, 1), %xmm3\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L102\r
-       ALIGN_4\r
-\r
-.L106:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L109\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 1), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L107:\r
-       vmovddup        -16 * SIZE(BO, %rax, 1), %xmm0\r
-       vfmaddpd %xmm8,-16 * SIZE(AO, %rax, 2), %xmm0,%xmm8\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L107\r
-       ALIGN_4\r
-\r
-.L109:\r
-       vaddpd  %xmm9,  %xmm8,%xmm8\r
-       vaddpd  %xmm11, %xmm10,%xmm10\r
-       vaddpd  %xmm10, %xmm8,%xmm8\r
-\r
-#ifndef TRMMKERNEL\r
-\r
-       vfmaddpd        (CO1),%xmm7, %xmm8,%xmm8\r
-#else\r
-       vmulpd  %xmm7, %xmm8,%xmm8\r
-\r
-#endif\r
-\r
-       vmovups %xmm8,  (CO1)\r
-\r
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KKK, %rax\r
-       leaq    (,%rax, SIZE), %rax\r
-       leaq    (AO, %rax, 2), AO\r
-       leaq    (BO, %rax, 1), BO\r
-#endif\r
-\r
-#if defined(TRMMKERNEL) && defined(LEFT)\r
-       addq    $2, KK\r
-#endif\r
-\r
-       addq    $2 * SIZE, CO1          # coffset += 4\r
-\r
-       ALIGN_4 \r
-\r
-.L110:\r
-       testq   $1, M\r
-       je      .L999\r
-       ALIGN_4\r
-\r
-.L111:\r
-#if !defined(TRMMKERNEL) || \\r
-       (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
-       (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
-       movq    B, BO\r
-#else\r
-       movq    KK, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (B,  %rax, 1), BO\r
-#endif \r
-\r
-       vmovups -16 * SIZE(AO), %xmm0\r
-       vxorps  %xmm8, %xmm8,%xmm8\r
-       movups  -14 * SIZE(AO), %xmm1\r
-       vxorps  %xmm9, %xmm9,%xmm9\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
-       movq    K, %rax\r
-       subq    KK, %rax\r
-       movq    %rax, KKK       \r
-#else\r
-       movq    KK, %rax\r
-#ifdef LEFT\r
-       addq    $1, %rax\r
-#else\r
-       addq    $1, %rax\r
-#endif\r
-       movq    %rax, KKK\r
-#endif\r
-\r
-       andq    $-4, %rax\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 1), BO\r
-       negq    %rax\r
-       NOBRANCH\r
-       je      .L116\r
-       ALIGN_4\r
-\r
-.L112:\r
-       vfmaddpd %xmm8,-16 * SIZE(BO, %rax, 1), %xmm0,%xmm8\r
-       vmovups -12 * SIZE(AO, %rax, 1), %xmm0\r
-\r
-       vfmaddpd %xmm9,-14 * SIZE(BO, %rax, 1), %xmm1,%xmm9\r
-       vmovups -10 * SIZE(AO, %rax, 1), %xmm1\r
-\r
-       addq    $4 * SIZE, %rax\r
-       BRANCH\r
-       jl      .L112\r
-       ALIGN_4\r
-\r
-.L116:\r
-       vmovddup        ALPHA, %xmm7\r
-\r
-#ifndef TRMMKERNEL\r
-       movq    K, %rax\r
-#else\r
-       movq    KKK, %rax\r
-#endif\r
-       andq    $3, %rax                # if (k & 1)\r
-       je .L118\r
-\r
-       leaq    (, %rax, SIZE), %rax\r
-       leaq    (AO, %rax, 1), AO\r
-       leaq    (BO, %rax, 1), BO\r
-       negq    %rax\r
-       ALIGN_4\r
-\r
-.L117:\r
-       vmulsd  -16 * SIZE(BO, %rax, 1), %xmm0,%xmm0\r
-       vaddsd  %xmm0, %xmm8,%xmm8\r
-       vmovsd  -15 * SIZE(AO, %rax, 1), %xmm0\r
-\r
-       addq    $SIZE, %rax\r
-       jl      .L117\r
-       ALIGN_4\r
-\r
-.L118:\r
-       vaddpd  %xmm9, %xmm8,%xmm8\r
-       vhaddpd %xmm8, %xmm8,%xmm8\r
-\r
-#ifndef TRMMKERNEL\r
-       vmovsd  (CO1), %xmm0\r
-#endif\r
-\r
-       vmulsd  %xmm7, %xmm8,%xmm8\r
-\r
-#ifndef TRMMKERNEL\r
-       vaddsd  %xmm0, %xmm8,%xmm8\r
-#endif\r
-\r
-       vmovsd  %xmm8,  (CO1)\r
-       ALIGN_4\r
-       \r
-.L999:\r
-       movq       (%rsp), %rbx\r
-       movq      8(%rsp), %rbp\r
-       movq     16(%rsp), %r12\r
-       movq     24(%rsp), %r13\r
-       movq     32(%rsp), %r14\r
-       movq     40(%rsp), %r15\r
-\r
-#ifdef WINDOWS_ABI\r
-       movq     48(%rsp), %rdi\r
-       movq     56(%rsp), %rsi\r
-       movups   64(%rsp), %xmm6\r
-       movups   80(%rsp), %xmm7\r
-       movups   96(%rsp), %xmm8\r
-       movups  112(%rsp), %xmm9\r
-       movups  128(%rsp), %xmm10\r
-       movups  144(%rsp), %xmm11\r
-       movups  160(%rsp), %xmm12\r
-       movups  176(%rsp), %xmm13\r
-       movups  192(%rsp), %xmm14\r
-       movups  208(%rsp), %xmm15\r
-#endif\r
-\r
-       addq    $STACKSIZE, %rsp\r
-       ret\r
-\r
-       EPILOGUE\r

diff --git a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S
new file mode 100644 (file)
index 0000000..dc980cb
--- /dev/null
+++ b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S
@@ -0,0 +1,3854 @@
+/*********************************************************************/\r
+/* Copyright 2009, 2010 The University of Texas at Austin.           */\r
+/* All rights reserved.                                              */\r
+/*                                                                   */\r
+/* Redistribution and use in source and binary forms, with or        */\r
+/* without modification, are permitted provided that the following   */\r
+/* conditions are met:                                               */\r
+/*                                                                   */\r
+/*   1. Redistributions of source code must retain the above         */\r
+/*      copyright notice, this list of conditions and the following  */\r
+/*      disclaimer.                                                  */\r
+/*                                                                   */\r
+/*   2. Redistributions in binary form must reproduce the above      */\r
+/*      copyright notice, this list of conditions and the following  */\r
+/*      disclaimer in the documentation and/or other materials       */\r
+/*      provided with the distribution.                              */\r
+/*                                                                   */\r
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */\r
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */\r
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */\r
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */\r
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */\r
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */\r
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */\r
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */\r
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */\r
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */\r
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */\r
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */\r
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */\r
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */\r
+/*                                                                   */\r
+/* The views and conclusions contained in the software and           */\r
+/* documentation are those of the authors and should not be          */\r
+/* interpreted as representing official policies, either expressed   */\r
+/* or implied, of The University of Texas at Austin.                 */\r
+/*********************************************************************/\r
+\r
+/*********************************************************************\r
+* 2013/06/02 Saar\r
+*\r
+* Parameter:\r
+*      UNROLL_M        8\r
+*      UNROLL_N        2\r
+*      DGEMM_P         360\r
+*      DGEMM_Q         160\r
+*\r
+* Performance at m x n without prefetch of BO:\r
+* \r
+* 5760x5760    93.4    GFLOPS with 8 threads on 4 modules (ACML: 90.8 GFLOPS)\r
+* 5760x5760    84.2    GFLOPS with 4 threads on 4 modules (ACML: 82.4 GFLOPS)\r
+* 3840x3840    50.3    GFLOPS with 2 threads on 2 modules (ACML: 49.5 GFLOPS)\r
+*\r
+* 5760x5760    56.4    GFLOPS with 4 threads on 2 modules (ACML: 58.5 GFLOPS)\r
+* 3840x3840    29.0    GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS)\r
+* 3840x3840    26.1    GFLOPS with 1 threads on 1 modules (ACML: 25.9 GFLOPS)\r
+*\r
+*********************************************************************/\r
+\r
+/*********************************************************************\r
+* 2013/06/03 Saar\r
+*\r
+* Parameter:\r
+*      UNROLL_M        8\r
+*      UNROLL_N        2\r
+*      DGEMM_P         336\r
+*      DGEMM_Q         168\r
+*      NO_WARMUP       1\r
+*      NO_AFFINITY     1\r
+*      GEMM_MULTITHREAD_THRESHOLD 4\r
+*\r
+* Performance at m x n with prefetch of BO:\r
+* \r
+* 8064x3840    93.7    GFLOPS with 8 threads on 4 modules (ACML: 93.6 GFLOPS)\r
+* 6048x2880    85.1    GFLOPS with 4 threads on 4 modules (ACML: 84.2 GFLOPS)\r
+* 6048x2880    52.0    GFLOPS with 2 threads on 2 modules (ACML: 50.0 GFLOPS)\r
+*\r
+* 6048x2880    56.3    GFLOPS with 4 threads on 2 modules (ACML: 57.6 GFLOPS)\r
+* 4032x1920    29.5    GFLOPS with 2 threads on 1 modules (ACML: 30.5 GFLOPS)\r
+* 4032x1920    26.9    GFLOPS with 1 threads on 1 modules (ACML: 26.1 GFLOPS)\r
+*\r
+*********************************************************************/\r
+\r
+/*********************************************************************\r
+* 2013/06/04 Saar\r
+*\r
+* Parameter:\r
+*      UNROLL_M        8\r
+*      UNROLL_N        2\r
+*      DGEMM_P         384\r
+*      DGEMM_Q         168\r
+*      NO_WARMUP       1\r
+*      NO_AFFINITY     1\r
+*      GEMM_MULTITHREAD_THRESHOLD 4\r
+*\r
+* Performance at m x n with prefetch of BO:\r
+* \r
+* 6144x5376    94.6    GFLOPS with 8 threads on 4 modules (ACML: 90.5 GFLOPS)\r
+* 6144x5376    86.0    GFLOPS with 4 threads on 4 modules (ACML: 81.5 GFLOPS)\r
+* 4608x4032    52.0    GFLOPS with 2 threads on 2 modules (ACML: 47.5 GFLOPS)\r
+*\r
+* 6144x5376    57.3    GFLOPS with 4 threads on 2 modules (ACML: 56.5 GFLOPS)\r
+* 4608x4032    29.6    GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS)\r
+* 4608x4032    26.9    GFLOPS with 1 threads on 1 modules (ACML: 25.6 GFLOPS)\r
+*\r
+*********************************************************************/\r
+\r
+\r
+\r
+#define ASSEMBLER\r
+#include "common.h"\r
+ \r
+#define OLD_M  %rdi\r
+#define OLD_N  %rsi\r
+#define M      %r13\r
+#define J      %r14\r
+#define OLD_K  %rdx\r
+\r
+#define A      %rcx\r
+#define B      %r8\r
+#define C      %r9\r
+#define LDC    %r10\r
+       \r
+#define I      %r11\r
+#define AO     %rdi\r
+#define BO     %rsi\r
+#define        CO1     %r15\r
+#define K      %r12\r
+#define BI     %rbp\r
+#define        SP      %rbx\r
+\r
+#define BO1    %rdi\r
+#define BO2    %r15\r
+\r
+#ifndef WINDOWS_ABI\r
+\r
+#define STACKSIZE 96\r
+\r
+#else\r
+\r
+#define STACKSIZE 256\r
+\r
+#define OLD_A          40 + STACKSIZE(%rsp)\r
+#define OLD_B          48 + STACKSIZE(%rsp)\r
+#define OLD_C          56 + STACKSIZE(%rsp)\r
+#define OLD_LDC                64 + STACKSIZE(%rsp)\r
+#define OLD_OFFSET     72 + STACKSIZE(%rsp)\r
+\r
+#endif\r
+\r
+#define L_BUFFER_SIZE 512*8*4\r
+#define LB2_OFFSET    512*8*2\r
+\r
+#define Ndiv6   24(%rsp)\r
+#define Nmod6   32(%rsp)\r
+#define N       40(%rsp)\r
+#define ALPHA   48(%rsp)\r
+#define OFFSET  56(%rsp)\r
+#define KK      64(%rsp)\r
+#define KKK     72(%rsp)\r
+#define BUFFER1                   128(%rsp)\r
+#define BUFFER2        LB2_OFFSET+128(%rsp)\r
+\r
+\r
+\r
+#define        A_PR1   384\r
+#define        B_PR1   192\r
+\r
+#define KERNEL8x3_1(xx) \\r
+       prefetcht0      A_PR1(AO,%rax,8)        ;\\r
+       vmovddup         -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vfmaddpd        %xmm12,%xmm3,%xmm0,%xmm12 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+       vfmaddpd        %xmm15,%xmm3,%xmm0,%xmm15 ;\\r
+\r
+#define KERNEL8x3_2(xx) \\r
+       prefetcht0      A_PR1+64(AO,%rax,8)     ;\\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups          -6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+       vmovups          -4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vfmaddpd        %xmm12,%xmm3,%xmm0,%xmm12 ;\\r
+       vmovups          -2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+       vfmaddpd        %xmm15,%xmm3,%xmm0,%xmm15 ;\\r
+\r
+#define KERNEL8x3_3(xx) \\r
+       prefetcht0      A_PR1+128(AO,%rax,8)    ;\\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups           0 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup          2 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups           2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+       vmovups           4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vfmaddpd        %xmm12,%xmm3,%xmm0,%xmm12 ;\\r
+       vmovups           6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+       vfmaddpd        %xmm15,%xmm3,%xmm0,%xmm15 ;\\r
+\r
+#define KERNEL8x3_4(xx) \\r
+       prefetcht0      A_PR1+192(AO,%rax,8)    ;\\r
+       vmovddup          3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups           8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          4 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup          5 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups          10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+       vmovups          12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vfmaddpd        %xmm12,%xmm3,%xmm0,%xmm12 ;\\r
+       vmovups          14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+       vfmaddpd        %xmm15,%xmm3,%xmm0,%xmm15 ;\\r
+       addq    $12, BI                           ;\\r
+       addq    $32, %rax                         ;\\r
+\r
+#define KERNEL8x3_SUB(xx) \\r
+       vmovddup         -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vfmaddpd        %xmm12,%xmm3,%xmm0,%xmm12 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+       vfmaddpd        %xmm15,%xmm3,%xmm0,%xmm15 ;\\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL4x3_1(xx) \\r
+       vmovddup         -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+\r
+#define KERNEL4x3_2(xx) \\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+\r
+#define KERNEL4x3_3(xx) \\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup          2 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups          -6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+\r
+#define KERNEL4x3_4(xx) \\r
+       vmovddup          3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          4 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup          5 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups          -2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+       addq    $12, BI                           ;\\r
+       addq    $16, %rax                         ;\\r
+\r
+#define KERNEL4x3_SUB(xx) \\r
+       vmovddup         -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vfmaddpd        %xmm9,%xmm3,%xmm0,%xmm9 ;\\r
+\r
+\r
+\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL2x3_1(xx) \\r
+       vmovddup         -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+#define KERNEL2x3_2(xx) \\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+#define KERNEL2x3_3(xx) \\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup          2 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+#define KERNEL2x3_4(xx) \\r
+       vmovddup          3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          4 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup          5 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       addq    $12, BI                           ;\\r
+       addq    $8, %rax                          ;\\r
+\r
+#define KERNEL2x3_SUB(xx) \\r
+       vmovddup         -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddpd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL1x3_1(xx) \\r
+       vmovsd   -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd   -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovsd   -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddsd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+#define KERNEL1x3_2(xx) \\r
+       vmovsd   -3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -15 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd   -2 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovsd   -1 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddsd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+#define KERNEL1x3_3(xx) \\r
+       vmovsd    0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd    1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovsd    2 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddsd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+#define KERNEL1x3_4(xx) \\r
+       vmovsd    3 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -13 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd    4 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovsd    5 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddsd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+       addq    $12, BI                           ;\\r
+       addq    $4, %rax                          ;\\r
+\r
+#define KERNEL1x3_SUB(xx) \\r
+       vmovsd   -6 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd   -5 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovsd   -4 * SIZE(BO, BI, 8), %xmm3 ;\\r
+       vfmaddsd        %xmm6,%xmm3,%xmm0,%xmm6 ;\\r
+\r
+\r
+\r
+/*******************************************************************************************\r
+* 2 lines of N\r
+*******************************************************************************************/\r
+\r
+#define KERNEL8x2_1(xx) \\r
+       prefetcht0      A_PR1(AO,%rax,8)        ;\\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+\r
+#define KERNEL8x2_2(xx) \\r
+       prefetcht0      A_PR1+64(AO,%rax,8)     ;\\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups          -6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vmovups          -4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vmovups          -2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+\r
+#define KERNEL8x2_3(xx) \\r
+       prefetcht0      A_PR1+128(AO,%rax,8)    ;\\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups           0 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups           2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vmovups           4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vmovups           6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+\r
+#define KERNEL8x2_4(xx) \\r
+       prefetcht0      A_PR1+192(AO,%rax,8)    ;\\r
+       vmovddup          2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups           8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups          10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vmovups          12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vmovups          14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+       addq    $8, BI                            ;\\r
+       addq    $32, %rax                         ;\\r
+\r
+#define KERNEL8x2_SUB(xx) \\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vfmaddpd        %xmm11,%xmm2,%xmm0,%xmm11 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       vfmaddpd        %xmm14,%xmm2,%xmm0,%xmm14 ;\\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL4x2_1(xx) \\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+\r
+#define KERNEL4x2_2(xx) \\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+\r
+#define KERNEL4x2_3(xx) \\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups          -6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+\r
+#define KERNEL4x2_4(xx) \\r
+       vmovddup          2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups          -2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+       addq    $8, BI                            ;\\r
+       addq    $16, %rax                         ;\\r
+\r
+#define KERNEL4x2_SUB(xx) \\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vfmaddpd        %xmm8,%xmm2,%xmm0,%xmm8 ;\\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL2x2_1(xx) \\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+#define KERNEL2x2_2(xx) \\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+#define KERNEL2x2_3(xx) \\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+#define KERNEL2x2_4(xx) \\r
+       vmovddup          2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup          3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       addq    $8, BI                            ;\\r
+       addq    $8, %rax                          ;\\r
+\r
+#define KERNEL2x2_SUB(xx) \\r
+       vmovddup         -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovddup         -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddpd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL1x2_1(xx) \\r
+       vmovsd   -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd   -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+#define KERNEL1x2_2(xx) \\r
+       vmovsd   -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -15 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd   -1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+#define KERNEL1x2_3(xx) \\r
+       vmovsd    0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd    1 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+#define KERNEL1x2_4(xx) \\r
+       vmovsd    2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -13 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd    3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+       addq    $8, BI                            ;\\r
+       addq    $4, %rax                          ;\\r
+\r
+#define KERNEL1x2_SUB(xx) \\r
+       vmovsd   -4 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovsd   -3 * SIZE(BO, BI, 8), %xmm2 ;\\r
+       vfmaddsd        %xmm5,%xmm2,%xmm0,%xmm5 ;\\r
+\r
+\r
+\r
+/*******************************************************************************************\r
+* 1 line of N\r
+*******************************************************************************************/\r
+\r
+#define KERNEL8x1_1(xx) \\r
+       prefetcht0      A_PR1(AO,%rax,8)        ;\\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+\r
+#define KERNEL8x1_2(xx) \\r
+       prefetcht0      A_PR1+64(AO,%rax,8)     ;\\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups          -6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vmovups          -4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vmovups          -2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+\r
+#define KERNEL8x1_3(xx) \\r
+       prefetcht0      A_PR1+128(AO,%rax,8)    ;\\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups           0 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups           2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vmovups           4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vmovups           6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+\r
+#define KERNEL8x1_4(xx) \\r
+       prefetcht0      A_PR1+192(AO,%rax,8)    ;\\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups           8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups          10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vmovups          12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vmovups          14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+       addq    $4, BI                            ;\\r
+       addq    $32, %rax                         ;\\r
+\r
+#define KERNEL8x1_SUB(xx) \\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm10,%xmm1,%xmm0,%xmm10 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm13,%xmm1,%xmm0,%xmm13 ;\\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL4x1_1(xx) \\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+\r
+#define KERNEL4x1_2(xx) \\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+\r
+#define KERNEL4x1_3(xx) \\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -8 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups          -6 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+\r
+#define KERNEL4x1_4(xx) \\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups          -4 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups          -2 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+       addq    $4, BI                            ;\\r
+       addq    $16, %rax                         ;\\r
+\r
+#define KERNEL4x1_SUB(xx) \\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm7,%xmm1,%xmm0,%xmm7 ;\\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL2x1_1(xx) \\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+#define KERNEL2x1_2(xx) \\r
+       vmovddup         -1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+#define KERNEL2x1_3(xx) \\r
+       vmovddup          0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -12 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+#define KERNEL2x1_4(xx) \\r
+       vmovddup          1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -10 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       addq    $4, BI                            ;\\r
+       addq    $8, %rax                          ;\\r
+\r
+#define KERNEL2x1_SUB(xx) \\r
+       vmovddup         -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovups         -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddpd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+/*******************************************************************************************/\r
+\r
+#define KERNEL1x1_1(xx) \\r
+       vmovsd   -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+#define KERNEL1x1_2(xx) \\r
+       vmovsd   -1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -15 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+#define KERNEL1x1_3(xx) \\r
+       vmovsd    0 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -14 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+#define KERNEL1x1_4(xx) \\r
+       vmovsd    1 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -13 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+       addq    $4, BI                            ;\\r
+       addq    $4, %rax                          ;\\r
+\r
+#define KERNEL1x1_SUB(xx) \\r
+       vmovsd   -2 * SIZE(BO, BI, 8), %xmm1 ;\\r
+       vmovsd  -16 * SIZE(AO, %rax, 8), %xmm0 ;\\r
+       vfmaddsd        %xmm4,%xmm1,%xmm0,%xmm4 ;\\r
+\r
+\r
+/*******************************************************************************************/\r
+\r
+\r
+\r
+\r
+\r
+       PROLOGUE\r
+       PROFCODE\r
+       \r
+       subq    $STACKSIZE, %rsp\r
+       movq    %rbx,   (%rsp)\r
+       movq    %rbp,  8(%rsp)\r
+       movq    %r12, 16(%rsp)\r
+       movq    %r13, 24(%rsp)\r
+       movq    %r14, 32(%rsp)\r
+       movq    %r15, 40(%rsp)\r
+\r
+       vzeroupper\r
+\r
+#ifdef WINDOWS_ABI\r
+       movq    %rdi,    48(%rsp)\r
+       movq    %rsi,    56(%rsp)\r
+       movups  %xmm6,   64(%rsp)\r
+       movups  %xmm7,   80(%rsp)\r
+       movups  %xmm8,   96(%rsp)\r
+       movups  %xmm9,  112(%rsp)\r
+       movups  %xmm10, 128(%rsp)\r
+       movups  %xmm11, 144(%rsp)\r
+       movups  %xmm12, 160(%rsp)\r
+       movups  %xmm13, 176(%rsp)\r
+       movups  %xmm14, 192(%rsp)\r
+       movups  %xmm15, 208(%rsp)\r
+\r
+       movq    ARG1,      OLD_M\r
+       movq    ARG2,      OLD_N\r
+       movq    ARG3,      OLD_K\r
+       movq    OLD_A,     A\r
+       movq    OLD_B,     B\r
+       movq    OLD_C,     C\r
+       movq    OLD_LDC,   LDC\r
+#ifdef TRMMKERNEL\r
+       movsd   OLD_OFFSET, %xmm12\r
+#endif\r
+       vmovaps %xmm3, %xmm0\r
+\r
+#else\r
+       movq    STACKSIZE +  8(%rsp), LDC\r
+#ifdef TRMMKERNEL\r
+       movsd   STACKSIZE + 16(%rsp), %xmm12\r
+#endif\r
+\r
+#endif\r
+\r
+       movq    %rsp, SP      # save old stack\r
+        subq    $128 + L_BUFFER_SIZE, %rsp\r
+        andq    $-4096, %rsp    # align stack\r
+\r
+        STACK_TOUCHING\r
+\r
+       cmpq    $0, OLD_M\r
+       je      .L999\r
+\r
+       cmpq    $0, OLD_N\r
+       je      .L999\r
+\r
+       cmpq    $0, OLD_K\r
+       je      .L999\r
+\r
+       movq    OLD_M, M\r
+       movq    OLD_N, N\r
+       movq    OLD_K, K\r
+\r
+       vmovsd   %xmm0, ALPHA\r
+\r
+       salq    $BASE_SHIFT, LDC\r
+\r
+       movq    N, %rax\r
+        xorq    %rdx, %rdx\r
+        movq    $6,  %rdi\r
+        divq    %rdi                    //    N / 6\r
+        movq    %rax, Ndiv6             //    N / 6\r
+        movq    %rdx, Nmod6             //    N % 6\r
+\r
+       \r
+\r
+#ifdef TRMMKERNEL\r
+       vmovsd  %xmm12, OFFSET\r
+       vmovsd  %xmm12, KK\r
+#ifndef LEFT\r
+       negq    KK\r
+#endif \r
+#endif\r
+\r
+       movq    Ndiv6,  J\r
+       cmpq    $0, J\r
+       je      .L2_0\r
+       ALIGN_4\r
+\r
+.L6_01:\r
+        // copy to sub buffer\r
+        movq    K, %rax\r
+        salq    $1,%rax                 // K * 2\r
+        movq    B, BO1\r
+        leaq    (B,%rax,8), BO2         // next offset to BO2\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        movq    K, %rax\r
+        sarq    $2, %rax                // K / 4\r
+        jz      .L6_02a\r
+        ALIGN_4\r
+\r
+.L6_02:\r
+       prefetcht0 512(BO1)\r
+       prefetcht0 512(BO2)\r
+       prefetchw  512(BO)\r
+       vmovups       (BO1), %xmm0\r
+       vmovups 2*SIZE(BO1), %xmm2\r
+       vmovups 4*SIZE(BO1), %xmm4\r
+       vmovups 6*SIZE(BO1), %xmm6\r
+       vmovsd        (BO2), %xmm1\r
+       vmovsd  2*SIZE(BO2), %xmm3\r
+       vmovsd  4*SIZE(BO2), %xmm5\r
+       vmovsd  6*SIZE(BO2), %xmm7\r
+       vmovups %xmm0,       (BO)\r
+       vmovsd  %xmm1, 2*SIZE(BO)\r
+       vmovups %xmm2, 3*SIZE(BO)\r
+       vmovsd  %xmm3, 5*SIZE(BO)\r
+       vmovups %xmm4, 6*SIZE(BO)\r
+       vmovsd  %xmm5, 8*SIZE(BO)\r
+       vmovups %xmm6, 9*SIZE(BO)\r
+       vmovsd  %xmm7,11*SIZE(BO)\r
+       addq    $8*SIZE,BO1\r
+       addq    $8*SIZE,BO2\r
+       addq    $12*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L6_02\r
+\r
+.L6_02a:\r
+\r
+       movq    K, %rax\r
+       andq    $3, %rax                // K % 4\r
+       jz      .L6_02c\r
+       ALIGN_4\r
+\r
+.L6_02b:\r
+\r
+       vmovups (BO1), %xmm0\r
+       vmovsd  (BO2), %xmm1\r
+       vmovups %xmm0,       (BO)\r
+       vmovsd  %xmm1, 2*SIZE(BO)\r
+       addq    $2*SIZE,BO1\r
+       addq    $2*SIZE,BO2\r
+       addq    $3*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L6_02b\r
+\r
+.L6_02c:\r
+\r
+       movq    K, %rax\r
+       salq    $1,%rax                 // K * 2\r
+       leaq    (B,%rax,8), BO1         // next offset to BO1\r
+       leaq    (BO1,%rax,8), BO2       // next offset to BO1\r
+       leaq    BUFFER2, BO             // second buffer to BO\r
+       movq    K, %rax\r
+       sarq    $2, %rax                // k / 4\r
+       jz      .L6_03a\r
+       ALIGN_4\r
+\r
+\r
+.L6_03:\r
+\r
+       prefetcht0 512(BO2)\r
+       prefetchw  512(BO)\r
+       vmovups       (BO2), %xmm0\r
+       vmovups 2*SIZE(BO2), %xmm2\r
+       vmovups 4*SIZE(BO2), %xmm4\r
+       vmovups 6*SIZE(BO2), %xmm6\r
+       vmovsd  1*SIZE(BO1), %xmm1\r
+       vmovsd  3*SIZE(BO1), %xmm3\r
+       vmovsd  5*SIZE(BO1), %xmm5\r
+       vmovsd  7*SIZE(BO1), %xmm7\r
+       vmovsd  %xmm1, 0*SIZE(BO)\r
+       vmovups %xmm0, 1*SIZE(BO)\r
+       vmovsd  %xmm3, 3*SIZE(BO)\r
+       vmovups %xmm2, 4*SIZE(BO)\r
+       vmovsd  %xmm5, 6*SIZE(BO)\r
+       vmovups %xmm4, 7*SIZE(BO)\r
+       vmovsd  %xmm7, 9*SIZE(BO)\r
+       vmovups %xmm6,10*SIZE(BO)\r
+       addq    $8*SIZE,BO1\r
+       addq    $8*SIZE,BO2\r
+       addq    $12*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L6_03\r
+\r
+.L6_03a:\r
+\r
+        movq    K, %rax\r
+        andq    $3, %rax                // K % 4\r
+        jz      .L6_03c\r
+        ALIGN_4\r
+\r
+\r
+.L6_03b:\r
+\r
+       vmovsd    1*SIZE(BO1), %xmm0\r
+       vmovups         (BO2), %xmm1\r
+       vmovsd  %xmm0,       (BO)\r
+       vmovups %xmm1, 1*SIZE(BO)\r
+       addq    $2*SIZE,BO1\r
+       addq    $2*SIZE,BO2\r
+       addq    $3*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L6_03b\r
+\r
+\r
+.L6_03c:\r
+\r
+       movq    BO2, B                  // next offset of B\r
+\r
+.L6_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 2), C           \r
+       leaq    (C, LDC, 1), C          // c += 3 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+       \r
+       movq    A, AO                   // aoffset = a\r
+       addq    $16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $3, I                   // i = (m >> 3)\r
+       je      .L6_20\r
+\r
+       ALIGN_4\r
+\r
+.L6_11:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $8, %rax       // number of values in AO\r
+#else\r
+        addq    $3, %rax       // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L6_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_12:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       KERNEL8x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL8x3_3(xxx)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       je      .L6_16\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       KERNEL8x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL8x3_3(xxx)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       je      .L6_16\r
+\r
+       jmp     .L6_12\r
+       ALIGN_4\r
+\r
+.L6_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_17:\r
+\r
+       KERNEL8x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $8, %rax\r
+       jl      .L6_17\r
+       ALIGN_4\r
+\r
+\r
+.L6_19:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+       vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10\r
+       vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13\r
+\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8\r
+       vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11\r
+       vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14\r
+\r
+       vfmaddpd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+       vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9\r
+       vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12\r
+       vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+       vmulpd  %xmm0, %xmm10,%xmm10\r
+       vmulpd  %xmm0, %xmm13,%xmm13\r
+\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm8,%xmm8\r
+       vmulpd  %xmm0, %xmm11,%xmm11\r
+       vmulpd  %xmm0, %xmm14,%xmm14\r
+\r
+       vmulpd  %xmm0, %xmm6,%xmm6\r
+       vmulpd  %xmm0, %xmm9,%xmm9\r
+       vmulpd  %xmm0, %xmm12,%xmm12\r
+       vmulpd  %xmm0, %xmm15,%xmm15\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+       vmovups %xmm10, 4 * SIZE(CO1)\r
+       vmovups %xmm13, 6 * SIZE(CO1)\r
+\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm8 , 2 * SIZE(CO1, LDC)\r
+       vmovups %xmm11, 4 * SIZE(CO1, LDC)\r
+       vmovups %xmm14, 6 * SIZE(CO1, LDC)\r
+\r
+       vmovups %xmm6 ,         (CO1, LDC, 2)\r
+       vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2)\r
+       vmovups %xmm12, 4 * SIZE(CO1, LDC, 2)\r
+       vmovups %xmm15, 6 * SIZE(CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $8, KK\r
+#endif\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       decq    I                       # i --\r
+       jg      .L6_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L6_20:\r
+       // Test rest of M\r
+\r
+       testq   $7, M\r
+       jz      .L7_10          // to next 3 lines of N\r
+\r
+       testq   $4, M           \r
+       jz      .L6_30\r
+\r
+       ALIGN_4\r
+\r
+.L6_21:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $4, %rax        // number of values in A\r
+#else\r
+        addq    $3, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L6_26\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_22:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       KERNEL4x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL4x3_3(xxx)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       je      .L6_26\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       KERNEL4x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL4x3_3(xxx)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       je      .L6_26\r
+\r
+       jmp     .L6_22\r
+       ALIGN_4\r
+\r
+.L6_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_27:\r
+\r
+       KERNEL4x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $4, %rax\r
+       jl      .L6_27\r
+       ALIGN_4\r
+\r
+\r
+.L6_29:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8\r
+\r
+       vfmaddpd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+       vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm8,%xmm8\r
+\r
+       vmulpd  %xmm0, %xmm6,%xmm6\r
+       vmulpd  %xmm0, %xmm9,%xmm9\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm8 , 2 * SIZE(CO1, LDC)\r
+\r
+       vmovups %xmm6 ,         (CO1, LDC, 2)\r
+       vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO         \r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L6_30:\r
+       testq   $2, M           \r
+       jz      .L6_40\r
+\r
+       ALIGN_4\r
+\r
+.L6_31:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $2, %rax        // number of values in AO\r
+#else\r
+        addq    $3, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L6_36\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_32:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       KERNEL2x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL2x3_3(xxx)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       je      .L6_36\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       KERNEL2x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL2x3_3(xxx)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       je      .L6_36\r
+\r
+       jmp     .L6_32\r
+       ALIGN_4\r
+\r
+.L6_36:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_39\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+       \r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_37:\r
+\r
+       KERNEL2x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $2, %rax\r
+       jl      .L6_37\r
+       ALIGN_4\r
+\r
+\r
+.L6_39:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm6,%xmm6\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm6 ,         (CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO         \r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $2, KK\r
+#endif\r
+\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       ALIGN_4\r
+\r
+.L6_40:\r
+       testq   $1, M           \r
+       jz      .L7_10          // to next 3 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L6_41:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $1, %rax        // number of values in AO\r
+#else\r
+        addq    $3, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax\r
+       je      .L6_46\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_42:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       KERNEL1x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL1x3_3(xxx)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       je      .L6_46\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       KERNEL1x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL1x3_3(xxx)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       je      .L6_46\r
+\r
+       jmp     .L6_42\r
+       ALIGN_4\r
+\r
+.L6_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L6_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L6_47:\r
+\r
+       KERNEL1x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $1, %rax\r
+       jl      .L6_47\r
+       ALIGN_4\r
+\r
+\r
+.L6_49:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddsd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddsd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddsd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+\r
+#else\r
+       vmulsd  %xmm0, %xmm4,%xmm4\r
+       vmulsd  %xmm0, %xmm5,%xmm5\r
+       vmulsd  %xmm0, %xmm6,%xmm6\r
+\r
+#endif\r
+\r
+       vmovsd  %xmm4 ,         (CO1)\r
+       vmovsd  %xmm5 ,         (CO1, LDC)\r
+       vmovsd  %xmm6 ,         (CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO         \r
+        leaq    (AO, %rax, 8), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $1, KK\r
+#endif\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       ALIGN_4\r
+       \r
+\r
+\r
+\r
+/***************************************************************************************************************/\r
+\r
+.L7_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 2), C           \r
+       leaq    (C, LDC, 1), C          // c += 3 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+\r
+       movq    A, AO                   // aoffset = a\r
+       addq    $16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $3, I                   // i = (m >> 3)\r
+       je      .L7_20\r
+       ALIGN_4\r
+\r
+.L7_11:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $8, %rax        // number of values in AO\r
+#else\r
+        addq    $3, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+\r
+       ALIGN_4\r
+\r
+.L7_12:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       KERNEL8x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL8x3_3(xxx)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       je      .L7_16\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       KERNEL8x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       KERNEL8x3_1(xxx)\r
+       KERNEL8x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL8x3_3(xxx)\r
+       KERNEL8x3_4(xxx)\r
+\r
+       je      .L7_16\r
+\r
+       jmp     .L7_12\r
+       ALIGN_4\r
+\r
+.L7_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L7_17:\r
+\r
+       KERNEL8x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $8, %rax\r
+       jl      .L7_17\r
+       ALIGN_4\r
+\r
+\r
+.L7_19:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+       vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10\r
+       vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13\r
+\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8\r
+       vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11\r
+       vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14\r
+\r
+       vfmaddpd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+       vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9\r
+       vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12\r
+       vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+       vmulpd  %xmm0, %xmm10,%xmm10\r
+       vmulpd  %xmm0, %xmm13,%xmm13\r
+\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm8,%xmm8\r
+       vmulpd  %xmm0, %xmm11,%xmm11\r
+       vmulpd  %xmm0, %xmm14,%xmm14\r
+\r
+       vmulpd  %xmm0, %xmm6,%xmm6\r
+       vmulpd  %xmm0, %xmm9,%xmm9\r
+       vmulpd  %xmm0, %xmm12,%xmm12\r
+       vmulpd  %xmm0, %xmm15,%xmm15\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+       vmovups %xmm10, 4 * SIZE(CO1)\r
+       vmovups %xmm13, 6 * SIZE(CO1)\r
+\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm8 , 2 * SIZE(CO1, LDC)\r
+       vmovups %xmm11, 4 * SIZE(CO1, LDC)\r
+       vmovups %xmm14, 6 * SIZE(CO1, LDC)\r
+\r
+       vmovups %xmm6 ,         (CO1, LDC, 2)\r
+       vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2)\r
+       vmovups %xmm12, 4 * SIZE(CO1, LDC, 2)\r
+       vmovups %xmm15, 6 * SIZE(CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO         \r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $8, KK\r
+#endif\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       decq    I                       # i --\r
+       jg      .L7_11\r
+       ALIGN_4 \r
+\r
+.L7_20:\r
+       // Test rest of M\r
+\r
+       testq   $7, M\r
+       jz      .L7_60          // to next 6 lines of N\r
+\r
+       testq   $4, M           \r
+       jz      .L7_30\r
+\r
+       ALIGN_4\r
+\r
+.L7_21:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $4, %rax        // number of values in AO\r
+#else\r
+        addq    $3, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_26\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L7_22:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       KERNEL4x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL4x3_3(xxx)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       je      .L7_26\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       KERNEL4x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       KERNEL4x3_1(xxx)\r
+       KERNEL4x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL4x3_3(xxx)\r
+       KERNEL4x3_4(xxx)\r
+\r
+       je      .L7_26\r
+\r
+       jmp     .L7_22\r
+       ALIGN_4\r
+\r
+.L7_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L7_27:\r
+\r
+       KERNEL4x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $4, %rax\r
+       jl      .L7_27\r
+       ALIGN_4\r
+\r
+\r
+.L7_29:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8\r
+\r
+       vfmaddpd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+       vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm8,%xmm8\r
+\r
+       vmulpd  %xmm0, %xmm6,%xmm6\r
+       vmulpd  %xmm0, %xmm9,%xmm9\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm8 , 2 * SIZE(CO1, LDC)\r
+\r
+       vmovups %xmm6 ,         (CO1, LDC, 2)\r
+       vmovups %xmm9 , 2 * SIZE(CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO         \r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L7_30:\r
+       testq   $2, M           \r
+       jz      .L7_40\r
+\r
+       ALIGN_4\r
+\r
+.L7_31:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $2, %rax        // number of values in AO\r
+#else\r
+        addq    $3, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_36\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L7_32:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       KERNEL2x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL2x3_3(xxx)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       je      .L7_36\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       KERNEL2x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       KERNEL2x3_1(xxx)\r
+       KERNEL2x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL2x3_3(xxx)\r
+       KERNEL2x3_4(xxx)\r
+\r
+       je      .L7_36\r
+\r
+       jmp     .L7_32\r
+       ALIGN_4\r
+\r
+.L7_36:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_39\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L7_37:\r
+\r
+       KERNEL2x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $2, %rax\r
+       jl      .L7_37\r
+       ALIGN_4\r
+\r
+\r
+.L7_39:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm6,%xmm6\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm6 ,         (CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO         \r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $2, KK\r
+#endif\r
+\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       ALIGN_4\r
+       \r
+\r
+\r
+\r
+\r
+.L7_40:\r
+       testq   $1, M           \r
+       jz      .L7_60          // to next 6 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L7_41:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER2, BO             // second buffer to BO\r
+        addq    $6 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $1, %rax        // number of values in AO\r
+#else\r
+        addq    $3, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L7_46\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L7_42:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       KERNEL1x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL1x3_3(xxx)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       je      .L7_46\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       KERNEL1x3_3(xxx)\r
+       prefetcht0      B_PR1+64(BO,BI,8)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       KERNEL1x3_1(xxx)\r
+       KERNEL1x3_2(xxx)\r
+       prefetcht0      B_PR1+32(BO,BI,8)\r
+       KERNEL1x3_3(xxx)\r
+       KERNEL1x3_4(xxx)\r
+\r
+       je      .L7_46\r
+\r
+       jmp     .L7_42\r
+       ALIGN_4\r
+\r
+.L7_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L7_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L7_47:\r
+\r
+       KERNEL1x3_SUB(xxx)\r
+       addq    $3, BI\r
+       addq    $1, %rax\r
+       jl      .L7_47\r
+       ALIGN_4\r
+\r
+\r
+.L7_49:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddsd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddsd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddsd         (CO1, LDC, 2),%xmm0, %xmm6,%xmm6\r
+\r
+#else\r
+       vmulsd  %xmm0, %xmm4,%xmm4\r
+       vmulsd  %xmm0, %xmm5,%xmm5\r
+       vmulsd  %xmm0, %xmm6,%xmm6\r
+\r
+#endif\r
+\r
+       vmovsd  %xmm4 ,         (CO1)\r
+       vmovsd  %xmm5 ,         (CO1, LDC)\r
+       vmovsd  %xmm6 ,         (CO1, LDC, 2)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values\r
+        leaq    (BO, BI, 8), BO         \r
+        leaq    (AO, %rax, 8), AO\r
+#endif  \r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $1, KK\r
+#endif\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+\r
+.L7_60:\r
+\r
+       decq    J                       // j --\r
+       jg      .L6_01\r
+\r
+\r
+.L2_0:\r
+       cmpq    $0, Nmod6               // N % 6 == 0\r
+       je      .L999\r
+\r
+/************************************************************************************************\r
+* Loop for Nmod6 / 2 > 0\r
+*************************************************************************************************/\r
+\r
+       movq    Nmod6, J                \r
+       sarq    $1, J                   // j = j / 2\r
+       je      .L1_0\r
+       ALIGN_4\r
+\r
+.L2_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       ALIGN_4\r
+\r
+.L2_02b:\r
+\r
+       vmovups (BO1), %xmm0\r
+       vmovups %xmm0,       (BO)\r
+       addq    $2*SIZE,BO1\r
+       addq    $2*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L2_02b\r
+\r
+.L2_02c:\r
+\r
+       movq    BO1, B                  // next offset of B\r
+\r
+.L2_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 2), C          // c += 2 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+       \r
+       movq    A, AO                   // aoffset = a\r
+       addq    $16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $3, I                   // i = (m >> 3)\r
+       je      .L2_20\r
+\r
+       ALIGN_4\r
+\r
+.L2_11:\r
+\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       addq    $4 * SIZE, BO\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $8, %rax        // number of values in AO\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L2_16\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_12:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x2_1(xxx)\r
+       KERNEL8x2_2(xxx)\r
+       KERNEL8x2_3(xxx)\r
+       KERNEL8x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x2_1(xxx)\r
+       KERNEL8x2_2(xxx)\r
+       KERNEL8x2_3(xxx)\r
+       KERNEL8x2_4(xxx)\r
+\r
+       je      .L2_16\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x2_1(xxx)\r
+       KERNEL8x2_2(xxx)\r
+       KERNEL8x2_3(xxx)\r
+       KERNEL8x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x2_1(xxx)\r
+       KERNEL8x2_2(xxx)\r
+       KERNEL8x2_3(xxx)\r
+       KERNEL8x2_4(xxx)\r
+\r
+       je      .L2_16\r
+\r
+       jmp     .L2_12\r
+       ALIGN_4\r
+\r
+.L2_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_17:\r
+\r
+       KERNEL8x2_SUB(xxx)\r
+       addq    $2, BI\r
+       addq    $8, %rax\r
+       jl      .L2_17\r
+       ALIGN_4\r
+\r
+\r
+.L2_19:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+       vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10\r
+       vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13\r
+\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8\r
+       vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11\r
+       vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14\r
+\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+       vmulpd  %xmm0, %xmm10,%xmm10\r
+       vmulpd  %xmm0, %xmm13,%xmm13\r
+\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm8,%xmm8\r
+       vmulpd  %xmm0, %xmm11,%xmm11\r
+       vmulpd  %xmm0, %xmm14,%xmm14\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+       vmovups %xmm10, 4 * SIZE(CO1)\r
+       vmovups %xmm13, 6 * SIZE(CO1)\r
+\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm8 , 2 * SIZE(CO1, LDC)\r
+       vmovups %xmm11, 4 * SIZE(CO1, LDC)\r
+       vmovups %xmm14, 6 * SIZE(CO1, LDC)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $8, KK\r
+#endif\r
+\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       decq    I                       # i --\r
+       jg      .L2_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L2_20:\r
+       // Test rest of M\r
+\r
+       testq   $7, M\r
+       jz      .L2_60          // to next 2 lines of N\r
+\r
+       testq   $4, M           \r
+       jz      .L2_30\r
+\r
+       ALIGN_4\r
+\r
+.L2_21:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $4, %rax        // number of values in AO\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L2_26\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_22:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x2_1(xxx)\r
+       KERNEL4x2_2(xxx)\r
+       KERNEL4x2_3(xxx)\r
+       KERNEL4x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x2_1(xxx)\r
+       KERNEL4x2_2(xxx)\r
+       KERNEL4x2_3(xxx)\r
+       KERNEL4x2_4(xxx)\r
+\r
+       je      .L2_26\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x2_1(xxx)\r
+       KERNEL4x2_2(xxx)\r
+       KERNEL4x2_3(xxx)\r
+       KERNEL4x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x2_1(xxx)\r
+       KERNEL4x2_2(xxx)\r
+       KERNEL4x2_3(xxx)\r
+       KERNEL4x2_4(xxx)\r
+\r
+       je      .L2_26\r
+\r
+       jmp     .L2_22\r
+       ALIGN_4\r
+\r
+.L2_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_27:\r
+\r
+       KERNEL4x2_SUB(xxx)\r
+       addq    $2, BI\r
+       addq    $4, %rax\r
+       jl      .L2_27\r
+       ALIGN_4\r
+\r
+\r
+.L2_29:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+       vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8\r
+\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+       vmulpd  %xmm0, %xmm8,%xmm8\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+       vmovups %xmm8 , 2 * SIZE(CO1, LDC)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L2_30:\r
+       testq   $2, M           \r
+       jz      .L2_40\r
+\r
+       ALIGN_4\r
+\r
+.L2_31:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $2, %rax        // number of values in AO\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L2_36\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_32:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x2_1(xxx)\r
+       KERNEL2x2_2(xxx)\r
+       KERNEL2x2_3(xxx)\r
+       KERNEL2x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x2_1(xxx)\r
+       KERNEL2x2_2(xxx)\r
+       KERNEL2x2_3(xxx)\r
+       KERNEL2x2_4(xxx)\r
+\r
+       je      .L2_36\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x2_1(xxx)\r
+       KERNEL2x2_2(xxx)\r
+       KERNEL2x2_3(xxx)\r
+       KERNEL2x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x2_1(xxx)\r
+       KERNEL2x2_2(xxx)\r
+       KERNEL2x2_3(xxx)\r
+       KERNEL2x2_4(xxx)\r
+\r
+       je      .L2_36\r
+\r
+       jmp     .L2_32\r
+       ALIGN_4\r
+\r
+.L2_36:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_39\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+       \r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_37:\r
+\r
+       KERNEL2x2_SUB(xxx)\r
+       addq    $2, BI\r
+       addq    $2, %rax\r
+       jl      .L2_37\r
+       ALIGN_4\r
+\r
+\r
+.L2_39:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm5,%xmm5\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm5 ,         (CO1, LDC)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax \r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $2, KK\r
+#endif\r
+\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       ALIGN_4\r
+       \r
+\r
+.L2_40:\r
+       testq   $1, M           \r
+       jz      .L2_60          // to next 2 lines of N\r
+\r
+       ALIGN_4\r
+\r
+.L2_41:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $4 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $1, %rax        // number of values in AO\r
+#else\r
+        addq    $2, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax\r
+       je      .L2_46\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_42:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x2_1(xxx)\r
+       KERNEL1x2_2(xxx)\r
+       KERNEL1x2_3(xxx)\r
+       KERNEL1x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x2_1(xxx)\r
+       KERNEL1x2_2(xxx)\r
+       KERNEL1x2_3(xxx)\r
+       KERNEL1x2_4(xxx)\r
+\r
+       je      .L2_46\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x2_1(xxx)\r
+       KERNEL1x2_2(xxx)\r
+       KERNEL1x2_3(xxx)\r
+       KERNEL1x2_4(xxx)\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x2_1(xxx)\r
+       KERNEL1x2_2(xxx)\r
+       KERNEL1x2_3(xxx)\r
+       KERNEL1x2_4(xxx)\r
+\r
+       je      .L2_46\r
+\r
+       jmp     .L2_42\r
+       ALIGN_4\r
+\r
+.L2_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L2_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L2_47:\r
+\r
+       KERNEL1x2_SUB(xxx)\r
+       addq    $2, BI\r
+       addq    $1, %rax\r
+       jl      .L2_47\r
+       ALIGN_4\r
+\r
+\r
+.L2_49:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddsd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddsd         (CO1, LDC),%xmm0, %xmm5,%xmm5\r
+\r
+#else\r
+       vmulsd  %xmm0, %xmm4,%xmm4\r
+       vmulsd  %xmm0, %xmm5,%xmm5\r
+\r
+#endif\r
+\r
+       vmovsd  %xmm4 ,         (CO1)\r
+       vmovsd  %xmm5 ,         (CO1, LDC)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values\r
+        leaq    (BO, BI, 8), BO\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $1, KK\r
+#endif\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       ALIGN_4\r
+       \r
+\r
+       \r
+.L2_60:\r
+\r
+       decq    J                       // j --\r
+       jg      .L2_01                  // next 2 lines of N\r
+\r
+\r
+\r
+.L1_0:\r
+\r
+/************************************************************************************************\r
+* Loop for Nmod6 % 2 > 0\r
+*************************************************************************************************/\r
+\r
+       movq    Nmod6, J                \r
+       andq    $1, J                   // j % 2\r
+       je      .L999\r
+       ALIGN_4\r
+\r
+.L1_01:\r
+       // copy to sub buffer\r
+       movq    B, BO1\r
+       leaq    BUFFER1, BO             // first buffer to BO\r
+       movq    K, %rax\r
+       ALIGN_4\r
+\r
+.L1_02b:\r
+\r
+       vmovsd  (BO1), %xmm0\r
+       vmovsd  %xmm0,       (BO)\r
+       addq    $1*SIZE,BO1\r
+       addq    $1*SIZE,BO\r
+       decq    %rax\r
+       jnz     .L1_02b\r
+\r
+.L1_02c:\r
+\r
+       movq    BO1, B                  // next offset of B\r
+\r
+.L1_10:\r
+       movq    C, CO1\r
+       leaq    (C, LDC, 1), C          // c += 1 * ldc\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        movq    OFFSET, %rax\r
+        movq    %rax, KK\r
+#endif\r
+\r
+       movq    A, AO                   // aoffset = a\r
+       addq    $16 * SIZE, AO\r
+\r
+       movq    M,  I\r
+       sarq    $3, I                   // i = (m >> 3)\r
+       je      .L1_20\r
+\r
+       ALIGN_4\r
+\r
+.L1_11:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $8, %rax        // number of values in AO\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax                       //  K = K - ( K % 8 )\r
+       je      .L1_16\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_12:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x1_1(xxx)\r
+       KERNEL8x1_2(xxx)\r
+       KERNEL8x1_3(xxx)\r
+       KERNEL8x1_4(xxx)\r
+\r
+       KERNEL8x1_1(xxx)\r
+       KERNEL8x1_2(xxx)\r
+       KERNEL8x1_3(xxx)\r
+       KERNEL8x1_4(xxx)\r
+\r
+       je      .L1_16\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL8x1_1(xxx)\r
+       KERNEL8x1_2(xxx)\r
+       KERNEL8x1_3(xxx)\r
+       KERNEL8x1_4(xxx)\r
+\r
+       KERNEL8x1_1(xxx)\r
+       KERNEL8x1_2(xxx)\r
+       KERNEL8x1_3(xxx)\r
+       KERNEL8x1_4(xxx)\r
+\r
+       je      .L1_16\r
+\r
+       jmp     .L1_12\r
+       ALIGN_4\r
+\r
+.L1_16:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_19\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_17:\r
+\r
+       KERNEL8x1_SUB(xxx)\r
+       addq    $1, BI\r
+       addq    $8, %rax\r
+       jl      .L1_17\r
+       ALIGN_4\r
+\r
+\r
+.L1_19:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+       vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10\r
+       vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+       vmulpd  %xmm0, %xmm10,%xmm10\r
+       vmulpd  %xmm0, %xmm13,%xmm13\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+       vmovups %xmm10, 4 * SIZE(CO1)\r
+       vmovups %xmm13, 6 * SIZE(CO1)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $3, %rax                        // rax = rax * 8 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $8, KK\r
+#endif\r
+       addq    $8 * SIZE, CO1          # coffset += 8\r
+       decq    I                       # i --\r
+       jg      .L1_11\r
+       ALIGN_4 \r
+\r
+/**************************************************************************\r
+* Rest of M \r
+***************************************************************************/\r
+.L1_20:\r
+       // Test rest of M\r
+\r
+       testq   $7, M\r
+       jz      .L999\r
+\r
+       testq   $4, M           \r
+       jz      .L1_30\r
+\r
+       ALIGN_4\r
+\r
+.L1_21:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $4, %rax        // number of values in AO\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L1_26\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_22:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x1_1(xxx)\r
+       KERNEL4x1_2(xxx)\r
+       KERNEL4x1_3(xxx)\r
+       KERNEL4x1_4(xxx)\r
+\r
+       KERNEL4x1_1(xxx)\r
+       KERNEL4x1_2(xxx)\r
+       KERNEL4x1_3(xxx)\r
+       KERNEL4x1_4(xxx)\r
+\r
+       je      .L1_26\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL4x1_1(xxx)\r
+       KERNEL4x1_2(xxx)\r
+       KERNEL4x1_3(xxx)\r
+       KERNEL4x1_4(xxx)\r
+\r
+       KERNEL4x1_1(xxx)\r
+       KERNEL4x1_2(xxx)\r
+       KERNEL4x1_3(xxx)\r
+       KERNEL4x1_4(xxx)\r
+\r
+       je      .L1_26\r
+\r
+       jmp     .L1_22\r
+       ALIGN_4\r
+\r
+.L1_26:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_29\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_27:\r
+\r
+       KERNEL4x1_SUB(xxx)\r
+       addq    $1, BI\r
+       addq    $4, %rax\r
+       jl      .L1_27\r
+       ALIGN_4\r
+\r
+\r
+.L1_29:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+       vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+       vmulpd  %xmm0, %xmm7,%xmm7\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+       vmovups %xmm7 , 2 * SIZE(CO1)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $2, %rax                        // rax = rax * 4 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $4, KK\r
+#endif\r
+\r
+       addq    $4 * SIZE, CO1          # coffset += 4\r
+       ALIGN_4\r
+       \r
+\r
+.L1_30:\r
+       testq   $2, M           \r
+       jz      .L1_40\r
+\r
+       ALIGN_4\r
+\r
+.L1_31:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $2, %rax        // number of values in AO\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+\r
+       andq    $-8, %rax\r
+       je      .L1_36\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_32:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL2x1_1(xxx)\r
+       KERNEL2x1_2(xxx)\r
+       KERNEL2x1_3(xxx)\r
+       KERNEL2x1_4(xxx)\r
+\r
+       KERNEL2x1_1(xxx)\r
+       KERNEL2x1_2(xxx)\r
+       KERNEL2x1_3(xxx)\r
+       KERNEL2x1_4(xxx)\r
+\r
+       je      .L1_36\r
+\r
+       KERNEL2x1_1(xxx)\r
+       KERNEL2x1_2(xxx)\r
+       KERNEL2x1_3(xxx)\r
+       KERNEL2x1_4(xxx)\r
+\r
+       KERNEL2x1_1(xxx)\r
+       KERNEL2x1_2(xxx)\r
+       KERNEL2x1_3(xxx)\r
+       KERNEL2x1_4(xxx)\r
+\r
+       je      .L1_36\r
+\r
+       jmp     .L1_32\r
+       ALIGN_4\r
+\r
+.L1_36:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_39\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+       \r
+       salq    $1, %rax                        // rax = rax *2 ; number of values\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_37:\r
+\r
+       KERNEL2x1_SUB(xxx)\r
+       addq    $1, BI\r
+       addq    $2, %rax\r
+       jl      .L1_37\r
+       ALIGN_4\r
+\r
+\r
+.L1_39:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddpd         (CO1),%xmm0, %xmm4,%xmm4\r
+\r
+#else\r
+       vmulpd  %xmm0, %xmm4,%xmm4\r
+\r
+#endif\r
+\r
+       vmovups %xmm4 ,         (CO1)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        salq    $1, %rax                        // rax = rax * 2 ; number of values\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $2, KK\r
+#endif\r
+       addq    $2 * SIZE, CO1          # coffset += 2\r
+       ALIGN_4\r
+       \r
+\r
+.L1_40:\r
+       testq   $1, M           \r
+       jz      .L999\r
+\r
+       ALIGN_4\r
+\r
+.L1_41:\r
+#if !defined(TRMMKERNEL) || \\r
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+#else\r
+        movq    KK, %rax\r
+        leaq    BUFFER1, BO             // first buffer to BO\r
+        addq    $2 * SIZE, BO\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+       vzeroall\r
+\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KK, %rax\r
+        movq    %rax, KKK\r
+#else\r
+        movq    KK, %rax\r
+#ifdef LEFT\r
+        addq    $1, %rax        // number of values in AO\r
+#else\r
+        addq    $1, %rax        // number of values in BO\r
+#endif\r
+        movq    %rax, KKK\r
+#endif\r
+\r
+       andq    $-8, %rax\r
+       je      .L1_46\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_42:\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x1_1(xxx)\r
+       KERNEL1x1_2(xxx)\r
+       KERNEL1x1_3(xxx)\r
+       KERNEL1x1_4(xxx)\r
+\r
+       KERNEL1x1_1(xxx)\r
+       KERNEL1x1_2(xxx)\r
+       KERNEL1x1_3(xxx)\r
+       KERNEL1x1_4(xxx)\r
+\r
+       je      .L1_46\r
+\r
+       prefetcht0      B_PR1(BO,BI,8)\r
+       KERNEL1x1_1(xxx)\r
+       KERNEL1x1_2(xxx)\r
+       KERNEL1x1_3(xxx)\r
+       KERNEL1x1_4(xxx)\r
+\r
+       KERNEL1x1_1(xxx)\r
+       KERNEL1x1_2(xxx)\r
+       KERNEL1x1_3(xxx)\r
+       KERNEL1x1_4(xxx)\r
+\r
+       je      .L1_46\r
+\r
+       jmp     .L1_42\r
+       ALIGN_4\r
+\r
+.L1_46:\r
+#ifndef TRMMKERNEL\r
+        movq    K, %rax\r
+#else\r
+        movq    KKK, %rax\r
+#endif\r
+\r
+       andq    $7, %rax                # if (k & 1)\r
+       je .L1_49\r
+\r
+       movq    %rax, BI                        //  Index for BO\r
+\r
+       leaq    (AO, %rax, 8), AO\r
+       leaq    (BO, BI, 8), BO\r
+       negq    BI\r
+       negq    %rax\r
+       ALIGN_4\r
+\r
+.L1_47:\r
+\r
+       KERNEL1x1_SUB(xxx)\r
+       addq    $1, BI\r
+       addq    $1, %rax\r
+       jl      .L1_47\r
+       ALIGN_4\r
+\r
+\r
+.L1_49:\r
+\r
+       vmovddup        ALPHA, %xmm0\r
+\r
+#ifndef TRMMKERNEL\r
+\r
+       vfmaddsd         (CO1),%xmm0, %xmm4,%xmm4\r
+\r
+#else\r
+       vmulsd  %xmm0, %xmm4,%xmm4\r
+\r
+#endif\r
+\r
+       vmovsd  %xmm4 ,         (CO1)\r
+\r
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \\r
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))\r
+        movq    K, %rax\r
+        subq    KKK, %rax\r
+        movq    %rax, BI                        //  Index for BO\r
+        leaq    (BO, BI, 8), BO\r
+        leaq    (AO, %rax, 8), AO\r
+#endif\r
+\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+        addq    $1, KK\r
+#endif\r
+\r
+       addq    $1 * SIZE, CO1          # coffset += 1\r
+       ALIGN_4\r
+       \r
+\r
+.L999:\r
+       movq            SP, %rsp\r
+       movq       (%rsp), %rbx\r
+       movq      8(%rsp), %rbp\r
+       movq     16(%rsp), %r12\r
+       movq     24(%rsp), %r13\r
+       movq     32(%rsp), %r14\r
+       movq     40(%rsp), %r15\r
+\r
+#ifdef WINDOWS_ABI\r
+       movq     48(%rsp), %rdi\r
+       movq     56(%rsp), %rsi\r
+       movups   64(%rsp), %xmm6\r
+       movups   80(%rsp), %xmm7\r
+       movups   96(%rsp), %xmm8\r
+       movups  112(%rsp), %xmm9\r
+       movups  128(%rsp), %xmm10\r
+       movups  144(%rsp), %xmm11\r
+       movups  160(%rsp), %xmm12\r
+       movups  176(%rsp), %xmm13\r
+       movups  192(%rsp), %xmm14\r
+       movups  208(%rsp), %xmm15\r
+#endif\r
+\r
+       addq    $STACKSIZE, %rsp\r
+       ret\r
+\r
+       EPILOGUE\r

diff --git a/param.h b/param.h
index d6c3a9b..95b8a69 100644 (file)
--- a/param.h
+++ b/param.h
@@ -153,7 +153,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GEMM_DEFAULT_ALIGN 0x0fffUL
 
 #define SGEMM_DEFAULT_UNROLL_N 4
+#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX)
+#define DGEMM_DEFAULT_UNROLL_N 2
+#else
 #define DGEMM_DEFAULT_UNROLL_N 4
+#endif
 #define QGEMM_DEFAULT_UNROLL_N 2
 #define CGEMM_DEFAULT_UNROLL_N 2
 #define ZGEMM_DEFAULT_UNROLL_N 2
@@ -161,14 +165,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifdef ARCH_X86
 #define SGEMM_DEFAULT_UNROLL_M 4
-#define DGEMM_DEFAULT_UNROLL_M 2
+#define DGEMM_DEFAULT_UNROLL_M 4
 #define QGEMM_DEFAULT_UNROLL_M 2
 #define CGEMM_DEFAULT_UNROLL_M 2
 #define ZGEMM_DEFAULT_UNROLL_M 1
 #define XGEMM_DEFAULT_UNROLL_M 1
 #else
 #define SGEMM_DEFAULT_UNROLL_M 8
+#if defined(BULLDOZER) && !defined(COMPLEX)
+#define DGEMM_DEFAULT_UNROLL_M 8
+#else
 #define DGEMM_DEFAULT_UNROLL_M 4
+#endif
 #define QGEMM_DEFAULT_UNROLL_M 2
 #define CGEMM_DEFAULT_UNROLL_M 4
 #define ZGEMM_DEFAULT_UNROLL_M 2
@@ -193,26 +201,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #else
 
 #define SGEMM_DEFAULT_P 448
-
-#if defined(BULLDOZER) && defined(ARCH_X86_64)
-#define DGEMM_DEFAULT_P 248
+#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX)
+#define DGEMM_DEFAULT_P 384
 #else
 #define DGEMM_DEFAULT_P 224
 #endif
-
 #define QGEMM_DEFAULT_P 112
 #define CGEMM_DEFAULT_P 224
 #define ZGEMM_DEFAULT_P 112 
 #define XGEMM_DEFAULT_P  56
 
 #define SGEMM_DEFAULT_Q 224
-
-#if defined(BULLDOZER) && defined(ARCH_X86_64)
-#define DGEMM_DEFAULT_Q 248
+#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX)
+#define DGEMM_DEFAULT_Q 168
 #else
 #define DGEMM_DEFAULT_Q 224
 #endif
-
 #define QGEMM_DEFAULT_Q 224
 #define CGEMM_DEFAULT_Q 224
 #define ZGEMM_DEFAULT_Q 224
@@ -230,7 +234,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define SYMV_P 16
 #define HAVE_EXCLUSIVE_CACHE
 
-#define GEMM_THREAD gemm_thread_mn
+#define GEMM_THREAD gemm_thread_m
 
 #endif