power9 makefile. dgemm based on power8 kernel with following changes : 32x unrolled...

diff --git a/Makefile.power b/Makefile.power
index a49372a..195f193 100644 (file)
--- a/Makefile.power
+++ b/Makefile.power
@@ -9,7 +9,15 @@ else
 USE_OPENMP = 1
 endif
 
-
+ifeq ($(CORE), POWER9)
+ifeq ($(USE_OPENMP), 1)
+COMMON_OPT += -Ofast -mcpu=power9 -mtune=power9 -mvsx -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+FCOMMON_OPT += -O2 -frecursive -mcpu=power9 -mtune=power9 -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+else
+COMMON_OPT += -Ofast -mcpu=power9 -mtune=power9 -mvsx -malign-power -fno-fast-math
+FCOMMON_OPT += -O2 -frecursive -mcpu=power9 -mtune=power9 -malign-power -fno-fast-math
+endif
+endif
 
 ifeq ($(CORE), POWER8)
 ifeq ($(USE_OPENMP), 1)

diff --git a/TargetList.txt b/TargetList.txt
index 3d04a57..44e539c 100644 (file)
--- a/TargetList.txt
+++ b/TargetList.txt
@@ -48,6 +48,7 @@ POWER5
 POWER6
 POWER7
 POWER8
+POWER9
 PPCG4
 PPC970
 PPC970MP

diff --git a/common.h b/common.h
index 7fcd5e3..b30a71f 100644 (file)
--- a/common.h
+++ b/common.h
@@ -348,6 +348,11 @@ typedef int blasint;
 #endif
 #endif
 
+#ifdef POWER9
+#ifndef YIELDING
+#define YIELDING        __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
+#endif
+#endif
 
 /*
 #ifdef PILEDRIVER

diff --git a/common_power.h b/common_power.h
index e3a1a7a..ddbee94 100644 (file)
--- a/common_power.h
+++ b/common_power.h
@@ -39,7 +39,7 @@
 #ifndef COMMON_POWER
 #define COMMON_POWER
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #define MB             __asm__ __volatile__ ("eieio":::"memory")
 #define WMB            __asm__ __volatile__ ("eieio":::"memory")
 #else
@@ -241,7 +241,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define HAVE_PREFETCH
 #endif
 
-#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL) || defined(POWER8)
+#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL) || defined(POWER8)  || defined(POWER9)
 #define DCBT_ARG       0
 #else
 #define DCBT_ARG       8
@@ -263,7 +263,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define L1_PREFETCH    dcbtst
 #endif
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #define L1_DUALFETCH
 #define L1_PREFETCHSIZE (16 + 128 * 100)
 #define L1_PREFETCH    dcbtst
@@ -802,7 +802,7 @@ Lmcount$lazy_ptr:
 #define BUFFER_SIZE     (  2 << 20)
 #elif defined(PPC440FP2)
 #define BUFFER_SIZE     ( 16 << 20)
-#elif defined(POWER8)
+#elif defined(POWER8) || defined(POWER9)
 #define BUFFER_SIZE     ( 64 << 20)
 #else
 #define BUFFER_SIZE     ( 16 << 20)

diff --git a/cpuid_power.c b/cpuid_power.c
index 82a3f4a..d5ba6fb 100644 (file)
--- a/cpuid_power.c
+++ b/cpuid_power.c
@@ -94,7 +94,7 @@ char *corename[] = {
   "CELL",
   "PPCG4",
   "POWER8",
-  "POWER8"     
+  "POWER9"     
 };
 
 int detect(void){
@@ -124,7 +124,7 @@ int detect(void){
   if (!strncasecmp(p, "POWER6", 6)) return CPUTYPE_POWER6;
   if (!strncasecmp(p, "POWER7", 6)) return CPUTYPE_POWER6;
   if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
-  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER8;
+  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9;
   if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
   if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;
 
@@ -156,7 +156,7 @@ int detect(void){
   if (!strncasecmp(p, "POWER6", 6)) return CPUTYPE_POWER6;
   if (!strncasecmp(p, "POWER7", 6)) return CPUTYPE_POWER6;
   if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
-  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER8;
+  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9;
   if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
   if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;
   return CPUTYPE_POWER5;
@@ -180,7 +180,7 @@ int id;
 __asm __volatile("mfpvr %0" : "=r"(id));
 switch ( id >> 16 ) {
   case 0x4e: // POWER9
-    return CPUTYPE_POWER8;
+    return CPUTYPE_POWER9;
     break;
   case 0x4d:
   case 0x4b: // POWER8/8E 

diff --git a/getarch.c b/getarch.c
index 78ba0fe..34d4690 100644 (file)
--- a/getarch.c
+++ b/getarch.c
@@ -618,6 +618,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CORENAME  "POWER8"
 #endif
 
+#if defined(FORCE_POWER9) 
+#define FORCE
+#define ARCHITECTURE    "POWER"
+#define SUBARCHITECTURE "POWER9"
+#define SUBDIRNAME      "power"
+#define ARCHCONFIG   "-DPOWER9 " \
+                    "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=128 " \
+                    "-DL2_SIZE=4194304 -DL2_LINESIZE=128 " \
+                    "-DDTB_DEFAULT_ENTRIES=128 -DDTB_SIZE=4096 -DL2_ASSOCIATIVE=8 "
+#define LIBNAME   "power9"
+#define CORENAME  "POWER9"
+#endif
 
 #ifdef FORCE_PPCG4
 #define FORCE

diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3
index 9258f21..db9fccd 100644 (file)
--- a/kernel/Makefile.L3
+++ b/kernel/Makefile.L3
@@ -44,6 +44,10 @@ ifeq ($(CORE), POWER8)
 USE_TRMM = 1
 endif
 
+ifeq ($(CORE), POWER9)
+USE_TRMM = 1
+endif
+
 ifeq ($(ARCH), zarch)
 USE_TRMM = 1
 endif

diff --git a/kernel/power/KERNEL.POWER9 b/kernel/power/KERNEL.POWER9
new file mode 100644 (file)
index 0000000..86a9319
--- /dev/null
+++ b/kernel/power/KERNEL.POWER9
@@ -0,0 +1,184 @@
+#SGEMM_BETA = ../generic/gemm_beta.c\r
+#DGEMM_BETA = ../generic/gemm_beta.c\r
+#CGEMM_BETA = ../generic/zgemm_beta.c\r
+#ZGEMM_BETA = ../generic/zgemm_beta.c\r
+\r
+STRMMKERNEL    = strmm_kernel_16x8_power8.S\r
+DTRMMKERNEL    = dgemm_kernel_power9.S\r
+CTRMMKERNEL    = ctrmm_kernel_8x4_power8.S\r
+ZTRMMKERNEL    = ztrmm_kernel_8x2_power8.S\r
+\r
+SGEMMKERNEL    =  sgemm_kernel_16x8_power8.S\r
+SGEMMINCOPY    = ../generic/gemm_ncopy_16.c\r
+SGEMMITCOPY    = sgemm_tcopy_16_power8.S\r
+SGEMMONCOPY    =  ../generic/gemm_ncopy_8.c\r
+SGEMMOTCOPY    = sgemm_tcopy_8_power8.S \r
+SGEMMINCOPYOBJ =  sgemm_incopy.o\r
+SGEMMITCOPYOBJ =  sgemm_itcopy.o\r
+SGEMMONCOPYOBJ =  sgemm_oncopy.o\r
+SGEMMOTCOPYOBJ =  sgemm_otcopy.o\r
+\r
+DGEMMKERNEL    =  dgemm_kernel_power9.S\r
+DGEMMINCOPY    = ../generic/gemm_ncopy_16.c\r
+DGEMMITCOPY    =  dgemm_tcopy_16_power8.S\r
+DGEMMONCOPY    =  dgemm_ncopy_4_power8.S\r
+DGEMMOTCOPY    =  ../generic/gemm_tcopy_4.c\r
+DGEMMINCOPYOBJ =  dgemm_incopy.o\r
+DGEMMITCOPYOBJ =  dgemm_itcopy.o\r
+DGEMMONCOPYOBJ =  dgemm_oncopy.o\r
+DGEMMOTCOPYOBJ =  dgemm_otcopy.o\r
+\r
+CGEMMKERNEL    = cgemm_kernel_8x4_power8.S\r
+CGEMMINCOPY    = ../generic/zgemm_ncopy_8.c\r
+CGEMMITCOPY    = cgemm_tcopy_8_power8.S\r
+CGEMMONCOPY    = ../generic/zgemm_ncopy_4.c\r
+CGEMMOTCOPY    = ../generic/zgemm_tcopy_4.c\r
+CGEMMONCOPYOBJ =  cgemm_oncopy.o\r
+CGEMMOTCOPYOBJ =  cgemm_otcopy.o\r
+CGEMMINCOPYOBJ =  cgemm_incopy.o\r
+CGEMMITCOPYOBJ =  cgemm_itcopy.o\r
+\r
+ZGEMMKERNEL    = zgemm_kernel_8x2_power8.S\r
+ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c\r
+ZGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c\r
+ZGEMMINCOPY    = ../generic/zgemm_ncopy_8.c\r
+ZGEMMITCOPY    = zgemm_tcopy_8_power8.S\r
+ZGEMMONCOPYOBJ =  zgemm_oncopy.o\r
+ZGEMMOTCOPYOBJ =  zgemm_otcopy.o\r
+ZGEMMINCOPYOBJ =  zgemm_incopy.o\r
+ZGEMMITCOPYOBJ =  zgemm_itcopy.o\r
+\r
+STRSMKERNEL_LN =  ../generic/trsm_kernel_LN.c\r
+STRSMKERNEL_LT =  ../generic/trsm_kernel_LT.c\r
+STRSMKERNEL_RN =  ../generic/trsm_kernel_RN.c\r
+STRSMKERNEL_RT =  ../generic/trsm_kernel_RT.c\r
+\r
+DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c\r
+DTRSMKERNEL_LT = dtrsm_kernel_LT_16x4_power8.S\r
+DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c\r
+DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c\r
+\r
+CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c\r
+CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c\r
+CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c\r
+CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c\r
+\r
+ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c\r
+ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c\r
+ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c\r
+ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c\r
+\r
+#Todo: CGEMM3MKERNEL should be 4x4 blocksizes.\r
+#CGEMM3MKERNEL    =  zgemm3m_kernel_8x4_sse3.S\r
+#ZGEMM3MKERNEL    =  zgemm3m_kernel_4x4_sse3.S\r
+\r
+#Pure C for other kernels\r
+#SAMAXKERNEL  = ../arm/amax.c\r
+#DAMAXKERNEL  = ../arm/amax.c\r
+#CAMAXKERNEL  = ../arm/zamax.c\r
+#ZAMAXKERNEL  = ../arm/zamax.c\r
+#\r
+#SAMINKERNEL  = ../arm/amin.c\r
+#DAMINKERNEL  = ../arm/amin.c\r
+#CAMINKERNEL  = ../arm/zamin.c\r
+#ZAMINKERNEL  = ../arm/zamin.c\r
+#\r
+#SMAXKERNEL   = ../arm/max.c\r
+#DMAXKERNEL   = ../arm/max.c\r
+#\r
+#SMINKERNEL   = ../arm/min.c\r
+#DMINKERNEL   = ../arm/min.c\r
+#\r
+ISAMAXKERNEL = isamax.c\r
+IDAMAXKERNEL = idamax.c\r
+ICAMAXKERNEL = icamax.c\r
+IZAMAXKERNEL = izamax.c\r
+#\r
+ISAMINKERNEL = isamin.c\r
+IDAMINKERNEL = idamin.c\r
+ICAMINKERNEL = icamin.c\r
+IZAMINKERNEL = izamin.c\r
+#\r
+#ISMAXKERNEL  = ../arm/imax.c\r
+#IDMAXKERNEL  = ../arm/imax.c\r
+#\r
+#ISMINKERNEL  = ../arm/imin.c\r
+#IDMINKERNEL  = ../arm/imin.c\r
+#\r
+SASUMKERNEL  = sasum.c\r
+DASUMKERNEL  = dasum.c\r
+CASUMKERNEL  = casum.c\r
+ZASUMKERNEL  = zasum.c\r
+#\r
+SAXPYKERNEL  = saxpy.c\r
+DAXPYKERNEL  = daxpy.c\r
+CAXPYKERNEL  = caxpy.c\r
+ZAXPYKERNEL  = zaxpy.c\r
+#\r
+SCOPYKERNEL  = scopy.c\r
+DCOPYKERNEL  = dcopy.c\r
+CCOPYKERNEL  = ccopy.c\r
+ZCOPYKERNEL  = zcopy.c\r
+#\r
+SDOTKERNEL   =  sdot.c\r
+DDOTKERNEL   =  ddot.c\r
+DSDOTKERNEL  =  sdot.c\r
+CDOTKERNEL   =  cdot.c\r
+ZDOTKERNEL   =  zdot.c\r
+#\r
+SNRM2KERNEL  = ../arm/nrm2.c\r
+DNRM2KERNEL  = ../arm/nrm2.c\r
+CNRM2KERNEL  = ../arm/znrm2.c\r
+ZNRM2KERNEL  = ../arm/znrm2.c\r
+#\r
+SROTKERNEL   = srot.c\r
+DROTKERNEL   = drot.c\r
+CROTKERNEL   = crot.c\r
+ZROTKERNEL   = zrot.c\r
+#\r
+SSCALKERNEL  = sscal.c\r
+DSCALKERNEL  = dscal.c\r
+CSCALKERNEL  = zscal.c\r
+ZSCALKERNEL  = zscal.c\r
+#\r
+SSWAPKERNEL  = sswap.c\r
+DSWAPKERNEL  = dswap.c\r
+CSWAPKERNEL  = cswap.c\r
+ZSWAPKERNEL  = zswap.c\r
+#\r
+\r
+SGEMVNKERNEL = sgemv_n.c\r
+DGEMVNKERNEL = dgemv_n.c\r
+CGEMVNKERNEL = cgemv_n.c\r
+ZGEMVNKERNEL = zgemv_n_4.c\r
+#\r
+SGEMVTKERNEL = sgemv_t.c\r
+DGEMVTKERNEL = dgemv_t.c\r
+CGEMVTKERNEL = cgemv_t.c\r
+ZGEMVTKERNEL = zgemv_t_4.c\r
+\r
+\r
+#SSYMV_U_KERNEL =  ../generic/symv_k.c\r
+#SSYMV_L_KERNEL =  ../generic/symv_k.c\r
+#DSYMV_U_KERNEL =  ../generic/symv_k.c\r
+#DSYMV_L_KERNEL =  ../generic/symv_k.c\r
+#QSYMV_U_KERNEL =  ../generic/symv_k.c\r
+#QSYMV_L_KERNEL =  ../generic/symv_k.c\r
+#CSYMV_U_KERNEL =  ../generic/zsymv_k.c\r
+#CSYMV_L_KERNEL =  ../generic/zsymv_k.c\r
+#ZSYMV_U_KERNEL =  ../generic/zsymv_k.c\r
+#ZSYMV_L_KERNEL =  ../generic/zsymv_k.c\r
+#XSYMV_U_KERNEL =  ../generic/zsymv_k.c\r
+#XSYMV_L_KERNEL =  ../generic/zsymv_k.c\r
+\r
+#ZHEMV_U_KERNEL =  ../generic/zhemv_k.c\r
+#ZHEMV_L_KERNEL =  ../generic/zhemv_k.c\r
+\r
+LSAME_KERNEL = ../generic/lsame.c\r
+SCABS_KERNEL   = ../generic/cabs.c\r
+DCABS_KERNEL   = ../generic/cabs.c\r
+QCABS_KERNEL   = ../generic/cabs.c\r
+\r
+#Dump kernel\r
+CGEMM3MKERNEL    = ../generic/zgemm3mkernel_dump.c\r
+ZGEMM3MKERNEL    = ../generic/zgemm3mkernel_dump.c\r

diff --git a/kernel/power/casum.c b/kernel/power/casum.c
index d110858..a9ece07 100644 (file)
--- a/kernel/power/casum.c
+++ b/kernel/power/casum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8)
+#if defined(POWER8)  || defined(POWER9)
 #include "casum_microk_power8.c"
 #endif
 

diff --git a/kernel/power/ccopy.c b/kernel/power/ccopy.c
index ce7d674..50df84c 100644 (file)
--- a/kernel/power/ccopy.c
+++ b/kernel/power/ccopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "ccopy_microk_power8.c"
 #endif
 

diff --git a/kernel/power/crot.c b/kernel/power/crot.c
index 40e350b..959a9ed 100644 (file)
--- a/kernel/power/crot.c
+++ b/kernel/power/crot.c
@@ -27,7 +27,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  \r
 #include "common.h"\r
  \r
-#if defined(POWER8)\r
+#if defined(POWER8) || defined(POWER9)\r
 \r
 static void crot_kernel_8 (long n, float *x, float *y, float c, float s)\r
 {\r

diff --git a/kernel/power/cswap.c b/kernel/power/cswap.c
index da97c89..31e02fe 100644 (file)
--- a/kernel/power/cswap.c
+++ b/kernel/power/cswap.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8)
+#if defined(POWER8)  || defined(POWER9)
 #include "cswap_microk_power8.c"
 #endif
 

diff --git a/kernel/power/dasum.c b/kernel/power/dasum.c
index 73962c2..d0e0609 100644 (file)
--- a/kernel/power/dasum.c
+++ b/kernel/power/dasum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "dasum_microk_power8.c"
 #endif
 

diff --git a/kernel/power/daxpy.c b/kernel/power/daxpy.c
index df0572e..f09611f 100644 (file)
--- a/kernel/power/daxpy.c
+++ b/kernel/power/daxpy.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "daxpy_microk_power8.c"
 #endif
 

diff --git a/kernel/power/dcopy.c b/kernel/power/dcopy.c
index 059c0e5..27b3914 100644 (file)
--- a/kernel/power/dcopy.c
+++ b/kernel/power/dcopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "dcopy_microk_power8.c"
 #endif
 

diff --git a/kernel/power/ddot.c b/kernel/power/ddot.c
index e43470e..f985df1 100644 (file)
--- a/kernel/power/ddot.c
+++ b/kernel/power/ddot.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8) 
+#if defined(POWER8)  || defined(POWER9)
 #include "ddot_microk_power8.c"
 #endif
 

diff --git a/kernel/power/dgemm_kernel_power9.S b/kernel/power/dgemm_kernel_power9.S
new file mode 100644 (file)
index 0000000..a1762dc
--- /dev/null
+++ b/kernel/power/dgemm_kernel_power9.S
@@ -0,0 +1,249 @@
+/***************************************************************************\r
+Copyright (c) 2013-2019, The OpenBLAS Project\r
+All rights reserved.\r
+Redistribution and use in source and binary forms, with or without\r
+modification, are permitted provided that the following conditions are\r
+met:\r
+1. Redistributions of source code must retain the above copyright\r
+notice, this list of conditions and the following disclaimer.\r
+2. Redistributions in binary form must reproduce the above copyright\r
+notice, this list of conditions and the following disclaimer in\r
+the documentation and/or other materials provided with the\r
+distribution.\r
+3. Neither the name of the OpenBLAS project nor the names of\r
+its contributors may be used to endorse or promote products\r
+derived from this software without specific prior written permission.\r
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"\r
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\r
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE\r
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\r
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+*****************************************************************************/\r
+ \r
+#define ASSEMBLER\r
+#include "common.h"\r
+#include "def_vsx.h"\r
+\r
+ \r
+#define LOAD   ld\r
+ \r
+ \r
+\r
+ \r
+#define STACKSIZE  (512 )\r
+#define ALPHA_SP   (296+192)(SP)\r
+#define FZERO  (304+192)(SP)\r
+ \r
+\r
+ \r
+#define        M       r3\r
+#define        N       r4\r
+#define        K       r5\r
+ \r
+#define A      r7\r
+#define        B       r8\r
+#define        C       r9\r
+#define        LDC     r10\r
+#define OFFSET r6\r
+ \r
+ \r
+\r
+#define alpha_r vs18\r
+\r
+#define o0     0\r
+\r
+\r
+#define T4     r12\r
+#define T3     r11\r
+#define C4     r14\r
+#define o8     r15\r
+#define o24    r16\r
+#define C2     r17\r
+#define L      r18\r
+#define T1     r19\r
+#define C3     r20\r
+#define TEMP_REG       r21\r
+#define        I       r22\r
+#define J      r23\r
+#define AO     r24\r
+#define        BO      r25\r
+#define        CO      r26\r
+#define o16    r27\r
+#define        o32     r28\r
+#define        o48     r29\r
+\r
+#define PRE    r30\r
+#define T2     r31\r
+\r
+#include "dgemm_macros_power9.S"\r
+\r
+\r
+#ifndef NEEDPARAM\r
+\r
+       PROLOGUE\r
+       PROFCODE\r
+\r
+       addi    SP, SP, -STACKSIZE\r
+       li      r0, 0\r
+\r
+       stfd    f14,    0(SP)\r
+       stfd    f15,    8(SP)\r
+       stfd    f16,   16(SP)\r
+       stfd    f17,   24(SP)\r
+\r
+       stfd    f18,   32(SP)\r
+       stfd    f19,   40(SP)\r
+       stfd    f20,   48(SP)\r
+       stfd    f21,   56(SP)\r
+\r
+       stfd    f22,   64(SP)\r
+       stfd    f23,   72(SP)\r
+       stfd    f24,   80(SP)\r
+       stfd    f25,   88(SP)\r
+\r
+       stfd    f26,   96(SP)\r
+       stfd    f27,  104(SP)\r
+       stfd    f28,  112(SP)\r
+       stfd    f29,  120(SP)\r
+\r
+       stfd    f30,  128(SP)\r
+       stfd    f31,  136(SP)\r
+\r
+ \r
+       std     r31,  144(SP)\r
+       std     r30,  152(SP)\r
+       std     r29,  160(SP)\r
+       std     r28,  168(SP)\r
+       std     r27,  176(SP)\r
+       std     r26,  184(SP)\r
+       std     r25,  192(SP)\r
+       std     r24,  200(SP)\r
+       std     r23,  208(SP)\r
+       std     r22,  216(SP)\r
+       std     r21,  224(SP)\r
+       std     r20,  232(SP)\r
+       std     r19,  240(SP)\r
+       std     r18,  248(SP)\r
+       std     r17,  256(SP)\r
+       std     r16,  264(SP)\r
+       std     r15,  272(SP)\r
+       std     r14,  280(SP)\r
+ \r
+ \r
+    stxv    v20,  288(SP)\r
+    stxv    v21,  304(SP)\r
+    stxv    v22,  320(SP)\r
+    stxv    v23,  336(SP)\r
+    stxv    v24,  352(SP)\r
+    stxv    v25,  368(SP)\r
+    stxv    v26,  384(SP)\r
+    stxv    v27,  400(SP)\r
+    stxv    v28,  416(SP)\r
+    stxv    v29,  432(SP)\r
+    stxv    v30,  448(SP)\r
+    stxv    v31,  464(SP)\r
+\r
+\r
+       stfd    f1,  ALPHA_SP\r
+       stw     r0,  FZERO \r
+\r
+       slwi    LDC, LDC, BASE_SHIFT\r
+\r
+#if defined(TRMMKERNEL)\r
+       ld      OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)\r
+#endif\r
+\r
+\r
+       cmpwi   cr0, M, 0\r
+       ble     .L999_H1\r
+       cmpwi   cr0, N, 0\r
+       ble     .L999_H1\r
+       cmpwi   cr0, K, 0\r
+       ble     .L999_H1\r
+\r
+ \r
+ \r
+       addi    T1, SP, 296+192\r
+ \r
+\r
+       li      PRE, 384\r
+       li      o8 , 8\r
+       li      o16, 16\r
+       li      o24, 24\r
+       li      o32, 32\r
+       li      o48, 48\r
+\r
+\r
+       lxvdsx  alpha_r, 0, T1\r
+\r
+#include "dgemm_logic_power9.S"\r
+\r
+.L999:\r
+       addi    r3, 0, 0\r
+\r
+       lfd     f14,    0(SP)\r
+       lfd     f15,    8(SP)\r
+       lfd     f16,   16(SP)\r
+       lfd     f17,   24(SP)\r
+\r
+       lfd     f18,   32(SP)\r
+       lfd     f19,   40(SP)\r
+       lfd     f20,   48(SP)\r
+       lfd     f21,   56(SP)\r
+\r
+       lfd     f22,   64(SP)\r
+       lfd     f23,   72(SP)\r
+       lfd     f24,   80(SP)\r
+       lfd     f25,   88(SP)\r
+\r
+       lfd     f26,   96(SP)\r
+       lfd     f27,  104(SP)\r
+       lfd     f28,  112(SP)\r
+       lfd     f29,  120(SP)\r
+\r
+       lfd     f30,  128(SP)\r
+       lfd     f31,  136(SP)\r
+\r
+ \r
+       ld      r31,  144(SP)\r
+       ld      r30,  152(SP)\r
+       ld      r29,  160(SP)\r
+       ld      r28,  168(SP)\r
+       ld      r27,  176(SP)\r
+       ld      r26,  184(SP)\r
+       ld      r25,  192(SP)\r
+       ld      r24,  200(SP)\r
+       ld      r23,  208(SP)\r
+       ld      r22,  216(SP)\r
+       ld      r21,  224(SP)\r
+       ld      r20,  232(SP)\r
+       ld      r19,  240(SP)\r
+       ld      r18,  248(SP)\r
+       ld      r17,  256(SP)\r
+       ld      r16,  264(SP)\r
+       ld      r15,  272(SP)\r
+       ld      r14,  280(SP)\r
+ \r
+    lxv    v20,  288(SP)\r
+    lxv    v21,  304(SP)\r
+    lxv    v22,  320(SP)\r
+    lxv    v23,  336(SP)\r
+    lxv    v24,  352(SP)\r
+    lxv    v25,  368(SP)\r
+    lxv    v26,  384(SP)\r
+    lxv    v27,  400(SP)\r
+    lxv    v28,  416(SP)\r
+    lxv    v29,  432(SP)\r
+    lxv    v30,  448(SP)\r
+    lxv    v31,  464(SP)\r
+\r
+       addi    SP, SP, STACKSIZE\r
+       blr\r
+\r
+       EPILOGUE\r
+#endif\r

diff --git a/kernel/power/dgemm_logic_power9.S b/kernel/power/dgemm_logic_power9.S
new file mode 100644 (file)
index 0000000..251839d
--- /dev/null
+++ b/kernel/power/dgemm_logic_power9.S
@@ -0,0 +1,1981 @@
+/***************************************************************************\r
+Copyright (c) 2013-2019 The OpenBLAS Project\r
+All rights reserved.\r
+Redistribution and use in source and binary forms, with or without\r
+modification, are permitted provided that the following conditions are\r
+met:\r
+1. Redistributions of source code must retain the above copyright\r
+notice, this list of conditions and the following disclaimer.\r
+2. Redistributions in binary form must reproduce the above copyright\r
+notice, this list of conditions and the following disclaimer in\r
+the documentation and/or other materials provided with the\r
+distribution.\r
+3. Neither the name of the OpenBLAS project nor the names of\r
+its contributors may be used to endorse or promote products\r
+derived from this software without specific prior written permission.\r
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"\r
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\r
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE\r
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\r
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+*****************************************************************************/\r
+ \r
+\r
+#define MY_ALIGN .align 3\r
+\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+   neg TEMP_REG, OFFSET \r
+#endif\r
+\r
+       srawi.          J,      N,      2\r
+       ble             LDGEMM_L4_END\r
+\r
+LDGEMM_L4_BEGIN:\r
+\r
+ \r
+       li              T1,     128\r
+       li              T2,     256\r
+ \r
+       mr              AO,     A\r
+       mr              CO,     C\r
+       slwi            T3,     LDC     ,       2\r
+       add             C,      C,      T3\r
+\r
+ \r
+       dcbt            A,      T1\r
+       dcbt            A,      T2\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+       mr TEMP_REG, OFFSET      /*off = offset;*/\r
+#endif \r
+       srawi.          I,      M,      4\r
+       ble             LDGEMM_L4x16_END\r
+\r
+       MY_ALIGN\r
+LDGEMM_L4x16_BEGIN:\r
+\r
+       li              L,      -128\r
+\r
+\r
+       SAVE4x16_REGS\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,4\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+ \r
+       and             T1,     CO,     L\r
+       and             T2,     C2,     L\r
+       and             T3,     C3,     L\r
+       and             T4,     C4,     L\r
+\r
+       dcbt            T1,     r0\r
+       dcbt            T2,     r0\r
+       dcbt            T3,     r0\r
+       dcbt            T4,     r0\r
+ \r
+\r
+       addi            T1, T1, 128\r
+       addi            T2, T2, 128\r
+       addi            T3, T3, 128\r
+       addi            T4, T4, 128\r
+\r
+       dcbt            T1,     r0\r
+       dcbt            T2,     r0\r
+       dcbt            T3,     r0\r
+       dcbt            T4,     r0\r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T3,K,TEMP_REG,16,4\r
+   srawi.              L, T3,  5\r
+#else\r
+   srawi.              L,      K,      5\r
+#endif \r
+       \r
+       ble             LDGEMM_L4x16_SUB0\r
+\r
+\r
+       MY_ALIGN\r
+LDGEMM_L4x16_LOOP_START:\r
+\r
+       li      T2,     512\r
+ \r
+ \r
+       LOAD4x16_1\r
+    ##OffsetA=128 OffsetB=32\r
+    addi AO,AO,2176\r
+ #   addi BO,BO,32 \r
+       addic.          L,      L,      -1\r
+\r
+       ble             LDGEMM_L4x16_LOOP_END\r
+\r
+       \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L4x16_LOOP:\r
+\r
+       #dcbt   AO,     PRE\r
+    KERNEL4x16_I1_L2_2  -2048,32, 0,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 1,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 2,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 3,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 4,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 5,0        \r
+    KERNEL4x16_I1_L2_2  -2048,32, 6,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 7,0  \r
+    KERNEL4x16_I1_L2_2  -2048,32, 8,0      \r
+    KERNEL4x16_I1_L2_2  -2048,32, 9,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 10,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 11,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 12,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 13,0    \r
+    KERNEL4x16_I1_L2_2  -2048,32, 14,0    \r
+    KERNEL4x16_I1_L2_2  -2048,32, 15,1         \r
+\r
+\r
+       bdnz            LDGEMM_L4x16_LOOP\r
+\r
+       MY_ALIGN\r
+       MY_ALIGN\r
+LDGEMM_L4x16_LOOP_END:\r
+\r
+    KERNEL4x16_I1_L2_2  -2048,32, 0,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 1,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 2,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 3,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 4,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 5,0        \r
+    KERNEL4x16_I1_L2_2  -2048,32, 6,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 7,0  \r
+    KERNEL4x16_I1_L2_2  -2048,32, 8,0      \r
+    KERNEL4x16_I1_L2_2  -2048,32, 9,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 10,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 11,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 12,0\r
+    KERNEL4x16_I1_L2_2  -2048,32, 13,0    \r
+    KERNEL4x16_I1_L2_2  -2048,32, 14,0    \r
+    KERNEL4x16_I1_L2_3  -2048,32, 15,1    \r
+       b               LDGEMM_L4x16_SUB1\r
+\r
+\r
+       MY_ALIGN\r
+LDGEMM_L4x16_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     31\r
+#else\r
+       andi.           L,      K,      31\r
+#endif\r
+       KERNEL4x16 1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L4x16_SAVE\r
+       b               LDGEMM_L4x16_SUB2\r
+       MY_ALIGN\r
+LDGEMM_L4x16_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     31\r
+#else\r
+       andi.           L,      K,      31\r
+#endif \r
+       ble             LDGEMM_L4x16_SAVE\r
+       MY_ALIGN\r
+LDGEMM_L4x16_SUB2:\r
+\r
+    andi.      T1,L, 16\r
+    ble LDGEMM_L4x16_SUB2_8\r
+       LOAD4x16_0 \r
+    KERNEL4x16_I1_L2_2  128,32, 0,0\r
+    KERNEL4x16_I1_L2_2  128,32, 1,0\r
+    KERNEL4x16_I1_L2_2  128,32, 2,0\r
+    KERNEL4x16_I1_L2_2  128,32, 3,0\r
+    KERNEL4x16_I1_L2_2  128,32, 4,0\r
+    KERNEL4x16_I1_L2_2  128,32, 5,0        \r
+    KERNEL4x16_I1_L2_2  128,32, 6,0\r
+    KERNEL4x16_I1_L2_3  128,32, 7,1 \r
+    MY_ALIGN\r
+LDGEMM_L4x16_SUB2_8:\r
+    andi.      T1,L, 8\r
+    ble LDGEMM_L4x16_SUB2_4\r
+       LOAD4x16_0\r
+    KERNEL4x16_I1_L2_2  128,32, 0,0\r
+    KERNEL4x16_I1_L2_2  128,32, 1,0\r
+    KERNEL4x16_I1_L2_2  128,32, 2,0\r
+    KERNEL4x16_I1_L2_3  128,32, 3,1\r
+       MY_ALIGN\r
+LDGEMM_L4x16_SUB2_4:\r
+    andi.      T1,L, 4\r
+    ble LDGEMM_L4x16_SUB2_2 \r
+       LOAD4x16_0\r
+    KERNEL4x16_I1_L2_2  128,32, 0,0\r
+    KERNEL4x16_I1_L2_3  128,32, 1,1\r
+       MY_ALIGN        \r
+LDGEMM_L4x16_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LDGEMM_L4x16_SUB2_1\r
+    LOAD4x16_0\r
+    KERNEL4x16_I1_L2_3  128,32, 0,1\r
+    MY_ALIGN\r
+LDGEMM_L4x16_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LDGEMM_L4x16_SAVE      \r
+    KERNEL4x16 0\r
+#      addic.          L,      L,      -1\r
+#      bgt             LDGEMM_L4x16_SUB2\r
+\r
+       MY_ALIGN\r
+LDGEMM_L4x16_SAVE:\r
+       SAVE4x16\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,16,4\r
+#endif \r
+       addic.          I,      I,      -1\r
+       bgt+            LDGEMM_L4x16_BEGIN\r
+\r
+LDGEMM_L4x16_END:\r
+\r
+LDGEMM_L4x8_BEGIN:\r
+\r
+       andi.           T2,     M,      15\r
+       ble             LDGEMM_L4x1_END\r
+\r
+       andi.           T1,     M,      8\r
+       ble             LDGEMM_L4x8_END\r
+\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,4\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,8,4\r
+    srawi.             L, T3,  4       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      4       \r
+#endif          \r
+ \r
+\r
+       ble             LDGEMM_L4x8_SUB0\r
+\r
+LDGEMM_L4x8_LOOP_START:\r
+\r
+\r
+       LOAD4x8_1\r
+    ##OffsetA=64 OffsetB=32\r
+\r
+\r
+       addic.          L,      L,      -1\r
+\r
+       ble             LDGEMM_L4x8_LOOP_END\r
+\r
+    mtctr              L\r
+       MY_ALIGN\r
+\r
+LDGEMM_L4x8_LOOP:\r
+\r
+    KERNEL4x8_I1_L2_2  64,32, 0,0\r
+    KERNEL4x8_I1_L2_2  64,32, 1,0\r
+    KERNEL4x8_I1_L2_2  64,32, 2,0\r
+    KERNEL4x8_I1_L2_2  64,32, 3,0\r
+    KERNEL4x8_I1_L2_2  64,32, 4,0\r
+    KERNEL4x8_I1_L2_2  64,32, 5,0        \r
+    KERNEL4x8_I1_L2_2  64,32, 6,0\r
+    KERNEL4x8_I1_L2_2  64,32, 7,1     \r
+\r
+       bdnz            LDGEMM_L4x8_LOOP\r
+       MY_ALIGN\r
+LDGEMM_L4x8_LOOP_END:\r
+\r
+    KERNEL4x8_I1_L2_2  64,32, 0,0\r
+    KERNEL4x8_I1_L2_2  64,32, 1,0\r
+    KERNEL4x8_I1_L2_2  64,32, 2,0\r
+    KERNEL4x8_I1_L2_2  64,32, 3,0\r
+    KERNEL4x8_I1_L2_2  64,32, 4,0\r
+    KERNEL4x8_I1_L2_2  64,32, 5,0        \r
+    KERNEL4x8_I1_L2_2  64,32, 6,0\r
+    KERNEL4x8_I1_L2_3  64,32, 7,1  \r
+\r
+       b               LDGEMM_L4x8_SUB1\r
+       MY_ALIGN\r
+LDGEMM_L4x8_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     15\r
+#else\r
+       andi.           L,      K,      15\r
+#endif\r
+       KERNEL4x8 1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L4x8_SAVE\r
+       b               LDGEMM_L4x8_SUB2\r
+       MY_ALIGN\r
+LDGEMM_L4x8_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     15\r
+#else\r
+       andi.           L,      K,      15\r
+#endif \r
+       ble             LDGEMM_L4x8_SAVE\r
+       MY_ALIGN\r
+LDGEMM_L4x8_SUB2:\r
+\r
+    andi.      T1,L, 8\r
+    ble LDGEMM_L4x8_SUB2_4\r
+       LOAD4x8_0\r
+    KERNEL4x8_I1_L2_2  64,32, 0,0\r
+    KERNEL4x8_I1_L2_2  64,32, 1,0\r
+    KERNEL4x8_I1_L2_2  64,32, 2,0\r
+    KERNEL4x8_I1_L2_3  64,32, 3,1\r
+       MY_ALIGN\r
+LDGEMM_L4x8_SUB2_4:\r
+    andi.      T1,L, 4\r
+    ble LDGEMM_L4x8_SUB2_2 \r
+       LOAD4x8_0\r
+    KERNEL4x8_I1_L2_2  64,32, 0,0\r
+    KERNEL4x8_I1_L2_3  64,32, 1,1\r
+       MY_ALIGN        \r
+LDGEMM_L4x8_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LDGEMM_L4x8_SUB2_1\r
+    LOAD4x8_0\r
+    KERNEL4x8_I1_L2_3  64,32, 0,1\r
+    MY_ALIGN\r
+LDGEMM_L4x8_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LDGEMM_L4x8_SAVE       \r
+    KERNEL4x8 0\r
+ \r
+       MY_ALIGN\r
+LDGEMM_L4x8_SAVE:\r
+       SAVE4x8\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,8,4\r
+#endif \r
+LDGEMM_L4x8_END:\r
+\r
+LDGEMM_L4x4_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      4\r
+       ble             LDGEMM_L4x4_END\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,4\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,4,4\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif         \r
+       ble             LDGEMM_L4x4_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L4x4_SUB4\r
+\r
+LDGEMM_L4x4_LOOP_START:\r
+\r
+       #dcbt   AO,     PRE\r
+       LOAD4x4_1\r
+       KERNEL4x4_I1\r
+       KERNEL4x4_2\r
+       KERNEL4x4_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL4x4_2\r
+\r
+       KERNEL4x4_1\r
+       KERNEL4x4_2\r
+       KERNEL4x4_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL4x4_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L4x4_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L4x4_LOOP:\r
+\r
+       KERNEL4x4_1\r
+       KERNEL4x4_2\r
+       KERNEL4x4_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL4x4_2\r
+\r
+       KERNEL4x4_1\r
+       KERNEL4x4_2\r
+       KERNEL4x4_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL4x4_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L4x4_LOOP\r
+\r
+LDGEMM_L4x4_LOOP_END:\r
+\r
+       KERNEL4x4_1\r
+       KERNEL4x4_2\r
+       KERNEL4x4_1\r
+       KERNEL4x4_2\r
+\r
+       KERNEL4x4_1\r
+       KERNEL4x4_2\r
+       KERNEL4x4_1\r
+       KERNEL4x4_E2\r
+\r
+       b               LDGEMM_L4x4_SUB1\r
+\r
+LDGEMM_L4x4_SUB4:\r
+\r
+       KERNEL4x4_SUBI1\r
+       KERNEL4x4_SUB1\r
+       KERNEL4x4_SUB1\r
+       KERNEL4x4_SUB1\r
+\r
+       KERNEL4x4_SUB1\r
+       KERNEL4x4_SUB1\r
+       KERNEL4x4_SUB1\r
+       KERNEL4x4_SUB1\r
+\r
+       b               LDGEMM_L4x4_SUB1\r
+\r
+LDGEMM_L4x4_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL4x4_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L4x4_SAVE\r
+       b               LDGEMM_L4x4_SUB2\r
+\r
+LDGEMM_L4x4_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L4x4_SAVE\r
+\r
+LDGEMM_L4x4_SUB2:\r
+\r
+       KERNEL4x4_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L4x4_SUB2\r
+\r
+LDGEMM_L4x4_SAVE:\r
+\r
+       SAVE4x4\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,4,4\r
+#endif \r
+LDGEMM_L4x4_END:\r
+\r
+LDGEMM_L4x2_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      2\r
+       ble             LDGEMM_L4x2_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,4\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,2,4\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L4x2_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L4x2_SUB4\r
+\r
+LDGEMM_L4x2_LOOP_START:\r
+\r
+       LOAD4x2_1\r
+       KERNEL4x2_I1\r
+       KERNEL4x2_2\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L4x2_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L4x2_LOOP:\r
+\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L4x2_LOOP\r
+\r
+LDGEMM_L4x2_LOOP_END:\r
+\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+\r
+       KERNEL4x2_1\r
+       KERNEL4x2_2\r
+       KERNEL4x2_1\r
+       KERNEL4x2_E2\r
+\r
+       b               LDGEMM_L4x2_SUB1\r
+\r
+LDGEMM_L4x2_SUB4:\r
+\r
+       KERNEL4x2_SUBI1\r
+       KERNEL4x2_SUB1\r
+       KERNEL4x2_SUB1\r
+       KERNEL4x2_SUB1\r
+\r
+       KERNEL4x2_SUB1\r
+       KERNEL4x2_SUB1\r
+       KERNEL4x2_SUB1\r
+       KERNEL4x2_SUB1\r
+\r
+       b               LDGEMM_L4x2_SUB1\r
+\r
+LDGEMM_L4x2_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL4x2_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L4x2_SAVE\r
+       b               LDGEMM_L4x2_SUB2\r
+\r
+LDGEMM_L4x2_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L4x2_SAVE\r
+\r
+LDGEMM_L4x2_SUB2:\r
+\r
+       KERNEL4x2_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L4x2_SUB2\r
+\r
+LDGEMM_L4x2_SAVE:\r
+\r
+       SAVE4x2\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,2,4\r
+#endif \r
+LDGEMM_L4x2_END:\r
+\r
+LDGEMM_L4x1_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      1\r
+       ble             LDGEMM_L4x1_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,4\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,1,4\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L4x1_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L4x1_SUB4\r
+\r
+LDGEMM_L4x1_LOOP_START:\r
+\r
+       LOAD4x1_1\r
+       KERNEL4x1_I1\r
+       KERNEL4x1_2\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L4x1_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L4x1_LOOP:\r
+\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L4x1_LOOP\r
+\r
+LDGEMM_L4x1_LOOP_END:\r
+\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+\r
+       KERNEL4x1_1\r
+       KERNEL4x1_2\r
+       KERNEL4x1_1\r
+       KERNEL4x1_E2\r
+\r
+       b               LDGEMM_L4x1_SUB1\r
+\r
+LDGEMM_L4x1_SUB4:\r
+\r
+       KERNEL4x1_SUBI1\r
+       KERNEL4x1_SUB1\r
+       KERNEL4x1_SUB1\r
+       KERNEL4x1_SUB1\r
+\r
+       KERNEL4x1_SUB1\r
+       KERNEL4x1_SUB1\r
+       KERNEL4x1_SUB1\r
+       KERNEL4x1_SUB1\r
+\r
+       b               LDGEMM_L4x1_SUB1\r
+\r
+LDGEMM_L4x1_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL4x1_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L4x1_SAVE\r
+       b               LDGEMM_L4x1_SUB2\r
+\r
+LDGEMM_L4x1_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L4x1_SAVE\r
+\r
+LDGEMM_L4x1_SUB2:\r
+\r
+       KERNEL4x1_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L4x1_SUB2\r
+\r
+LDGEMM_L4x1_SAVE:\r
+\r
+       SAVE4x1\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,1,4\r
+#endif \r
+LDGEMM_L4x1_END:\r
+\r
+       slwi            T1,     K,      5\r
+       add             B,      B,      T1\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+    addi TEMP_REG, TEMP_REG, 4\r
+#endif\r
+       addic.          J,      J,      -1\r
+       bgt             LDGEMM_L4_BEGIN\r
+\r
+       andi.           T2,     N,      3\r
+       ble             .L999\r
+\r
+LDGEMM_L4_END:\r
+\r
+       b               LDGEMM_L2_BEGIN\r
+\r
+.L999_H1:\r
+\r
+       b               .L999\r
+\r
+LDGEMM_L2_BEGIN:\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+       mr TEMP_REG, OFFSET      /*off = offset;*/\r
+#endif \r
+       andi.           T1,     N,      2\r
+       ble             LDGEMM_L2_END\r
+       mr              CO,     C\r
+       mr              AO,     A\r
+       slwi            T1,     LDC     ,       1\r
+       add             C,      C,      T1\r
+       srawi.          I,      M,      4\r
+       ble             LDGEMM_L2x16_END\r
+\r
+LDGEMM_L2x16_BEGIN:\r
+\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,2\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,16,2\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L2x16_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L2x16_SUB4\r
+\r
+LDGEMM_L2x16_LOOP_START:\r
+\r
+       #dcbt           AO,     PRE\r
+       LOAD2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_I1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L2x16_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L2x16_LOOP:\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x16_LOOP\r
+\r
+LDGEMM_L2x16_LOOP_END:\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_1\r
+       KERNEL2x16_E2\r
+\r
+       b               LDGEMM_L2x16_SUB1\r
+\r
+LDGEMM_L2x16_SUB4:\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_SUBI1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_SUB1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_SUB1\r
+       #dcbt           AO,     PRE\r
+       KERNEL2x16_SUB1\r
+\r
+       KERNEL2x16_SUB1\r
+       KERNEL2x16_SUB1\r
+       KERNEL2x16_SUB1\r
+       KERNEL2x16_SUB1\r
+\r
+       b               LDGEMM_L2x16_SUB1\r
+\r
+LDGEMM_L2x16_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL2x16_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L2x16_SAVE\r
+       b               LDGEMM_L2x16_SUB2\r
+\r
+LDGEMM_L2x16_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L2x16_SAVE\r
+\r
+LDGEMM_L2x16_SUB2:\r
+\r
+       KERNEL2x16_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x16_SUB2\r
+\r
+LDGEMM_L2x16_SAVE:\r
+\r
+       SAVE2x16\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,16,2\r
+#endif \r
+       addic.          I,      I,      -1\r
+       bgt             LDGEMM_L2x16_BEGIN\r
+\r
+LDGEMM_L2x16_END:\r
+\r
+LDGEMM_L2x8_BEGIN:\r
+\r
+       andi.           T2,     M,      15\r
+       ble             LDGEMM_L2x1_END\r
+\r
+       andi.           T1,     M,      8\r
+       ble             LDGEMM_L2x8_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,2\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,8,2\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L2x8_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L2x8_SUB4\r
+\r
+LDGEMM_L2x8_LOOP_START:\r
+\r
+       #dcbt   AO,     PRE\r
+       LOAD2x8_1\r
+       KERNEL2x8_I1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+       KERNEL2x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+\r
+       KERNEL2x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+       KERNEL2x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L2x8_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L2x8_LOOP:\r
+\r
+       KERNEL2x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+       KERNEL2x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+\r
+       KERNEL2x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+       KERNEL2x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL2x8_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x8_LOOP\r
+\r
+LDGEMM_L2x8_LOOP_END:\r
+\r
+       KERNEL2x8_1\r
+       KERNEL2x8_2\r
+       KERNEL2x8_1\r
+       KERNEL2x8_2\r
+\r
+       KERNEL2x8_1\r
+       KERNEL2x8_2\r
+       KERNEL2x8_1\r
+       KERNEL2x8_E2\r
+\r
+       b               LDGEMM_L2x8_SUB1\r
+\r
+LDGEMM_L2x8_SUB4:\r
+\r
+       KERNEL2x8_SUBI1\r
+       KERNEL2x8_SUB1\r
+       KERNEL2x8_SUB1\r
+       KERNEL2x8_SUB1\r
+\r
+       KERNEL2x8_SUB1\r
+       KERNEL2x8_SUB1\r
+       KERNEL2x8_SUB1\r
+       KERNEL2x8_SUB1\r
+\r
+       b               LDGEMM_L2x8_SUB1\r
+\r
+LDGEMM_L2x8_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL2x8_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L2x8_SAVE\r
+       b               LDGEMM_L2x8_SUB2\r
+\r
+LDGEMM_L2x8_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L2x8_SAVE\r
+\r
+LDGEMM_L2x8_SUB2:\r
+\r
+       KERNEL2x8_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x8_SUB2\r
+\r
+LDGEMM_L2x8_SAVE:\r
+\r
+       SAVE2x8\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,8,2\r
+#endif\r
+LDGEMM_L2x8_END:\r
+\r
+LDGEMM_L2x4_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      4\r
+       ble             LDGEMM_L2x4_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,2\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,4,2\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L2x4_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L2x4_SUB4\r
+\r
+LDGEMM_L2x4_LOOP_START:\r
+\r
+       LOAD2x4_1\r
+       KERNEL2x4_I1\r
+       KERNEL2x4_2\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L2x4_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L2x4_LOOP:\r
+\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x4_LOOP\r
+\r
+LDGEMM_L2x4_LOOP_END:\r
+\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+\r
+       KERNEL2x4_1\r
+       KERNEL2x4_2\r
+       KERNEL2x4_1\r
+       KERNEL2x4_E2\r
+\r
+       b               LDGEMM_L2x4_SUB1\r
+\r
+LDGEMM_L2x4_SUB4:\r
+\r
+       KERNEL2x4_SUBI1\r
+       KERNEL2x4_SUB1\r
+       KERNEL2x4_SUB1\r
+       KERNEL2x4_SUB1\r
+\r
+       KERNEL2x4_SUB1\r
+       KERNEL2x4_SUB1\r
+       KERNEL2x4_SUB1\r
+       KERNEL2x4_SUB1\r
+\r
+       b               LDGEMM_L2x4_SUB1\r
+\r
+LDGEMM_L2x4_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL2x4_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L2x4_SAVE\r
+       b               LDGEMM_L2x4_SUB2\r
+\r
+LDGEMM_L2x4_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L2x4_SAVE\r
+\r
+LDGEMM_L2x4_SUB2:\r
+\r
+       KERNEL2x4_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x4_SUB2\r
+\r
+LDGEMM_L2x4_SAVE:\r
+\r
+       SAVE2x4\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,4,2\r
+#endif\r
+LDGEMM_L2x4_END:\r
+\r
+LDGEMM_L2x2_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      2\r
+       ble             LDGEMM_L2x2_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,2\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,2,2\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L2x2_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L2x2_SUB4\r
+\r
+LDGEMM_L2x2_LOOP_START:\r
+\r
+       LOAD2x2_1\r
+       KERNEL2x2_I1\r
+       KERNEL2x2_2\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L2x2_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L2x2_LOOP:\r
+\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x2_LOOP\r
+\r
+LDGEMM_L2x2_LOOP_END:\r
+\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+\r
+       KERNEL2x2_1\r
+       KERNEL2x2_2\r
+       KERNEL2x2_1\r
+       KERNEL2x2_E2\r
+\r
+       b               LDGEMM_L2x2_SUB1\r
+\r
+LDGEMM_L2x2_SUB4:\r
+\r
+       KERNEL2x2_SUBI1\r
+       KERNEL2x2_SUB1\r
+       KERNEL2x2_SUB1\r
+       KERNEL2x2_SUB1\r
+\r
+       KERNEL2x2_SUB1\r
+       KERNEL2x2_SUB1\r
+       KERNEL2x2_SUB1\r
+       KERNEL2x2_SUB1\r
+\r
+       b               LDGEMM_L2x2_SUB1\r
+\r
+LDGEMM_L2x2_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL2x2_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L2x2_SAVE\r
+       b               LDGEMM_L2x2_SUB2\r
+\r
+LDGEMM_L2x2_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L2x2_SAVE\r
+\r
+LDGEMM_L2x2_SUB2:\r
+\r
+       KERNEL2x2_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x2_SUB2\r
+\r
+LDGEMM_L2x2_SAVE:\r
+\r
+       SAVE2x2\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,2,2\r
+#endif\r
+LDGEMM_L2x2_END:\r
+\r
+LDGEMM_L2x1_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      1\r
+       ble             LDGEMM_L2x1_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,2\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,1,2\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L2x1_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L2x1_SUB4\r
+\r
+LDGEMM_L2x1_LOOP_START:\r
+\r
+       LOAD2x1_1\r
+       KERNEL2x1_I1\r
+       KERNEL2x1_2\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L2x1_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L2x1_LOOP:\r
+\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x1_LOOP\r
+\r
+LDGEMM_L2x1_LOOP_END:\r
+\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+\r
+       KERNEL2x1_1\r
+       KERNEL2x1_2\r
+       KERNEL2x1_1\r
+       KERNEL2x1_E2\r
+\r
+       b               LDGEMM_L2x1_SUB1\r
+\r
+LDGEMM_L2x1_SUB4:\r
+\r
+       KERNEL2x1_SUBI1\r
+       KERNEL2x1_SUB1\r
+       KERNEL2x1_SUB1\r
+       KERNEL2x1_SUB1\r
+\r
+       KERNEL2x1_SUB1\r
+       KERNEL2x1_SUB1\r
+       KERNEL2x1_SUB1\r
+       KERNEL2x1_SUB1\r
+\r
+       b               LDGEMM_L2x1_SUB1\r
+\r
+LDGEMM_L2x1_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL2x1_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L2x1_SAVE\r
+       b               LDGEMM_L2x1_SUB2\r
+\r
+LDGEMM_L2x1_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L2x1_SAVE\r
+\r
+LDGEMM_L2x1_SUB2:\r
+\r
+       KERNEL2x1_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L2x1_SUB2\r
+\r
+LDGEMM_L2x1_SAVE:\r
+\r
+       SAVE2x1\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,1,2\r
+#endif\r
+LDGEMM_L2x1_END:\r
+\r
+       slwi            T1,     K,      4\r
+       add             B,      B,      T1\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+    addi TEMP_REG, TEMP_REG, 2\r
+#endif\r
+LDGEMM_L2_END:\r
+LDGEMM_L1_BEGIN:\r
+\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+       mr TEMP_REG, OFFSET      /*off = offset;*/\r
+#endif \r
+       andi.           T1,     N,      1\r
+       ble             LDGEMM_L1_END\r
+       mr              CO,     C\r
+       mr              AO,     A\r
+       srawi.          I,      M,      4\r
+       ble             LDGEMM_L1x16_END\r
+\r
+LDGEMM_L1x16_BEGIN:\r
+\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,1\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,16,1\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L1x16_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L1x16_SUB4\r
+\r
+LDGEMM_L1x16_LOOP_START:\r
+\r
+       #dcbt           AO,     PRE\r
+       LOAD1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_I1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L1x16_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L1x16_LOOP:\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x16_LOOP\r
+\r
+LDGEMM_L1x16_LOOP_END:\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_2\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_1\r
+       KERNEL1x16_E2\r
+\r
+       b               LDGEMM_L1x16_SUB1\r
+\r
+LDGEMM_L1x16_SUB4:\r
+\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_SUBI1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_SUB1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_SUB1\r
+       #dcbt           AO,     PRE\r
+       KERNEL1x16_SUB1\r
+\r
+       KERNEL1x16_SUB1\r
+       KERNEL1x16_SUB1\r
+       KERNEL1x16_SUB1\r
+       KERNEL1x16_SUB1\r
+\r
+       b               LDGEMM_L1x16_SUB1\r
+\r
+LDGEMM_L1x16_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL1x16_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L1x16_SAVE\r
+       b               LDGEMM_L1x16_SUB2\r
+\r
+LDGEMM_L1x16_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L1x16_SAVE\r
+\r
+LDGEMM_L1x16_SUB2:\r
+\r
+       KERNEL1x16_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x16_SUB2\r
+\r
+LDGEMM_L1x16_SAVE:\r
+\r
+       SAVE1x16\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,16,1\r
+#endif\r
+       addic.          I,      I,      -1\r
+       bgt             LDGEMM_L1x16_BEGIN\r
+\r
+LDGEMM_L1x16_END:\r
+\r
+LDGEMM_L1x8_BEGIN:\r
+\r
+       andi.           T2,     M,      15\r
+       ble             LDGEMM_L1x1_END\r
+\r
+       andi.           T1,     M,      8\r
+       ble             LDGEMM_L1x8_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,1\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,8,1\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L1x8_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L1x8_SUB4\r
+\r
+LDGEMM_L1x8_LOOP_START:\r
+\r
+       #dcbt   AO,     PRE\r
+       LOAD1x8_1\r
+       KERNEL1x8_I1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+       KERNEL1x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+\r
+       KERNEL1x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+       KERNEL1x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L1x8_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L1x8_LOOP:\r
+\r
+       KERNEL1x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+       KERNEL1x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+\r
+       KERNEL1x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+       KERNEL1x8_1\r
+       #dcbt   AO,     PRE\r
+       KERNEL1x8_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x8_LOOP\r
+\r
+LDGEMM_L1x8_LOOP_END:\r
+\r
+       KERNEL1x8_1\r
+       KERNEL1x8_2\r
+       KERNEL1x8_1\r
+       KERNEL1x8_2\r
+\r
+       KERNEL1x8_1\r
+       KERNEL1x8_2\r
+       KERNEL1x8_1\r
+       KERNEL1x8_E2\r
+\r
+       b               LDGEMM_L1x8_SUB1\r
+\r
+LDGEMM_L1x8_SUB4:\r
+\r
+       KERNEL1x8_SUBI1\r
+       KERNEL1x8_SUB1\r
+       KERNEL1x8_SUB1\r
+       KERNEL1x8_SUB1\r
+\r
+       KERNEL1x8_SUB1\r
+       KERNEL1x8_SUB1\r
+       KERNEL1x8_SUB1\r
+       KERNEL1x8_SUB1\r
+\r
+       b               LDGEMM_L1x8_SUB1\r
+\r
+LDGEMM_L1x8_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL1x8_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L1x8_SAVE\r
+       b               LDGEMM_L1x8_SUB2\r
+\r
+LDGEMM_L1x8_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L1x8_SAVE\r
+\r
+LDGEMM_L1x8_SUB2:\r
+\r
+       KERNEL1x8_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x8_SUB2\r
+\r
+LDGEMM_L1x8_SAVE:\r
+\r
+       SAVE1x8\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,8,1\r
+#endif\r
+LDGEMM_L1x8_END:\r
+\r
+LDGEMM_L1x4_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      4\r
+       ble             LDGEMM_L1x4_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,1\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,4,1\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L1x4_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L1x4_SUB4\r
+\r
+LDGEMM_L1x4_LOOP_START:\r
+\r
+       LOAD1x4_1\r
+       KERNEL1x4_I1\r
+       KERNEL1x4_2\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L1x4_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L1x4_LOOP:\r
+\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x4_LOOP\r
+\r
+LDGEMM_L1x4_LOOP_END:\r
+\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+\r
+       KERNEL1x4_1\r
+       KERNEL1x4_2\r
+       KERNEL1x4_1\r
+       KERNEL1x4_E2\r
+\r
+       b               LDGEMM_L1x4_SUB1\r
+\r
+LDGEMM_L1x4_SUB4:\r
+\r
+       KERNEL1x4_SUBI1\r
+       KERNEL1x4_SUB1\r
+       KERNEL1x4_SUB1\r
+       KERNEL1x4_SUB1\r
+\r
+       KERNEL1x4_SUB1\r
+       KERNEL1x4_SUB1\r
+       KERNEL1x4_SUB1\r
+       KERNEL1x4_SUB1\r
+\r
+       b               LDGEMM_L1x4_SUB1\r
+\r
+LDGEMM_L1x4_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL1x4_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L1x4_SAVE\r
+       b               LDGEMM_L1x4_SUB2\r
+\r
+LDGEMM_L1x4_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L1x4_SAVE\r
+\r
+LDGEMM_L1x4_SUB2:\r
+\r
+       KERNEL1x4_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x4_SUB2\r
+\r
+LDGEMM_L1x4_SAVE:\r
+\r
+       SAVE1x4\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,4,1\r
+#endif\r
+LDGEMM_L1x4_END:\r
+\r
+LDGEMM_L1x2_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      2\r
+       ble             LDGEMM_L1x2_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,1\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,2,1\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L1x2_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L1x2_SUB4\r
+\r
+LDGEMM_L1x2_LOOP_START:\r
+\r
+       LOAD1x2_1\r
+       KERNEL1x2_I1\r
+       KERNEL1x2_2\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L1x2_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L1x2_LOOP:\r
+\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x2_LOOP\r
+\r
+LDGEMM_L1x2_LOOP_END:\r
+\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+\r
+       KERNEL1x2_1\r
+       KERNEL1x2_2\r
+       KERNEL1x2_1\r
+       KERNEL1x2_E2\r
+\r
+       b               LDGEMM_L1x2_SUB1\r
+\r
+LDGEMM_L1x2_SUB4:\r
+\r
+       KERNEL1x2_SUBI1\r
+       KERNEL1x2_SUB1\r
+       KERNEL1x2_SUB1\r
+       KERNEL1x2_SUB1\r
+\r
+       KERNEL1x2_SUB1\r
+       KERNEL1x2_SUB1\r
+       KERNEL1x2_SUB1\r
+       KERNEL1x2_SUB1\r
+\r
+       b               LDGEMM_L1x2_SUB1\r
+\r
+LDGEMM_L1x2_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL1x2_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L1x2_SAVE\r
+       b               LDGEMM_L1x2_SUB2\r
+\r
+LDGEMM_L1x2_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L1x2_SAVE\r
+\r
+LDGEMM_L1x2_SUB2:\r
+\r
+       KERNEL1x2_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x2_SUB2\r
+\r
+LDGEMM_L1x2_SAVE:\r
+\r
+       SAVE1x2\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,2,1\r
+#endif\r
+LDGEMM_L1x2_END:\r
+\r
+LDGEMM_L1x1_BEGIN:\r
+\r
+\r
+       andi.           T1,     M,      1\r
+       ble             LDGEMM_L1x1_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,1\r
+    REFRESH_TEMP_BK T3,K,TEMP_REG,1,1\r
+    srawi.             L, T3,  3       \r
+#else\r
+       mr              BO,     B\r
+       srawi.          L,      K,      3       \r
+#endif \r
+       ble             LDGEMM_L1x1_SUB0\r
+       cmpwi           cr0,    L,      1\r
+       ble             LDGEMM_L1x1_SUB4\r
+\r
+LDGEMM_L1x1_LOOP_START:\r
+\r
+       LOAD1x1_1\r
+       KERNEL1x1_I1\r
+       KERNEL1x1_2\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+\r
+       addic.          L,      L,      -2\r
+       ble             LDGEMM_L1x1_LOOP_END\r
+\r
+       MY_ALIGN\r
+\r
+LDGEMM_L1x1_LOOP:\r
+\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x1_LOOP\r
+\r
+LDGEMM_L1x1_LOOP_END:\r
+\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+\r
+       KERNEL1x1_1\r
+       KERNEL1x1_2\r
+       KERNEL1x1_1\r
+       KERNEL1x1_E2\r
+\r
+       b               LDGEMM_L1x1_SUB1\r
+\r
+LDGEMM_L1x1_SUB4:\r
+\r
+       KERNEL1x1_SUBI1\r
+       KERNEL1x1_SUB1\r
+       KERNEL1x1_SUB1\r
+       KERNEL1x1_SUB1\r
+\r
+       KERNEL1x1_SUB1\r
+       KERNEL1x1_SUB1\r
+       KERNEL1x1_SUB1\r
+       KERNEL1x1_SUB1\r
+\r
+       b               LDGEMM_L1x1_SUB1\r
+\r
+LDGEMM_L1x1_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+\r
+       KERNEL1x1_SUBI1\r
+\r
+       addic.          L,      L,      -1\r
+       ble             LDGEMM_L1x1_SAVE\r
+       b               LDGEMM_L1x1_SUB2\r
+\r
+LDGEMM_L1x1_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T3,     7\r
+#else\r
+       andi.           L,      K,      7\r
+#endif\r
+       ble             LDGEMM_L1x1_SAVE\r
+\r
+LDGEMM_L1x1_SUB2:\r
+\r
+       KERNEL1x1_SUB1\r
+\r
+       addic.          L,      L,      -1\r
+       bgt             LDGEMM_L1x1_SUB2\r
+\r
+LDGEMM_L1x1_SAVE:\r
+\r
+       SAVE1x1\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,1,1\r
+#endif\r
+LDGEMM_L1x1_END:\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+    addi TEMP_REG, TEMP_REG, 1\r
+#endif\r
+LDGEMM_L1_END:\r

diff --git a/kernel/power/dgemm_macros_power9.S b/kernel/power/dgemm_macros_power9.S
new file mode 100644 (file)
index 0000000..c4b8270
--- /dev/null
+++ b/kernel/power/dgemm_macros_power9.S
@@ -0,0 +1,3623 @@
+/***************************************************************************\r
+Copyright (c) 2013-2019, The OpenBLAS Project\r
+All rights reserved.\r
+Redistribution and use in source and binary forms, with or without\r
+modification, are permitted provided that the following conditions are\r
+met:\r
+1. Redistributions of source code must retain the above copyright\r
+notice, this list of conditions and the following disclaimer.\r
+2. Redistributions in binary form must reproduce the above copyright\r
+notice, this list of conditions and the following disclaimer in\r
+the documentation and/or other materials provided with the\r
+distribution.\r
+3. Neither the name of the OpenBLAS project nor the names of\r
+its contributors may be used to endorse or promote products\r
+derived from this software without specific prior written permission.\r
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"\r
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\r
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE\r
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\r
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+*****************************************************************************/\r
+\r
+/**************************************************************************************\r
+* Abdelrauf(quickwritereader@googlemail.com)\r
+*       BLASTEST               : OK\r
+*       CTEST                  : OK\r
+*       TEST                   : OK\r
+*       LAPACK-TEST            : OK\r
+**************************************************************************************/\r
+\r
+/*********************************************************************\r
+* Macros for N=4, M=16                                               *\r
+*********************************************************************/\r
+.macro LOAD4x16_1\r
+   LOAD4x16 1\r
+.endm\r
+\r
+.macro LOAD4x16_0\r
+   LOAD4x16 0\r
+.endm\r
+.macro LOAD4x16  Zero\r
+\r
+       lxv     vs24,   0(BO)\r
+       lxv     vs26,   16(BO)\r
+       xxpermdi        vs25,   vs24,   vs24,2  \r
+       xxpermdi        vs27,   vs26,   vs26,2\r
+\r
+       lxv     vs0,     0(AO)\r
+       lxv     vs1,    16(AO)\r
+       lxv     vs2,    32(AO)\r
+       lxv     vs3,    48(AO)\r
+ \r
+\r
+       lxv     vs4,    64(AO)\r
+       lxv     vs5,    80(AO)\r
+       lxv     vs6,    96(AO)\r
+       lxv     vs7,    112(AO)\r
+.if \Zero==1 \r
+    xxlxor             vs32,vs32,vs32\r
+    xxlxor             vs33,vs33,vs33\r
+       xxlxor          vs34,vs34,vs34\r
+       xxlxor          vs35,vs35,vs35\r
+       xxlxor          vs36,vs36,vs36\r
+       xxlxor          vs37,vs37,vs37\r
+       xxlxor          vs38,vs38,vs38\r
+       xxlxor          vs39,vs39,vs39\r
+       xxlxor          vs40,   vs40,   vs40\r
+       xxlxor          vs41,   vs41,   vs41\r
+       xxlxor          vs42,   vs42,   vs42\r
+       xxlxor          vs43,   vs43,   vs43\r
+       xxlxor          vs44,   vs44,   vs44\r
+       xxlxor          vs45,   vs45,   vs45\r
+       xxlxor          vs46,   vs46,   vs46\r
+       xxlxor          vs47,   vs47,   vs47\r
+       xxlxor          vs48,   vs48,   vs48\r
+       xxlxor          vs49,   vs49,   vs49\r
+       xxlxor          vs50,   vs50,   vs50\r
+       xxlxor          vs51,   vs51,   vs51 \r
+       xxlxor          vs52,   vs52,   vs52\r
+       xxlxor          vs53,   vs53,   vs53\r
+       xxlxor          vs54,   vs54,   vs54\r
+       xxlxor          vs55,   vs55,   vs55 \r
+       xxlxor          vs56,   vs56,   vs56\r
+       xxlxor          vs57,   vs57,   vs57\r
+       xxlxor          vs58,   vs58,   vs58\r
+       xxlxor          vs59,   vs59,   vs59 \r
+       xxlxor          vs60,   vs60,   vs60\r
+       xxlxor          vs61,   vs61,   vs61\r
+       xxlxor          vs62,   vs62,   vs62\r
+       xxlxor          vs63,   vs63,   vs63    \r
+.endif\r
+.endm\r
+\r
+  \r
+#define unit_size 8\r
+#define DISP32(ind,disp) (ind*unit_size*32+disp)\r
+#define DISP16(ind,disp) (ind*unit_size*16+disp)\r
+#define DISP8(ind,disp) (ind*unit_size*8+disp)\r
+#define DISP4(ind,disp) (ind*unit_size*4+disp)\r
+#define DISP2(ind,disp) (ind*unit_size*2+disp)\r
+#define DISP1(ind,disp) (ind*unit_size+disp)\r
+\r
+.macro KERNEL4x16_L1_L2  Index,IsLast\r
+  KERNEL4x16_L1_L2_I AO,BO, 0,0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+\r
+\r
+.macro KERNEL4x16_I1_L2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L2_I  AO,BO,1,\OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I1_L2_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L2_I  AO,BO, 0,\OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L2_I  AO,BO, 0,\OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL4x16_I2_L2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L2_I  \AREG,\BREG,1,\OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I2_L2_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L2_I  \AREG,\BREG, 0,\OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I2_L2_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L2_I \AREG,\BREG, 0,\OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL4x16_L1_L2_I  AREG,BREG, First, OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+.if \First ==1\r
+       xvmuldp         vs32,   vs0,    vs24\r
+       xvmuldp         vs33,   vs1,    vs24\r
+       xvmuldp         vs34,   vs2,    vs24\r
+       xvmuldp         vs35,   vs3,    vs24\r
+.else\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+.endif\r
+       lxv     vs8,    DISP32(\Index,0+\OffsetA)(\AREG)\r
+       lxv     vs9,    DISP32(\Index,16+\OffsetA)(\AREG)\r
+       lxv     vs10,   DISP32(\Index,32+\OffsetA)(\AREG)\r
+       lxv     vs11,   DISP32(\Index,48+\OffsetA)(\AREG)\r
+.if \First ==1\r
+       xvmuldp         vs36,   vs4,    vs24\r
+       xvmuldp         vs37,   vs5,    vs24\r
+       xvmuldp         vs38,   vs6,    vs24\r
+       xvmuldp         vs39,   vs7,    vs24\r
+.else\r
+       xvmaddadp               vs36,   vs4,    vs24\r
+       xvmaddadp               vs37,   vs5,    vs24\r
+       xvmaddadp               vs38,   vs6,    vs24\r
+       xvmaddadp               vs39,   vs7,    vs24\r
+.endif\r
+       lxv     vs28,   DISP8(\Index,0  +\OffsetB)(\BREG)\r
+       lxv     vs30,   DISP8(\Index,16  +\OffsetB)(\BREG)\r
+       xxpermdi        vs29,   vs28,   vs28,2  \r
+       xxpermdi        vs31,   vs30,   vs30,2\r
+.if \First ==1\r
+       xvmuldp         vs40,   vs0,    vs25\r
+       xvmuldp         vs41,   vs1,    vs25\r
+       xvmuldp         vs42,   vs2,    vs25\r
+       xvmuldp         vs43,   vs3,    vs25\r
+\r
+\r
+       xvmuldp         vs44,   vs4,    vs25\r
+       xvmuldp         vs45,   vs5,    vs25\r
+       xvmuldp         vs46,   vs6,    vs25\r
+       xvmuldp         vs47,   vs7,    vs25\r
+\r
+\r
+       xvmuldp         vs48,   vs0,    vs26\r
+       xvmuldp         vs49,   vs1,    vs26\r
+       xvmuldp         vs50,   vs2,    vs26\r
+       xvmuldp         vs51,   vs3,    vs26\r
+\r
+\r
+.else\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+\r
+\r
+       xvmaddadp               vs44,   vs4,    vs25\r
+       xvmaddadp               vs45,   vs5,    vs25\r
+       xvmaddadp               vs46,   vs6,    vs25\r
+       xvmaddadp               vs47,   vs7,    vs25\r
+\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+       xvmaddadp               vs49,   vs1,    vs26\r
+       xvmaddadp               vs50,   vs2,    vs26\r
+       xvmaddadp               vs51,   vs3,    vs26\r
+\r
+.endif\r
+       lxv     vs12,  DISP32(\Index,64+\OffsetA)(\AREG)\r
+       lxv     vs13,  DISP32(\Index,80+\OffsetA)(\AREG)\r
+.if \First ==1\r
+       xvmuldp         vs52,   vs4,    vs26\r
+       xvmuldp         vs53,   vs5,    vs26\r
+       xvmuldp         vs54,   vs6,    vs26\r
+       xvmuldp         vs55,   vs7,    vs26\r
+\r
+.else\r
+       xvmaddadp               vs52,   vs4,    vs26\r
+       xvmaddadp               vs53,   vs5,    vs26\r
+       xvmaddadp               vs54,   vs6,    vs26\r
+       xvmaddadp               vs55,   vs7,    vs26\r
+.endif\r
+       lxv     vs14,  DISP32(\Index,96+\OffsetA)(\AREG)\r
+       lxv     vs15,  DISP32(\Index,112+\OffsetA)(\AREG)\r
+.if \First ==1\r
+       xvmuldp         vs56,   vs0,    vs27\r
+       xvmuldp         vs57,   vs1,    vs27\r
+       xvmuldp         vs58,   vs2,    vs27\r
+       xvmuldp         vs59,   vs3,    vs27\r
+\r
+ \r
+\r
+       xvmuldp         vs60,   vs4,    vs27\r
+       xvmuldp         vs61,   vs5,    vs27\r
+       xvmuldp         vs62,   vs6,    vs27\r
+       xvmuldp         vs63,   vs7,    vs27\r
+\r
+.else\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+       xvmaddadp               vs57,   vs1,    vs27\r
+       xvmaddadp               vs58,   vs2,    vs27\r
+       xvmaddadp               vs59,   vs3,    vs27\r
+\r
+ \r
+\r
+       xvmaddadp               vs60,   vs4,    vs27\r
+       xvmaddadp               vs61,   vs5,    vs27\r
+       xvmaddadp               vs62,   vs6,    vs27\r
+       xvmaddadp               vs63,   vs7,    vs27\r
+.endif\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+.if \Complete==0\r
+       lxv     vs0,    DISP32(\Index,128+\OffsetA)(\AREG)\r
+       lxv     vs1,    DISP32(\Index,144+\OffsetA)(\AREG)\r
+.endif\r
+       xvmaddadp               vs36,   vs12,   vs28\r
+       xvmaddadp               vs37,   vs13,   vs28\r
+       xvmaddadp               vs38,   vs14,   vs28\r
+       xvmaddadp               vs39,   vs15,   vs28\r
+.if \Complete==0\r
+       lxv     vs24,   DISP8(\Index,32  +\OffsetB)(\BREG)\r
+       lxv     vs26,   DISP8(\Index,48  +\OffsetB)(\BREG)\r
+       xxpermdi        vs25,   vs24,   vs24,2  \r
+       xxpermdi        vs27,   vs26,   vs26,2\r
+.endif\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+       xvmaddadp               vs42,   vs10,   vs29\r
+       xvmaddadp               vs43,   vs11,   vs29\r
+.if \Complete==0\r
+       lxv     vs2,    DISP32(\Index,160+\OffsetA)(\AREG)\r
+       lxv     vs3,    DISP32(\Index,176+\OffsetA)(\AREG)\r
+.endif\r
+       xvmaddadp               vs44,   vs12,   vs29\r
+       xvmaddadp               vs45,   vs13,   vs29\r
+       xvmaddadp               vs46,   vs14,   vs29\r
+       xvmaddadp               vs47,   vs15,   vs29\r
+\r
+\r
+       xvmaddadp               vs48,   vs8,    vs30\r
+       xvmaddadp               vs49,   vs9,    vs30\r
+       xvmaddadp               vs50,   vs10,   vs30\r
+       xvmaddadp               vs51,   vs11,   vs30\r
+.if \Complete==0\r
+       lxv     vs4,    DISP32(\Index,192+\OffsetA)(\AREG)\r
+       lxv     vs5,    DISP32(\Index,208+\OffsetA)(\AREG)\r
+.endif\r
+       xvmaddadp               vs52,   vs12,   vs30\r
+       xvmaddadp               vs53,   vs13,   vs30\r
+       xvmaddadp               vs54,   vs14,   vs30\r
+       xvmaddadp               vs55,   vs15,   vs30\r
+.if \Complete==0\r
+       lxv     vs6,    DISP32(\Index,224+\OffsetA)(\AREG)\r
+       lxv     vs7,    DISP32(\Index,240+\OffsetA)(\AREG)\r
+.endif\r
+       xvmaddadp               vs56,   vs8,    vs31\r
+       xvmaddadp               vs57,   vs9,    vs31\r
+       xvmaddadp               vs58,   vs10,   vs31\r
+       xvmaddadp               vs59,   vs11,   vs31\r
+ \r
+\r
+       xvmaddadp               vs60,   vs12,   vs31\r
+       \r
+       xvmaddadp               vs61,   vs13,   vs31\r
+       xvmaddadp               vs62,   vs14,   vs31\r
+       \r
+       xvmaddadp               vs63,   vs15,   vs31\r
+  .if \IsLast==1       \r
+  .if \Complete==1\r
+       addi            \AREG, \AREG, DISP32(\Index,128+\OffsetA)\r
+       addi            \BREG, \BREG,  DISP8(\Index,32+\OffsetB)\r
+  .else\r
+       addi            \AREG, \AREG, DISP32(\Index,256)\r
+       addi            \BREG, \BREG,  DISP8(\Index,64)\r
+  .endif\r
+  .endif\r
+  \r
+\r
+.endm\r
+\r
+ \r
+\r
+.macro KERNEL4x16 First\r
+\r
+       lxv     vs24,   0(BO)\r
+       lxv     vs26,   16(BO)\r
+       xxpermdi        vs25,   vs24,   vs24,2  \r
+       xxpermdi        vs27,   vs26,   vs26,2\r
+\r
+       lxv     vs0,    0(AO)\r
+       lxv     vs1,    16(AO)\r
+       lxv     vs2,    32(AO)\r
+       lxv     vs3,    48(AO) \r
+\r
+       lxv     vs4,    64(AO)\r
+       lxv     vs5,    80(AO)\r
+       lxv     vs6,    96(AO)\r
+       lxv     vs7,    112(AO)\r
+\r
+\r
+ \r
+       addi            BO, BO, 32\r
+  addi         AO, AO, 128\r
+\r
+.if \First==1\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+       xvmuldp                 vs36,   vs4,    vs24\r
+       xvmuldp                 vs37,   vs5,    vs24\r
+       xvmuldp                 vs38,   vs6,    vs24\r
+       xvmuldp                 vs39,   vs7,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+       xvmuldp                 vs42,   vs2,    vs25\r
+       xvmuldp                 vs43,   vs3,    vs25\r
+       xvmuldp                 vs44,   vs4,    vs25\r
+       xvmuldp                 vs45,   vs5,    vs25\r
+       xvmuldp                 vs46,   vs6,    vs25\r
+       xvmuldp                 vs47,   vs7,    vs25\r
+\r
+       xvmuldp                 vs48,   vs0,    vs26\r
+       xvmuldp                 vs49,   vs1,    vs26\r
+       xvmuldp                 vs50,   vs2,    vs26\r
+       xvmuldp                 vs51,   vs3,    vs26\r
+       xvmuldp                 vs52,   vs4,    vs26\r
+       xvmuldp                 vs53,   vs5,    vs26\r
+       xvmuldp                 vs54,   vs6,    vs26\r
+       xvmuldp                 vs55,   vs7,    vs26\r
+\r
+       xvmuldp                 vs56,   vs0,    vs27\r
+       xvmuldp                 vs57,   vs1,    vs27\r
+       xvmuldp                 vs58,   vs2,    vs27\r
+       xvmuldp                 vs59,   vs3,    vs27\r
+       xvmuldp                 vs60,   vs4,    vs27\r
+       xvmuldp                 vs61,   vs5,    vs27\r
+       xvmuldp                 vs62,   vs6,    vs27\r
+       xvmuldp                 vs63,   vs7,    vs27\r
+.else\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+       xvmaddadp               vs36,   vs4,    vs24\r
+       xvmaddadp               vs37,   vs5,    vs24\r
+       xvmaddadp               vs38,   vs6,    vs24\r
+       xvmaddadp               vs39,   vs7,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+ \r
+       xvmaddadp               vs44,   vs4,    vs25\r
+       xvmaddadp               vs45,   vs5,    vs25\r
+       xvmaddadp               vs46,   vs6,    vs25\r
+       xvmaddadp               vs47,   vs7,    vs25\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+       xvmaddadp               vs49,   vs1,    vs26\r
+       xvmaddadp               vs50,   vs2,    vs26\r
+       xvmaddadp               vs51,   vs3,    vs26\r
+ \r
+       xvmaddadp               vs52,   vs4,    vs26\r
+       xvmaddadp               vs53,   vs5,    vs26\r
+       xvmaddadp               vs54,   vs6,    vs26\r
+       xvmaddadp               vs55,   vs7,    vs26\r
+\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+       xvmaddadp               vs57,   vs1,    vs27\r
+       xvmaddadp               vs58,   vs2,    vs27\r
+       xvmaddadp               vs59,   vs3,    vs27\r
+       xvmaddadp               vs60,   vs4,    vs27\r
+       xvmaddadp               vs61,   vs5,    vs27\r
+       xvmaddadp               vs62,   vs6,    vs27\r
+       xvmaddadp               vs63,   vs7,    vs27\r
+\r
+.endif\r
+.endm\r
+\r
+.macro SAVE4x16_REGS\r
+       add             C2,     CO,     LDC\r
+       add             C3,     C2,     LDC\r
+       add             C4,     C3,     LDC\r
+.endm\r
+\r
+.macro SAVE4x16\r
+#ifndef TRMMKERNEL\r
+       lxv             vs0,    0(CO)\r
+       lxv             vs2,    16(CO)\r
+       lxv             vs4,    32(CO)\r
+       lxv             vs6,    48(CO)\r
+#endif \r
+       xxpermdi  vs8, vs40,vs32,1\r
+       xxpermdi  vs9 ,vs32,vs40,1\r
+#ifndef TRMMKERNEL\r
+       lxv             vs24,   64(CO)\r
+       lxv             vs26,   80(CO)\r
+       lxv             vs28,   96(CO)\r
+       lxv             vs30,   112(CO)\r
+#endif \r
+       xxpermdi  vs10, vs41,vs33,1              \r
+       xxpermdi  vs11 ,vs33,vs41,1\r
+#ifndef TRMMKERNEL      \r
+       lxv             vs1,    0(C2)\r
+       lxv             vs3,    16(C2)\r
+       lxv             vs5,    32(C2)\r
+       lxv             vs7,    48(C2)\r
+#endif \r
+       xxpermdi  vs12, vs42,vs34,1\r
+       xxpermdi  vs13 ,vs34,vs42,1\r
+#ifndef TRMMKERNEL\r
+       lxv             vs25,   64(C2)\r
+       lxv             vs27,   80(C2)\r
+#endif \r
+       xxpermdi  vs14, vs43,vs35,1              \r
+       xxpermdi  vs15 ,vs35,vs43,1     \r
+#ifndef TRMMKERNEL      \r
+       lxv             vs29,   96(C2)\r
+       lxv             vs31,   112(C2) \r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs8,    alpha_r \r
+       xvmaddadp       vs1,    vs9,    alpha_r \r
+       xvmaddadp       vs2,    vs10,   alpha_r \r
+       xvmaddadp       vs3,    vs11,   alpha_r \r
+#else\r
+       xvmuldp vs0,    vs8,    alpha_r \r
+       xvmuldp vs1,    vs9,    alpha_r \r
+       xvmuldp vs2,    vs10,   alpha_r \r
+       xvmuldp vs3,    vs11,   alpha_r \r
+\r
+#endif\r
+       xxpermdi  vs8, vs44,vs36,1\r
+       xxpermdi  vs9 ,vs36,vs44,1\r
+       xxpermdi  vs10, vs45,vs37,1              \r
+       xxpermdi  vs11 ,vs37,vs45,1\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs4,    vs12,   alpha_r \r
+       xvmaddadp       vs5,    vs13,   alpha_r \r
+       xvmaddadp       vs6,    vs14,   alpha_r \r
+       xvmaddadp       vs7,    vs15,   alpha_r \r
+#else\r
+       xvmuldp vs4,    vs12,   alpha_r \r
+       xvmuldp vs5,    vs13,   alpha_r \r
+       xvmuldp vs6,    vs14,   alpha_r \r
+       xvmuldp vs7,    vs15,   alpha_r \r
+#endif\r
+       xxpermdi  vs12, vs46,vs38,1\r
+       xxpermdi  vs13 ,vs38,vs46,1\r
+       xxpermdi  vs14, vs47,vs39,1              \r
+       xxpermdi  vs15 ,vs39,vs47,1\r
+\r
+#ifndef TRMMKERNEL \r
+       xvmaddadp       vs24,   vs8,    alpha_r \r
+       xvmaddadp       vs25,   vs9,    alpha_r \r
+       xvmaddadp       vs26,   vs10,   alpha_r \r
+       xvmaddadp       vs27,   vs11,   alpha_r \r
+\r
+       xvmaddadp       vs28,   vs12,   alpha_r \r
+       xvmaddadp       vs29,   vs13,   alpha_r \r
+       xvmaddadp       vs30,   vs14,   alpha_r \r
+       xvmaddadp       vs31,   vs15,   alpha_r \r
+#else\r
+       xvmuldp vs24,   vs8,    alpha_r \r
+       xvmuldp vs25,   vs9,    alpha_r \r
+       xvmuldp vs26,   vs10,   alpha_r \r
+       xvmuldp vs27,   vs11,   alpha_r \r
+\r
+       xvmuldp vs28,   vs12,   alpha_r \r
+       xvmuldp vs29,   vs13,   alpha_r \r
+       xvmuldp vs30,   vs14,   alpha_r \r
+       xvmuldp vs31,   vs15,   alpha_r \r
+\r
+#endif\r
+       stxv            vs0,    0(CO)\r
+       stxv            vs2,    16(CO)\r
+       stxv            vs4,    32(CO)\r
+       stxv            vs6,    48(CO)\r
+\r
+       stxv            vs24,   64(CO)\r
+       stxv            vs26,   80(CO)\r
+       stxv            vs28,   96(CO)\r
+       stxv            vs30,   112(CO)\r
+\r
+       stxv            vs1,    0(C2)\r
+       stxv            vs3,    16(C2)\r
+       stxv            vs5,    32(C2)\r
+       stxv            vs7,    48(C2)\r
+       \r
+       stxv            vs25,   64(C2)\r
+       stxv            vs27,   80(C2)\r
+       stxv            vs29,   96(C2)\r
+       stxv            vs31,   112(C2) \r
+#ifndef TRMMKERNEL\r
+       lxv             vs0,    0(C3)\r
+       lxv             vs2,    16(C3)\r
+       lxv             vs4,    32(C3)\r
+       lxv             vs6,    48(C3)\r
+#endif \r
+       xxpermdi  vs8, vs56,vs48,1\r
+       xxpermdi  vs9 ,vs48,vs56,1\r
+#ifndef TRMMKERNEL      \r
+       lxv             vs24,   64(C3)\r
+       lxv             vs26,   80(C3)\r
+#endif \r
+       xxpermdi  vs10, vs57,vs49,1              \r
+       xxpermdi  vs11 ,vs49,vs57,1     \r
+#ifndef TRMMKERNEL      \r
+       lxv             vs28,   96(C3)\r
+       lxv             vs30,   112(C3)\r
+#endif \r
+       xxpermdi  vs12, vs58,vs50,1\r
+       xxpermdi  vs13 ,vs50,vs58,1\r
+#ifndef TRMMKERNEL      \r
+       lxv             vs1,    0(C4)\r
+       lxv             vs3,    16(C4)\r
+#endif \r
+       xxpermdi  vs14, vs59,vs51,1              \r
+       xxpermdi  vs15 ,vs51,vs59,1     \r
+#ifndef TRMMKERNEL      \r
+       lxv             vs5,    32(C4)\r
+       lxv             vs7,    48(C4)\r
+\r
+       lxv             vs25,   64(C4)\r
+       lxv             vs27,   80(C4)\r
+       lxv             vs29,   96(C4)\r
+       lxv             vs31,   112(C4) \r
+#endif\r
+ \r
+#ifndef TRMMKERNEL \r
+       xvmaddadp       vs0,    vs8,    alpha_r \r
+       xvmaddadp       vs1,    vs9,    alpha_r \r
+       xvmaddadp       vs2,    vs10,   alpha_r \r
+       xvmaddadp       vs3,    vs11,   alpha_r \r
+#else\r
+       xvmuldp vs0,    vs8,    alpha_r \r
+       xvmuldp vs1,    vs9,    alpha_r \r
+       xvmuldp vs2,    vs10,   alpha_r \r
+       xvmuldp vs3,    vs11,   alpha_r \r
+\r
+#endif\r
+\r
+       xxpermdi  vs8, vs60,vs52,1\r
+       xxpermdi  vs9 ,vs52,vs60,1\r
+       xxpermdi  vs10, vs61,vs53,1              \r
+       xxpermdi  vs11 ,vs53,vs61,1\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs4,    vs12,   alpha_r \r
+       xvmaddadp       vs5,    vs13,   alpha_r \r
+       xvmaddadp       vs6,    vs14,   alpha_r \r
+       xvmaddadp       vs7,    vs15,   alpha_r \r
+#else\r
+       xvmuldp vs4,    vs12,   alpha_r \r
+       xvmuldp vs5,    vs13,   alpha_r \r
+       xvmuldp vs6,    vs14,   alpha_r \r
+       xvmuldp vs7,    vs15,   alpha_r \r
+#endif\r
+\r
+\r
+       xxpermdi  vs12, vs62,vs54,1\r
+       xxpermdi  vs13 ,vs54,vs62,1\r
+       xxpermdi  vs14, vs63,vs55,1              \r
+       xxpermdi  vs15 ,vs55,vs63,1\r
+#ifndef TRMMKERNEL \r
+       xvmaddadp       vs24,   vs8,    alpha_r \r
+       xvmaddadp       vs25,   vs9,    alpha_r \r
+       xvmaddadp       vs26,   vs10,   alpha_r \r
+       xvmaddadp       vs27,   vs11,   alpha_r \r
+\r
+       xvmaddadp       vs28,   vs12,   alpha_r \r
+       xvmaddadp       vs29,   vs13,   alpha_r \r
+       xvmaddadp       vs30,   vs14,   alpha_r \r
+       xvmaddadp       vs31,   vs15,   alpha_r \r
+#else\r
+       xvmuldp vs24,   vs8,    alpha_r \r
+       xvmuldp vs25,   vs9,    alpha_r \r
+       xvmuldp vs26,   vs10,   alpha_r \r
+       xvmuldp vs27,   vs11,   alpha_r \r
+\r
+       xvmuldp vs28,   vs12,   alpha_r \r
+       xvmuldp vs29,   vs13,   alpha_r \r
+       xvmuldp vs30,   vs14,   alpha_r \r
+       xvmuldp vs31,   vs15,   alpha_r \r
+#endif\r
+       stxv            vs0,    0(C3)\r
+       stxv            vs2,    16(C3)\r
+       stxv            vs4,    32(C3)\r
+       stxv            vs6,    48(C3)\r
+\r
+       stxv            vs24,   64(C3)\r
+       stxv            vs26,   80(C3)\r
+       stxv            vs28,   96(C3)\r
+       stxv            vs30,   112(C3)\r
+\r
+       stxv            vs1,    0(C4)\r
+       stxv            vs3,    16(C4)\r
+       stxv            vs5,    32(C4)\r
+       stxv            vs7,    48(C4)\r
+       \r
+       stxv            vs25,   64(C4)\r
+       stxv            vs27,   80(C4)\r
+       stxv            vs29,   96(C4)\r
+       stxv            vs31,   112(C4) \r
+\r
+       addi            CO,     CO,     128\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=4, M=8                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD4x8_1\r
+   LOAD4x8 1\r
+.endm\r
+\r
+.macro LOAD4x8_0\r
+   LOAD4x8 0\r
+.endm\r
+.macro LOAD4x8  Zero\r
+\r
+       lxv     vs24,   0(BO)\r
+       lxv     vs26,   16(BO)\r
+       xxpermdi        vs25,   vs24,   vs24,2  \r
+       xxpermdi        vs27,   vs26,   vs26,2\r
+\r
+       lxv     vs0,     0(AO)\r
+       lxv     vs1,    16(AO)\r
+       lxv     vs2,    32(AO)\r
+       lxv     vs3,    48(AO)\r
+ \r
+\r
+\r
+.if \Zero==1 \r
+    xxlxor             vs32,vs32,vs32\r
+    xxlxor             vs33,vs33,vs33\r
+       xxlxor          vs34,vs34,vs34\r
+       xxlxor          vs35,vs35,vs35\r
+\r
+       xxlxor          vs40,   vs40,   vs40\r
+       xxlxor          vs41,   vs41,   vs41\r
+       xxlxor          vs42,   vs42,   vs42\r
+       xxlxor          vs43,   vs43,   vs43\r
+\r
+       xxlxor          vs48,   vs48,   vs48\r
+       xxlxor          vs49,   vs49,   vs49\r
+       xxlxor          vs50,   vs50,   vs50\r
+       xxlxor          vs51,   vs51,   vs51 \r
+\r
+       xxlxor          vs56,   vs56,   vs56\r
+       xxlxor          vs57,   vs57,   vs57\r
+       xxlxor          vs58,   vs58,   vs58\r
+       xxlxor          vs59,   vs59,   vs59 \r
+\r
+.endif\r
+.endm\r
+\r
+  \r
+ \r
+.macro KERNEL4x8_L1_L2  Index,IsLast\r
+  KERNEL4x8_L1_L2_I  0,0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+\r
+\r
+.macro KERNEL4x8_I1_L2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L2_I  1,\OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x8_I1_L2_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L2_I  0,\OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x8_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L2_I  0,\OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL4x8_L1_L2_I  First, OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+       lxv     vs8,    DISP16(\Index,0+\OffsetA)(AO)\r
+       lxv     vs9,    DISP16(\Index,16+\OffsetA)(AO)\r
+.if \First ==1\r
+       xvmuldp         vs32,   vs0,    vs24\r
+       xvmuldp         vs33,   vs1,    vs24\r
+       xvmuldp         vs34,   vs2,    vs24\r
+       xvmuldp         vs35,   vs3,    vs24\r
+.else\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+.endif\r
+\r
+       lxv     vs10,   DISP16(\Index,32+\OffsetA)(AO)\r
+       lxv     vs11,   DISP16(\Index,48+\OffsetA)(AO)\r
+\r
+\r
+\r
+.if \First ==1\r
+       xvmuldp         vs40,   vs0,    vs25\r
+       xvmuldp         vs41,   vs1,    vs25\r
+       xvmuldp         vs42,   vs2,    vs25\r
+       xvmuldp         vs43,   vs3,    vs25\r
+\r
+\r
+       xvmuldp         vs48,   vs0,    vs26\r
+       xvmuldp         vs49,   vs1,    vs26\r
+       xvmuldp         vs50,   vs2,    vs26\r
+       xvmuldp         vs51,   vs3,    vs26\r
+\r
+\r
+.else\r
+\r
+       lxv     vs28,   DISP8(\Index,0  +\OffsetB)(BO)\r
+       lxv     vs30,   DISP8(\Index,16  +\OffsetB)(BO)\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+       xvmaddadp               vs49,   vs1,    vs26\r
+       xvmaddadp               vs50,   vs2,    vs26\r
+       xvmaddadp               vs51,   vs3,    vs26\r
+\r
+.endif\r
+       xxpermdi        vs29,   vs28,   vs28,2  \r
+       xxpermdi        vs31,   vs30,   vs30,2\r
+.if \First ==1\r
+       xvmuldp         vs56,   vs0,    vs27\r
+       xvmuldp         vs57,   vs1,    vs27\r
+       xvmuldp         vs58,   vs2,    vs27\r
+       xvmuldp         vs59,   vs3,    vs27\r
+\r
+.else\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+       xvmaddadp               vs57,   vs1,    vs27\r
+       xvmaddadp               vs58,   vs2,    vs27\r
+       xvmaddadp               vs59,   vs3,    vs27\r
+\r
+.endif\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+.if \Complete==0\r
+       lxv     vs0,    DISP16(\Index,64+\OffsetA)(AO)\r
+       lxv     vs1,    DISP16(\Index,80+\OffsetA)(AO) \r
+.endif\r
+\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+       xvmaddadp               vs42,   vs10,   vs29\r
+       xvmaddadp               vs43,   vs11,   vs29\r
+\r
+.if \Complete==0 \r
+       lxv     vs2,    DISP16(\Index,96+\OffsetA)(AO)\r
+       lxv     vs3,    DISP16(\Index,112+\OffsetA)(AO)\r
+.endif \r
+\r
+\r
+       xvmaddadp               vs48,   vs8,    vs30\r
+       xvmaddadp               vs49,   vs9,    vs30\r
+       xvmaddadp               vs50,   vs10,   vs30\r
+       xvmaddadp               vs51,   vs11,   vs30\r
+.if \Complete==0\r
+       lxv     vs24,   DISP8(\Index,32  +\OffsetB)(BO)\r
+       lxv     vs26,   DISP8(\Index,48  +\OffsetB)(BO) \r
+.endif\r
+ \r
+       xvmaddadp               vs56,   vs8,    vs31\r
+       xvmaddadp               vs57,   vs9,    vs31\r
+       xvmaddadp               vs58,   vs10,   vs31\r
+       xvmaddadp               vs59,   vs11,   vs31\r
+.if \Complete==0 \r
+       xxpermdi        vs25,   vs24,   vs24,2  \r
+       xxpermdi        vs27,   vs26,   vs26,2\r
+.endif\r
+\r
+  .if \IsLast==1       \r
+  .if \Complete==1\r
+       addi            AO, AO, DISP16(\Index,64+\OffsetA)\r
+       addi            BO, BO,  DISP8(\Index,32+\OffsetB)\r
+  .else\r
+       addi            AO, AO, DISP16(\Index,128)\r
+       addi            BO, BO,  DISP8(\Index,64)\r
+  .endif\r
+  .endif\r
+  \r
+\r
+.endm\r
+\r
+ \r
+\r
+.macro KERNEL4x8 First\r
+\r
+       lxv     vs24,   0(BO)\r
+       lxv     vs26,   16(BO)\r
+       xxpermdi        vs25,   vs24,   vs24,2  \r
+       xxpermdi        vs27,   vs26,   vs26,2\r
+\r
+       lxv     vs0,    0(AO)\r
+       lxv     vs1,    16(AO)\r
+       lxv     vs2,    32(AO)\r
+       lxv     vs3,    48(AO) \r
+\r
+\r
+\r
+ \r
+       addi            BO, BO, 32\r
+    addi               AO, AO, 64\r
+\r
+.if \First==1\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+ \r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+       xvmuldp                 vs42,   vs2,    vs25\r
+       xvmuldp                 vs43,   vs3,    vs25\r
+ \r
+\r
+       xvmuldp                 vs48,   vs0,    vs26\r
+       xvmuldp                 vs49,   vs1,    vs26\r
+       xvmuldp                 vs50,   vs2,    vs26\r
+       xvmuldp                 vs51,   vs3,    vs26\r
+ \r
+\r
+       xvmuldp                 vs56,   vs0,    vs27\r
+       xvmuldp                 vs57,   vs1,    vs27\r
+       xvmuldp                 vs58,   vs2,    vs27\r
+       xvmuldp                 vs59,   vs3,    vs27\r
+ \r
+.else\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+ \r
+\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+       xvmaddadp               vs49,   vs1,    vs26\r
+       xvmaddadp               vs50,   vs2,    vs26\r
+       xvmaddadp               vs51,   vs3,    vs26\r
+ \r
+\r
+\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+       xvmaddadp               vs57,   vs1,    vs27\r
+       xvmaddadp               vs58,   vs2,    vs27\r
+       xvmaddadp               vs59,   vs3,    vs27\r
+\r
+\r
+.endif\r
+.endm\r
+\r
+ \r
+\r
+.macro SAVE4x8\r
+       add             T2,     CO,     LDC\r
+       add             T3,     T2,     LDC\r
+       add             T4,     T3,     LDC\r
+#ifndef TRMMKERNEL\r
+       lxv             vs0,    0(CO)\r
+       lxv             vs2,    16(CO)\r
+#endif \r
+       xxpermdi  vs8, vs40,vs32,1\r
+       xxpermdi  vs9 ,vs32,vs40,1\r
+#ifndef TRMMKERNEL      \r
+       lxv             vs4,    32(CO)\r
+       lxv             vs6,    48(CO)\r
+#endif \r
+       xxpermdi  vs10, vs41,vs33,1              \r
+       xxpermdi  vs11 ,vs33,vs41,1\r
+#ifndef TRMMKERNEL      \r
+       lxv             vs1,    0(T2)\r
+       lxv             vs3,    16(T2)\r
+#endif \r
+       xxpermdi  vs12, vs42,vs34,1\r
+       xxpermdi  vs13 ,vs34,vs42,1\r
+#ifndef TRMMKERNEL      \r
+       lxv             vs5,    32(T2)\r
+       lxv             vs7,    48(T2)\r
+#endif \r
+       xxpermdi  vs14, vs43,vs35,1              \r
+       xxpermdi  vs15 ,vs35,vs43,1     \r
+ \r
+\r
+\r
+#ifndef TRMMKERNEL \r
+       xvmaddadp       vs0,    vs8,    alpha_r \r
+       xvmaddadp       vs1,    vs9,    alpha_r \r
+       xvmaddadp       vs2,    vs10,   alpha_r \r
+       xvmaddadp       vs3,    vs11,   alpha_r \r
+\r
+       xvmaddadp       vs4,    vs12,   alpha_r \r
+       xvmaddadp       vs5,    vs13,   alpha_r \r
+       xvmaddadp       vs6,    vs14,   alpha_r \r
+       xvmaddadp       vs7,    vs15,   alpha_r \r
+#else\r
+       xvmuldp vs0,    vs8,    alpha_r \r
+       xvmuldp vs1,    vs9,    alpha_r \r
+       xvmuldp vs2,    vs10,   alpha_r \r
+       xvmuldp vs3,    vs11,   alpha_r \r
+\r
+       xvmuldp vs4,    vs12,   alpha_r \r
+       xvmuldp vs5,    vs13,   alpha_r \r
+       xvmuldp vs6,    vs14,   alpha_r \r
+       xvmuldp vs7,    vs15,   alpha_r \r
+\r
+#endif\r
+ \r
+\r
+       stxv            vs0,    0(CO)\r
+       stxv            vs2,    16(CO)\r
+       stxv            vs4,    32(CO)\r
+       stxv            vs6,    48(CO)\r
+\r
+ \r
+       stxv            vs1,    0(T2)\r
+       stxv            vs3,    16(T2)\r
+       stxv            vs5,    32(T2)\r
+       stxv            vs7,    48(T2)\r
+       \r
+ \r
+       xxpermdi  vs8, vs56,vs48,1\r
+       xxpermdi  vs9 ,vs48,vs56,1\r
+#ifndef TRMMKERNEL \r
+       lxv             vs0,    0(T3)\r
+       lxv             vs2,    16(T3)\r
+#endif \r
+       xxpermdi  vs10, vs57,vs49,1              \r
+       xxpermdi  vs11 ,vs49,vs57,1     \r
+#ifndef TRMMKERNEL      \r
+       lxv             vs4,    32(T3)\r
+       lxv             vs6,    48(T3)\r
+#endif \r
+       xxpermdi  vs12, vs58,vs50,1\r
+       xxpermdi  vs13 ,vs50,vs58,1\r
+#ifndef TRMMKERNEL      \r
+       lxv             vs1,    0(T4)\r
+       lxv             vs3,    16(T4)\r
+#endif \r
+       xxpermdi  vs14, vs59,vs51,1              \r
+       xxpermdi  vs15 ,vs51,vs59,1     \r
+#ifndef TRMMKERNEL      \r
+       lxv             vs5,    32(T4)\r
+       lxv             vs7,    48(T4)\r
+ \r
+ \r
+       xvmaddadp       vs0,    vs8,    alpha_r \r
+       xvmaddadp       vs1,    vs9,    alpha_r \r
+       xvmaddadp       vs2,    vs10,   alpha_r \r
+       xvmaddadp       vs3,    vs11,   alpha_r \r
+       \r
+\r
+\r
+       xvmaddadp       vs4,    vs12,   alpha_r \r
+       xvmaddadp       vs5,    vs13,   alpha_r \r
+       xvmaddadp       vs6,    vs14,   alpha_r \r
+       xvmaddadp       vs7,    vs15,   alpha_r \r
+#else\r
+       xvmuldp vs0,    vs8,    alpha_r \r
+       xvmuldp vs1,    vs9,    alpha_r \r
+       xvmuldp vs2,    vs10,   alpha_r \r
+       xvmuldp vs3,    vs11,   alpha_r \r
+       \r
+\r
+\r
+       xvmuldp vs4,    vs12,   alpha_r \r
+       xvmuldp vs5,    vs13,   alpha_r \r
+       xvmuldp vs6,    vs14,   alpha_r \r
+       xvmuldp vs7,    vs15,   alpha_r \r
+\r
+#endif\r
+\r
+\r
+       stxv            vs0,    0(T3)\r
+       stxv            vs2,    16(T3)\r
+       stxv            vs4,    32(T3)\r
+       stxv            vs6,    48(T3)\r
+\r
+ \r
+       stxv            vs1,    0(T4)\r
+       stxv            vs3,    16(T4)\r
+       stxv            vs5,    32(T4)\r
+       stxv            vs7,    48(T4)\r
+       \r
+ \r
+\r
+       addi            CO,     CO,     64\r
+.endm\r
+\r
+\r
+/*********************************************************************\r
+* Macros for N=4, M=4                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD4x4_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 32\r
+\r
+.endm\r
+\r
+.macro KERNEL4x4_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+       lxvdsx  vs30,   o16,    BO\r
+       lxvdsx  vs31,   o24,    BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+\r
+       xvmuldp                 vs48,   vs0,    vs26\r
+       xvmuldp                 vs49,   vs1,    vs26\r
+\r
+       xvmuldp                 vs56,   vs0,    vs27\r
+       xvmuldp                 vs57,   vs1,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x4_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+       lxvdsx  vs30,   o16,    BO\r
+       lxvdsx  vs31,   o24,    BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+       xvmaddadp               vs49,   vs1,    vs26\r
+\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+       xvmaddadp               vs57,   vs1,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x4_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+\r
+       xvmaddadp               vs48,   vs8,    vs30\r
+       xvmaddadp               vs49,   vs9,    vs30\r
+\r
+       xvmaddadp               vs56,   vs8,    vs31\r
+       xvmaddadp               vs57,   vs9,    vs31\r
+\r
+.endm\r
+\r
+.macro KERNEL4x4_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+\r
+       xvmaddadp               vs48,   vs8,    vs30\r
+       xvmaddadp               vs49,   vs9,    vs30\r
+\r
+       xvmaddadp               vs56,   vs8,    vs31\r
+       xvmaddadp               vs57,   vs9,    vs31\r
+\r
+.endm\r
+\r
+.macro KERNEL4x4_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+\r
+       xvmuldp                 vs48,   vs0,    vs26\r
+       xvmuldp                 vs49,   vs1,    vs26\r
+\r
+       xvmuldp                 vs56,   vs0,    vs27\r
+       xvmuldp                 vs57,   vs1,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x4_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+       xvmaddadp               vs49,   vs1,    vs26\r
+\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+       xvmaddadp               vs57,   vs1,    vs27\r
+\r
+.endm\r
+\r
+.macro SAVE4x4\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+       xvmaddadp       vs1,    vs33,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+       xvmuldp         vs1,    vs33,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+       lxvd2x          vs9,    o16,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs40,   alpha_r\r
+       xvmaddadp       vs9,    vs41,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs40,   alpha_r\r
+       xvmuldp         vs9,    vs41,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+       stxvd2x         vs9,    o16,    T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs48,   alpha_r\r
+       xvmaddadp       vs1,    vs49,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs48,   alpha_r\r
+       xvmuldp         vs1,    vs49,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+       lxvd2x          vs9,    o16,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs56,   alpha_r\r
+       xvmaddadp       vs9,    vs57,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs56,   alpha_r\r
+       xvmuldp         vs9,    vs57,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+       stxvd2x         vs9,    o16,    T1\r
+\r
+       addi            CO,     CO,     32\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=4, M=2                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD4x2_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 32\r
+\r
+.endm\r
+\r
+.macro KERNEL4x2_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+       lxvdsx  vs30,   o16,    BO\r
+       lxvdsx  vs31,   o24,    BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+\r
+       xvmuldp                 vs48,   vs0,    vs26\r
+\r
+       xvmuldp                 vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x2_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+       lxvdsx  vs30,   o16,    BO\r
+       lxvdsx  vs31,   o24,    BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x2_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+\r
+       xvmaddadp               vs48,   vs8,    vs30\r
+\r
+       xvmaddadp               vs56,   vs8,    vs31\r
+\r
+.endm\r
+\r
+.macro KERNEL4x2_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+\r
+       xvmaddadp               vs48,   vs8,    vs30\r
+\r
+       xvmaddadp               vs56,   vs8,    vs31\r
+\r
+.endm\r
+\r
+.macro KERNEL4x2_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+\r
+       xvmuldp                 vs48,   vs0,    vs26\r
+\r
+       xvmuldp                 vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x2_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+       lxvdsx  vs26,   o16,    BO\r
+       lxvdsx  vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+\r
+       xvmaddadp               vs48,   vs0,    vs26\r
+\r
+       xvmaddadp               vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro SAVE4x2\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs40,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs40,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs48,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs48,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs56,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs56,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+\r
+       addi            CO,     CO,     16\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=4, M=1                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD4x1_1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+       lxsdx   vs26,   o16,    BO\r
+       lxsdx   vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 32\r
+\r
+.endm\r
+\r
+.macro KERNEL4x1_I1\r
+\r
+       lxsdx   vs8,    0,      AO\r
+\r
+       lxsdx   vs28,   0,      BO\r
+       lxsdx   vs29,   o8,     BO\r
+       lxsdx   vs30,   o16,    BO\r
+       lxsdx   vs31,   o24,    BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xsmuldp                 vs32,   vs0,    vs24\r
+\r
+       xsmuldp                 vs40,   vs0,    vs25\r
+\r
+       xsmuldp                 vs48,   vs0,    vs26\r
+\r
+       xsmuldp                 vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x1_1\r
+\r
+       lxsdx   vs8,    0,      AO\r
+\r
+       lxsdx   vs28,   0,      BO\r
+       lxsdx   vs29,   o8,     BO\r
+       lxsdx   vs30,   o16,    BO\r
+       lxsdx   vs31,   o24,    BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xsmaddadp               vs32,   vs0,    vs24\r
+\r
+       xsmaddadp               vs40,   vs0,    vs25\r
+\r
+       xsmaddadp               vs48,   vs0,    vs26\r
+\r
+       xsmaddadp               vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x1_2\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+       lxsdx   vs26,   o16,    BO\r
+       lxsdx   vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xsmaddadp               vs32,   vs8,    vs28\r
+\r
+       xsmaddadp               vs40,   vs8,    vs29\r
+\r
+       xsmaddadp               vs48,   vs8,    vs30\r
+\r
+       xsmaddadp               vs56,   vs8,    vs31\r
+\r
+.endm\r
+\r
+.macro KERNEL4x1_E2\r
+\r
+\r
+       xsmaddadp               vs32,   vs8,    vs28\r
+\r
+       xsmaddadp               vs40,   vs8,    vs29\r
+\r
+       xsmaddadp               vs48,   vs8,    vs30\r
+\r
+       xsmaddadp               vs56,   vs8,    vs31\r
+\r
+.endm\r
+\r
+.macro KERNEL4x1_SUBI1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+       lxsdx   vs26,   o16,    BO\r
+       lxsdx   vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xsmuldp                 vs32,   vs0,    vs24\r
+\r
+       xsmuldp                 vs40,   vs0,    vs25\r
+\r
+       xsmuldp                 vs48,   vs0,    vs26\r
+\r
+       xsmuldp                 vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro KERNEL4x1_SUB1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+       lxsdx   vs26,   o16,    BO\r
+       lxsdx   vs27,   o24,    BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 32\r
+\r
+\r
+       xsmaddadp               vs32,   vs0,    vs24\r
+\r
+       xsmaddadp               vs40,   vs0,    vs25\r
+\r
+       xsmaddadp               vs48,   vs0,    vs26\r
+\r
+       xsmaddadp               vs56,   vs0,    vs27\r
+\r
+.endm\r
+\r
+.macro SAVE4x1\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxsdx           vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xsmaddadp       vs0,    vs32,   alpha_r\r
+#else\r
+       xsmuldp         vs0,    vs32,   alpha_r\r
+#endif\r
+\r
+       stxsdx          vs0,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxsdx           vs8,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xsmaddadp       vs8,    vs40,   alpha_r\r
+#else\r
+       xsmuldp         vs8,    vs40,   alpha_r\r
+#endif\r
+\r
+       stxsdx          vs8,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxsdx           vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xsmaddadp       vs0,    vs48,   alpha_r\r
+#else\r
+       xsmuldp         vs0,    vs48,   alpha_r\r
+#endif\r
+\r
+       stxsdx          vs0,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxsdx           vs8,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xsmaddadp       vs8,    vs56,   alpha_r\r
+#else\r
+       xsmuldp         vs8,    vs56,   alpha_r\r
+#endif\r
+\r
+       stxsdx          vs8,    0,      T1\r
+\r
+       addi            CO,     CO,     8\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=2, M=16                                               *\r
+*********************************************************************/\r
+\r
+.macro LOAD2x16_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+.endm\r
+\r
+.macro KERNEL2x16_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+       lxvd2x  vs12,   0,      AO\r
+       lxvd2x  vs13,   o16,    AO\r
+       lxvd2x  vs14,   o32,    AO\r
+       lxvd2x  vs15,   o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+       xvmuldp                 vs36,   vs4,    vs24\r
+       xvmuldp                 vs37,   vs5,    vs24\r
+       xvmuldp                 vs38,   vs6,    vs24\r
+       xvmuldp                 vs39,   vs7,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+       xvmuldp                 vs42,   vs2,    vs25\r
+       xvmuldp                 vs43,   vs3,    vs25\r
+       xvmuldp                 vs44,   vs4,    vs25\r
+       xvmuldp                 vs45,   vs5,    vs25\r
+       xvmuldp                 vs46,   vs6,    vs25\r
+       xvmuldp                 vs47,   vs7,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x16_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+       lxvd2x  vs12,   0,      AO\r
+       lxvd2x  vs13,   o16,    AO\r
+       lxvd2x  vs14,   o32,    AO\r
+       lxvd2x  vs15,   o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+       xvmaddadp               vs36,   vs4,    vs24\r
+       xvmaddadp               vs37,   vs5,    vs24\r
+       xvmaddadp               vs38,   vs6,    vs24\r
+       xvmaddadp               vs39,   vs7,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+       xvmaddadp               vs44,   vs4,    vs25\r
+       xvmaddadp               vs45,   vs5,    vs25\r
+       xvmaddadp               vs46,   vs6,    vs25\r
+       xvmaddadp               vs47,   vs7,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x16_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+       xvmaddadp               vs36,   vs12,   vs28\r
+       xvmaddadp               vs37,   vs13,   vs28\r
+       xvmaddadp               vs38,   vs14,   vs28\r
+       xvmaddadp               vs39,   vs15,   vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+       xvmaddadp               vs42,   vs10,   vs29\r
+       xvmaddadp               vs43,   vs11,   vs29\r
+       xvmaddadp               vs44,   vs12,   vs29\r
+       xvmaddadp               vs45,   vs13,   vs29\r
+       xvmaddadp               vs46,   vs14,   vs29\r
+       xvmaddadp               vs47,   vs15,   vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x16_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+       xvmaddadp               vs36,   vs12,   vs28\r
+       xvmaddadp               vs37,   vs13,   vs28\r
+       xvmaddadp               vs38,   vs14,   vs28\r
+       xvmaddadp               vs39,   vs15,   vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+       xvmaddadp               vs42,   vs10,   vs29\r
+       xvmaddadp               vs43,   vs11,   vs29\r
+       xvmaddadp               vs44,   vs12,   vs29\r
+       xvmaddadp               vs45,   vs13,   vs29\r
+       xvmaddadp               vs46,   vs14,   vs29\r
+       xvmaddadp               vs47,   vs15,   vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x16_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+       xvmuldp                 vs36,   vs4,    vs24\r
+       xvmuldp                 vs37,   vs5,    vs24\r
+       xvmuldp                 vs38,   vs6,    vs24\r
+       xvmuldp                 vs39,   vs7,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+       xvmuldp                 vs42,   vs2,    vs25\r
+       xvmuldp                 vs43,   vs3,    vs25\r
+       xvmuldp                 vs44,   vs4,    vs25\r
+       xvmuldp                 vs45,   vs5,    vs25\r
+       xvmuldp                 vs46,   vs6,    vs25\r
+       xvmuldp                 vs47,   vs7,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x16_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+       xvmaddadp               vs36,   vs4,    vs24\r
+       xvmaddadp               vs37,   vs5,    vs24\r
+       xvmaddadp               vs38,   vs6,    vs24\r
+       xvmaddadp               vs39,   vs7,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+       xvmaddadp               vs44,   vs4,    vs25\r
+       xvmaddadp               vs45,   vs5,    vs25\r
+       xvmaddadp               vs46,   vs6,    vs25\r
+       xvmaddadp               vs47,   vs7,    vs25\r
+\r
+.endm\r
+\r
+.macro SAVE2x16\r
+\r
+       mr              T1,     CO\r
+       addi            T2,     T1,     64\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+       lxvd2x          vs2,    o32,    T1\r
+       lxvd2x          vs3,    o48,    T1\r
+\r
+       lxvd2x          vs4,    0,      T2\r
+       lxvd2x          vs5,    o16,    T2\r
+       lxvd2x          vs6,    o32,    T2\r
+       lxvd2x          vs7,    o48,    T2\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+       xvmaddadp       vs1,    vs33,   alpha_r\r
+       xvmaddadp       vs2,    vs34,   alpha_r\r
+       xvmaddadp       vs3,    vs35,   alpha_r\r
+       xvmaddadp       vs4,    vs36,   alpha_r\r
+       xvmaddadp       vs5,    vs37,   alpha_r\r
+       xvmaddadp       vs6,    vs38,   alpha_r\r
+       xvmaddadp       vs7,    vs39,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+       xvmuldp         vs1,    vs33,   alpha_r\r
+       xvmuldp         vs2,    vs34,   alpha_r\r
+       xvmuldp         vs3,    vs35,   alpha_r\r
+       xvmuldp         vs4,    vs36,   alpha_r\r
+       xvmuldp         vs5,    vs37,   alpha_r\r
+       xvmuldp         vs6,    vs38,   alpha_r\r
+       xvmuldp         vs7,    vs39,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+       stxvd2x         vs2,    o32,    T1\r
+       stxvd2x         vs3,    o48,    T1\r
+\r
+       stxvd2x         vs4,    0,      T2\r
+       stxvd2x         vs5,    o16,    T2\r
+       stxvd2x         vs6,    o32,    T2\r
+       stxvd2x         vs7,    o48,    T2\r
+\r
+       add             T1,     T1,     LDC\r
+       add             T2,     T2,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+       lxvd2x          vs9,    o16,    T1\r
+       lxvd2x          vs10,   o32,    T1\r
+       lxvd2x          vs11,   o48,    T1\r
+\r
+       lxvd2x          vs12,   0,      T2\r
+       lxvd2x          vs13,   o16,    T2\r
+       lxvd2x          vs14,   o32,    T2\r
+       lxvd2x          vs15,   o48,    T2\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs40,   alpha_r\r
+       xvmaddadp       vs9,    vs41,   alpha_r\r
+       xvmaddadp       vs10,   vs42,   alpha_r\r
+       xvmaddadp       vs11,   vs43,   alpha_r\r
+       xvmaddadp       vs12,   vs44,   alpha_r\r
+       xvmaddadp       vs13,   vs45,   alpha_r\r
+       xvmaddadp       vs14,   vs46,   alpha_r\r
+       xvmaddadp       vs15,   vs47,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs40,   alpha_r\r
+       xvmuldp         vs9,    vs41,   alpha_r\r
+       xvmuldp         vs10,   vs42,   alpha_r\r
+       xvmuldp         vs11,   vs43,   alpha_r\r
+       xvmuldp         vs12,   vs44,   alpha_r\r
+       xvmuldp         vs13,   vs45,   alpha_r\r
+       xvmuldp         vs14,   vs46,   alpha_r\r
+       xvmuldp         vs15,   vs47,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+       stxvd2x         vs9,    o16,    T1\r
+       stxvd2x         vs10,   o32,    T1\r
+       stxvd2x         vs11,   o48,    T1\r
+\r
+       stxvd2x         vs12,   0,      T2\r
+       stxvd2x         vs13,   o16,    T2\r
+       stxvd2x         vs14,   o32,    T2\r
+       stxvd2x         vs15,   o48,    T2\r
+\r
+       addi            CO,     CO,     128\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=4, M=8                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD2x8_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+.endm\r
+\r
+.macro KERNEL2x8_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+       xvmuldp                 vs42,   vs2,    vs25\r
+       xvmuldp                 vs43,   vs3,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x8_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x8_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+       xvmaddadp               vs42,   vs10,   vs29\r
+       xvmaddadp               vs43,   vs11,   vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x8_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+       xvmaddadp               vs42,   vs10,   vs29\r
+       xvmaddadp               vs43,   vs11,   vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x8_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+       xvmuldp                 vs42,   vs2,    vs25\r
+       xvmuldp                 vs43,   vs3,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x8_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+       xvmaddadp               vs42,   vs2,    vs25\r
+       xvmaddadp               vs43,   vs3,    vs25\r
+\r
+.endm\r
+\r
+.macro SAVE2x8\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+       lxvd2x          vs2,    o32,    T1\r
+       lxvd2x          vs3,    o48,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+       xvmaddadp       vs1,    vs33,   alpha_r\r
+       xvmaddadp       vs2,    vs34,   alpha_r\r
+       xvmaddadp       vs3,    vs35,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+       xvmuldp         vs1,    vs33,   alpha_r\r
+       xvmuldp         vs2,    vs34,   alpha_r\r
+       xvmuldp         vs3,    vs35,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+       stxvd2x         vs2,    o32,    T1\r
+       stxvd2x         vs3,    o48,    T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+       lxvd2x          vs9,    o16,    T1\r
+       lxvd2x          vs10,   o32,    T1\r
+       lxvd2x          vs11,   o48,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs40,   alpha_r\r
+       xvmaddadp       vs9,    vs41,   alpha_r\r
+       xvmaddadp       vs10,   vs42,   alpha_r\r
+       xvmaddadp       vs11,   vs43,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs40,   alpha_r\r
+       xvmuldp         vs9,    vs41,   alpha_r\r
+       xvmuldp         vs10,   vs42,   alpha_r\r
+       xvmuldp         vs11,   vs43,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+       stxvd2x         vs9,    o16,    T1\r
+       stxvd2x         vs10,   o32,    T1\r
+       stxvd2x         vs11,   o48,    T1\r
+\r
+       addi            CO,     CO,     64\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=2, M=4                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD2x4_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 16\r
+\r
+.endm\r
+\r
+.macro KERNEL2x4_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x4_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x4_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x4_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+       xvmaddadp               vs41,   vs9,    vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x4_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+       xvmuldp                 vs41,   vs1,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x4_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+       xvmaddadp               vs41,   vs1,    vs25\r
+\r
+.endm\r
+\r
+.macro SAVE2x4\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+       xvmaddadp       vs1,    vs33,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+       xvmuldp         vs1,    vs33,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+       lxvd2x          vs9,    o16,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs40,   alpha_r\r
+       xvmaddadp       vs9,    vs41,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs40,   alpha_r\r
+       xvmuldp         vs9,    vs41,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+       stxvd2x         vs9,    o16,    T1\r
+\r
+       addi            CO,     CO,     32\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=2, M=2                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD2x2_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 16\r
+\r
+.endm\r
+\r
+.macro KERNEL2x2_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x2_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+       lxvdsx  vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x2_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x2_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+\r
+       xvmaddadp               vs40,   vs8,    vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x2_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+\r
+       xvmuldp                 vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x2_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+       lxvdsx  vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+\r
+       xvmaddadp               vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro SAVE2x2\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs8,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs8,    vs40,   alpha_r\r
+#else\r
+       xvmuldp         vs8,    vs40,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs8,    0,      T1\r
+\r
+       addi            CO,     CO,     16\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=2, M=1                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD2x1_1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 16\r
+\r
+.endm\r
+\r
+.macro KERNEL2x1_I1\r
+\r
+       lxsdx   vs8,    0,      AO\r
+\r
+       lxsdx   vs28,   0,      BO\r
+       lxsdx   vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xsmuldp                 vs32,   vs0,    vs24\r
+\r
+       xsmuldp                 vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x1_1\r
+\r
+       lxsdx   vs8,    0,      AO\r
+\r
+       lxsdx   vs28,   0,      BO\r
+       lxsdx   vs29,   o8,     BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xsmaddadp               vs32,   vs0,    vs24\r
+\r
+       xsmaddadp               vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x1_2\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xsmaddadp               vs32,   vs8,    vs28\r
+\r
+       xsmaddadp               vs40,   vs8,    vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x1_E2\r
+\r
+\r
+       xsmaddadp               vs32,   vs8,    vs28\r
+\r
+       xsmaddadp               vs40,   vs8,    vs29\r
+\r
+.endm\r
+\r
+.macro KERNEL2x1_SUBI1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xsmuldp                 vs32,   vs0,    vs24\r
+\r
+       xsmuldp                 vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro KERNEL2x1_SUB1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+       lxsdx   vs25,   o8,     BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 16\r
+\r
+\r
+       xsmaddadp               vs32,   vs0,    vs24\r
+\r
+       xsmaddadp               vs40,   vs0,    vs25\r
+\r
+.endm\r
+\r
+.macro SAVE2x1\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxsdx           vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xsmaddadp       vs0,    vs32,   alpha_r\r
+#else\r
+       xsmuldp         vs0,    vs32,   alpha_r\r
+#endif\r
+\r
+       stxsdx          vs0,    0,      T1\r
+\r
+       add             T1,     T1,     LDC\r
+\r
+#ifndef TRMMKERNEL\r
+       lxsdx           vs8,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xsmaddadp       vs8,    vs40,   alpha_r\r
+#else\r
+       xsmuldp         vs8,    vs40,   alpha_r\r
+#endif\r
+\r
+       stxsdx          vs8,    0,      T1\r
+\r
+       addi            CO,     CO,     8\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=1, M=16                                               *\r
+*********************************************************************/\r
+\r
+.macro LOAD1x16_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+.endm\r
+\r
+.macro KERNEL1x16_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+       lxvd2x  vs12,   0,      AO\r
+       lxvd2x  vs13,   o16,    AO\r
+       lxvd2x  vs14,   o32,    AO\r
+       lxvd2x  vs15,   o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+       xvmuldp                 vs36,   vs4,    vs24\r
+       xvmuldp                 vs37,   vs5,    vs24\r
+       xvmuldp                 vs38,   vs6,    vs24\r
+       xvmuldp                 vs39,   vs7,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x16_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+       lxvd2x  vs12,   0,      AO\r
+       lxvd2x  vs13,   o16,    AO\r
+       lxvd2x  vs14,   o32,    AO\r
+       lxvd2x  vs15,   o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+       xvmaddadp               vs36,   vs4,    vs24\r
+       xvmaddadp               vs37,   vs5,    vs24\r
+       xvmaddadp               vs38,   vs6,    vs24\r
+       xvmaddadp               vs39,   vs7,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x16_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+       xvmaddadp               vs36,   vs12,   vs28\r
+       xvmaddadp               vs37,   vs13,   vs28\r
+       xvmaddadp               vs38,   vs14,   vs28\r
+       xvmaddadp               vs39,   vs15,   vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x16_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+       xvmaddadp               vs36,   vs12,   vs28\r
+       xvmaddadp               vs37,   vs13,   vs28\r
+       xvmaddadp               vs38,   vs14,   vs28\r
+       xvmaddadp               vs39,   vs15,   vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x16_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+       xvmuldp                 vs36,   vs4,    vs24\r
+       xvmuldp                 vs37,   vs5,    vs24\r
+       xvmuldp                 vs38,   vs6,    vs24\r
+       xvmuldp                 vs39,   vs7,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x16_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+       lxvd2x  vs4,    0,      AO\r
+       lxvd2x  vs5,    o16,    AO\r
+       lxvd2x  vs6,    o32,    AO\r
+       lxvd2x  vs7,    o48,    AO\r
+\r
+       addi            AO, AO, 64\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+       xvmaddadp               vs36,   vs4,    vs24\r
+       xvmaddadp               vs37,   vs5,    vs24\r
+       xvmaddadp               vs38,   vs6,    vs24\r
+       xvmaddadp               vs39,   vs7,    vs24\r
+\r
+.endm\r
+\r
+.macro SAVE1x16\r
+\r
+       mr              T1,     CO\r
+       addi            T2,     T1,     64\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+       lxvd2x          vs2,    o32,    T1\r
+       lxvd2x          vs3,    o48,    T1\r
+\r
+       lxvd2x          vs4,    0,      T2\r
+       lxvd2x          vs5,    o16,    T2\r
+       lxvd2x          vs6,    o32,    T2\r
+       lxvd2x          vs7,    o48,    T2\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+       xvmaddadp       vs1,    vs33,   alpha_r\r
+       xvmaddadp       vs2,    vs34,   alpha_r\r
+       xvmaddadp       vs3,    vs35,   alpha_r\r
+       xvmaddadp       vs4,    vs36,   alpha_r\r
+       xvmaddadp       vs5,    vs37,   alpha_r\r
+       xvmaddadp       vs6,    vs38,   alpha_r\r
+       xvmaddadp       vs7,    vs39,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+       xvmuldp         vs1,    vs33,   alpha_r\r
+       xvmuldp         vs2,    vs34,   alpha_r\r
+       xvmuldp         vs3,    vs35,   alpha_r\r
+       xvmuldp         vs4,    vs36,   alpha_r\r
+       xvmuldp         vs5,    vs37,   alpha_r\r
+       xvmuldp         vs6,    vs38,   alpha_r\r
+       xvmuldp         vs7,    vs39,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+       stxvd2x         vs2,    o32,    T1\r
+       stxvd2x         vs3,    o48,    T1\r
+\r
+       stxvd2x         vs4,    0,      T2\r
+       stxvd2x         vs5,    o16,    T2\r
+       stxvd2x         vs6,    o32,    T2\r
+       stxvd2x         vs7,    o48,    T2\r
+\r
+       addi            CO,     CO,     128\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=4, M=8                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD1x8_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+.endm\r
+\r
+.macro KERNEL1x8_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x8_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+       lxvd2x  vs10,   o32,    AO\r
+       lxvd2x  vs11,   o48,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x8_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x8_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+       xvmaddadp               vs34,   vs10,   vs28\r
+       xvmaddadp               vs35,   vs11,   vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x8_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+       xvmuldp                 vs34,   vs2,    vs24\r
+       xvmuldp                 vs35,   vs3,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x8_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+       lxvd2x  vs2,    o32,    AO\r
+       lxvd2x  vs3,    o48,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 64\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+       xvmaddadp               vs34,   vs2,    vs24\r
+       xvmaddadp               vs35,   vs3,    vs24\r
+\r
+.endm\r
+\r
+.macro SAVE1x8\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+       lxvd2x          vs2,    o32,    T1\r
+       lxvd2x          vs3,    o48,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+       xvmaddadp       vs1,    vs33,   alpha_r\r
+       xvmaddadp       vs2,    vs34,   alpha_r\r
+       xvmaddadp       vs3,    vs35,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+       xvmuldp         vs1,    vs33,   alpha_r\r
+       xvmuldp         vs2,    vs34,   alpha_r\r
+       xvmuldp         vs3,    vs35,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+       stxvd2x         vs2,    o32,    T1\r
+       stxvd2x         vs3,    o48,    T1\r
+\r
+       addi            CO,     CO,     64\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=1, M=4                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD1x4_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 8\r
+\r
+.endm\r
+\r
+.macro KERNEL1x4_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x4_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+       lxvd2x  vs9,    o16,    AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x4_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x4_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+       xvmaddadp               vs33,   vs9,    vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x4_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+       xvmuldp                 vs33,   vs1,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x4_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+       lxvd2x  vs1,    o16,    AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 32\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+       xvmaddadp               vs33,   vs1,    vs24\r
+\r
+.endm\r
+\r
+.macro SAVE1x4\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+       lxvd2x          vs1,    o16,    T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+       xvmaddadp       vs1,    vs33,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+       xvmuldp         vs1,    vs33,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+       stxvd2x         vs1,    o16,    T1\r
+\r
+       addi            CO,     CO,     32\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=1, M=2                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD1x2_1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 8\r
+\r
+.endm\r
+\r
+.macro KERNEL1x2_I1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x2_1\r
+\r
+       lxvd2x  vs8,    0,      AO\r
+\r
+       lxvdsx  vs28,   0,      BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x2_2\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x2_E2\r
+\r
+\r
+       xvmaddadp               vs32,   vs8,    vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x2_SUBI1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmuldp                 vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x2_SUB1\r
+\r
+       lxvd2x  vs0,    0,      AO\r
+\r
+       lxvdsx  vs24,   0,      BO\r
+\r
+       addi            AO, AO, 16\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xvmaddadp               vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro SAVE1x2\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxvd2x          vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xvmaddadp       vs0,    vs32,   alpha_r\r
+#else\r
+       xvmuldp         vs0,    vs32,   alpha_r\r
+#endif\r
+\r
+       stxvd2x         vs0,    0,      T1\r
+\r
+       addi            CO,     CO,     16\r
+\r
+.endm\r
+\r
+/*********************************************************************\r
+* Macros for N=1, M=1                                                *\r
+*********************************************************************/\r
+\r
+.macro LOAD1x1_1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 8\r
+\r
+.endm\r
+\r
+.macro KERNEL1x1_I1\r
+\r
+       lxsdx   vs8,    0,      AO\r
+\r
+       lxsdx   vs28,   0,      BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xsmuldp                 vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x1_1\r
+\r
+       lxsdx   vs8,    0,      AO\r
+\r
+       lxsdx   vs28,   0,      BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xsmaddadp               vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x1_2\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xsmaddadp               vs32,   vs8,    vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x1_E2\r
+\r
+\r
+       xsmaddadp               vs32,   vs8,    vs28\r
+\r
+.endm\r
+\r
+.macro KERNEL1x1_SUBI1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xsmuldp                 vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro KERNEL1x1_SUB1\r
+\r
+       lxsdx   vs0,    0,      AO\r
+\r
+       lxsdx   vs24,   0,      BO\r
+\r
+       addi            AO, AO, 8\r
+       addi            BO, BO, 8\r
+\r
+\r
+       xsmaddadp               vs32,   vs0,    vs24\r
+\r
+.endm\r
+\r
+.macro SAVE1x1\r
+\r
+       mr              T1,     CO\r
+\r
+#ifndef TRMMKERNEL\r
+       lxsdx           vs0,    0,      T1\r
+#endif\r
+\r
+#ifndef TRMMKERNEL\r
+       xsmaddadp       vs0,    vs32,   alpha_r\r
+#else\r
+       xsmuldp         vs0,    vs32,   alpha_r\r
+#endif\r
+\r
+       stxsdx          vs0,    0,      T1\r
+\r
+       addi            CO,     CO,     8\r
+\r
+.endm\r
+\r
+\r
+\r
+\r
+/****************************TRMM POINTER REFRESH MACROSES*************************/\r
+\r
+.macro SHIFT_REG  REG1,REG2,SHIFT_VAL\r
+               .if \SHIFT_VAL==16 \r
+                       slwi            \REG1,  \REG2,  7                       \r
+               .elseif \SHIFT_VAL==8  \r
+                       slwi            \REG1,  \REG2,  6                        \r
+               .elseif \SHIFT_VAL==4\r
+                       slwi            \REG1,  \REG2,  5                         \r
+               .elseif \SHIFT_VAL==2\r
+                       slwi            \REG1,  \REG2,  4                        \r
+               .elseif \SHIFT_VAL==1\r
+                       slwi            \REG1,  \REG2,  3                        \r
+               .endif\r
+.endm\r
+\r
+/*\r
+//#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))\r
+//             ptrbb = bb;\r
+// #else\r
+//             ptrba += off*16;\r
+//             ptrbb = bb + off*2;\r
+// #endif\r
+*/\r
+.macro REFRESH_POINTERS  PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B\r
+    #if (defined(LEFT) &&  defined(TRANSA)) ||  (!defined(LEFT) && !defined(TRANSA))\r
+        /* ptrbb = bb;*/\r
+        mr \PTR_B,\B_VAL     /* refresh BPOINT */\r
+\r
+    #else\r
+                   /*\r
+        // ptrba  =ptrba+ off*C_A;\r
+        // ptrbb = bb + off*C_B; \r
+                               */\r
+               SHIFT_REG T4,\OFF_VAL,\C_B              /* Number of values in B shifted  */\r
+               SHIFT_REG T2,\OFF_VAL,\C_A              /* Number of values in A shifted  */\r
+               add             \PTR_B, \B_VAL ,        T4                              /* Add values to BO */\r
+               add             \PTR_A, \PTR_A, T2                              /* Add values to AO  */\r
+    #endif \r
+.endm\r
+\r
+\r
+/*\r
+// #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+//             temp = bk-off;\r
+// #elif defined(LEFT)\r
+//             temp = off+16;  // number of values in A\r
+// #else\r
+//             temp = off+2;   // number of values in B\r
+// #endif\r
+*/\r
+.macro REFRESH_TEMP_BK TEMP_BK,BK_VAL,OFF_VAL,INCR_A,INCR_B\r
+    #if (defined(LEFT) && !defined(TRANSA)) ||  (!defined(LEFT) && defined(TRANSA))\r
+                            /* temp = bk-off;*/\r
+           sub \TEMP_BK,\BK_VAL,\OFF_VAL\r
+\r
+    #elif defined(LEFT)\r
+                            /* temp = off+INCR_A;      // number of values in A */\r
+           addi \TEMP_BK, \OFF_VAL, \INCR_A\r
+    #else\r
+                            /* temp = off+INCR_B       // number of values in B*/\r
+           addi \TEMP_BK,\OFF_VAL, \INCR_B\r
+    #endif\r
+\r
+.endm\r
+/*\r
+// #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))\r
+//             temp = bk - off;\r
+// #ifdef LEFT\r
+//             temp -= 16; // number of values in A\r
+// #else\r
+//             temp -= 2; // number of values in B\r
+// #endif\r
+//             ptrba += temp*16;\r
+//             ptrbb += temp*2;\r
+// #endif\r
+\r
+// #ifdef LEFT\r
+//             off += 16; // number of values in A\r
+// #endif\r
+*/\r
+ \r
+\r
+.macro REFRESH_AFTER_SAVE TEMP_BK,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B\r
+\r
+    #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))\r
+                    /*temp = bk - off;*/\r
+                sub \TEMP_BK,\BK_VAL,\OFF_VAL\r
+    #ifdef LEFT\r
+                    /*temp -= 8; // number of values in A*/\r
+                addi \TEMP_BK,\TEMP_BK,-\C_A\r
+    #else\r
+                    /*temp -= 4; // number of values in B*/\r
+                addi \TEMP_BK,\TEMP_BK,-\C_B \r
+    #endif\r
+                    /*ptrba += temp*C_A;\r
+                    ptrbb += temp*C_B;*/ \r
+                SHIFT_REG T4,\TEMP_BK,\C_A\r
+                                                               SHIFT_REG T2,\TEMP_BK,\C_B\r
+                add \PTR_A, \PTR_A,T4/*ptrba+temp*C_A*/ \r
+                                                               add \PTR_B, \PTR_B,T2 \r
+\r
+    #endif\r
+\r
+    #ifdef LEFT\r
+                    /*off += 8; // number of values in A*/\r
+                 addi \OFF_VAL,\OFF_VAL,\C_A\r
+    #endif\r
+.endm
\ No newline at end of file

diff --git a/kernel/power/dgemv_n.c b/kernel/power/dgemv_n.c
index 57f9f9e..b458e11 100644 (file)
--- a/kernel/power/dgemv_n.c
+++ b/kernel/power/dgemv_n.c
@@ -38,7 +38,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "dgemv_n_microk_power8.c"
 #endif
 

diff --git a/kernel/power/drot.c b/kernel/power/drot.c
index 3e10748..baeb542 100644 (file)
--- a/kernel/power/drot.c
+++ b/kernel/power/drot.c
@@ -39,7 +39,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #pragma GCC optimize "O1"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "drot_microk_power8.c"
 #endif
 

diff --git a/kernel/power/dscal.c b/kernel/power/dscal.c
index f32dc4b..779a08e 100644 (file)
--- a/kernel/power/dscal.c
+++ b/kernel/power/dscal.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) 
+#if defined(POWER8) || defined(POWER9)
 #include "dscal_microk_power8.c"
 #endif
 

diff --git a/kernel/power/dswap.c b/kernel/power/dswap.c
index fd2dec9..52b7f50 100644 (file)
--- a/kernel/power/dswap.c
+++ b/kernel/power/dswap.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "dswap_microk_power8.c"
 #endif
 

diff --git a/kernel/power/sasum.c b/kernel/power/sasum.c
index fb10b1d..5908347 100644 (file)
--- a/kernel/power/sasum.c
+++ b/kernel/power/sasum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "sasum_microk_power8.c"
 #endif
 

diff --git a/kernel/power/scopy.c b/kernel/power/scopy.c
index 167c29b..5e3fe45 100644 (file)
--- a/kernel/power/scopy.c
+++ b/kernel/power/scopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "scopy_microk_power8.c"
 #endif
 

diff --git a/kernel/power/sdot.c b/kernel/power/sdot.c
index 4fdc2f5..ae527dd 100644 (file)
--- a/kernel/power/sdot.c
+++ b/kernel/power/sdot.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) 
+#if defined(POWER8)  || defined(POWER9)
 #include "sdot_microk_power8.c"
 #endif
 

diff --git a/kernel/power/srot.c b/kernel/power/srot.c
index d2910ff..6af813c 100644 (file)
--- a/kernel/power/srot.c
+++ b/kernel/power/srot.c
@@ -39,7 +39,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #pragma GCC optimize "O1"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "srot_microk_power8.c"
 #endif
 

diff --git a/kernel/power/sscal.c b/kernel/power/sscal.c
index bd5cdc4..4f3ba56 100644 (file)
--- a/kernel/power/sscal.c
+++ b/kernel/power/sscal.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8) 
+#if defined(POWER8) || defined(POWER9)
 #include "sscal_microk_power8.c"
 #endif
 

diff --git a/kernel/power/sswap.c b/kernel/power/sswap.c
index 932652b..23d1328 100644 (file)
--- a/kernel/power/sswap.c
+++ b/kernel/power/sswap.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "sswap_microk_power8.c"
 #endif
 

diff --git a/kernel/power/zasum.c b/kernel/power/zasum.c
index 0b6b87d..f61c62e 100644 (file)
--- a/kernel/power/zasum.c
+++ b/kernel/power/zasum.c
@@ -46,7 +46,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "zasum_microk_power8.c"
 #endif
 

diff --git a/kernel/power/zaxpy.c b/kernel/power/zaxpy.c
index dd7ab6c..f0f8c69 100644 (file)
--- a/kernel/power/zaxpy.c
+++ b/kernel/power/zaxpy.c
@@ -36,19 +36,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "zaxpy_microk_power8.c"
 #endif
 
 
 #ifndef HAVE_KERNEL_4
 
-static void zaxpy_kernel_4(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
+static void zaxpy_kernel_4(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT da_r,FLOAT da_i)
 {
        BLASLONG register i  = 0;
        BLASLONG register ix = 0;
-       FLOAT da_r = alpha[0];
-       FLOAT da_i = alpha[1];
+ 
        
 
        while(i < n)

diff --git a/kernel/power/zcopy.c b/kernel/power/zcopy.c
index a7658f7..b21d6ef 100644 (file)
--- a/kernel/power/zcopy.c
+++ b/kernel/power/zcopy.c
@@ -35,7 +35,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "zcopy_microk_power8.c"
 #endif
 

diff --git a/kernel/power/zdot.c b/kernel/power/zdot.c
index b83f832..fd36c7f 100644 (file)
--- a/kernel/power/zdot.c
+++ b/kernel/power/zdot.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8) 
+#if defined(POWER8) || defined(POWER9)
 #include "zdot_microk_power8.c"
 #endif
 

diff --git a/kernel/power/zscal.c b/kernel/power/zscal.c
index 14d677f..a1b441d 100644 (file)
--- a/kernel/power/zscal.c
+++ b/kernel/power/zscal.c
@@ -38,7 +38,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #pragma GCC optimize "O1"
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #if defined(DOUBLE)
 #include "zscal_microk_power8.c"
 #endif

diff --git a/kernel/power/zswap.c b/kernel/power/zswap.c
index 5ec1eee..1d8826f 100644 (file)
--- a/kernel/power/zswap.c
+++ b/kernel/power/zswap.c
@@ -36,7 +36,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #include "zswap_microk_power8.c"
 #endif
 

diff --git a/param.h b/param.h
index fa67302..938a82a 100644 (file)
--- a/param.h
+++ b/param.h
@@ -2230,6 +2230,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #endif
 
+#if defined(POWER9)
+
+#define SNUMOPT                16
+#define DNUMOPT                8
+
+#define GEMM_DEFAULT_OFFSET_A 0 
+#define GEMM_DEFAULT_OFFSET_B 65536
+#define GEMM_DEFAULT_ALIGN 0x0ffffUL
+
+#define SGEMM_DEFAULT_UNROLL_M 16
+#define SGEMM_DEFAULT_UNROLL_N 8
+#define DGEMM_DEFAULT_UNROLL_M 16
+#define DGEMM_DEFAULT_UNROLL_N 4
+#define CGEMM_DEFAULT_UNROLL_M 8
+#define CGEMM_DEFAULT_UNROLL_N 4
+#define ZGEMM_DEFAULT_UNROLL_M 8
+#define ZGEMM_DEFAULT_UNROLL_N 2
+
+#define SGEMM_DEFAULT_P  1280
+#define DGEMM_DEFAULT_P  128
+#define CGEMM_DEFAULT_P  640
+#define ZGEMM_DEFAULT_P  320
+
+#define SGEMM_DEFAULT_Q  640
+#define DGEMM_DEFAULT_Q  384
+#define CGEMM_DEFAULT_Q  640
+#define ZGEMM_DEFAULT_Q  640
+
+#define SYMV_P  8
+
+#endif
 
 #if defined(SPARC) && defined(V7)