sgemm/strmm

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 08f8cc6..3859a9c 100644 (file)
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -167,4 +167,7 @@ In chronological order:
   * [2017-02-26] ztrmm kernel for IBM z13
   * [2017-03-13] strmm and ctrmm kernel for IBM z13
   * [2017-09-01] initial Blas Level-1,2 (double precision) for IBM z13
-
+  * [2018-03-07] added missing Blas Level 1-2  (double precision) simd codes
+  * [2019-02-01] added missing Blas Level-1,2 (single precision)  simd codes
+  * [2019-03-14] power9 dgemm/dtrmm kernel
+  * [2019-04-29] power9 sgemm/strmm kernel 

diff --git a/kernel/power/KERNEL.POWER9 b/kernel/power/KERNEL.POWER9
index 86a9319..6d5cf90 100644 (file)
--- a/kernel/power/KERNEL.POWER9
+++ b/kernel/power/KERNEL.POWER9
@@ -3,16 +3,16 @@
 #CGEMM_BETA = ../generic/zgemm_beta.c\r
 #ZGEMM_BETA = ../generic/zgemm_beta.c\r
 \r
-STRMMKERNEL    = strmm_kernel_16x8_power8.S\r
+STRMMKERNEL    = sgemm_kernel_power9.S\r
 DTRMMKERNEL    = dgemm_kernel_power9.S\r
 CTRMMKERNEL    = ctrmm_kernel_8x4_power8.S\r
 ZTRMMKERNEL    = ztrmm_kernel_8x2_power8.S\r
 \r
-SGEMMKERNEL    =  sgemm_kernel_16x8_power8.S\r
+SGEMMKERNEL    =  sgemm_kernel_power9.S\r
 SGEMMINCOPY    = ../generic/gemm_ncopy_16.c\r
 SGEMMITCOPY    = sgemm_tcopy_16_power8.S\r
 SGEMMONCOPY    =  ../generic/gemm_ncopy_8.c\r
-SGEMMOTCOPY    = sgemm_tcopy_8_power8.S \r
+SGEMMOTCOPY    = sgemm_tcopy_8_power8.S\r
 SGEMMINCOPYOBJ =  sgemm_incopy.o\r
 SGEMMITCOPYOBJ =  sgemm_itcopy.o\r
 SGEMMONCOPYOBJ =  sgemm_oncopy.o\r

diff --git a/kernel/power/sgemm_kernel_power9.S b/kernel/power/sgemm_kernel_power9.S
new file mode 100644 (file)
index 0000000..a446594
--- /dev/null
+++ b/kernel/power/sgemm_kernel_power9.S
@@ -0,0 +1,286 @@
+/***************************************************************************\r
+Copyright (c) 2013-2019, The OpenBLAS Project\r
+All rights reserved.\r
+Redistribution and use in source and binary forms, with or without\r
+modification, are permitted provided that the following conditions are\r
+met:\r
+1. Redistributions of source code must retain the above copyright\r
+notice, this list of conditions and the following disclaimer.\r
+2. Redistributions in binary form must reproduce the above copyright\r
+notice, this list of conditions and the following disclaimer in\r
+the documentation and/or other materials provided with the\r
+distribution.\r
+3. Neither the name of the OpenBLAS project nor the names of\r
+its contributors may be used to endorse or promote products\r
+derived from this software without specific prior written permission.\r
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"\r
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\r
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE\r
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\r
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+*****************************************************************************/\r
+ \r
+#define ASSEMBLER\r
+#include "common.h"\r
+#include "def_vsx.h"\r
+\r
+ \r
+#define LOAD   ld\r
+#define STACKSIZE  (512 )  \r
+  \r
+#define        M       r3\r
+#define        N       r4\r
+#define        K       r5\r
+\r
+ \r
+#define A      r7\r
+#define        B       r8\r
+#define        C       r9\r
+#define        LDC     r10\r
+#define OFFSET r6\r
+ \r
+ \r
+\r
+#define alpha_r vs20\r
+#define save_permute_1 vs21\r
+#define save_permute_2 vs22\r
+#define permute_mask vs23\r
+#define o0     0\r
+ \r
+\r
+#define T1     r11\r
+#define T2     r12\r
+#define T3     r14\r
+#define T4     r15\r
+#define T5     r16\r
+#define T6     r17\r
+#define L      r18\r
+#define T7     r19\r
+#define T8     r20\r
+#define TEMP_REG       r21\r
+#define        I       r22\r
+#define J      r23\r
+#define AO     r24\r
+#define        BO      r25\r
+#define        CO      r26\r
+#define T9     r27\r
+#define        T10     r28\r
+#define        T11     r29\r
+\r
+#define T12    r30\r
+#define T13    r31\r
+\r
+#include "sgemm_macros_power9.S"\r
+\r
+.equ    perm_const1, 0x0405060700010203\r
+.equ    perm_const2, 0x0c0d0e0f08090a0b\r
+.equ save_permute_11, 0x1415161718191a1b\r
+.equ save_permute_12, 0x0405060708090a0b\r
+.equ save_permute_21, 0x101112131c1d1e1f\r
+.equ save_permute_22, 0x000102030c0d0e0f \r
+\r
+\r
+#ifndef NEEDPARAM\r
+\r
+       PROLOGUE\r
+       PROFCODE\r
+\r
+       addi    SP, SP, -STACKSIZE\r
+       li      r0, 0\r
+\r
+       stfd    f14,    0(SP)\r
+       stfd    f15,    8(SP)\r
+       stfd    f16,   16(SP)\r
+       stfd    f17,   24(SP)\r
+\r
+       stfd    f18,   32(SP)\r
+       stfd    f19,   40(SP)\r
+       stfd    f20,   48(SP)\r
+       stfd    f21,   56(SP)\r
+\r
+       stfd    f22,   64(SP)\r
+       stfd    f23,   72(SP)\r
+       stfd    f24,   80(SP)\r
+       stfd    f25,   88(SP)\r
+\r
+       stfd    f26,   96(SP)\r
+       stfd    f27,  104(SP)\r
+       stfd    f28,  112(SP)\r
+       stfd    f29,  120(SP)\r
+\r
+       stfd    f30,  128(SP)\r
+       stfd    f31,  136(SP)\r
+\r
+ \r
+       std     r31,  144(SP)\r
+       std     r30,  152(SP)\r
+       std     r29,  160(SP)\r
+       std     r28,  168(SP)\r
+       std     r27,  176(SP)\r
+       std     r26,  184(SP)\r
+       std     r25,  192(SP)\r
+       std     r24,  200(SP)\r
+       std     r23,  208(SP)\r
+       std     r22,  216(SP)\r
+       std     r21,  224(SP)\r
+       std     r20,  232(SP)\r
+       std     r19,  240(SP)\r
+       std     r18,  248(SP)\r
+       std     r17,  256(SP)\r
+       std     r16,  264(SP)\r
+       std     r15,  272(SP)\r
+       std     r14,  280(SP)\r
+ \r
+ \r
+       stxv    v20,  288(SP)\r
+       stxv    v21,  304(SP)\r
+       stxv    v22,  320(SP)\r
+       stxv    v23,  336(SP)\r
+       stxv    v24,  352(SP)\r
+       stxv    v25,  368(SP)\r
+       stxv    v26,  384(SP)\r
+       stxv    v27,  400(SP)\r
+       stxv    v28,  416(SP)\r
+       stxv    v29,  432(SP)\r
+       stxv    v30,  448(SP)\r
+       stxv    v31,  464(SP)\r
+\r
+ \r
+\r
+#if defined(TRMMKERNEL) \r
+       ld      OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)\r
+#endif\r
+   slwi    LDC, LDC, 2\r
+\r
+\r
+/*     cmpwi   cr0, M, 0\r
+       ble     .L999_H1\r
+       cmpwi   cr0, N, 0\r
+       ble     .L999_H1\r
+       cmpwi   cr0, K, 0\r
+       ble     .L999_H1\r
+*/\r
+ \r
+ \r
+       /*alpha is stored in f1. convert to single and splat*/\r
+  xscvdpspn alpha_r,vs1 \r
+       xxspltw   alpha_r,alpha_r,0\r
+ \r
+ \r
+/*load reverse permute mask for big endian\r
+  uint128 = 0xc0d0e0f08090a0b0405060700010203\r
+*/ \r
+               \r
+       lis T2, perm_const2@highest\r
+       ori T2, T2, perm_const2@higher\r
+       rldicr T2, T2, 32, 31\r
+       oris T2, T2, perm_const2@h\r
+       ori T2, T2, perm_const2@l \r
+\r
+       lis T1, perm_const1@highest\r
+       ori T1, T1, perm_const1@higher\r
+       rldicr T1, T1, 32, 31\r
+       oris T1, T1, perm_const1@h\r
+       ori T1, T1, perm_const1@l\r
+\r
+       mtvsrdd permute_mask,T2,T1\r
+\r
+       lis T2, save_permute_12@highest\r
+       ori T2, T2, save_permute_12@higher\r
+       rldicr T2, T2, 32, 31\r
+       oris T2, T2, save_permute_12@h\r
+       ori T2, T2, save_permute_12@l \r
+\r
+       lis T1, save_permute_11@highest\r
+       ori T1, T1, save_permute_11@higher\r
+       rldicr T1, T1, 32, 31\r
+       oris T1, T1, save_permute_11@h\r
+       ori T1, T1, save_permute_11@l\r
+\r
+       mtvsrdd save_permute_1,T2,T1    \r
+\r
+       lis T2, save_permute_22@highest\r
+       ori T2, T2, save_permute_22@higher\r
+       rldicr T2, T2, 32, 31\r
+       oris T2, T2, save_permute_22@h\r
+       ori T2, T2, save_permute_22@l \r
+\r
+       lis T1, save_permute_21@highest\r
+       ori T1, T1, save_permute_21@higher\r
+       rldicr T1, T1, 32, 31\r
+       oris T1, T1, save_permute_21@h\r
+       ori T1, T1, save_permute_21@l\r
+\r
+       mtvsrdd save_permute_2,T2,T1    \r
+\r
+#include "sgemm_logic_power9.S"\r
+\r
+.L999:\r
+       addi    r3, 0, 0\r
+\r
+       lfd     f14,    0(SP)\r
+       lfd     f15,    8(SP)\r
+       lfd     f16,   16(SP)\r
+       lfd     f17,   24(SP)\r
+\r
+       lfd     f18,   32(SP)\r
+       lfd     f19,   40(SP)\r
+       lfd     f20,   48(SP)\r
+       lfd     f21,   56(SP)\r
+\r
+       lfd     f22,   64(SP)\r
+       lfd     f23,   72(SP)\r
+       lfd     f24,   80(SP)\r
+       lfd     f25,   88(SP)\r
+\r
+       lfd     f26,   96(SP)\r
+       lfd     f27,  104(SP)\r
+       lfd     f28,  112(SP)\r
+       lfd     f29,  120(SP)\r
+\r
+       lfd     f30,  128(SP)\r
+       lfd     f31,  136(SP)\r
+\r
+       ld      r31,  144(SP)\r
+       ld      r30,  152(SP)\r
+       ld      r29,  160(SP)\r
+       ld      r28,  168(SP)\r
+       ld      r27,  176(SP)\r
+       ld      r26,  184(SP)\r
+       ld      r25,  192(SP)\r
+       ld      r24,  200(SP)\r
+       ld      r23,  208(SP)\r
+       ld      r22,  216(SP)\r
+       ld      r21,  224(SP)\r
+       ld      r20,  232(SP)\r
+       ld      r19,  240(SP)\r
+       ld      r18,  248(SP)\r
+       ld      r17,  256(SP)\r
+       ld      r16,  264(SP)\r
+       ld      r15,  272(SP)\r
+       ld      r14,  280(SP)\r
+ \r
+       lxv    v20,  288(SP)\r
+       lxv    v21,  304(SP)\r
+       lxv    v22,  320(SP)\r
+       lxv    v23,  336(SP)\r
+       lxv    v24,  352(SP)\r
+       lxv    v25,  368(SP)\r
+       lxv    v26,  384(SP)\r
+       lxv    v27,  400(SP)\r
+       lxv    v28,  416(SP)\r
+       lxv    v29,  432(SP)\r
+       lxv    v30,  448(SP)\r
+       lxv    v31,  464(SP)\r
+\r
+ \r
+       addi    SP, SP, STACKSIZE\r
+       blr\r
+\r
+       EPILOGUE\r
+#endif\r

diff --git a/kernel/power/sgemm_logic_power9.S b/kernel/power/sgemm_logic_power9.S
new file mode 100644 (file)
index 0000000..300e304
--- /dev/null
+++ b/kernel/power/sgemm_logic_power9.S
@@ -0,0 +1,2133 @@
+#define MY_ALIGN .align 3\r
+\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+   neg TEMP_REG, OFFSET \r
+#endif\r
+\r
+       srawi.          J,      N,      3\r
+\r
+       ble             LSGEMM_L8_END\r
+\r
+LSGEMM_L8_BEGIN:\r
+\r
+       li              T1,     128\r
+       li              T2,     256\r
+ \r
+       mr              AO,     A\r
+       mr              CO,     C\r
+       slwi            T3,     LDC     ,       3\r
+       add             C,      C,      T3\r
+\r
+       dcbt            A,      T1\r
+       dcbt            A,      T2\r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+       mr TEMP_REG, OFFSET      /*off = offset;*/\r
+#endif \r
+       srawi.          I,      M,      4\r
+       ble             LSGEMM_L8x16_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_L8x16_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,8\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,16,8\r
+   mr T12, T11\r
+   addi T12,T12, -1\r
+   srawi.              L, T12, 6 /**(T11-1) % 64x */\r
+#else\r
+   mr T12, K\r
+   addi T12,T12, -1\r
+   srawi.              L,      T12,    6 /**(K-1) % 64x */\r
+#endif \r
+ \r
+    ZERO8x16\r
+       ble             LSGEMM_L8x16_SUB0\r
+\r
+       MY_ALIGN\r
+LSGEMM_L8x16_LOOP_START:\r
+ \r
+       LOAD8x16_0  /*we already zeroed */\r
+    ##OffsetA=64 OffsetB=32\r
+    addi AO,AO,2112\r
+    addi BO,BO,32  \r
+\r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_L8x16_LOOP:\r
+\r
+    KERNEL8x16_I1_L4_2  -2048,0, 0,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 1,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 2,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 3,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 4,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 5,0        \r
+    KERNEL8x16_I1_L4_2  -2048,0, 6,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 7,0  \r
+    KERNEL8x16_I1_L4_2  -2048,0, 8,0      \r
+    KERNEL8x16_I1_L4_2  -2048,0, 9,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 10,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 11,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 12,0\r
+    KERNEL8x16_I1_L4_2  -2048,0, 13,0    \r
+    KERNEL8x16_I1_L4_2  -2048,0, 14,0    \r
+    KERNEL8x16_I1_L4_2  -2048,0, 15,1          \r
+\r
+       bdnz            LSGEMM_L8x16_LOOP\r
+\r
+       MY_ALIGN\r
+LSGEMM_L8x16_LOOP_END:\r
+\r
+    END8x16 0, AO, BO, -2048, 0    \r
+\r
+       b               LSGEMM_L8x16_SUB1 \r
+       MY_ALIGN\r
+LSGEMM_L8x16_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    127\r
+#else\r
+       andi.           L,      K,      127\r
+#endif   \r
+       b               LSGEMM_L8x16_SUB2\r
+       MY_ALIGN\r
+LSGEMM_L8x16_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T12,    63\r
+#else\r
+       andi.           L,  T12,        63\r
+#endif \r
+       ble             LSGEMM_L8x16_SAVE\r
+       MY_ALIGN\r
+LSGEMM_L8x16_SUB2:\r
+\r
+    srawi.      T10,L, 5\r
+    ble LSGEMM_L8x16_SUB2_16\r
+    mtctr              T10\r
+    MY_ALIGN\r
+LSGEMM_L8x16_SUB2_LOOP:\r
+       LOAD8x16_0 \r
+    KERNEL8x16_I1_L4_2  64,32, 0,0\r
+    KERNEL8x16_I1_L4_2  64,32, 1,0\r
+    KERNEL8x16_I1_L4_2  64,32, 2,0\r
+    KERNEL8x16_I1_L4_2  64,32, 3,0\r
+    KERNEL8x16_I1_L4_2  64,32, 4,0\r
+    KERNEL8x16_I1_L4_2  64,32, 5,0\r
+    KERNEL8x16_I1_L4_2  64,32, 6,0\r
+    KERNEL8x16_I1_L4_3  64,32, 7,1\r
+    bdnz LSGEMM_L8x16_SUB2_LOOP \r
+    MY_ALIGN        \r
+LSGEMM_L8x16_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_L8x16_SUB2_8\r
+       LOAD8x16_0 \r
+    KERNEL8x16_I1_L4_2  64,32, 0,0\r
+    KERNEL8x16_I1_L4_2  64,32, 1,0\r
+    KERNEL8x16_I1_L4_2  64,32, 2,0\r
+    KERNEL8x16_I1_L4_3  64,32, 3,1\r
+    MY_ALIGN \r
+LSGEMM_L8x16_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_L8x16_SUB2_4 \r
+       LOAD8x16_0\r
+    KERNEL8x16_I1_L4_2  64,32, 0,0\r
+    KERNEL8x16_I1_L4_3  64,32, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_L8x16_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_L8x16_SUB2_2\r
+    LOAD8x16_0\r
+    KERNEL8x16_I1_L4_3  64,32, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L8x16_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_L8x16_SUB2_1\r
+    LOAD8x16_0\r
+    KERNEL8x16_I1_L2_3  64,32, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L8x16_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_L8x16_SAVE      \r
+    KERNEL8x16 0\r
+#      addic.          L,      L,      -1\r
+#      bgt             LSGEMM_L8x16_SUB2\r
+\r
+       MY_ALIGN\r
+LSGEMM_L8x16_SAVE:\r
+       SAVE8x16\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,8\r
+#endif \r
+       addic.          I,      I,      -1\r
+       bgt+            LSGEMM_L8x16_BEGIN\r
+    MY_ALIGN\r
+LSGEMM_L8x16_END:\r
+LSGEMM_L8x8_BEGIN:\r
+    andi.       T2, M,  15\r
+    ble     LSGEMM_L8x1_END\r
+\r
+    andi.       T1, M,  8\r
+    ble     LSGEMM_L8x8_END\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,8\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,8,8\r
+   mr T12, T11\r
+   addi T12,T12, -1\r
+   srawi.       L, T12, 4 /**(T11-1) % 16x */\r
+#else\r
+   mr T12, K\r
+   addi T12,T12, -1\r
+   srawi.       L,  T12,    4 /**(K-1) % 16x */\r
+#endif \r
+    \r
+    ZERO8x8\r
+    ble     LSGEMM_L8x8_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x8_LOOP_START:\r
+ \r
+    LOAD8x8_0  /*we already zeroed */ \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L8x8_LOOP:\r
+\r
+    KERNEL8x8_I1_L4_2  32,32, 0,0\r
+    KERNEL8x8_I1_L4_2  32,32, 1,0\r
+    KERNEL8x8_I1_L4_2  32,32, 2,0\r
+    KERNEL8x8_I1_L4_2  32,32, 3,1    \r
+\r
+    bdnz        LSGEMM_L8x8_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x8_LOOP_END:\r
+\r
+    END8x8 0, AO, BO, 32, 32    \r
+\r
+    b       LSGEMM_L8x8_SUB1 \r
+    MY_ALIGN\r
+LSGEMM_L8x8_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    31\r
+#else\r
+    andi.       L,  K,  31\r
+#endif   \r
+    b       LSGEMM_L8x8_SUB2\r
+    MY_ALIGN\r
+LSGEMM_L8x8_SUB1:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T12,    15\r
+#else\r
+    andi.       L,  T12,    15\r
+#endif  \r
+    ble     LSGEMM_L8x8_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L8x8_SUB2:\r
+ \r
+    srawi.      T1,L, 3\r
+    ble LSGEMM_L8x8_SUB2_4 \r
+    mtctr              T1\r
+    MY_ALIGN\r
+LSGEMM_L8x8_SUB2_LOOP:    \r
+    LOAD8x8_0\r
+    KERNEL8x8_I1_L4_2  32,32, 0,0\r
+    KERNEL8x8_I1_L4_3  32,32, 1,1\r
+    bdnz LSGEMM_L8x8_SUB2_LOOP\r
+    MY_ALIGN    \r
+LSGEMM_L8x8_SUB2_4:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L8x8_SUB2_2\r
+    LOAD8x8_0\r
+    KERNEL8x8_I1_L4_3  32,32, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L8x8_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L8x8_SUB2_1\r
+    LOAD8x8_0\r
+    KERNEL8x8_I1_L2_3  32,32, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L8x8_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L8x8_SAVE   \r
+    KERNEL8x8 0\r
+ \r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x8_SAVE:\r
+    SAVE8x8\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,8\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L8x8_END:\r
+LSGEMM_L8x4_BEGIN:\r
+    andi.       T2, M,  15\r
+    ble     LSGEMM_L8x1_END\r
+\r
+    andi.       T1, M,  4\r
+    ble     LSGEMM_L8x4_END\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,8\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,4,8\r
+   mr T12, T11\r
+   addi T12,T12, -1\r
+   srawi.       L, T12, 4 /**(T11-1) % 16x */\r
+#else\r
+   mr T12, K\r
+   addi T12,T12, -1\r
+   srawi.       L,  T12,    4 /**(K-1) % 16x */\r
+#endif \r
+    \r
+    ZERO8x4\r
+    ble     LSGEMM_L8x4_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x4_LOOP_START:\r
+ \r
+    LOAD8x4_0  /*we already zeroed */ \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L8x4_LOOP:\r
+\r
+    KERNEL8x4_I1_L4_2  16,32, 0,0\r
+    KERNEL8x4_I1_L4_2  16,32, 1,0\r
+    KERNEL8x4_I1_L4_2  16,32, 2,0\r
+    KERNEL8x4_I1_L4_2  16,32, 3,1    \r
+\r
+    bdnz        LSGEMM_L8x4_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x4_LOOP_END:\r
+\r
+    END8x4 0, AO, BO, 16, 32    \r
+\r
+    b       LSGEMM_L8x4_SUB1 \r
+    MY_ALIGN\r
+LSGEMM_L8x4_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    31\r
+#else\r
+    andi.       L,  K,  31\r
+#endif   \r
+    b       LSGEMM_L8x4_SUB2\r
+    MY_ALIGN\r
+LSGEMM_L8x4_SUB1:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T12,    15\r
+#else\r
+    andi.       L,  T12,    15\r
+#endif  \r
+    ble     LSGEMM_L8x4_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L8x4_SUB2:\r
+\r
+    srawi.      T1,L, 3\r
+    ble LSGEMM_L8x4_SUB2_4 \r
+    mtctr              T1\r
+    MY_ALIGN\r
+LSGEMM_L8x4_SUB2_LOOP:      \r
+    LOAD8x4_0\r
+    KERNEL8x4_I1_L4_2  16,32, 0,0\r
+    KERNEL8x4_I1_L4_3  16,32, 1,1\r
+    bdnz LSGEMM_L8x4_SUB2_LOOP\r
+    MY_ALIGN    \r
+LSGEMM_L8x4_SUB2_4:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L8x4_SUB2_2\r
+    LOAD8x4_0\r
+    KERNEL8x4_I1_L4_3  16,32, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L8x4_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L8x4_SUB2_1\r
+    LOAD8x4_0\r
+    KERNEL8x4_I1_L2_3  16,32, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L8x4_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L8x4_SAVE   \r
+    KERNEL8x4 0\r
+ \r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x4_SAVE:\r
+    SAVE8x4\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,8\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L8x4_END:\r
+LSGEMM_L8x2_BEGIN:\r
+    andi.       T1, M,  2\r
+    ble     LSGEMM_L8x2_END\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,8\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,2,8 \r
+   srawi.       L, T11, 3 /**(T11) % 8x */\r
+#else\r
+   srawi.       L,  K,    3 /**(K) % 8x */\r
+#endif \r
+    \r
+    ZERO8x2\r
+    ble     LSGEMM_L8x2_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x2_LOOP_START: \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L8x2_LOOP:\r
+\r
+    KERNEL8x2_2  0,0, 0,0\r
+    KERNEL8x2_2  0,0, 1,0\r
+    KERNEL8x2_2  0,0, 2,0\r
+    KERNEL8x2_2  0,0, 3,1    \r
+\r
+    bdnz        LSGEMM_L8x2_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x2_LOOP_END:   \r
+ \r
+LSGEMM_L8x2_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    7\r
+#else\r
+    andi.       L,  K,  7\r
+#endif    \r
+    ble     LSGEMM_L8x2_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L8x2_SUB2:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L8x2_SUB2_2\r
+    KERNEL8x2_2  0,0, 0,0\r
+    KERNEL8x2_2  0,0, 1,1\r
+    MY_ALIGN\r
+LSGEMM_L8x2_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L8x2_SUB2_1\r
+    KERNEL8x2_2  0,0, 0,1 \r
+    MY_ALIGN    \r
+LSGEMM_L8x2_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L8x2_SAVE   \r
+    KERNEL8x2\r
+  \r
+    MY_ALIGN\r
+LSGEMM_L8x2_SAVE:\r
+    SAVE8x2\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,8\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L8x2_END:\r
+LSGEMM_L8x1_BEGIN: \r
+    andi.       T1, M,  1\r
+    ble     LSGEMM_L8x1_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,8\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,1,8 \r
+   srawi.       L, T11, 3 /**(T11) % 8x */\r
+#else\r
+   srawi.       L,  K,    3 /**(K) % 8x */\r
+#endif \r
+    \r
+    ZERO8x1\r
+    ble     LSGEMM_L8x1_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x1_LOOP_START: \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L8x1_LOOP:\r
+\r
+    KERNEL8x1_4  0,0, 0,0\r
+    KERNEL8x1_4  0,0, 1,1     \r
+\r
+    bdnz        LSGEMM_L8x1_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L8x1_LOOP_END:   \r
+ \r
+LSGEMM_L8x1_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    7\r
+#else\r
+    andi.       L,  K,  7\r
+#endif    \r
+    ble     LSGEMM_L8x1_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L8x1_SUB2:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L8x1_SUB2_2\r
+    KERNEL8x1_4  0,0, 0,1 \r
+    MY_ALIGN\r
+LSGEMM_L8x1_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L8x1_SUB2_1\r
+    KERNEL8x1_2 \r
+    MY_ALIGN    \r
+LSGEMM_L8x1_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L8x1_SAVE   \r
+    KERNEL8x1\r
+  \r
+    MY_ALIGN\r
+LSGEMM_L8x1_SAVE:\r
+    SAVE8x1\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,8\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L8x1_END:\r
+\r
+       slwi            T1,     K,      5\r
+       add             B,      B,      T1\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+    addi TEMP_REG, TEMP_REG, 8\r
+#endif\r
+       addic.          J,      J,      -1\r
+       bgt             LSGEMM_L8_BEGIN\r
+ \r
+\r
+LSGEMM_L8_END:\r
+\r
+/*     b               LSGEMM_L4_BEGIN*/\r
+    andi.       T1, N,  4\r
+    ble     LSGEMM_L4_END\r
+LSGEMM_L4_BEGIN:\r
+  \r
+\r
+       mr              AO,     A\r
+       mr              CO,     C\r
+       slwi            T3,     LDC     ,       2\r
+       add             C,      C,      T3\r
+ \r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+       mr TEMP_REG, OFFSET      /*off = offset;*/\r
+#endif \r
+       srawi.          I,      M,      4\r
+       ble             LSGEMM_L4x16_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_L4x16_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,4\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,16,4\r
+   mr T12, T11\r
+   addi T12,T12, -1\r
+   srawi.              L, T12, 6 /**(T11-1) % 64x */\r
+#else\r
+   mr T12, K\r
+   addi T12,T12, -1\r
+   srawi.              L,      T12,    6 /**(K-1) % 64x */\r
+#endif \r
+ \r
+    ZERO4x16\r
+       ble             LSGEMM_L4x16_SUB0\r
+\r
+       MY_ALIGN\r
+LSGEMM_L4x16_LOOP_START:\r
+ \r
+       LOAD4x16_0  /*we already zeroed */\r
+    ##OffsetA=64 OffsetB=16\r
+    addi AO,AO,2112\r
+    addi BO,BO,16  \r
+\r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_L4x16_LOOP:\r
+\r
+    KERNEL4x16_I1_L4_2  -2048,0, 0,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 1,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 2,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 3,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 4,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 5,0        \r
+    KERNEL4x16_I1_L4_2  -2048,0, 6,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 7,0  \r
+    KERNEL4x16_I1_L4_2  -2048,0, 8,0      \r
+    KERNEL4x16_I1_L4_2  -2048,0, 9,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 10,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 11,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 12,0\r
+    KERNEL4x16_I1_L4_2  -2048,0, 13,0    \r
+    KERNEL4x16_I1_L4_2  -2048,0, 14,0    \r
+    KERNEL4x16_I1_L4_2  -2048,0, 15,1          \r
+\r
+       bdnz            LSGEMM_L4x16_LOOP\r
+\r
+       MY_ALIGN\r
+LSGEMM_L4x16_LOOP_END:\r
+\r
+    END4x16 0, AO, BO, -2048, 0    \r
+\r
+       b               LSGEMM_L4x16_SUB1 \r
+       MY_ALIGN\r
+LSGEMM_L4x16_SUB0:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    127\r
+#else\r
+       andi.           L,      K,      127\r
+#endif   \r
+       b               LSGEMM_L4x16_SUB2\r
+       MY_ALIGN\r
+LSGEMM_L4x16_SUB1:\r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T12,    63\r
+#else\r
+       andi.           L,  T12,        63\r
+#endif \r
+       ble             LSGEMM_L4x16_SAVE\r
+       MY_ALIGN\r
+LSGEMM_L4x16_SUB2:\r
+\r
+    srawi.      T10,L, 5\r
+    ble LSGEMM_L4x16_SUB2_16\r
+    mtctr              T10\r
+    MY_ALIGN\r
+LSGEMM_L4x16_SUB2_LOOP:\r
+       LOAD4x16_0 \r
+    KERNEL4x16_I1_L4_2  64,16, 0,0\r
+    KERNEL4x16_I1_L4_2  64,16, 1,0\r
+    KERNEL4x16_I1_L4_2  64,16, 2,0\r
+    KERNEL4x16_I1_L4_2  64,16, 3,0\r
+    KERNEL4x16_I1_L4_2  64,16, 4,0\r
+    KERNEL4x16_I1_L4_2  64,16, 5,0\r
+    KERNEL4x16_I1_L4_2  64,16, 6,0\r
+    KERNEL4x16_I1_L4_3  64,16, 7,1\r
+    bdnz LSGEMM_L4x16_SUB2_LOOP \r
+    MY_ALIGN        \r
+LSGEMM_L4x16_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_L4x16_SUB2_8\r
+       LOAD4x16_0 \r
+    KERNEL4x16_I1_L4_2  64,16, 0,0\r
+    KERNEL4x16_I1_L4_2  64,16, 1,0\r
+    KERNEL4x16_I1_L4_2  64,16, 2,0\r
+    KERNEL4x16_I1_L4_3  64,16, 3,1\r
+    MY_ALIGN \r
+LSGEMM_L4x16_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_L4x16_SUB2_4 \r
+       LOAD4x16_0\r
+    KERNEL4x16_I1_L4_2  64,16, 0,0\r
+    KERNEL4x16_I1_L4_3  64,16, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_L4x16_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_L4x16_SUB2_2\r
+    LOAD4x16_0\r
+    KERNEL4x16_I1_L4_3  64,16, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L4x16_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_L4x16_SUB2_1\r
+    LOAD4x16_0\r
+    KERNEL4x16_I1_L2_3  64,16, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L4x16_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_L4x16_SAVE      \r
+    KERNEL4x16 0\r
+#      addic.          L,      L,      -1\r
+#      bgt             LSGEMM_L4x16_SUB2\r
+\r
+       MY_ALIGN\r
+LSGEMM_L4x16_SAVE:\r
+       SAVE4x16\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,4\r
+#endif \r
+       addic.          I,      I,      -1\r
+       bgt+            LSGEMM_L4x16_BEGIN\r
+    MY_ALIGN\r
+LSGEMM_L4x16_END:\r
+LSGEMM_L4x8_BEGIN:\r
+    andi.       T2, M,  15\r
+    ble     LSGEMM_L4x1_END\r
+\r
+    andi.       T1, M,  8\r
+    ble     LSGEMM_L4x8_END\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,4\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,8,4\r
+   mr T12, T11\r
+   addi T12,T12, -1\r
+   srawi.       L, T12, 4 /**(T11-1) % 16x */\r
+#else\r
+   mr T12, K\r
+   addi T12,T12, -1\r
+   srawi.       L,  T12,    4 /**(K-1) % 16x */\r
+#endif \r
+    \r
+    ZERO4x8\r
+    ble     LSGEMM_L4x8_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x8_LOOP_START:\r
+ \r
+    LOAD4x8_0  /*we already zeroed */ \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L4x8_LOOP:\r
+\r
+    KERNEL4x8_I1_L4_2  32,16, 0,0\r
+    KERNEL4x8_I1_L4_2  32,16, 1,0\r
+    KERNEL4x8_I1_L4_2  32,16, 2,0\r
+    KERNEL4x8_I1_L4_2  32,16, 3,1    \r
+\r
+    bdnz        LSGEMM_L4x8_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x8_LOOP_END:\r
+\r
+    END4x8 0, AO, BO, 32, 16    \r
+\r
+    b       LSGEMM_L4x8_SUB1 \r
+    MY_ALIGN\r
+LSGEMM_L4x8_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    31\r
+#else\r
+    andi.       L,  K,  31\r
+#endif   \r
+    b       LSGEMM_L4x8_SUB2\r
+    MY_ALIGN\r
+LSGEMM_L4x8_SUB1:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T12,    15\r
+#else\r
+    andi.       L,  T12,    15\r
+#endif  \r
+    ble     LSGEMM_L4x8_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L4x8_SUB2:\r
+ \r
+    srawi.      T1,L, 3\r
+    ble LSGEMM_L4x8_SUB2_4 \r
+    mtctr              T1\r
+    MY_ALIGN\r
+LSGEMM_L4x8_SUB2_LOOP:    \r
+    LOAD4x8_0\r
+    KERNEL4x8_I1_L4_2  32,16, 0,0\r
+    KERNEL4x8_I1_L4_3  32,16, 1,1\r
+    bdnz LSGEMM_L4x8_SUB2_LOOP\r
+    MY_ALIGN    \r
+LSGEMM_L4x8_SUB2_4:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L4x8_SUB2_2\r
+    LOAD4x8_0\r
+    KERNEL4x8_I1_L4_3  32,16, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L4x8_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L4x8_SUB2_1\r
+    LOAD4x8_0\r
+    KERNEL4x8_I1_L2_3  32,16, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L4x8_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L4x8_SAVE   \r
+    KERNEL4x8 0\r
+ \r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x8_SAVE:\r
+    SAVE4x8\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,4\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L4x8_END:\r
+LSGEMM_L4x4_BEGIN:\r
+    andi.       T2, M,  15\r
+    ble     LSGEMM_L4x1_END\r
+\r
+    andi.       T1, M,  4\r
+    ble     LSGEMM_L4x4_END\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,4\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,4,4\r
+   mr T12, T11\r
+   addi T12,T12, -1\r
+   srawi.       L, T12, 4 /**(T11-1) % 16x */\r
+#else\r
+   mr T12, K\r
+   addi T12,T12, -1\r
+   srawi.       L,  T12,    4 /**(K-1) % 16x */\r
+#endif \r
+    \r
+    ZERO4x4\r
+    ble     LSGEMM_L4x4_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x4_LOOP_START:\r
+ \r
+    LOAD4x4_0  /*we already zeroed */ \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L4x4_LOOP:\r
+\r
+    KERNEL4x4_I1_L4_2  16,16, 0,0\r
+    KERNEL4x4_I1_L4_2  16,16, 1,0\r
+    KERNEL4x4_I1_L4_2  16,16, 2,0\r
+    KERNEL4x4_I1_L4_2  16,16, 3,1    \r
+\r
+    bdnz        LSGEMM_L4x4_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x4_LOOP_END:\r
+\r
+    END4x4 0, AO, BO, 16, 16    \r
+\r
+    b       LSGEMM_L4x4_SUB1 \r
+    MY_ALIGN\r
+LSGEMM_L4x4_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    31\r
+#else\r
+    andi.       L,  K,  31\r
+#endif   \r
+    b       LSGEMM_L4x4_SUB2\r
+    MY_ALIGN\r
+LSGEMM_L4x4_SUB1:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T12,    15\r
+#else\r
+    andi.       L,  T12,    15\r
+#endif  \r
+    ble     LSGEMM_L4x4_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L4x4_SUB2:\r
+\r
+    srawi.      T1,L, 3 \r
+    ble LSGEMM_L4x4_SUB2_4  \r
+    mtctr              T1\r
+    MY_ALIGN\r
+LSGEMM_L4x4_SUB2_LOOP:     \r
+    LOAD4x4_0\r
+    KERNEL4x4_I1_L4_2  16,16, 0,0\r
+    KERNEL4x4_I1_L4_3  16,16, 1,1\r
+    bdnz LSGEMM_L4x4_SUB2_LOOP\r
+    MY_ALIGN    \r
+LSGEMM_L4x4_SUB2_4:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L4x4_SUB2_2\r
+    LOAD4x4_0\r
+    KERNEL4x4_I1_L4_3  16,16, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L4x4_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L4x4_SUB2_1\r
+    LOAD4x4_0\r
+    KERNEL4x4_I1_L2_3  16,16, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L4x4_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L4x4_SAVE   \r
+    KERNEL4x4 0\r
+ \r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x4_SAVE:\r
+    SAVE4x4\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,4\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L4x4_END:\r
+LSGEMM_L4x2_BEGIN:\r
+    andi.       T1, M,  2\r
+    ble     LSGEMM_L4x2_END\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,4\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,2,4 \r
+   srawi.       L, T11, 3 /**(T11) % 8x */\r
+#else\r
+   srawi.       L,  K,    3 /**(K) % 8x */\r
+#endif \r
+    \r
+    ZERO4x2\r
+    ble     LSGEMM_L4x2_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x2_LOOP_START: \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L4x2_LOOP:\r
+\r
+    KERNEL4x2_2  0,0, 0,0\r
+    KERNEL4x2_2  0,0, 1,0\r
+    KERNEL4x2_2  0,0, 2,0\r
+    KERNEL4x2_2  0,0, 3,1    \r
+\r
+    bdnz        LSGEMM_L4x2_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x2_LOOP_END:   \r
+ \r
+LSGEMM_L4x2_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    7\r
+#else\r
+    andi.       L,  K,  7\r
+#endif    \r
+    ble     LSGEMM_L4x2_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L4x2_SUB2:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L4x2_SUB2_2\r
+    KERNEL4x2_2  0,0, 0,0\r
+    KERNEL4x2_2  0,0, 1,1\r
+    MY_ALIGN\r
+LSGEMM_L4x2_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L4x2_SUB2_1\r
+    KERNEL4x2_2  0,0, 0,1 \r
+    MY_ALIGN    \r
+LSGEMM_L4x2_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L4x2_SAVE   \r
+    KERNEL4x2\r
+  \r
+    MY_ALIGN\r
+LSGEMM_L4x2_SAVE:\r
+    SAVE4x2\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,4\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L4x2_END:\r
+LSGEMM_L4x1_BEGIN: \r
+    andi.       T1, M,  1\r
+    ble     LSGEMM_L4x1_END\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,4\r
+#else\r
+    mr      BO, B\r
+#endif  \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,1,4 \r
+   srawi.       L, T11, 3 /**(T11) % 8x */\r
+#else\r
+   srawi.       L,  K,    3 /**(K) % 8x */\r
+#endif \r
+    \r
+    ZERO4x1\r
+    ble     LSGEMM_L4x1_SUB0\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x1_LOOP_START: \r
+    mtctr       L\r
+\r
+    MY_ALIGN\r
+\r
+LSGEMM_L4x1_LOOP:\r
+\r
+    KERNEL4x1_4  0,0, 0,0\r
+    KERNEL4x1_4  0,0, 1,1     \r
+\r
+    bdnz        LSGEMM_L4x1_LOOP\r
+\r
+    MY_ALIGN\r
+LSGEMM_L4x1_LOOP_END:   \r
+ \r
+LSGEMM_L4x1_SUB0:\r
+#if defined(TRMMKERNEL)\r
+    andi.       L,  T11,    7\r
+#else\r
+    andi.       L,  K,  7\r
+#endif    \r
+    ble     LSGEMM_L4x1_SAVE\r
+    MY_ALIGN\r
+LSGEMM_L4x1_SUB2:\r
+    andi.      T1,L, 4\r
+    ble LSGEMM_L4x1_SUB2_2\r
+    KERNEL4x1_4  0,0, 0,1 \r
+    MY_ALIGN\r
+LSGEMM_L4x1_SUB2_2:\r
+    andi.      T1,L, 2\r
+    ble LSGEMM_L4x1_SUB2_1\r
+    KERNEL4x1_2 \r
+    MY_ALIGN    \r
+LSGEMM_L4x1_SUB2_1:\r
+    andi.      T1,L, 1\r
+    ble LSGEMM_L4x1_SAVE   \r
+    KERNEL4x1\r
+  \r
+    MY_ALIGN\r
+LSGEMM_L4x1_SAVE:\r
+    SAVE4x1\r
+#if defined(TRMMKERNEL) \r
+    REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,4\r
+#endif  \r
+    MY_ALIGN  \r
+LSGEMM_L4x1_END:\r
+\r
+       slwi            T1,     K,      4\r
+       add             B,      B,      T1\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+    addi TEMP_REG, TEMP_REG, 4\r
+#endif\r
+\r
+       andi.           T2,     N,      3\r
+       ble             .L999\r
+\r
+LSGEMM_L4_END:\r
+    andi.       T1, N,  2\r
+    ble     LSGEMM_L2_END\r
+LSGEMM_L2_BEGIN:\r
+  \r
+\r
+       mr              AO,     A\r
+       mr              CO,     C\r
+       slwi            T3,     LDC     ,       1\r
+       add             C,      C,      T3\r
+ \r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+       mr TEMP_REG, OFFSET      /*off = offset;*/\r
+#endif \r
+       srawi.          I,      M,      4\r
+       ble             LSGEMM_L2x16_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x16_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,2\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,16,2 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO2x16\r
+       ble             LSGEMM_L2x16_SUB0\r
+    addi AO,AO,2048\r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_L2x16_LOOP:\r
+\r
+    KERNEL2x16_4  -2048,0, 0,0\r
+    KERNEL2x16_4  -2048,0, 1,0\r
+    KERNEL2x16_4  -2048,0, 2,0\r
+    KERNEL2x16_4  -2048,0, 3,0\r
+    KERNEL2x16_4  -2048,0, 4,0\r
+    KERNEL2x16_4  -2048,0, 5,0        \r
+    KERNEL2x16_4  -2048,0, 6,0\r
+    KERNEL2x16_4  -2048,0, 7,0  \r
+    KERNEL2x16_4  -2048,0, 8,0      \r
+    KERNEL2x16_4  -2048,0, 9,0\r
+    KERNEL2x16_4  -2048,0, 10,0\r
+    KERNEL2x16_4  -2048,0, 11,0\r
+    KERNEL2x16_4  -2048,0, 12,0\r
+    KERNEL2x16_4  -2048,0, 13,0    \r
+    KERNEL2x16_4  -2048,0, 14,0    \r
+    KERNEL2x16_4  -2048,0, 15,1        \r
+\r
+       bdnz            LSGEMM_L2x16_LOOP\r
+    MY_ALIGN\r
+    addi AO,AO, -2048\r
+       MY_ALIGN\r
+LSGEMM_L2x16_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_L2x16_SAVE\r
+       MY_ALIGN\r
+LSGEMM_L2x16_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_L2x16_SUB2_16 \r
+    KERNEL2x16_4  0,0, 0,0\r
+    KERNEL2x16_4  0,0, 1,0\r
+    KERNEL2x16_4  0,0, 2,0\r
+    KERNEL2x16_4  0,0, 3,0\r
+    KERNEL2x16_4  0,0, 4,0\r
+    KERNEL2x16_4  0,0, 5,0\r
+    KERNEL2x16_4  0,0, 6,0\r
+    KERNEL2x16_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_L2x16_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_L2x16_SUB2_8 \r
+    KERNEL2x16_4  0,0, 0,0\r
+    KERNEL2x16_4  0,0, 1,0\r
+    KERNEL2x16_4  0,0, 2,0\r
+    KERNEL2x16_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_L2x16_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_L2x16_SUB2_4  \r
+    KERNEL2x16_4  0,0, 0,0\r
+    KERNEL2x16_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_L2x16_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_L2x16_SUB2_2 \r
+    KERNEL2x16_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L2x16_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_L2x16_SUB2_1 \r
+    KERNEL2x16_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L2x16_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_L2x16_SAVE      \r
+    KERNEL2x16\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x16_SAVE:\r
+       SAVE2x16\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,2\r
+#endif \r
+       addic.          I,      I,      -1\r
+       bgt+            LSGEMM_L2x16_BEGIN\r
+    MY_ALIGN\r
+LSGEMM_L2x16_END:\r
+       andi.           I,      M,      8\r
+       ble             LSGEMM_L2x8_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x8_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,2\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,8,2 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO2x8\r
+       ble             LSGEMM_L2x8_SUB0\r
+    addi AO,AO,2048\r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_L2x8_LOOP:\r
+\r
+    KERNEL2x8_4  -2048,0, 0,0\r
+    KERNEL2x8_4  -2048,0, 1,0\r
+    KERNEL2x8_4  -2048,0, 2,0\r
+    KERNEL2x8_4  -2048,0, 3,0\r
+    KERNEL2x8_4  -2048,0, 4,0\r
+    KERNEL2x8_4  -2048,0, 5,0        \r
+    KERNEL2x8_4  -2048,0, 6,0\r
+    KERNEL2x8_4  -2048,0, 7,0  \r
+    KERNEL2x8_4  -2048,0, 8,0      \r
+    KERNEL2x8_4  -2048,0, 9,0\r
+    KERNEL2x8_4  -2048,0, 10,0\r
+    KERNEL2x8_4  -2048,0, 11,0\r
+    KERNEL2x8_4  -2048,0, 12,0\r
+    KERNEL2x8_4  -2048,0, 13,0    \r
+    KERNEL2x8_4  -2048,0, 14,0    \r
+    KERNEL2x8_4  -2048,0, 15,1         \r
+\r
+       bdnz            LSGEMM_L2x8_LOOP\r
+    MY_ALIGN\r
+    addi AO,AO, -2048\r
+       MY_ALIGN\r
+LSGEMM_L2x8_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_L2x8_SAVE\r
+       MY_ALIGN\r
+LSGEMM_L2x8_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_L2x8_SUB2_16 \r
+    KERNEL2x8_4  0,0, 0,0\r
+    KERNEL2x8_4  0,0, 1,0\r
+    KERNEL2x8_4  0,0, 2,0\r
+    KERNEL2x8_4  0,0, 3,0\r
+    KERNEL2x8_4  0,0, 4,0\r
+    KERNEL2x8_4  0,0, 5,0\r
+    KERNEL2x8_4  0,0, 6,0\r
+    KERNEL2x8_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_L2x8_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_L2x8_SUB2_8 \r
+    KERNEL2x8_4  0,0, 0,0\r
+    KERNEL2x8_4  0,0, 1,0\r
+    KERNEL2x8_4  0,0, 2,0\r
+    KERNEL2x8_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_L2x8_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_L2x8_SUB2_4  \r
+    KERNEL2x8_4  0,0, 0,0\r
+    KERNEL2x8_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_L2x8_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_L2x8_SUB2_2 \r
+    KERNEL2x8_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L2x8_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_L2x8_SUB2_1 \r
+    KERNEL2x8_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L2x8_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_L2x8_SAVE       \r
+    KERNEL2x8\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x8_SAVE:\r
+       SAVE2x8\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,2\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_L2x8_END:\r
+       andi.           I,      M,      4\r
+       ble             LSGEMM_L2x4_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x4_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,2\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,4,2 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO2x4\r
+       ble             LSGEMM_L2x4_SUB0\r
+ \r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_L2x4_LOOP:\r
+\r
+    KERNEL2x4_4  0,0, 0,0\r
+    KERNEL2x4_4  0,0, 1,0\r
+    KERNEL2x4_4  0,0, 2,0\r
+    KERNEL2x4_4  0,0, 3,0\r
+    KERNEL2x4_4  0,0, 4,0\r
+    KERNEL2x4_4  0,0, 5,0        \r
+    KERNEL2x4_4  0,0, 6,0\r
+    KERNEL2x4_4  0,0, 7,0  \r
+    KERNEL2x4_4  0,0, 8,0      \r
+    KERNEL2x4_4  0,0, 9,0\r
+    KERNEL2x4_4  0,0, 10,0\r
+    KERNEL2x4_4  0,0, 11,0\r
+    KERNEL2x4_4  0,0, 12,0\r
+    KERNEL2x4_4  0,0, 13,0    \r
+    KERNEL2x4_4  0,0, 14,0    \r
+    KERNEL2x4_4  0,0, 15,1     \r
+\r
+       bdnz            LSGEMM_L2x4_LOOP\r
+    MY_ALIGN\r
+ \r
+       MY_ALIGN\r
+LSGEMM_L2x4_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_L2x4_SAVE\r
+       MY_ALIGN\r
+LSGEMM_L2x4_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_L2x4_SUB2_16 \r
+    KERNEL2x4_4  0,0, 0,0\r
+    KERNEL2x4_4  0,0, 1,0\r
+    KERNEL2x4_4  0,0, 2,0\r
+    KERNEL2x4_4  0,0, 3,0\r
+    KERNEL2x4_4  0,0, 4,0\r
+    KERNEL2x4_4  0,0, 5,0\r
+    KERNEL2x4_4  0,0, 6,0\r
+    KERNEL2x4_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_L2x4_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_L2x4_SUB2_8 \r
+    KERNEL2x4_4  0,0, 0,0\r
+    KERNEL2x4_4  0,0, 1,0\r
+    KERNEL2x4_4  0,0, 2,0\r
+    KERNEL2x4_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_L2x4_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_L2x4_SUB2_4  \r
+    KERNEL2x4_4  0,0, 0,0\r
+    KERNEL2x4_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_L2x4_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_L2x4_SUB2_2 \r
+    KERNEL2x4_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L2x4_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_L2x4_SUB2_1 \r
+    KERNEL2x4_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L2x4_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_L2x4_SAVE       \r
+    KERNEL2x4\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x4_SAVE:\r
+       SAVE2x4\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,2\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_L2x4_END:\r
+       andi.           I,      M,      2\r
+       ble             LSGEMM_L2x2_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x2_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,2\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,2,2 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO2x2\r
+       ble             LSGEMM_L2x2_SUB0\r
+ \r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_L2x2_LOOP:\r
+\r
+    KERNEL2x2_4  0,0, 0,0\r
+    KERNEL2x2_4  0,0, 1,0\r
+    KERNEL2x2_4  0,0, 2,0\r
+    KERNEL2x2_4  0,0, 3,0\r
+    KERNEL2x2_4  0,0, 4,0\r
+    KERNEL2x2_4  0,0, 5,0        \r
+    KERNEL2x2_4  0,0, 6,0\r
+    KERNEL2x2_4  0,0, 7,0  \r
+    KERNEL2x2_4  0,0, 8,0      \r
+    KERNEL2x2_4  0,0, 9,0\r
+    KERNEL2x2_4  0,0, 10,0\r
+    KERNEL2x2_4  0,0, 11,0\r
+    KERNEL2x2_4  0,0, 12,0\r
+    KERNEL2x2_4  0,0, 13,0    \r
+    KERNEL2x2_4  0,0, 14,0    \r
+    KERNEL2x2_4  0,0, 15,1     \r
+\r
+       bdnz            LSGEMM_L2x2_LOOP\r
+    MY_ALIGN\r
+ \r
+       MY_ALIGN\r
+LSGEMM_L2x2_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_L2x2_SAVE\r
+       MY_ALIGN\r
+LSGEMM_L2x2_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_L2x2_SUB2_16 \r
+    KERNEL2x2_4  0,0, 0,0\r
+    KERNEL2x2_4  0,0, 1,0\r
+    KERNEL2x2_4  0,0, 2,0\r
+    KERNEL2x2_4  0,0, 3,0\r
+    KERNEL2x2_4  0,0, 4,0\r
+    KERNEL2x2_4  0,0, 5,0\r
+    KERNEL2x2_4  0,0, 6,0\r
+    KERNEL2x2_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_L2x2_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_L2x2_SUB2_8 \r
+    KERNEL2x2_4  0,0, 0,0\r
+    KERNEL2x2_4  0,0, 1,0\r
+    KERNEL2x2_4  0,0, 2,0\r
+    KERNEL2x2_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_L2x2_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_L2x2_SUB2_4  \r
+    KERNEL2x2_4  0,0, 0,0\r
+    KERNEL2x2_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_L2x2_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_L2x2_SUB2_2 \r
+    KERNEL2x2_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L2x2_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_L2x2_SUB2_1 \r
+    KERNEL2x2_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L2x2_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_L2x2_SAVE       \r
+    KERNEL2x2\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x2_SAVE:\r
+       SAVE2x2\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,2\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_L2x2_END:\r
+       andi.           I,      M,      1\r
+       ble             LSGEMM_L2x1_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x1_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,2\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,1,2 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO2x1\r
+       ble             LSGEMM_L2x1_SUB0\r
+ \r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_L2x1_LOOP:\r
+\r
+    KERNEL2x1_4  0,0, 0,0\r
+    KERNEL2x1_4  0,0, 1,0\r
+    KERNEL2x1_4  0,0, 2,0\r
+    KERNEL2x1_4  0,0, 3,0\r
+    KERNEL2x1_4  0,0, 4,0\r
+    KERNEL2x1_4  0,0, 5,0        \r
+    KERNEL2x1_4  0,0, 6,0\r
+    KERNEL2x1_4  0,0, 7,0  \r
+    KERNEL2x1_4  0,0, 8,0      \r
+    KERNEL2x1_4  0,0, 9,0\r
+    KERNEL2x1_4  0,0, 10,0\r
+    KERNEL2x1_4  0,0, 11,0\r
+    KERNEL2x1_4  0,0, 12,0\r
+    KERNEL2x1_4  0,0, 13,0    \r
+    KERNEL2x1_4  0,0, 14,0    \r
+    KERNEL2x1_4  0,0, 15,1     \r
+\r
+       bdnz            LSGEMM_L2x1_LOOP\r
+    MY_ALIGN\r
+ \r
+       MY_ALIGN\r
+LSGEMM_L2x1_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_L2x1_SAVE\r
+       MY_ALIGN\r
+LSGEMM_L2x1_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_L2x1_SUB2_16 \r
+    KERNEL2x1_4  0,0, 0,0\r
+    KERNEL2x1_4  0,0, 1,0\r
+    KERNEL2x1_4  0,0, 2,0\r
+    KERNEL2x1_4  0,0, 3,0\r
+    KERNEL2x1_4  0,0, 4,0\r
+    KERNEL2x1_4  0,0, 5,0\r
+    KERNEL2x1_4  0,0, 6,0\r
+    KERNEL2x1_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_L2x1_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_L2x1_SUB2_8 \r
+    KERNEL2x1_4  0,0, 0,0\r
+    KERNEL2x1_4  0,0, 1,0\r
+    KERNEL2x1_4  0,0, 2,0\r
+    KERNEL2x1_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_L2x1_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_L2x1_SUB2_4  \r
+    KERNEL2x1_4  0,0, 0,0\r
+    KERNEL2x1_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_L2x1_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_L2x1_SUB2_2 \r
+    KERNEL2x1_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_L2x1_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_L2x1_SUB2_1 \r
+    KERNEL2x1_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_L2x1_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_L2x1_SAVE       \r
+    KERNEL2x1\r
+\r
+       MY_ALIGN\r
+LSGEMM_L2x1_SAVE:\r
+       SAVE2x1\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,2\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_L2x1_END:\r
+       slwi            T1,     K,      3\r
+       add             B,      B,      T1\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+    addi TEMP_REG, TEMP_REG, 2\r
+#endif \r
+LSGEMM_L2_END:\r
+   andi.       T1, N,  1\r
+   ble     LSGEMM_END\r
+LSGEMM_1_BEGIN:\r
+  \r
+\r
+       mr              AO,     A\r
+       mr              CO,     C \r
+       add             C,      C,      LDC\r
+ \r
+#if defined(TRMMKERNEL) && defined(LEFT)\r
+       mr TEMP_REG, OFFSET      /*off = offset;*/\r
+#endif \r
+       srawi.          I,      M,      4\r
+       ble             LSGEMM_1x16_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x16_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,1\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,16,1 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO1x16\r
+       ble             LSGEMM_1x16_SUB0\r
+    addi AO,AO,2048\r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_1x16_LOOP:\r
+\r
+    KERNEL1x16_4  -2048,0, 0,0\r
+    KERNEL1x16_4  -2048,0, 1,0\r
+    KERNEL1x16_4  -2048,0, 2,0\r
+    KERNEL1x16_4  -2048,0, 3,0\r
+    KERNEL1x16_4  -2048,0, 4,0\r
+    KERNEL1x16_4  -2048,0, 5,0        \r
+    KERNEL1x16_4  -2048,0, 6,0\r
+    KERNEL1x16_4  -2048,0, 7,0  \r
+    KERNEL1x16_4  -2048,0, 8,0      \r
+    KERNEL1x16_4  -2048,0, 9,0\r
+    KERNEL1x16_4  -2048,0, 10,0\r
+    KERNEL1x16_4  -2048,0, 11,0\r
+    KERNEL1x16_4  -2048,0, 12,0\r
+    KERNEL1x16_4  -2048,0, 13,0    \r
+    KERNEL1x16_4  -2048,0, 14,0    \r
+    KERNEL1x16_4  -2048,0, 15,1        \r
+\r
+       bdnz            LSGEMM_1x16_LOOP\r
+    MY_ALIGN\r
+    addi AO,AO, -2048\r
+       MY_ALIGN\r
+LSGEMM_1x16_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_1x16_SAVE\r
+       MY_ALIGN\r
+LSGEMM_1x16_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_1x16_SUB2_16 \r
+    KERNEL1x16_4  0,0, 0,0\r
+    KERNEL1x16_4  0,0, 1,0\r
+    KERNEL1x16_4  0,0, 2,0\r
+    KERNEL1x16_4  0,0, 3,0\r
+    KERNEL1x16_4  0,0, 4,0\r
+    KERNEL1x16_4  0,0, 5,0\r
+    KERNEL1x16_4  0,0, 6,0\r
+    KERNEL1x16_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_1x16_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_1x16_SUB2_8 \r
+    KERNEL1x16_4  0,0, 0,0\r
+    KERNEL1x16_4  0,0, 1,0\r
+    KERNEL1x16_4  0,0, 2,0\r
+    KERNEL1x16_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_1x16_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_1x16_SUB2_4  \r
+    KERNEL1x16_4  0,0, 0,0\r
+    KERNEL1x16_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_1x16_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_1x16_SUB2_2 \r
+    KERNEL1x16_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_1x16_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_1x16_SUB2_1 \r
+    KERNEL1x16_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_1x16_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_1x16_SAVE       \r
+    KERNEL1x16\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x16_SAVE:\r
+       SAVE1x16\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,1\r
+#endif \r
+       addic.          I,      I,      -1\r
+       bgt+            LSGEMM_1x16_BEGIN\r
+    MY_ALIGN\r
+LSGEMM_1x16_END:\r
+       andi.           I,      M,      8\r
+       ble             LSGEMM_1x8_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x8_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,1\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,8,1 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO1x8\r
+       ble             LSGEMM_1x8_SUB0\r
+    addi AO,AO,2048\r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_1x8_LOOP:\r
+\r
+    KERNEL1x8_4  -2048,0, 0,0\r
+    KERNEL1x8_4  -2048,0, 1,0\r
+    KERNEL1x8_4  -2048,0, 2,0\r
+    KERNEL1x8_4  -2048,0, 3,0\r
+    KERNEL1x8_4  -2048,0, 4,0\r
+    KERNEL1x8_4  -2048,0, 5,0        \r
+    KERNEL1x8_4  -2048,0, 6,0\r
+    KERNEL1x8_4  -2048,0, 7,0  \r
+    KERNEL1x8_4  -2048,0, 8,0      \r
+    KERNEL1x8_4  -2048,0, 9,0\r
+    KERNEL1x8_4  -2048,0, 10,0\r
+    KERNEL1x8_4  -2048,0, 11,0\r
+    KERNEL1x8_4  -2048,0, 12,0\r
+    KERNEL1x8_4  -2048,0, 13,0    \r
+    KERNEL1x8_4  -2048,0, 14,0    \r
+    KERNEL1x8_4  -2048,0, 15,1         \r
+\r
+       bdnz            LSGEMM_1x8_LOOP\r
+    MY_ALIGN\r
+    addi AO,AO, -2048\r
+       MY_ALIGN\r
+LSGEMM_1x8_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_1x8_SAVE\r
+       MY_ALIGN\r
+LSGEMM_1x8_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_1x8_SUB2_16 \r
+    KERNEL1x8_4  0,0, 0,0\r
+    KERNEL1x8_4  0,0, 1,0\r
+    KERNEL1x8_4  0,0, 2,0\r
+    KERNEL1x8_4  0,0, 3,0\r
+    KERNEL1x8_4  0,0, 4,0\r
+    KERNEL1x8_4  0,0, 5,0\r
+    KERNEL1x8_4  0,0, 6,0\r
+    KERNEL1x8_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_1x8_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_1x8_SUB2_8 \r
+    KERNEL1x8_4  0,0, 0,0\r
+    KERNEL1x8_4  0,0, 1,0\r
+    KERNEL1x8_4  0,0, 2,0\r
+    KERNEL1x8_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_1x8_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_1x8_SUB2_4  \r
+    KERNEL1x8_4  0,0, 0,0\r
+    KERNEL1x8_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_1x8_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_1x8_SUB2_2 \r
+    KERNEL1x8_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_1x8_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_1x8_SUB2_1 \r
+    KERNEL1x8_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_1x8_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_1x8_SAVE        \r
+    KERNEL1x8\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x8_SAVE:\r
+       SAVE1x8\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,1\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_1x8_END:\r
+       andi.           I,      M,      4\r
+       ble             LSGEMM_1x4_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x4_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,1\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,4,1 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO1x4\r
+       ble             LSGEMM_1x4_SUB0\r
+ \r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_1x4_LOOP:\r
+\r
+    KERNEL1x4_4  0,0, 0,0\r
+    KERNEL1x4_4  0,0, 1,0\r
+    KERNEL1x4_4  0,0, 2,0\r
+    KERNEL1x4_4  0,0, 3,0\r
+    KERNEL1x4_4  0,0, 4,0\r
+    KERNEL1x4_4  0,0, 5,0        \r
+    KERNEL1x4_4  0,0, 6,0\r
+    KERNEL1x4_4  0,0, 7,0  \r
+    KERNEL1x4_4  0,0, 8,0      \r
+    KERNEL1x4_4  0,0, 9,0\r
+    KERNEL1x4_4  0,0, 10,0\r
+    KERNEL1x4_4  0,0, 11,0\r
+    KERNEL1x4_4  0,0, 12,0\r
+    KERNEL1x4_4  0,0, 13,0    \r
+    KERNEL1x4_4  0,0, 14,0    \r
+    KERNEL1x4_4  0,0, 15,1     \r
+\r
+       bdnz            LSGEMM_1x4_LOOP\r
+    MY_ALIGN\r
+ \r
+       MY_ALIGN\r
+LSGEMM_1x4_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_1x4_SAVE\r
+       MY_ALIGN\r
+LSGEMM_1x4_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_1x4_SUB2_16 \r
+    KERNEL1x4_4  0,0, 0,0\r
+    KERNEL1x4_4  0,0, 1,0\r
+    KERNEL1x4_4  0,0, 2,0\r
+    KERNEL1x4_4  0,0, 3,0\r
+    KERNEL1x4_4  0,0, 4,0\r
+    KERNEL1x4_4  0,0, 5,0\r
+    KERNEL1x4_4  0,0, 6,0\r
+    KERNEL1x4_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_1x4_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_1x4_SUB2_8 \r
+    KERNEL1x4_4  0,0, 0,0\r
+    KERNEL1x4_4  0,0, 1,0\r
+    KERNEL1x4_4  0,0, 2,0\r
+    KERNEL1x4_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_1x4_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_1x4_SUB2_4  \r
+    KERNEL1x4_4  0,0, 0,0\r
+    KERNEL1x4_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_1x4_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_1x4_SUB2_2 \r
+    KERNEL1x4_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_1x4_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_1x4_SUB2_1 \r
+    KERNEL1x4_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_1x4_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_1x4_SAVE        \r
+    KERNEL1x4\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x4_SAVE:\r
+       SAVE1x4\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,1\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_1x4_END:\r
+       andi.           I,      M,      2\r
+       ble             LSGEMM_1x2_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x2_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,1\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,2,1 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO1x2\r
+       ble             LSGEMM_1x2_SUB0\r
+ \r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_1x2_LOOP:\r
+\r
+    KERNEL1x2_4  0,0, 0,0\r
+    KERNEL1x2_4  0,0, 1,0\r
+    KERNEL1x2_4  0,0, 2,0\r
+    KERNEL1x2_4  0,0, 3,0\r
+    KERNEL1x2_4  0,0, 4,0\r
+    KERNEL1x2_4  0,0, 5,0        \r
+    KERNEL1x2_4  0,0, 6,0\r
+    KERNEL1x2_4  0,0, 7,0  \r
+    KERNEL1x2_4  0,0, 8,0      \r
+    KERNEL1x2_4  0,0, 9,0\r
+    KERNEL1x2_4  0,0, 10,0\r
+    KERNEL1x2_4  0,0, 11,0\r
+    KERNEL1x2_4  0,0, 12,0\r
+    KERNEL1x2_4  0,0, 13,0    \r
+    KERNEL1x2_4  0,0, 14,0    \r
+    KERNEL1x2_4  0,0, 15,1     \r
+\r
+       bdnz            LSGEMM_1x2_LOOP\r
+    MY_ALIGN\r
+ \r
+       MY_ALIGN\r
+LSGEMM_1x2_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_1x2_SAVE\r
+       MY_ALIGN\r
+LSGEMM_1x2_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_1x2_SUB2_16 \r
+    KERNEL1x2_4  0,0, 0,0\r
+    KERNEL1x2_4  0,0, 1,0\r
+    KERNEL1x2_4  0,0, 2,0\r
+    KERNEL1x2_4  0,0, 3,0\r
+    KERNEL1x2_4  0,0, 4,0\r
+    KERNEL1x2_4  0,0, 5,0\r
+    KERNEL1x2_4  0,0, 6,0\r
+    KERNEL1x2_4  0,0, 7,1 \r
+    MY_ALIGN        \r
+LSGEMM_1x2_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_1x2_SUB2_8 \r
+    KERNEL1x2_4  0,0, 0,0\r
+    KERNEL1x2_4  0,0, 1,0\r
+    KERNEL1x2_4  0,0, 2,0\r
+    KERNEL1x2_4  0,0, 3,1\r
+    MY_ALIGN \r
+LSGEMM_1x2_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_1x2_SUB2_4  \r
+    KERNEL1x2_4  0,0, 0,0\r
+    KERNEL1x2_4  0,0, 1,1\r
+       MY_ALIGN        \r
+LSGEMM_1x2_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_1x2_SUB2_2 \r
+    KERNEL1x2_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_1x2_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_1x2_SUB2_1 \r
+    KERNEL1x2_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_1x2_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_1x2_SAVE        \r
+    KERNEL1x2\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x2_SAVE:\r
+       SAVE1x2\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,1\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_1x2_END:\r
+    andi.              I,      M,      1\r
+       ble             LSGEMM_1x1_END\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x1_BEGIN:\r
+\r
+#if defined(TRMMKERNEL)\r
+    REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,1\r
+#else\r
+       mr              BO,     B\r
+#endif \r
+\r
+#if defined(TRMMKERNEL)\r
+   REFRESH_TEMP_BK T11,K,TEMP_REG,1,1 \r
+   srawi.              L, T11, 6 /**(T11 ) % 64x */\r
+#else \r
+   srawi.              L,      K,      6 /**(K ) % 64x */\r
+#endif \r
+ \r
+    ZERO1x1\r
+       ble             LSGEMM_1x1_SUB0\r
+ \r
+  \r
+       mtctr           L\r
+\r
+       MY_ALIGN\r
+\r
+LSGEMM_1x1_LOOP:\r
+\r
+    KERNEL1x1_16  0,0, 0,0\r
+    KERNEL1x1_16  0,0, 1,0\r
+    KERNEL1x1_16  0,0, 2,0\r
+    KERNEL1x1_16  0,0, 3,1     \r
+\r
+       bdnz            LSGEMM_1x1_LOOP\r
+    MY_ALIGN\r
+ \r
+       MY_ALIGN\r
+LSGEMM_1x1_SUB0: \r
+#if defined(TRMMKERNEL)\r
+       andi.           L,      T11,    63\r
+#else\r
+       andi.           L,  K,  63\r
+#endif \r
+       ble             LSGEMM_1x1_SAVE\r
+       MY_ALIGN\r
+LSGEMM_1x1_SUB2:\r
+    andi.      T10,L, 32\r
+    ble LSGEMM_1x1_SUB2_16 \r
+    KERNEL1x1_16  0,0, 0,0\r
+    KERNEL1x1_16  0,0, 1,1 \r
+    MY_ALIGN        \r
+LSGEMM_1x1_SUB2_16:\r
+    andi.      T10,L, 16\r
+    ble LSGEMM_1x1_SUB2_8 \r
+    KERNEL1x1_16  0,0, 0,1\r
+    MY_ALIGN \r
+LSGEMM_1x1_SUB2_8:\r
+    andi.      T10,L, 8\r
+    ble LSGEMM_1x1_SUB2_4  \r
+    KERNEL1x1_8  0,0, 0,1\r
+       MY_ALIGN        \r
+LSGEMM_1x1_SUB2_4:\r
+    andi.      T10,L, 4\r
+    ble LSGEMM_1x1_SUB2_2 \r
+    KERNEL1x1_4  0,0, 0,1\r
+    MY_ALIGN\r
+LSGEMM_1x1_SUB2_2:\r
+    andi.      T10,L, 2\r
+    ble LSGEMM_1x1_SUB2_1 \r
+    KERNEL1x1_2  0,0, 0,1\r
+    MY_ALIGN    \r
+LSGEMM_1x1_SUB2_1:\r
+    andi.      T10,L, 1\r
+    ble LSGEMM_1x1_SAVE        \r
+    KERNEL1x1\r
+\r
+       MY_ALIGN\r
+LSGEMM_1x1_SAVE:\r
+       SAVE1x1\r
+#if defined(TRMMKERNEL)        \r
+       REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,1\r
+#endif  \r
+    MY_ALIGN\r
+LSGEMM_1x1_END:\r
+       slwi            T1,     K,      2\r
+       add             B,      B,      T1\r
+#if defined(TRMMKERNEL) && !defined(LEFT)\r
+    addi TEMP_REG, TEMP_REG, 1\r
+#endif \r
+LSGEMM_END:
\ No newline at end of file

diff --git a/kernel/power/sgemm_macros_power9.S b/kernel/power/sgemm_macros_power9.S
new file mode 100644 (file)
index 0000000..c61f419
--- /dev/null
+++ b/kernel/power/sgemm_macros_power9.S
@@ -0,0 +1,5828 @@
+/***************************************************************************\r
+Copyright (c) 2013-2019, The OpenBLAS Project\r
+All rights reserved.\r
+Redistribution and use in source and binary forms, with or without\r
+modification, are permitted provided that the following conditions are\r
+met:\r
+1. Redistributions of source code must retain the above copyright\r
+notice, this list of conditions and the following disclaimer.\r
+2. Redistributions in binary form must reproduce the above copyright\r
+notice, this list of conditions and the following disclaimer in\r
+the documentation and/or other materials provided with the\r
+distribution.\r
+3. Neither the name of the OpenBLAS project nor the names of\r
+its contributors may be used to endorse or promote products\r
+derived from this software without specific prior written permission.\r
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"\r
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\r
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE\r
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE\r
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
+*****************************************************************************/\r
+ \r
+#define unit_size 4\r
+#define DISP64(ind,disp) (ind*unit_size*64+disp)\r
+#define DISP32(ind,disp) (ind*unit_size*32+disp)\r
+#define DISP16(ind,disp) (ind*unit_size*16+disp)\r
+#define DISP8(ind,disp) (ind*unit_size*8+disp)\r
+#define DISP4(ind,disp) (ind*unit_size*4+disp)\r
+#define DISP2(ind,disp) (ind*unit_size*2+disp)\r
+#define DISP1(ind,disp) (ind*unit_size+disp)\r
+\r
+/**********************************************************************************************\r
+* Macros for N=8 and M=16\r
+**********************************************************************************************/\r
+\r
+.macro LOAD8x16_1\r
+   LOAD8x16 1\r
+.endm\r
+\r
+.macro LOAD8x16_0\r
+   LOAD8x16 0\r
+.endm\r
+\r
+.macro KERNEL8x16_L1_L4  Index,IsLast\r
+  KERNEL8x16_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x16_I1_L4  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x16_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x16_I1_L4_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x16_I1_L4_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+.macro KERNEL8x16_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+   KERNEL8x16_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL8x16_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x16_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x16_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x16_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro Zero8X16\r
+    xxlxor             vs32,   vs32,   vs32\r
+    xxlxor             vs33,   vs33,   vs33\r
+       xxlxor          vs34,   vs34,   vs34\r
+       xxlxor          vs35,   vs35,   vs35\r
+       xxlxor          vs36,   vs36,   vs36\r
+       xxlxor          vs37,   vs37,   vs37\r
+       xxlxor          vs38,   vs38,   vs38\r
+       xxlxor          vs39,   vs39,   vs39\r
+       xxlxor          vs40,   vs40,   vs40\r
+       xxlxor          vs41,   vs41,   vs41\r
+       xxlxor          vs42,   vs42,   vs42\r
+       xxlxor          vs43,   vs43,   vs43\r
+       xxlxor          vs44,   vs44,   vs44\r
+       xxlxor          vs45,   vs45,   vs45\r
+       xxlxor          vs46,   vs46,   vs46\r
+       xxlxor          vs47,   vs47,   vs47\r
+       xxlxor          vs48,   vs48,   vs48\r
+       xxlxor          vs49,   vs49,   vs49\r
+       xxlxor          vs50,   vs50,   vs50\r
+       xxlxor          vs51,   vs51,   vs51 \r
+       xxlxor          vs52,   vs52,   vs52\r
+       xxlxor          vs53,   vs53,   vs53\r
+       xxlxor          vs54,   vs54,   vs54\r
+       xxlxor          vs55,   vs55,   vs55 \r
+       xxlxor          vs56,   vs56,   vs56\r
+       xxlxor          vs57,   vs57,   vs57\r
+       xxlxor          vs58,   vs58,   vs58\r
+       xxlxor          vs59,   vs59,   vs59 \r
+       xxlxor          vs60,   vs60,   vs60\r
+       xxlxor          vs61,   vs61,   vs61\r
+       xxlxor          vs62,   vs62,   vs62\r
+       xxlxor          vs63,   vs63,   vs63    \r
+.endm\r
+\r
+.macro LOAD8x16  Zero\r
+\r
+       lxv     vs24,   0(BO)\r
+       lxv     vs28,   16(BO)\r
+       lxv     vs0,     0(AO)\r
+       lxv     vs1,    16(AO)\r
+       lxv     vs2,    32(AO)\r
+       lxv     vs3,    48(AO)\r
+       xxperm          vs26,   vs24,           permute_mask\r
+       xxperm          vs30,   vs28,           permute_mask    \r
+       xxpermdi        vs25,   vs24,   vs24,2     \r
+       xxpermdi        vs29,   vs28,   vs28,2    \r
+\r
+       xxpermdi        vs27,   vs26,   vs26,2  \r
+       xxpermdi        vs31,   vs30,   vs30,2          \r
+\r
+.if \Zero==1 \r
+    xxlxor             vs32,   vs32,   vs32\r
+    xxlxor             vs33,   vs33,   vs33\r
+       xxlxor          vs34,   vs34,   vs34\r
+       xxlxor          vs35,   vs35,   vs35\r
+       xxlxor          vs36,   vs36,   vs36\r
+       xxlxor          vs37,   vs37,   vs37\r
+       xxlxor          vs38,   vs38,   vs38\r
+       xxlxor          vs39,   vs39,   vs39\r
+       xxlxor          vs40,   vs40,   vs40\r
+       xxlxor          vs41,   vs41,   vs41\r
+       xxlxor          vs42,   vs42,   vs42\r
+       xxlxor          vs43,   vs43,   vs43\r
+       xxlxor          vs44,   vs44,   vs44\r
+       xxlxor          vs45,   vs45,   vs45\r
+       xxlxor          vs46,   vs46,   vs46\r
+       xxlxor          vs47,   vs47,   vs47\r
+       xxlxor          vs48,   vs48,   vs48\r
+       xxlxor          vs49,   vs49,   vs49\r
+       xxlxor          vs50,   vs50,   vs50\r
+       xxlxor          vs51,   vs51,   vs51 \r
+       xxlxor          vs52,   vs52,   vs52\r
+       xxlxor          vs53,   vs53,   vs53\r
+       xxlxor          vs54,   vs54,   vs54\r
+       xxlxor          vs55,   vs55,   vs55 \r
+       xxlxor          vs56,   vs56,   vs56\r
+       xxlxor          vs57,   vs57,   vs57\r
+       xxlxor          vs58,   vs58,   vs58\r
+       xxlxor          vs59,   vs59,   vs59 \r
+       xxlxor          vs60,   vs60,   vs60\r
+       xxlxor          vs61,   vs61,   vs61\r
+       xxlxor          vs62,   vs62,   vs62\r
+       xxlxor          vs63,   vs63,   vs63    \r
+.endif\r
+.endm\r
+\r
+.macro END8x16_NORMAL\r
+  END8x16 0, AO, BO, 64,32 \r
+.endm\r
+\r
+.macro END8x16 First, AREG, BREG, OffsetA, OffsetB\r
+\r
+.if \OffsetB != 0 \r
+    addi        \BREG, \BREG, \OffsetB \r
+.endif\r
+.if \OffsetA != 0 \r
+    addi        \AREG, \AREG, \OffsetA \r
+.endif  \r
+\r
+.if \First==1\r
+    xvmulsp     vs32, vs0,vs24\r
+    xvmulsp     vs33, vs1,vs24\r
+    xvmulsp     vs34, vs2,vs24  \r
+    xvmulsp     vs35, vs3,vs24  \r
+\r
+    xvmulsp     vs36, vs0,vs25\r
+    xvmulsp     vs37, vs1,vs25\r
+    xvmulsp     vs38, vs2,vs25  \r
+    xvmulsp     vs39, vs3,vs25\r
+\r
+    xvmulsp     vs40, vs0,vs26\r
+    xvmulsp     vs41, vs1,vs26\r
+    xvmulsp     vs42, vs2,vs26  \r
+    xvmulsp     vs43, vs3,vs26\r
+\r
+    xvmulsp     vs44, vs0,vs27\r
+    xvmulsp     vs45, vs1,vs27\r
+    xvmulsp     vs46, vs2,vs27  \r
+    xvmulsp     vs47, vs3,vs27\r
+\r
+    xvmulsp     vs48, vs0,vs28\r
+    xvmulsp     vs49, vs1,vs28\r
+    xvmulsp     vs50, vs2,vs28  \r
+    xvmulsp     vs51, vs3,vs28  \r
+\r
+    xvmulsp     vs52, vs0,vs29\r
+    xvmulsp     vs53, vs1,vs29\r
+    xvmulsp     vs54, vs2,vs29  \r
+    xvmulsp     vs55, vs3,vs29\r
+\r
+    xvmulsp     vs56, vs0,vs30\r
+    xvmulsp     vs57, vs1,vs30\r
+    xvmulsp     vs58, vs2,vs30  \r
+    xvmulsp     vs59, vs3,vs30\r
+\r
+    xvmulsp     vs60, vs0,vs31\r
+    xvmulsp     vs61, vs1,vs31\r
+    xvmulsp     vs62, vs2,vs31  \r
+    xvmulsp     vs63, vs3,vs31\r
+\r
+.else\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+    xvmaddasp       vs34, vs2,vs24  \r
+    xvmaddasp       vs35, vs3,vs24  \r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+    xvmaddasp       vs38, vs2,vs25  \r
+    xvmaddasp       vs39, vs3,vs25 \r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+    xvmaddasp       vs42, vs2,vs26  \r
+    xvmaddasp       vs43, vs3,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+    xvmaddasp       vs46, vs2,vs27  \r
+    xvmaddasp       vs47, vs3,vs27\r
+\r
+    xvmaddasp       vs48, vs0,vs28\r
+    xvmaddasp       vs49, vs1,vs28\r
+    xvmaddasp       vs50, vs2,vs28  \r
+    xvmaddasp       vs51, vs3,vs28  \r
+\r
+    xvmaddasp       vs52, vs0,vs29\r
+    xvmaddasp       vs53, vs1,vs29\r
+    xvmaddasp       vs54, vs2,vs29  \r
+    xvmaddasp       vs55, vs3,vs29\r
+\r
+    xvmaddasp       vs56, vs0,vs30\r
+    xvmaddasp       vs57, vs1,vs30\r
+    xvmaddasp       vs58, vs2,vs30  \r
+    xvmaddasp       vs59, vs3,vs30\r
+\r
+    xvmaddasp       vs60, vs0,vs31\r
+    xvmaddasp       vs61, vs1,vs31\r
+    xvmaddasp       vs62, vs2,vs31  \r
+    xvmaddasp       vs63, vs3,vs31 \r
+\r
+.endif\r
+.endm  \r
+\r
+.macro KERNEL8x16_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+       lxv     vs8,    DISP32(\Index, 0+\OffsetB)(\BREG)\r
+       lxv     vs12,   DISP32(\Index,16+\OffsetB)(\BREG)\r
+\r
+       lxv     vs4,    DISP64(\Index, 0+\OffsetA)(\AREG)\r
+       lxv     vs5,    DISP64(\Index,16+\OffsetA)(\AREG)\r
+       lxv     vs6,    DISP64(\Index,32+\OffsetA)(\AREG)\r
+       lxv     vs7,    DISP64(\Index,48+\OffsetA)(\AREG) \r
+\r
+       xxperm          vs10,   vs8,            permute_mask\r
+       xxperm          vs14,   vs12,           permute_mask    \r
+       xxpermdi        vs9,    vs8,    vs8,2    \r
+       xxpermdi        vs13,   vs12,   vs12,2   \r
+\r
+    xvmaddasp          vs32, vs0,vs24\r
+       xvmaddasp               vs33, vs1,vs24\r
+       xvmaddasp               vs34, vs2,vs24  \r
+       xvmaddasp               vs35, vs3,vs24   \r
+\r
+    xvmaddasp          vs36, vs0,vs25\r
+       xvmaddasp               vs37, vs1,vs25\r
+       xvmaddasp               vs38, vs2,vs25  \r
+       xvmaddasp               vs39, vs3,vs25 \r
+\r
+       xxpermdi        vs11,   vs10,   vs10,2  \r
+       xxpermdi        vs15,   vs14,   vs14,2  \r
+\r
+    xvmaddasp          vs40, vs0,vs26\r
+       xvmaddasp               vs41, vs1,vs26\r
+       xvmaddasp               vs42, vs2,vs26  \r
+       xvmaddasp               vs43, vs3,vs26\r
+\r
+    xvmaddasp          vs44, vs0,vs27\r
+       xvmaddasp               vs45, vs1,vs27\r
+       xvmaddasp               vs46, vs2,vs27  \r
+       xvmaddasp               vs47, vs3,vs27\r
+\r
+    xvmaddasp          vs48, vs0,vs28\r
+       xvmaddasp               vs49, vs1,vs28\r
+       xvmaddasp               vs50, vs2,vs28  \r
+       xvmaddasp               vs51, vs3,vs28  \r
+\r
+    xvmaddasp          vs52, vs0,vs29\r
+       xvmaddasp               vs53, vs1,vs29\r
+       xvmaddasp               vs54, vs2,vs29  \r
+       xvmaddasp               vs55, vs3,vs29\r
+\r
+    xvmaddasp          vs56, vs0,vs30\r
+       xvmaddasp               vs57, vs1,vs30\r
+       xvmaddasp               vs58, vs2,vs30  \r
+       xvmaddasp               vs59, vs3,vs30\r
+\r
+    xvmaddasp          vs60, vs0,vs31\r
+       xvmaddasp               vs61, vs1,vs31\r
+       xvmaddasp               vs62, vs2,vs31  \r
+       xvmaddasp               vs63, vs3,vs31 \r
+\r
+       lxv     vs24,   DISP32(\Index,32+\OffsetB)(\BREG)\r
+       lxv     vs28,   DISP32(\Index,32+16+\OffsetB)(\BREG)\r
+\r
+       lxv     vs0,    DISP64(\Index,64+\OffsetA)(\AREG)\r
+       lxv     vs1,    DISP64(\Index,64+16+\OffsetA)(\AREG)\r
+       lxv     vs2,    DISP64(\Index,64+32+\OffsetA)(\AREG)\r
+       lxv     vs3,    DISP64(\Index,64+48+\OffsetA)(\AREG)\r
+\r
+       xxperm          vs26,   vs24,           permute_mask\r
+       xxperm          vs30,   vs28,           permute_mask    \r
+       xxpermdi        vs25,   vs24,   vs24,2     \r
+       xxpermdi        vs29,   vs28,   vs28,2    \r
+ \r
+\r
+    xvmaddasp          vs32, vs4,vs8\r
+       xvmaddasp               vs33, vs5,vs8\r
+       xvmaddasp               vs34, vs6,vs8   \r
+       xvmaddasp               vs35, vs7,vs8   \r
+ \r
+    xvmaddasp          vs36, vs4,vs9\r
+       xvmaddasp               vs37, vs5,vs9\r
+       xvmaddasp               vs38, vs6,vs9   \r
+       xvmaddasp               vs39, vs7,vs9\r
+         \r
+       xxpermdi        vs27,   vs26,   vs26,2  \r
+       xxpermdi        vs31,   vs30,   vs30,2          \r
+\r
+    xvmaddasp          vs40, vs4,vs10\r
+       xvmaddasp               vs41, vs5,vs10\r
+       xvmaddasp               vs42, vs6,vs10  \r
+       xvmaddasp               vs43, vs7,vs10\r
+\r
+    xvmaddasp          vs44, vs4,vs11\r
+       xvmaddasp               vs45, vs5,vs11\r
+       xvmaddasp               vs46, vs6,vs11  \r
+       xvmaddasp               vs47, vs7,vs11\r
+\r
+    xvmaddasp          vs48, vs4,vs12\r
+       xvmaddasp               vs49, vs5,vs12\r
+       xvmaddasp               vs50, vs6,vs12  \r
+       xvmaddasp               vs51, vs7,vs12  \r
+\r
+    xvmaddasp          vs52, vs4,vs13\r
+       xvmaddasp               vs53, vs5,vs13\r
+       xvmaddasp               vs54, vs6,vs13  \r
+       xvmaddasp               vs55, vs7,vs13\r
+\r
+    xvmaddasp          vs56, vs4,vs14\r
+       xvmaddasp               vs57, vs5,vs14\r
+       xvmaddasp               vs58, vs6,vs14  \r
+       xvmaddasp               vs59, vs7,vs14\r
+\r
+    xvmaddasp          vs60, vs4,vs15\r
+       xvmaddasp               vs61, vs5,vs15\r
+       xvmaddasp               vs62, vs6,vs15  \r
+       xvmaddasp               vs63, vs7,vs15\r
+\r
+       lxv     vs8,    DISP32(\Index,64+\OffsetB)(\BREG)\r
+       lxv     vs12,   DISP32(\Index,64+16+\OffsetB)(\BREG)\r
+\r
+       lxv     vs4,    DISP64(\Index,128+0+\OffsetA)(\AREG)\r
+       lxv     vs5,    DISP64(\Index,128+16+\OffsetA)(\AREG)\r
+       lxv     vs6,    DISP64(\Index,128+32+\OffsetA)(\AREG)\r
+       lxv     vs7,    DISP64(\Index,128+48+\OffsetA)(\AREG) \r
+\r
+       xxperm          vs10,   vs8,            permute_mask\r
+       xxperm          vs14,   vs12,           permute_mask    \r
+       xxpermdi        vs9,    vs8,    vs8,2    \r
+       xxpermdi        vs13,   vs12,   vs12,2  \r
+\r
+    xvmaddasp          vs32, vs0,vs24\r
+       xvmaddasp               vs33, vs1,vs24\r
+       xvmaddasp               vs34, vs2,vs24  \r
+       xvmaddasp               vs35, vs3,vs24   \r
+\r
+    xvmaddasp          vs36, vs0,vs25\r
+       xvmaddasp               vs37, vs1,vs25\r
+       xvmaddasp               vs38, vs2,vs25  \r
+       xvmaddasp               vs39, vs3,vs25\r
+\r
+       xxpermdi        vs11,   vs10,   vs10,2  \r
+       xxpermdi        vs15,   vs14,   vs14,2  \r
+\r
+    xvmaddasp          vs40, vs0,vs26\r
+       xvmaddasp               vs41, vs1,vs26\r
+       xvmaddasp               vs42, vs2,vs26  \r
+       xvmaddasp               vs43, vs3,vs26\r
+\r
+    xvmaddasp          vs44, vs0,vs27\r
+       xvmaddasp               vs45, vs1,vs27\r
+       xvmaddasp               vs46, vs2,vs27  \r
+       xvmaddasp               vs47, vs3,vs27\r
+\r
+    xvmaddasp          vs48, vs0,vs28\r
+       xvmaddasp               vs49, vs1,vs28\r
+       xvmaddasp               vs50, vs2,vs28  \r
+       xvmaddasp               vs51, vs3,vs28  \r
+\r
+    xvmaddasp          vs52, vs0,vs29\r
+       xvmaddasp               vs53, vs1,vs29\r
+       xvmaddasp               vs54, vs2,vs29  \r
+       xvmaddasp               vs55, vs3,vs29\r
+\r
+    xvmaddasp          vs56, vs0,vs30\r
+       xvmaddasp               vs57, vs1,vs30\r
+       xvmaddasp               vs58, vs2,vs30  \r
+       xvmaddasp               vs59, vs3,vs30\r
+\r
+    xvmaddasp          vs60, vs0,vs31\r
+       xvmaddasp               vs61, vs1,vs31\r
+       xvmaddasp               vs62, vs2,vs31  \r
+       xvmaddasp               vs63, vs3,vs31 \r
+ \r
+.if \Complete==0\r
+       lxv     vs24,   DISP32(\Index,96+\OffsetB)(\BREG)\r
+       lxv     vs28,   DISP32(\Index,96+16+\OffsetB)(\BREG)\r
+\r
+       lxv     vs0,    DISP64(\Index,192+\OffsetA)(\AREG)\r
+       lxv     vs1,    DISP64(\Index,192+16+\OffsetA)(\AREG) \r
+       lxv     vs2,    DISP64(\Index,192+32+\OffsetA)(\AREG)\r
+       lxv     vs3,    DISP64(\Index,192+48+\OffsetA)(\AREG)\r
+\r
+       xxperm          vs26,   vs24,   permute_mask\r
+       xxperm          vs30,   vs28,   permute_mask    \r
+       xxpermdi        vs25,   vs24,   vs24,2 \r
+       xxpermdi        vs29,   vs28,   vs28,2          \r
+\r
+.endif \r
+.if \IsLast==1 \r
+.if \Complete==1\r
+  \r
+       addi            \BREG, \BREG,  DISP32(\Index,32*3+\OffsetB)\r
+       addi            \AREG, \AREG, DISP64(\Index,64*3+\OffsetA)\r
+.else\r
+  \r
+       addi            \BREG, \BREG,  DISP32(\Index,128)\r
+       addi            \AREG, \AREG, DISP64(\Index,256)\r
+.endif\r
+.endif   \r
+ \r
+    xvmaddasp          vs32, vs4,vs8\r
+       xvmaddasp               vs33, vs5,vs8\r
+       xvmaddasp               vs34, vs6,vs8   \r
+       xvmaddasp               vs35, vs7,vs8    \r
+ \r
+    xvmaddasp          vs36, vs4,vs9\r
+       xvmaddasp               vs37, vs5,vs9\r
+       xvmaddasp               vs38, vs6,vs9   \r
+       xvmaddasp               vs39, vs7,vs9\r
+  \r
+.if \Complete==0        \r
+       xxpermdi        vs27,   vs26,   vs26,2  \r
+       xxpermdi        vs31,   vs30,   vs30,2  \r
+       \r
+.endif\r
+ \r
+    xvmaddasp          vs40, vs4,vs10\r
+       xvmaddasp               vs41, vs5,vs10\r
+       xvmaddasp               vs42, vs6,vs10  \r
+       xvmaddasp               vs43, vs7,vs10\r
+\r
+    xvmaddasp          vs44, vs4,vs11\r
+       xvmaddasp               vs45, vs5,vs11\r
+       xvmaddasp               vs46, vs6,vs11  \r
+       xvmaddasp               vs47, vs7,vs11\r
+\r
+    xvmaddasp          vs48, vs4,vs12\r
+       xvmaddasp               vs49, vs5,vs12\r
+       xvmaddasp               vs50, vs6,vs12  \r
+       xvmaddasp               vs51, vs7,vs12  \r
+\r
+    xvmaddasp          vs52, vs4,vs13\r
+       xvmaddasp               vs53, vs5,vs13\r
+       xvmaddasp               vs54, vs6,vs13  \r
+       xvmaddasp               vs55, vs7,vs13\r
+\r
+    xvmaddasp          vs56, vs4,vs14\r
+       xvmaddasp               vs57, vs5,vs14\r
+       xvmaddasp               vs58, vs6,vs14  \r
+       xvmaddasp               vs59, vs7,vs14\r
+\r
+    xvmaddasp          vs60, vs4,vs15\r
+       xvmaddasp               vs61, vs5,vs15\r
+       xvmaddasp               vs62, vs6,vs15  \r
+       xvmaddasp               vs63, vs7,vs15\r
+\r
+.endm\r
+\r
+.macro KERNEL8x16 First\r
+\r
+  LOAD8x16 0\r
+  END8x16 \First, AO, BO, 64,32 \r
+.endm\r
+\r
+.macro KERNEL8x16_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete\r
+       \r
+       lxv     vs8,    DISP16(\Index, 0+\OffsetB)(\BREG)\r
+       lxv     vs12,   DISP16(\Index,16+\OffsetB)(\BREG)\r
+\r
+       lxv     vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)\r
+       lxv     vs5,    DISP32(\Index,16+\OffsetA)(\AREG)\r
+       lxv     vs6,    DISP32(\Index,32+\OffsetA)(\AREG)\r
+       lxv     vs7,    DISP32(\Index,48+\OffsetA)(\AREG) \r
+\r
+       xxperm          vs10,   vs8,            permute_mask\r
+       xxperm          vs14,   vs12,           permute_mask    \r
+       xxpermdi        vs9,    vs8,    vs8,2    \r
+       xxpermdi        vs13,   vs12,   vs12,2   \r
+.if \First==1\r
+    xvmulsp            vs32, vs0,vs24\r
+       xvmulsp         vs33, vs1,vs24\r
+       xvmulsp         vs34, vs2,vs24  \r
+       xvmulsp         vs35, vs3,vs24  \r
+\r
+    xvmulsp            vs36, vs0,vs25\r
+       xvmulsp         vs37, vs1,vs25\r
+       xvmulsp         vs38, vs2,vs25  \r
+       xvmulsp         vs39, vs3,vs25  \r
+.else\r
+    xvmaddasp          vs32, vs0,vs24\r
+       xvmaddasp               vs33, vs1,vs24\r
+       xvmaddasp               vs34, vs2,vs24  \r
+       xvmaddasp               vs35, vs3,vs24\r
+\r
+    xvmaddasp          vs36, vs0,vs25\r
+       xvmaddasp               vs37, vs1,vs25\r
+       xvmaddasp               vs38, vs2,vs25  \r
+       xvmaddasp               vs39, vs3,vs25          \r
+.endif\r
+\r
+       xxpermdi        vs11,   vs10,   vs10,2  \r
+       xxpermdi        vs15,   vs14,   vs14,2  \r
+ \r
+.if \First==1  \r
+    xvmulsp            vs40, vs0,vs26\r
+       xvmulsp         vs41, vs1,vs26\r
+       xvmulsp         vs42, vs2,vs26  \r
+       xvmulsp         vs43, vs3,vs26\r
+\r
+    xvmulsp            vs44, vs0,vs27\r
+       xvmulsp         vs45, vs1,vs27\r
+       xvmulsp         vs46, vs2,vs27  \r
+       xvmulsp         vs47, vs3,vs27\r
+\r
+    xvmulsp            vs48, vs0,vs28\r
+       xvmulsp         vs49, vs1,vs28\r
+       xvmulsp         vs50, vs2,vs28  \r
+       xvmulsp         vs51, vs3,vs28  \r
+\r
+    xvmulsp            vs52, vs0,vs29\r
+       xvmulsp         vs53, vs1,vs29\r
+       xvmulsp         vs54, vs2,vs29  \r
+       xvmulsp         vs55, vs3,vs29\r
+\r
+    xvmulsp            vs56, vs0,vs30\r
+       xvmulsp         vs57, vs1,vs30\r
+       xvmulsp         vs58, vs2,vs30  \r
+       xvmulsp         vs59, vs3,vs30\r
+\r
+    xvmulsp            vs60, vs0,vs31\r
+       xvmulsp         vs61, vs1,vs31\r
+       xvmulsp         vs62, vs2,vs31  \r
+       xvmulsp         vs63, vs3,vs31\r
+\r
+.else \r
+    xvmaddasp          vs40, vs0,vs26\r
+       xvmaddasp               vs41, vs1,vs26\r
+       xvmaddasp               vs42, vs2,vs26  \r
+       xvmaddasp               vs43, vs3,vs26\r
+\r
+    xvmaddasp          vs44, vs0,vs27\r
+       xvmaddasp               vs45, vs1,vs27\r
+       xvmaddasp               vs46, vs2,vs27  \r
+       xvmaddasp               vs47, vs3,vs27\r
+\r
+    xvmaddasp          vs48, vs0,vs28\r
+       xvmaddasp               vs49, vs1,vs28\r
+       xvmaddasp               vs50, vs2,vs28  \r
+       xvmaddasp               vs51, vs3,vs28  \r
+\r
+    xvmaddasp          vs52, vs0,vs29\r
+       xvmaddasp               vs53, vs1,vs29\r
+       xvmaddasp               vs54, vs2,vs29  \r
+       xvmaddasp               vs55, vs3,vs29\r
+\r
+    xvmaddasp          vs56, vs0,vs30\r
+       xvmaddasp               vs57, vs1,vs30\r
+       xvmaddasp               vs58, vs2,vs30  \r
+       xvmaddasp               vs59, vs3,vs30\r
+\r
+    xvmaddasp          vs60, vs0,vs31\r
+       xvmaddasp               vs61, vs1,vs31\r
+       xvmaddasp               vs62, vs2,vs31  \r
+       xvmaddasp               vs63, vs3,vs31 \r
+\r
+.endif\r
+.if \Complete==0\r
+       lxv     vs24,   DISP16(\Index,32+\OffsetB)(\BREG)\r
+       lxv     vs28,   DISP16(\Index,32+16+\OffsetB)(\BREG)\r
+\r
+       lxv     vs0,    DISP32(\Index,64+\OffsetA)(\AREG)\r
+       lxv     vs1,    DISP32(\Index,64+16+\OffsetA)(\AREG)\r
+       lxv     vs2,    DISP32(\Index,64+32+\OffsetA)(\AREG)\r
+       lxv     vs3,    DISP32(\Index,64+48+\OffsetA)(\AREG)\r
+\r
+       xxperm          vs26,   vs24,   permute_mask\r
+       xxperm          vs30,   vs28,   permute_mask    \r
+       xxpermdi        vs25,   vs24,   vs24,2   \r
+       xxpermdi        vs29,   vs28,   vs28,2  \r
+.endif    \r
+.if \IsLast==1 \r
+.if \Complete==1\r
+       addi            \BREG, \BREG,  DISP16(\Index,32+\OffsetB) \r
+       addi            \AREG, \AREG, DISP32(\Index,64+\OffsetA)\r
+\r
+.else\r
+       addi            \BREG, \BREG,  DISP16(\Index,64)\r
+       addi            \AREG, \AREG, DISP32(\Index,128) \r
+.endif\r
+.endif\r
+\r
+.if \First==1\r
+    xvmulsp            vs32, vs4,vs8\r
+       xvmulsp         vs33, vs5,vs8\r
+       xvmulsp         vs34, vs6,vs8   \r
+       xvmulsp         vs35, vs7,vs8\r
+\r
+    xvmulsp            vs36, vs4,vs9\r
+       xvmulsp         vs37, vs5,vs9\r
+       xvmulsp         vs38, vs6,vs9   \r
+       xvmulsp         vs39, vs7,vs9\r
+.else\r
+    xvmaddasp          vs32, vs4,vs8\r
+       xvmaddasp               vs33, vs5,vs8\r
+       xvmaddasp               vs34, vs6,vs8   \r
+       xvmaddasp               vs35, vs7,vs8   \r
+\r
+    xvmaddasp          vs36, vs4,vs9\r
+       xvmaddasp               vs37, vs5,vs9\r
+       xvmaddasp               vs38, vs6,vs9   \r
+       xvmaddasp               vs39, vs7,vs9\r
+.endif \r
+ \r
+.if \Complete==0        \r
+       xxpermdi        vs27,   vs26,   vs26,2  \r
+       xxpermdi        vs31,   vs30,   vs30,2  \r
+ \r
+.endif\r
+.if \First==1  \r
+    xvmulsp            vs40, vs4,vs10\r
+       xvmulsp         vs41, vs5,vs10\r
+       xvmulsp         vs42, vs6,vs10  \r
+       xvmulsp         vs43, vs7,vs10\r
+\r
+    xvmulsp            vs44, vs4,vs11\r
+       xvmulsp         vs45, vs5,vs11\r
+       xvmulsp         vs46, vs6,vs11  \r
+       xvmulsp         vs47, vs7,vs11\r
+\r
+    xvmulsp            vs48, vs4,vs12\r
+       xvmulsp         vs49, vs5,vs12\r
+       xvmulsp         vs50, vs6,vs12  \r
+       xvmulsp         vs51, vs7,vs12  \r
+\r
+    xvmulsp            vs52, vs4,vs13\r
+       xvmulsp         vs53, vs5,vs13\r
+       xvmulsp         vs54, vs6,vs13  \r
+       xvmulsp         vs55, vs7,vs13\r
+\r
+    xvmulsp            vs56, vs4,vs14\r
+       xvmulsp         vs57, vs5,vs14\r
+       xvmulsp         vs58, vs6,vs14  \r
+       xvmulsp         vs59, vs7,vs14\r
+\r
+    xvmulsp            vs60, vs4,vs15\r
+       xvmulsp         vs61, vs5,vs15\r
+       xvmulsp         vs62, vs6,vs15  \r
+       xvmulsp         vs63, vs7,vs15\r
+\r
+.else \r
+    xvmaddasp          vs40, vs4,vs10\r
+       xvmaddasp               vs41, vs5,vs10\r
+       xvmaddasp               vs42, vs6,vs10  \r
+       xvmaddasp               vs43, vs7,vs10\r
+\r
+    xvmaddasp          vs44, vs4,vs11\r
+       xvmaddasp               vs45, vs5,vs11\r
+       xvmaddasp               vs46, vs6,vs11  \r
+       xvmaddasp               vs47, vs7,vs11\r
+\r
+    xvmaddasp          vs48, vs4,vs12\r
+       xvmaddasp               vs49, vs5,vs12\r
+       xvmaddasp               vs50, vs6,vs12  \r
+       xvmaddasp               vs51, vs7,vs12  \r
+\r
+    xvmaddasp          vs52, vs4,vs13\r
+       xvmaddasp               vs53, vs5,vs13\r
+       xvmaddasp               vs54, vs6,vs13  \r
+       xvmaddasp               vs55, vs7,vs13\r
+\r
+    xvmaddasp          vs56, vs4,vs14\r
+       xvmaddasp               vs57, vs5,vs14\r
+       xvmaddasp               vs58, vs6,vs14  \r
+       xvmaddasp               vs59, vs7,vs14\r
+\r
+    xvmaddasp          vs60, vs4,vs15\r
+       xvmaddasp               vs61, vs5,vs15\r
+       xvmaddasp               vs62, vs6,vs15  \r
+       xvmaddasp               vs63, vs7,vs15\r
+\r
+.endif\r
+\r
+.endm\r
+\r
+ \r
+.macro SAVE8x16\r
+\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC \r
+\r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10  \r
+\r
+  add     T4, T2, T10  \r
+  add     T5, T3, T10 \r
+\r
+  add     T6, T4, T10 \r
+  add     T7, T5, T10 \r
+\r
+\r
+\r
+   /* permute to restore butterfly rank 1 updateto normal promoted one */  \r
+    /* permute 16 vs8 MEM(CO) vs9 MEM(CO+LDC) vs10 MEM(CO+2*LDC)  vs11 MEM(CO+3*LDC) */\r
+    /* permute 16 vs12 MEM(16+CO) vs13 MEM(16+CO+LDC) vs14 MEM(16+CO+2*LDC)  vs15 MEM(16+CO+3*LDC) */\r
+    /* permute 16 vs16 MEM(32+CO) vs17 MEM(32+CO+LDC) vs18 MEM(32+CO+2*LDC)  vs19 MEM(32+CO+3*LDC) */\r
+    /* permute 16 vs24 MEM(32+CO) vs25 MEM(32+CO+LDC) vs26 MEM(32+CO+2*LDC)  vs27 MEM(32+CO+3*LDC) */\r
+\r
+    xxmrglw     vs8,    vs32,   vs44\r
+    xxmrglw     vs10,   vs36,   vs40  \r
+\r
+    xxmrghw     vs1,    vs32,   vs44\r
+    xxmrghw     vs0,    vs36,   vs40\r
+\r
+    xxmrglw     vs12,   vs33,   vs45\r
+    xxmrglw     vs14,   vs37,   vs41  \r
+\r
+    xxmrghw     vs2,    vs37,   vs41\r
+    xxmrghw     vs3,    vs33,   vs45\r
+\r
+    xxmrglw     vs16,   vs34,   vs46\r
+    xxmrglw     vs18,   vs38,   vs42   \r
+\r
+    xxlor      vs9, vs8,    vs8\r
+    xxlor      vs11,    vs10,   vs10 \r
+\r
+    xxmrghw     vs4,    vs38,   vs42\r
+    xxmrghw     vs5,    vs34,   vs46\r
+\r
+    xxlor      vs13,    vs12,   vs12\r
+    xxlor      vs15,    vs14,   vs14\r
+\r
+    xxmrglw     vs24,   vs35,   vs47\r
+    xxmrglw     vs26,   vs39,   vs43  \r
+\r
+    xxlor      vs17,    vs16,   vs16\r
+    xxlor      vs19,    vs18,   vs18\r
+\r
+    xxmrghw     vs30,   vs39,   vs43 \r
+    xxmrghw     vs31,   vs35,   vs47\r
+\r
+    xxperm      vs8,    vs0,    save_permute_1\r
+    xxperm      vs10,   vs1,    save_permute_1\r
+    xxperm      vs9,    vs0,    save_permute_2  \r
+    xxperm      vs11,   vs1,    save_permute_2      \r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs32, 0(CO)\r
+    lxv        vs33, 16(CO) \r
+    lxv        vs34, 32(CO)  \r
+    lxv        vs35, 48(CO)      \r
+#endif\r
+    xxlor      vs25,    vs24,   vs24\r
+    xxlor      vs27,    vs26,   vs26 \r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs36, 0(T1)\r
+    lxv        vs37, 16(T1) \r
+    lxv        vs38, 32(T1)  \r
+    lxv        vs39, 48(T1)     \r
+#endif\r
+#ifndef TRMMKERNEL       \r
+    lxv        vs40, 0(T2)\r
+    lxv        vs41, 16(T2) \r
+    lxv        vs42, 32(T2)  \r
+    lxv        vs43, 48(T2)     \r
+#endif  \r
+#ifndef TRMMKERNEL    \r
+    lxv        vs44, 0(T3)\r
+    lxv        vs45, 16(T3) \r
+    lxv        vs46, 32(T3)  \r
+    lxv        vs47, 48(T3)                 \r
+#endif  \r
+\r
+    xxperm     vs12,    vs2,    save_permute_1\r
+    xxperm     vs14,    vs3,    save_permute_1\r
+       \r
+    xxperm     vs13,    vs2,    save_permute_2   \r
+    xxperm     vs15,    vs3,    save_permute_2      \r
+\r
+    xxperm     vs16,    vs4,    save_permute_1\r
+    xxperm     vs18,    vs5,    save_permute_1\r
+      \r
+    xxperm     vs17,    vs4,    save_permute_2   \r
+    xxperm     vs19,    vs5,    save_permute_2      \r
+\r
+    xxperm     vs24,    vs30,   save_permute_1\r
+    xxperm     vs26,    vs31,   save_permute_1 \r
+         \r
+    xxperm     vs25,    vs30,   save_permute_2   \r
+    xxperm     vs27,    vs31,   save_permute_2  \r
+\r
+\r
+    /* multiply add normal way */\r
+ \r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs32,   vs8,    alpha_r \r
+    xvmulsp     vs33,   vs12,   alpha_r   \r
+    xvmulsp     vs34,   vs16,   alpha_r \r
+    xvmulsp     vs35,   vs24,   alpha_r  \r
+    xvmulsp     vs36,   vs9,    alpha_r \r
+    xvmulsp     vs37,   vs13,   alpha_r  \r
+    xvmulsp     vs38,   vs17,   alpha_r \r
+    xvmulsp     vs39,   vs25,   alpha_r               \r
+#else \r
+    xvmaddasp   vs32,   vs8,    alpha_r \r
+    xvmaddasp   vs33,   vs12,   alpha_r   \r
+    xvmaddasp   vs34,   vs16,   alpha_r \r
+    xvmaddasp   vs35,   vs24,   alpha_r  \r
+    xvmaddasp   vs36,   vs9,    alpha_r \r
+    xvmaddasp   vs37,   vs13,   alpha_r   \r
+    xvmaddasp   vs38,   vs17,   alpha_r \r
+    xvmaddasp   vs39,   vs25,   alpha_r         \r
+#endif \r
+\r
+\r
+\r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs40,   vs10,   alpha_r \r
+    xvmulsp     vs41,   vs14,   alpha_r \r
+    xvmulsp     vs42,   vs18,   alpha_r \r
+    xvmulsp     vs43,   vs26,   alpha_r  \r
+    xvmulsp     vs44,   vs11,   alpha_r \r
+    xvmulsp     vs45,   vs15,   alpha_r  \r
+    xvmulsp     vs46,   vs19,   alpha_r \r
+    xvmulsp     vs47,   vs27,   alpha_r                   \r
+#else\r
+\r
+    xvmaddasp   vs40,   vs10,   alpha_r \r
+    xvmaddasp   vs41,   vs14,   alpha_r   \r
+    xvmaddasp   vs42,   vs18,   alpha_r \r
+    xvmaddasp   vs43,   vs26,   alpha_r  \r
+    xvmaddasp   vs44,   vs11,   alpha_r \r
+    xvmaddasp   vs45,   vs15,   alpha_r \r
+    xvmaddasp   vs46,   vs19,   alpha_r \r
+    xvmaddasp   vs47,   vs27,   alpha_r  \r
+        \r
+#endif  \r
+\r
+    stxv        vs32, 0(CO)\r
+    stxv        vs33, 16(CO) \r
+    stxv        vs34, 32(CO)  \r
+    stxv        vs35, 48(CO)  \r
+\r
+    stxv        vs36, 0(T1)\r
+    stxv        vs37, 16(T1)  \r
+    stxv        vs38, 32(T1)  \r
+    stxv        vs39, 48(T1)\r
+\r
+    stxv        vs40, 0(T2)\r
+    stxv        vs41, 16(T2)  \r
+    stxv        vs42, 32(T2)  \r
+    stxv        vs43, 48(T2)  \r
+    stxv        vs44, 0(T3)\r
+    stxv        vs45, 16(T3) \r
+    stxv        vs46, 32(T3)  \r
+    stxv        vs47, 48(T3)\r
+  \r
+ /*****the same with the second 8X8 ****/\r
+#ifndef TRMMKERNEL\r
+  \r
+    lxv        vs32, 0(T4)\r
+    lxv        vs33, 16(T4) \r
+    lxv        vs34, 32(T4)  \r
+    lxv        vs35, 48(T4)      \r
+    lxv        vs36, 0(T5)\r
+    lxv        vs37, 16(T5) \r
+    lxv        vs38,32(T5)  \r
+    lxv        vs39, 48(T5)     \r
+#endif  \r
+ \r
+    xxmrglw     vs8,    vs48,   vs60\r
+    xxmrglw     vs10,   vs52,   vs56  \r
+\r
+    xxmrghw     vs1,    vs48,   vs60\r
+    xxmrghw     vs0,    vs52,   vs56\r
+    xxmrglw     vs12,   vs49,   vs61\r
+    xxmrglw     vs14,   vs53,   vs57  \r
+\r
+#ifndef TRMMKERNEL   \r
+    lxv        vs40, 0(T6)\r
+    lxv        vs41, 16(T6)  \r
+    lxv        vs42, 32(T6)  \r
+    lxv        vs43, 48(T6)           \r
+    lxv        vs44, 0(T7)\r
+    lxv        vs45, 16(T7) \r
+    lxv        vs46, 32(T7)  \r
+    lxv        vs47, 48(T7)     \r
+#endif  \r
+    xxmrghw     vs2,    vs53,   vs57\r
+    xxmrghw     vs3,    vs49,   vs61\r
+\r
+    xxmrglw     vs16,   vs50,   vs62\r
+    xxmrglw     vs18,   vs54,   vs58   \r
+\r
+    xxlor      vs9, vs8,    vs8\r
+    xxlor      vs11,    vs10,   vs10 \r
+    xxmrghw     vs4,    vs54,   vs58\r
+    xxmrghw     vs5,    vs50,   vs62\r
+\r
+    xxlor      vs13,    vs12,   vs12\r
+    xxlor      vs15,    vs14,   vs14\r
+ \r
+    xxmrglw     vs24,   vs51,   vs63\r
+    xxmrglw     vs26,   vs55,   vs59  \r
+\r
+    xxlor      vs17,    vs16,   vs16\r
+    xxlor      vs19,    vs18,   vs18\r
+    xxmrghw     vs30,   vs55,   vs59 \r
+    xxmrghw     vs31,   vs51,   vs63\r
+\r
+    xxperm      vs8,    vs0,    save_permute_1\r
+    xxperm      vs10,   vs1,    save_permute_1\r
+     \r
+    xxperm      vs9,    vs0,    save_permute_2  \r
+    xxperm      vs11,   vs1,    save_permute_2      \r
+\r
+    xxlor      vs25,    vs24,   vs24\r
+    xxlor      vs27,    vs26,   vs26 \r
+    xxperm     vs12,    vs2,    save_permute_1\r
+    xxperm     vs14,    vs3,    save_permute_1\r
+    xxperm     vs13,    vs2,    save_permute_2   \r
+    xxperm     vs15,    vs3,    save_permute_2      \r
+ \r
+    xxperm     vs16,    vs4,    save_permute_1\r
+    xxperm     vs18,    vs5,    save_permute_1\r
+    xxperm     vs17,    vs4,    save_permute_2   \r
+    xxperm     vs19,    vs5,    save_permute_2      \r
+    xxperm     vs24,    vs30,   save_permute_1\r
+    xxperm     vs26,    vs31,   save_permute_1 \r
+    xxperm     vs25,    vs30,   save_permute_2   \r
+    xxperm     vs27,    vs31,   save_permute_2      \r
+\r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs32,   vs8,    alpha_r \r
+    xvmulsp     vs33,   vs12,   alpha_r   \r
+    xvmulsp     vs34,   vs16,   alpha_r \r
+    xvmulsp     vs35,   vs24,   alpha_r  \r
+    xvmulsp     vs36,   vs9,    alpha_r \r
+    xvmulsp     vs37,   vs13,   alpha_r  \r
+    xvmulsp     vs38,   vs17,   alpha_r \r
+    xvmulsp     vs39,   vs25,   alpha_r               \r
+#else \r
+    xvmaddasp   vs32,   vs8,    alpha_r \r
+    xvmaddasp   vs33,   vs12,   alpha_r   \r
+    xvmaddasp   vs34,   vs16,   alpha_r \r
+    xvmaddasp   vs35,   vs24,   alpha_r  \r
+    xvmaddasp   vs36,   vs9,    alpha_r \r
+    xvmaddasp   vs37,   vs13,   alpha_r   \r
+    xvmaddasp   vs38,   vs17,   alpha_r \r
+    xvmaddasp   vs39,   vs25,   alpha_r         \r
+#endif \r
+\r
+    stxv        vs32, 0(T4)\r
+    stxv        vs33, 16(T4) \r
+    stxv        vs34, 32(T4)  \r
+    stxv        vs35, 48(T4)  \r
+\r
+    stxv        vs36, 0(T5)\r
+    stxv        vs37, 16(T5)  \r
+    stxv        vs38, 32(T5)  \r
+    stxv        vs39, 48(T5)\r
+\r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs40,   vs10,   alpha_r \r
+    xvmulsp     vs41,   vs14,   alpha_r \r
+    xvmulsp     vs42,   vs18,   alpha_r \r
+    xvmulsp     vs43,   vs26,   alpha_r  \r
+    xvmulsp     vs44,   vs11,   alpha_r \r
+    xvmulsp     vs45,   vs15,   alpha_r  \r
+    xvmulsp     vs46,   vs19,   alpha_r \r
+    xvmulsp     vs47,   vs27,   alpha_r                   \r
+#else\r
+\r
+    xvmaddasp   vs40,   vs10,   alpha_r \r
+    xvmaddasp   vs41,   vs14,   alpha_r   \r
+    xvmaddasp   vs42,   vs18,   alpha_r \r
+    xvmaddasp   vs43,   vs26,   alpha_r  \r
+    xvmaddasp   vs44,   vs11,   alpha_r \r
+    xvmaddasp   vs45,   vs15,   alpha_r \r
+    xvmaddasp   vs46,   vs19,   alpha_r \r
+    xvmaddasp   vs47,   vs27,   alpha_r  \r
+        \r
+#endif  \r
+\r
+    stxv        vs40, 0(T6)\r
+    stxv        vs41, 16(T6)  \r
+    stxv        vs42, 32(T6)  \r
+    stxv        vs43, 48(T6)  \r
+    stxv        vs44, 0(T7)\r
+    stxv        vs45, 16(T7) \r
+    stxv        vs46, 32(T7)  \r
+    stxv        vs47, 48(T7)\r
+  \r
+\r
+    addi CO,CO,64\r
+\r
+\r
+.endm\r
+\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=8 and M=8\r
+**********************************************************************************************/\r
+\r
+.macro LOAD8x8_1\r
+   LOAD8x8 1\r
+.endm\r
+\r
+.macro LOAD8x8_0\r
+   LOAD8x8 0\r
+.endm\r
+\r
+.macro KERNEL8x8_L1_L4  Index,IsLast\r
+  KERNEL8x8_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x8_I1_L4  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x8_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x8_I1_L4_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x8_I1_L4_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+.macro KERNEL8x8_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+   KERNEL8x8_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL8x8_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x8_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x8_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x8_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro END8x8_NORMAL\r
+  END8x8 0, AO, BO, 32,32 \r
+.endm\r
+\r
+.macro Zero8X8\r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33\r
+ \r
+    xxlxor      vs36,   vs36,   vs36\r
+    xxlxor      vs37,   vs37,   vs37\r
+ \r
+    xxlxor      vs40,   vs40,   vs40\r
+    xxlxor      vs41,   vs41,   vs41\r
+ \r
+    xxlxor      vs44,   vs44,   vs44\r
+    xxlxor      vs45,   vs45,   vs45\r
+ \r
+    xxlxor      vs48,   vs48,   vs48\r
+    xxlxor      vs49,   vs49,   vs49\r
+ \r
+    xxlxor      vs52,   vs52,   vs52\r
+    xxlxor      vs53,   vs53,   vs53\r
+ \r
+    xxlxor      vs56,   vs56,   vs56\r
+    xxlxor      vs57,   vs57,   vs57\r
+  \r
+    xxlxor      vs60,   vs60,   vs60\r
+    xxlxor      vs61,   vs61,   vs61\r
+    \r
+.endm\r
+\r
+.macro LOAD8x8  Zero\r
+\r
+    lxv vs24,   0(BO)\r
+    lxv vs28,   16(BO)\r
+    lxv vs0,     0(AO)\r
+    lxv vs1,    16(AO)\r
+\r
+    xxperm      vs26,   vs24,       permute_mask\r
+    xxperm      vs30,   vs28,       permute_mask    \r
+    xxpermdi    vs25,   vs24,   vs24,2     \r
+    xxpermdi    vs29,   vs28,   vs28,2    \r
+\r
+    xxpermdi    vs27,   vs26,   vs26,2  \r
+    xxpermdi    vs31,   vs30,   vs30,2      \r
+\r
+.if \Zero==1 \r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33 \r
+    xxlxor      vs36,   vs36,   vs36\r
+    xxlxor      vs37,   vs37,   vs37\r
+    xxlxor      vs40,   vs40,   vs40\r
+    xxlxor      vs41,   vs41,   vs41 \r
+    xxlxor      vs44,   vs44,   vs44\r
+    xxlxor      vs45,   vs45,   vs45 \r
+    xxlxor      vs48,   vs48,   vs48\r
+    xxlxor      vs49,   vs49,   vs49 \r
+    xxlxor      vs52,   vs52,   vs52\r
+    xxlxor      vs53,   vs53,   vs53 \r
+    xxlxor      vs56,   vs56,   vs56\r
+    xxlxor      vs57,   vs57,   vs57  \r
+    xxlxor      vs60,   vs60,   vs60\r
+    xxlxor      vs61,   vs61,   vs61  \r
+.endif\r
+.endm\r
+\r
+\r
+.macro END8x8 First, AREG, BREG, OffsetA, OffsetB\r
+\r
+.if \OffsetB != 0 \r
+    addi        \BREG, \BREG, \OffsetB \r
+.endif\r
+.if \OffsetA != 0 \r
+    addi        \AREG, \AREG, \OffsetA \r
+.endif  \r
+\r
+.if \First==1\r
+    xvmulsp     vs32, vs0,vs24\r
+    xvmulsp     vs33, vs1,vs24\r
+\r
+    xvmulsp     vs36, vs0,vs25\r
+    xvmulsp     vs37, vs1,vs25\r
+\r
+    xvmulsp     vs40, vs0,vs26\r
+    xvmulsp     vs41, vs1,vs26\r
+\r
+    xvmulsp     vs44, vs0,vs27\r
+    xvmulsp     vs45, vs1,vs27\r
+\r
+    xvmulsp     vs48, vs0,vs28\r
+    xvmulsp     vs49, vs1,vs28\r
+\r
+    xvmulsp     vs52, vs0,vs29\r
+    xvmulsp     vs53, vs1,vs29\r
+\r
+    xvmulsp     vs56, vs0,vs30\r
+    xvmulsp     vs57, vs1,vs30\r
+\r
+    xvmulsp     vs60, vs0,vs31\r
+    xvmulsp     vs61, vs1,vs31\r
+\r
+.else\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+\r
+    xvmaddasp       vs48, vs0,vs28\r
+    xvmaddasp       vs49, vs1,vs28\r
+\r
+    xvmaddasp       vs52, vs0,vs29\r
+    xvmaddasp       vs53, vs1,vs29\r
+\r
+    xvmaddasp       vs56, vs0,vs30\r
+    xvmaddasp       vs57, vs1,vs30\r
+\r
+    xvmaddasp       vs60, vs0,vs31\r
+    xvmaddasp       vs61, vs1,vs31\r
+\r
+.endif\r
+.endm  \r
+\r
+.macro KERNEL8x8_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+    lxv vs8,    DISP32(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs12,   DISP32(\Index,16+\OffsetB)(\BREG)\r
+\r
+    lxv vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs5,    DISP32(\Index,16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs10,   vs8,        permute_mask\r
+    xxperm      vs14,   vs12,       permute_mask    \r
+    xxpermdi    vs9,    vs8,    vs8,2    \r
+    xxpermdi    vs13,   vs12,   vs12,2   \r
+\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+    xxpermdi    vs11,   vs10,   vs10,2  \r
+    xxpermdi    vs15,   vs14,   vs14,2  \r
+\r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+\r
+    xvmaddasp       vs48, vs0,vs28\r
+    xvmaddasp       vs49, vs1,vs28\r
+\r
+    xvmaddasp       vs52, vs0,vs29\r
+    xvmaddasp       vs53, vs1,vs29\r
+\r
+    xvmaddasp       vs56, vs0,vs30\r
+    xvmaddasp       vs57, vs1,vs30\r
+\r
+    xvmaddasp       vs60, vs0,vs31\r
+    xvmaddasp       vs61, vs1,vs31\r
+\r
+    lxv vs24,   DISP32(\Index,32+\OffsetB)(\BREG)\r
+    lxv vs28,   DISP32(\Index,32+16+\OffsetB)(\BREG)\r
+\r
+    lxv vs0,    DISP32(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs1,    DISP32(\Index,32+16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs26,   vs24,       permute_mask\r
+    xxperm      vs30,   vs28,       permute_mask    \r
+    xxpermdi    vs25,   vs24,   vs24,2     \r
+    xxpermdi    vs29,   vs28,   vs28,2    \r
+\r
+    xvmaddasp       vs32, vs4,vs8\r
+    xvmaddasp       vs33, vs5,vs8\r
+\r
+    xvmaddasp       vs36, vs4,vs9\r
+    xvmaddasp       vs37, vs5,vs9\r
+\r
+    xxpermdi    vs27,   vs26,   vs26,2  \r
+    xxpermdi    vs31,   vs30,   vs30,2      \r
+\r
+    xvmaddasp       vs40, vs4,vs10\r
+    xvmaddasp       vs41, vs5,vs10\r
+\r
+    xvmaddasp       vs44, vs4,vs11\r
+    xvmaddasp       vs45, vs5,vs11\r
+\r
+    xvmaddasp       vs48, vs4,vs12\r
+    xvmaddasp       vs49, vs5,vs12\r
+\r
+    xvmaddasp       vs52, vs4,vs13\r
+    xvmaddasp       vs53, vs5,vs13\r
+\r
+    xvmaddasp       vs56, vs4,vs14\r
+    xvmaddasp       vs57, vs5,vs14\r
+\r
+    xvmaddasp       vs60, vs4,vs15\r
+    xvmaddasp       vs61, vs5,vs15\r
+\r
+    lxv vs8,    DISP32(\Index,64+\OffsetB)(\BREG)\r
+    lxv vs12,   DISP32(\Index,64+16+\OffsetB)(\BREG)\r
+\r
+    lxv vs4,    DISP32(\Index,64+0+\OffsetA)(\AREG)\r
+    lxv vs5,    DISP32(\Index,64+16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs10,   vs8,        permute_mask\r
+    xxperm      vs14,   vs12,       permute_mask    \r
+    xxpermdi    vs9,    vs8,    vs8,2    \r
+    xxpermdi    vs13,   vs12,   vs12,2  \r
+\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+    xxpermdi    vs11,   vs10,   vs10,2  \r
+    xxpermdi    vs15,   vs14,   vs14,2  \r
+\r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+\r
+    xvmaddasp       vs48, vs0,vs28\r
+    xvmaddasp       vs49, vs1,vs28\r
+\r
+    xvmaddasp       vs52, vs0,vs29\r
+    xvmaddasp       vs53, vs1,vs29\r
+\r
+    xvmaddasp       vs56, vs0,vs30\r
+    xvmaddasp       vs57, vs1,vs30\r
+\r
+    xvmaddasp       vs60, vs0,vs31\r
+    xvmaddasp       vs61, vs1,vs31\r
+\r
+.if \Complete==0\r
+    lxv vs24,   DISP32(\Index,96+\OffsetB)(\BREG)\r
+    lxv vs28,   DISP32(\Index,96+16+\OffsetB)(\BREG)\r
+\r
+    lxv vs0,    DISP32(\Index,96+\OffsetA)(\AREG)\r
+    lxv vs1,    DISP32(\Index,96+16+\OffsetA)(\AREG) \r
+\r
+    xxperm      vs26,   vs24,   permute_mask\r
+    xxperm      vs30,   vs28,   permute_mask    \r
+    xxpermdi    vs25,   vs24,   vs24,2 \r
+    xxpermdi    vs29,   vs28,   vs28,2      \r
+\r
+.endif \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+  \r
+    addi        \BREG, \BREG,  DISP32(\Index,32*3+\OffsetB)\r
+    addi        \AREG, \AREG, DISP32(\Index,32*3+\OffsetA)\r
+.else\r
+  \r
+    addi        \BREG, \BREG,  DISP32(\Index,128)\r
+    addi        \AREG, \AREG, DISP32(\Index,128)\r
+.endif\r
+.endif   \r
+ \r
+    xvmaddasp       vs32, vs4,vs8\r
+    xvmaddasp       vs33, vs5,vs8\r
+\r
+    xvmaddasp       vs36, vs4,vs9\r
+    xvmaddasp       vs37, vs5,vs9\r
+\r
+.if \Complete==0        \r
+    xxpermdi    vs27,   vs26,   vs26,2  \r
+    xxpermdi    vs31,   vs30,   vs30,2  \r
+    \r
+.endif\r
+ \r
+    xvmaddasp       vs40, vs4,vs10\r
+    xvmaddasp       vs41, vs5,vs10\r
+\r
+    xvmaddasp       vs44, vs4,vs11\r
+    xvmaddasp       vs45, vs5,vs11\r
+\r
+    xvmaddasp       vs48, vs4,vs12\r
+    xvmaddasp       vs49, vs5,vs12\r
+\r
+    xvmaddasp       vs52, vs4,vs13\r
+    xvmaddasp       vs53, vs5,vs13\r
+\r
+    xvmaddasp       vs56, vs4,vs14\r
+    xvmaddasp       vs57, vs5,vs14\r
+\r
+    xvmaddasp       vs60, vs4,vs15\r
+    xvmaddasp       vs61, vs5,vs15\r
+\r
+.endm\r
+\r
+.macro KERNEL8x8 First\r
+\r
+  LOAD8x8 0\r
+  END8x8 \First, AO, BO, 32,32  \r
+.endm\r
+\r
+.macro KERNEL8x8_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete\r
+    \r
+    lxv vs8,    DISP16(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs12,   DISP16(\Index,16+\OffsetB)(\BREG)\r
+\r
+    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs5,    DISP16(\Index,16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs10,   vs8,        permute_mask\r
+    xxperm      vs14,   vs12,       permute_mask    \r
+    xxpermdi    vs9,    vs8,    vs8,2    \r
+    xxpermdi    vs13,   vs12,   vs12,2   \r
+.if \First==1\r
+    xvmulsp     vs32, vs0,vs24\r
+    xvmulsp     vs33, vs1,vs24\r
+\r
+    xvmulsp     vs36, vs0,vs25\r
+    xvmulsp     vs37, vs1,vs25\r
+\r
+.else\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+.endif\r
+\r
+    xxpermdi    vs11,   vs10,   vs10,2  \r
+    xxpermdi    vs15,   vs14,   vs14,2  \r
+ \r
+.if \First==1  \r
+    xvmulsp     vs40, vs0,vs26\r
+    xvmulsp     vs41, vs1,vs26\r
+\r
+    xvmulsp     vs44, vs0,vs27\r
+    xvmulsp     vs45, vs1,vs27\r
+\r
+    xvmulsp     vs48, vs0,vs28\r
+    xvmulsp     vs49, vs1,vs28\r
+\r
+    xvmulsp     vs52, vs0,vs29\r
+    xvmulsp     vs53, vs1,vs29\r
+\r
+    xvmulsp     vs56, vs0,vs30\r
+    xvmulsp     vs57, vs1,vs30\r
+\r
+    xvmulsp     vs60, vs0,vs31\r
+    xvmulsp     vs61, vs1,vs31\r
+\r
+.else \r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+\r
+    xvmaddasp       vs48, vs0,vs28\r
+    xvmaddasp       vs49, vs1,vs28\r
+\r
+    xvmaddasp       vs52, vs0,vs29\r
+    xvmaddasp       vs53, vs1,vs29\r
+\r
+    xvmaddasp       vs56, vs0,vs30\r
+    xvmaddasp       vs57, vs1,vs30\r
+\r
+    xvmaddasp       vs60, vs0,vs31\r
+    xvmaddasp       vs61, vs1,vs31\r
+\r
+.endif\r
+.if \Complete==0\r
+    lxv vs24,   DISP16(\Index,32+\OffsetB)(\BREG)\r
+    lxv vs28,   DISP16(\Index,32+16+\OffsetB)(\BREG)\r
+\r
+    lxv vs0,    DISP16(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs1,    DISP16(\Index,32+16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs26,   vs24,   permute_mask\r
+    xxperm      vs30,   vs28,   permute_mask    \r
+    xxpermdi    vs25,   vs24,   vs24,2   \r
+    xxpermdi    vs29,   vs28,   vs28,2  \r
+.endif    \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+    addi        \BREG, \BREG,  DISP16(\Index,32+\OffsetB) \r
+    addi        \AREG, \AREG,  DISP16(\Index,32+\OffsetA)\r
+\r
+.else\r
+    addi        \BREG, \BREG,  DISP16(\Index,64)\r
+    addi        \AREG, \AREG,  DISP16(\Index,64) \r
+.endif\r
+.endif\r
+\r
+.if \First==1\r
+    xvmulsp     vs32, vs4,vs8\r
+    xvmulsp     vs33, vs5,vs8\r
+\r
+    xvmulsp     vs36, vs4,vs9\r
+    xvmulsp     vs37, vs5,vs9\r
+\r
+.else\r
+    xvmaddasp       vs32, vs4,vs8\r
+    xvmaddasp       vs33, vs5,vs8\r
+\r
+    xvmaddasp       vs36, vs4,vs9\r
+    xvmaddasp       vs37, vs5,vs9\r
+\r
+.endif \r
+ \r
+.if \Complete==0        \r
+    xxpermdi    vs27,   vs26,   vs26,2  \r
+    xxpermdi    vs31,   vs30,   vs30,2  \r
+ \r
+.endif\r
+.if \First==1  \r
+    xvmulsp     vs40, vs4,vs10\r
+    xvmulsp     vs41, vs5,vs10\r
+\r
+    xvmulsp     vs44, vs4,vs11\r
+    xvmulsp     vs45, vs5,vs11\r
+\r
+    xvmulsp     vs48, vs4,vs12\r
+    xvmulsp     vs49, vs5,vs12\r
+\r
+    xvmulsp     vs52, vs4,vs13\r
+    xvmulsp     vs53, vs5,vs13\r
+\r
+    xvmulsp     vs56, vs4,vs14\r
+    xvmulsp     vs57, vs5,vs14\r
+\r
+    xvmulsp     vs60, vs4,vs15\r
+    xvmulsp     vs61, vs5,vs15\r
+\r
+.else \r
+    xvmaddasp       vs40, vs4,vs10\r
+    xvmaddasp       vs41, vs5,vs10\r
+\r
+    xvmaddasp       vs44, vs4,vs11\r
+    xvmaddasp       vs45, vs5,vs11\r
+\r
+    xvmaddasp       vs48, vs4,vs12\r
+    xvmaddasp       vs49, vs5,vs12\r
+\r
+    xvmaddasp       vs52, vs4,vs13\r
+    xvmaddasp       vs53, vs5,vs13\r
+\r
+    xvmaddasp       vs56, vs4,vs14\r
+    xvmaddasp       vs57, vs5,vs14\r
+\r
+    xvmaddasp       vs60, vs4,vs15\r
+    xvmaddasp       vs61, vs5,vs15\r
+\r
+.endif\r
+\r
+.endm\r
+\r
+\r
+.macro SAVE8x8 \r
+ \r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC \r
+\r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10  \r
+\r
+  add     T4, T2, T10  \r
+  add     T5, T3, T10 \r
+\r
+  add     T6, T4, T10 \r
+  add     T7, T5, T10 \r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs34, 0(CO)\r
+    lxv        vs35, 16(CO)      \r
+    lxv        vs38, 0(T1)\r
+    lxv        vs39, 16(T1)  \r
+    lxv        vs42, 0(T2)\r
+    lxv        vs43, 16(T2)     \r
+    lxv        vs46, 0(T3)\r
+    lxv        vs47, 16(T3)  \r
+\r
+    lxv        vs50, 0(T4)\r
+    lxv        vs51, 16(T4)      \r
+    lxv        vs54, 0(T5)\r
+    lxv        vs55, 16(T5)  \r
+    lxv        vs58, 0(T6)\r
+    lxv        vs59, 16(T6)     \r
+    lxv        vs62, 0(T7)\r
+    lxv        vs63, 16(T7) \r
+#endif  \r
+\r
+    xxmrglw     vs8,    vs32,   vs44\r
+    xxmrglw     vs10,   vs36,   vs40  \r
+\r
+    xxmrghw     vs1,    vs32,   vs44\r
+    xxmrghw     vs0,    vs36,   vs40\r
+\r
+    xxmrglw     vs12,   vs33,   vs45\r
+    xxmrglw     vs14,   vs37,   vs41  \r
+\r
+    xxmrghw     vs2,    vs37,   vs41\r
+    xxmrghw     vs3,    vs33,   vs45\r
+\r
+    xxlor      vs9, vs8,    vs8\r
+    xxlor      vs11,    vs10,   vs10 \r
+ \r
+    xxlor      vs13,    vs12,   vs12\r
+    xxlor      vs15,    vs14,   vs14\r
+\r
+    xxperm      vs8,    vs0,    save_permute_1\r
+    xxperm      vs10,   vs1,    save_permute_1\r
+    xxperm      vs9,    vs0,    save_permute_2  \r
+    xxperm      vs11,   vs1,    save_permute_2      \r
+\r
+    xxperm     vs12,    vs2,    save_permute_1\r
+    xxperm     vs14,    vs3,    save_permute_1\r
+      \r
+    xxperm     vs13,    vs2,    save_permute_2   \r
+    xxperm     vs15,    vs3,    save_permute_2      \r
+\r
+\r
+    /* multiply add normal way */\r
+ \r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs34,   vs8,    alpha_r \r
+    xvmulsp     vs35,   vs12,   alpha_r \r
+    xvmulsp     vs38,   vs9,    alpha_r \r
+    xvmulsp     vs39,   vs13,   alpha_r \r
+    xvmulsp     vs42,   vs10,   alpha_r \r
+    xvmulsp     vs43,   vs14,   alpha_r \r
+    xvmulsp     vs46,   vs11,   alpha_r \r
+    xvmulsp     vs47,   vs15,   alpha_r                    \r
+#else \r
+    xvmaddasp   vs34,   vs8,    alpha_r \r
+    xvmaddasp   vs35,   vs12,   alpha_r \r
+    xvmaddasp   vs38,   vs9,    alpha_r \r
+    xvmaddasp   vs39,   vs13,   alpha_r  \r
+    xvmaddasp   vs42,   vs10,   alpha_r \r
+    xvmaddasp   vs43,   vs14,   alpha_r   \r
+    xvmaddasp   vs46,   vs11,   alpha_r \r
+    xvmaddasp   vs47,   vs15,   alpha_r                     \r
+#endif     \r
+ \r
+   \r
+    xxmrglw     vs8,    vs48,   vs60\r
+    xxmrglw     vs10,   vs52,   vs56  \r
+\r
+    xxmrghw     vs1,    vs48,   vs60\r
+    xxmrghw     vs0,    vs52,   vs56\r
+    stxv        vs34, 0(CO)\r
+    stxv        vs35, 16(CO) \r
+    xxmrglw     vs12,   vs49,   vs61\r
+    xxmrglw     vs14,   vs53,   vs57  \r
+    stxv        vs38, 0(T1)\r
+    stxv        vs39, 16(T1) \r
+    xxmrghw     vs2,    vs53,   vs57\r
+    xxmrghw     vs3,    vs49,   vs61\r
+    stxv        vs42, 0(T2)\r
+    stxv        vs43, 16(T2)   \r
+    xxlor      vs9, vs8,    vs8\r
+    xxlor      vs11,    vs10,   vs10  \r
+    stxv        vs46, 0(T3)\r
+    stxv        vs47, 16(T3)  \r
+    xxlor      vs13,    vs12,   vs12\r
+    xxlor      vs15,    vs14,   vs14\r
+   \r
+    xxperm      vs8,    vs0,    save_permute_1\r
+    xxperm      vs10,   vs1,    save_permute_1\r
+    \r
+ \r
+    xxperm      vs9,    vs0,    save_permute_2  \r
+    xxperm      vs11,   vs1,    save_permute_2      \r
+ \r
+    xxperm     vs12,    vs2,    save_permute_1\r
+    xxperm     vs14,    vs3,    save_permute_1\r
+    xxperm     vs13,    vs2,    save_permute_2   \r
+    xxperm     vs15,    vs3,    save_permute_2      \r
+    \r
+ #ifdef TRMMKERNEL\r
+    xvmulsp     vs50,   vs8,    alpha_r \r
+    xvmulsp     vs51,   vs12,   alpha_r \r
+    xvmulsp     vs54,   vs9,    alpha_r \r
+    xvmulsp     vs55,   vs13,   alpha_r \r
+    xvmulsp     vs58,   vs10,   alpha_r \r
+    xvmulsp     vs59,   vs14,   alpha_r \r
+    xvmulsp     vs62,   vs11,   alpha_r \r
+    xvmulsp     vs63,   vs15,   alpha_r                    \r
+#else \r
+    xvmaddasp     vs50,   vs8,    alpha_r \r
+    xvmaddasp     vs51,   vs12,   alpha_r \r
+    xvmaddasp     vs54,   vs9,    alpha_r \r
+    xvmaddasp     vs55,   vs13,   alpha_r \r
+    xvmaddasp     vs58,   vs10,   alpha_r \r
+    xvmaddasp     vs59,   vs14,   alpha_r \r
+    xvmaddasp     vs62,   vs11,   alpha_r \r
+    xvmaddasp     vs63,   vs15,   alpha_r                     \r
+#endif  \r
+\r
+    stxv        vs50, 0(T4)\r
+    stxv        vs51, 16(T4)      \r
+    stxv        vs54, 0(T5)\r
+    stxv        vs55, 16(T5)  \r
+    stxv        vs58, 0(T6)\r
+    stxv        vs59, 16(T6)     \r
+    stxv        vs62, 0(T7)\r
+    stxv        vs63, 16(T7)   \r
+\r
+    addi CO,CO,32\r
+\r
+.endm\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=8 and M=4\r
+**********************************************************************************************/\r
+\r
+.macro LOAD8x4_1\r
+   LOAD8x4 1\r
+.endm\r
+\r
+.macro LOAD8x4_0\r
+   LOAD8x4 0\r
+.endm\r
+\r
+.macro KERNEL8x4_L1_L4  Index,IsLast\r
+  KERNEL8x4_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x4_I1_L4  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x4_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x4_I1_L4_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x4_I1_L4_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+.macro KERNEL8x4_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+   KERNEL8x4_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL8x4_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x4_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL8x4_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x4_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro Zero8X4\r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33 \r
+    xxlxor      vs34,   vs34,   vs34\r
+    xxlxor      vs35,   vs35,   vs35\r
+    \r
+    xxlxor      vs48,   vs48,   vs48\r
+    xxlxor      vs49,   vs49,   vs49\r
+    xxlxor      vs50,   vs50,   vs50\r
+    xxlxor      vs51,   vs51,   vs51  \r
+    \r
+.endm\r
+\r
+.macro LOAD8x4  Zero\r
+\r
+    lxv vs0,     0(AO)\r
+    lxv vs24,   0(BO)\r
+    lxv vs25,   16(BO)\r
+\r
+\r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2        \r
+\r
+.if \Zero==1 \r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33 \r
+    xxlxor      vs34,   vs34,   vs34\r
+    xxlxor      vs35,   vs35,   vs35\r
+\r
+    xxlxor      vs48,   vs48,   vs48\r
+    xxlxor      vs49,   vs49,   vs49\r
+    xxlxor      vs50,   vs50,   vs50\r
+    xxlxor      vs51,   vs51,   vs51  \r
+.endif\r
+.endm\r
+\r
+.macro END8x4_NORMAL\r
+  END8x4 0, AO, BO, 16,32 \r
+.endm\r
+\r
+.macro END8x4 First, AREG, BREG, OffsetA, OffsetB\r
+\r
+.if \OffsetB != 0 \r
+    addi        \BREG, \BREG, \OffsetB \r
+.endif\r
+.if \OffsetA != 0 \r
+    addi        \AREG, \AREG, \OffsetA \r
+.endif  \r
+\r
+.if \First==1\r
+    xvmulsp      vs32,   vs24,   vs0\r
+    xvmulsp      vs33,   vs24,   vs1 \r
+    xvmulsp      vs34,   vs24,   vs2\r
+    xvmulsp      vs35,   vs24,   vs3\r
+\r
+    xvmulsp      vs48,   vs25,   vs0\r
+    xvmulsp      vs49,   vs25,   vs1\r
+    xvmulsp      vs50,   vs25,   vs2\r
+    xvmulsp      vs51,   vs25,   vs3  \r
+.else\r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+\r
+    xvmaddasp      vs48,   vs25,   vs0\r
+    xvmaddasp      vs49,   vs25,   vs1\r
+    xvmaddasp      vs50,   vs25,   vs2\r
+    xvmaddasp      vs51,   vs25,   vs3 \r
+\r
+.endif\r
+.endm  \r
+\r
+.macro KERNEL8x4_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP32(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs27,   DISP32(\Index,16+\OffsetB)(\BREG)\r
+\r
+    xxperm      vs6,   vs4,       permute_mask  \r
+    xxpermdi    vs5,   vs4,   vs4,2      \r
+    xxpermdi    vs7,   vs6,   vs6,2 \r
+ \r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+\r
+    xvmaddasp      vs48,   vs25,   vs0\r
+    xvmaddasp      vs49,   vs25,   vs1\r
+    xvmaddasp      vs50,   vs25,   vs2\r
+    xvmaddasp      vs51,   vs25,   vs3 \r
+\r
+    lxv vs0,    DISP16(\Index, 16+\OffsetA)(\AREG)\r
+    lxv vs24,   DISP32(\Index, 32+\OffsetB)(\BREG)\r
+    lxv vs25,   DISP32(\Index, 48+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2   \r
+\r
+    xvmaddasp      vs32,   vs26,   vs4\r
+    xvmaddasp      vs33,   vs26,   vs5 \r
+    xvmaddasp      vs34,   vs26,   vs6\r
+    xvmaddasp      vs35,   vs26,   vs7\r
+\r
+    xvmaddasp      vs48,   vs27,   vs4\r
+    xvmaddasp      vs49,   vs27,   vs5\r
+    xvmaddasp      vs50,   vs27,   vs6\r
+    xvmaddasp      vs51,   vs27,   vs7\r
+ \r
+\r
+    lxv vs4,    DISP16(\Index, 32+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP32(\Index, 64+\OffsetB)(\BREG)\r
+    lxv vs27,   DISP32(\Index, 80+\OffsetB)(\BREG)\r
+\r
+    xxperm      vs6,   vs4,       permute_mask  \r
+    xxpermdi    vs5,   vs4,   vs4,2      \r
+    xxpermdi    vs7,   vs6,   vs6,2 \r
+ \r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+\r
+    xvmaddasp      vs48,   vs25,   vs0\r
+    xvmaddasp      vs49,   vs25,   vs1\r
+    xvmaddasp      vs50,   vs25,   vs2\r
+    xvmaddasp      vs51,   vs25,   vs3 \r
+\r
+.if \Complete==0 \r
+\r
+    lxv vs0,    DISP16(\Index, 48+\OffsetA)(\AREG)\r
+    lxv vs24,   DISP32(\Index, 96+\OffsetB)(\BREG)\r
+    lxv vs25,   DISP32(\Index, 96+16+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2   \r
+.endif\r
+    xvmaddasp      vs32,   vs26,   vs4\r
+    xvmaddasp      vs33,   vs26,   vs5 \r
+    xvmaddasp      vs34,   vs26,   vs6\r
+    xvmaddasp      vs35,   vs26,   vs7\r
+\r
+    xvmaddasp      vs48,   vs27,   vs4\r
+    xvmaddasp      vs49,   vs27,   vs5\r
+    xvmaddasp      vs50,   vs27,   vs6\r
+    xvmaddasp      vs51,   vs27,   vs7\r
+\r
+ \r
+ \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+    addi        \AREG, \AREG, DISP16(\Index,16*3+\OffsetA)  \r
+    addi        \BREG, \BREG,  DISP32(\Index,32*3+\OffsetB)\r
+\r
+.else\r
+    addi        \AREG, \AREG, DISP16(\Index,64)  \r
+    addi        \BREG, \BREG,  DISP32(\Index,128)\r
+\r
+.endif\r
+.endif   \r
+ \r
+ \r
+.endm\r
+\r
+.macro KERNEL8x4 First\r
+    LOAD8x4 0\r
+    END8x4 \First, AO, BO, 16,32  \r
+.endm\r
+\r
+.macro KERNEL8x4_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+    lxv vs4,    DISP8(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs27,   DISP16(\Index,16+\OffsetB)(\BREG)\r
+\r
+    xxperm      vs6,   vs4,       permute_mask  \r
+    xxpermdi    vs5,   vs4,   vs4,2      \r
+    xxpermdi    vs7,   vs6,   vs6,2 \r
+.if \First==1\r
+    xvmulsp      vs32,   vs24,   vs0\r
+    xvmulsp      vs33,   vs24,   vs1 \r
+    xvmulsp      vs34,   vs24,   vs2\r
+    xvmulsp      vs35,   vs24,   vs3\r
+\r
+    xvmulsp      vs48,   vs25,   vs0\r
+    xvmulsp      vs49,   vs25,   vs1\r
+    xvmulsp      vs50,   vs25,   vs2\r
+    xvmulsp      vs51,   vs25,   vs3  \r
+.else \r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+\r
+    xvmaddasp      vs48,   vs25,   vs0\r
+    xvmaddasp      vs49,   vs25,   vs1\r
+    xvmaddasp      vs50,   vs25,   vs2\r
+    xvmaddasp      vs51,   vs25,   vs3 \r
+.endif\r
+\r
+.if \Complete==0 \r
+\r
+    lxv vs0,    DISP8(\Index, 16+\OffsetA)(\AREG)\r
+    lxv vs24,   DISP16(\Index, 32+\OffsetB)(\BREG)\r
+    lxv vs25,   DISP16(\Index, 48+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2   \r
+.endif\r
+\r
+.if \First==1\r
+    xvmulsp      vs32,   vs26,   vs4\r
+    xvmulsp      vs33,   vs26,   vs5 \r
+    xvmulsp      vs34,   vs26,   vs6\r
+    xvmulsp      vs35,   vs26,   vs7\r
+\r
+    xvmulsp      vs48,   vs27,   vs4\r
+    xvmulsp      vs49,   vs27,   vs5\r
+    xvmulsp      vs50,   vs27,   vs6\r
+    xvmulsp      vs51,   vs27,   vs7\r
+\r
+\r
+.else\r
+    xvmaddasp      vs32,   vs26,   vs4\r
+    xvmaddasp      vs33,   vs26,   vs5 \r
+    xvmaddasp      vs34,   vs26,   vs6\r
+    xvmaddasp      vs35,   vs26,   vs7\r
+\r
+    xvmaddasp      vs48,   vs27,   vs4\r
+    xvmaddasp      vs49,   vs27,   vs5\r
+    xvmaddasp      vs50,   vs27,   vs6\r
+    xvmaddasp      vs51,   vs27,   vs7\r
+.endif\r
+ \r
+ \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+    addi        \AREG, \AREG, DISP8(\Index,16+\OffsetA)  \r
+    addi        \BREG, \BREG,  DISP16(\Index,32+\OffsetB)\r
+\r
+.else\r
+    addi        \AREG, \AREG, DISP8(\Index,32)  \r
+    addi        \BREG, \BREG,  DISP16(\Index,64)\r
+\r
+.endif\r
+.endif   \r
+     \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE8x4\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC \r
+#if !defined(TRMMKERNEL)  \r
+  lxv        vs36, 0(CO)\r
+  lxv        vs37, 0(T1)\r
+#endif  \r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10 \r
+#if !defined(TRMMKERNEL)    \r
+  lxv        vs38, 0(T2)\r
+  lxv        vs39, 0(T3)   \r
+#endif   \r
+  add     T4, T2, T10 \r
+  add     T5, T3, T10\r
+#if !defined(TRMMKERNEL)    \r
+  lxv        vs40, 0(T4)\r
+  lxv        vs41, 0(T5)\r
+#endif  \r
+  add     T6, T4, T10 \r
+  add     T7, T5, T10\r
+#if !defined(TRMMKERNEL)    \r
+  lxv        vs42, 0(T6)\r
+  lxv        vs43, 0(T7)\r
+#endif\r
+  xxmrglw  vs0, vs35,vs32\r
+  xxmrglw  vs1, vs34,vs33 \r
+  xxmrglw  vs4, vs32,vs35\r
+  xxmrglw  vs5, vs33,vs34 \r
+\r
+\r
+  xxmrghw  vs2, vs35,vs32\r
+  xxmrghw  vs3, vs34,vs33 \r
+  xxmrghw  vs6, vs32,vs35\r
+  xxmrghw  vs7, vs33,vs34  \r
+\r
+  xxmrgld  vs24, vs1, vs0  \r
+  xxmrghd  vs25,vs5,vs4 \r
+\r
+  xxmrgld  vs26, vs2, vs3  \r
+  xxmrghd  vs27,vs6,vs7\r
+\r
+\r
+  xxmrglw  vs0, vs51,vs48\r
+  xxmrglw  vs1, vs50,vs49  \r
+  xxmrglw  vs4, vs48,vs51\r
+  xxmrglw  vs5, vs49,vs50 \r
+\r
+  xxmrghw  vs2, vs51,vs48\r
+  xxmrghw  vs3, vs50,vs49  \r
+  xxmrghw  vs6, vs48,vs51\r
+  xxmrghw  vs7, vs49,vs50   \r
+\r
+  xxmrgld  vs28, vs1, vs0  \r
+  xxmrghd  vs29,vs5,vs4\r
+\r
+  xxmrgld  vs30, vs2, vs3   \r
+  xxmrghd  vs31,vs6,vs7\r
+#if defined(TRMMKERNEL)\r
+\r
+  xvmulsp        vs36, vs24, alpha_r\r
+  xvmulsp        vs37, vs25, alpha_r \r
+  xvmulsp        vs38, vs26, alpha_r\r
+  xvmulsp        vs39, vs27, alpha_r   \r
+  xvmulsp        vs40, vs28, alpha_r\r
+  xvmulsp        vs41, vs29, alpha_r \r
+  xvmulsp        vs42, vs30, alpha_r\r
+  xvmulsp        vs43, vs31, alpha_r\r
+#else\r
+  xvmaddasp        vs36, vs24, alpha_r\r
+  xvmaddasp        vs37, vs25, alpha_r \r
+  xvmaddasp        vs38, vs26, alpha_r\r
+  xvmaddasp        vs39, vs27, alpha_r   \r
+  xvmaddasp        vs40, vs28, alpha_r\r
+  xvmaddasp        vs41, vs29, alpha_r \r
+  xvmaddasp        vs42, vs30, alpha_r\r
+  xvmaddasp        vs43, vs31, alpha_r\r
+#endif\r
+\r
+  stxv        vs36, 0(CO)\r
+  stxv        vs37, 0(T1) \r
+  stxv        vs38, 0(T2)\r
+  stxv        vs39, 0(T3)   \r
+  stxv        vs40, 0(T4)\r
+  stxv        vs41, 0(T5) \r
+  stxv        vs42, 0(T6)\r
+  stxv        vs43, 0(T7)\r
+\r
+\r
+  addi CO,CO,16\r
+.endm\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=8 and M=2\r
+**********************************************************************************************/\r
+\r
+ \r
+.macro KERNEL8x2_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x2_I_2 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ \r
+\r
+.macro Zero8x2\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+    xxlxor      vs2,   vs2,   vs2\r
+    xxlxor      vs3,   vs3,   vs3\r
+       \r
+.endm\r
+ \r
+.macro KERNEL8x2\r
+  KERNEL8x2_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL8x2_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxsd v4,   DISP2(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs27,   DISP8(\Index,16+\OffsetB)(\BREG)      \r
+    xxspltw   vs8,  vs36, 0 \r
+    xxspltw   vs9,  vs36, 1  \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8 \r
+    xvmulsp      vs2,   vs26,   vs9\r
+    xvmulsp      vs3,   vs27,   vs9 \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs26,   vs9\r
+    xvmaddasp      vs3,   vs27,   vs9\r
+ \r
+ .endif\r
+   \r
+    addi        \AREG, \AREG, DISP2(\Index,8)  \r
+    addi        \BREG, \BREG, DISP8(\Index,32)\r
+ \r
+.endm\r
+\r
+.macro KERNEL8x2_I_2  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs27,   DISP16(\Index,16+\OffsetB)(\BREG)\r
+    lxv vs28,   DISP16(\Index,32+\OffsetB)(\BREG)\r
+    lxv vs29,   DISP16(\Index,48+\OffsetB)(\BREG)      \r
+    xxspltw   vs8,  vs4, 2  \r
+    xxspltw   vs9,  vs4, 3 \r
+    xxspltw   vs10, vs4, 0 \r
+    xxspltw   vs11, vs4, 1\r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8 \r
+    xvmulsp      vs2,   vs26,   vs9\r
+    xvmulsp      vs3,   vs27,   vs9 \r
+\r
+    xvmulsp      vs0,   vs28,   vs10\r
+    xvmulsp      vs1,   vs29,   vs10 \r
+    xvmulsp      vs2,   vs28,   vs11\r
+    xvmulsp      vs3,   vs29,   vs11     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs26,   vs9\r
+    xvmaddasp      vs3,   vs27,   vs9\r
+\r
+    xvmaddasp      vs0,   vs28,   vs10\r
+    xvmaddasp      vs1,   vs29,   vs10 \r
+    xvmaddasp      vs2,   vs28,   vs11\r
+    xvmaddasp      vs3,   vs29,   vs11  \r
+ .endif\r
+\r
+ \r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP4(\Index,16)  \r
+    addi        \BREG, \BREG, DISP16(\Index,64)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE8x2\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC  \r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10     \r
+  add     T4, T2, T10 \r
+  add     T5, T3, T10 \r
+  add     T6, T4, T10 \r
+  add     T7, T5, T10 \r
+  /*convert alpha_r for multiply*/\r
+  xscvspdp  vs4,alpha_r\r
+/* v0 corresponds to vs32, do not forget*/\r
+#if !defined(TRMMKERNEL)\r
+  lxssp  v0,0(CO) \r
+  lxssp  v1,4(CO) \r
+\r
+  lxssp  v2,0(T1)\r
+  lxssp  v3,4(T1)\r
+\r
+  lxssp  v4,0(T2)\r
+  lxssp  v5,4(T2)\r
+\r
+  lxssp  v6,0(T3)\r
+  lxssp  v7,4(T3)\r
+\r
+  lxssp  v8,0(T4)\r
+  lxssp  v9,4(T4)\r
+\r
+  lxssp  v10,0(T5)\r
+  lxssp  v11,4(T5)\r
+\r
+  lxssp  v12,0(T6)\r
+  lxssp  v13,4(T6)\r
+\r
+  lxssp  v14,0(T7)\r
+  lxssp  v15,4(T7)\r
+#endif\r
+  xscvspdp  vs5, vs2\r
+  xxspltw   vs6, vs2, 1 \r
+  xxspltw   vs7, vs2, 2 \r
+  xxspltw   vs8, vs2, 3  \r
+  xscvspdp  vs6,vs6\r
+  xscvspdp  vs7,vs7\r
+  xscvspdp  vs8,vs8\r
+\r
+  xscvspdp  vs24, vs0\r
+  xxspltw   vs25, vs0, 1 \r
+  xxspltw   vs26, vs0, 2 \r
+  xxspltw   vs27, vs0, 3  \r
+  xscvspdp  vs25,vs25\r
+  xscvspdp  vs26,vs26\r
+  xscvspdp  vs27,vs27\r
+\r
+  xscvspdp  vs9, vs3\r
+  xxspltw   vs10, vs3, 1 \r
+  xxspltw   vs11, vs3, 2 \r
+  xxspltw   vs12, vs3, 3  \r
+  xscvspdp  vs10,vs10\r
+  xscvspdp  vs11,vs11\r
+  xscvspdp  vs12,vs12\r
+\r
+  xscvspdp  vs28, vs1\r
+  xxspltw   vs29, vs1, 1 \r
+  xxspltw   vs30, vs1, 2 \r
+  xxspltw   vs31, vs1, 3  \r
+  xscvspdp  vs29,vs29\r
+  xscvspdp  vs30,vs30\r
+  xscvspdp  vs31,vs31\r
+\r
+\r
+\r
+\r
+#if defined(TRMMKERNEL)\r
+  xsmuldp  vs32,vs8, vs4 \r
+  xsmuldp  vs33,vs27, vs4 \r
+\r
+  xsmuldp  vs34,vs7, vs4 \r
+  xsmuldp  vs35,vs26, vs4 \r
+\r
+  xsmuldp  vs36,vs6, vs4 \r
+  xsmuldp  vs37,vs25, vs4  \r
+\r
+  xsmuldp  vs38,vs5, vs4 \r
+  xsmuldp  vs39,vs24, vs4  \r
+\r
+  xsmuldp  vs40,vs12, vs4 \r
+  xsmuldp  vs41,vs31, vs4\r
+\r
+  xsmuldp  vs42,vs11, vs4 \r
+  xsmuldp  vs43,vs30, vs4  \r
+\r
+  xsmuldp  vs44,vs10, vs4 \r
+  xsmuldp  vs45,vs29, vs4 \r
+\r
+  xsmuldp  vs46,vs9, vs4 \r
+  xsmuldp  vs47,vs28, vs4      \r
+#else\r
+  xsmaddadp  vs32,vs8, vs4 \r
+  xsmaddadp  vs33,vs27, vs4 \r
+\r
+  xsmaddadp  vs34,vs7, vs4 \r
+  xsmaddadp  vs35,vs26, vs4 \r
+\r
+  xsmaddadp  vs36,vs6, vs4 \r
+  xsmaddadp  vs37,vs25, vs4  \r
+\r
+  xsmaddadp  vs38,vs5, vs4 \r
+  xsmaddadp  vs39,vs24, vs4  \r
+\r
+  xsmaddadp  vs40,vs12, vs4 \r
+  xsmaddadp  vs41,vs31, vs4\r
+\r
+  xsmaddadp  vs42,vs11, vs4 \r
+  xsmaddadp  vs43,vs30, vs4  \r
+\r
+  xsmaddadp  vs44,vs10, vs4 \r
+  xsmaddadp  vs45,vs29, vs4 \r
+\r
+  xsmaddadp  vs46,vs9, vs4 \r
+  xsmaddadp  vs47,vs28, vs4     \r
+#endif  \r
+\r
+  stxssp  v0,0(CO) \r
+  stxssp  v1,4(CO) \r
+\r
+  stxssp  v2,0(T1)\r
+  stxssp  v3,4(T1)\r
+\r
+  stxssp  v4,0(T2)\r
+  stxssp  v5,4(T2)\r
+\r
+  stxssp  v6,0(T3)\r
+  stxssp  v7,4(T3)\r
+\r
+  stxssp  v8,0(T4)\r
+  stxssp  v9,4(T4)\r
+\r
+  stxssp  v10,0(T5)\r
+  stxssp  v11,4(T5)\r
+\r
+  stxssp  v12,0(T6)\r
+  stxssp  v13,4(T6)\r
+\r
+  stxssp  v14,0(T7)\r
+  stxssp  v15,4(T7)\r
+ \r
+\r
+  addi CO,CO,8\r
+.endm\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=8 and M=1\r
+**********************************************************************************************/\r
+.macro KERNEL8x1_4   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL8x1_I_4 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro Zero8x1\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1  \r
+.endm\r
+\r
+.macro KERNEL8x1\r
+  KERNEL8x1_1 AO,BO, 0 \r
+.endm\r
+\r
+.macro KERNEL8x1_2\r
+  KERNEL8x1_2_1 AO,BO, 0 \r
+.endm\r
+\r
+.macro KERNEL8x1_1 AREG,BREG,First \r
+    lxvwsx vs8,  0, \AREG\r
+    lxv vs26,   0(\BREG)\r
+    lxv vs27,   16(\BREG)      \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8  \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8  \r
+ .endif\r
+    addi        \AREG, \AREG,  4  \r
+    addi        \BREG, \BREG,  32\r
+.endm\r
+\r
+.macro KERNEL8x1_2_1 AREG,BREG,First \r
+    lxsd v4,    0(\AREG)\r
+    lxv vs26,   0(\BREG)\r
+    lxv vs27,  16(\BREG)      \r
+    lxv vs28,  32(\BREG)\r
+    lxv vs29,  48(\BREG) \r
+    xxspltw   vs8,  vs36, 1 \r
+    xxspltw   vs9,  vs36, 0  \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8  \r
+    xvmulsp      vs0,   vs28,   vs9\r
+    xvmulsp      vs1,   vs29,   vs9     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8  \r
+    xvmaddasp      vs0,   vs28,   vs9\r
+    xvmaddasp      vs1,   vs29,   vs9 \r
+ .endif\r
+    addi        \AREG, \AREG,  8 \r
+    addi        \BREG, \BREG,  64\r
+.endm\r
+\r
+.macro KERNEL8x1_I_4  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  \r
+    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)\r
+    xxspltw   vs8,  vs4, 3 \r
+    xxspltw   vs9,  vs4, 2 \r
+    xxspltw   vs10, vs4, 1 \r
+    xxspltw   vs11, vs4, 0\r
+    lxv vs26,   DISP32(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs27,   DISP32(\Index,16+\OffsetB)(\BREG)\r
+    lxv vs28,   DISP32(\Index,32+\OffsetB)(\BREG)\r
+    lxv vs29,   DISP32(\Index,48+\OffsetB)(\BREG) \r
+    lxv vs30,   DISP32(\Index,64+ 0+\OffsetB)(\BREG)\r
+    lxv vs31,   DISP32(\Index,64+16+\OffsetB)(\BREG)\r
+    lxv vs32,   DISP32(\Index,64+32+\OffsetB)(\BREG)\r
+    lxv vs33,   DISP32(\Index,64+48+\OffsetB)(\BREG)         \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8  \r
+    xvmulsp      vs0,   vs28,   vs9\r
+    xvmulsp      vs1,   vs29,   vs9     \r
+    xvmulsp      vs0,   vs30,   vs10\r
+    xvmulsp      vs1,   vs31,   vs10  \r
+    xvmulsp      vs0,   vs32,   vs11\r
+    xvmulsp      vs1,   vs33,   vs11     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8  \r
+    xvmaddasp      vs0,   vs28,   vs9\r
+    xvmaddasp      vs1,   vs29,   vs9     \r
+    xvmaddasp      vs0,   vs30,   vs10\r
+    xvmaddasp      vs1,   vs31,   vs10  \r
+    xvmaddasp      vs0,   vs32,   vs11\r
+    xvmaddasp      vs1,   vs33,   vs11  \r
+ .endif\r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP4(\Index,16)  \r
+    addi        \BREG, \BREG, DISP32(\Index,128)\r
+.endif \r
+.endm\r
+\r
+.macro SAVE8x1\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC  \r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10     \r
+  add     T4, T2, T10 \r
+  add     T5, T3, T10 \r
+  add     T6, T4, T10 \r
+  add     T7, T5, T10 \r
+  /*convert alpha_r for multiply*/\r
+  xscvspdp  vs4,alpha_r\r
+/* v0 corresponds to vs32, do not forget*/\r
+#if !defined(TRMMKERNEL)\r
+  lxssp  v0,0(CO)  \r
+  lxssp  v2,0(T1) \r
+  lxssp  v4,0(T2) \r
+  lxssp  v6,0(T3) \r
+  lxssp  v8,0(T4) \r
+  lxssp  v10,0(T5) \r
+  lxssp  v12,0(T6) \r
+  lxssp  v14,0(T7)\r
+#endif\r
+  xscvspdp  vs24, vs0\r
+  xxspltw   vs25, vs0, 1 \r
+  xxspltw   vs26, vs0, 2 \r
+  xxspltw   vs27, vs0, 3  \r
+  xscvspdp  vs25,vs25\r
+  xscvspdp  vs26,vs26\r
+  xscvspdp  vs27,vs27\r
+  xscvspdp  vs28, vs1\r
+  xxspltw   vs29, vs1, 1 \r
+  xxspltw   vs30, vs1, 2 \r
+  xxspltw   vs31, vs1, 3  \r
+  xscvspdp  vs29,vs29\r
+  xscvspdp  vs30,vs30\r
+  xscvspdp  vs31,vs31\r
+#if defined(TRMMKERNEL)\r
+  xsmuldp  vs32,vs27, vs4 \r
+  xsmuldp  vs34,vs26, vs4 \r
+  xsmuldp  vs36,vs25, vs4 \r
+  xsmuldp  vs38,vs24, vs4 \r
+  xsmuldp  vs40,vs31, vs4 \r
+  xsmuldp  vs42,vs30, vs4 \r
+  xsmuldp  vs44,vs29, vs4 \r
+  xsmuldp  vs46,vs28, vs4 \r
+#else\r
+  xsmaddadp  vs32,vs27, vs4 \r
+  xsmaddadp  vs34,vs26, vs4 \r
+  xsmaddadp  vs36,vs25, vs4 \r
+  xsmaddadp  vs38,vs24, vs4 \r
+  xsmaddadp  vs40,vs31, vs4 \r
+  xsmaddadp  vs42,vs30, vs4 \r
+  xsmaddadp  vs44,vs29, vs4 \r
+  xsmaddadp  vs46,vs28, vs4  \r
+#endif  \r
+  stxssp  v0,0(CO)  \r
+  stxssp  v2,0(T1) \r
+  stxssp  v4,0(T2) \r
+  stxssp  v6,0(T3) \r
+  stxssp  v8,0(T4) \r
+  stxssp  v10,0(T5) \r
+  stxssp  v12,0(T6) \r
+  stxssp  v14,0(T7) \r
+  addi CO,CO,4\r
+.endm\r
+\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=4 and M=16\r
+**********************************************************************************************/\r
+\r
+.macro LOAD4x16_1\r
+   LOAD4x16 1\r
+.endm\r
+\r
+.macro LOAD4x16_0\r
+   LOAD4x16 0\r
+.endm\r
+\r
+.macro KERNEL4x16_L1_L4  Index,IsLast\r
+  KERNEL4x16_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I1_L4  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I1_L4_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I1_L4_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+.macro KERNEL4x16_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+   KERNEL4x16_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL4x16_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x16_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x16_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro Zero4X16\r
+    xxlxor             vs32,   vs32,   vs32\r
+    xxlxor             vs33,   vs33,   vs33\r
+       xxlxor          vs34,   vs34,   vs34\r
+       xxlxor          vs35,   vs35,   vs35\r
+       xxlxor          vs36,   vs36,   vs36\r
+       xxlxor          vs37,   vs37,   vs37\r
+       xxlxor          vs38,   vs38,   vs38\r
+       xxlxor          vs39,   vs39,   vs39\r
+       xxlxor          vs40,   vs40,   vs40\r
+       xxlxor          vs41,   vs41,   vs41\r
+       xxlxor          vs42,   vs42,   vs42\r
+       xxlxor          vs43,   vs43,   vs43\r
+       xxlxor          vs44,   vs44,   vs44\r
+       xxlxor          vs45,   vs45,   vs45\r
+       xxlxor          vs46,   vs46,   vs46\r
+       xxlxor          vs47,   vs47,   vs47    \r
+.endm\r
+\r
+.macro LOAD4x16  Zero\r
+\r
+       lxv     vs24,   0(BO) \r
+       lxv     vs0,     0(AO)\r
+       lxv     vs1,    16(AO)\r
+       lxv     vs2,    32(AO)\r
+       lxv     vs3,    48(AO)\r
+       xxperm          vs26,   vs24,           permute_mask    \r
+       xxpermdi        vs25,   vs24,   vs24,2 \r
+       xxpermdi        vs27,   vs26,   vs26,2          \r
+\r
+.if \Zero==1 \r
+    xxlxor             vs32,   vs32,   vs32\r
+    xxlxor             vs33,   vs33,   vs33\r
+       xxlxor          vs34,   vs34,   vs34\r
+       xxlxor          vs35,   vs35,   vs35\r
+       xxlxor          vs36,   vs36,   vs36\r
+       xxlxor          vs37,   vs37,   vs37\r
+       xxlxor          vs38,   vs38,   vs38\r
+       xxlxor          vs39,   vs39,   vs39\r
+       xxlxor          vs40,   vs40,   vs40\r
+       xxlxor          vs41,   vs41,   vs41\r
+       xxlxor          vs42,   vs42,   vs42\r
+       xxlxor          vs43,   vs43,   vs43\r
+       xxlxor          vs44,   vs44,   vs44\r
+       xxlxor          vs45,   vs45,   vs45\r
+       xxlxor          vs46,   vs46,   vs46\r
+       xxlxor          vs47,   vs47,   vs47\r
+ \r
+.endif\r
+.endm\r
+\r
+.macro END4x16_NORMAL\r
+  END4x16 0, AO, BO, 64,16 \r
+.endm\r
+\r
+.macro END4x16 First, AREG, BREG, OffsetA, OffsetB\r
+\r
+.if \OffsetB != 0 \r
+    addi        \BREG, \BREG, \OffsetB \r
+.endif\r
+.if \OffsetA != 0 \r
+    addi        \AREG, \AREG, \OffsetA \r
+.endif  \r
+\r
+.if \First==1\r
+    xvmulsp     vs32, vs0,vs24\r
+    xvmulsp     vs33, vs1,vs24\r
+    xvmulsp     vs34, vs2,vs24  \r
+    xvmulsp     vs35, vs3,vs24  \r
+\r
+    xvmulsp     vs36, vs0,vs25\r
+    xvmulsp     vs37, vs1,vs25\r
+    xvmulsp     vs38, vs2,vs25  \r
+    xvmulsp     vs39, vs3,vs25\r
+\r
+    xvmulsp     vs40, vs0,vs26\r
+    xvmulsp     vs41, vs1,vs26\r
+    xvmulsp     vs42, vs2,vs26  \r
+    xvmulsp     vs43, vs3,vs26\r
+\r
+    xvmulsp     vs44, vs0,vs27\r
+    xvmulsp     vs45, vs1,vs27\r
+    xvmulsp     vs46, vs2,vs27  \r
+    xvmulsp     vs47, vs3,vs27\r
+\r
+.else\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+    xvmaddasp       vs34, vs2,vs24  \r
+    xvmaddasp       vs35, vs3,vs24  \r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+    xvmaddasp       vs38, vs2,vs25  \r
+    xvmaddasp       vs39, vs3,vs25 \r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+    xvmaddasp       vs42, vs2,vs26  \r
+    xvmaddasp       vs43, vs3,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+    xvmaddasp       vs46, vs2,vs27  \r
+    xvmaddasp       vs47, vs3,vs27\r
+\r
+.endif\r
+.endm  \r
+\r
+.macro KERNEL4x16_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+       lxv     vs8,    DISP16(\Index, 0+\OffsetB)(\BREG) \r
+\r
+       lxv     vs4,    DISP64(\Index, 0+\OffsetA)(\AREG)\r
+       lxv     vs5,    DISP64(\Index,16+\OffsetA)(\AREG)\r
+       lxv     vs6,    DISP64(\Index,32+\OffsetA)(\AREG)\r
+       lxv     vs7,    DISP64(\Index,48+\OffsetA)(\AREG) \r
+\r
+       xxperm          vs10,   vs8,            permute_mask \r
+       xxpermdi        vs9,    vs8,    vs8,2     \r
+\r
+    xvmaddasp          vs32, vs0,vs24\r
+       xvmaddasp               vs33, vs1,vs24\r
+       xvmaddasp               vs34, vs2,vs24  \r
+       xvmaddasp               vs35, vs3,vs24   \r
+\r
+    xvmaddasp          vs36, vs0,vs25\r
+       xvmaddasp               vs37, vs1,vs25\r
+       xvmaddasp               vs38, vs2,vs25  \r
+       xvmaddasp               vs39, vs3,vs25 \r
+\r
+       xxpermdi        vs11,   vs10,   vs10,2   \r
+\r
+    xvmaddasp          vs40, vs0,vs26\r
+       xvmaddasp               vs41, vs1,vs26\r
+       xvmaddasp               vs42, vs2,vs26  \r
+       xvmaddasp               vs43, vs3,vs26\r
+\r
+    xvmaddasp          vs44, vs0,vs27\r
+       xvmaddasp               vs45, vs1,vs27\r
+       xvmaddasp               vs46, vs2,vs27  \r
+       xvmaddasp               vs47, vs3,vs27\r
+\r
+\r
+\r
+       lxv     vs24,   DISP16(\Index,16+\OffsetB)(\BREG) \r
+\r
+       lxv     vs0,    DISP64(\Index,64+\OffsetA)(\AREG)\r
+       lxv     vs1,    DISP64(\Index,64+16+\OffsetA)(\AREG)\r
+       lxv     vs2,    DISP64(\Index,64+32+\OffsetA)(\AREG)\r
+       lxv     vs3,    DISP64(\Index,64+48+\OffsetA)(\AREG)\r
+\r
+       xxperm          vs26,   vs24,           permute_mask \r
+       xxpermdi        vs25,   vs24,   vs24,2      \r
+ \r
+\r
+    xvmaddasp          vs32, vs4,vs8\r
+       xvmaddasp               vs33, vs5,vs8\r
+       xvmaddasp               vs34, vs6,vs8   \r
+       xvmaddasp               vs35, vs7,vs8   \r
+ \r
+    xvmaddasp          vs36, vs4,vs9\r
+       xvmaddasp               vs37, vs5,vs9\r
+       xvmaddasp               vs38, vs6,vs9   \r
+       xvmaddasp               vs39, vs7,vs9\r
+         \r
+       xxpermdi        vs27,   vs26,   vs26,2          \r
+\r
+    xvmaddasp          vs40, vs4,vs10\r
+       xvmaddasp               vs41, vs5,vs10\r
+       xvmaddasp               vs42, vs6,vs10  \r
+       xvmaddasp               vs43, vs7,vs10\r
+\r
+    xvmaddasp          vs44, vs4,vs11\r
+       xvmaddasp               vs45, vs5,vs11\r
+       xvmaddasp               vs46, vs6,vs11  \r
+       xvmaddasp               vs47, vs7,vs11\r
+ \r
+\r
+       lxv     vs8,    DISP16(\Index,32+\OffsetB)(\BREG) \r
+\r
+       lxv     vs4,    DISP64(\Index,128+0+\OffsetA)(\AREG)\r
+       lxv     vs5,    DISP64(\Index,128+16+\OffsetA)(\AREG)\r
+       lxv     vs6,    DISP64(\Index,128+32+\OffsetA)(\AREG)\r
+       lxv     vs7,    DISP64(\Index,128+48+\OffsetA)(\AREG) \r
+\r
+       xxperm          vs10,   vs8,            permute_mask \r
+       xxpermdi        vs9,    vs8,    vs8,2     \r
+\r
+    xvmaddasp          vs32, vs0,vs24\r
+       xvmaddasp               vs33, vs1,vs24\r
+       xvmaddasp               vs34, vs2,vs24  \r
+       xvmaddasp               vs35, vs3,vs24   \r
+\r
+    xvmaddasp          vs36, vs0,vs25\r
+       xvmaddasp               vs37, vs1,vs25\r
+       xvmaddasp               vs38, vs2,vs25  \r
+       xvmaddasp               vs39, vs3,vs25\r
+\r
+       xxpermdi        vs11,   vs10,   vs10,2   \r
+\r
+    xvmaddasp          vs40, vs0,vs26\r
+       xvmaddasp               vs41, vs1,vs26\r
+       xvmaddasp               vs42, vs2,vs26  \r
+       xvmaddasp               vs43, vs3,vs26\r
+\r
+    xvmaddasp          vs44, vs0,vs27\r
+       xvmaddasp               vs45, vs1,vs27\r
+       xvmaddasp               vs46, vs2,vs27  \r
+       xvmaddasp               vs47, vs3,vs27\r
+\r
+ \r
+ \r
+.if \Complete==0\r
+       lxv     vs24,   DISP16(\Index,48+\OffsetB)(\BREG) \r
+\r
+       lxv     vs0,    DISP64(\Index,192+\OffsetA)(\AREG)\r
+       lxv     vs1,    DISP64(\Index,192+16+\OffsetA)(\AREG) \r
+       lxv     vs2,    DISP64(\Index,192+32+\OffsetA)(\AREG)\r
+       lxv     vs3,    DISP64(\Index,192+48+\OffsetA)(\AREG)\r
+\r
+       xxperm          vs26,   vs24,   permute_mask    \r
+       xxpermdi        vs25,   vs24,   vs24,2          \r
+\r
+.endif \r
+.if \IsLast==1 \r
+.if \Complete==1\r
+  \r
+       addi            \BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)\r
+       addi            \AREG, \AREG, DISP64(\Index,64*3+\OffsetA)\r
+.else\r
+  \r
+       addi            \BREG, \BREG,  DISP16(\Index,64)\r
+       addi            \AREG, \AREG, DISP64(\Index,256)\r
+.endif\r
+.endif   \r
+ \r
+    xvmaddasp          vs32, vs4,vs8\r
+       xvmaddasp               vs33, vs5,vs8\r
+       xvmaddasp               vs34, vs6,vs8   \r
+       xvmaddasp               vs35, vs7,vs8    \r
+ \r
+    xvmaddasp          vs36, vs4,vs9\r
+       xvmaddasp               vs37, vs5,vs9\r
+       xvmaddasp               vs38, vs6,vs9   \r
+       xvmaddasp               vs39, vs7,vs9\r
+  \r
+.if \Complete==0        \r
+       xxpermdi        vs27,   vs26,   vs26,2   \r
+       \r
+.endif\r
+ \r
+    xvmaddasp          vs40, vs4,vs10\r
+       xvmaddasp               vs41, vs5,vs10\r
+       xvmaddasp               vs42, vs6,vs10  \r
+       xvmaddasp               vs43, vs7,vs10\r
+\r
+    xvmaddasp          vs44, vs4,vs11\r
+       xvmaddasp               vs45, vs5,vs11\r
+       xvmaddasp               vs46, vs6,vs11  \r
+       xvmaddasp               vs47, vs7,vs11\r
+\r
+ \r
+\r
+.endm\r
+\r
+.macro KERNEL4x16 First\r
+\r
+  LOAD4x16 0\r
+  END4x16 \First, AO, BO, 64,16 \r
+.endm\r
+\r
+.macro KERNEL4x16_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete\r
+       \r
+       lxv     vs8,    DISP8(\Index, 0+\OffsetB)(\BREG) \r
+       lxv     vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)\r
+       lxv     vs5,    DISP32(\Index,16+\OffsetA)(\AREG)\r
+       lxv     vs6,    DISP32(\Index,32+\OffsetA)(\AREG)\r
+       lxv     vs7,    DISP32(\Index,48+\OffsetA)(\AREG) \r
+\r
+       xxperm          vs10,   vs8,            permute_mask \r
+       xxpermdi        vs9,    vs8,    vs8,2     \r
+.if \First==1\r
+    xvmulsp            vs32, vs0,vs24\r
+       xvmulsp         vs33, vs1,vs24\r
+       xvmulsp         vs34, vs2,vs24  \r
+       xvmulsp         vs35, vs3,vs24  \r
+\r
+    xvmulsp            vs36, vs0,vs25\r
+       xvmulsp         vs37, vs1,vs25\r
+       xvmulsp         vs38, vs2,vs25  \r
+       xvmulsp         vs39, vs3,vs25  \r
+.else\r
+    xvmaddasp          vs32, vs0,vs24\r
+       xvmaddasp               vs33, vs1,vs24\r
+       xvmaddasp               vs34, vs2,vs24  \r
+       xvmaddasp               vs35, vs3,vs24\r
+\r
+    xvmaddasp          vs36, vs0,vs25\r
+       xvmaddasp               vs37, vs1,vs25\r
+       xvmaddasp               vs38, vs2,vs25  \r
+       xvmaddasp               vs39, vs3,vs25          \r
+.endif\r
+\r
+       xxpermdi        vs11,   vs10,   vs10,2          \r
+ \r
+.if \First==1  \r
+    xvmulsp            vs40, vs0,vs26\r
+       xvmulsp         vs41, vs1,vs26\r
+       xvmulsp         vs42, vs2,vs26  \r
+       xvmulsp         vs43, vs3,vs26\r
+\r
+    xvmulsp            vs44, vs0,vs27\r
+       xvmulsp         vs45, vs1,vs27\r
+       xvmulsp         vs46, vs2,vs27  \r
+       xvmulsp         vs47, vs3,vs27\r
+\r
+  \r
+.else \r
+    xvmaddasp          vs40, vs0,vs26\r
+       xvmaddasp               vs41, vs1,vs26\r
+       xvmaddasp               vs42, vs2,vs26  \r
+       xvmaddasp               vs43, vs3,vs26\r
+\r
+    xvmaddasp          vs44, vs0,vs27\r
+       xvmaddasp               vs45, vs1,vs27\r
+       xvmaddasp               vs46, vs2,vs27  \r
+       xvmaddasp               vs47, vs3,vs27\r
+ \r
+\r
+.endif\r
+.if \Complete==0\r
+       lxv     vs24,   DISP8(\Index,16+\OffsetB)(\BREG) \r
+       lxv     vs0,    DISP32(\Index,64+\OffsetA)(\AREG)\r
+       lxv     vs1,    DISP32(\Index,64+16+\OffsetA)(\AREG)\r
+       lxv     vs2,    DISP32(\Index,64+32+\OffsetA)(\AREG)\r
+       lxv     vs3,    DISP32(\Index,64+48+\OffsetA)(\AREG)\r
+\r
+       xxperm          vs26,   vs24,   permute_mask \r
+       xxpermdi        vs25,   vs24,   vs24,2    \r
+.endif    \r
+.if \IsLast==1 \r
+.if \Complete==1\r
+       addi            \BREG, \BREG,  DISP8(\Index,16+\OffsetB) \r
+       addi            \AREG, \AREG, DISP32(\Index,64+\OffsetA)\r
+\r
+.else\r
+       addi            \BREG, \BREG,  DISP8(\Index,32)\r
+       addi            \AREG, \AREG, DISP32(\Index,128) \r
+.endif\r
+.endif\r
+\r
+.if \First==1\r
+    xvmulsp            vs32, vs4,vs8\r
+       xvmulsp         vs33, vs5,vs8\r
+       xvmulsp         vs34, vs6,vs8   \r
+       xvmulsp         vs35, vs7,vs8\r
+\r
+    xvmulsp            vs36, vs4,vs9\r
+       xvmulsp         vs37, vs5,vs9\r
+       xvmulsp         vs38, vs6,vs9   \r
+       xvmulsp         vs39, vs7,vs9\r
+.else\r
+    xvmaddasp          vs32, vs4,vs8\r
+       xvmaddasp               vs33, vs5,vs8\r
+       xvmaddasp               vs34, vs6,vs8   \r
+       xvmaddasp               vs35, vs7,vs8   \r
+\r
+    xvmaddasp          vs36, vs4,vs9\r
+       xvmaddasp               vs37, vs5,vs9\r
+       xvmaddasp               vs38, vs6,vs9   \r
+       xvmaddasp               vs39, vs7,vs9\r
+.endif \r
+ \r
+.if \Complete==0        \r
+       xxpermdi        vs27,   vs26,   vs26,2   \r
+ \r
+.endif\r
+.if \First==1  \r
+    xvmulsp            vs40, vs4,vs10\r
+       xvmulsp         vs41, vs5,vs10\r
+       xvmulsp         vs42, vs6,vs10  \r
+       xvmulsp         vs43, vs7,vs10\r
+\r
+    xvmulsp            vs44, vs4,vs11\r
+       xvmulsp         vs45, vs5,vs11\r
+       xvmulsp         vs46, vs6,vs11  \r
+       xvmulsp         vs47, vs7,vs11\r
+\r
+ \r
+\r
+.else \r
+    xvmaddasp          vs40, vs4,vs10\r
+       xvmaddasp               vs41, vs5,vs10\r
+       xvmaddasp               vs42, vs6,vs10  \r
+       xvmaddasp               vs43, vs7,vs10\r
+\r
+    xvmaddasp          vs44, vs4,vs11\r
+       xvmaddasp               vs45, vs5,vs11\r
+       xvmaddasp               vs46, vs6,vs11  \r
+       xvmaddasp               vs47, vs7,vs11\r
+\r
+ \r
+\r
+.endif\r
+\r
+.endm\r
+\r
+ \r
+.macro SAVE4x16\r
+\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC \r
+\r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10  \r
+\r
+  \r
+ \r
+    xxmrglw     vs8,    vs32,   vs44\r
+    xxmrglw     vs10,   vs36,   vs40  \r
+\r
+    xxmrghw     vs1,    vs32,   vs44\r
+    xxmrghw     vs0,    vs36,   vs40\r
+\r
+    xxmrglw     vs12,   vs33,   vs45\r
+    xxmrglw     vs14,   vs37,   vs41  \r
+\r
+    xxmrghw     vs2,    vs37,   vs41\r
+    xxmrghw     vs3,    vs33,   vs45\r
+\r
+    xxmrglw     vs16,   vs34,   vs46\r
+    xxmrglw     vs18,   vs38,   vs42   \r
+\r
+    xxlor      vs9, vs8,    vs8\r
+    xxlor      vs11,    vs10,   vs10 \r
+\r
+    xxmrghw     vs4,    vs38,   vs42\r
+    xxmrghw     vs5,    vs34,   vs46\r
+\r
+    xxlor      vs13,    vs12,   vs12\r
+    xxlor      vs15,    vs14,   vs14\r
+\r
+    xxmrglw     vs24,   vs35,   vs47\r
+    xxmrglw     vs26,   vs39,   vs43  \r
+\r
+    xxlor      vs17,    vs16,   vs16\r
+    xxlor      vs19,    vs18,   vs18\r
+\r
+    xxmrghw     vs30,   vs39,   vs43 \r
+    xxmrghw     vs31,   vs35,   vs47\r
+\r
+    xxperm      vs8,    vs0,    save_permute_1\r
+    xxperm      vs10,   vs1,    save_permute_1\r
+    xxperm      vs9,    vs0,    save_permute_2  \r
+    xxperm      vs11,   vs1,    save_permute_2      \r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs32, 0(CO)\r
+    lxv        vs33, 16(CO) \r
+    lxv        vs34, 32(CO)  \r
+    lxv        vs35, 48(CO)      \r
+#endif\r
+    xxlor      vs25,    vs24,   vs24\r
+    xxlor      vs27,    vs26,   vs26 \r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs36, 0(T1)\r
+    lxv        vs37, 16(T1) \r
+    lxv        vs38, 32(T1)  \r
+    lxv        vs39, 48(T1)     \r
+#endif\r
+#ifndef TRMMKERNEL       \r
+    lxv        vs40, 0(T2)\r
+    lxv        vs41, 16(T2) \r
+    lxv        vs42, 32(T2)  \r
+    lxv        vs43, 48(T2)     \r
+#endif  \r
+#ifndef TRMMKERNEL    \r
+    lxv        vs44, 0(T3)\r
+    lxv        vs45, 16(T3) \r
+    lxv        vs46, 32(T3)  \r
+    lxv        vs47, 48(T3)                 \r
+#endif  \r
+\r
+    xxperm     vs12,    vs2,    save_permute_1\r
+    xxperm     vs14,    vs3,    save_permute_1\r
+       \r
+    xxperm     vs13,    vs2,    save_permute_2   \r
+    xxperm     vs15,    vs3,    save_permute_2      \r
+\r
+    xxperm     vs16,    vs4,    save_permute_1\r
+    xxperm     vs18,    vs5,    save_permute_1\r
+      \r
+    xxperm     vs17,    vs4,    save_permute_2   \r
+    xxperm     vs19,    vs5,    save_permute_2      \r
+\r
+    xxperm     vs24,    vs30,   save_permute_1\r
+    xxperm     vs26,    vs31,   save_permute_1 \r
+         \r
+    xxperm     vs25,    vs30,   save_permute_2   \r
+    xxperm     vs27,    vs31,   save_permute_2  \r
+\r
+\r
+    /* multiply add normal way */\r
+ \r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs32,   vs8,    alpha_r \r
+    xvmulsp     vs33,   vs12,   alpha_r   \r
+    xvmulsp     vs34,   vs16,   alpha_r \r
+    xvmulsp     vs35,   vs24,   alpha_r  \r
+    xvmulsp     vs36,   vs9,    alpha_r \r
+    xvmulsp     vs37,   vs13,   alpha_r  \r
+    xvmulsp     vs38,   vs17,   alpha_r \r
+    xvmulsp     vs39,   vs25,   alpha_r               \r
+#else \r
+    xvmaddasp   vs32,   vs8,    alpha_r \r
+    xvmaddasp   vs33,   vs12,   alpha_r   \r
+    xvmaddasp   vs34,   vs16,   alpha_r \r
+    xvmaddasp   vs35,   vs24,   alpha_r  \r
+    xvmaddasp   vs36,   vs9,    alpha_r \r
+    xvmaddasp   vs37,   vs13,   alpha_r   \r
+    xvmaddasp   vs38,   vs17,   alpha_r \r
+    xvmaddasp   vs39,   vs25,   alpha_r         \r
+#endif \r
+\r
+\r
+\r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs40,   vs10,   alpha_r \r
+    xvmulsp     vs41,   vs14,   alpha_r \r
+    xvmulsp     vs42,   vs18,   alpha_r \r
+    xvmulsp     vs43,   vs26,   alpha_r  \r
+    xvmulsp     vs44,   vs11,   alpha_r \r
+    xvmulsp     vs45,   vs15,   alpha_r  \r
+    xvmulsp     vs46,   vs19,   alpha_r \r
+    xvmulsp     vs47,   vs27,   alpha_r                   \r
+#else\r
+\r
+    xvmaddasp   vs40,   vs10,   alpha_r \r
+    xvmaddasp   vs41,   vs14,   alpha_r   \r
+    xvmaddasp   vs42,   vs18,   alpha_r \r
+    xvmaddasp   vs43,   vs26,   alpha_r  \r
+    xvmaddasp   vs44,   vs11,   alpha_r \r
+    xvmaddasp   vs45,   vs15,   alpha_r \r
+    xvmaddasp   vs46,   vs19,   alpha_r \r
+    xvmaddasp   vs47,   vs27,   alpha_r  \r
+        \r
+#endif  \r
+\r
+    stxv        vs32, 0(CO)\r
+    stxv        vs33, 16(CO) \r
+    stxv        vs34, 32(CO)  \r
+    stxv        vs35, 48(CO)  \r
+\r
+    stxv        vs36, 0(T1)\r
+    stxv        vs37, 16(T1)  \r
+    stxv        vs38, 32(T1)  \r
+    stxv        vs39, 48(T1)\r
+\r
+    stxv        vs40, 0(T2)\r
+    stxv        vs41, 16(T2)  \r
+    stxv        vs42, 32(T2)  \r
+    stxv        vs43, 48(T2)  \r
+    stxv        vs44, 0(T3)\r
+    stxv        vs45, 16(T3) \r
+    stxv        vs46, 32(T3)  \r
+    stxv        vs47, 48(T3)\r
+   \r
+    addi CO,CO,64\r
+\r
+\r
+.endm\r
+\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=4 and M=8\r
+**********************************************************************************************/\r
+\r
+.macro LOAD4x8_1\r
+   LOAD4x8 1\r
+.endm\r
+\r
+.macro LOAD4x8_0\r
+   LOAD4x8 0\r
+.endm\r
+\r
+.macro KERNEL4x8_L1_L4  Index,IsLast\r
+  KERNEL4x8_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x8_I1_L4  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x8_I1_L4_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x8_I1_L4_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+.macro KERNEL4x8_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+   KERNEL4x8_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL4x8_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x8_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x8_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro END4x8_NORMAL\r
+  END4x8 0, AO, BO, 32,16 \r
+.endm\r
+\r
+.macro Zero4X8\r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33\r
+ \r
+    xxlxor      vs36,   vs36,   vs36\r
+    xxlxor      vs37,   vs37,   vs37\r
+ \r
+    xxlxor      vs40,   vs40,   vs40\r
+    xxlxor      vs41,   vs41,   vs41\r
+ \r
+    xxlxor      vs44,   vs44,   vs44\r
+    xxlxor      vs45,   vs45,   vs45\r
+    \r
+.endm\r
+\r
+.macro LOAD4x8  Zero\r
+\r
+    lxv vs24,   0(BO) \r
+    lxv vs0,     0(AO)\r
+    lxv vs1,    16(AO)\r
+\r
+    xxperm      vs26,   vs24,       permute_mask    \r
+    xxpermdi    vs25,   vs24,   vs24,2      \r
+\r
+    xxpermdi    vs27,   vs26,   vs26,2      \r
+\r
+.if \Zero==1 \r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33 \r
+    xxlxor      vs36,   vs36,   vs36\r
+    xxlxor      vs37,   vs37,   vs37\r
+    xxlxor      vs40,   vs40,   vs40\r
+    xxlxor      vs41,   vs41,   vs41 \r
+    xxlxor      vs44,   vs44,   vs44\r
+    xxlxor      vs45,   vs45,   vs45 \r
+ \r
+.endif\r
+.endm\r
+\r
+\r
+.macro END4x8 First, AREG, BREG, OffsetA, OffsetB\r
+\r
+.if \OffsetB != 0 \r
+    addi        \BREG, \BREG, \OffsetB \r
+.endif\r
+.if \OffsetA != 0 \r
+    addi        \AREG, \AREG, \OffsetA \r
+.endif  \r
+\r
+.if \First==1\r
+    xvmulsp     vs32, vs0,vs24\r
+    xvmulsp     vs33, vs1,vs24\r
+\r
+    xvmulsp     vs36, vs0,vs25\r
+    xvmulsp     vs37, vs1,vs25\r
+\r
+    xvmulsp     vs40, vs0,vs26\r
+    xvmulsp     vs41, vs1,vs26\r
+\r
+    xvmulsp     vs44, vs0,vs27\r
+    xvmulsp     vs45, vs1,vs27\r
+ \r
+\r
+.else\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+ \r
+\r
+.endif\r
+.endm  \r
+\r
+.macro KERNEL4x8_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+    lxv vs8,    DISP16(\Index, 0+\OffsetB)(\BREG) \r
+\r
+    lxv vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs5,    DISP32(\Index,16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs10,   vs8,        permute_mask    \r
+    xxpermdi    vs9,    vs8,    vs8,2     \r
+\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+    xxpermdi    vs11,   vs10,   vs10,2   \r
+\r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+\r
+ \r
+\r
+    lxv vs24,   DISP16(\Index,16+\OffsetB)(\BREG) \r
+\r
+    lxv vs0,    DISP32(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs1,    DISP32(\Index,32+16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs26,   vs24,       permute_mask   \r
+    xxpermdi    vs25,   vs24,   vs24,2      \r
+\r
+    xvmaddasp       vs32, vs4,vs8\r
+    xvmaddasp       vs33, vs5,vs8\r
+\r
+    xvmaddasp       vs36, vs4,vs9\r
+    xvmaddasp       vs37, vs5,vs9\r
+\r
+    xxpermdi    vs27,   vs26,   vs26,2       \r
+\r
+    xvmaddasp       vs40, vs4,vs10\r
+    xvmaddasp       vs41, vs5,vs10\r
+\r
+    xvmaddasp       vs44, vs4,vs11\r
+    xvmaddasp       vs45, vs5,vs11\r
+\r
+ \r
+\r
+    lxv vs8,    DISP16(\Index,32+\OffsetB)(\BREG) \r
+\r
+    lxv vs4,    DISP32(\Index,64+0+\OffsetA)(\AREG)\r
+    lxv vs5,    DISP32(\Index,64+16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs10,   vs8,        permute_mask     \r
+    xxpermdi    vs9,    vs8,    vs8,2     \r
+\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+    xxpermdi    vs11,   vs10,   vs10,2   \r
+\r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+\r
+ \r
+\r
+.if \Complete==0\r
+    lxv vs24,   DISP16(\Index,48+\OffsetB)(\BREG) \r
+\r
+    lxv vs0,    DISP32(\Index,96+\OffsetA)(\AREG)\r
+    lxv vs1,    DISP32(\Index,96+16+\OffsetA)(\AREG) \r
+\r
+    xxperm      vs26,   vs24,   permute_mask     \r
+    xxpermdi    vs25,   vs24,   vs24,2      \r
+\r
+.endif \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+  \r
+    addi        \BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)\r
+    addi        \AREG, \AREG, DISP32(\Index,32*3+\OffsetA)\r
+.else\r
+  \r
+    addi        \BREG, \BREG,  DISP16(\Index,64)\r
+    addi        \AREG, \AREG, DISP32(\Index,128)\r
+.endif\r
+.endif   \r
+ \r
+    xvmaddasp       vs32, vs4,vs8\r
+    xvmaddasp       vs33, vs5,vs8\r
+\r
+    xvmaddasp       vs36, vs4,vs9\r
+    xvmaddasp       vs37, vs5,vs9\r
+\r
+.if \Complete==0        \r
+    xxpermdi    vs27,   vs26,   vs26,2    \r
+    \r
+.endif\r
+ \r
+    xvmaddasp       vs40, vs4,vs10\r
+    xvmaddasp       vs41, vs5,vs10\r
+\r
+    xvmaddasp       vs44, vs4,vs11\r
+    xvmaddasp       vs45, vs5,vs11\r
+\r
+ \r
+\r
+.endm\r
+\r
+.macro KERNEL4x8 First\r
+\r
+  LOAD4x8 0\r
+  END4x8 \First, AO, BO, 32,16  \r
+.endm\r
+\r
+.macro KERNEL4x8_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete\r
+    \r
+    lxv vs8,     DISP8(\Index, 0+\OffsetB)(\BREG) \r
+    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs5,    DISP16(\Index,16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs10,   vs8,        permute_mask  \r
+    xxpermdi    vs9,    vs8,    vs8,2     \r
+.if \First==1\r
+    xvmulsp     vs32, vs0,vs24\r
+    xvmulsp     vs33, vs1,vs24\r
+\r
+    xvmulsp     vs36, vs0,vs25\r
+    xvmulsp     vs37, vs1,vs25\r
+\r
+.else\r
+    xvmaddasp       vs32, vs0,vs24\r
+    xvmaddasp       vs33, vs1,vs24\r
+\r
+    xvmaddasp       vs36, vs0,vs25\r
+    xvmaddasp       vs37, vs1,vs25\r
+\r
+.endif\r
+\r
+    xxpermdi    vs11,   vs10,   vs10,2    \r
+ \r
+.if \First==1  \r
+    xvmulsp     vs40, vs0,vs26\r
+    xvmulsp     vs41, vs1,vs26\r
+\r
+    xvmulsp     vs44, vs0,vs27\r
+    xvmulsp     vs45, vs1,vs27\r
+ \r
+\r
+.else \r
+    xvmaddasp       vs40, vs0,vs26\r
+    xvmaddasp       vs41, vs1,vs26\r
+\r
+    xvmaddasp       vs44, vs0,vs27\r
+    xvmaddasp       vs45, vs1,vs27\r
+ \r
+\r
+.endif\r
+.if \Complete==0\r
+    lxv vs24,    DISP8(\Index,16+\OffsetB)(\BREG) \r
+\r
+    lxv vs0,    DISP16(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs1,    DISP16(\Index,32+16+\OffsetA)(\AREG)\r
+\r
+    xxperm      vs26,   vs24,   permute_mask   \r
+    xxpermdi    vs25,   vs24,   vs24,2    \r
+.endif    \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+    addi        \BREG, \BREG,   DISP8(\Index,16+\OffsetB) \r
+    addi        \AREG, \AREG,  DISP16(\Index,32+\OffsetA)\r
+\r
+.else\r
+    addi        \BREG, \BREG,   DISP8(\Index,32)\r
+    addi        \AREG, \AREG,  DISP16(\Index,64) \r
+.endif\r
+.endif\r
+\r
+.if \First==1\r
+    xvmulsp     vs32, vs4,vs8\r
+    xvmulsp     vs33, vs5,vs8\r
+\r
+    xvmulsp     vs36, vs4,vs9\r
+    xvmulsp     vs37, vs5,vs9\r
+\r
+.else\r
+    xvmaddasp       vs32, vs4,vs8\r
+    xvmaddasp       vs33, vs5,vs8\r
+\r
+    xvmaddasp       vs36, vs4,vs9\r
+    xvmaddasp       vs37, vs5,vs9\r
+\r
+.endif \r
+ \r
+.if \Complete==0        \r
+    xxpermdi    vs27,   vs26,   vs26,2   \r
+ \r
+.endif\r
+.if \First==1  \r
+    xvmulsp     vs40, vs4,vs10\r
+    xvmulsp     vs41, vs5,vs10\r
+\r
+    xvmulsp     vs44, vs4,vs11\r
+    xvmulsp     vs45, vs5,vs11\r
+ \r
+.else \r
+    xvmaddasp       vs40, vs4,vs10\r
+    xvmaddasp       vs41, vs5,vs10\r
+\r
+    xvmaddasp       vs44, vs4,vs11\r
+    xvmaddasp       vs45, vs5,vs11 \r
+\r
+.endif\r
+\r
+.endm\r
+\r
+\r
+.macro SAVE4x8 \r
+ \r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC \r
+\r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10  \r
+\r
+ \r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs34, 0(CO)\r
+    lxv        vs35, 16(CO)      \r
+    lxv        vs38, 0(T1)\r
+    lxv        vs39, 16(T1)  \r
+    lxv        vs42, 0(T2)\r
+    lxv        vs43, 16(T2)     \r
+    lxv        vs46, 0(T3)\r
+    lxv        vs47, 16(T3)  \r
+\r
+ \r
+#endif  \r
+\r
+    xxmrglw     vs8,    vs32,   vs44\r
+    xxmrglw     vs10,   vs36,   vs40  \r
+\r
+    xxmrghw     vs1,    vs32,   vs44\r
+    xxmrghw     vs0,    vs36,   vs40\r
+\r
+    xxmrglw     vs12,   vs33,   vs45\r
+    xxmrglw     vs14,   vs37,   vs41  \r
+\r
+    xxmrghw     vs2,    vs37,   vs41\r
+    xxmrghw     vs3,    vs33,   vs45\r
+\r
+    xxlor      vs9, vs8,    vs8\r
+    xxlor      vs11,    vs10,   vs10 \r
+ \r
+    xxlor      vs13,    vs12,   vs12\r
+    xxlor      vs15,    vs14,   vs14\r
+\r
+    xxperm      vs8,    vs0,    save_permute_1\r
+    xxperm      vs10,   vs1,    save_permute_1\r
+    xxperm      vs9,    vs0,    save_permute_2  \r
+    xxperm      vs11,   vs1,    save_permute_2      \r
+\r
+    xxperm     vs12,    vs2,    save_permute_1\r
+    xxperm     vs14,    vs3,    save_permute_1\r
+      \r
+    xxperm     vs13,    vs2,    save_permute_2   \r
+    xxperm     vs15,    vs3,    save_permute_2      \r
+\r
+\r
+    /* multiply add normal way */\r
+ \r
+#ifdef TRMMKERNEL\r
+    xvmulsp     vs34,   vs8,    alpha_r \r
+    xvmulsp     vs35,   vs12,   alpha_r \r
+    xvmulsp     vs38,   vs9,    alpha_r \r
+    xvmulsp     vs39,   vs13,   alpha_r \r
+    xvmulsp     vs42,   vs10,   alpha_r \r
+    xvmulsp     vs43,   vs14,   alpha_r \r
+    xvmulsp     vs46,   vs11,   alpha_r \r
+    xvmulsp     vs47,   vs15,   alpha_r                    \r
+#else \r
+    xvmaddasp   vs34,   vs8,    alpha_r \r
+    xvmaddasp   vs35,   vs12,   alpha_r \r
+    xvmaddasp   vs38,   vs9,    alpha_r \r
+    xvmaddasp   vs39,   vs13,   alpha_r  \r
+    xvmaddasp   vs42,   vs10,   alpha_r \r
+    xvmaddasp   vs43,   vs14,   alpha_r   \r
+    xvmaddasp   vs46,   vs11,   alpha_r \r
+    xvmaddasp   vs47,   vs15,   alpha_r                     \r
+#endif     \r
+ \r
+    \r
+    stxv        vs34, 0(CO)\r
+    stxv        vs35, 16(CO)  \r
+    stxv        vs38, 0(T1)\r
+    stxv        vs39, 16(T1)  \r
+    stxv        vs42, 0(T2)\r
+    stxv        vs43, 16(T2)     \r
+    stxv        vs46, 0(T3)\r
+    stxv        vs47, 16(T3)  \r
+  \r
+\r
+    addi CO,CO,32\r
+\r
+.endm\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=4 and M=4\r
+**********************************************************************************************/\r
+\r
+.macro LOAD4x4_1\r
+   LOAD4x4 1\r
+.endm\r
+\r
+.macro LOAD4x4_0\r
+   LOAD4x4 0\r
+.endm\r
+\r
+.macro KERNEL4x4_L1_L4  Index,IsLast\r
+  KERNEL4x4_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x4_I1_L4  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x4_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x4_I1_L4_2  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x4_I1_L4_3  OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+.macro KERNEL4x4_I1_L2_3  OffsetA,OffsetB, Index,IsLast\r
+   KERNEL4x4_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro KERNEL4x4_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x4_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0\r
+.endm\r
+\r
+.macro KERNEL4x4_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x4_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1\r
+.endm\r
+\r
+.macro Zero4X4\r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33 \r
+    xxlxor      vs34,   vs34,   vs34\r
+    xxlxor      vs35,   vs35,   vs35\r
+ \r
+.endm\r
+\r
+.macro LOAD4x4  Zero\r
+\r
+    lxv vs0,     0(AO)\r
+    lxv vs24,   0(BO) \r
+\r
+\r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2        \r
+\r
+.if \Zero==1 \r
+    xxlxor      vs32,   vs32,   vs32\r
+    xxlxor      vs33,   vs33,   vs33 \r
+    xxlxor      vs34,   vs34,   vs34\r
+    xxlxor      vs35,   vs35,   vs35\r
+ \r
+.endif\r
+.endm\r
+\r
+.macro END4x4_NORMAL\r
+  END4x4 0, AO, BO, 16,16 \r
+.endm\r
+\r
+.macro END4x4 First, AREG, BREG, OffsetA, OffsetB\r
+\r
+.if \OffsetB != 0 \r
+    addi        \BREG, \BREG, \OffsetB \r
+.endif\r
+.if \OffsetA != 0 \r
+    addi        \AREG, \AREG, \OffsetA \r
+.endif  \r
+\r
+.if \First==1\r
+    xvmulsp      vs32,   vs24,   vs0\r
+    xvmulsp      vs33,   vs24,   vs1 \r
+    xvmulsp      vs34,   vs24,   vs2\r
+    xvmulsp      vs35,   vs24,   vs3  \r
+.else\r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+ \r
+\r
+.endif\r
+.endm  \r
+\r
+.macro KERNEL4x4_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs6,   vs4,       permute_mask  \r
+    xxpermdi    vs5,   vs4,   vs4,2      \r
+    xxpermdi    vs7,   vs6,   vs6,2 \r
+ \r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+ \r
+\r
+    lxv vs0,    DISP16(\Index, 16+\OffsetA)(\AREG)\r
+    lxv vs24,   DISP16(\Index, 16+\OffsetB)(\BREG)  \r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2   \r
+\r
+    xvmaddasp      vs32,   vs26,   vs4\r
+    xvmaddasp      vs33,   vs26,   vs5 \r
+    xvmaddasp      vs34,   vs26,   vs6\r
+    xvmaddasp      vs35,   vs26,   vs7\r
+ \r
+ \r
+\r
+    lxv vs4,    DISP16(\Index, 32+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP16(\Index, 32+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs6,   vs4,       permute_mask  \r
+    xxpermdi    vs5,   vs4,   vs4,2      \r
+    xxpermdi    vs7,   vs6,   vs6,2 \r
+ \r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+ \r
+\r
+.if \Complete==0 \r
+\r
+    lxv vs0,    DISP16(\Index, 48+\OffsetA)(\AREG)\r
+    lxv vs24,   DISP16(\Index, 48+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2   \r
+.endif\r
+    xvmaddasp      vs32,   vs26,   vs4\r
+    xvmaddasp      vs33,   vs26,   vs5 \r
+    xvmaddasp      vs34,   vs26,   vs6\r
+    xvmaddasp      vs35,   vs26,   vs7\r
+ \r
+\r
+ \r
+ \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+    addi        \AREG, \AREG, DISP16(\Index,16*3+\OffsetA)  \r
+    addi        \BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)\r
+\r
+.else\r
+    addi        \AREG, \AREG, DISP16(\Index,64)  \r
+    addi        \BREG, \BREG,  DISP16(\Index,64)\r
+\r
+.endif\r
+.endif   \r
+ \r
+ \r
+.endm\r
+\r
+.macro KERNEL4x4 First\r
+    LOAD4x4 0\r
+    END4x4 \First, AO, BO, 16,16  \r
+.endm\r
+\r
+.macro KERNEL4x4_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete\r
+\r
+    lxv vs4,    DISP8(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs6,   vs4,       permute_mask  \r
+    xxpermdi    vs5,   vs4,   vs4,2      \r
+    xxpermdi    vs7,   vs6,   vs6,2 \r
+.if \First==1\r
+    xvmulsp      vs32,   vs24,   vs0\r
+    xvmulsp      vs33,   vs24,   vs1 \r
+    xvmulsp      vs34,   vs24,   vs2\r
+    xvmulsp      vs35,   vs24,   vs3\r
+ \r
+.else \r
+    xvmaddasp      vs32,   vs24,   vs0\r
+    xvmaddasp      vs33,   vs24,   vs1 \r
+    xvmaddasp      vs34,   vs24,   vs2\r
+    xvmaddasp      vs35,   vs24,   vs3\r
+ \r
+.endif\r
+\r
+.if \Complete==0 \r
+\r
+    lxv vs0,    DISP8(\Index, 16+\OffsetA)(\AREG)\r
+    lxv vs24,   DISP8(\Index, 16+\OffsetB)(\BREG) \r
+\r
+    xxperm      vs2,   vs0,       permute_mask  \r
+    xxpermdi    vs1,   vs0,   vs0,2      \r
+    xxpermdi    vs3,   vs2,   vs2,2   \r
+.endif\r
+\r
+.if \First==1\r
+    xvmulsp      vs32,   vs26,   vs4\r
+    xvmulsp      vs33,   vs26,   vs5 \r
+    xvmulsp      vs34,   vs26,   vs6\r
+    xvmulsp      vs35,   vs26,   vs7 \r
+\r
+\r
+.else\r
+    xvmaddasp      vs32,   vs26,   vs4\r
+    xvmaddasp      vs33,   vs26,   vs5 \r
+    xvmaddasp      vs34,   vs26,   vs6\r
+    xvmaddasp      vs35,   vs26,   vs7\r
+ \r
+.endif\r
+ \r
+ \r
+.if \IsLast==1  \r
+.if \Complete==1\r
+    addi        \AREG, \AREG, DISP8(\Index,16+\OffsetA)  \r
+    addi        \BREG, \BREG,  DISP8(\Index,16+\OffsetB)\r
+\r
+.else\r
+    addi        \AREG, \AREG, DISP8(\Index,32)  \r
+    addi        \BREG, \BREG,  DISP8(\Index,32)\r
+\r
+.endif\r
+.endif   \r
+     \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE4x4\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC \r
+#if !defined(TRMMKERNEL)  \r
+  lxv        vs36, 0(CO)\r
+  lxv        vs37, 0(T1)\r
+#endif\r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10 \r
+#if !defined(TRMMKERNEL)   \r
+  lxv        vs38, 0(T2)\r
+  lxv        vs39, 0(T3)    \r
+#endif   \r
+\r
+  xxmrglw  vs0, vs35,vs32\r
+  xxmrglw  vs1, vs34,vs33 \r
+  xxmrglw  vs4, vs32,vs35\r
+  xxmrglw  vs5, vs33,vs34 \r
+\r
+\r
+  xxmrghw  vs2, vs35,vs32\r
+  xxmrghw  vs3, vs34,vs33 \r
+  xxmrghw  vs6, vs32,vs35\r
+  xxmrghw  vs7, vs33,vs34  \r
+\r
+  xxmrgld  vs24, vs1, vs0  \r
+  xxmrghd  vs25,vs5,vs4 \r
+\r
+  xxmrgld  vs26, vs2, vs3  \r
+  xxmrghd  vs27,vs6,vs7\r
+\r
+ #if defined(TRMMKERNEL)\r
+  xvmulsp        vs36, vs24, alpha_r\r
+  xvmulsp        vs37, vs25, alpha_r \r
+  xvmulsp        vs38, vs26, alpha_r\r
+  xvmulsp        vs39, vs27, alpha_r \r
+#else\r
+  xvmaddasp        vs36, vs24, alpha_r\r
+  xvmaddasp        vs37, vs25, alpha_r \r
+  xvmaddasp        vs38, vs26, alpha_r\r
+  xvmaddasp        vs39, vs27, alpha_r   \r
+ #endif\r
+  stxv        vs36, 0(CO)\r
+  stxv        vs37, 0(T1) \r
+  stxv        vs38, 0(T2)\r
+  stxv        vs39, 0(T3)   \r
+ \r
+\r
+\r
+  addi CO,CO,16\r
+.endm\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=4 and M=2\r
+**********************************************************************************************/\r
+\r
+ \r
+.macro KERNEL4x2_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x2_I_2 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ \r
+\r
+.macro Zero4x2\r
+    xxlxor      vs0,   vs0,   vs0 \r
+    xxlxor      vs2,   vs2,   vs2 \r
+       \r
+.endm\r
+ \r
+.macro KERNEL4x2\r
+  KERNEL4x2_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL4x2_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxsd v4,   DISP2(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG)      \r
+    xxspltw   vs8,  vs36, 0 \r
+    xxspltw   vs9,  vs36, 1  \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8 \r
+    xvmulsp      vs2,   vs26,   vs9 \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8  \r
+    xvmaddasp      vs2,   vs26,   vs9 \r
+ \r
+ .endif\r
+   \r
+    addi        \AREG, \AREG, DISP2(\Index,8)  \r
+    addi        \BREG, \BREG, DISP4(\Index,16)\r
+ \r
+.endm\r
+\r
+.macro KERNEL4x2_I_2  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG) \r
+    lxv vs28,   DISP8(\Index,16+\OffsetB)(\BREG)       \r
+    xxspltw   vs8,  vs4, 2  \r
+    xxspltw   vs9,  vs4, 3 \r
+    xxspltw   vs10, vs4, 0 \r
+    xxspltw   vs11, vs4, 1\r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8 \r
+    xvmulsp      vs2,   vs26,   vs9  \r
+\r
+    xvmulsp      vs0,   vs28,   vs10 \r
+    xvmulsp      vs2,   vs28,   vs11     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8 \r
+    xvmaddasp      vs2,   vs26,   vs9 \r
+\r
+    xvmaddasp      vs0,   vs28,   vs10 \r
+    xvmaddasp      vs2,   vs28,   vs11   \r
+ .endif\r
+\r
+ \r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP4(\Index,16)  \r
+    addi        \BREG, \BREG, DISP8(\Index,32)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE4x2\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC  \r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10     \r
+  /*convert alpha_r for multiply*/\r
+  xscvspdp  vs4,alpha_r\r
+/* v0 corresponds to vs32, do not forget*/\r
+#if !defined(TRMMKERNEL)\r
+  lxssp  v0,0(CO) \r
+  lxssp  v1,4(CO) \r
+\r
+  lxssp  v2,0(T1)\r
+  lxssp  v3,4(T1)\r
+\r
+  lxssp  v4,0(T2)\r
+  lxssp  v5,4(T2)\r
+\r
+  lxssp  v6,0(T3)\r
+  lxssp  v7,4(T3)\r
+\r
+   \r
+#endif\r
+  xscvspdp  vs5, vs2\r
+  xxspltw   vs6, vs2, 1 \r
+  xxspltw   vs7, vs2, 2 \r
+  xxspltw   vs8, vs2, 3  \r
+  xscvspdp  vs6,vs6\r
+  xscvspdp  vs7,vs7\r
+  xscvspdp  vs8,vs8\r
+\r
+  xscvspdp  vs24, vs0\r
+  xxspltw   vs25, vs0, 1 \r
+  xxspltw   vs26, vs0, 2 \r
+  xxspltw   vs27, vs0, 3  \r
+  xscvspdp  vs25,vs25\r
+  xscvspdp  vs26,vs26\r
+  xscvspdp  vs27,vs27\r
+ \r
+\r
+#if defined(TRMMKERNEL)\r
+  xsmuldp  vs32,vs8, vs4 \r
+  xsmuldp  vs33,vs27, vs4 \r
+\r
+  xsmuldp  vs34,vs7, vs4 \r
+  xsmuldp  vs35,vs26, vs4 \r
+\r
+  xsmuldp  vs36,vs6, vs4 \r
+  xsmuldp  vs37,vs25, vs4  \r
+\r
+  xsmuldp  vs38,vs5, vs4 \r
+  xsmuldp  vs39,vs24, vs4  \r
+\r
+      \r
+#else\r
+  xsmaddadp  vs32,vs8, vs4 \r
+  xsmaddadp  vs33,vs27, vs4 \r
+\r
+  xsmaddadp  vs34,vs7, vs4 \r
+  xsmaddadp  vs35,vs26, vs4 \r
+\r
+  xsmaddadp  vs36,vs6, vs4 \r
+  xsmaddadp  vs37,vs25, vs4  \r
+\r
+  xsmaddadp  vs38,vs5, vs4 \r
+  xsmaddadp  vs39,vs24, vs4  \r
+\r
+    \r
+#endif  \r
+\r
+  stxssp  v0,0(CO) \r
+  stxssp  v1,4(CO) \r
+\r
+  stxssp  v2,0(T1)\r
+  stxssp  v3,4(T1)\r
+\r
+  stxssp  v4,0(T2)\r
+  stxssp  v5,4(T2)\r
+\r
+  stxssp  v6,0(T3)\r
+  stxssp  v7,4(T3)\r
+\r
+ \r
+ \r
+\r
+  addi CO,CO,8\r
+.endm\r
+\r
+\r
+/**********************************************************************************************\r
+* Macros for N=4 and M=1\r
+**********************************************************************************************/\r
+.macro KERNEL4x1_4   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL4x1_I_4 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro Zero4x1\r
+    xxlxor      vs0,   vs0,   vs0 \r
+.endm\r
+\r
+.macro KERNEL4x1\r
+  KERNEL4x1_1 AO,BO, 0 \r
+.endm\r
+\r
+.macro KERNEL4x1_2\r
+  KERNEL4x1_2_1 AO,BO, 0 \r
+.endm\r
+\r
+.macro KERNEL4x1_1 AREG,BREG,First \r
+    lxvwsx vs8,  0, \AREG\r
+    lxv vs26,   0(\BREG)       \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8  \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8  \r
+ .endif\r
+    addi        \AREG, \AREG,  4  \r
+    addi        \BREG, \BREG,  16\r
+.endm\r
+\r
+.macro KERNEL4x1_2_1 AREG,BREG,First \r
+    lxsd v4,    0(\AREG)\r
+    lxv vs26,   0(\BREG)      \r
+    lxv vs28,  16(\BREG) \r
+    xxspltw   vs8,  vs36, 1 \r
+    xxspltw   vs9,  vs36, 0  \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8 \r
+    xvmulsp      vs0,   vs28,   vs9     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8  \r
+    xvmaddasp      vs0,   vs28,   vs9  \r
+ .endif\r
+    addi        \AREG, \AREG,  8 \r
+    addi        \BREG, \BREG,  32\r
+.endm\r
+\r
+.macro KERNEL4x1_I_4  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  \r
+    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)\r
+    xxspltw   vs8,  vs4, 3 \r
+    xxspltw   vs9,  vs4, 2 \r
+    xxspltw   vs10, vs4, 1 \r
+    xxspltw   vs11, vs4, 0\r
+    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG) \r
+    lxv vs28,   DISP16(\Index,16+\OffsetB)(\BREG)  \r
+    lxv vs30,   DISP16(\Index,32+\OffsetB)(\BREG) \r
+    lxv vs32,   DISP16(\Index,48+\OffsetB)(\BREG)          \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8  \r
+    xvmulsp      vs0,   vs28,   vs9      \r
+    xvmulsp      vs0,   vs30,   vs10  \r
+    xvmulsp      vs0,   vs32,   vs11     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8  \r
+    xvmaddasp      vs0,   vs28,   vs9     \r
+    xvmaddasp      vs0,   vs30,   vs10  \r
+    xvmaddasp      vs0,   vs32,   vs11  \r
+ .endif\r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP4(\Index,16)  \r
+    addi        \BREG, \BREG, DISP16(\Index,64)\r
+.endif \r
+.endm\r
+\r
+.macro SAVE4x1\r
+  slwi    T10, LDC ,   1 \r
+  add     T1, CO, LDC  \r
+  add     T2, CO, T10  \r
+  add     T3, T1, T10     \r
+  /*convert alpha_r for multiply*/\r
+  xscvspdp  vs4,alpha_r\r
+/* v0 corresponds to vs32, do not forget*/\r
+#if !defined(TRMMKERNEL)\r
+  lxssp  v0,0(CO)  \r
+  lxssp  v2,0(T1) \r
+  lxssp  v4,0(T2) \r
+  lxssp  v6,0(T3)  \r
+#endif\r
+  xscvspdp  vs24, vs0\r
+  xxspltw   vs25, vs0, 1 \r
+  xxspltw   vs26, vs0, 2 \r
+  xxspltw   vs27, vs0, 3  \r
+  xscvspdp  vs25,vs25\r
+  xscvspdp  vs26,vs26\r
+  xscvspdp  vs27,vs27\r
+\r
+#if defined(TRMMKERNEL)\r
+  xsmuldp  vs32,vs27, vs4 \r
+  xsmuldp  vs34,vs26, vs4 \r
+  xsmuldp  vs36,vs25, vs4 \r
+  xsmuldp  vs38,vs24, vs4  \r
+#else\r
+  xsmaddadp  vs32,vs27, vs4 \r
+  xsmaddadp  vs34,vs26, vs4 \r
+  xsmaddadp  vs36,vs25, vs4 \r
+  xsmaddadp  vs38,vs24, vs4   \r
+#endif  \r
+  stxssp  v0,0(CO)  \r
+  stxssp  v2,0(T1) \r
+  stxssp  v4,0(T2) \r
+  stxssp  v6,0(T3)  \r
+  addi CO,CO,4\r
+.endm\r
+\r
+/****************************N=2 section*****************/\r
+\r
+.macro KERNEL2x16_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL2x16_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ \r
+.macro Zero2x16\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+    xxlxor      vs2,   vs2,   vs2\r
+    xxlxor      vs3,   vs3,   vs3\r
+    xxlxor      vs4,   vs4,   vs4\r
+    xxlxor      vs5,   vs5,   vs5 \r
+    xxlxor      vs6,   vs6,   vs6\r
+    xxlxor      vs7,   vs7,   vs7      \r
+.endm\r
+ \r
+.macro KERNEL2x16\r
+  KERNEL2x16_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL2x16_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL2x16_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ \r
+.macro KERNEL2x16_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 1 \r
+    xxspltw   vs9,  vs36, 0 \r
+    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)   \r
+    lxv vs28,   DISP16(\Index, 32+\OffsetA)(\AREG)\r
+    lxv vs29,   DISP16(\Index,48+\OffsetA)(\AREG)        \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8 \r
+    xvmulsp      vs2,   vs28,   vs8\r
+    xvmulsp      vs3,   vs29,   vs8 \r
+\r
+    xvmulsp      vs4,   vs26,   vs9\r
+    xvmulsp      vs5,   vs27,   vs9 \r
+    xvmulsp      vs6,   vs28,   vs9\r
+    xvmulsp      vs7,   vs29,   vs9     \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs28,   vs8\r
+    xvmaddasp      vs3,   vs29,   vs8 \r
+\r
+    xvmaddasp      vs4,   vs26,   vs9\r
+    xvmaddasp      vs5,   vs27,   vs9 \r
+    xvmaddasp      vs6,   vs28,   vs9\r
+    xvmaddasp      vs7,   vs29,   vs9\r
+ \r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)\r
+    addi        \AREG, \AREG, DISP16(\Index,64)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL2x16_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP64(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP64(\Index,16+\OffsetA)(\AREG)\r
+    lxv vs28,   DISP64(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs29,   DISP64(\Index,48+\OffsetA)(\AREG)  \r
+\r
+    lxv vs16,   DISP64(\Index,64+ 0+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP64(\Index,64+ 16+\OffsetA)(\AREG)\r
+    lxv vs18,   DISP64(\Index,64+ 32+\OffsetA)(\AREG)\r
+    lxv vs19,   DISP64(\Index,64+ 48+\OffsetA)(\AREG) \r
+\r
+    lxv vs30,   DISP64(\Index,128+ 0+\OffsetA)(\AREG)\r
+    lxv vs31,   DISP64(\Index,128+ 16+\OffsetA)(\AREG)\r
+    lxv vs32,   DISP64(\Index,128+ 32+\OffsetA)(\AREG)\r
+    lxv vs33,   DISP64(\Index,128+ 48+\OffsetA)(\AREG)  \r
+        \r
+    lxv vs34,   DISP64(\Index,128+ 64+ 0+\OffsetA)(\AREG)\r
+    lxv vs35,   DISP64(\Index,128+ 64+ 16+\OffsetA)(\AREG)\r
+    lxv vs36,   DISP64(\Index,128+ 64+ 32+\OffsetA)(\AREG)\r
+    lxv vs37,   DISP64(\Index,128+ 64+ 48+\OffsetA)(\AREG) \r
+\r
+    xxspltw   vs8,  vs38, 3  \r
+    xxspltw   vs9,  vs38, 2 \r
+    xxspltw   vs10, vs38, 1 \r
+    xxspltw   vs11, vs38, 0  \r
+\r
+    xxspltw   vs12,  vs39, 3  \r
+    xxspltw   vs13,  vs39, 2 \r
+    xxspltw   vs14, vs39, 1 \r
+    xxspltw   vs15, vs39, 0  \r
+\r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs28,   vs8\r
+    xvmaddasp      vs3,   vs29,   vs8 \r
+\r
+    xvmaddasp      vs4,   vs26,   vs9\r
+    xvmaddasp      vs5,   vs27,   vs9 \r
+    xvmaddasp      vs6,   vs28,   vs9\r
+    xvmaddasp      vs7,   vs29,   vs9 \r
+\r
+    xvmaddasp      vs0,   vs16,   vs10\r
+    xvmaddasp      vs1,   vs17,   vs10 \r
+    xvmaddasp      vs2,   vs18,   vs10\r
+    xvmaddasp      vs3,   vs19,   vs10 \r
+\r
+    xvmaddasp      vs4,   vs16,   vs11\r
+    xvmaddasp      vs5,   vs17,   vs11 \r
+    xvmaddasp      vs6,   vs18,   vs11\r
+    xvmaddasp      vs7,   vs19,   vs11  \r
+\r
+    xvmaddasp      vs0,   vs30,   vs12\r
+    xvmaddasp      vs1,   vs31,   vs12 \r
+    xvmaddasp      vs2,   vs32,   vs12\r
+    xvmaddasp      vs3,   vs33,   vs12 \r
+\r
+    xvmaddasp      vs4,   vs30,   vs13\r
+    xvmaddasp      vs5,   vs31,   vs13 \r
+    xvmaddasp      vs6,   vs32,   vs13\r
+    xvmaddasp      vs7,   vs33,   vs13 \r
+\r
+    xvmaddasp      vs0,   vs34,   vs14\r
+    xvmaddasp      vs1,   vs35,   vs14 \r
+    xvmaddasp      vs2,   vs36,   vs14\r
+    xvmaddasp      vs3,   vs37,   vs14 \r
+\r
+    xvmaddasp      vs4,   vs34,   vs15\r
+    xvmaddasp      vs5,   vs35,   vs15 \r
+    xvmaddasp      vs6,   vs36,   vs15\r
+    xvmaddasp      vs7,   vs37,   vs15    \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP8(\Index,32)  \r
+    addi        \AREG, \AREG, DISP64(\Index,256)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL2x16_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 3  \r
+    xxspltw   vs9,  vs36, 2 \r
+    xxspltw   vs10, vs36, 1 \r
+    xxspltw   vs11, vs36, 0    \r
+    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG)\r
+    lxv vs28,   DISP32(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs29,   DISP32(\Index,48+\OffsetA)(\AREG)      \r
+    lxv vs16,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)\r
+    lxv vs18,   DISP32(\Index,64+ 32+\OffsetA)(\AREG)\r
+    lxv vs19,   DISP32(\Index,64+ 48+\OffsetA)(\AREG) \r
+ \r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs28,   vs8\r
+    xvmaddasp      vs3,   vs29,   vs8 \r
+\r
+    xvmaddasp      vs4,   vs26,   vs9\r
+    xvmaddasp      vs5,   vs27,   vs9 \r
+    xvmaddasp      vs6,   vs28,   vs9\r
+    xvmaddasp      vs7,   vs29,   vs9 \r
+\r
+    xvmaddasp      vs0,   vs16,   vs10\r
+    xvmaddasp      vs1,   vs17,   vs10 \r
+    xvmaddasp      vs2,   vs18,   vs10\r
+    xvmaddasp      vs3,   vs19,   vs10 \r
+\r
+    xvmaddasp      vs4,   vs16,   vs11\r
+    xvmaddasp      vs5,   vs17,   vs11 \r
+    xvmaddasp      vs6,   vs18,   vs11\r
+    xvmaddasp      vs7,   vs19,   vs11   \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)  \r
+    addi        \AREG, \AREG, DISP32(\Index,128)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE2x16\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs16, 0(CO)\r
+    lxv        vs17, 16(CO) \r
+    lxv        vs18, 32(CO)  \r
+    lxv        vs19, 48(CO)      \r
+#endif\r
+  add     T1, CO, LDC \r
+#ifndef TRMMKERNEL    \r
+    lxv        vs26, 0(T1)\r
+    lxv        vs27, 16(T1) \r
+    lxv        vs28, 32(T1)  \r
+    lxv        vs29, 48(T1)      \r
+#endif\r
+\r
+#if defined(TRMMKERNEL)\r
+  xvmulsp        vs16, vs0, alpha_r\r
+  xvmulsp        vs17, vs1, alpha_r \r
+  xvmulsp        vs18, vs2, alpha_r\r
+  xvmulsp        vs19, vs3, alpha_r   \r
+  xvmulsp        vs26, vs4, alpha_r\r
+  xvmulsp        vs27, vs5, alpha_r \r
+  xvmulsp        vs28, vs6, alpha_r\r
+  xvmulsp        vs29, vs7, alpha_r\r
+#else\r
+  xvmaddasp        vs16, vs0, alpha_r\r
+  xvmaddasp        vs17, vs1, alpha_r \r
+  xvmaddasp        vs18, vs2, alpha_r\r
+  xvmaddasp        vs19, vs3, alpha_r   \r
+  xvmaddasp        vs26, vs4, alpha_r\r
+  xvmaddasp        vs27, vs5, alpha_r \r
+  xvmaddasp        vs28, vs6, alpha_r\r
+  xvmaddasp        vs29, vs7, alpha_r\r
+#endif\r
+    stxv        vs16, 0(CO)\r
+    stxv        vs17, 16(CO) \r
+    stxv        vs18, 32(CO)  \r
+    stxv        vs19, 48(CO)      \r
+    \r
+    stxv        vs26, 0(T1)\r
+    stxv        vs27, 16(T1) \r
+    stxv        vs28, 32(T1)  \r
+    stxv        vs29, 48(T1) \r
+ \r
+  addi CO,CO,64\r
+\r
+.endm\r
+\r
+/*       M=8 N=2 */\r
+\r
+.macro KERNEL2x8_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL2x8_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ \r
+.macro Zero2x8\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+ \r
+    xxlxor      vs4,   vs4,   vs4\r
+    xxlxor      vs5,   vs5,   vs5 \r
+     \r
+.endm\r
+ \r
+.macro KERNEL2x8\r
+  KERNEL2x8_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL2x8_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL2x8_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ \r
+.macro KERNEL2x8_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 1 \r
+    xxspltw   vs9,  vs36, 0 \r
+    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)          \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8  \r
+\r
+    xvmulsp      vs4,   vs26,   vs9\r
+    xvmulsp      vs5,   vs27,   vs9      \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8   \r
+\r
+    xvmaddasp      vs4,   vs26,   vs9\r
+    xvmaddasp      vs5,   vs27,   vs9  \r
+ \r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)\r
+    addi        \AREG, \AREG, DISP8(\Index,32)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL2x8_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG) \r
+\r
+    lxv vs16,   DISP32(\Index,32+ 0+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP32(\Index,32+ 16+\OffsetA)(\AREG) \r
+\r
+    lxv vs30,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)\r
+    lxv vs31,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)  \r
+        \r
+    lxv vs34,   DISP32(\Index, 96+ 0+\OffsetA)(\AREG)\r
+    lxv vs35,   DISP32(\Index, 96+ 16+\OffsetA)(\AREG) \r
+\r
+    xxspltw   vs8,  vs38, 3  \r
+    xxspltw   vs9,  vs38, 2 \r
+    xxspltw   vs10, vs38, 1 \r
+    xxspltw   vs11, vs38, 0  \r
+\r
+    xxspltw   vs12,  vs39, 3  \r
+    xxspltw   vs13,  vs39, 2 \r
+    xxspltw   vs14, vs39, 1 \r
+    xxspltw   vs15, vs39, 0  \r
+\r
+ \r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs4,   vs26,   vs9\r
+    xvmaddasp      vs5,   vs27,   vs9 \r
+ \r
+\r
+    xvmaddasp      vs0,   vs16,   vs10\r
+    xvmaddasp      vs1,   vs17,   vs10 \r
+    xvmaddasp      vs4,   vs16,   vs11\r
+    xvmaddasp      vs5,   vs17,   vs11 \r
+ \r
+\r
+    xvmaddasp      vs0,   vs30,   vs12\r
+    xvmaddasp      vs1,   vs31,   vs12 \r
+    xvmaddasp      vs4,   vs30,   vs13\r
+    xvmaddasp      vs5,   vs31,   vs13 \r
+\r
+    xvmaddasp      vs0,   vs34,   vs14\r
+    xvmaddasp      vs1,   vs35,   vs14 \r
+    xvmaddasp      vs4,   vs34,   vs15\r
+    xvmaddasp      vs5,   vs35,   vs15 \r
+   \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP8(\Index,32)  \r
+    addi        \AREG, \AREG, DISP32(\Index,128)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL2x8_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 3  \r
+    xxspltw   vs9,  vs36, 2 \r
+    xxspltw   vs10, vs36, 1 \r
+    xxspltw   vs11, vs36, 0    \r
+    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)\r
+    lxv vs16,   DISP16(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP16(\Index,48+\OffsetA)(\AREG)      \r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8  \r
+\r
+    xvmaddasp      vs4,   vs26,   vs9\r
+    xvmaddasp      vs5,   vs27,   vs9  \r
+\r
+    xvmaddasp      vs0,   vs16,   vs10\r
+    xvmaddasp      vs1,   vs17,   vs10   \r
+\r
+    xvmaddasp      vs4,   vs16,   vs11\r
+    xvmaddasp      vs5,   vs17,   vs11     \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)  \r
+    addi        \AREG, \AREG, DISP16(\Index,64)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE2x8\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs16, 0(CO)\r
+    lxv        vs17, 16(CO)     \r
+#endif\r
+  add     T1, CO, LDC \r
+#ifndef TRMMKERNEL    \r
+    lxv        vs26, 0(T1)\r
+    lxv        vs27, 16(T1) \r
+    \r
+#endif\r
+\r
+#if defined(TRMMKERNEL)\r
+  xvmulsp        vs16, vs0, alpha_r\r
+  xvmulsp        vs17, vs1, alpha_r  \r
+  xvmulsp        vs26, vs4, alpha_r\r
+  xvmulsp        vs27, vs5, alpha_r \r
+#else\r
+  xvmaddasp        vs16, vs0, alpha_r\r
+  xvmaddasp        vs17, vs1, alpha_r  \r
+  xvmaddasp        vs26, vs4, alpha_r\r
+  xvmaddasp        vs27, vs5, alpha_r \r
+#endif\r
+\r
+    stxv        vs16, 0(CO)\r
+    stxv        vs17, 16(CO) \r
+     \r
+    \r
+    stxv        vs26, 0(T1)\r
+    stxv        vs27, 16(T1) \r
+\r
+  addi CO,CO,32\r
+\r
+.endm\r
+\r
+\r
+/*M=4*/\r
+\r
+\r
+.macro KERNEL2x4_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL2x4_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ /* we will aggregate on save vs0 +vs4 vs11+vs5 */\r
+.macro Zero2x4\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+ \r
+    xxlxor      vs4,   vs4,   vs4\r
+    xxlxor      vs5,   vs5,   vs5 \r
+    \r
+.endm\r
+ \r
+.macro KERNEL2x4\r
+  KERNEL2x4_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL2x4_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL2x4_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ \r
+.macro KERNEL2x4_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 1 \r
+    xxspltw   vs9,  vs36, 0 \r
+    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)        \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8 \r
+    xvmulsp      vs1,   vs26,   vs9     \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8 \r
+    xvmaddasp      vs1,   vs26,   vs9 \r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)\r
+    addi        \AREG, \AREG, DISP4(\Index,16)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL2x4_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs16,   DISP16(\Index,16+\OffsetA)(\AREG) \r
+\r
+    lxv vs30,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)\r
+    lxv vs34,   DISP16(\Index,32+ 16+\OffsetA)(\AREG) \r
+ \r
+\r
+    xxspltw   vs8,  vs38, 3  \r
+    xxspltw   vs9,  vs38, 2 \r
+    xxspltw   vs10, vs38, 1 \r
+    xxspltw   vs11, vs38, 0  \r
+\r
+    xxspltw   vs12,  vs39, 3  \r
+    xxspltw   vs13,  vs39, 2 \r
+    xxspltw   vs14, vs39, 1 \r
+    xxspltw   vs15, vs39, 0  \r
+\r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs26,   vs9 \r
+    xvmaddasp      vs4,   vs16,   vs10\r
+    xvmaddasp      vs5,   vs16,   vs11 \r
+ \r
+\r
+    xvmaddasp      vs0,   vs30,   vs12\r
+    xvmaddasp      vs1,   vs30,   vs13 \r
+    xvmaddasp      vs4,   vs34,   vs14\r
+    xvmaddasp      vs5,   vs34,   vs15 \r
+ \r
+   \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP8(\Index,32)  \r
+    addi        \AREG, \AREG, DISP16(\Index,64)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL2x4_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 3  \r
+    xxspltw   vs9,  vs36, 2 \r
+    xxspltw   vs10, vs36, 1 \r
+    xxspltw   vs11, vs36, 0    \r
+    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs16,   DISP8(\Index, 16+\OffsetA)(\AREG)      \r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs26,   vs9 \r
+    xvmaddasp      vs4,   vs16,   vs10\r
+    xvmaddasp      vs5,   vs16,   vs11     \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)  \r
+    addi        \AREG, \AREG, DISP8(\Index,32)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE2x4\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs16, 0(CO)     \r
+#endif\r
+  add     T1, CO, LDC \r
+#ifndef TRMMKERNEL    \r
+    lxv        vs26, 0(T1) \r
+    \r
+#endif\r
+    /*aggregate vectors*/\r
+  xvaddsp         vs0,vs0,vs4\r
+  xvaddsp         vs1,vs1,vs5 \r
+#if defined(TRMMKERNEL)\r
+  xvmulsp        vs16, vs0, alpha_r \r
+  xvmulsp        vs26, vs1, alpha_r \r
+#else\r
+  xvmaddasp        vs16, vs0, alpha_r \r
+  xvmaddasp        vs26, vs1, alpha_r \r
+#endif\r
+\r
+  stxv        vs16, 0(CO) \r
+  stxv        vs26, 0(T1)  \r
+\r
+  addi CO,CO,16\r
+\r
+.endm\r
+\r
+\r
+/* M=2 N=2 we will have inner pemrute action before permute was revrsing 3,2,1,0 not iw 2ill inner reverse 1,0,3,2  */\r
+.macro SWITCH_PERMUTE_INNER\r
+    xxpermdi   permute_mask,   permute_mask,   permute_mask,2\r
+.endm\r
+\r
+.macro Zero2x2\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+    SWITCH_PERMUTE_INNER\r
+.endm\r
+ \r
+.macro KERNEL2x2\r
+  KERNEL2x2_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL2x2_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL2x2_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro KERNEL2x2_2 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL2x2_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ \r
+.macro KERNEL2x2_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    xxperm   vs9,  vs36, permute_mask \r
+    lxsd v5,   DISP2(\Index, 0+\OffsetA)(\AREG)        \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs37,   vs36 \r
+    xvmulsp      vs1,   vs37,   vs9     \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs37,   vs36 \r
+    xvmaddasp      vs1,   vs37,   vs9 \r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)\r
+    addi        \AREG, \AREG, DISP2(\Index,8)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL2x2_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs10,    DISP8(\Index, 16+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs16,   DISP8(\Index,16+\OffsetA)(\AREG) \r
+\r
+ \r
+    xxperm   vs9,  vs8, permute_mask   \r
+    xxperm   vs11, vs10, permute_mask  \r
+\r
+\r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs26,   vs9 \r
+    xvmaddasp      vs0,   vs16,   vs10\r
+    xvmaddasp      vs1,   vs16,   vs11 \r
+ \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP8(\Index,32)  \r
+    addi        \AREG, \AREG, DISP8(\Index,32)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL2x2_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs8,    DISP4(\Index, 0+\OffsetB)(\BREG)  \r
+    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG) \r
+\r
+ \r
+    xxperm   vs9,  vs8, permute_mask    \r
+\r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs26,   vs9  \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)  \r
+    addi        \AREG, \AREG, DISP4(\Index,16)\r
+.endif \r
+.endm\r
+\r
+\r
+.macro SAVE2x2\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxsd v4   , 0(CO)     \r
+#endif\r
+  add     T1, CO, LDC \r
+#ifndef TRMMKERNEL    \r
+    lxsd v5   , 0(T1) \r
+    \r
+#endif\r
+    /*aggregate vectors*/\r
+  xxpermdi         vs4,vs0,vs0,2\r
+  xxpermdi         vs5,vs1,vs1,2  \r
+  xvaddsp          vs0,vs0,vs4\r
+  xvaddsp         vs1,vs1,vs5 \r
+  /*   */\r
+  /* lets correct the order to 00 10 and 10 ,11 from {00,11} {01,10}  */\r
+  xxperm    vs1,vs1, permute_mask\r
+\r
+\r
+  xxmrghw   vs2 ,vs1,vs0\r
+  xxpermdi         vs2,vs2,vs2,2  \r
+  xxmrghw   vs3 ,vs0,vs1  \r
+#if defined(TRMMKERNEL)\r
+  xvmulsp        vs36, vs2, alpha_r \r
+  xvmulsp        vs37, vs3, alpha_r \r
+#else\r
+  xvmaddasp        vs36, vs2, alpha_r \r
+  xvmaddasp        vs37, vs3, alpha_r \r
+#endif\r
+  /**** store last two words*/\r
+\r
+\r
+  stxsd       v4, 0(CO) \r
+  stxsd        v5, 0(T1)  \r
+\r
+  addi CO,CO,8\r
+\r
+.endm\r
+\r
+/*--------------------------- M=1 N=2 */\r
+.macro Zero2x1\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+    xxlxor    vs2,vs2,vs2 \r
+    xxlxor    vs3,vs3,vs3     \r
+.endm\r
+ \r
+.macro KERNEL2x1\r
+  KERNEL2x1_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL2x1_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL2x1_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro KERNEL2x1_2 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL2x1_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ /*\r
+   we will calculate 1 alone then will add it to batched ones\r
+ */\r
+.macro KERNEL2x1_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxssp v3,   DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    lxssp v4,   DISP2(\Index, 4+\OffsetB)(\BREG) \r
+    lxssp v5,   DISP1(\Index, 0+\OffsetA)(\AREG)        \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs2,   vs37,   vs35 \r
+    xvmulsp      vs3,   vs37,   vs36     \r
+     \r
+.else \r
+    xsmaddadp     vs2,   vs37,   vs35\r
+    xsmaddadp      vs3,   vs37,   vs36\r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)\r
+    addi        \AREG, \AREG, DISP1(\Index,4)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL2x1_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\BREG)\r
+    lxv vs10,   DISP8(\Index, 16+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)\r
+   \r
+    xxmrglw   vs5, vs26,vs26\r
+    xxmrghw   vs6, vs26,vs26 \r
+ \r
+    xvmaddasp      vs0,   vs8,   vs5\r
+    xvmaddasp      vs1,   vs10,   vs6 \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP8(\Index,32)  \r
+    addi        \AREG, \AREG, DISP4(\Index,16)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL2x1_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxssp v3,   DISP4(\Index, 0+\OffsetB)(\BREG)\r
+    lxssp v4,   DISP4(\Index, 4+\OffsetB)(\BREG) \r
+    lxssp v7,   DISP4(\Index, 8+\OffsetB)(\BREG)\r
+    lxssp v8,   DISP4(\Index, 12+\OffsetB)(\BREG)    \r
+    lxssp v5,   DISP2(\Index, 0+\OffsetA)(\AREG)        \r
+    lxssp v6,   DISP2(\Index, 4+\OffsetA)(\AREG)  \r
+ \r
+ \r
+    xsmaddadp      vs2,   vs37,   vs35\r
+    xsmaddadp      vs3,   vs37,   vs36\r
+\r
+    xsmaddadp      vs2,   vs38,   vs39 \r
+    xsmaddadp      vs3,   vs38,   vs40      \r
+ \r
+   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)\r
+    addi        \AREG, \AREG, DISP2(\Index,8) \r
+.endm\r
+\r
+\r
+.macro SAVE2x1\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxssp v4   , 0(CO)     \r
+#endif\r
+  add     T1, CO, LDC \r
+#ifndef TRMMKERNEL    \r
+    lxssp v5   , 0(T1) \r
+    \r
+#endif\r
+\r
+  /*convert alpha_r for multiply*/\r
+  xscvspdp  vs16,alpha_r\r
+\r
+ /*aggregate vectors 2x2_4   */ \r
+      xxpermdi         vs4,vs0,vs0,2\r
+      xxpermdi         vs5,vs1,vs1,2  \r
+      xvaddsp          vs0,vs0,vs4\r
+      xvaddsp         vs1,vs1,vs5 \r
+      xvaddsp         vs0,vs0,vs1 \r
+/*aggregate vectors 2x1_2 and 2x1_1 into 2x2_4*/\r
+  xscvspdp  vs5, vs0\r
+  xxspltw   vs6, vs0, 1  \r
+  xscvspdp  vs6,vs6 \r
+  xsadddp  vs2,vs2,vs6\r
+  xsadddp  vs3,vs3,vs5  \r
+\r
+  /**** store last two words*/\r
+#if defined(TRMMKERNEL) \r
+  xsmuldp  vs36,vs2, vs16 \r
+  xsmuldp  vs37,vs3, vs16  \r
+ \r
+#else\r
+  xsmaddadp  vs36,vs2, vs16 \r
+  xsmaddadp  vs37,vs3, vs16 \r
+#endif  \r
+\r
+  stxssp       v4, 0(CO) \r
+  stxssp        v5, 0(T1)  \r
+\r
+  addi CO,CO,4\r
+\r
+.endm\r
+\r
+\r
+\r
+/****************************N=1 section*****************/\r
+\r
+.macro KERNEL1x16_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL1x16_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ \r
+.macro Zero1x16\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+    xxlxor      vs2,   vs2,   vs2\r
+    xxlxor      vs3,   vs3,   vs3       \r
+.endm\r
+ \r
+.macro KERNEL1x16\r
+  KERNEL1x16_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL1x16_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x16_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ \r
+.macro KERNEL1x16_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)\r
+    xscvdpspn   vs36,vs36\r
+    xxspltw     vs8,  vs36, 0\r
+    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)   \r
+    lxv vs28,   DISP16(\Index, 32+\OffsetA)(\AREG)\r
+    lxv vs29,   DISP16(\Index,48+\OffsetA)(\AREG)        \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8 \r
+    xvmulsp      vs2,   vs28,   vs8\r
+    xvmulsp      vs3,   vs29,   vs8 \r
+  \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs28,   vs8\r
+    xvmaddasp      vs3,   vs29,   vs8 \r
+ \r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP1(\Index,4)\r
+    addi        \AREG, \AREG, DISP16(\Index,64)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL1x16_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP64(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP64(\Index,16+\OffsetA)(\AREG)\r
+    lxv vs28,   DISP64(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs29,   DISP64(\Index,48+\OffsetA)(\AREG)  \r
+\r
+    lxv vs16,   DISP64(\Index,64+ 0+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP64(\Index,64+ 16+\OffsetA)(\AREG)\r
+    lxv vs18,   DISP64(\Index,64+ 32+\OffsetA)(\AREG)\r
+    lxv vs19,   DISP64(\Index,64+ 48+\OffsetA)(\AREG) \r
+\r
+    xxspltw   vs8,  vs38, 3  \r
+    xxspltw   vs9,  vs38, 2 \r
+\r
+    lxv vs30,   DISP64(\Index,128+ 0+\OffsetA)(\AREG)\r
+    lxv vs31,   DISP64(\Index,128+ 16+\OffsetA)(\AREG)\r
+    lxv vs32,   DISP64(\Index,128+ 32+\OffsetA)(\AREG)\r
+    lxv vs33,   DISP64(\Index,128+ 48+\OffsetA)(\AREG)  \r
+        \r
+    lxv vs34,   DISP64(\Index,128+ 64+ 0+\OffsetA)(\AREG)\r
+    lxv vs35,   DISP64(\Index,128+ 64+ 16+\OffsetA)(\AREG)\r
+    lxv vs36,   DISP64(\Index,128+ 64+ 32+\OffsetA)(\AREG)\r
+    lxv vs37,   DISP64(\Index,128+ 64+ 48+\OffsetA)(\AREG) \r
+\r
+    xxspltw   vs10, vs38, 1 \r
+    xxspltw   vs11, vs38, 0    \r
+\r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs28,   vs8\r
+    xvmaddasp      vs3,   vs29,   vs8 \r
+ \r
+\r
+    xvmaddasp      vs0,   vs16,   vs9\r
+    xvmaddasp      vs1,   vs17,   vs9 \r
+    xvmaddasp      vs2,   vs18,   vs9\r
+    xvmaddasp      vs3,   vs19,   vs9 \r
+ \r
+\r
+    xvmaddasp      vs0,   vs30,   vs10\r
+    xvmaddasp      vs1,   vs31,   vs10 \r
+    xvmaddasp      vs2,   vs32,   vs10\r
+    xvmaddasp      vs3,   vs33,   vs10 \r
+ \r
+\r
+    xvmaddasp      vs0,   vs34,   vs11\r
+    xvmaddasp      vs1,   vs35,   vs11 \r
+    xvmaddasp      vs2,   vs36,   vs11\r
+    xvmaddasp      vs3,   vs37,   vs11 \r
+\r
+ \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)  \r
+    addi        \AREG, \AREG, DISP64(\Index,256)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL1x16_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 1  \r
+    xxspltw   vs9,  vs36, 0      \r
+    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG)\r
+    lxv vs28,   DISP32(\Index,32+\OffsetA)(\AREG)\r
+    lxv vs29,   DISP32(\Index,48+\OffsetA)(\AREG)      \r
+    lxv vs16,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)\r
+    lxv vs18,   DISP32(\Index,64+ 32+\OffsetA)(\AREG)\r
+    lxv vs19,   DISP32(\Index,64+ 48+\OffsetA)(\AREG) \r
+ \r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8 \r
+    xvmaddasp      vs2,   vs28,   vs8\r
+    xvmaddasp      vs3,   vs29,   vs8 \r
+ \r
+\r
+    xvmaddasp      vs0,   vs16,   vs9\r
+    xvmaddasp      vs1,   vs17,   vs9 \r
+    xvmaddasp      vs2,   vs18,   vs9\r
+    xvmaddasp      vs3,   vs19,   vs9 \r
+  \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)  \r
+    addi        \AREG, \AREG, DISP32(\Index,128)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE1x16\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs16, 0(CO)\r
+    lxv        vs17, 16(CO) \r
+    lxv        vs18, 32(CO)  \r
+    lxv        vs19, 48(CO)      \r
+#endif\r
+ \r
+\r
+#if defined(TRMMKERNEL)\r
+  xvmulsp        vs16, vs0, alpha_r\r
+  xvmulsp        vs17, vs1, alpha_r \r
+  xvmulsp        vs18, vs2, alpha_r\r
+  xvmulsp        vs19, vs3, alpha_r   \r
+#else\r
+  xvmaddasp        vs16, vs0, alpha_r\r
+  xvmaddasp        vs17, vs1, alpha_r \r
+  xvmaddasp        vs18, vs2, alpha_r\r
+  xvmaddasp        vs19, vs3, alpha_r   \r
+#endif\r
+    stxv        vs16, 0(CO)\r
+    stxv        vs17, 16(CO) \r
+    stxv        vs18, 32(CO)  \r
+    stxv        vs19, 48(CO)      \r
+    \r
+  addi CO,CO,64\r
+\r
+.endm\r
+\r
+/*       M=8 N=1 */\r
+\r
+.macro KERNEL1x8_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL1x8_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ \r
+.macro Zero1x8\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1  \r
+    xxlxor      vs2,   vs2,   vs2\r
+    xxlxor      vs3,   vs3,   vs3          \r
+.endm\r
+ \r
+.macro KERNEL1x8\r
+  KERNEL1x8_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL1x8_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x8_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ \r
+.macro KERNEL1x8_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)\r
+    xscvdpspn   vs36,vs36\r
+    xxspltw     vs8,  vs36, 0\r
+    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)         \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8\r
+    xvmulsp      vs1,   vs27,   vs8  \r
+  \r
+     \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8  \r
+ \r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP1(\Index,4)\r
+    addi        \AREG, \AREG, DISP8(\Index,32)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL1x8_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG) \r
+\r
+    lxv vs16,   DISP32(\Index,32+ 0+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP32(\Index,32+ 16+\OffsetA)(\AREG) \r
+\r
+    xxspltw   vs8,  vs38, 3  \r
+    xxspltw   vs9,  vs38, 2 \r
+\r
+    lxv vs30,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)\r
+    lxv vs31,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)   \r
+        \r
+    lxv vs34,   DISP32(\Index,64+ 32+ 0+\OffsetA)(\AREG)\r
+    lxv vs35,   DISP32(\Index,64+ 32+ 16+\OffsetA)(\AREG)  \r
+\r
+    xxspltw   vs10, vs38, 1 \r
+    xxspltw   vs11, vs38, 0    \r
+\r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8  \r
+ \r
+\r
+    xvmaddasp      vs2,   vs16,   vs9\r
+    xvmaddasp      vs3,   vs17,   vs9  \r
+ \r
+\r
+    xvmaddasp      vs0,   vs30,   vs10\r
+    xvmaddasp      vs1,   vs31,   vs10  \r
+ \r
+\r
+    xvmaddasp      vs2,   vs34,   vs11\r
+    xvmaddasp      vs3,   vs35,   vs11  \r
+\r
+ \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)  \r
+    addi        \AREG, \AREG, DISP32(\Index,128)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL1x8_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 1  \r
+    xxspltw   vs9,  vs36, 0      \r
+    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)     \r
+    lxv vs16,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)\r
+    lxv vs17,   DISP16(\Index,32+ 16+\OffsetA)(\AREG) \r
+ \r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs8  \r
+ \r
+\r
+    xvmaddasp      vs2,   vs16,   vs9\r
+    xvmaddasp      vs3,   vs17,   vs9   \r
+  \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)  \r
+    addi        \AREG, \AREG, DISP16(\Index,64)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE1x8\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs16, 0(CO)\r
+    lxv        vs17, 16(CO)       \r
+#endif\r
+   /* aggregate vs0 vs2 and vs1 vs3*/\r
+  xvaddsp vs0,vs0,vs2\r
+  xvaddsp  vs1,vs1,vs3\r
+#if defined(TRMMKERNEL)\r
+  xvmulsp        vs16, vs0, alpha_r\r
+  xvmulsp        vs17, vs1, alpha_r     \r
+#else\r
+  xvmaddasp        vs16, vs0, alpha_r\r
+  xvmaddasp        vs17, vs1, alpha_r  \r
+#endif\r
+    stxv        vs16, 0(CO)\r
+    stxv        vs17, 16(CO)      \r
+    \r
+  addi CO,CO,32\r
+\r
+.endm\r
+/*M=4*/\r
+\r
+.macro KERNEL1x4_2   OffsetA,OffsetB, Index,IsLast\r
+  KERNEL1x4_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+ \r
+.macro Zero1x4\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1  \r
+    xxlxor      vs2,   vs2,   vs2\r
+    xxlxor      vs3,   vs3,   vs3          \r
+.endm\r
+ \r
+.macro KERNEL1x4\r
+  KERNEL1x4_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL1x4_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x4_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ \r
+.macro KERNEL1x4_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)\r
+    xscvdpspn   vs36,vs36\r
+    xxspltw     vs8,  vs36, 0\r
+    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)         \r
+ \r
+ \r
+.if \First==1\r
+    xvmulsp      vs0,   vs26,   vs8 \r
+.else \r
+    xvmaddasp      vs0,   vs26,   vs8 \r
+ \r
+ .endif\r
+   \r
+    addi        \BREG, \BREG, DISP1(\Index,4)\r
+    addi        \AREG, \AREG, DISP4(\Index,16)  \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL1x4_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) \r
+\r
+    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG) \r
+ \r
+\r
+    xxspltw   vs8,  vs38, 3  \r
+    xxspltw   vs9,  vs38, 2 \r
+\r
+    lxv vs30,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)\r
+    lxv vs31,   DISP16(\Index,32+ 16+\OffsetA)(\AREG)   \r
+          \r
+\r
+    xxspltw   vs10, vs38, 1 \r
+    xxspltw   vs11, vs38, 0    \r
+\r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8 \r
+\r
+    xvmaddasp      vs1,   vs27,   vs9 \r
+\r
+    xvmaddasp      vs2,   vs30,   vs10   \r
+ \r
+\r
+    xvmaddasp      vs3,   vs31,   vs11   \r
+\r
+ \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP4(\Index,16)  \r
+    addi        \AREG, \AREG, DISP16(\Index,64)\r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL1x4_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)\r
+    xxspltw   vs8,  vs36, 1  \r
+    xxspltw   vs9,  vs36, 0      \r
+    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)\r
+    lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)      \r
+ \r
+ \r
+    xvmaddasp      vs0,   vs26,   vs8\r
+    xvmaddasp      vs1,   vs27,   vs9\r
+  \r
+ \r
+.if \IsLast==1   \r
+    addi        \BREG, \BREG, DISP2(\Index,8)  \r
+    addi        \AREG, \AREG, DISP8(\Index,32)\r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro SAVE1x4\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxv        vs16, 0(CO)       \r
+#endif\r
+   /* aggregate */\r
+  xvaddsp vs0,vs0,vs2\r
+  xvaddsp  vs1,vs1,vs3\r
+  xvaddsp  vs0,vs1,vs0\r
+#if defined(TRMMKERNEL)\r
+  xvmulsp        vs16, vs0, alpha_r     \r
+#else\r
+  xvmaddasp        vs16, vs0, alpha_r  \r
+#endif\r
+    stxv        vs16, 0(CO)      \r
+    \r
+  addi CO,CO,16\r
+\r
+.endm\r
+\r
+/* M=2 N=1*/ \r
+.macro Zero1x2\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+    xxlxor    vs2,vs2,vs2 \r
+    xxlxor    vs3,vs3,vs3     \r
+.endm\r
+ \r
+.macro KERNEL1x2\r
+  KERNEL1x2_1 AO,BO, 0, 0,0,0\r
+.endm\r
+.macro KERNEL1x2_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x2_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro KERNEL1x2_2 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x2_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ /*\r
+   we will calculate 1 alone then will add it to batched ones\r
+ */\r
+.macro KERNEL1x2_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxssp v3,   DISP2(\Index, 0+\OffsetB)(\AREG)\r
+    lxssp v4,   DISP2(\Index, 4+\OffsetB)(\AREG) \r
+    lxssp v5,   DISP1(\Index, 0+\OffsetA)(\BREG)        \r
+ \r
+ \r
+.if \First==1\r
+    xvmuldp      vs2,   vs37,   vs35 \r
+    xvmuldp      vs3,   vs37,   vs36     \r
+     \r
+.else \r
+    xsmaddadp     vs2,   vs37,   vs35\r
+    xsmaddadp      vs3,   vs37,   vs36\r
+ .endif\r
+   \r
+    addi        \AREG, \AREG,  DISP2(\Index,8) \r
+    addi        \BREG, \BREG, DISP1(\Index,4) \r
+ \r
+.endm\r
+\r
+\r
+\r
+\r
+.macro KERNEL1x2_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\AREG)\r
+    lxv vs10,   DISP8(\Index, 16+\OffsetB)(\AREG) \r
+\r
+    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\BREG)\r
+   \r
+    xxmrglw   vs5, vs26,vs26\r
+    xxmrghw   vs6, vs26,vs26 \r
+ \r
+    xvmaddasp      vs0,   vs8,   vs5\r
+    xvmaddasp      vs1,   vs10,   vs6 \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP8(\Index,32)\r
+    addi        \BREG, \BREG,  DISP4(\Index,16)  \r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL1x2_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxssp v3,   DISP4(\Index, 0+\OffsetB)(\AREG)\r
+    lxssp v4,   DISP4(\Index, 4+\OffsetB)(\AREG) \r
+    lxssp v7,   DISP4(\Index, 8+\OffsetB)(\AREG)\r
+    lxssp v8,   DISP4(\Index, 12+\OffsetB)(\AREG)    \r
+    lxssp v5,   DISP2(\Index, 0+\OffsetA)(\BREG)        \r
+    lxssp v6,   DISP2(\Index, 4+\OffsetA)(\BREG)  \r
+ \r
+ \r
+    xsmaddadp      vs2,   vs37,   vs35\r
+    xsmaddadp      vs3,   vs37,   vs36\r
+\r
+    xsmaddadp      vs2,   vs38,   vs39 \r
+    xsmaddadp      vs3,   vs38,   vs40      \r
+ \r
+   \r
+    addi        \AREG, \AREG, DISP4(\Index,16)\r
+    addi        \BREG, \BREG, DISP2(\Index,8) \r
+.endm\r
+\r
+\r
+.macro SAVE1x2\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxssp v4   , 0(CO)      \r
+    lxssp v5   , 4(CO) \r
+    \r
+#endif\r
+\r
+  /*convert alpha_r for multiply*/\r
+  xscvspdp  vs16,alpha_r\r
+\r
+ /*aggregate vectors 1x2_4   */ \r
+      xxpermdi         vs4,vs0,vs0,2\r
+      xxpermdi         vs5,vs1,vs1,2  \r
+      xvaddsp          vs0,vs0,vs4\r
+      xvaddsp         vs1,vs1,vs5 \r
+      xvaddsp         vs0,vs0,vs1 \r
+/*aggregate vectors 1x1_2 and 1x1_1 into 1x2_4*/\r
+  xscvspdp  vs5, vs0\r
+  xxspltw   vs6, vs0, 1  \r
+  xscvspdp  vs6,vs6 \r
+  xsadddp  vs2,vs2,vs6\r
+  xsadddp  vs3,vs3,vs5  \r
+\r
+  /**** store last two words*/\r
+#if defined(TRMMKERNEL) \r
+  xsmuldp  vs36,vs2, vs16 \r
+  xsmuldp  vs37,vs3, vs16  \r
+ \r
+#else\r
+  xsmaddadp  vs36,vs2, vs16 \r
+  xsmaddadp  vs37,vs3, vs16 \r
+#endif  \r
+\r
+  stxssp       v4, 0(CO) \r
+  stxssp        v5, 4(CO)  \r
+\r
+  addi CO,CO,8\r
+\r
+.endm\r
+/*///////////////// N=1 M=1 //////////////////*/\r
+.macro Zero1x1\r
+    xxlxor      vs0,   vs0,   vs0\r
+    xxlxor      vs1,   vs1,   vs1 \r
+    xxlxor      vs2, vs2,vs2 \r
+    xxlxor      vs3,vs3,vs3 \r
+    xxlxor      vs4,vs4,vs4       \r
+.endm\r
+ \r
+.macro KERNEL1x1\r
+  KERNEL1x1_1 AO,BO, 1, 0,0,0\r
+.endm\r
+\r
+.macro KERNEL1x1_16 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x1_I_16 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro KERNEL1x1_8 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x1_I_8 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro KERNEL1x1_4 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x1_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+\r
+.macro KERNEL1x1_2 OffsetA,OffsetB, Index,IsLast  \r
+  KERNEL1x1_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast\r
+.endm\r
+ /*\r
+   we will calculate 1 alone ( FIRST==1 to zero vs4) \r
+ */\r
+.macro KERNEL1x1_1 AREG,BREG,First,OffsetA,OffsetB,Index\r
+\r
+\r
+    lxssp v3,   DISP1(\Index, 0+\OffsetB)(\AREG) \r
+    lxssp v5,   DISP1(\Index, 0+\OffsetA)(\BREG)        \r
+ \r
+ \r
+.if \First==1\r
+    xvmuldp      vs4,   vs37,   vs35       \r
+     \r
+.else \r
+    xsmaddadp     vs4,   vs37,   vs35 \r
+ .endif\r
+   \r
+    addi        \AREG, \AREG,  DISP1(\Index,4) \r
+    addi        \BREG, \BREG, DISP1(\Index,4) \r
+ \r
+.endm\r
+\r
+\r
+.macro KERNEL1x1_I_16 AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs8,    DISP16(\Index, 0+\OffsetB)(\AREG) \r
+    lxv vs9,    DISP16(\Index, 16+\OffsetB)(\AREG) \r
+    lxv vs10,   DISP16(\Index, 32+0+\OffsetB)(\AREG) \r
+    lxv vs11,   DISP16(\Index, 32+ 16+\OffsetB)(\AREG)        \r
+    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\BREG) \r
+    lxv vs16,   DISP16(\Index, 16+\OffsetA)(\BREG) \r
+    lxv vs17,   DISP16(\Index, 32+0+\OffsetA)(\BREG) \r
+    lxv vs18,   DISP16(\Index, 32+16+\OffsetA)(\BREG)     \r
+    xvmaddasp      vs0,   vs8,   vs26 \r
+    xvmaddasp      vs1,   vs9,   vs16  \r
+    xvmaddasp      vs2,   vs10,  vs17 \r
+    xvmaddasp      vs3,   vs11,  vs18\r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP16(\Index,64)\r
+    addi        \BREG, \BREG,  DISP16(\Index,64)  \r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL1x1_I_8  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\AREG) \r
+    lxv vs9,    DISP8(\Index, 16+\OffsetB)(\AREG)     \r
+    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\BREG) \r
+    lxv vs16,   DISP8(\Index, 16+\OffsetA)(\BREG) \r
+    xvmaddasp      vs0,   vs8,   vs26 \r
+    xvmaddasp      vs1,   vs9,   vs16 \r
+ \r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP8(\Index,32)\r
+    addi        \BREG, \BREG,  DISP8(\Index,32)  \r
+.endif \r
+  \r
+.endm\r
+\r
+\r
+.macro KERNEL1x1_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxv vs8,    DISP4(\Index, 0+\OffsetB)(\AREG) \r
+    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\BREG) \r
+ \r
+    xvmaddasp      vs0,   vs8,   vs26 \r
+ \r
+ \r
+.if \IsLast==1   \r
+    addi        \AREG, \AREG, DISP4(\Index,16)\r
+    addi        \BREG, \BREG,  DISP4(\Index,16)  \r
+.endif \r
+  \r
+.endm\r
+\r
+.macro KERNEL1x1_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  \r
+\r
+    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\AREG) \r
+    lxsd v5,   DISP2(\Index, 0+\OffsetA)(\BREG) \r
+ \r
+    xvmaddasp      vs0,   vs36,   vs37 \r
+ \r
+    addi        \AREG, \AREG, DISP2(\Index,8)\r
+    addi        \BREG, \BREG, DISP2(\Index,8) \r
+.endm\r
+\r
+\r
+.macro SAVE1x1\r
+\r
+#ifndef TRMMKERNEL    \r
+    lxssp v4   , 0(CO)    \r
+    \r
+#endif\r
+\r
+  /*convert alpha_r for multiply*/\r
+  xscvspdp  vs16,alpha_r\r
+\r
+ /*aggregate vectors   */ \r
+      xvaddsp          vs0,vs0,vs1\r
+      xvaddsp          vs2,vs2,vs3\r
+      xvaddsp          vs0,vs0,vs2\r
+\r
+      xxpermdi         vs7,vs0,vs0,2   \r
+      xvaddsp          vs0,vs0,vs7 \r
+/*aggregate vectors 1x1_2 and 1x1_1 into 1x1_4*/\r
+  xscvspdp  vs5, vs0\r
+  xxspltw   vs6, vs0, 1  \r
+  xscvspdp  vs6,vs6 \r
+  xsadddp  vs7,vs5,vs6\r
+  xsadddp  vs4,vs4,vs7  \r
+\r
+  /**** store last two words*/\r
+#if defined(TRMMKERNEL) \r
+  xsmuldp  vs36,vs4, vs16   \r
+ \r
+#else\r
+  xsmaddadp  vs36,vs4, vs16   \r
+#endif  \r
+\r
+  stxssp       v4, 0(CO)    \r
+\r
+  addi CO,CO,4\r
+\r
+.endm\r
+\r
+\r
+\r
+\r
+/****************************TRMM POINTER REFRESH MACROSES*************************/\r
+\r
+.macro SHIFT_REG  REG1,REG2,SHIFT_VAL\r
+               .if \SHIFT_VAL==16 \r
+                       slwi            \REG1,  \REG2,  6                       \r
+               .elseif \SHIFT_VAL==8  \r
+                       slwi            \REG1,  \REG2,  5                        \r
+               .elseif \SHIFT_VAL==4\r
+                       slwi            \REG1,  \REG2,  4                         \r
+               .elseif \SHIFT_VAL==2\r
+                       slwi            \REG1,  \REG2,  3                        \r
+               .elseif \SHIFT_VAL==1\r
+                       slwi            \REG1,  \REG2,  2                        \r
+               .endif\r
+.endm\r
+\r
+/*\r
+//#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))\r
+//             ptrbb = bb;\r
+// #else\r
+//             ptrba += off*16;\r
+//             ptrbb = bb + off*2;\r
+// #endif\r
+*/\r
+.macro REFRESH_POINTERS  PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B\r
+    #if (defined(LEFT) &&  defined(TRANSA)) ||  (!defined(LEFT) && !defined(TRANSA))\r
+        /* ptrbb = bb;*/\r
+        mr \PTR_B,\B_VAL     /* refresh BPOINT */\r
+\r
+    #else\r
+                   /*\r
+        // ptrba  =ptrba+ off*C_A;\r
+        // ptrbb = bb + off*C_B; \r
+                               */\r
+               SHIFT_REG T4,\OFF_VAL,\C_B              /* Number of values in B shifted  */\r
+               SHIFT_REG T2,\OFF_VAL,\C_A              /* Number of values in A shifted  */\r
+               add             \PTR_B, \B_VAL ,        T4                              /* Add values to BO */\r
+               add             \PTR_A, \PTR_A, T2                              /* Add values to AO  */\r
+    #endif \r
+.endm\r
+\r
+\r
+/*\r
+// #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))\r
+//             temp = bk-off;\r
+// #elif defined(LEFT)\r
+//             temp = off+16;  // number of values in A\r
+// #else\r
+//             temp = off+2;   // number of values in B\r
+// #endif\r
+*/\r
+.macro REFRESH_TEMP_BK TEMP_BK,BK_VAL,OFF_VAL,INCR_A,INCR_B\r
+    #if (defined(LEFT) && !defined(TRANSA)) ||  (!defined(LEFT) && defined(TRANSA))\r
+                            /* temp = bk-off;*/\r
+           sub \TEMP_BK,\BK_VAL,\OFF_VAL\r
+\r
+    #elif defined(LEFT)\r
+                            /* temp = off+INCR_A;      // number of values in A */\r
+           addi \TEMP_BK, \OFF_VAL, \INCR_A\r
+    #else\r
+                            /* temp = off+INCR_B       // number of values in B*/\r
+           addi \TEMP_BK,\OFF_VAL, \INCR_B\r
+    #endif\r
+\r
+.endm\r
+/*\r
+// #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))\r
+//             temp = bk - off;\r
+// #ifdef LEFT\r
+//             temp -= 16; // number of values in A\r
+// #else\r
+//             temp -= 2; // number of values in B\r
+// #endif\r
+//             ptrba += temp*16;\r
+//             ptrbb += temp*2;\r
+// #endif\r
+\r
+// #ifdef LEFT\r
+//             off += 16; // number of values in A\r
+// #endif\r
+*/\r
+ \r
+\r
+.macro REFRESH_AFTER_SAVE TEMP_BK,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B\r
+\r
+    #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))\r
+                    /*temp = bk - off;*/\r
+                sub \TEMP_BK,\BK_VAL,\OFF_VAL\r
+    #ifdef LEFT\r
+                    /*temp -= 8; // number of values in A*/\r
+                addi \TEMP_BK,\TEMP_BK,-\C_A\r
+    #else\r
+                    /*temp -= 4; // number of values in B*/\r
+                addi \TEMP_BK,\TEMP_BK,-\C_B \r
+    #endif\r
+                    /*ptrba += temp*C_A;\r
+                    ptrbb += temp*C_B;*/ \r
+                SHIFT_REG T4,\TEMP_BK,\C_A\r
+                                                               SHIFT_REG T2,\TEMP_BK,\C_B\r
+                add \PTR_A, \PTR_A,T4/*ptrba+temp*C_A*/ \r
+                                                               add \PTR_B, \PTR_B,T2 \r
+\r
+    #endif\r
+\r
+    #ifdef LEFT\r
+                    /*off += 8; // number of values in A*/\r
+                 addi \OFF_VAL,\OFF_VAL,\C_A\r
+    #endif\r
+.endm
\ No newline at end of file

diff --git a/param.h b/param.h
index 938a82a..d59cb16 100644 (file)
--- a/param.h
+++ b/param.h
@@ -2248,12 +2248,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define ZGEMM_DEFAULT_UNROLL_M 8
 #define ZGEMM_DEFAULT_UNROLL_N 2
 
-#define SGEMM_DEFAULT_P  1280
+#define SGEMM_DEFAULT_P 640
 #define DGEMM_DEFAULT_P  128
 #define CGEMM_DEFAULT_P  640
 #define ZGEMM_DEFAULT_P  320
 
-#define SGEMM_DEFAULT_Q  640
+#define SGEMM_DEFAULT_Q 1408
 #define DGEMM_DEFAULT_Q  384
 #define CGEMM_DEFAULT_Q  640
 #define ZGEMM_DEFAULT_Q  640