POWER10: Avoid setting accumulators to zero in gemm kernels

For the first iteration, it is better to use xvf*ger instead of xvf*gerpp
builtins which helps to avoid setting accumulators to zero. This helps
to reduce few instructions.

diff --git a/kernel/power/dgemm_kernel_power10.c b/kernel/power/dgemm_kernel_power10.c
index a0bc1a7..b2a2914 100644 (file)
--- a/kernel/power/dgemm_kernel_power10.c
+++ b/kernel/power/dgemm_kernel_power10.c
@@ -87,22 +87,6 @@ typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
           rowC[0] += result[1] * alpha;
 #endif
 
-#define SET_ACC_ZERO4() \
-          __builtin_mma_xxsetaccz (&acc0); \
-          __builtin_mma_xxsetaccz (&acc1); \
-          __builtin_mma_xxsetaccz (&acc2); \
-          __builtin_mma_xxsetaccz (&acc3);
-
-#define SET_ACC_ZERO8() \
-          __builtin_mma_xxsetaccz (&acc0); \
-          __builtin_mma_xxsetaccz (&acc1); \
-          __builtin_mma_xxsetaccz (&acc2); \
-          __builtin_mma_xxsetaccz (&acc3); \
-          __builtin_mma_xxsetaccz (&acc4); \
-          __builtin_mma_xxsetaccz (&acc5); \
-          __builtin_mma_xxsetaccz (&acc6); \
-          __builtin_mma_xxsetaccz (&acc7);
-
 #define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");
 
 #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
@@ -210,12 +194,22 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          PREFETCH1 (CO + ldc + ldc, 128);
          PREFETCH1 (CO + ldc + ldc + ldc, 128);
          __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
-         SET_ACC_ZERO8 ();
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & BO[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]);
+         __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]);
+         __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]);
+         __builtin_mma_xvf64ger (&acc4, rowB, rowA[4]);
+         __builtin_mma_xvf64ger (&acc5, rowB, rowA[5]);
+         __builtin_mma_xvf64ger (&acc6, rowB, rowA[6]);
+         __builtin_mma_xvf64ger (&acc7, rowB, rowA[7]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 4];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 4];
+             rb = (vec_t *) & BO[l << 2];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
              __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
@@ -254,13 +248,19 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1, acc2, acc3;
-         SET_ACC_ZERO4 ();
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & BO[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]);
+         __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]);
+         __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 3];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 3];
+             rb = (vec_t *) & BO[l << 2];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
              __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
@@ -291,14 +291,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1;
-         __builtin_mma_xxsetaccz (&acc0);
-         __builtin_mma_xxsetaccz (&acc1);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & BO[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 2];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 2];
+             rb = (vec_t *) & BO[l << 2];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
              __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
@@ -325,13 +328,16 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0;
-         __builtin_mma_xxsetaccz (&acc0);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & BO[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 1];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 1];
+             rb = (vec_t *) & BO[l << 2];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
            }
@@ -414,16 +420,27 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
-         SET_ACC_ZERO8 ();
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0, 0, 0, 0 };
+         t[0] = BO[0], t[1] = BO[1];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & t[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         vec_t *rowA = (vec_t *) & AO[0];
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]);
+         __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]);
+         __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]);
+         __builtin_mma_xvf64ger (&acc4, rowB, rowA[4]);
+         __builtin_mma_xvf64ger (&acc5, rowB, rowA[5]);
+         __builtin_mma_xvf64ger (&acc6, rowB, rowA[6]);
+         __builtin_mma_xvf64ger (&acc7, rowB, rowA[7]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0, 0, 0, 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & t[0];
+             rb = (vec_t *) & t[0];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
-             vec_t *rowA = (vec_t *) & AO[l << 4];
+             rowA = (vec_t *) & AO[l << 4];
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
              __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
              __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
@@ -461,16 +478,23 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1, acc2, acc3;
-         SET_ACC_ZERO4 ();
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0, 0, 0, 0 };
+         t[0] = BO[0], t[1] = BO[1];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & t[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         vec_t *rowA = (vec_t *) & AO[0];
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]);
+         __builtin_mma_xvf64ger (&acc2, rowB, rowA[2]);
+         __builtin_mma_xvf64ger (&acc3, rowB, rowA[3]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0, 0, 0, 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & t[0];
+             rb = (vec_t *) & t[0];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
-             vec_t *rowA = (vec_t *) & AO[l << 3];
+             rowA = (vec_t *) & AO[l << 3];
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
              __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
              __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);
@@ -500,17 +524,21 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1;
-         __builtin_mma_xxsetaccz (&acc0);
-         __builtin_mma_xxsetaccz (&acc1);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0, 0, 0, 0 };
+         t[0] = BO[0], t[1] = BO[1];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & t[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         vec_t *rowA = (vec_t *) & AO[0];
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         __builtin_mma_xvf64ger (&acc1, rowB, rowA[1]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0, 0, 0, 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & t[0];
+             rb = (vec_t *) & t[0];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
-             vec_t *rowA = (vec_t *) & AO[l << 2];
+             rowA = (vec_t *) & AO[l << 2];
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
              __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);
            }
@@ -536,16 +564,20 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0;
-         __builtin_mma_xxsetaccz (&acc0);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0, 0, 0, 0 };
+         t[0] = BO[0], t[1] = BO[1];
+         __vector_pair rowB;
+         vec_t *rb = (vec_t *) & t[0];
+         __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
+         vec_t *rowA = (vec_t *) & AO[0];
+         __builtin_mma_xvf64ger (&acc0, rowB, rowA[0]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0, 0, 0, 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             __vector_pair rowB;
-             vec_t *rb = (vec_t *) & t[0];
+             rb = (vec_t *) & t[0];
              __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);
-             vec_t *rowA = (vec_t *) & AO[l << 1];
+             rowA = (vec_t *) & AO[l << 1];
              __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
            }
          SAVE2x4_ACC (&acc0, 0);

diff --git a/kernel/power/sgemm_kernel_power10.c b/kernel/power/sgemm_kernel_power10.c
index 81a5ec7..9fbf846 100644 (file)
--- a/kernel/power/sgemm_kernel_power10.c
+++ b/kernel/power/sgemm_kernel_power10.c
@@ -134,21 +134,6 @@ typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));
           __builtin_mma_xvf32gerpp (&acc5, rowB[i+1], rowA[j+2]); \
           __builtin_mma_xvf32gerpp (&acc6, rowB[i], rowA[j+3]); \
           __builtin_mma_xvf32gerpp (&acc7, rowB[i+1], rowA[j+3]);
-#define SET_ACC_ZERO4() \
-          __builtin_mma_xxsetaccz (&acc0); \
-          __builtin_mma_xxsetaccz (&acc1); \
-          __builtin_mma_xxsetaccz (&acc2); \
-          __builtin_mma_xxsetaccz (&acc3);
-
-#define SET_ACC_ZERO8() \
-          __builtin_mma_xxsetaccz (&acc0); \
-          __builtin_mma_xxsetaccz (&acc1); \
-          __builtin_mma_xxsetaccz (&acc2); \
-          __builtin_mma_xxsetaccz (&acc3); \
-          __builtin_mma_xxsetaccz (&acc4); \
-          __builtin_mma_xxsetaccz (&acc5); \
-          __builtin_mma_xxsetaccz (&acc6); \
-          __builtin_mma_xxsetaccz (&acc7);
 
 #define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");
 
@@ -249,8 +234,20 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
-         SET_ACC_ZERO8 ();
          BLASLONG l = 0;
+         vec_t *rowA1 = (vec_t *) & AO[0];
+         vec_t *rowB1 = (vec_t *) & BO[0];
+          __builtin_mma_xvf32ger (&acc0, rowB1[0], rowA1[0]);
+          __builtin_mma_xvf32ger (&acc1, rowB1[1], rowA1[0]);
+          __builtin_mma_xvf32ger (&acc2, rowB1[0], rowA1[1]);
+          __builtin_mma_xvf32ger (&acc3, rowB1[1], rowA1[1]);
+          __builtin_mma_xvf32ger (&acc4, rowB1[0], rowA1[2]);
+          __builtin_mma_xvf32ger (&acc5, rowB1[1], rowA1[2]);
+          __builtin_mma_xvf32ger (&acc6, rowB1[0], rowA1[3]);
+          __builtin_mma_xvf32ger (&acc7, rowB1[1], rowA1[3]);
+         AO += 16;
+         BO += 8;
+         temp--;
          BLASLONG K = temp / 64;
          for (l = 0; l < K; l++)
            {
@@ -454,12 +451,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1, acc2, acc3;
-         SET_ACC_ZERO4 ();
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[1], rowA[0]);
+         __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[1]);
+         __builtin_mma_xvf32ger (&acc3, rowB[1], rowA[1]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 3];
-             vec_t *rowB = (vec_t *) & BO[l << 3];
+             rowA = (vec_t *) & AO[l << 3];
+             rowB = (vec_t *) & BO[l << 3];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[1]);
@@ -489,13 +491,15 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1;
-         __builtin_mma_xxsetaccz (&acc0);
-         __builtin_mma_xxsetaccz (&acc1);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[1], rowA[0]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 2];
-             vec_t *rowB = (vec_t *) & BO[l << 3];
+             rowA = (vec_t *) & AO[l << 2];
+             rowB = (vec_t *) & BO[l << 3];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);
            }
@@ -522,15 +526,18 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v2sf_t *rowC;
          v2sf_t result[8];
          __vector_quad acc0, acc1;
-         __builtin_mma_xxsetaccz (&acc0);
-         __builtin_mma_xxsetaccz (&acc1);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0 };
+         t[0] = AO[0], t[1] = AO[1];
+         vec_t *rowA = (vec_t *) & t[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[1], rowA[0]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0 };
              t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1];
-             vec_t *rowA = (vec_t *) & t[0];
-             vec_t *rowB = (vec_t *) & BO[l << 3];
+             rowA = (vec_t *) & t[0];
+             rowB = (vec_t *) & BO[l << 3];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);
            }
@@ -625,13 +632,23 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          FLOAT *A1;
          A1 = AO + (16 * k);
          __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
-         SET_ACC_ZERO8 ();
          BLASLONG l = 0;
-         for (l = 0; l < k; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         vec_t *rowA1 = (vec_t *) & A1[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]);
+         __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]);
+         __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]);
+         __builtin_mma_xvf32ger (&acc4, rowB[0], rowA1[0]);
+         __builtin_mma_xvf32ger (&acc5, rowB[0], rowA1[1]);
+         __builtin_mma_xvf32ger (&acc6, rowB[0], rowA1[2]);
+         __builtin_mma_xvf32ger (&acc7, rowB[0], rowA1[3]);
+         for (l = 1; l < k; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 4];
-             vec_t *rowA1 = (vec_t *) & A1[l << 4];
-             vec_t *rowB = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 4];
+             rowA1 = (vec_t *) & A1[l << 4];
+             rowB = (vec_t *) & BO[l << 2];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
              __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
@@ -673,12 +690,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1, acc2, acc3;
-         SET_ACC_ZERO4 ();
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]);
+         __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]);
+         __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 4];
-             vec_t *rowB = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 4];
+             rowB = (vec_t *) & BO[l << 2];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
              __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
@@ -710,13 +732,15 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1;
-         __builtin_mma_xxsetaccz (&acc0);
-         __builtin_mma_xxsetaccz (&acc1);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 3];
-             vec_t *rowB = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 3];
+             rowB = (vec_t *) & BO[l << 2];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
            }
@@ -742,12 +766,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          __vector_quad acc0;
          v4sf_t result[4];
-         __builtin_mma_xxsetaccz (&acc0);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         vec_t *rowA = (vec_t *) & AO[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         for (l = 1; l < temp; l++)
            {
-             vec_t *rowA = (vec_t *) & AO[l << 2];
-             vec_t *rowB = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & AO[l << 2];
+             rowB = (vec_t *) & BO[l << 2];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
            }
          SAVE_ACC (&acc0, 0);
@@ -771,14 +797,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v2sf_t *rowC;
          v2sf_t result[8];
          __vector_quad acc0;
-         __builtin_mma_xxsetaccz (&acc0);
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0 };
+         t[0] = AO[0], t[1] = AO[1];
+         vec_t *rowA = (vec_t *) & t[0];
+         vec_t *rowB = (vec_t *) & BO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0 };
              t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1];
-             vec_t *rowA = (vec_t *) & t[0];
-             vec_t *rowB = (vec_t *) & BO[l << 2];
+             rowA = (vec_t *) & t[0];
+             rowB = (vec_t *) & BO[l << 2];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
            }
          SAVE4x2_ACC (&acc0, 0);
@@ -856,15 +885,26 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          FLOAT *A1;
          A1 = AO + (16 * k);
          __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;
-         SET_ACC_ZERO8 ();
          BLASLONG l = 0;
-         for (l = 0; l < k; l++)
+         FLOAT t[4] = { 0 };
+         t[0] = BO[0], t[1] = BO[1];
+         vec_t *rowB = (vec_t *) & t[0];
+         vec_t *rowA = (vec_t *) & AO[0];
+         vec_t *rowA1 = (vec_t *) & A1[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]);
+         __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]);
+         __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]);
+         __builtin_mma_xvf32ger (&acc4, rowB[0], rowA1[0]);
+         __builtin_mma_xvf32ger (&acc5, rowB[0], rowA1[1]);
+         __builtin_mma_xvf32ger (&acc6, rowB[0], rowA1[2]);
+         __builtin_mma_xvf32ger (&acc7, rowB[0], rowA1[3]);
+         for (l = 1; l < k; l++)
            {
-             FLOAT t[4] = { 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             vec_t *rowB = (vec_t *) & t[0];
-             vec_t *rowA = (vec_t *) & AO[l << 4];
-             vec_t *rowA1 = (vec_t *) & A1[l << 4];
+             rowB = (vec_t *) & t[0];
+             rowA = (vec_t *) & AO[l << 4];
+             rowA1 = (vec_t *) & A1[l << 4];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
              __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
@@ -897,7 +937,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1, acc2, acc3;
-         SET_ACC_ZERO4 ();
          BLASLONG l = 0;
 #if defined(TRMMKERNEL)
          REFRESH_POINTERS (16, 2)
@@ -905,12 +944,19 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          BO = B;
          temp = k;
 #endif
-         for (l = 0; l < temp; l++)
+        FLOAT t[4] = { 0 };
+        t[0] = BO[0], t[1] = BO[1];
+        vec_t *rowB = (vec_t *) & t[0];
+        vec_t *rowA = (vec_t *) & AO[0];
+        __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+        __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]);
+        __builtin_mma_xvf32ger (&acc2, rowB[0], rowA[2]);
+        __builtin_mma_xvf32ger (&acc3, rowB[0], rowA[3]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             vec_t *rowB = (vec_t *) & t[0];
-             vec_t *rowA = (vec_t *) & AO[l << 4];
+             rowB = (vec_t *) & t[0];
+             rowA = (vec_t *) & AO[l << 4];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
              __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);
@@ -934,8 +980,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0, acc1;
-         __builtin_mma_xxsetaccz (&acc0);
-         __builtin_mma_xxsetaccz (&acc1);
 #if defined(TRMMKERNEL)
          REFRESH_POINTERS (8, 2)
 #else
@@ -943,12 +987,17 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          temp = k;
 #endif
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0 };
+         t[0] = BO[0], t[1] = BO[1];
+         vec_t *rowB = (vec_t *) & t[0];
+         vec_t *rowA = (vec_t *) & AO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         __builtin_mma_xvf32ger (&acc1, rowB[0], rowA[1]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             vec_t *rowB = (vec_t *) & t[0];
-             vec_t *rowA = (vec_t *) & AO[l << 3];
+             rowB = (vec_t *) & t[0];
+             rowA = (vec_t *) & AO[l << 3];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
              __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);
            }
@@ -968,7 +1017,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          v4sf_t *rowC;
          v4sf_t result[4];
          __vector_quad acc0;
-         __builtin_mma_xxsetaccz (&acc0);
 #if defined(TRMMKERNEL)
          REFRESH_POINTERS (4, 2)
 #else
@@ -976,12 +1024,16 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
          temp = k;
 #endif
          BLASLONG l = 0;
-         for (l = 0; l < temp; l++)
+         FLOAT t[4] = { 0 };
+         t[0] = BO[0], t[1] = BO[1];
+         vec_t *rowB = (vec_t *) & t[0];
+         vec_t *rowA = (vec_t *) & AO[0];
+         __builtin_mma_xvf32ger (&acc0, rowB[0], rowA[0]);
+         for (l = 1; l < temp; l++)
            {
-             FLOAT t[4] = { 0 };
              t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];
-             vec_t *rowB = (vec_t *) & t[0];
-             vec_t *rowA = (vec_t *) & AO[l << 2];
+             rowB = (vec_t *) & t[0];
+             rowA = (vec_t *) & AO[l << 2];
              __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);
            }
          SAVE2x4_ACC (&acc0, 0);