SBGEMM_BETA = ../generic/gemm_beta.c
SBGEMMKERNEL = sbgemm_kernel_power10.c
-SBGEMMINCOPY = ../generic/gemm_ncopy_16.c
-SBGEMMITCOPY = ../generic/gemm_tcopy_16.c
-SBGEMMONCOPY = ../generic/gemm_ncopy_8.c
-SBGEMMOTCOPY = ../generic/gemm_tcopy_8.c
+SBGEMMINCOPY = sbgemm_ncopy_16_power10.c
+SBGEMMITCOPY = sbgemm_tcopy_16_power10.c
+SBGEMMONCOPY = sbgemm_ncopy_8_power10.c
+SBGEMMOTCOPY = sbgemm_tcopy_8_power10.c
SBGEMMINCOPYOBJ = sbgemm_incopy$(TSUFFIX).$(SUFFIX)
SBGEMMITCOPYOBJ = sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
SBGEMMONCOPYOBJ = sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
IFLOAT * B, FLOAT * C, BLASLONG ldc)
{
- BLASLONG N = n;
BLASLONG i1;
v4sf_t valpha = { alpha, alpha, alpha, alpha };
vector short vzero = { 0, 0, 0, 0, 0, 0, 0, 0 };
- N = n >> 3;
/* Loop for n >= 8. */
- for (i1 = 0; i1 < N; i1++)
+ for (i1 = 0; i1 < (n >> 3); i1++)
{
- BLASLONG i, j;
+ BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
AO = A;
PREFETCH1 (A, 128);
PREFETCH1 (A, 256);
- i = m >> 4;
/* Loop for m >= 16. */
- for (j = 0; j < i; j++)
+ for (j = 0; j < (m >> 4); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
{
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
- vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
- vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
- vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[2]);
- vec_t rowA_l = MERGE_LOW (rowA[0], rowA[2]);
- vec_t rowA2_h = MERGE_HIGH (rowA[1], rowA[3]);
- vec_t rowA2_l = MERGE_LOW (rowA[1], rowA[3]);
- MMA (&acc0, rowB_h, rowA_h);
- MMA (&acc1, rowB_l, rowA_h);
- MMA (&acc2, rowB_h, rowA_l);
- MMA (&acc3, rowB_l, rowA_l);
- MMA (&acc4, rowB_h, rowA2_h);
- MMA (&acc5, rowB_l, rowA2_h);
- MMA (&acc6, rowB_h, rowA2_l);
- MMA (&acc7, rowB_l, rowA2_l);
+ MMA (&acc0, rowB[0], rowA[0]);
+ MMA (&acc1, rowB[1], rowA[0]);
+ MMA (&acc2, rowB[0], rowA[1]);
+ MMA (&acc3, rowB[1], rowA[1]);
+ MMA (&acc4, rowB[0], rowA[2]);
+ MMA (&acc5, rowB[1], rowA[2]);
+ MMA (&acc6, rowB[0], rowA[3]);
+ MMA (&acc7, rowB[1], rowA[3]);
}
if (k % 2 == 1)
{
AO += (k << 4);
BO += (k << 3);
}
- i = (m & 15) >> 3;
- /* Loop for m >= 8. */
- for (j = 0; j < i; j++)
+ if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
{
vec_t *rowA = (vec_t *) & (AO[l << 4]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
- vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
- vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
- vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[1]);
- vec_t rowA_l = MERGE_LOW (rowA[0], rowA[1]);
- MMA (&acc0, rowB_h, rowA_h);
- MMA (&acc1, rowB_l, rowA_h);
- MMA (&acc2, rowB_h, rowA_l);
- MMA (&acc3, rowB_l, rowA_l);
+
+ MMA (&acc0, rowB[0], rowA[0]);
+ MMA (&acc1, rowB[1], rowA[0]);
+ MMA (&acc2, rowB[0], rowA[1]);
+ MMA (&acc3, rowB[1], rowA[1]);
}
if (k % 2 == 1)
{
AO += (k << 3);
BO += (k << 3);
}
- i = (m & 7) >> 2;
- /* Loop for m >= 4. */
- for (j = 0; j < i; j++)
+ if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
{
vec_t *rowA = (vec_t *) & (AO[l << 3]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
- vec_t rowA_mrg = MERGE_ROW (rowA[0]);
- MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), rowA_mrg);
- MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), rowA_mrg);
+ MMA (&acc0, rowB[0], rowA[0]);
+ MMA (&acc1, rowB[1], rowA[0]);
}
if (k % 2 == 1)
{
AO += (k << 2);
BO += (k << 3);
}
- i = (m & 3) >> 1;
- /* Loop for m >= 2. */
- for (j = 0; j < i; j++)
+ if (m & 2)
{
IFLOAT *BO = B;
v2sf_t *rowC;
0, 0, 0, 0
};
vec_t *rowB = (vec_t *) & (BO[l << 4]);
- MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
- MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
+ MMA (&acc0, rowB[0], (vec_t) rowA);
+ MMA (&acc1, rowB[1], (vec_t) rowA);
}
if (k % 2 == 1)
{
AO += (k << 1);
BO += (k << 3);
}
- i = (m & 1) >> 0;
- /* Loop for m = 1. */
- for (j = 0; j < i; j++)
+ if (m & 1)
{
IFLOAT *BO = B;
+ v2sf_t *rowC;
+ v2sf_t result[8];
+ __vector_quad acc0, acc1;
+ __builtin_mma_xxsetaccz (&acc0);
+ __builtin_mma_xxsetaccz (&acc1);
BLASLONG l = 0;
- v4sf_t t = { 0, 0, 0, 0 }
- , t1 =
- {
- 0, 0, 0, 0};
- for (l = 0; l < k; l++)
+ for (l = 0; l < k / 2; l++)
{
- v4sf_t rowA =
- { BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
- BF16TOF32 (AO[l])
- };
- v4sf_t rowB =
- { BF16TOF32 (BO[l << 3]), BF16TOF32 (BO[(l << 3) + 1]),
- BF16TOF32 (BO[(l << 3) + 2]),
- BF16TOF32 (BO[(l << 3) + 3])
- };
- v4sf_t rowB1 =
- { BF16TOF32 (BO[(l << 3) + 4]), BF16TOF32 (BO[(l << 3) + 5]),
- BF16TOF32 (BO[(l << 3) + 6]),
- BF16TOF32 (BO[(l << 3) + 7])
- };
- t += rowA * rowB;
- t1 += rowA * rowB1;
+ vector short rowA =
+ { AO[(l << 1) + 0], AO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
+ vec_t *rowB = (vec_t *) & (BO[l << 4]);
+ MMA (&acc0, rowB[0], (vec_t) rowA);
+ MMA (&acc1, rowB[1], (vec_t) rowA);
}
- t = t * valpha;
- t1 = t1 * valpha;
- CO[0 * ldc] += t[0];
- CO[1 * ldc] += t[1];
- CO[2 * ldc] += t[2];
- CO[3 * ldc] += t[3];
- CO[4 * ldc] += t1[0];
- CO[5 * ldc] += t1[1];
- CO[6 * ldc] += t1[2];
- CO[7 * ldc] += t1[3];
+ if (k % 2 == 1)
+ {
+ if (k > 1)
+ l = (k / 2) << 1;
+ vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 };
+ vec_t *rowB = (vec_t *) & (BO[(l << 3)]);
+ MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
+ MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
+ }
+ SAVE4x2_ACC (&acc0, 0);
+ SAVE4x2_ACC1 (&acc1, 0);
CO += 1;
AO += k;
BO += (k << 3);
}
B += k << 3;
}
- N = (n & 7) >> 2;
- /* Loop for n >= 4. */
- for (i1 = 0; i1 < N; i1++)
+ if (n & 4)
{
- BLASLONG i, j;
+ BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc << 2;
AO = A;
- i = m >> 5;
/* Loop for m >= 32. */
- for (j = 0; j < i; j++)
+ for (j = 0; j < (m >> 5); j++)
{
IFLOAT *BO = B;
IFLOAT *A1 = AO + (16 * k);
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
- vec_t rowB_mrg = MERGE_ROW (rowB[0]);
- MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
- MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
- MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
- MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
- MMA (&acc4, rowB_mrg, MERGE_HIGH (rowA1[0], rowA1[2]));
- MMA (&acc5, rowB_mrg, MERGE_LOW (rowA1[0], rowA1[2]));
- MMA (&acc6, rowB_mrg, MERGE_HIGH (rowA1[1], rowA1[3]));
- MMA (&acc7, rowB_mrg, MERGE_LOW (rowA1[1], rowA1[3]));
+ MMA (&acc0, rowB[0], rowA[0]);
+ MMA (&acc1, rowB[0], rowA[1]);
+ MMA (&acc2, rowB[0], rowA[2]);
+ MMA (&acc3, rowB[0], rowA[3]);
+ MMA (&acc4, rowB[0], rowA1[0]);
+ MMA (&acc5, rowB[0], rowA1[1]);
+ MMA (&acc6, rowB[0], rowA1[2]);
+ MMA (&acc7, rowB[0], rowA1[3]);
}
if (k % 2 == 1)
{
AO += k << 5;
BO += k << 2;
}
- i = (m & 31) >> 4;
- /* Loop for m >= 16. */
- for (j = 0; j < i; j++)
+ if (m & 16)
{
IFLOAT *BO = B;
v4sf_t *rowC;
{
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
- vec_t rowB_mrg = MERGE_ROW (rowB[0]);
- MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
- MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
- MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
- MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
+ MMA (&acc0, rowB[0], rowA[0]);
+ MMA (&acc1, rowB[0], rowA[1]);
+ MMA (&acc2, rowB[0], rowA[2]);
+ MMA (&acc3, rowB[0], rowA[3]);
}
if (k % 2 == 1)
{
AO += k << 4;
BO += k << 2;
}
- i = (m & 15) >> 3;
- /* Loop for m >= 8. */
- for (j = 0; j < i; j++)
+ if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
{
vec_t *rowA = (vec_t *) & (AO[l << 4]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
- vec_t rowB_mrg = MERGE_ROW (rowB[0]);
- MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[1]));
- MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[1]));
+ MMA (&acc0, rowB[0], rowA[0]);
+ MMA (&acc1, rowB[0], rowA[1]);
}
if (k % 2 == 1)
{
AO += k << 3;
BO += k << 2;
}
- i = (m & 7) >> 2;
- /* Loop for m >= 4. */
- for (j = 0; j < i; j++)
+ if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
{
vec_t *rowA = (vec_t *) & (AO[l << 3]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
- MMA (&acc0, MERGE_ROW (rowB[0]), MERGE_ROW (rowA[0]));
+ MMA (&acc0, rowB[0], rowA[0]);
}
if (k % 2 == 1)
{
AO += k << 2;
BO += k << 2;
}
- i = (m & 3) >> 1;
- /* Loop for m >= 2. */
- for (j = 0; j < i; j++)
+ if (m & 2)
{
IFLOAT *BO = B;
v2sf_t *rowC;
0, 0, 0, 0
};
vec_t *rowB = (vec_t *) & (BO[l << 3]);
- MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
+ MMA (&acc0, rowB[0], (vec_t) rowA);
}
if (k % 2 == 1)
{
AO += k << 1;
BO += k << 2;
}
- i = (m & 1) >> 0;
- /* Loop for m = 1. */
- for (j = 0; j < i; j++)
+ if (m & 1)
{
IFLOAT *BO = B;
+ v2sf_t *rowC;
+ v2sf_t result[8];
+ __vector_quad acc0;
BLASLONG l = 0;
- v4sf_t t = { 0, 0, 0, 0 };
- for (l = 0; l < k; l++)
+ __builtin_mma_xxsetaccz (&acc0);
+ for (l = 0; l < k / 2; l++)
{
- v4sf_t rowA =
- { BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
- BF16TOF32 (AO[l])
- };
- v4sf_t rowB =
- { BF16TOF32 (BO[l << 2]), BF16TOF32 (BO[(l << 2) + 1]),
- BF16TOF32 (BO[(l << 2) + 2]),
- BF16TOF32 (BO[(l << 2) + 3])
+ vector short rowA =
+ { AO[(l << 1) + 0], AO[(l << 1) + 1], 0,
+ 0, 0, 0, 0
};
- t += rowA * rowB;
+ vec_t *rowB = (vec_t *) & (BO[l << 3]);
+ MMA (&acc0, rowB[0], (vec_t) rowA);
}
- t = t * valpha;
- CO[0 * ldc] += t[0];
- CO[1 * ldc] += t[1];
- CO[2 * ldc] += t[2];
- CO[3 * ldc] += t[3];
+ if (k % 2 == 1)
+ {
+ if (k > 1)
+ l = (k / 2) << 1;
+ vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 };
+ vec_t *rowB = (vec_t *) & (BO[l << 2]);
+ MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
+ }
+ SAVE4x2_ACC (&acc0, 0);
AO += k;
BO += (k << 2);
CO += 1;
B += k << 2;
}
- N = (n & 3) >> 1;
- /* Loop for n >= 2. */
- for (i1 = 0; i1 < N; i1++)
+ if (n & 2)
{
- BLASLONG i, j;
+ BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc << 1;
AO = A;
- i = m >> 5;
/* Loop for m >= 32. */
- for (j = 0; j < i; j++)
+ for (j = 0; j < (m >> 5); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
- MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
- MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
- MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
- MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
- MMA (&acc4, (vec_t) rowB, MERGE_HIGH (rowA1[0], rowA1[2]));
- MMA (&acc5, (vec_t) rowB, MERGE_LOW (rowA1[0], rowA1[2]));
- MMA (&acc6, (vec_t) rowB, MERGE_HIGH (rowA1[1], rowA1[3]));
- MMA (&acc7, (vec_t) rowB, MERGE_LOW (rowA1[1], rowA1[3]));
+ MMA (&acc0, (vec_t) rowB, rowA[0]);
+ MMA (&acc1, (vec_t) rowB, rowA[1]);
+ MMA (&acc2, (vec_t) rowB, rowA[2]);
+ MMA (&acc3, (vec_t) rowB, rowA[3]);
+ MMA (&acc4, (vec_t) rowB, rowA1[0]);
+ MMA (&acc5, (vec_t) rowB, rowA1[1]);
+ MMA (&acc6, (vec_t) rowB, rowA1[2]);
+ MMA (&acc7, (vec_t) rowB, rowA1[3]);
}
if (k % 2 == 1)
{
AO += k << 5;
BO += k << 1;
}
- i = (m & 31) >> 4;
- /* Loop for m >= 16. */
- for (j = 0; j < i; j++)
+ if (m & 16)
{
IFLOAT *BO = B;
v4sf_t *rowC;
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
- MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
- MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
- MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
- MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
+ MMA (&acc0, (vec_t) rowB, rowA[0]);
+ MMA (&acc1, (vec_t) rowB, rowA[1]);
+ MMA (&acc2, (vec_t) rowB, rowA[2]);
+ MMA (&acc3, (vec_t) rowB, rowA[3]);
}
if (k % 2 == 1)
{
AO += k << 4;
BO += k << 1;
}
- i = (m & 15) >> 3;
- /* Loop for m >= 8. */
- for (j = 0; j < i; j++)
+ if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 4]);
- MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
- MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
+ MMA (&acc0, (vec_t) rowB, rowA[0]);
+ MMA (&acc1, (vec_t) rowB, rowA[1]);
}
if (k % 2 == 1)
{
AO += k << 3;
BO += k << 1;
}
- i = (m & 7) >> 2;
- /* Loop for m >= 4. */
- for (j = 0; j < i; j++)
+ if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 3]);
- MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
+ MMA (&acc0, (vec_t) rowB, rowA[0]);
}
if (k % 2 == 1)
{
AO += k << 2;
BO += k << 1;
}
- i = (m & 3) >> 1;
- /* Loop for m >= 2. */
- for (j = 0; j < i; j++)
+ if (m & 2)
{
IFLOAT *BO = B;
BLASLONG l = 0;
AO += k << 1;
BO += k << 1;
}
- i = (m & 1) >> 0;
- /* Loop for m = 1. */
- for (j = 0; j < i; j++)
+ if (m & 1)
{
IFLOAT *BO = B;
BLASLONG l = 0;
}
B += k << 1;
}
- N = (n & 1) >> 0;
- /* Loop for n = 1. */
- for (i1 = 0; i1 < N; i1++)
+ if (n & 1)
{
- BLASLONG i;
+ BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc;
AO = A;
- i = m;
/* Loop for m >= 16. */
- while (i >= 16)
+ for (j = 0; j < (m >> 4); j++)
{
IFLOAT *BO = B;
+ v4sf_t *rowC;
+ v4sf_t result[4];
+ __vector_quad acc0, acc1, acc2, acc3;
+ SET_ACC_ZERO4 ();
BLASLONG l = 0;
- v4sf_t t = { 0, 0, 0, 0 };
- v4sf_t t1 = { 0, 0, 0, 0 };
- v4sf_t t2 = { 0, 0, 0, 0 };
- v4sf_t t3 = { 0, 0, 0, 0 };
- for (l = 0; l < k; l++)
+ for (l = 0; l < k / 2; l++)
{
- v4sf_t rowB =
- { BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
- BF16TOF32 (BO[l])
- };
- v4sf_t rowA =
- { BF16TOF32 (AO[l << 4]), BF16TOF32 (AO[(l << 4) + 1]),
- BF16TOF32 (AO[(l << 4) + 2]),
- BF16TOF32 (AO[(l << 4) + 3])
- };
- v4sf_t rowA1 =
- { BF16TOF32 (AO[(l << 4) + 4]), BF16TOF32 (AO[(l << 4) + 5]),
- BF16TOF32 (AO[(l << 4) + 6]),
- BF16TOF32 (AO[(l << 4) + 7])
- };
- v4sf_t rowA2 =
- { BF16TOF32 (AO[(l << 4) + 8]), BF16TOF32 (AO[(l << 4) + 9]),
- BF16TOF32 (AO[(l << 4) + 10]),
- BF16TOF32 (AO[(l << 4) + 11])
- };
- v4sf_t rowA3 = { BF16TOF32 (AO[(l << 4) + 12]),
- BF16TOF32 (AO[(l << 4) + 13]), BF16TOF32 (AO[(l << 4) + 14]),
- BF16TOF32 (AO[(l << 4) + 15])
- };
- t += rowA * rowB;
- t1 += rowA1 * rowB;
- t2 += rowA2 * rowB;
- t3 += rowA3 * rowB;
+ vector short rowB =
+ { BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
+ vec_t *rowA = (vec_t *) & (AO[l << 5]);
+ MMA (&acc0, (vec_t) rowB, rowA[0]);
+ MMA (&acc1, (vec_t) rowB, rowA[1]);
+ MMA (&acc2, (vec_t) rowB, rowA[2]);
+ MMA (&acc3, (vec_t) rowB, rowA[3]);
}
- t = t * valpha;
- t1 = t1 * valpha;
- t2 = t2 * valpha;
- t3 = t3 * valpha;
- CO[0] += t[0];
- CO[1] += t[1];
- CO[2] += t[2];
- CO[3] += t[3];
- CO[4] += t1[0];
- CO[5] += t1[1];
- CO[6] += t1[2];
- CO[7] += t1[3];
- CO[8] += t2[0];
- CO[9] += t2[1];
- CO[10] += t2[2];
- CO[11] += t2[3];
- CO[12] += t3[0];
- CO[13] += t3[1];
- CO[14] += t3[2];
- CO[15] += t3[3];
+ if (k % 2 == 1)
+ {
+ if (k > 1)
+ l = (k / 2) << 1;
+ vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
+ vec_t *rowA = (vec_t *) & (AO[(l << 4)]);
+ MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
+ MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
+ MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
+ MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
+ }
+ rowC = (v4sf_t *) &CO[0];
+ __builtin_mma_disassemble_acc ((void *)result, &acc0);
+ rowC[0] += result[0] * alpha;
+ __builtin_mma_disassemble_acc ((void *)result, &acc1);
+ rowC[1] += result[0] * alpha;
+ __builtin_mma_disassemble_acc ((void *)result, &acc2);
+ rowC[2] += result[0] * alpha;
+ __builtin_mma_disassemble_acc ((void *)result, &acc3);
+ rowC[3] += result[0] * alpha;
AO += k << 4;
BO += k;
CO += 16;
- i -= 16;
}
/* Loop for m >= 8. */
- while (i >= 8)
+ if (m & 8)
{
IFLOAT *BO = B;
+ v4sf_t *rowC;
+ v4sf_t result[4];
+ __vector_quad acc0, acc1;
+ __builtin_mma_xxsetaccz (&acc0);
+ __builtin_mma_xxsetaccz (&acc1);
BLASLONG l = 0;
- v4sf_t t = { 0, 0, 0, 0 };
- v4sf_t t1 = { 0, 0, 0, 0 };
- for (l = 0; l < k; l++)
+ for (l = 0; l < k / 2; l++)
{
- v4sf_t rowB =
- { BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
- BF16TOF32 (BO[l])
- };
- v4sf_t rowA =
- { BF16TOF32 (AO[l << 3]), BF16TOF32 (AO[(l << 3) + 1]),
- BF16TOF32 (AO[(l << 3) + 2]),
- BF16TOF32 (AO[(l << 3) + 3])
- };
- v4sf_t rowA1 =
- { BF16TOF32 (AO[(l << 3) + 4]), BF16TOF32 (AO[(l << 3) + 5]),
- BF16TOF32 (AO[(l << 3) + 6]),
- BF16TOF32 (AO[(l << 3) + 7])
- };
- t += rowA * rowB;
- t1 += rowA1 * rowB;
+ vector short rowB =
+ { BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
+ vec_t *rowA = (vec_t *) & (AO[l << 4]);
+ MMA (&acc0, (vec_t) rowB, rowA[0]);
+ MMA (&acc1, (vec_t) rowB, rowA[1]);
}
- t = t * valpha;
- t1 = t1 * valpha;
- CO[0] += t[0];
- CO[1] += t[1];
- CO[2] += t[2];
- CO[3] += t[3];
- CO[4] += t1[0];
- CO[5] += t1[1];
- CO[6] += t1[2];
- CO[7] += t1[3];
+ if (k % 2 == 1)
+ {
+ if (k > 1)
+ l = (k / 2) << 1;
+ vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
+ vec_t *rowA = (vec_t *) & (AO[(l << 3)]);
+ MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
+ MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
+ }
+ rowC = (v4sf_t *) &CO[0];
+ __builtin_mma_disassemble_acc ((void *)result, &acc0);
+ rowC[0] += result[0] * alpha;
+ __builtin_mma_disassemble_acc ((void *)result, &acc1);
+ rowC[1] += result[0] * alpha;
AO += k << 3;
BO += k;
CO += 8;
- i -= 8;
}
/* Loop for m >= 4. */
- while (i >= 4)
+ if (m & 4)
{
IFLOAT *BO = B;
+ v4sf_t *rowC;
+ v4sf_t result[4];
+ __vector_quad acc0;
+ __builtin_mma_xxsetaccz (&acc0);
BLASLONG l = 0;
- v4sf_t t = { 0, 0, 0, 0 };
- for (l = 0; l < k; l++)
+ for (l = 0; l < k / 2; l++)
{
- v4sf_t rowB =
- { BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
- BF16TOF32 (BO[l])
- };
- v4sf_t rowA =
- { BF16TOF32 (AO[l << 2]), BF16TOF32 (AO[(l << 2) + 1]),
- BF16TOF32 (AO[(l << 2) + 2]),
- BF16TOF32 (AO[(l << 2) + 3])
- };
- t += rowA * rowB;
+ vector short rowB =
+ { BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
+ vec_t *rowA = (vec_t *) & (AO[l << 3]);
+ MMA (&acc0, (vec_t) rowB, rowA[0]);
}
- t = t * valpha;
- CO[0] += t[0];
- CO[1] += t[1];
- CO[2] += t[2];
- CO[3] += t[3];
+ if (k % 2 == 1)
+ {
+ if (k > 1)
+ l = (k / 2) << 1;
+ vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
+ vec_t *rowA = (vec_t *) & (AO[(l << 2)]);
+ MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
+ }
+ rowC = (v4sf_t *) &CO[0];
+ __builtin_mma_disassemble_acc ((void *)result, &acc0);
+ rowC[0] += result[0] * alpha;
AO += k << 2;
BO += k;
CO += 4;
- i -= 4;
}
/* Loop for m >= 2. */
- while (i >= 2)
+ if (m & 2)
{
IFLOAT *BO = B;
BLASLONG l = 0;
AO += k << 1;
BO += k;
CO += 2;
- i -= 2;
}
/* Loop for m = 1. */
- while (i >= 1)
+ if (m & 1)
{
IFLOAT *BO = B;
BLASLONG l = 0;
BO += k;
CO[0] += t * alpha;
CO += 1;
- i -= 1;
}
B += k;
--- /dev/null
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin. */
+/* All rights reserved. */
+/* */
+/* Redistribution and use in source and binary forms, with or */
+/* without modification, are permitted provided that the following */
+/* conditions are met: */
+/* */
+/* 1. Redistributions of source code must retain the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer. */
+/* */
+/* 2. Redistributions in binary form must reproduce the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer in the documentation and/or other materials */
+/* provided with the distribution. */
+/* */
+/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
+/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
+/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
+/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
+/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
+/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
+/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
+/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
+/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
+/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
+/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
+/* POSSIBILITY OF SUCH DAMAGE. */
+/* */
+/* The views and conclusions contained in the software and */
+/* documentation are those of the authors and should not be */
+/* interpreted as representing official policies, either expressed */
+/* or implied, of The University of Texas at Austin. */
+/*********************************************************************/
+
+#include <stdio.h>
+#include "common.h"
+
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
+ BLASLONG i, j;
+
+ IFLOAT *aoffset;
+ IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
+ IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
+ IFLOAT *aoffset9, *aoffset10, *aoffset11, *aoffset12;
+ IFLOAT *aoffset13, *aoffset14, *aoffset15, *aoffset16;
+
+ IFLOAT *boffset;
+ IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+ IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+ IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
+ IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
+ IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
+ IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
+ IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
+ IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;
+
+ aoffset = a;
+ boffset = b;
+
+ j = (n >> 4);
+ if (j > 0){
+ do{
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset3 = aoffset2 + lda;
+ aoffset4 = aoffset3 + lda;
+ aoffset5 = aoffset4 + lda;
+ aoffset6 = aoffset5 + lda;
+ aoffset7 = aoffset6 + lda;
+ aoffset8 = aoffset7 + lda;
+ aoffset9 = aoffset8 + lda;
+ aoffset10 = aoffset9 + lda;
+ aoffset11 = aoffset10 + lda;
+ aoffset12 = aoffset11 + lda;
+ aoffset13 = aoffset12 + lda;
+ aoffset14 = aoffset13 + lda;
+ aoffset15 = aoffset14 + lda;
+ aoffset16 = aoffset15 + lda;
+ aoffset += 16 * lda;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+
+ ctemp05 = *(aoffset3 + 0);
+ ctemp06 = *(aoffset3 + 1);
+ ctemp07 = *(aoffset4 + 0);
+ ctemp08 = *(aoffset4 + 1);
+
+ ctemp09 = *(aoffset5 + 0);
+ ctemp10 = *(aoffset5 + 1);
+ ctemp11 = *(aoffset6 + 0);
+ ctemp12 = *(aoffset6 + 1);
+
+ ctemp13 = *(aoffset7 + 0);
+ ctemp14 = *(aoffset7 + 1);
+ ctemp15 = *(aoffset8 + 0);
+ ctemp16 = *(aoffset8 + 1);
+
+ ctemp17 = *(aoffset9 + 0);
+ ctemp18 = *(aoffset9 + 1);
+ ctemp19 = *(aoffset10 + 0);
+ ctemp20 = *(aoffset10 + 1);
+
+ ctemp21 = *(aoffset11 + 0);
+ ctemp22 = *(aoffset11 + 1);
+ ctemp23 = *(aoffset12 + 0);
+ ctemp24 = *(aoffset12 + 1);
+
+ ctemp25 = *(aoffset13 + 0);
+ ctemp26 = *(aoffset13 + 1);
+ ctemp27 = *(aoffset14 + 0);
+ ctemp28 = *(aoffset14 + 1);
+
+ ctemp29 = *(aoffset15 + 0);
+ ctemp30 = *(aoffset15 + 1);
+ ctemp31 = *(aoffset16 + 0);
+ ctemp32 = *(aoffset16 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ *(boffset + 2) = ctemp03;
+ *(boffset + 3) = ctemp04;
+ *(boffset + 4) = ctemp05;
+ *(boffset + 5) = ctemp06;
+ *(boffset + 6) = ctemp07;
+ *(boffset + 7) = ctemp08;
+
+ *(boffset + 8) = ctemp09;
+ *(boffset + 9) = ctemp10;
+ *(boffset + 10) = ctemp11;
+ *(boffset + 11) = ctemp12;
+ *(boffset + 12) = ctemp13;
+ *(boffset + 13) = ctemp14;
+ *(boffset + 14) = ctemp15;
+ *(boffset + 15) = ctemp16;
+
+ *(boffset + 16) = ctemp17;
+ *(boffset + 17) = ctemp18;
+ *(boffset + 18) = ctemp19;
+ *(boffset + 19) = ctemp20;
+ *(boffset + 20) = ctemp21;
+ *(boffset + 21) = ctemp22;
+ *(boffset + 22) = ctemp23;
+ *(boffset + 23) = ctemp24;
+
+ *(boffset + 24) = ctemp25;
+ *(boffset + 25) = ctemp26;
+ *(boffset + 26) = ctemp27;
+ *(boffset + 27) = ctemp28;
+ *(boffset + 28) = ctemp29;
+ *(boffset + 29) = ctemp30;
+ *(boffset + 30) = ctemp31;
+ *(boffset + 31) = ctemp32;
+
+ aoffset1 += 2;
+ aoffset2 += 2;
+ aoffset3 += 2;
+ aoffset4 += 2;
+ aoffset5 += 2;
+ aoffset6 += 2;
+ aoffset7 += 2;
+ aoffset8 += 2;
+
+ aoffset9 += 2;
+ aoffset10 += 2;
+ aoffset11 += 2;
+ aoffset12 += 2;
+ aoffset13 += 2;
+ aoffset14 += 2;
+ aoffset15 += 2;
+ aoffset16 += 2;
+ boffset += 32;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp05 = *(aoffset3 + 0);
+ ctemp07 = *(aoffset4 + 0);
+ ctemp09 = *(aoffset5 + 0);
+ ctemp11 = *(aoffset6 + 0);
+ ctemp13 = *(aoffset7 + 0);
+ ctemp15 = *(aoffset8 + 0);
+
+ ctemp17 = *(aoffset9 + 0);
+ ctemp19 = *(aoffset10 + 0);
+ ctemp21 = *(aoffset11 + 0);
+ ctemp23 = *(aoffset12 + 0);
+ ctemp25 = *(aoffset13 + 0);
+ ctemp27 = *(aoffset14 + 0);
+ ctemp29 = *(aoffset15 + 0);
+ ctemp31 = *(aoffset16 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp03;
+ *(boffset + 2) = ctemp05;
+ *(boffset + 3) = ctemp07;
+ *(boffset + 4) = ctemp09;
+ *(boffset + 5) = ctemp11;
+ *(boffset + 6) = ctemp13;
+ *(boffset + 7) = ctemp15;
+
+ *(boffset + 8) = ctemp17;
+ *(boffset + 9) = ctemp19;
+ *(boffset + 10) = ctemp21;
+ *(boffset + 11) = ctemp23;
+ *(boffset + 12) = ctemp25;
+ *(boffset + 13) = ctemp27;
+ *(boffset + 14) = ctemp29;
+ *(boffset + 15) = ctemp31;
+
+ boffset += 16;
+ }
+ j--;
+ }while(j > 0);
+ } /* end of if(j > 0) */
+
+ if (n & 8){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset3 = aoffset2 + lda;
+ aoffset4 = aoffset3 + lda;
+ aoffset5 = aoffset4 + lda;
+ aoffset6 = aoffset5 + lda;
+ aoffset7 = aoffset6 + lda;
+ aoffset8 = aoffset7 + lda;
+ aoffset += 8 * lda;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+
+ ctemp05 = *(aoffset3 + 0);
+ ctemp06 = *(aoffset3 + 1);
+ ctemp07 = *(aoffset4 + 0);
+ ctemp08 = *(aoffset4 + 1);
+
+ ctemp09 = *(aoffset5 + 0);
+ ctemp10 = *(aoffset5 + 1);
+ ctemp11 = *(aoffset6 + 0);
+ ctemp12 = *(aoffset6 + 1);
+
+ ctemp13 = *(aoffset7 + 0);
+ ctemp14 = *(aoffset7 + 1);
+ ctemp15 = *(aoffset8 + 0);
+ ctemp16 = *(aoffset8 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ *(boffset + 2) = ctemp03;
+ *(boffset + 3) = ctemp04;
+ *(boffset + 4) = ctemp05;
+ *(boffset + 5) = ctemp06;
+ *(boffset + 6) = ctemp07;
+ *(boffset + 7) = ctemp08;
+
+ *(boffset + 8) = ctemp09;
+ *(boffset + 9) = ctemp10;
+ *(boffset + 10) = ctemp11;
+ *(boffset + 11) = ctemp12;
+ *(boffset + 12) = ctemp13;
+ *(boffset + 13) = ctemp14;
+ *(boffset + 14) = ctemp15;
+ *(boffset + 15) = ctemp16;
+
+ aoffset1 += 2;
+ aoffset2 += 2;
+ aoffset3 += 2;
+ aoffset4 += 2;
+ aoffset5 += 2;
+ aoffset6 += 2;
+ aoffset7 += 2;
+ aoffset8 += 2;
+
+ boffset += 16;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp05 = *(aoffset3 + 0);
+ ctemp07 = *(aoffset4 + 0);
+ ctemp09 = *(aoffset5 + 0);
+ ctemp11 = *(aoffset6 + 0);
+ ctemp13 = *(aoffset7 + 0);
+ ctemp15 = *(aoffset8 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp03;
+ *(boffset + 2) = ctemp05;
+ *(boffset + 3) = ctemp07;
+ *(boffset + 4) = ctemp09;
+ *(boffset + 5) = ctemp11;
+ *(boffset + 6) = ctemp13;
+ *(boffset + 7) = ctemp15;
+
+ boffset += 8;
+ }
+ }
+
+ if (n & 4){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset3 = aoffset2 + lda;
+ aoffset4 = aoffset3 + lda;
+ aoffset += 4 * lda;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+
+ ctemp05 = *(aoffset3 + 0);
+ ctemp06 = *(aoffset3 + 1);
+ ctemp07 = *(aoffset4 + 0);
+ ctemp08 = *(aoffset4 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ *(boffset + 2) = ctemp03;
+ *(boffset + 3) = ctemp04;
+ *(boffset + 4) = ctemp05;
+ *(boffset + 5) = ctemp06;
+ *(boffset + 6) = ctemp07;
+ *(boffset + 7) = ctemp08;
+
+ aoffset1 += 2;
+ aoffset2 += 2;
+ aoffset3 += 2;
+ aoffset4 += 2;
+ boffset += 8;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp05 = *(aoffset3 + 0);
+ ctemp07 = *(aoffset4 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp03;
+ *(boffset + 2) = ctemp05;
+ *(boffset + 3) = ctemp07;
+ boffset += 4;
+ }
+ }
+
+ if (n & 2){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset += 2 * lda;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp03;
+ *(boffset + 2) = ctemp02;
+ *(boffset + 3) = ctemp04;
+
+ aoffset1 += 2;
+ aoffset2 += 2;
+ boffset += 4;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp03 = *(aoffset2 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp03;
+ boffset += 2;
+ }
+ }
+
+ if (n & 1){
+ aoffset1 = aoffset;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+
+ aoffset1 += 2;
+ boffset += 2;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+
+ *(boffset + 0) = ctemp01;
+ // boffset += 1;
+ }
+ }
+
+ return 0;
+}
--- /dev/null
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin. */
+/* All rights reserved. */
+/* */
+/* Redistribution and use in source and binary forms, with or */
+/* without modification, are permitted provided that the following */
+/* conditions are met: */
+/* */
+/* 1. Redistributions of source code must retain the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer. */
+/* */
+/* 2. Redistributions in binary form must reproduce the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer in the documentation and/or other materials */
+/* provided with the distribution. */
+/* */
+/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
+/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
+/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
+/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
+/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
+/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
+/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
+/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
+/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
+/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
+/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
+/* POSSIBILITY OF SUCH DAMAGE. */
+/* */
+/* The views and conclusions contained in the software and */
+/* documentation are those of the authors and should not be */
+/* interpreted as representing official policies, either expressed */
+/* or implied, of The University of Texas at Austin. */
+/*********************************************************************/
+
+#include <stdio.h>
+#include <altivec.h>
+#include "common.h"
+
+typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
+ BLASLONG i, j;
+
+ IFLOAT *aoffset;
+ IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
+ IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
+
+ IFLOAT *boffset;
+ vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
+ vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08;
+ vec_bf16 vtemp09, vtemp10, vtemp11, vtemp12;
+ vector char mask =
+ { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 };
+ vector char mask1 =
+ { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 };
+ IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+ IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+ IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
+ IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
+ IFLOAT ctemp17;
+ IFLOAT ctemp25;
+ IFLOAT ctemp33;
+ IFLOAT ctemp41;
+ IFLOAT ctemp49;
+ IFLOAT ctemp57;
+
+
+ aoffset = a;
+ boffset = b;
+
+ j = (n >> 3);
+ if (j > 0){
+ do{
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset3 = aoffset2 + lda;
+ aoffset4 = aoffset3 + lda;
+ aoffset5 = aoffset4 + lda;
+ aoffset6 = aoffset5 + lda;
+ aoffset7 = aoffset6 + lda;
+ aoffset8 = aoffset7 + lda;
+ aoffset += 8 * lda;
+
+ i = (m >> 3);
+ if (i > 0){
+ do{
+ vtemp01 = *(vec_bf16 *)(aoffset1);
+ vtemp02 = *(vec_bf16 *)(aoffset2);
+ vtemp03 = *(vec_bf16 *)(aoffset3);
+ vtemp04 = *(vec_bf16 *)(aoffset4);
+ vtemp05 = *(vec_bf16 *)(aoffset5);
+ vtemp06 = *(vec_bf16 *)(aoffset6);
+ vtemp07 = *(vec_bf16 *)(aoffset7);
+ vtemp08 = *(vec_bf16 *)(aoffset8);
+
+ vtemp09 = vec_perm(vtemp01, vtemp02, mask);
+ vtemp10 = vec_perm(vtemp03, vtemp04, mask);
+ vtemp11 = vec_perm(vtemp05, vtemp06, mask);
+ vtemp12 = vec_perm(vtemp07, vtemp08, mask);
+
+ *(vec_bf16 *)(boffset + 0) = vec_xxpermdi(vtemp09, vtemp10, 0);
+ *(vec_bf16 *)(boffset + 8) = vec_xxpermdi(vtemp11, vtemp12, 0);
+ *(vec_bf16 *)(boffset + 16) = vec_xxpermdi(vtemp09, vtemp10, 3);
+ *(vec_bf16 *)(boffset + 24) = vec_xxpermdi(vtemp11, vtemp12, 3);
+
+ vtemp09 = vec_perm(vtemp01, vtemp02, mask1);
+ vtemp10 = vec_perm(vtemp03, vtemp04, mask1);
+ vtemp11 = vec_perm(vtemp05, vtemp06, mask1);
+ vtemp12 = vec_perm(vtemp07, vtemp08, mask1);
+
+ *(vec_bf16 *)(boffset + 32) = vec_xxpermdi(vtemp09, vtemp10, 0);
+ *(vec_bf16 *)(boffset + 40) = vec_xxpermdi(vtemp11, vtemp12, 0);
+ *(vec_bf16 *)(boffset + 48) = vec_xxpermdi(vtemp09, vtemp10, 3);
+ *(vec_bf16 *)(boffset + 56) = vec_xxpermdi(vtemp11, vtemp12, 3);
+
+ aoffset1 += 8;
+ aoffset2 += 8;
+ aoffset3 += 8;
+ aoffset4 += 8;
+ aoffset5 += 8;
+ aoffset6 += 8;
+ aoffset7 += 8;
+ aoffset8 += 8;
+ boffset += 64;
+ i --;
+ }while(i > 0);
+ }
+
+ i = (m & 7);
+ if (i >= 2){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp09 = *(aoffset1 + 1);
+ ctemp17 = *(aoffset2 + 0);
+ ctemp25 = *(aoffset2 + 1);
+ ctemp33 = *(aoffset3 + 0);
+ ctemp41 = *(aoffset3 + 1);
+ ctemp49 = *(aoffset4 + 0);
+ ctemp57 = *(aoffset4 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp09;
+ *(boffset + 2) = ctemp17;
+ *(boffset + 3) = ctemp25;
+ *(boffset + 4) = ctemp33;
+ *(boffset + 5) = ctemp41;
+ *(boffset + 6) = ctemp49;
+ *(boffset + 7) = ctemp57;
+ aoffset1 += 2;
+ aoffset2 += 2;
+ aoffset3 += 2;
+ aoffset4 += 2;
+
+ ctemp01 = *(aoffset5 + 0);
+ ctemp09 = *(aoffset5 + 1);
+ ctemp17 = *(aoffset6 + 0);
+ ctemp25 = *(aoffset6 + 1);
+ ctemp33 = *(aoffset7 + 0);
+ ctemp41 = *(aoffset7 + 1);
+ ctemp49 = *(aoffset8 + 0);
+ ctemp57 = *(aoffset8 + 1);
+ *(boffset + 8) = ctemp01;
+ *(boffset + 9) = ctemp09;
+ *(boffset + 10) = ctemp17;
+ *(boffset + 11) = ctemp25;
+ *(boffset + 12) = ctemp33;
+ *(boffset + 13) = ctemp41;
+ *(boffset + 14) = ctemp49;
+ *(boffset + 15) = ctemp57;
+
+ aoffset5 += 2;
+ aoffset6 += 2;
+ aoffset7 += 2;
+ aoffset8 += 2;
+
+ boffset += 16;
+ i -= 2;
+ }while(i > 1);
+ }
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp09 = *(aoffset2 + 0);
+ ctemp17 = *(aoffset3 + 0);
+ ctemp25 = *(aoffset4 + 0);
+ ctemp33 = *(aoffset5 + 0);
+ ctemp41 = *(aoffset6 + 0);
+ ctemp49 = *(aoffset7 + 0);
+ ctemp57 = *(aoffset8 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp09;
+ *(boffset + 2) = ctemp17;
+ *(boffset + 3) = ctemp25;
+ *(boffset + 4) = ctemp33;
+ *(boffset + 5) = ctemp41;
+ *(boffset + 6) = ctemp49;
+ *(boffset + 7) = ctemp57;
+
+ aoffset1 ++;
+ aoffset2 ++;
+ aoffset3 ++;
+ aoffset4 ++;
+ aoffset5 ++;
+ aoffset6 ++;
+ aoffset7 ++;
+ aoffset8 ++;
+
+ boffset += 8;
+ }
+
+ j--;
+ }while(j > 0);
+ } /* end of if(j > 0) */
+
+ if (n & 4){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset3 = aoffset2 + lda;
+ aoffset4 = aoffset3 + lda;
+ aoffset += 4 * lda;
+
+ i = (m >> 2);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+
+ ctemp05 = *(aoffset2 + 0);
+ ctemp06 = *(aoffset2 + 1);
+ ctemp07 = *(aoffset2 + 2);
+ ctemp08 = *(aoffset2 + 3);
+
+ ctemp09 = *(aoffset3 + 0);
+ ctemp10 = *(aoffset3 + 1);
+ ctemp11 = *(aoffset3 + 2);
+ ctemp12 = *(aoffset3 + 3);
+
+ ctemp13 = *(aoffset4 + 0);
+ ctemp14 = *(aoffset4 + 1);
+ ctemp15 = *(aoffset4 + 2);
+ ctemp16 = *(aoffset4 + 3);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ *(boffset + 2) = ctemp05;
+ *(boffset + 3) = ctemp06;
+
+ *(boffset + 4) = ctemp09;
+ *(boffset + 5) = ctemp10;
+ *(boffset + 6) = ctemp13;
+ *(boffset + 7) = ctemp14;
+
+ *(boffset + 8) = ctemp03;
+ *(boffset + 9) = ctemp04;
+ *(boffset + 10) = ctemp07;
+ *(boffset + 11) = ctemp08;
+
+ *(boffset + 12) = ctemp11;
+ *(boffset + 13) = ctemp12;
+ *(boffset + 14) = ctemp15;
+ *(boffset + 15) = ctemp16;
+
+ aoffset1 += 4;
+ aoffset2 += 4;
+ aoffset3 += 4;
+ aoffset4 += 4;
+ boffset += 16;
+ i --;
+ }while(i > 0);
+ }
+
+ i = (m & 3);
+ if (i >= 2){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp09 = *(aoffset1 + 1);
+ ctemp17 = *(aoffset2 + 0);
+ ctemp25 = *(aoffset2 + 1);
+ ctemp33 = *(aoffset3 + 0);
+ ctemp41 = *(aoffset3 + 1);
+ ctemp49 = *(aoffset4 + 0);
+ ctemp57 = *(aoffset4 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp09;
+ *(boffset + 2) = ctemp17;
+ *(boffset + 3) = ctemp25;
+ *(boffset + 4) = ctemp33;
+ *(boffset + 5) = ctemp41;
+ *(boffset + 6) = ctemp49;
+ *(boffset + 7) = ctemp57;
+ aoffset1 += 2;
+ aoffset2 += 2;
+ aoffset3 += 2;
+ aoffset4 += 2;
+
+ boffset += 8;
+ i -= 2;
+ }while(i > 1);
+ }
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset2 + 0);
+ ctemp03 = *(aoffset3 + 0);
+ ctemp04 = *(aoffset4 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ *(boffset + 2) = ctemp03;
+ *(boffset + 3) = ctemp04;
+
+ aoffset1 ++;
+ aoffset2 ++;
+ aoffset3 ++;
+ aoffset4 ++;
+
+ boffset += 4;
+ }
+ }
+
+ if (n & 2){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset += 2 * lda;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp03;
+ *(boffset + 2) = ctemp02;
+ *(boffset + 3) = ctemp04;
+
+ aoffset1 += 2;
+ aoffset2 += 2;
+ boffset += 4;
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset2 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+
+ aoffset1 ++;
+ aoffset2 ++;
+ boffset += 2;
+ }
+ } /* end of if(j > 0) */
+
+ if (n & 1){
+ aoffset1 = aoffset;
+
+ i = m;
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+
+ *(boffset + 0) = ctemp01;
+
+ aoffset1 ++;
+ boffset ++;
+ i --;
+ }while(i > 0);
+ }
+
+ } /* end of if(j > 0) */
+
+ return 0;
+}
--- /dev/null
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin. */
+/* All rights reserved. */
+/* */
+/* Redistribution and use in source and binary forms, with or */
+/* without modification, are permitted provided that the following */
+/* conditions are met: */
+/* */
+/* 1. Redistributions of source code must retain the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer. */
+/* */
+/* 2. Redistributions in binary form must reproduce the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer in the documentation and/or other materials */
+/* provided with the distribution. */
+/* */
+/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
+/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
+/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
+/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
+/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
+/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
+/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
+/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
+/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
+/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
+/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
+/* POSSIBILITY OF SUCH DAMAGE. */
+/* */
+/* The views and conclusions contained in the software and */
+/* documentation are those of the authors and should not be */
+/* interpreted as representing official policies, either expressed */
+/* or implied, of The University of Texas at Austin. */
+/*********************************************************************/
+
+#include <stdio.h>
+#include <altivec.h>
+#include "common.h"
+typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));
+
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
+
+ BLASLONG i, j;
+
+ IFLOAT *aoffset;
+ IFLOAT *aoffset1, *aoffset2;
+ IFLOAT *boffset;
+
+ vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
+ IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+ IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+
+ aoffset = a;
+ boffset = b;
+
+#if 0
+ fprintf(stderr, "m = %d n = %d\n", m, n);
+#endif
+
+ j = (n >> 4);
+ if (j > 0){
+ do{
+ aoffset1 = aoffset;
+ aoffset2 = aoffset + lda;
+ aoffset += 16;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ vtemp01 = *(vec_bf16 *)(aoffset1);
+ vtemp02 = *(vec_bf16 *)(aoffset1+8);
+ vtemp03 = *(vec_bf16 *)(aoffset2);
+ vtemp04 = *(vec_bf16 *)(aoffset2+8);
+ *(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03);
+ *(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03);
+ *(vec_bf16 *)(boffset + 16) = vec_mergeh(vtemp02, vtemp04);
+ *(vec_bf16 *)(boffset + 24) = vec_mergel(vtemp02, vtemp04);
+ aoffset1 += 2 * lda;
+ aoffset2 += 2 * lda;
+ boffset += 32;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ vtemp01 = *(vec_bf16 *)(aoffset1);
+ vtemp02 = *(vec_bf16 *)(aoffset1+8);
+ *(vec_bf16 *)(boffset + 0) = vtemp01;
+ *(vec_bf16 *)(boffset + 8) = vtemp02;
+ boffset += 16;
+ }
+
+ j--;
+ }while(j > 0);
+ } /* end of if(j > 0) */
+
+ if (n & 8){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset + lda;
+ aoffset += 8;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ vtemp01 = *(vec_bf16 *)(aoffset1);
+ vtemp03 = *(vec_bf16 *)(aoffset2);
+ *(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03);
+ *(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03);
+
+ aoffset1 += 2 * lda;
+ aoffset2 += 2 * lda;
+ boffset += 16;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ vtemp01 = *(vec_bf16 *)(aoffset1);
+ *(vec_bf16 *)(boffset + 0) = vtemp01;
+ boffset += 8;
+ }
+ }
+
+ if (n & 4){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset + lda;
+ aoffset += 4;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+
+ ctemp05 = *(aoffset2 + 0);
+ ctemp06 = *(aoffset2 + 1);
+ ctemp07 = *(aoffset2 + 2);
+ ctemp08 = *(aoffset2 + 3);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp05;
+ *(boffset + 2) = ctemp02;
+ *(boffset + 3) = ctemp06;
+ *(boffset + 4) = ctemp03;
+ *(boffset + 5) = ctemp07;
+ *(boffset + 6) = ctemp04;
+ *(boffset + 7) = ctemp08;
+
+ aoffset1 += 2 * lda;
+ aoffset2 += 2 * lda;
+ boffset += 8;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ *(boffset + 2) = ctemp03;
+ *(boffset + 3) = ctemp04;
+
+ boffset += 4;
+ }
+ }
+
+ if (n & 2){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset + lda;
+ aoffset += 2;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ *(boffset + 2) = ctemp03;
+ *(boffset + 3) = ctemp04;
+
+ aoffset1 += 2 * lda;
+ aoffset2 += 2 * lda;
+ boffset += 4;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+ boffset += 2;
+ }
+ }
+
+ if (n & 1){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset + lda;
+
+ i = (m >> 1);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset2 + 0);
+
+ *(boffset + 0) = ctemp01;
+ *(boffset + 1) = ctemp02;
+
+ aoffset1 += 2 * lda;
+ aoffset2 += 2 * lda;
+ boffset += 2;
+
+ i --;
+ }while(i > 0);
+ }
+
+ if (m & 1){
+ ctemp01 = *(aoffset1 + 0);
+ *(boffset + 0) = ctemp01;
+ // boffset += 1;
+ }
+ }
+
+ return 0;
+}
--- /dev/null
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin. */
+/* All rights reserved. */
+/* */
+/* Redistribution and use in source and binary forms, with or */
+/* without modification, are permitted provided that the following */
+/* conditions are met: */
+/* */
+/* 1. Redistributions of source code must retain the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer. */
+/* */
+/* 2. Redistributions in binary form must reproduce the above */
+/* copyright notice, this list of conditions and the following */
+/* disclaimer in the documentation and/or other materials */
+/* provided with the distribution. */
+/* */
+/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
+/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
+/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
+/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
+/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
+/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
+/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
+/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
+/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
+/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
+/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
+/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
+/* POSSIBILITY OF SUCH DAMAGE. */
+/* */
+/* The views and conclusions contained in the software and */
+/* documentation are those of the authors and should not be */
+/* interpreted as representing official policies, either expressed */
+/* or implied, of The University of Texas at Austin. */
+/*********************************************************************/
+
+#include <stdio.h>
+#include "common.h"
+#include <altivec.h>
+
+typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));
+
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
+
+ BLASLONG i, j;
+
+ IFLOAT *aoffset;
+ IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
+ IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
+
+ IFLOAT *boffset, *boffset1, *boffset2, *boffset3, *boffset4;
+ vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
+ vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08;
+ IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
+ IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
+ IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
+ IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
+ IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
+ IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
+ IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
+ IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;
+
+ aoffset = a;
+ boffset = b;
+
+#if 0
+ fprintf(stderr, "M = %d N = %d\n", m, n);
+#endif
+
+ boffset2 = b + m * (n & ~7);
+ boffset3 = b + m * (n & ~3);
+ boffset4 = b + m * (n & ~1);
+
+ j = (m >> 3);
+ if (j > 0){
+ do{
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset3 = aoffset2 + lda;
+ aoffset4 = aoffset3 + lda;
+ aoffset5 = aoffset4 + lda;
+ aoffset6 = aoffset5 + lda;
+ aoffset7 = aoffset6 + lda;
+ aoffset8 = aoffset7 + lda;
+ aoffset += 8 * lda;
+
+ boffset1 = boffset;
+ boffset += 64;
+
+ i = (n >> 3);
+ if (i > 0){
+ do{
+ vtemp01 = *(vec_bf16 *)(aoffset1);
+ vtemp02 = *(vec_bf16 *)(aoffset2);
+ vtemp03 = *(vec_bf16 *)(aoffset3);
+ vtemp04 = *(vec_bf16 *)(aoffset4);
+ vtemp05 = *(vec_bf16 *)(aoffset5);
+ vtemp06 = *(vec_bf16 *)(aoffset6);
+ vtemp07 = *(vec_bf16 *)(aoffset7);
+ vtemp08 = *(vec_bf16 *)(aoffset8);
+ aoffset1 += 8;
+ aoffset2 += 8;
+ aoffset3 += 8;
+ aoffset4 += 8;
+ aoffset5 += 8;
+ aoffset6 += 8;
+ aoffset7 += 8;
+ aoffset8 += 8;
+
+ *(vec_bf16 *)(boffset1 + 0) = vec_mergeh(vtemp01, vtemp02);
+ *(vec_bf16 *)(boffset1 + 8) = vec_mergel(vtemp01, vtemp02);
+ *(vec_bf16 *)(boffset1 + 16) = vec_mergeh(vtemp03, vtemp04);
+ *(vec_bf16 *)(boffset1 + 24) = vec_mergel(vtemp03, vtemp04);
+ *(vec_bf16 *)(boffset1 + 32) = vec_mergeh(vtemp05, vtemp06);
+ *(vec_bf16 *)(boffset1 + 40) = vec_mergel(vtemp05, vtemp06);
+ *(vec_bf16 *)(boffset1 + 48) = vec_mergeh(vtemp07, vtemp08);
+ *(vec_bf16 *)(boffset1 + 56) = vec_mergel(vtemp07, vtemp08);
+
+ boffset1 += m * 8;
+ i --;
+ }while(i > 0);
+ }
+
+ if (n & 4){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+ aoffset1 += 4;
+
+ ctemp05 = *(aoffset2 + 0);
+ ctemp06 = *(aoffset2 + 1);
+ ctemp07 = *(aoffset2 + 2);
+ ctemp08 = *(aoffset2 + 3);
+ aoffset2 += 4;
+
+ ctemp09 = *(aoffset3 + 0);
+ ctemp10 = *(aoffset3 + 1);
+ ctemp11 = *(aoffset3 + 2);
+ ctemp12 = *(aoffset3 + 3);
+ aoffset3 += 4;
+
+ ctemp13 = *(aoffset4 + 0);
+ ctemp14 = *(aoffset4 + 1);
+ ctemp15 = *(aoffset4 + 2);
+ ctemp16 = *(aoffset4 + 3);
+ aoffset4 += 4;
+
+ ctemp17 = *(aoffset5 + 0);
+ ctemp18 = *(aoffset5 + 1);
+ ctemp19 = *(aoffset5 + 2);
+ ctemp20 = *(aoffset5 + 3);
+ aoffset5 += 4;
+
+ ctemp21 = *(aoffset6 + 0);
+ ctemp22 = *(aoffset6 + 1);
+ ctemp23 = *(aoffset6 + 2);
+ ctemp24 = *(aoffset6 + 3);
+ aoffset6 += 4;
+
+ ctemp25 = *(aoffset7 + 0);
+ ctemp26 = *(aoffset7 + 1);
+ ctemp27 = *(aoffset7 + 2);
+ ctemp28 = *(aoffset7 + 3);
+ aoffset7 += 4;
+
+ ctemp29 = *(aoffset8 + 0);
+ ctemp30 = *(aoffset8 + 1);
+ ctemp31 = *(aoffset8 + 2);
+ ctemp32 = *(aoffset8 + 3);
+ aoffset8 += 4;
+
+ *(boffset2 + 0) = ctemp01;
+ *(boffset2 + 1) = ctemp05;
+ *(boffset2 + 2) = ctemp02;
+ *(boffset2 + 3) = ctemp06;
+ *(boffset2 + 4) = ctemp03;
+ *(boffset2 + 5) = ctemp07;
+ *(boffset2 + 6) = ctemp04;
+ *(boffset2 + 7) = ctemp08;
+
+ *(boffset2 + 8) = ctemp09;
+ *(boffset2 + 9) = ctemp13;
+ *(boffset2 + 10) = ctemp10;
+ *(boffset2 + 11) = ctemp14;
+ *(boffset2 + 12) = ctemp11;
+ *(boffset2 + 13) = ctemp15;
+ *(boffset2 + 14) = ctemp12;
+ *(boffset2 + 15) = ctemp16;
+
+ *(boffset2 + 16) = ctemp17;
+ *(boffset2 + 17) = ctemp21;
+ *(boffset2 + 18) = ctemp18;
+ *(boffset2 + 19) = ctemp22;
+ *(boffset2 + 20) = ctemp19;
+ *(boffset2 + 21) = ctemp23;
+ *(boffset2 + 22) = ctemp20;
+ *(boffset2 + 23) = ctemp24;
+
+ *(boffset2 + 24) = ctemp25;
+ *(boffset2 + 25) = ctemp29;
+ *(boffset2 + 26) = ctemp26;
+ *(boffset2 + 27) = ctemp30;
+ *(boffset2 + 28) = ctemp27;
+ *(boffset2 + 29) = ctemp31;
+ *(boffset2 + 30) = ctemp28;
+ *(boffset2 + 31) = ctemp32;
+
+ boffset2 += 32;
+ }
+
+ if (n & 2){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ aoffset1 += 2;
+
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+ aoffset2 += 2;
+
+ ctemp05 = *(aoffset3 + 0);
+ ctemp06 = *(aoffset3 + 1);
+ aoffset3 += 2;
+
+ ctemp07 = *(aoffset4 + 0);
+ ctemp08 = *(aoffset4 + 1);
+ aoffset4 += 2;
+
+ ctemp09 = *(aoffset5 + 0);
+ ctemp10 = *(aoffset5 + 1);
+ aoffset5 += 2;
+
+ ctemp11 = *(aoffset6 + 0);
+ ctemp12 = *(aoffset6 + 1);
+ aoffset6 += 2;
+
+ ctemp13 = *(aoffset7 + 0);
+ ctemp14 = *(aoffset7 + 1);
+ aoffset7 += 2;
+
+ ctemp15 = *(aoffset8 + 0);
+ ctemp16 = *(aoffset8 + 1);
+ aoffset8 += 2;
+
+ *(boffset3 + 0) = ctemp01;
+ *(boffset3 + 1) = ctemp02;
+ *(boffset3 + 2) = ctemp03;
+ *(boffset3 + 3) = ctemp04;
+ *(boffset3 + 4) = ctemp05;
+ *(boffset3 + 5) = ctemp06;
+ *(boffset3 + 6) = ctemp07;
+ *(boffset3 + 7) = ctemp08;
+ *(boffset3 + 8) = ctemp09;
+ *(boffset3 + 9) = ctemp10;
+ *(boffset3 + 10) = ctemp11;
+ *(boffset3 + 11) = ctemp12;
+ *(boffset3 + 12) = ctemp13;
+ *(boffset3 + 13) = ctemp14;
+ *(boffset3 + 14) = ctemp15;
+ *(boffset3 + 15) = ctemp16;
+ boffset3 += 16;
+ }
+
+ if (n & 1){
+ ctemp01 = *(aoffset1 + 0);
+ aoffset1 ++;
+ ctemp02 = *(aoffset2 + 0);
+ aoffset2 ++;
+ ctemp03 = *(aoffset3 + 0);
+ aoffset3 ++;
+ ctemp04 = *(aoffset4 + 0);
+ aoffset4 ++;
+ ctemp05 = *(aoffset5 + 0);
+ aoffset5 ++;
+ ctemp06 = *(aoffset6 + 0);
+ aoffset6 ++;
+ ctemp07 = *(aoffset7 + 0);
+ aoffset7 ++;
+ ctemp08 = *(aoffset8 + 0);
+ aoffset8 ++;
+
+ *(boffset4 + 0) = ctemp01;
+ *(boffset4 + 1) = ctemp02;
+ *(boffset4 + 2) = ctemp03;
+ *(boffset4 + 3) = ctemp04;
+ *(boffset4 + 4) = ctemp05;
+ *(boffset4 + 5) = ctemp06;
+ *(boffset4 + 6) = ctemp07;
+ *(boffset4 + 7) = ctemp08;
+ boffset4 += 8;
+ }
+
+ j--;
+ }while(j > 0);
+ }
+
+ if (m & 4){
+
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset3 = aoffset2 + lda;
+ aoffset4 = aoffset3 + lda;
+ aoffset += 4 * lda;
+
+ boffset1 = boffset;
+ boffset += 32;
+
+ i = (n >> 3);
+ if (i > 0){
+
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+ ctemp05 = *(aoffset1 + 4);
+ ctemp06 = *(aoffset1 + 5);
+ ctemp07 = *(aoffset1 + 6);
+ ctemp08 = *(aoffset1 + 7);
+ aoffset1 += 8;
+
+ ctemp09 = *(aoffset2 + 0);
+ ctemp10 = *(aoffset2 + 1);
+ ctemp11 = *(aoffset2 + 2);
+ ctemp12 = *(aoffset2 + 3);
+ ctemp13 = *(aoffset2 + 4);
+ ctemp14 = *(aoffset2 + 5);
+ ctemp15 = *(aoffset2 + 6);
+ ctemp16 = *(aoffset2 + 7);
+ aoffset2 += 8;
+
+ ctemp17 = *(aoffset3 + 0);
+ ctemp18 = *(aoffset3 + 1);
+ ctemp19 = *(aoffset3 + 2);
+ ctemp20 = *(aoffset3 + 3);
+ ctemp21 = *(aoffset3 + 4);
+ ctemp22 = *(aoffset3 + 5);
+ ctemp23 = *(aoffset3 + 6);
+ ctemp24 = *(aoffset3 + 7);
+ aoffset3 += 8;
+
+ ctemp25 = *(aoffset4 + 0);
+ ctemp26 = *(aoffset4 + 1);
+ ctemp27 = *(aoffset4 + 2);
+ ctemp28 = *(aoffset4 + 3);
+ ctemp29 = *(aoffset4 + 4);
+ ctemp30 = *(aoffset4 + 5);
+ ctemp31 = *(aoffset4 + 6);
+ ctemp32 = *(aoffset4 + 7);
+ aoffset4 += 8;
+
+ *(boffset1 + 0) = ctemp01;
+ *(boffset1 + 1) = ctemp09;
+ *(boffset1 + 2) = ctemp02;
+ *(boffset1 + 3) = ctemp10;
+ *(boffset1 + 4) = ctemp03;
+ *(boffset1 + 5) = ctemp11;
+ *(boffset1 + 6) = ctemp04;
+ *(boffset1 + 7) = ctemp12;
+
+ *(boffset1 + 8) = ctemp05;
+ *(boffset1 + 9) = ctemp13;
+ *(boffset1 + 10) = ctemp06;
+ *(boffset1 + 11) = ctemp14;
+ *(boffset1 + 12) = ctemp07;
+ *(boffset1 + 13) = ctemp15;
+ *(boffset1 + 14) = ctemp08;
+ *(boffset1 + 15) = ctemp16;
+
+ *(boffset1 + 16) = ctemp17;
+ *(boffset1 + 17) = ctemp25;
+ *(boffset1 + 18) = ctemp18;
+ *(boffset1 + 19) = ctemp26;
+ *(boffset1 + 20) = ctemp19;
+ *(boffset1 + 21) = ctemp27;
+ *(boffset1 + 22) = ctemp20;
+ *(boffset1 + 23) = ctemp28;
+
+ *(boffset1 + 24) = ctemp21;
+ *(boffset1 + 25) = ctemp29;
+ *(boffset1 + 26) = ctemp22;
+ *(boffset1 + 27) = ctemp30;
+ *(boffset1 + 28) = ctemp23;
+ *(boffset1 + 29) = ctemp31;
+ *(boffset1 + 30) = ctemp24;
+ *(boffset1 + 31) = ctemp32;
+
+ boffset1 += 8 * m;
+ i --;
+ }while(i > 0);
+ }
+
+ if (n & 4) {
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+ aoffset1 += 4;
+
+ ctemp05 = *(aoffset2 + 0);
+ ctemp06 = *(aoffset2 + 1);
+ ctemp07 = *(aoffset2 + 2);
+ ctemp08 = *(aoffset2 + 3);
+ aoffset2 += 4;
+
+ ctemp09 = *(aoffset3 + 0);
+ ctemp10 = *(aoffset3 + 1);
+ ctemp11 = *(aoffset3 + 2);
+ ctemp12 = *(aoffset3 + 3);
+ aoffset3 += 4;
+
+ ctemp13 = *(aoffset4 + 0);
+ ctemp14 = *(aoffset4 + 1);
+ ctemp15 = *(aoffset4 + 2);
+ ctemp16 = *(aoffset4 + 3);
+ aoffset4 += 4;
+
+ *(boffset2 + 0) = ctemp01;
+ *(boffset2 + 1) = ctemp05;
+ *(boffset2 + 2) = ctemp02;
+ *(boffset2 + 3) = ctemp06;
+ *(boffset2 + 4) = ctemp03;
+ *(boffset2 + 5) = ctemp07;
+ *(boffset2 + 6) = ctemp04;
+ *(boffset2 + 7) = ctemp08;
+
+ *(boffset2 + 8) = ctemp09;
+ *(boffset2 + 9) = ctemp13;
+ *(boffset2 + 10) = ctemp10;
+ *(boffset2 + 11) = ctemp14;
+ *(boffset2 + 12) = ctemp11;
+ *(boffset2 + 13) = ctemp15;
+ *(boffset2 + 14) = ctemp12;
+ *(boffset2 + 15) = ctemp16;
+ boffset2 += 16;
+ }
+
+ if (n & 2){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ aoffset1 += 2;
+
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+ aoffset2 += 2;
+
+ ctemp05 = *(aoffset3 + 0);
+ ctemp06 = *(aoffset3 + 1);
+ aoffset3 += 2;
+
+ ctemp07 = *(aoffset4 + 0);
+ ctemp08 = *(aoffset4 + 1);
+ aoffset4 += 2;
+
+ *(boffset3 + 0) = ctemp01;
+ *(boffset3 + 1) = ctemp02;
+ *(boffset3 + 2) = ctemp03;
+ *(boffset3 + 3) = ctemp04;
+ *(boffset3 + 4) = ctemp05;
+ *(boffset3 + 5) = ctemp06;
+ *(boffset3 + 6) = ctemp07;
+ *(boffset3 + 7) = ctemp08;
+ boffset3 += 8;
+ }
+
+ if (n & 1){
+ ctemp01 = *(aoffset1 + 0);
+ aoffset1 ++;
+ ctemp02 = *(aoffset2 + 0);
+ aoffset2 ++;
+ ctemp03 = *(aoffset3 + 0);
+ aoffset3 ++;
+ ctemp04 = *(aoffset4 + 0);
+ aoffset4 ++;
+
+ *(boffset4 + 0) = ctemp01;
+ *(boffset4 + 1) = ctemp02;
+ *(boffset4 + 2) = ctemp03;
+ *(boffset4 + 3) = ctemp04;
+ boffset4 += 4;
+ }
+ }
+
+ if (m & 2){
+ aoffset1 = aoffset;
+ aoffset2 = aoffset1 + lda;
+ aoffset += 2 * lda;
+
+ boffset1 = boffset;
+ boffset += 16;
+
+ i = (n >> 3);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+ ctemp05 = *(aoffset1 + 4);
+ ctemp06 = *(aoffset1 + 5);
+ ctemp07 = *(aoffset1 + 6);
+ ctemp08 = *(aoffset1 + 7);
+ aoffset1 += 8;
+
+ ctemp09 = *(aoffset2 + 0);
+ ctemp10 = *(aoffset2 + 1);
+ ctemp11 = *(aoffset2 + 2);
+ ctemp12 = *(aoffset2 + 3);
+ ctemp13 = *(aoffset2 + 4);
+ ctemp14 = *(aoffset2 + 5);
+ ctemp15 = *(aoffset2 + 6);
+ ctemp16 = *(aoffset2 + 7);
+ aoffset2 += 8;
+
+ *(boffset1 + 0) = ctemp01;
+ *(boffset1 + 1) = ctemp09;
+ *(boffset1 + 2) = ctemp02;
+ *(boffset1 + 3) = ctemp10;
+ *(boffset1 + 4) = ctemp03;
+ *(boffset1 + 5) = ctemp11;
+ *(boffset1 + 6) = ctemp04;
+ *(boffset1 + 7) = ctemp12;
+
+ *(boffset1 + 8) = ctemp05;
+ *(boffset1 + 9) = ctemp13;
+ *(boffset1 + 10) = ctemp06;
+ *(boffset1 + 11) = ctemp14;
+ *(boffset1 + 12) = ctemp07;
+ *(boffset1 + 13) = ctemp15;
+ *(boffset1 + 14) = ctemp08;
+ *(boffset1 + 15) = ctemp16;
+
+ boffset1 += 8 * m;
+ i --;
+ }while(i > 0);
+ }
+
+ if (n & 4){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+ aoffset1 += 4;
+
+ ctemp05 = *(aoffset2 + 0);
+ ctemp06 = *(aoffset2 + 1);
+ ctemp07 = *(aoffset2 + 2);
+ ctemp08 = *(aoffset2 + 3);
+ aoffset2 += 4;
+
+ *(boffset2 + 0) = ctemp01;
+ *(boffset2 + 1) = ctemp05;
+ *(boffset2 + 2) = ctemp02;
+ *(boffset2 + 3) = ctemp06;
+ *(boffset2 + 4) = ctemp03;
+ *(boffset2 + 5) = ctemp07;
+ *(boffset2 + 6) = ctemp04;
+ *(boffset2 + 7) = ctemp08;
+ boffset2 += 8;
+ }
+
+ if (n & 2){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ aoffset1 += 2;
+
+ ctemp03 = *(aoffset2 + 0);
+ ctemp04 = *(aoffset2 + 1);
+ aoffset2 += 2;
+
+ *(boffset3 + 0) = ctemp01;
+ *(boffset3 + 1) = ctemp02;
+ *(boffset3 + 2) = ctemp03;
+ *(boffset3 + 3) = ctemp04;
+ boffset3 += 4;
+ }
+
+ if (n & 1){
+ ctemp01 = *(aoffset1 + 0);
+ aoffset1 ++;
+ ctemp02 = *(aoffset2 + 0);
+ aoffset2 ++;
+
+ *(boffset4 + 0) = ctemp01;
+ *(boffset4 + 1) = ctemp02;
+ boffset4 += 2;
+ }
+ }
+
+ if (m & 1){
+ aoffset1 = aoffset;
+ // aoffset += lda;
+
+ boffset1 = boffset;
+ // boffset += 8;
+
+ i = (n >> 3);
+ if (i > 0){
+ do{
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+ ctemp05 = *(aoffset1 + 4);
+ ctemp06 = *(aoffset1 + 5);
+ ctemp07 = *(aoffset1 + 6);
+ ctemp08 = *(aoffset1 + 7);
+ aoffset1 += 8;
+
+ *(boffset1 + 0) = ctemp01;
+ *(boffset1 + 1) = ctemp02;
+ *(boffset1 + 2) = ctemp03;
+ *(boffset1 + 3) = ctemp04;
+ *(boffset1 + 4) = ctemp05;
+ *(boffset1 + 5) = ctemp06;
+ *(boffset1 + 6) = ctemp07;
+ *(boffset1 + 7) = ctemp08;
+
+ boffset1 += 8 * m;
+ i --;
+ }while(i > 0);
+ }
+
+ if (n & 4){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ ctemp03 = *(aoffset1 + 2);
+ ctemp04 = *(aoffset1 + 3);
+ aoffset1 += 4;
+
+ *(boffset2 + 0) = ctemp01;
+ *(boffset2 + 1) = ctemp02;
+ *(boffset2 + 2) = ctemp03;
+ *(boffset2 + 3) = ctemp04;
+ // boffset2 += 4;
+ }
+
+ if (n & 2){
+ ctemp01 = *(aoffset1 + 0);
+ ctemp02 = *(aoffset1 + 1);
+ aoffset1 += 2;
+
+ *(boffset3 + 0) = ctemp01;
+ *(boffset3 + 1) = ctemp02;
+ // boffset3 += 2;
+ }
+
+ if (n & 1){
+ ctemp01 = *(aoffset1 + 0);
+ aoffset1 ++;
+ *(boffset4 + 0) = ctemp01;
+ boffset4 ++;
+ }
+ }
+
+ return 0;
+}
projects / platform / upstream / openblas.git / commitdiff