From 0826d68f93ef1fed021c426911c464728d60ccb3 Mon Sep 17 00:00:00 2001 From: Rajalakshmi Srinivasaraghavan Date: Tue, 13 Oct 2020 16:05:10 -0500 Subject: [PATCH] POWER10: Change the packing format for bfloat16 As the new MMA instructions need the inputs in 4x2 order for bfloat16, changing the format in copy/packing code. This avoids permute instructions in the gemm kernel inner loop. --- kernel/power/KERNEL.POWER10 | 8 +- kernel/power/sbgemm_kernel_power10.c | 477 ++++++++++-------------- kernel/power/sbgemm_ncopy_16_power10.c | 437 ++++++++++++++++++++++ kernel/power/sbgemm_ncopy_8_power10.c | 383 +++++++++++++++++++ kernel/power/sbgemm_tcopy_16_power10.c | 244 ++++++++++++ kernel/power/sbgemm_tcopy_8_power10.c | 659 +++++++++++++++++++++++++++++++++ 6 files changed, 1923 insertions(+), 285 deletions(-) create mode 100644 kernel/power/sbgemm_ncopy_16_power10.c create mode 100644 kernel/power/sbgemm_ncopy_8_power10.c create mode 100644 kernel/power/sbgemm_tcopy_16_power10.c create mode 100644 kernel/power/sbgemm_tcopy_8_power10.c diff --git a/kernel/power/KERNEL.POWER10 b/kernel/power/KERNEL.POWER10 index 5cf1660..031d965 100644 --- a/kernel/power/KERNEL.POWER10 +++ b/kernel/power/KERNEL.POWER10 @@ -9,10 +9,10 @@ else SBGEMM_BETA = ../generic/gemm_beta.c SBGEMMKERNEL = sbgemm_kernel_power10.c -SBGEMMINCOPY = ../generic/gemm_ncopy_16.c -SBGEMMITCOPY = ../generic/gemm_tcopy_16.c -SBGEMMONCOPY = ../generic/gemm_ncopy_8.c -SBGEMMOTCOPY = ../generic/gemm_tcopy_8.c +SBGEMMINCOPY = sbgemm_ncopy_16_power10.c +SBGEMMITCOPY = sbgemm_tcopy_16_power10.c +SBGEMMONCOPY = sbgemm_ncopy_8_power10.c +SBGEMMOTCOPY = sbgemm_tcopy_8_power10.c SBGEMMINCOPYOBJ = sbgemm_incopy$(TSUFFIX).$(SUFFIX) SBGEMMITCOPYOBJ = sbgemm_itcopy$(TSUFFIX).$(SUFFIX) SBGEMMONCOPYOBJ = sbgemm_oncopy$(TSUFFIX).$(SUFFIX) diff --git a/kernel/power/sbgemm_kernel_power10.c b/kernel/power/sbgemm_kernel_power10.c index 46d8259..d155867 100644 --- a/kernel/power/sbgemm_kernel_power10.c +++ b/kernel/power/sbgemm_kernel_power10.c @@ -137,15 +137,13 @@ int CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc) { - BLASLONG N = n; BLASLONG i1; v4sf_t valpha = { alpha, alpha, alpha, alpha }; vector short vzero = { 0, 0, 0, 0, 0, 0, 0, 0 }; - N = n >> 3; /* Loop for n >= 8. */ - for (i1 = 0; i1 < N; i1++) + for (i1 = 0; i1 < (n >> 3); i1++) { - BLASLONG i, j; + BLASLONG j; FLOAT *CO; IFLOAT *AO; CO = C; @@ -153,9 +151,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO = A; PREFETCH1 (A, 128); PREFETCH1 (A, 256); - i = m >> 4; /* Loop for m >= 16. */ - for (j = 0; j < i; j++) + for (j = 0; j < (m >> 4); j++) { IFLOAT *BO = B; v4sf_t *rowC; @@ -167,20 +164,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, { vec_t *rowA = (vec_t *) & (AO[l << 5]); vec_t *rowB = (vec_t *) & (BO[l << 4]); - vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]); - vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]); - vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[2]); - vec_t rowA_l = MERGE_LOW (rowA[0], rowA[2]); - vec_t rowA2_h = MERGE_HIGH (rowA[1], rowA[3]); - vec_t rowA2_l = MERGE_LOW (rowA[1], rowA[3]); - MMA (&acc0, rowB_h, rowA_h); - MMA (&acc1, rowB_l, rowA_h); - MMA (&acc2, rowB_h, rowA_l); - MMA (&acc3, rowB_l, rowA_l); - MMA (&acc4, rowB_h, rowA2_h); - MMA (&acc5, rowB_l, rowA2_h); - MMA (&acc6, rowB_h, rowA2_l); - MMA (&acc7, rowB_l, rowA2_l); + MMA (&acc0, rowB[0], rowA[0]); + MMA (&acc1, rowB[1], rowA[0]); + MMA (&acc2, rowB[0], rowA[1]); + MMA (&acc3, rowB[1], rowA[1]); + MMA (&acc4, rowB[0], rowA[2]); + MMA (&acc5, rowB[1], rowA[2]); + MMA (&acc6, rowB[0], rowA[3]); + MMA (&acc7, rowB[1], rowA[3]); } if (k % 2 == 1) { @@ -216,9 +207,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += (k << 4); BO += (k << 3); } - i = (m & 15) >> 3; - /* Loop for m >= 8. */ - for (j = 0; j < i; j++) + if (m & 8) { IFLOAT *BO = B; v4sf_t *rowC; @@ -230,14 +219,11 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, { vec_t *rowA = (vec_t *) & (AO[l << 4]); vec_t *rowB = (vec_t *) & (BO[l << 4]); - vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]); - vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]); - vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[1]); - vec_t rowA_l = MERGE_LOW (rowA[0], rowA[1]); - MMA (&acc0, rowB_h, rowA_h); - MMA (&acc1, rowB_l, rowA_h); - MMA (&acc2, rowB_h, rowA_l); - MMA (&acc3, rowB_l, rowA_l); + + MMA (&acc0, rowB[0], rowA[0]); + MMA (&acc1, rowB[1], rowA[0]); + MMA (&acc2, rowB[0], rowA[1]); + MMA (&acc3, rowB[1], rowA[1]); } if (k % 2 == 1) { @@ -262,9 +248,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += (k << 3); BO += (k << 3); } - i = (m & 7) >> 2; - /* Loop for m >= 4. */ - for (j = 0; j < i; j++) + if (m & 4) { IFLOAT *BO = B; v4sf_t *rowC; @@ -277,9 +261,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, { vec_t *rowA = (vec_t *) & (AO[l << 3]); vec_t *rowB = (vec_t *) & (BO[l << 4]); - vec_t rowA_mrg = MERGE_ROW (rowA[0]); - MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), rowA_mrg); - MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), rowA_mrg); + MMA (&acc0, rowB[0], rowA[0]); + MMA (&acc1, rowB[1], rowA[0]); } if (k % 2 == 1) { @@ -297,9 +280,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += (k << 2); BO += (k << 3); } - i = (m & 3) >> 1; - /* Loop for m >= 2. */ - for (j = 0; j < i; j++) + if (m & 2) { IFLOAT *BO = B; v2sf_t *rowC; @@ -316,8 +297,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, 0, 0, 0, 0 }; vec_t *rowB = (vec_t *) & (BO[l << 4]); - MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA); - MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA); + MMA (&acc0, rowB[0], (vec_t) rowA); + MMA (&acc1, rowB[1], (vec_t) rowA); } if (k % 2 == 1) { @@ -334,64 +315,50 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += (k << 1); BO += (k << 3); } - i = (m & 1) >> 0; - /* Loop for m = 1. */ - for (j = 0; j < i; j++) + if (m & 1) { IFLOAT *BO = B; + v2sf_t *rowC; + v2sf_t result[8]; + __vector_quad acc0, acc1; + __builtin_mma_xxsetaccz (&acc0); + __builtin_mma_xxsetaccz (&acc1); BLASLONG l = 0; - v4sf_t t = { 0, 0, 0, 0 } - , t1 = - { - 0, 0, 0, 0}; - for (l = 0; l < k; l++) + for (l = 0; l < k / 2; l++) { - v4sf_t rowA = - { BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), - BF16TOF32 (AO[l]) - }; - v4sf_t rowB = - { BF16TOF32 (BO[l << 3]), BF16TOF32 (BO[(l << 3) + 1]), - BF16TOF32 (BO[(l << 3) + 2]), - BF16TOF32 (BO[(l << 3) + 3]) - }; - v4sf_t rowB1 = - { BF16TOF32 (BO[(l << 3) + 4]), BF16TOF32 (BO[(l << 3) + 5]), - BF16TOF32 (BO[(l << 3) + 6]), - BF16TOF32 (BO[(l << 3) + 7]) - }; - t += rowA * rowB; - t1 += rowA * rowB1; + vector short rowA = + { AO[(l << 1) + 0], AO[(l << 1) + 1], 0, 0, 0, 0, 0, 0}; + vec_t *rowB = (vec_t *) & (BO[l << 4]); + MMA (&acc0, rowB[0], (vec_t) rowA); + MMA (&acc1, rowB[1], (vec_t) rowA); } - t = t * valpha; - t1 = t1 * valpha; - CO[0 * ldc] += t[0]; - CO[1 * ldc] += t[1]; - CO[2 * ldc] += t[2]; - CO[3 * ldc] += t[3]; - CO[4 * ldc] += t1[0]; - CO[5 * ldc] += t1[1]; - CO[6 * ldc] += t1[2]; - CO[7 * ldc] += t1[3]; + if (k % 2 == 1) + { + if (k > 1) + l = (k / 2) << 1; + vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 }; + vec_t *rowB = (vec_t *) & (BO[(l << 3)]); + MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA); + MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA); + } + SAVE4x2_ACC (&acc0, 0); + SAVE4x2_ACC1 (&acc1, 0); CO += 1; AO += k; BO += (k << 3); } B += k << 3; } - N = (n & 7) >> 2; - /* Loop for n >= 4. */ - for (i1 = 0; i1 < N; i1++) + if (n & 4) { - BLASLONG i, j; + BLASLONG j; FLOAT *CO; IFLOAT *AO; CO = C; C += ldc << 2; AO = A; - i = m >> 5; /* Loop for m >= 32. */ - for (j = 0; j < i; j++) + for (j = 0; j < (m >> 5); j++) { IFLOAT *BO = B; IFLOAT *A1 = AO + (16 * k); @@ -405,15 +372,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, vec_t *rowA = (vec_t *) & (AO[l << 5]); vec_t *rowA1 = (vec_t *) & (A1[l << 5]); vec_t *rowB = (vec_t *) & (BO[l << 3]); - vec_t rowB_mrg = MERGE_ROW (rowB[0]); - MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2])); - MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2])); - MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3])); - MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3])); - MMA (&acc4, rowB_mrg, MERGE_HIGH (rowA1[0], rowA1[2])); - MMA (&acc5, rowB_mrg, MERGE_LOW (rowA1[0], rowA1[2])); - MMA (&acc6, rowB_mrg, MERGE_HIGH (rowA1[1], rowA1[3])); - MMA (&acc7, rowB_mrg, MERGE_LOW (rowA1[1], rowA1[3])); + MMA (&acc0, rowB[0], rowA[0]); + MMA (&acc1, rowB[0], rowA[1]); + MMA (&acc2, rowB[0], rowA[2]); + MMA (&acc3, rowB[0], rowA[3]); + MMA (&acc4, rowB[0], rowA1[0]); + MMA (&acc5, rowB[0], rowA1[1]); + MMA (&acc6, rowB[0], rowA1[2]); + MMA (&acc7, rowB[0], rowA1[3]); } if (k % 2 == 1) { @@ -448,9 +414,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 5; BO += k << 2; } - i = (m & 31) >> 4; - /* Loop for m >= 16. */ - for (j = 0; j < i; j++) + if (m & 16) { IFLOAT *BO = B; v4sf_t *rowC; @@ -462,11 +426,10 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, { vec_t *rowA = (vec_t *) & (AO[l << 5]); vec_t *rowB = (vec_t *) & (BO[l << 3]); - vec_t rowB_mrg = MERGE_ROW (rowB[0]); - MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2])); - MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2])); - MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3])); - MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3])); + MMA (&acc0, rowB[0], rowA[0]); + MMA (&acc1, rowB[0], rowA[1]); + MMA (&acc2, rowB[0], rowA[2]); + MMA (&acc3, rowB[0], rowA[3]); } if (k % 2 == 1) { @@ -490,9 +453,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 4; BO += k << 2; } - i = (m & 15) >> 3; - /* Loop for m >= 8. */ - for (j = 0; j < i; j++) + if (m & 8) { IFLOAT *BO = B; v4sf_t *rowC; @@ -505,9 +466,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, { vec_t *rowA = (vec_t *) & (AO[l << 4]); vec_t *rowB = (vec_t *) & (BO[l << 3]); - vec_t rowB_mrg = MERGE_ROW (rowB[0]); - MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[1])); - MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[1])); + MMA (&acc0, rowB[0], rowA[0]); + MMA (&acc1, rowB[0], rowA[1]); } if (k % 2 == 1) { @@ -525,9 +485,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 3; BO += k << 2; } - i = (m & 7) >> 2; - /* Loop for m >= 4. */ - for (j = 0; j < i; j++) + if (m & 4) { IFLOAT *BO = B; v4sf_t *rowC; @@ -539,7 +497,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, { vec_t *rowA = (vec_t *) & (AO[l << 3]); vec_t *rowB = (vec_t *) & (BO[l << 3]); - MMA (&acc0, MERGE_ROW (rowB[0]), MERGE_ROW (rowA[0])); + MMA (&acc0, rowB[0], rowA[0]); } if (k % 2 == 1) { @@ -555,9 +513,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 2; BO += k << 2; } - i = (m & 3) >> 1; - /* Loop for m >= 2. */ - for (j = 0; j < i; j++) + if (m & 2) { IFLOAT *BO = B; v2sf_t *rowC; @@ -573,7 +529,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, 0, 0, 0, 0 }; vec_t *rowB = (vec_t *) & (BO[l << 3]); - MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA); + MMA (&acc0, rowB[0], (vec_t) rowA); } if (k % 2 == 1) { @@ -588,31 +544,32 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 1; BO += k << 2; } - i = (m & 1) >> 0; - /* Loop for m = 1. */ - for (j = 0; j < i; j++) + if (m & 1) { IFLOAT *BO = B; + v2sf_t *rowC; + v2sf_t result[8]; + __vector_quad acc0; BLASLONG l = 0; - v4sf_t t = { 0, 0, 0, 0 }; - for (l = 0; l < k; l++) + __builtin_mma_xxsetaccz (&acc0); + for (l = 0; l < k / 2; l++) { - v4sf_t rowA = - { BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), - BF16TOF32 (AO[l]) - }; - v4sf_t rowB = - { BF16TOF32 (BO[l << 2]), BF16TOF32 (BO[(l << 2) + 1]), - BF16TOF32 (BO[(l << 2) + 2]), - BF16TOF32 (BO[(l << 2) + 3]) + vector short rowA = + { AO[(l << 1) + 0], AO[(l << 1) + 1], 0, + 0, 0, 0, 0 }; - t += rowA * rowB; + vec_t *rowB = (vec_t *) & (BO[l << 3]); + MMA (&acc0, rowB[0], (vec_t) rowA); } - t = t * valpha; - CO[0 * ldc] += t[0]; - CO[1 * ldc] += t[1]; - CO[2 * ldc] += t[2]; - CO[3 * ldc] += t[3]; + if (k % 2 == 1) + { + if (k > 1) + l = (k / 2) << 1; + vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 }; + vec_t *rowB = (vec_t *) & (BO[l << 2]); + MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA); + } + SAVE4x2_ACC (&acc0, 0); AO += k; BO += (k << 2); CO += 1; @@ -620,19 +577,16 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, B += k << 2; } - N = (n & 3) >> 1; - /* Loop for n >= 2. */ - for (i1 = 0; i1 < N; i1++) + if (n & 2) { - BLASLONG i, j; + BLASLONG j; FLOAT *CO; IFLOAT *AO; CO = C; C += ldc << 1; AO = A; - i = m >> 5; /* Loop for m >= 32. */ - for (j = 0; j < i; j++) + for (j = 0; j < (m >> 5); j++) { IFLOAT *BO = B; v4sf_t *rowC; @@ -650,14 +604,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, }; vec_t *rowA = (vec_t *) & (AO[l << 5]); vec_t *rowA1 = (vec_t *) & (A1[l << 5]); - MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2])); - MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2])); - MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3])); - MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3])); - MMA (&acc4, (vec_t) rowB, MERGE_HIGH (rowA1[0], rowA1[2])); - MMA (&acc5, (vec_t) rowB, MERGE_LOW (rowA1[0], rowA1[2])); - MMA (&acc6, (vec_t) rowB, MERGE_HIGH (rowA1[1], rowA1[3])); - MMA (&acc7, (vec_t) rowB, MERGE_LOW (rowA1[1], rowA1[3])); + MMA (&acc0, (vec_t) rowB, rowA[0]); + MMA (&acc1, (vec_t) rowB, rowA[1]); + MMA (&acc2, (vec_t) rowB, rowA[2]); + MMA (&acc3, (vec_t) rowB, rowA[3]); + MMA (&acc4, (vec_t) rowB, rowA1[0]); + MMA (&acc5, (vec_t) rowB, rowA1[1]); + MMA (&acc6, (vec_t) rowB, rowA1[2]); + MMA (&acc7, (vec_t) rowB, rowA1[3]); } if (k % 2 == 1) { @@ -688,9 +642,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 5; BO += k << 1; } - i = (m & 31) >> 4; - /* Loop for m >= 16. */ - for (j = 0; j < i; j++) + if (m & 16) { IFLOAT *BO = B; v4sf_t *rowC; @@ -706,10 +658,10 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, 0, 0, 0, 0 }; vec_t *rowA = (vec_t *) & (AO[l << 5]); - MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2])); - MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2])); - MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3])); - MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3])); + MMA (&acc0, (vec_t) rowB, rowA[0]); + MMA (&acc1, (vec_t) rowB, rowA[1]); + MMA (&acc2, (vec_t) rowB, rowA[2]); + MMA (&acc3, (vec_t) rowB, rowA[3]); } if (k % 2 == 1) { @@ -730,9 +682,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 4; BO += k << 1; } - i = (m & 15) >> 3; - /* Loop for m >= 8. */ - for (j = 0; j < i; j++) + if (m & 8) { IFLOAT *BO = B; v4sf_t *rowC; @@ -749,8 +699,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, 0, 0, 0, 0 }; vec_t *rowA = (vec_t *) & (AO[l << 4]); - MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1])); - MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1])); + MMA (&acc0, (vec_t) rowB, rowA[0]); + MMA (&acc1, (vec_t) rowB, rowA[1]); } if (k % 2 == 1) { @@ -767,9 +717,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 3; BO += k << 1; } - i = (m & 7) >> 2; - /* Loop for m >= 4. */ - for (j = 0; j < i; j++) + if (m & 4) { IFLOAT *BO = B; v4sf_t *rowC; @@ -785,7 +733,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, 0, 0, 0, 0 }; vec_t *rowA = (vec_t *) & (AO[l << 3]); - MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0])); + MMA (&acc0, (vec_t) rowB, rowA[0]); } if (k % 2 == 1) { @@ -800,9 +748,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 2; BO += k << 1; } - i = (m & 3) >> 1; - /* Loop for m >= 2. */ - for (j = 0; j < i; j++) + if (m & 2) { IFLOAT *BO = B; BLASLONG l = 0; @@ -828,9 +774,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 1; BO += k << 1; } - i = (m & 1) >> 0; - /* Loop for m = 1. */ - for (j = 0; j < i; j++) + if (m & 1) { IFLOAT *BO = B; BLASLONG l = 0; @@ -852,153 +796,126 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, } B += k << 1; } - N = (n & 1) >> 0; - /* Loop for n = 1. */ - for (i1 = 0; i1 < N; i1++) + if (n & 1) { - BLASLONG i; + BLASLONG j; FLOAT *CO; IFLOAT *AO; CO = C; C += ldc; AO = A; - i = m; /* Loop for m >= 16. */ - while (i >= 16) + for (j = 0; j < (m >> 4); j++) { IFLOAT *BO = B; + v4sf_t *rowC; + v4sf_t result[4]; + __vector_quad acc0, acc1, acc2, acc3; + SET_ACC_ZERO4 (); BLASLONG l = 0; - v4sf_t t = { 0, 0, 0, 0 }; - v4sf_t t1 = { 0, 0, 0, 0 }; - v4sf_t t2 = { 0, 0, 0, 0 }; - v4sf_t t3 = { 0, 0, 0, 0 }; - for (l = 0; l < k; l++) + for (l = 0; l < k / 2; l++) { - v4sf_t rowB = - { BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), - BF16TOF32 (BO[l]) - }; - v4sf_t rowA = - { BF16TOF32 (AO[l << 4]), BF16TOF32 (AO[(l << 4) + 1]), - BF16TOF32 (AO[(l << 4) + 2]), - BF16TOF32 (AO[(l << 4) + 3]) - }; - v4sf_t rowA1 = - { BF16TOF32 (AO[(l << 4) + 4]), BF16TOF32 (AO[(l << 4) + 5]), - BF16TOF32 (AO[(l << 4) + 6]), - BF16TOF32 (AO[(l << 4) + 7]) - }; - v4sf_t rowA2 = - { BF16TOF32 (AO[(l << 4) + 8]), BF16TOF32 (AO[(l << 4) + 9]), - BF16TOF32 (AO[(l << 4) + 10]), - BF16TOF32 (AO[(l << 4) + 11]) - }; - v4sf_t rowA3 = { BF16TOF32 (AO[(l << 4) + 12]), - BF16TOF32 (AO[(l << 4) + 13]), BF16TOF32 (AO[(l << 4) + 14]), - BF16TOF32 (AO[(l << 4) + 15]) - }; - t += rowA * rowB; - t1 += rowA1 * rowB; - t2 += rowA2 * rowB; - t3 += rowA3 * rowB; + vector short rowB = + { BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0}; + vec_t *rowA = (vec_t *) & (AO[l << 5]); + MMA (&acc0, (vec_t) rowB, rowA[0]); + MMA (&acc1, (vec_t) rowB, rowA[1]); + MMA (&acc2, (vec_t) rowB, rowA[2]); + MMA (&acc3, (vec_t) rowB, rowA[3]); } - t = t * valpha; - t1 = t1 * valpha; - t2 = t2 * valpha; - t3 = t3 * valpha; - CO[0] += t[0]; - CO[1] += t[1]; - CO[2] += t[2]; - CO[3] += t[3]; - CO[4] += t1[0]; - CO[5] += t1[1]; - CO[6] += t1[2]; - CO[7] += t1[3]; - CO[8] += t2[0]; - CO[9] += t2[1]; - CO[10] += t2[2]; - CO[11] += t2[3]; - CO[12] += t3[0]; - CO[13] += t3[1]; - CO[14] += t3[2]; - CO[15] += t3[3]; + if (k % 2 == 1) + { + if (k > 1) + l = (k / 2) << 1; + vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 }; + vec_t *rowA = (vec_t *) & (AO[(l << 4)]); + MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2])); + MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2])); + MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3])); + MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3])); + } + rowC = (v4sf_t *) &CO[0]; + __builtin_mma_disassemble_acc ((void *)result, &acc0); + rowC[0] += result[0] * alpha; + __builtin_mma_disassemble_acc ((void *)result, &acc1); + rowC[1] += result[0] * alpha; + __builtin_mma_disassemble_acc ((void *)result, &acc2); + rowC[2] += result[0] * alpha; + __builtin_mma_disassemble_acc ((void *)result, &acc3); + rowC[3] += result[0] * alpha; AO += k << 4; BO += k; CO += 16; - i -= 16; } /* Loop for m >= 8. */ - while (i >= 8) + if (m & 8) { IFLOAT *BO = B; + v4sf_t *rowC; + v4sf_t result[4]; + __vector_quad acc0, acc1; + __builtin_mma_xxsetaccz (&acc0); + __builtin_mma_xxsetaccz (&acc1); BLASLONG l = 0; - v4sf_t t = { 0, 0, 0, 0 }; - v4sf_t t1 = { 0, 0, 0, 0 }; - for (l = 0; l < k; l++) + for (l = 0; l < k / 2; l++) { - v4sf_t rowB = - { BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), - BF16TOF32 (BO[l]) - }; - v4sf_t rowA = - { BF16TOF32 (AO[l << 3]), BF16TOF32 (AO[(l << 3) + 1]), - BF16TOF32 (AO[(l << 3) + 2]), - BF16TOF32 (AO[(l << 3) + 3]) - }; - v4sf_t rowA1 = - { BF16TOF32 (AO[(l << 3) + 4]), BF16TOF32 (AO[(l << 3) + 5]), - BF16TOF32 (AO[(l << 3) + 6]), - BF16TOF32 (AO[(l << 3) + 7]) - }; - t += rowA * rowB; - t1 += rowA1 * rowB; + vector short rowB = + { BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0}; + vec_t *rowA = (vec_t *) & (AO[l << 4]); + MMA (&acc0, (vec_t) rowB, rowA[0]); + MMA (&acc1, (vec_t) rowB, rowA[1]); } - t = t * valpha; - t1 = t1 * valpha; - CO[0] += t[0]; - CO[1] += t[1]; - CO[2] += t[2]; - CO[3] += t[3]; - CO[4] += t1[0]; - CO[5] += t1[1]; - CO[6] += t1[2]; - CO[7] += t1[3]; + if (k % 2 == 1) + { + if (k > 1) + l = (k / 2) << 1; + vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 }; + vec_t *rowA = (vec_t *) & (AO[(l << 3)]); + MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1])); + MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1])); + } + rowC = (v4sf_t *) &CO[0]; + __builtin_mma_disassemble_acc ((void *)result, &acc0); + rowC[0] += result[0] * alpha; + __builtin_mma_disassemble_acc ((void *)result, &acc1); + rowC[1] += result[0] * alpha; AO += k << 3; BO += k; CO += 8; - i -= 8; } /* Loop for m >= 4. */ - while (i >= 4) + if (m & 4) { IFLOAT *BO = B; + v4sf_t *rowC; + v4sf_t result[4]; + __vector_quad acc0; + __builtin_mma_xxsetaccz (&acc0); BLASLONG l = 0; - v4sf_t t = { 0, 0, 0, 0 }; - for (l = 0; l < k; l++) + for (l = 0; l < k / 2; l++) { - v4sf_t rowB = - { BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), - BF16TOF32 (BO[l]) - }; - v4sf_t rowA = - { BF16TOF32 (AO[l << 2]), BF16TOF32 (AO[(l << 2) + 1]), - BF16TOF32 (AO[(l << 2) + 2]), - BF16TOF32 (AO[(l << 2) + 3]) - }; - t += rowA * rowB; + vector short rowB = + { BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0}; + vec_t *rowA = (vec_t *) & (AO[l << 3]); + MMA (&acc0, (vec_t) rowB, rowA[0]); } - t = t * valpha; - CO[0] += t[0]; - CO[1] += t[1]; - CO[2] += t[2]; - CO[3] += t[3]; + if (k % 2 == 1) + { + if (k > 1) + l = (k / 2) << 1; + vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 }; + vec_t *rowA = (vec_t *) & (AO[(l << 2)]); + MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0])); + } + rowC = (v4sf_t *) &CO[0]; + __builtin_mma_disassemble_acc ((void *)result, &acc0); + rowC[0] += result[0] * alpha; AO += k << 2; BO += k; CO += 4; - i -= 4; } /* Loop for m >= 2. */ - while (i >= 2) + if (m & 2) { IFLOAT *BO = B; BLASLONG l = 0; @@ -1018,10 +935,9 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, AO += k << 1; BO += k; CO += 2; - i -= 2; } /* Loop for m = 1. */ - while (i >= 1) + if (m & 1) { IFLOAT *BO = B; BLASLONG l = 0; @@ -1034,7 +950,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, BO += k; CO[0] += t * alpha; CO += 1; - i -= 1; } B += k; diff --git a/kernel/power/sbgemm_ncopy_16_power10.c b/kernel/power/sbgemm_ncopy_16_power10.c new file mode 100644 index 0000000..c6b6330 --- /dev/null +++ b/kernel/power/sbgemm_ncopy_16_power10.c @@ -0,0 +1,437 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" + +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){ + BLASLONG i, j; + + IFLOAT *aoffset; + IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4; + IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8; + IFLOAT *aoffset9, *aoffset10, *aoffset11, *aoffset12; + IFLOAT *aoffset13, *aoffset14, *aoffset15, *aoffset16; + + IFLOAT *boffset; + IFLOAT ctemp01, ctemp02, ctemp03, ctemp04; + IFLOAT ctemp05, ctemp06, ctemp07, ctemp08; + IFLOAT ctemp09, ctemp10, ctemp11, ctemp12; + IFLOAT ctemp13, ctemp14, ctemp15, ctemp16; + IFLOAT ctemp17, ctemp18, ctemp19, ctemp20; + IFLOAT ctemp21, ctemp22, ctemp23, ctemp24; + IFLOAT ctemp25, ctemp26, ctemp27, ctemp28; + IFLOAT ctemp29, ctemp30, ctemp31, ctemp32; + + aoffset = a; + boffset = b; + + j = (n >> 4); + if (j > 0){ + do{ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset5 = aoffset4 + lda; + aoffset6 = aoffset5 + lda; + aoffset7 = aoffset6 + lda; + aoffset8 = aoffset7 + lda; + aoffset9 = aoffset8 + lda; + aoffset10 = aoffset9 + lda; + aoffset11 = aoffset10 + lda; + aoffset12 = aoffset11 + lda; + aoffset13 = aoffset12 + lda; + aoffset14 = aoffset13 + lda; + aoffset15 = aoffset14 + lda; + aoffset16 = aoffset15 + lda; + aoffset += 16 * lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + ctemp05 = *(aoffset3 + 0); + ctemp06 = *(aoffset3 + 1); + ctemp07 = *(aoffset4 + 0); + ctemp08 = *(aoffset4 + 1); + + ctemp09 = *(aoffset5 + 0); + ctemp10 = *(aoffset5 + 1); + ctemp11 = *(aoffset6 + 0); + ctemp12 = *(aoffset6 + 1); + + ctemp13 = *(aoffset7 + 0); + ctemp14 = *(aoffset7 + 1); + ctemp15 = *(aoffset8 + 0); + ctemp16 = *(aoffset8 + 1); + + ctemp17 = *(aoffset9 + 0); + ctemp18 = *(aoffset9 + 1); + ctemp19 = *(aoffset10 + 0); + ctemp20 = *(aoffset10 + 1); + + ctemp21 = *(aoffset11 + 0); + ctemp22 = *(aoffset11 + 1); + ctemp23 = *(aoffset12 + 0); + ctemp24 = *(aoffset12 + 1); + + ctemp25 = *(aoffset13 + 0); + ctemp26 = *(aoffset13 + 1); + ctemp27 = *(aoffset14 + 0); + ctemp28 = *(aoffset14 + 1); + + ctemp29 = *(aoffset15 + 0); + ctemp30 = *(aoffset15 + 1); + ctemp31 = *(aoffset16 + 0); + ctemp32 = *(aoffset16 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + *(boffset + 2) = ctemp03; + *(boffset + 3) = ctemp04; + *(boffset + 4) = ctemp05; + *(boffset + 5) = ctemp06; + *(boffset + 6) = ctemp07; + *(boffset + 7) = ctemp08; + + *(boffset + 8) = ctemp09; + *(boffset + 9) = ctemp10; + *(boffset + 10) = ctemp11; + *(boffset + 11) = ctemp12; + *(boffset + 12) = ctemp13; + *(boffset + 13) = ctemp14; + *(boffset + 14) = ctemp15; + *(boffset + 15) = ctemp16; + + *(boffset + 16) = ctemp17; + *(boffset + 17) = ctemp18; + *(boffset + 18) = ctemp19; + *(boffset + 19) = ctemp20; + *(boffset + 20) = ctemp21; + *(boffset + 21) = ctemp22; + *(boffset + 22) = ctemp23; + *(boffset + 23) = ctemp24; + + *(boffset + 24) = ctemp25; + *(boffset + 25) = ctemp26; + *(boffset + 26) = ctemp27; + *(boffset + 27) = ctemp28; + *(boffset + 28) = ctemp29; + *(boffset + 29) = ctemp30; + *(boffset + 30) = ctemp31; + *(boffset + 31) = ctemp32; + + aoffset1 += 2; + aoffset2 += 2; + aoffset3 += 2; + aoffset4 += 2; + aoffset5 += 2; + aoffset6 += 2; + aoffset7 += 2; + aoffset8 += 2; + + aoffset9 += 2; + aoffset10 += 2; + aoffset11 += 2; + aoffset12 += 2; + aoffset13 += 2; + aoffset14 += 2; + aoffset15 += 2; + aoffset16 += 2; + boffset += 32; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp03 = *(aoffset2 + 0); + ctemp05 = *(aoffset3 + 0); + ctemp07 = *(aoffset4 + 0); + ctemp09 = *(aoffset5 + 0); + ctemp11 = *(aoffset6 + 0); + ctemp13 = *(aoffset7 + 0); + ctemp15 = *(aoffset8 + 0); + + ctemp17 = *(aoffset9 + 0); + ctemp19 = *(aoffset10 + 0); + ctemp21 = *(aoffset11 + 0); + ctemp23 = *(aoffset12 + 0); + ctemp25 = *(aoffset13 + 0); + ctemp27 = *(aoffset14 + 0); + ctemp29 = *(aoffset15 + 0); + ctemp31 = *(aoffset16 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp05; + *(boffset + 3) = ctemp07; + *(boffset + 4) = ctemp09; + *(boffset + 5) = ctemp11; + *(boffset + 6) = ctemp13; + *(boffset + 7) = ctemp15; + + *(boffset + 8) = ctemp17; + *(boffset + 9) = ctemp19; + *(boffset + 10) = ctemp21; + *(boffset + 11) = ctemp23; + *(boffset + 12) = ctemp25; + *(boffset + 13) = ctemp27; + *(boffset + 14) = ctemp29; + *(boffset + 15) = ctemp31; + + boffset += 16; + } + j--; + }while(j > 0); + } /* end of if(j > 0) */ + + if (n & 8){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset5 = aoffset4 + lda; + aoffset6 = aoffset5 + lda; + aoffset7 = aoffset6 + lda; + aoffset8 = aoffset7 + lda; + aoffset += 8 * lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + ctemp05 = *(aoffset3 + 0); + ctemp06 = *(aoffset3 + 1); + ctemp07 = *(aoffset4 + 0); + ctemp08 = *(aoffset4 + 1); + + ctemp09 = *(aoffset5 + 0); + ctemp10 = *(aoffset5 + 1); + ctemp11 = *(aoffset6 + 0); + ctemp12 = *(aoffset6 + 1); + + ctemp13 = *(aoffset7 + 0); + ctemp14 = *(aoffset7 + 1); + ctemp15 = *(aoffset8 + 0); + ctemp16 = *(aoffset8 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + *(boffset + 2) = ctemp03; + *(boffset + 3) = ctemp04; + *(boffset + 4) = ctemp05; + *(boffset + 5) = ctemp06; + *(boffset + 6) = ctemp07; + *(boffset + 7) = ctemp08; + + *(boffset + 8) = ctemp09; + *(boffset + 9) = ctemp10; + *(boffset + 10) = ctemp11; + *(boffset + 11) = ctemp12; + *(boffset + 12) = ctemp13; + *(boffset + 13) = ctemp14; + *(boffset + 14) = ctemp15; + *(boffset + 15) = ctemp16; + + aoffset1 += 2; + aoffset2 += 2; + aoffset3 += 2; + aoffset4 += 2; + aoffset5 += 2; + aoffset6 += 2; + aoffset7 += 2; + aoffset8 += 2; + + boffset += 16; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp03 = *(aoffset2 + 0); + ctemp05 = *(aoffset3 + 0); + ctemp07 = *(aoffset4 + 0); + ctemp09 = *(aoffset5 + 0); + ctemp11 = *(aoffset6 + 0); + ctemp13 = *(aoffset7 + 0); + ctemp15 = *(aoffset8 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp05; + *(boffset + 3) = ctemp07; + *(boffset + 4) = ctemp09; + *(boffset + 5) = ctemp11; + *(boffset + 6) = ctemp13; + *(boffset + 7) = ctemp15; + + boffset += 8; + } + } + + if (n & 4){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset += 4 * lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + ctemp05 = *(aoffset3 + 0); + ctemp06 = *(aoffset3 + 1); + ctemp07 = *(aoffset4 + 0); + ctemp08 = *(aoffset4 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + *(boffset + 2) = ctemp03; + *(boffset + 3) = ctemp04; + *(boffset + 4) = ctemp05; + *(boffset + 5) = ctemp06; + *(boffset + 6) = ctemp07; + *(boffset + 7) = ctemp08; + + aoffset1 += 2; + aoffset2 += 2; + aoffset3 += 2; + aoffset4 += 2; + boffset += 8; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp03 = *(aoffset2 + 0); + ctemp05 = *(aoffset3 + 0); + ctemp07 = *(aoffset4 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp05; + *(boffset + 3) = ctemp07; + boffset += 4; + } + } + + if (n & 2){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset += 2 * lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp02; + *(boffset + 3) = ctemp04; + + aoffset1 += 2; + aoffset2 += 2; + boffset += 4; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp03 = *(aoffset2 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + boffset += 2; + } + } + + if (n & 1){ + aoffset1 = aoffset; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + + aoffset1 += 2; + boffset += 2; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + + *(boffset + 0) = ctemp01; + // boffset += 1; + } + } + + return 0; +} diff --git a/kernel/power/sbgemm_ncopy_8_power10.c b/kernel/power/sbgemm_ncopy_8_power10.c new file mode 100644 index 0000000..0e4a680 --- /dev/null +++ b/kernel/power/sbgemm_ncopy_8_power10.c @@ -0,0 +1,383 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include +#include "common.h" + +typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16))); +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){ + BLASLONG i, j; + + IFLOAT *aoffset; + IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4; + IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8; + + IFLOAT *boffset; + vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04; + vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08; + vec_bf16 vtemp09, vtemp10, vtemp11, vtemp12; + vector char mask = + { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 }; + vector char mask1 = + { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 }; + IFLOAT ctemp01, ctemp02, ctemp03, ctemp04; + IFLOAT ctemp05, ctemp06, ctemp07, ctemp08; + IFLOAT ctemp09, ctemp10, ctemp11, ctemp12; + IFLOAT ctemp13, ctemp14, ctemp15, ctemp16; + IFLOAT ctemp17; + IFLOAT ctemp25; + IFLOAT ctemp33; + IFLOAT ctemp41; + IFLOAT ctemp49; + IFLOAT ctemp57; + + + aoffset = a; + boffset = b; + + j = (n >> 3); + if (j > 0){ + do{ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset5 = aoffset4 + lda; + aoffset6 = aoffset5 + lda; + aoffset7 = aoffset6 + lda; + aoffset8 = aoffset7 + lda; + aoffset += 8 * lda; + + i = (m >> 3); + if (i > 0){ + do{ + vtemp01 = *(vec_bf16 *)(aoffset1); + vtemp02 = *(vec_bf16 *)(aoffset2); + vtemp03 = *(vec_bf16 *)(aoffset3); + vtemp04 = *(vec_bf16 *)(aoffset4); + vtemp05 = *(vec_bf16 *)(aoffset5); + vtemp06 = *(vec_bf16 *)(aoffset6); + vtemp07 = *(vec_bf16 *)(aoffset7); + vtemp08 = *(vec_bf16 *)(aoffset8); + + vtemp09 = vec_perm(vtemp01, vtemp02, mask); + vtemp10 = vec_perm(vtemp03, vtemp04, mask); + vtemp11 = vec_perm(vtemp05, vtemp06, mask); + vtemp12 = vec_perm(vtemp07, vtemp08, mask); + + *(vec_bf16 *)(boffset + 0) = vec_xxpermdi(vtemp09, vtemp10, 0); + *(vec_bf16 *)(boffset + 8) = vec_xxpermdi(vtemp11, vtemp12, 0); + *(vec_bf16 *)(boffset + 16) = vec_xxpermdi(vtemp09, vtemp10, 3); + *(vec_bf16 *)(boffset + 24) = vec_xxpermdi(vtemp11, vtemp12, 3); + + vtemp09 = vec_perm(vtemp01, vtemp02, mask1); + vtemp10 = vec_perm(vtemp03, vtemp04, mask1); + vtemp11 = vec_perm(vtemp05, vtemp06, mask1); + vtemp12 = vec_perm(vtemp07, vtemp08, mask1); + + *(vec_bf16 *)(boffset + 32) = vec_xxpermdi(vtemp09, vtemp10, 0); + *(vec_bf16 *)(boffset + 40) = vec_xxpermdi(vtemp11, vtemp12, 0); + *(vec_bf16 *)(boffset + 48) = vec_xxpermdi(vtemp09, vtemp10, 3); + *(vec_bf16 *)(boffset + 56) = vec_xxpermdi(vtemp11, vtemp12, 3); + + aoffset1 += 8; + aoffset2 += 8; + aoffset3 += 8; + aoffset4 += 8; + aoffset5 += 8; + aoffset6 += 8; + aoffset7 += 8; + aoffset8 += 8; + boffset += 64; + i --; + }while(i > 0); + } + + i = (m & 7); + if (i >= 2){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp09 = *(aoffset1 + 1); + ctemp17 = *(aoffset2 + 0); + ctemp25 = *(aoffset2 + 1); + ctemp33 = *(aoffset3 + 0); + ctemp41 = *(aoffset3 + 1); + ctemp49 = *(aoffset4 + 0); + ctemp57 = *(aoffset4 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp09; + *(boffset + 2) = ctemp17; + *(boffset + 3) = ctemp25; + *(boffset + 4) = ctemp33; + *(boffset + 5) = ctemp41; + *(boffset + 6) = ctemp49; + *(boffset + 7) = ctemp57; + aoffset1 += 2; + aoffset2 += 2; + aoffset3 += 2; + aoffset4 += 2; + + ctemp01 = *(aoffset5 + 0); + ctemp09 = *(aoffset5 + 1); + ctemp17 = *(aoffset6 + 0); + ctemp25 = *(aoffset6 + 1); + ctemp33 = *(aoffset7 + 0); + ctemp41 = *(aoffset7 + 1); + ctemp49 = *(aoffset8 + 0); + ctemp57 = *(aoffset8 + 1); + *(boffset + 8) = ctemp01; + *(boffset + 9) = ctemp09; + *(boffset + 10) = ctemp17; + *(boffset + 11) = ctemp25; + *(boffset + 12) = ctemp33; + *(boffset + 13) = ctemp41; + *(boffset + 14) = ctemp49; + *(boffset + 15) = ctemp57; + + aoffset5 += 2; + aoffset6 += 2; + aoffset7 += 2; + aoffset8 += 2; + + boffset += 16; + i -= 2; + }while(i > 1); + } + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp09 = *(aoffset2 + 0); + ctemp17 = *(aoffset3 + 0); + ctemp25 = *(aoffset4 + 0); + ctemp33 = *(aoffset5 + 0); + ctemp41 = *(aoffset6 + 0); + ctemp49 = *(aoffset7 + 0); + ctemp57 = *(aoffset8 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp09; + *(boffset + 2) = ctemp17; + *(boffset + 3) = ctemp25; + *(boffset + 4) = ctemp33; + *(boffset + 5) = ctemp41; + *(boffset + 6) = ctemp49; + *(boffset + 7) = ctemp57; + + aoffset1 ++; + aoffset2 ++; + aoffset3 ++; + aoffset4 ++; + aoffset5 ++; + aoffset6 ++; + aoffset7 ++; + aoffset8 ++; + + boffset += 8; + } + + j--; + }while(j > 0); + } /* end of if(j > 0) */ + + if (n & 4){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset += 4 * lda; + + i = (m >> 2); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + + ctemp05 = *(aoffset2 + 0); + ctemp06 = *(aoffset2 + 1); + ctemp07 = *(aoffset2 + 2); + ctemp08 = *(aoffset2 + 3); + + ctemp09 = *(aoffset3 + 0); + ctemp10 = *(aoffset3 + 1); + ctemp11 = *(aoffset3 + 2); + ctemp12 = *(aoffset3 + 3); + + ctemp13 = *(aoffset4 + 0); + ctemp14 = *(aoffset4 + 1); + ctemp15 = *(aoffset4 + 2); + ctemp16 = *(aoffset4 + 3); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + *(boffset + 2) = ctemp05; + *(boffset + 3) = ctemp06; + + *(boffset + 4) = ctemp09; + *(boffset + 5) = ctemp10; + *(boffset + 6) = ctemp13; + *(boffset + 7) = ctemp14; + + *(boffset + 8) = ctemp03; + *(boffset + 9) = ctemp04; + *(boffset + 10) = ctemp07; + *(boffset + 11) = ctemp08; + + *(boffset + 12) = ctemp11; + *(boffset + 13) = ctemp12; + *(boffset + 14) = ctemp15; + *(boffset + 15) = ctemp16; + + aoffset1 += 4; + aoffset2 += 4; + aoffset3 += 4; + aoffset4 += 4; + boffset += 16; + i --; + }while(i > 0); + } + + i = (m & 3); + if (i >= 2){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp09 = *(aoffset1 + 1); + ctemp17 = *(aoffset2 + 0); + ctemp25 = *(aoffset2 + 1); + ctemp33 = *(aoffset3 + 0); + ctemp41 = *(aoffset3 + 1); + ctemp49 = *(aoffset4 + 0); + ctemp57 = *(aoffset4 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp09; + *(boffset + 2) = ctemp17; + *(boffset + 3) = ctemp25; + *(boffset + 4) = ctemp33; + *(boffset + 5) = ctemp41; + *(boffset + 6) = ctemp49; + *(boffset + 7) = ctemp57; + aoffset1 += 2; + aoffset2 += 2; + aoffset3 += 2; + aoffset4 += 2; + + boffset += 8; + i -= 2; + }while(i > 1); + } + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset2 + 0); + ctemp03 = *(aoffset3 + 0); + ctemp04 = *(aoffset4 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + *(boffset + 2) = ctemp03; + *(boffset + 3) = ctemp04; + + aoffset1 ++; + aoffset2 ++; + aoffset3 ++; + aoffset4 ++; + + boffset += 4; + } + } + + if (n & 2){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset += 2 * lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp03; + *(boffset + 2) = ctemp02; + *(boffset + 3) = ctemp04; + + aoffset1 += 2; + aoffset2 += 2; + boffset += 4; + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset2 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + + aoffset1 ++; + aoffset2 ++; + boffset += 2; + } + } /* end of if(j > 0) */ + + if (n & 1){ + aoffset1 = aoffset; + + i = m; + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + + *(boffset + 0) = ctemp01; + + aoffset1 ++; + boffset ++; + i --; + }while(i > 0); + } + + } /* end of if(j > 0) */ + + return 0; +} diff --git a/kernel/power/sbgemm_tcopy_16_power10.c b/kernel/power/sbgemm_tcopy_16_power10.c new file mode 100644 index 0000000..120c5ab --- /dev/null +++ b/kernel/power/sbgemm_tcopy_16_power10.c @@ -0,0 +1,244 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include +#include "common.h" +typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16))); + +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){ + + BLASLONG i, j; + + IFLOAT *aoffset; + IFLOAT *aoffset1, *aoffset2; + IFLOAT *boffset; + + vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04; + IFLOAT ctemp01, ctemp02, ctemp03, ctemp04; + IFLOAT ctemp05, ctemp06, ctemp07, ctemp08; + + aoffset = a; + boffset = b; + +#if 0 + fprintf(stderr, "m = %d n = %d\n", m, n); +#endif + + j = (n >> 4); + if (j > 0){ + do{ + aoffset1 = aoffset; + aoffset2 = aoffset + lda; + aoffset += 16; + + i = (m >> 1); + if (i > 0){ + do{ + vtemp01 = *(vec_bf16 *)(aoffset1); + vtemp02 = *(vec_bf16 *)(aoffset1+8); + vtemp03 = *(vec_bf16 *)(aoffset2); + vtemp04 = *(vec_bf16 *)(aoffset2+8); + *(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03); + *(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03); + *(vec_bf16 *)(boffset + 16) = vec_mergeh(vtemp02, vtemp04); + *(vec_bf16 *)(boffset + 24) = vec_mergel(vtemp02, vtemp04); + aoffset1 += 2 * lda; + aoffset2 += 2 * lda; + boffset += 32; + + i --; + }while(i > 0); + } + + if (m & 1){ + vtemp01 = *(vec_bf16 *)(aoffset1); + vtemp02 = *(vec_bf16 *)(aoffset1+8); + *(vec_bf16 *)(boffset + 0) = vtemp01; + *(vec_bf16 *)(boffset + 8) = vtemp02; + boffset += 16; + } + + j--; + }while(j > 0); + } /* end of if(j > 0) */ + + if (n & 8){ + aoffset1 = aoffset; + aoffset2 = aoffset + lda; + aoffset += 8; + + i = (m >> 1); + if (i > 0){ + do{ + vtemp01 = *(vec_bf16 *)(aoffset1); + vtemp03 = *(vec_bf16 *)(aoffset2); + *(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03); + *(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03); + + aoffset1 += 2 * lda; + aoffset2 += 2 * lda; + boffset += 16; + + i --; + }while(i > 0); + } + + if (m & 1){ + vtemp01 = *(vec_bf16 *)(aoffset1); + *(vec_bf16 *)(boffset + 0) = vtemp01; + boffset += 8; + } + } + + if (n & 4){ + aoffset1 = aoffset; + aoffset2 = aoffset + lda; + aoffset += 4; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + + ctemp05 = *(aoffset2 + 0); + ctemp06 = *(aoffset2 + 1); + ctemp07 = *(aoffset2 + 2); + ctemp08 = *(aoffset2 + 3); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp05; + *(boffset + 2) = ctemp02; + *(boffset + 3) = ctemp06; + *(boffset + 4) = ctemp03; + *(boffset + 5) = ctemp07; + *(boffset + 6) = ctemp04; + *(boffset + 7) = ctemp08; + + aoffset1 += 2 * lda; + aoffset2 += 2 * lda; + boffset += 8; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + *(boffset + 2) = ctemp03; + *(boffset + 3) = ctemp04; + + boffset += 4; + } + } + + if (n & 2){ + aoffset1 = aoffset; + aoffset2 = aoffset + lda; + aoffset += 2; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + *(boffset + 2) = ctemp03; + *(boffset + 3) = ctemp04; + + aoffset1 += 2 * lda; + aoffset2 += 2 * lda; + boffset += 4; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + boffset += 2; + } + } + + if (n & 1){ + aoffset1 = aoffset; + aoffset2 = aoffset + lda; + + i = (m >> 1); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset2 + 0); + + *(boffset + 0) = ctemp01; + *(boffset + 1) = ctemp02; + + aoffset1 += 2 * lda; + aoffset2 += 2 * lda; + boffset += 2; + + i --; + }while(i > 0); + } + + if (m & 1){ + ctemp01 = *(aoffset1 + 0); + *(boffset + 0) = ctemp01; + // boffset += 1; + } + } + + return 0; +} diff --git a/kernel/power/sbgemm_tcopy_8_power10.c b/kernel/power/sbgemm_tcopy_8_power10.c new file mode 100644 index 0000000..aceb0c9 --- /dev/null +++ b/kernel/power/sbgemm_tcopy_8_power10.c @@ -0,0 +1,659 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" +#include + +typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16))); + +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){ + + BLASLONG i, j; + + IFLOAT *aoffset; + IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4; + IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8; + + IFLOAT *boffset, *boffset1, *boffset2, *boffset3, *boffset4; + vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04; + vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08; + IFLOAT ctemp01, ctemp02, ctemp03, ctemp04; + IFLOAT ctemp05, ctemp06, ctemp07, ctemp08; + IFLOAT ctemp09, ctemp10, ctemp11, ctemp12; + IFLOAT ctemp13, ctemp14, ctemp15, ctemp16; + IFLOAT ctemp17, ctemp18, ctemp19, ctemp20; + IFLOAT ctemp21, ctemp22, ctemp23, ctemp24; + IFLOAT ctemp25, ctemp26, ctemp27, ctemp28; + IFLOAT ctemp29, ctemp30, ctemp31, ctemp32; + + aoffset = a; + boffset = b; + +#if 0 + fprintf(stderr, "M = %d N = %d\n", m, n); +#endif + + boffset2 = b + m * (n & ~7); + boffset3 = b + m * (n & ~3); + boffset4 = b + m * (n & ~1); + + j = (m >> 3); + if (j > 0){ + do{ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset5 = aoffset4 + lda; + aoffset6 = aoffset5 + lda; + aoffset7 = aoffset6 + lda; + aoffset8 = aoffset7 + lda; + aoffset += 8 * lda; + + boffset1 = boffset; + boffset += 64; + + i = (n >> 3); + if (i > 0){ + do{ + vtemp01 = *(vec_bf16 *)(aoffset1); + vtemp02 = *(vec_bf16 *)(aoffset2); + vtemp03 = *(vec_bf16 *)(aoffset3); + vtemp04 = *(vec_bf16 *)(aoffset4); + vtemp05 = *(vec_bf16 *)(aoffset5); + vtemp06 = *(vec_bf16 *)(aoffset6); + vtemp07 = *(vec_bf16 *)(aoffset7); + vtemp08 = *(vec_bf16 *)(aoffset8); + aoffset1 += 8; + aoffset2 += 8; + aoffset3 += 8; + aoffset4 += 8; + aoffset5 += 8; + aoffset6 += 8; + aoffset7 += 8; + aoffset8 += 8; + + *(vec_bf16 *)(boffset1 + 0) = vec_mergeh(vtemp01, vtemp02); + *(vec_bf16 *)(boffset1 + 8) = vec_mergel(vtemp01, vtemp02); + *(vec_bf16 *)(boffset1 + 16) = vec_mergeh(vtemp03, vtemp04); + *(vec_bf16 *)(boffset1 + 24) = vec_mergel(vtemp03, vtemp04); + *(vec_bf16 *)(boffset1 + 32) = vec_mergeh(vtemp05, vtemp06); + *(vec_bf16 *)(boffset1 + 40) = vec_mergel(vtemp05, vtemp06); + *(vec_bf16 *)(boffset1 + 48) = vec_mergeh(vtemp07, vtemp08); + *(vec_bf16 *)(boffset1 + 56) = vec_mergel(vtemp07, vtemp08); + + boffset1 += m * 8; + i --; + }while(i > 0); + } + + if (n & 4){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + aoffset1 += 4; + + ctemp05 = *(aoffset2 + 0); + ctemp06 = *(aoffset2 + 1); + ctemp07 = *(aoffset2 + 2); + ctemp08 = *(aoffset2 + 3); + aoffset2 += 4; + + ctemp09 = *(aoffset3 + 0); + ctemp10 = *(aoffset3 + 1); + ctemp11 = *(aoffset3 + 2); + ctemp12 = *(aoffset3 + 3); + aoffset3 += 4; + + ctemp13 = *(aoffset4 + 0); + ctemp14 = *(aoffset4 + 1); + ctemp15 = *(aoffset4 + 2); + ctemp16 = *(aoffset4 + 3); + aoffset4 += 4; + + ctemp17 = *(aoffset5 + 0); + ctemp18 = *(aoffset5 + 1); + ctemp19 = *(aoffset5 + 2); + ctemp20 = *(aoffset5 + 3); + aoffset5 += 4; + + ctemp21 = *(aoffset6 + 0); + ctemp22 = *(aoffset6 + 1); + ctemp23 = *(aoffset6 + 2); + ctemp24 = *(aoffset6 + 3); + aoffset6 += 4; + + ctemp25 = *(aoffset7 + 0); + ctemp26 = *(aoffset7 + 1); + ctemp27 = *(aoffset7 + 2); + ctemp28 = *(aoffset7 + 3); + aoffset7 += 4; + + ctemp29 = *(aoffset8 + 0); + ctemp30 = *(aoffset8 + 1); + ctemp31 = *(aoffset8 + 2); + ctemp32 = *(aoffset8 + 3); + aoffset8 += 4; + + *(boffset2 + 0) = ctemp01; + *(boffset2 + 1) = ctemp05; + *(boffset2 + 2) = ctemp02; + *(boffset2 + 3) = ctemp06; + *(boffset2 + 4) = ctemp03; + *(boffset2 + 5) = ctemp07; + *(boffset2 + 6) = ctemp04; + *(boffset2 + 7) = ctemp08; + + *(boffset2 + 8) = ctemp09; + *(boffset2 + 9) = ctemp13; + *(boffset2 + 10) = ctemp10; + *(boffset2 + 11) = ctemp14; + *(boffset2 + 12) = ctemp11; + *(boffset2 + 13) = ctemp15; + *(boffset2 + 14) = ctemp12; + *(boffset2 + 15) = ctemp16; + + *(boffset2 + 16) = ctemp17; + *(boffset2 + 17) = ctemp21; + *(boffset2 + 18) = ctemp18; + *(boffset2 + 19) = ctemp22; + *(boffset2 + 20) = ctemp19; + *(boffset2 + 21) = ctemp23; + *(boffset2 + 22) = ctemp20; + *(boffset2 + 23) = ctemp24; + + *(boffset2 + 24) = ctemp25; + *(boffset2 + 25) = ctemp29; + *(boffset2 + 26) = ctemp26; + *(boffset2 + 27) = ctemp30; + *(boffset2 + 28) = ctemp27; + *(boffset2 + 29) = ctemp31; + *(boffset2 + 30) = ctemp28; + *(boffset2 + 31) = ctemp32; + + boffset2 += 32; + } + + if (n & 2){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + aoffset1 += 2; + + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + aoffset2 += 2; + + ctemp05 = *(aoffset3 + 0); + ctemp06 = *(aoffset3 + 1); + aoffset3 += 2; + + ctemp07 = *(aoffset4 + 0); + ctemp08 = *(aoffset4 + 1); + aoffset4 += 2; + + ctemp09 = *(aoffset5 + 0); + ctemp10 = *(aoffset5 + 1); + aoffset5 += 2; + + ctemp11 = *(aoffset6 + 0); + ctemp12 = *(aoffset6 + 1); + aoffset6 += 2; + + ctemp13 = *(aoffset7 + 0); + ctemp14 = *(aoffset7 + 1); + aoffset7 += 2; + + ctemp15 = *(aoffset8 + 0); + ctemp16 = *(aoffset8 + 1); + aoffset8 += 2; + + *(boffset3 + 0) = ctemp01; + *(boffset3 + 1) = ctemp02; + *(boffset3 + 2) = ctemp03; + *(boffset3 + 3) = ctemp04; + *(boffset3 + 4) = ctemp05; + *(boffset3 + 5) = ctemp06; + *(boffset3 + 6) = ctemp07; + *(boffset3 + 7) = ctemp08; + *(boffset3 + 8) = ctemp09; + *(boffset3 + 9) = ctemp10; + *(boffset3 + 10) = ctemp11; + *(boffset3 + 11) = ctemp12; + *(boffset3 + 12) = ctemp13; + *(boffset3 + 13) = ctemp14; + *(boffset3 + 14) = ctemp15; + *(boffset3 + 15) = ctemp16; + boffset3 += 16; + } + + if (n & 1){ + ctemp01 = *(aoffset1 + 0); + aoffset1 ++; + ctemp02 = *(aoffset2 + 0); + aoffset2 ++; + ctemp03 = *(aoffset3 + 0); + aoffset3 ++; + ctemp04 = *(aoffset4 + 0); + aoffset4 ++; + ctemp05 = *(aoffset5 + 0); + aoffset5 ++; + ctemp06 = *(aoffset6 + 0); + aoffset6 ++; + ctemp07 = *(aoffset7 + 0); + aoffset7 ++; + ctemp08 = *(aoffset8 + 0); + aoffset8 ++; + + *(boffset4 + 0) = ctemp01; + *(boffset4 + 1) = ctemp02; + *(boffset4 + 2) = ctemp03; + *(boffset4 + 3) = ctemp04; + *(boffset4 + 4) = ctemp05; + *(boffset4 + 5) = ctemp06; + *(boffset4 + 6) = ctemp07; + *(boffset4 + 7) = ctemp08; + boffset4 += 8; + } + + j--; + }while(j > 0); + } + + if (m & 4){ + + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset3 = aoffset2 + lda; + aoffset4 = aoffset3 + lda; + aoffset += 4 * lda; + + boffset1 = boffset; + boffset += 32; + + i = (n >> 3); + if (i > 0){ + + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + ctemp05 = *(aoffset1 + 4); + ctemp06 = *(aoffset1 + 5); + ctemp07 = *(aoffset1 + 6); + ctemp08 = *(aoffset1 + 7); + aoffset1 += 8; + + ctemp09 = *(aoffset2 + 0); + ctemp10 = *(aoffset2 + 1); + ctemp11 = *(aoffset2 + 2); + ctemp12 = *(aoffset2 + 3); + ctemp13 = *(aoffset2 + 4); + ctemp14 = *(aoffset2 + 5); + ctemp15 = *(aoffset2 + 6); + ctemp16 = *(aoffset2 + 7); + aoffset2 += 8; + + ctemp17 = *(aoffset3 + 0); + ctemp18 = *(aoffset3 + 1); + ctemp19 = *(aoffset3 + 2); + ctemp20 = *(aoffset3 + 3); + ctemp21 = *(aoffset3 + 4); + ctemp22 = *(aoffset3 + 5); + ctemp23 = *(aoffset3 + 6); + ctemp24 = *(aoffset3 + 7); + aoffset3 += 8; + + ctemp25 = *(aoffset4 + 0); + ctemp26 = *(aoffset4 + 1); + ctemp27 = *(aoffset4 + 2); + ctemp28 = *(aoffset4 + 3); + ctemp29 = *(aoffset4 + 4); + ctemp30 = *(aoffset4 + 5); + ctemp31 = *(aoffset4 + 6); + ctemp32 = *(aoffset4 + 7); + aoffset4 += 8; + + *(boffset1 + 0) = ctemp01; + *(boffset1 + 1) = ctemp09; + *(boffset1 + 2) = ctemp02; + *(boffset1 + 3) = ctemp10; + *(boffset1 + 4) = ctemp03; + *(boffset1 + 5) = ctemp11; + *(boffset1 + 6) = ctemp04; + *(boffset1 + 7) = ctemp12; + + *(boffset1 + 8) = ctemp05; + *(boffset1 + 9) = ctemp13; + *(boffset1 + 10) = ctemp06; + *(boffset1 + 11) = ctemp14; + *(boffset1 + 12) = ctemp07; + *(boffset1 + 13) = ctemp15; + *(boffset1 + 14) = ctemp08; + *(boffset1 + 15) = ctemp16; + + *(boffset1 + 16) = ctemp17; + *(boffset1 + 17) = ctemp25; + *(boffset1 + 18) = ctemp18; + *(boffset1 + 19) = ctemp26; + *(boffset1 + 20) = ctemp19; + *(boffset1 + 21) = ctemp27; + *(boffset1 + 22) = ctemp20; + *(boffset1 + 23) = ctemp28; + + *(boffset1 + 24) = ctemp21; + *(boffset1 + 25) = ctemp29; + *(boffset1 + 26) = ctemp22; + *(boffset1 + 27) = ctemp30; + *(boffset1 + 28) = ctemp23; + *(boffset1 + 29) = ctemp31; + *(boffset1 + 30) = ctemp24; + *(boffset1 + 31) = ctemp32; + + boffset1 += 8 * m; + i --; + }while(i > 0); + } + + if (n & 4) { + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + aoffset1 += 4; + + ctemp05 = *(aoffset2 + 0); + ctemp06 = *(aoffset2 + 1); + ctemp07 = *(aoffset2 + 2); + ctemp08 = *(aoffset2 + 3); + aoffset2 += 4; + + ctemp09 = *(aoffset3 + 0); + ctemp10 = *(aoffset3 + 1); + ctemp11 = *(aoffset3 + 2); + ctemp12 = *(aoffset3 + 3); + aoffset3 += 4; + + ctemp13 = *(aoffset4 + 0); + ctemp14 = *(aoffset4 + 1); + ctemp15 = *(aoffset4 + 2); + ctemp16 = *(aoffset4 + 3); + aoffset4 += 4; + + *(boffset2 + 0) = ctemp01; + *(boffset2 + 1) = ctemp05; + *(boffset2 + 2) = ctemp02; + *(boffset2 + 3) = ctemp06; + *(boffset2 + 4) = ctemp03; + *(boffset2 + 5) = ctemp07; + *(boffset2 + 6) = ctemp04; + *(boffset2 + 7) = ctemp08; + + *(boffset2 + 8) = ctemp09; + *(boffset2 + 9) = ctemp13; + *(boffset2 + 10) = ctemp10; + *(boffset2 + 11) = ctemp14; + *(boffset2 + 12) = ctemp11; + *(boffset2 + 13) = ctemp15; + *(boffset2 + 14) = ctemp12; + *(boffset2 + 15) = ctemp16; + boffset2 += 16; + } + + if (n & 2){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + aoffset1 += 2; + + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + aoffset2 += 2; + + ctemp05 = *(aoffset3 + 0); + ctemp06 = *(aoffset3 + 1); + aoffset3 += 2; + + ctemp07 = *(aoffset4 + 0); + ctemp08 = *(aoffset4 + 1); + aoffset4 += 2; + + *(boffset3 + 0) = ctemp01; + *(boffset3 + 1) = ctemp02; + *(boffset3 + 2) = ctemp03; + *(boffset3 + 3) = ctemp04; + *(boffset3 + 4) = ctemp05; + *(boffset3 + 5) = ctemp06; + *(boffset3 + 6) = ctemp07; + *(boffset3 + 7) = ctemp08; + boffset3 += 8; + } + + if (n & 1){ + ctemp01 = *(aoffset1 + 0); + aoffset1 ++; + ctemp02 = *(aoffset2 + 0); + aoffset2 ++; + ctemp03 = *(aoffset3 + 0); + aoffset3 ++; + ctemp04 = *(aoffset4 + 0); + aoffset4 ++; + + *(boffset4 + 0) = ctemp01; + *(boffset4 + 1) = ctemp02; + *(boffset4 + 2) = ctemp03; + *(boffset4 + 3) = ctemp04; + boffset4 += 4; + } + } + + if (m & 2){ + aoffset1 = aoffset; + aoffset2 = aoffset1 + lda; + aoffset += 2 * lda; + + boffset1 = boffset; + boffset += 16; + + i = (n >> 3); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + ctemp05 = *(aoffset1 + 4); + ctemp06 = *(aoffset1 + 5); + ctemp07 = *(aoffset1 + 6); + ctemp08 = *(aoffset1 + 7); + aoffset1 += 8; + + ctemp09 = *(aoffset2 + 0); + ctemp10 = *(aoffset2 + 1); + ctemp11 = *(aoffset2 + 2); + ctemp12 = *(aoffset2 + 3); + ctemp13 = *(aoffset2 + 4); + ctemp14 = *(aoffset2 + 5); + ctemp15 = *(aoffset2 + 6); + ctemp16 = *(aoffset2 + 7); + aoffset2 += 8; + + *(boffset1 + 0) = ctemp01; + *(boffset1 + 1) = ctemp09; + *(boffset1 + 2) = ctemp02; + *(boffset1 + 3) = ctemp10; + *(boffset1 + 4) = ctemp03; + *(boffset1 + 5) = ctemp11; + *(boffset1 + 6) = ctemp04; + *(boffset1 + 7) = ctemp12; + + *(boffset1 + 8) = ctemp05; + *(boffset1 + 9) = ctemp13; + *(boffset1 + 10) = ctemp06; + *(boffset1 + 11) = ctemp14; + *(boffset1 + 12) = ctemp07; + *(boffset1 + 13) = ctemp15; + *(boffset1 + 14) = ctemp08; + *(boffset1 + 15) = ctemp16; + + boffset1 += 8 * m; + i --; + }while(i > 0); + } + + if (n & 4){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + aoffset1 += 4; + + ctemp05 = *(aoffset2 + 0); + ctemp06 = *(aoffset2 + 1); + ctemp07 = *(aoffset2 + 2); + ctemp08 = *(aoffset2 + 3); + aoffset2 += 4; + + *(boffset2 + 0) = ctemp01; + *(boffset2 + 1) = ctemp05; + *(boffset2 + 2) = ctemp02; + *(boffset2 + 3) = ctemp06; + *(boffset2 + 4) = ctemp03; + *(boffset2 + 5) = ctemp07; + *(boffset2 + 6) = ctemp04; + *(boffset2 + 7) = ctemp08; + boffset2 += 8; + } + + if (n & 2){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + aoffset1 += 2; + + ctemp03 = *(aoffset2 + 0); + ctemp04 = *(aoffset2 + 1); + aoffset2 += 2; + + *(boffset3 + 0) = ctemp01; + *(boffset3 + 1) = ctemp02; + *(boffset3 + 2) = ctemp03; + *(boffset3 + 3) = ctemp04; + boffset3 += 4; + } + + if (n & 1){ + ctemp01 = *(aoffset1 + 0); + aoffset1 ++; + ctemp02 = *(aoffset2 + 0); + aoffset2 ++; + + *(boffset4 + 0) = ctemp01; + *(boffset4 + 1) = ctemp02; + boffset4 += 2; + } + } + + if (m & 1){ + aoffset1 = aoffset; + // aoffset += lda; + + boffset1 = boffset; + // boffset += 8; + + i = (n >> 3); + if (i > 0){ + do{ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + ctemp05 = *(aoffset1 + 4); + ctemp06 = *(aoffset1 + 5); + ctemp07 = *(aoffset1 + 6); + ctemp08 = *(aoffset1 + 7); + aoffset1 += 8; + + *(boffset1 + 0) = ctemp01; + *(boffset1 + 1) = ctemp02; + *(boffset1 + 2) = ctemp03; + *(boffset1 + 3) = ctemp04; + *(boffset1 + 4) = ctemp05; + *(boffset1 + 5) = ctemp06; + *(boffset1 + 6) = ctemp07; + *(boffset1 + 7) = ctemp08; + + boffset1 += 8 * m; + i --; + }while(i > 0); + } + + if (n & 4){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + ctemp03 = *(aoffset1 + 2); + ctemp04 = *(aoffset1 + 3); + aoffset1 += 4; + + *(boffset2 + 0) = ctemp01; + *(boffset2 + 1) = ctemp02; + *(boffset2 + 2) = ctemp03; + *(boffset2 + 3) = ctemp04; + // boffset2 += 4; + } + + if (n & 2){ + ctemp01 = *(aoffset1 + 0); + ctemp02 = *(aoffset1 + 1); + aoffset1 += 2; + + *(boffset3 + 0) = ctemp01; + *(boffset3 + 1) = ctemp02; + // boffset3 += 2; + } + + if (n & 1){ + ctemp01 = *(aoffset1 + 0); + aoffset1 ++; + *(boffset4 + 0) = ctemp01; + boffset4 ++; + } + } + + return 0; +} -- 2.7.4