SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX)
SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX)
-DGEMMKERNEL = dgemm_kernel_sve_v1x$(DGEMM_UNROLL_N).S
+DGEMMKERNEL = dgemm_kernel_sve_v2x$(DGEMM_UNROLL_N).S
DTRMMKERNEL = dtrmm_kernel_sve_v1x$(DGEMM_UNROLL_N).S
DGEMMINCOPY = dgemm_ncopy_sve_v1.c
#define alpha0 d10
#define alphaZ z2.d
-#define A_PRE_SIZE 2560
+#define A_PRE_SIZE 1536
#define B_PRE_SIZE 512
#define C_PRE_SIZE 128
.macro KERNELv1x8_I
ld1d z0.d, p1/z, [pA]
ld1d z1.d, p1/z, [pA, lanes, lsl #3] // next one
- //incb pA, all, mul #2
add pA, pA, lanes, lsl #4 // pA = pA + lanes * 2 * 8
ld1rd z8.d, p0/z, [pB]
ptrue p0.d // create true predicate
mov pB, origPB
-
+// Loop over N
mov counterJ, origN
asr counterJ, counterJ, #3 // J = J / 8
cmp counterJ, #0
ble .Ldgemm_kernel_L4_BEGIN
/******************************************************************************/
+/* Repeat this as long as there are 8 left in N */
.align 5
.Ldgemm_kernel_L8_BEGIN:
.Ldgemm_kernel_L8_Mv1_BEGIN:
+/* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */
mov counterI, #0
- whilelt p1.d, counterI, origM //SVE instruction
+ whilelt p1.d, counterI, origM
cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension
.align 5
bgt .Ldgemm_kernel_L8_BEGIN
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 4 left in N */
.align 5
.Ldgemm_kernel_L4_BEGIN:
add origPB, origPB, temp // B = B + K * 4 * 8
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 2 left in N */
.align 5
.Ldgemm_kernel_L2_BEGIN:
add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 1 left in N */
.align 5
.Ldgemm_kernel_L1_BEGIN:
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
+/* This is an SVE dgemm kernel with size 2*SVE_LEN x 8.
+However, the data layout is the same as for the kernel 1*SVE_LEN x 8.
+This means that we sweep two panels of packed A when iterating in a loop over K.
+With this approach, we can reuse dgemm_n|tcopy_sve_v1.c packing functions. */
+
#define ASSEMBLER
#include "common.h"
#define alpha0 d10
#define alphaZ z7.d
-#define A_PRE_SIZE 2560
+#define A_PRE_SIZE 1536
#define B_PRE_SIZE 512
#define C_PRE_SIZE 128
//v00 ALPHA -> pA10_0
//v01 pA10_1
-//v02
-//v03
+//v02 pA20_0
+//v03 pA20_1
//v04
//v05
//v06
//v21 must save C5
//v22 must save C6
//v23 must save C7
+//v24 must save C8
+//v25 must save C9
+//v26 must save C10
+//v27 must save C11
+//v28 must save C12
+//v29 must save C13
+//v30 must save C14
+//v31 must save C15
/*******************************************************************************
* Macro definitions
.macro KERNELv1x8_I
ld1d z0.d, p1/z, [pA1]
ld1d z1.d, p1/z, [pA1, lanes, lsl #3] // next one
- //incb pA1, all, mul #2
add pA1, pA1, lanes, lsl #4 // pA1 = pA1 + lanes * 2 * 8
ld1rd z8.d, p0/z, [pB]
ptrue p0.d // create true predicate
mov pB, origPB
-
+// Loop over N
mov counterJ, origN
asr counterJ, counterJ, #3 // J = J / 8
cmp counterJ, #0
ble .Ldgemm_kernel_L4_BEGIN
/******************************************************************************/
+/* Repeat this as long as there are 8 left in N */
.align 5
.Ldgemm_kernel_L8_BEGIN:
.Ldgemm_kernel_L8_Mv2_BEGIN:
mov counterI, #0
- cmp origM, vec_lenx2
+ cmp origM, vec_lenx2 // Check if M < 2*SVE_LEN
blt .Ldgemm_kernel_L8_Mv1_BEGIN
mov counterI, origM
+/* Until we have at least 2*SVE_LEN iters left in M, we do them with V2*8 kernel */
mul temp, vec_len, origK // generate address of pA2
add pA2, pA1, temp, lsl #3 // pA1 = start of A array
prfm PLDL1KEEP, [pA2]
cmp counterI, origM
beq .Ldgemm_kernel_L8_END
-//////////////////////////////////
+//////////////////////////////////////////
+// We have less than 2*SVE_LEN left. We do this with V1x8 kernel.
.Ldgemm_kernel_L8_Mv1_BEGIN:
whilelt p1.d, counterI, origM //SVE instruction
bgt .Ldgemm_kernel_L8_BEGIN
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 4 left in N */
.align 5
.Ldgemm_kernel_L4_BEGIN:
beq .Ldgemm_kernel_L4_END
//////////////////////////////////
+// We have less than 2*SVE_LEN left. We do this with V1x4 kernel.
.Ldgemm_kernel_L4_Mv1_BEGIN:
whilelt p1.d, counterI, origM //SVE instruction
add origPB, origPB, temp // B = B + K * 4 * 8
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 2 left in N */
.align 5
.Ldgemm_kernel_L2_BEGIN:
//////////////////////////////////
+// We have less than 2*SVE_LEN left. We do this with V1x2 kernel.
.Ldgemm_kernel_L2_Mv1_BEGIN:
whilelt p1.d, counterI, origM //SVE instruction
add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 1 left in N */
.align 5
.Ldgemm_kernel_L1_BEGIN:
//////////////////////////////////
+// We have less than 2*SVE_LEN left. We do this with V1x1 kernel.
.Ldgemm_kernel_L1_Mv1_BEGIN:
whilelt p1.d, counterI, origM //SVE instruction
#include "common.h"
#include <arm_sve.h>
-// TODO: write in assembly with proper unrolling
+// TODO: write in assembly with proper unrolling of inner loop
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
- BLASLONG j;
- IFLOAT *aoffset, *aoffset1, *boffset;
+ BLASLONG j;
+ IFLOAT *aoffset, *aoffset1, *boffset;
- svint64_t lda_vec = svindex_s64(0LL, lda);
- uint64_t sve_size = svcntd();
+ svint64_t lda_vec = svindex_s64(0LL, lda);
+ uint64_t sve_size = svcntd();
- aoffset = a;
- boffset = b;
+ aoffset = a;
+ boffset = b;
- j = 0;
- svbool_t pg = svwhilelt_b64(j, n);
- uint64_t active = svcntp_b64(svptrue_b64(), pg);
- do {
+ j = 0;
+ svbool_t pg = svwhilelt_b64(j, n);
+ uint64_t active = svcntp_b64(svptrue_b64(), pg);
+ do {
- aoffset1 = aoffset;
+ aoffset1 = aoffset;
- uint64_t i_cnt = m;
- while (i_cnt--) {
- svfloat64_t a_vec = svld1_gather_index(pg, (double *) aoffset1, lda_vec);
- svst1_f64(pg, (double *) boffset, a_vec);
- aoffset1++;
- boffset += active;
- }
- aoffset += sve_size * lda;
+ uint64_t i_cnt = m;
+ while (i_cnt--) {
+ svfloat64_t a_vec = svld1_gather_index(pg, (double *) aoffset1, lda_vec);
+ svst1_f64(pg, (double *) boffset, a_vec);
+ aoffset1++;
+ boffset += active;
+ }
+ aoffset += sve_size * lda;
- j += svcntd();
- pg = svwhilelt_b64(j, n);
- active = svcntp_b64(svptrue_b64(), pg);
+ j += svcntd();
+ pg = svwhilelt_b64(j, n);
+ active = svcntp_b64(svptrue_b64(), pg);
- } while (svptest_any(svptrue_b64(), pg));
+ } while (svptest_any(svptrue_b64(), pg));
- return 0;
+ return 0;
}
#include "common.h"
#include <arm_sve.h>
-// TODO: write in assembly with proper unrolling
+// TODO: write in assembly with proper unrolling of inner loop
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
- BLASLONG j;
- IFLOAT *aoffset, *aoffset1, *boffset;
+ BLASLONG j;
+ IFLOAT *aoffset, *aoffset1, *boffset;
- uint64_t sve_size = svcntd();
+ uint64_t sve_size = svcntd();
- aoffset = a;
- boffset = b;
+ aoffset = a;
+ boffset = b;
- j = 0;
- svbool_t pg = svwhilelt_b64(j, n);
- uint64_t active = svcntp_b64(svptrue_b64(), pg);
- do {
+ j = 0;
+ svbool_t pg = svwhilelt_b64(j, n);
+ uint64_t active = svcntp_b64(svptrue_b64(), pg);
+ do {
- aoffset1 = aoffset;
+ aoffset1 = aoffset;
- uint64_t i_cnt = m;
- while (i_cnt--) {
- svfloat64_t a_vec = svld1(pg, (double *)aoffset1);
- svst1_f64(pg, (double *) boffset, a_vec);
- aoffset1 += lda;
- boffset += active;
- }
- aoffset += sve_size;
+ uint64_t i_cnt = m;
+ while (i_cnt--) {
+ svfloat64_t a_vec = svld1(pg, (double *)aoffset1);
+ svst1_f64(pg, (double *) boffset, a_vec);
+ aoffset1 += lda;
+ boffset += active;
+ }
+ aoffset += sve_size;
- j += svcntd();
- pg = svwhilelt_b64(j, n);
- active = svcntp_b64(svptrue_b64(), pg);
+ j += svcntd();
+ pg = svwhilelt_b64(j, n);
+ active = svcntp_b64(svptrue_b64(), pg);
- } while (svptest_any(svptrue_b64(), pg));
+ } while (svptest_any(svptrue_b64(), pg));
- return 0;
+ return 0;
}
#define alpha0 d10
#define alphaZ z2.d
-#define A_PRE_SIZE 2560
+#define A_PRE_SIZE 1536
#define B_PRE_SIZE 512
#define C_PRE_SIZE 128
.macro KERNELv1x8_I
ld1d z0.d, p1/z, [pA]
ld1d z1.d, p1/z, [pA, lanes, lsl #3] // next one
- //incb pA, all, mul #2
add pA, pA, lanes, lsl #4 // pA = pA + lanes * 2 * 8
ld1rd z8.d, p0/z, [pB]
#endif
mov pB, origPB
-
+// Loop over N
mov counterJ, origN
asr counterJ, counterJ, #3 // J = J / 8
cmp counterJ, #0
ble .Ldtrmm_kernel_L4_BEGIN
/******************************************************************************/
+/* Repeat this as long as there are 8 left in N */
.align 5
.Ldtrmm_kernel_L8_BEGIN:
.Ldtrmm_kernel_L8_Mv1_BEGIN:
+/* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */
mov counterI, #0
- whilelt p1.d, counterI, origM //SVE instruction
- cntp lanes, p0, p1.d
+ whilelt p1.d, counterI, origM
+ cntp lanes, p0, p1.d // lanes contain number of active SVE lanes in M dimension
.align 5
.Ldtrmm_kernel_L8_Mv1_20:
bgt .Ldtrmm_kernel_L8_BEGIN
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 4 left in N */
.align 5
.Ldtrmm_kernel_L4_BEGIN:
#endif
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 2 left in N */
.align 5
.Ldtrmm_kernel_L2_BEGIN:
#endif
/******************************************************************************/
-/******************************************************************************/
+/* Repeat the same thing if 1 left in N */
.align 5
.Ldtrmm_kernel_L1_BEGIN:
BLASLONG i, js;
BLASLONG X;
- //printf("Using trmm_ln.\n");
int sve_len = svcntd();
svint64_t index = svindex_s64(0LL, lda);
}
i = 0;
- /* svbool_t pm = svwhilelt_b64(i, m); */
- /* int m_active = svcntp_b64(svptrue_b64(), pm); */
do
{
- if (X > posY) { // optimize this: unroll over DGEMM_UNROLL_M: vl
+ if (X > posY) {
svfloat64_t aj_vec = svld1_gather_index(pn, ao, index);
svst1(pn, b, aj_vec);
ao ++;
X ++;
i ++;
} else {
+ /* I did not find a way to unroll this while preserving vector-length-agnostic code. */
#ifdef UNIT
int temp = 0;
for (int j = 0; j < n_active; j++) {
}
} while (i < m);
- //printf("\n");
-
-
posY += n_active;
js += n_active;
pn = svwhilelt_b64(js, n);
BLASLONG i, js;
BLASLONG X;
- //printf("Using trmm_lt.\n");
-
int sve_len = svcntd();
FLOAT *ao;
}
i = 0;
- /* svbool_t pm = svwhilelt_b64(i, m); */
- /* int m_active = svcntp_b64(svptrue_b64(), pm); */
do
{
- if (X > posY) { // optimize this: unroll over DGEMM_UNROLL_M: vl
+ if (X > posY) {
ao ++;
b += n_active;
X ++;
X ++;
i ++;
} else {
+ /* I did not find a way to unroll this while preserving vector-length-agnostic code. */
#ifdef UNIT
int temp = 0;
for (int j = 0; j < n_active; j++) {
}
} while (i < m);
- //printf("\n");
-
posY += n_active;
js += n_active;
BLASLONG i, js;
BLASLONG X;
- //printf("Using trmm_un.\n");
- //printf("Using m %ld, n %ld.\n", m, n);
- //printf("Using lda %ld.\n", lda);
- //printf("Using posX %ld, posY %ld.\n", posX, posY);
int sve_len = svcntd();
svint64_t index = svindex_s64(0LL, lda);
}
i = 0;
- /* svbool_t pm = svwhilelt_b64(i, m); */
- /* int m_active = svcntp_b64(svptrue_b64(), pm); */
do
{
- if (X < posY) { // optimize this: unroll over DGEMM_UNROLL_M: vl
+ if (X < posY) {
svfloat64_t aj_vec = svld1_gather_index(pn, ao, index);
svst1(pn, b, aj_vec);
ao ++;
X ++;
i ++;
} else {
+ /* I did not find a way to unroll this while preserving vector-length-agnostic code. */
#ifdef UNIT
int temp = 0;
for (int j = 0; j < n_active; j++) {
}
} while (i < m);
- //printf("\n");
-
-
posY += n_active;
js += n_active;
pn = svwhilelt_b64(js, n);
BLASLONG i, js;
BLASLONG X;
- //printf("Using trmm_ut.\n");
int sve_len = svcntd();
}
i = 0;
- /* svbool_t pm = svwhilelt_b64(i, m); */
- /* int m_active = svcntp_b64(svptrue_b64(), pm); */
do
{
- if (X < posY) { // optimize this: unroll over DGEMM_UNROLL_M: vl
+ if (X < posY) {
ao ++;
b += n_active;
X ++;
b += n_active;
X ++;
i ++;
- } else {
+ } else {
+ /* I did not find a way to unroll this while preserving vector-length-agnostic code. */
#ifdef UNIT
int temp = 0;
for (int j = 0; j < n_active; j++) {
}
} while (i < m);
- //printf("\n");
-
-
posY += n_active;
js += n_active;
pn = svwhilelt_b64(js, n);
projects / platform / upstream / openblas.git / commitdiff