CGEMVTKERNEL = zgemv_t.S
ZGEMVTKERNEL = zgemv_t.S
-STRMMKERNEL = strmm_kernel_4x4.S
-DTRMMKERNEL = dtrmm_kernel_4x4.S
-CTRMMKERNEL = ctrmm_kernel_4x4.S
-ZTRMMKERNEL = ztrmm_kernel_4x4.S
-
-SGEMMKERNEL = sgemm_kernel_4x4.S
-SGEMMONCOPY = ../generic/gemm_ncopy_4.c
-SGEMMOTCOPY = ../generic/gemm_tcopy_4.c
+SGEMMKERNEL = sgemm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S
+STRMMKERNEL = strmm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N).S
+ifneq ($(SGEMM_UNROLL_M), $(SGEMM_UNROLL_N))
+SGEMMINCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_M).c
+SGEMMITCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_M).c
+SGEMMINCOPYOBJ = sgemm_incopy.o
+SGEMMITCOPYOBJ = sgemm_itcopy.o
+endif
+SGEMMONCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_N).c
+SGEMMOTCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_N).c
SGEMMONCOPYOBJ = sgemm_oncopy.o
SGEMMOTCOPYOBJ = sgemm_otcopy.o
-DGEMMKERNEL = dgemm_kernel_4x4.S
-DGEMMONCOPY = ../generic/gemm_ncopy_4.c
-DGEMMOTCOPY = ../generic/gemm_tcopy_4.c
+DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S
+DTRMMKERNEL = dtrmm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N).S
+ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N))
+DGEMMINCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c
+DGEMMITCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c
+DGEMMINCOPYOBJ = dgemm_incopy.o
+DGEMMITCOPYOBJ = dgemm_itcopy.o
+endif
+DGEMMONCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c
+DGEMMOTCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c
DGEMMONCOPYOBJ = dgemm_oncopy.o
DGEMMOTCOPYOBJ = dgemm_otcopy.o
-CGEMMKERNEL = cgemm_kernel_4x4.S
-CGEMMONCOPY = ../generic/zgemm_ncopy_4.c
-CGEMMOTCOPY = ../generic/zgemm_tcopy_4.c
+CGEMMKERNEL = cgemm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S
+CTRMMKERNEL = ctrmm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N).S
+ifneq ($(CGEMM_UNROLL_M), $(CGEMM_UNROLL_N))
+CGEMMINCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_M).c
+CGEMMITCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_M).c
+CGEMMINCOPYOBJ = cgemm_incopy.o
+CGEMMITCOPYOBJ = cgemm_itcopy.o
+endif
+CGEMMONCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_N).c
+CGEMMOTCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_N).c
CGEMMONCOPYOBJ = cgemm_oncopy.o
CGEMMOTCOPYOBJ = cgemm_otcopy.o
-ZGEMMKERNEL = zgemm_kernel_4x4.S
-ZGEMMONCOPY = ../generic/zgemm_ncopy_4.c
-ZGEMMOTCOPY = ../generic/zgemm_tcopy_4.c
+ZGEMMKERNEL = zgemm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S
+ZTRMMKERNEL = ztrmm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N).S
+ifneq ($(ZGEMM_UNROLL_M), $(ZGEMM_UNROLL_N))
+ZGEMMINCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_M).c
+ZGEMMITCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_M).c
+ZGEMMINCOPYOBJ = zgemm_incopy.o
+ZGEMMITCOPYOBJ = zgemm_itcopy.o
+endif
+ZGEMMONCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_N).c
+ZGEMMOTCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c
ZGEMMONCOPYOBJ = zgemm_oncopy.o
ZGEMMOTCOPYOBJ = zgemm_otcopy.o
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define temp x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+
+#define alpha0_R s10
+#define alphaV0_R v10.s[0]
+#define alpha0_I s11
+#define alphaV0_I v11.s[0]
+
+#define alpha1_R s14
+#define alphaV1_R v14.s[0]
+#define alpha1_I s15
+#define alphaV1_I v15.s[0]
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
+#define OP_rr fmla
+#define OP_ii fmls
+#define OP_ri fmla
+#define OP_ir fmla
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+#define OP_rr fmla
+#define OP_ii fmla
+#define OP_ri fmls
+#define OP_ir fmla
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+#define OP_rr fmla
+#define OP_ii fmla
+#define OP_ri fmla
+#define OP_ir fmls
+#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
+#define OP_rr fmla
+#define OP_ii fmls
+#define OP_ri fmls
+#define OP_ir fmls
+#endif
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset -> temp
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16
+// 17
+// 18 must save
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA_R -> pA0_00_R, pA0_01_R, pA0_02_R, pA0_03_R
+//v01 ALPHA_I -> pA0_00_I, pA0_01_I, pA0_02_I, pA0_03_I
+//v02 pA0_04_R, pA0_05_R, pA0_06_R, pA0_07_R
+//v03 pA0_04_I, pA0_05_I, pA0_06_I, pA0_07_I
+//v04 pA1_00_R, pA1_01_R, pA1_02_R, pA1_03_R
+//v05 pA1_00_I, pA1_01_I, pA1_02_I, pA1_03_I
+//v06 pA1_04_R, pA1_05_R, pA1_06_R, pA1_07_R
+//v07 pA1_04_I, pA1_05_I, pA1_06_I, pA1_07_I
+//v08 must save pB0_00_R, pB0_01_R, pB0_02_R, pB0_03_R
+//v09 must save pB0_00_I, pB0_01_I, pB0_02_I, pB0_03_I
+//v10 must save ALPHA0_R
+//v11 must save ALPHA0_I
+//v12 must save pB1_00_R, pB1_01_R, pB1_02_R, pB1_03_R
+//v13 must save pB1_00_I, pB1_01_I, pB1_02_I, pB1_03_I
+//v14 must save ALPHA1_R
+//v15 must save ALPHA1_I
+//v16 must save pC_00_R, pC_01_R, pC_02_R, pC_03_R
+//v17 must save pC_00_I, pC_01_I, pC_02_I, pC_03_I
+//v18 pC_04_R, pC_05_R, pC_06_R, pC_07_R
+//v19 pC_04_I, pC_05_I, pC_06_I, pC_07_I
+//v20 pC_08_R, pC_09_R, pC_10_R, pC_11_R
+//v21 pC_08_I, pC_09_I, pC_10_I, pC_11_I
+//v22 pC_12_R, pC_13_R, pC_14_R, pC_15_R
+//v23 pC_12_I, pC_13_I, pC_14_I, pC_15_I
+//v24 pC_16_R, pC_17_R, pC_18_R, pC_19_R
+//v25 pC_16_I, pC_17_I, pC_18_I, pC_19_I
+//v26 pC_20_R, pC_21_R, pC_22_R, pC_23_R
+//v27 pC_20_I, pC_21_I, pC_22_I, pC_23_I
+//v28 pC_24_R, pC_25_R, pC_26_R, pC_27_R
+//v29 pC_24_I, pC_25_I, pC_26_I, pC_27_I
+//v30 pC_28_R, pC_29_R, pC_30_R, pC_31_R
+//v31 pC_28_I, pC_29_I, pC_30_I, pC_31_I
+
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT8x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s26, s17
+ fmov s27, s18
+ fmov s28, wzr
+ fmov s29, s16
+ fmov s30, s17
+ fmov s31, s18
+.endm
+
+.macro KERNEL8x4_I
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ fmul v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v17.16b, v17.16b, v17.16b
+ fmls v17.4s, v0.4s, v9.4s[0]
+#else
+ fmul v17.4s, v0.4s, v9.4s[0]
+#endif
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ fmul v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v19.16b, v19.16b, v19.16b
+ fmls v19.4s, v2.4s, v9.4s[0]
+#else
+ fmul v19.4s, v2.4s, v9.4s[0]
+#endif
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ fmul v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v21.16b, v21.16b, v21.16b
+ fmls v21.4s, v0.4s, v9.4s[1]
+#else
+ fmul v21.4s, v0.4s, v9.4s[1]
+#endif
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ fmul v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v23.16b, v23.16b, v23.16b
+ fmls v23.4s, v2.4s, v9.4s[1]
+#else
+ fmul v23.4s, v2.4s, v9.4s[1]
+#endif
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ fmul v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v25.16b, v25.16b, v25.16b
+ fmls v25.4s, v0.4s, v9.4s[2]
+#else
+ fmul v25.4s, v0.4s, v9.4s[2]
+#endif
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ fmul v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v27.16b, v27.16b, v27.16b
+ fmls v27.4s, v2.4s, v9.4s[2]
+#else
+ fmul v27.4s, v2.4s, v9.4s[2]
+#endif
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ fmul v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v29.16b, v29.16b, v29.16b
+ fmls v29.4s, v0.4s, v9.4s[3]
+#else
+ fmul v29.4s, v0.4s, v9.4s[3]
+#endif
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ fmul v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v31.16b, v31.16b, v31.16b
+ fmls v31.4s, v2.4s, v9.4s[3]
+#else
+ fmul v31.4s, v2.4s, v9.4s[3]
+#endif
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+
+ ld2 {v12.4s, v13.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v4.4s, v5.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v6.4s, v7.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL8x4_M1
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+ OP_ri v19.4s, v2.4s, v9.4s[0]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ OP_rr v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+ OP_ri v23.4s, v2.4s, v9.4s[1]
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+ OP_ri v27.4s, v2.4s, v9.4s[2]
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ OP_rr v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+ OP_ri v31.4s, v2.4s, v9.4s[3]
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+
+ ld2 {v12.4s, v13.4s}, [pB] // For next round
+ add pB, pB, #32
+ ld2 {v4.4s, v5.4s}, [pA] // For next round
+ add pA, pA, #32
+ ld2 {v6.4s, v7.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL8x4_M2
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ OP_rr v18.4s, v6.4s, v12.4s[0]
+ OP_ii v18.4s, v7.4s, v13.4s[0]
+ OP_ri v19.4s, v6.4s, v13.4s[0]
+ OP_ir v19.4s, v7.4s, v12.4s[0]
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ OP_rr v22.4s, v6.4s, v12.4s[1]
+ OP_ii v22.4s, v7.4s, v13.4s[1]
+ OP_ri v23.4s, v6.4s, v13.4s[1]
+ OP_ir v23.4s, v7.4s, v12.4s[1]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ OP_rr v26.4s, v6.4s, v12.4s[2]
+ OP_ii v26.4s, v7.4s, v13.4s[2]
+ OP_ri v27.4s, v6.4s, v13.4s[2]
+ OP_ir v27.4s, v7.4s, v12.4s[2]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+
+ OP_rr v30.4s, v6.4s, v12.4s[3]
+ OP_ii v30.4s, v7.4s, v13.4s[3]
+ OP_ri v31.4s, v6.4s, v13.4s[3]
+ OP_ir v31.4s, v7.4s, v12.4s[3]
+
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL8x4_E
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ OP_rr v18.4s, v6.4s, v12.4s[0]
+ OP_ii v18.4s, v7.4s, v13.4s[0]
+ OP_ri v19.4s, v6.4s, v13.4s[0]
+ OP_ir v19.4s, v7.4s, v12.4s[0]
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ OP_rr v22.4s, v6.4s, v12.4s[1]
+ OP_ii v22.4s, v7.4s, v13.4s[1]
+ OP_ri v23.4s, v6.4s, v13.4s[1]
+ OP_ir v23.4s, v7.4s, v12.4s[1]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ OP_rr v26.4s, v6.4s, v12.4s[2]
+ OP_ii v26.4s, v7.4s, v13.4s[2]
+ OP_ri v27.4s, v6.4s, v13.4s[2]
+ OP_ir v27.4s, v7.4s, v12.4s[2]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+
+ OP_rr v30.4s, v6.4s, v12.4s[3]
+ OP_ii v30.4s, v7.4s, v13.4s[3]
+ OP_ri v31.4s, v6.4s, v13.4s[3]
+ OP_ir v31.4s, v7.4s, v12.4s[3]
+
+.endm
+
+.macro KERNEL8x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+ OP_ri v19.4s, v2.4s, v9.4s[0]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ OP_rr v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+ OP_ri v23.4s, v2.4s, v9.4s[1]
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+ OP_ri v27.4s, v2.4s, v9.4s[2]
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ OP_rr v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+ OP_ri v31.4s, v2.4s, v9.4s[3]
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+
+.endm
+
+.macro SAVE8x4
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v2.4s, v3.4s}, [pCRow2]
+ fmla v2.4s, v18.4s, alphaV0_R
+ fmls v2.4s, v19.4s, alphaV0_I
+ fmla v3.4s, v18.4s, alphaV1_I
+ fmla v3.4s, v19.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmla v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v6.4s, v7.4s}, [pCRow2]
+ fmla v6.4s, v22.4s, alphaV0_R
+ fmls v6.4s, v23.4s, alphaV0_I
+ fmla v7.4s, v22.4s, alphaV1_I
+ fmla v7.4s, v23.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v24.4s, alphaV0_R
+ fmls v0.4s, v25.4s, alphaV0_I
+ fmla v1.4s, v24.4s, alphaV1_I
+ fmla v1.4s, v25.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v2.4s, v3.4s}, [pCRow2]
+ fmla v2.4s, v26.4s, alphaV0_R
+ fmls v2.4s, v27.4s, alphaV0_I
+ fmla v3.4s, v26.4s, alphaV1_I
+ fmla v3.4s, v27.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v28.4s, alphaV0_R
+ fmls v4.4s, v29.4s, alphaV0_I
+ fmla v5.4s, v28.4s, alphaV1_I
+ fmla v5.4s, v29.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v6.4s, v7.4s}, [pCRow2]
+ fmla v6.4s, v30.4s, alphaV0_R
+ fmls v6.4s, v31.4s, alphaV0_I
+ fmla v7.4s, v30.4s, alphaV1_I
+ fmla v7.4s, v31.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s28, s17
+ fmov s29, s16
+.endm
+
+.macro KERNEL4x4_I
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ fmul v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v17.16b, v17.16b, v17.16b
+ fmls v17.4s, v0.4s, v9.4s[0]
+#else
+ fmul v17.4s, v0.4s, v9.4s[0]
+#endif
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ fmul v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v21.16b, v21.16b, v21.16b
+ fmls v21.4s, v0.4s, v9.4s[1]
+#else
+ fmul v21.4s, v0.4s, v9.4s[1]
+#endif
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ fmul v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v25.16b, v25.16b, v25.16b
+ fmls v25.4s, v0.4s, v9.4s[2]
+#else
+ fmul v25.4s, v0.4s, v9.4s[2]
+#endif
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ fmul v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v29.16b, v29.16b, v29.16b
+ fmls v29.4s, v0.4s, v9.4s[3]
+#else
+ fmul v29.4s, v0.4s, v9.4s[3]
+#endif
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ ld2 {v12.4s, v13.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v4.4s, v5.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL4x4_M1
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ ld2 {v12.4s, v13.4s}, [pB] // For next round
+ add pB, pB, #32
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ ld2 {v4.4s, v5.4s}, [pA] // For next round
+ add pA, pA, #32
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+.endm
+
+.macro KERNEL4x4_M2
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ ld2 {v8.4s, v9.4s}, [pB] // For next round
+ add pB, pB, #32
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ ld2 {v0.4s, v1.4s}, [pA] // For next round
+ add pA, pA, #32
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+.endm
+
+.macro KERNEL4x4_E
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+.endm
+
+.macro SAVE4x4
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmla v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v24.4s, alphaV0_R
+ fmls v0.4s, v25.4s, alphaV0_I
+ fmla v1.4s, v24.4s, alphaV1_I
+ fmla v1.4s, v25.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v28.4s, alphaV0_R
+ fmls v4.4s, v29.4s, alphaV0_I
+ fmla v5.4s, v28.4s, alphaV1_I
+ fmla v5.4s, v29.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+ fmov s24, s16
+ fmov s25, s17
+ fmov s28, s16
+ fmov s29, s17
+.endm
+
+.macro KERNEL2x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.4s[0]
+ OP_ii v16.2s, v1.2s, v9.4s[0]
+ OP_ri v17.2s, v0.2s, v9.4s[0]
+ OP_ir v17.2s, v1.2s, v8.4s[0]
+
+ OP_rr v20.2s, v0.2s, v8.4s[1]
+ OP_ii v20.2s, v1.2s, v9.4s[1]
+ OP_ri v21.2s, v0.2s, v9.4s[1]
+ OP_ir v21.2s, v1.2s, v8.4s[1]
+
+ OP_rr v24.2s, v0.2s, v8.4s[2]
+ OP_ii v24.2s, v1.2s, v9.4s[2]
+ OP_ri v25.2s, v0.2s, v9.4s[2]
+ OP_ir v25.2s, v1.2s, v8.4s[2]
+
+ OP_rr v28.2s, v0.2s, v8.4s[3]
+ OP_ii v28.2s, v1.2s, v9.4s[3]
+ OP_ri v29.2s, v0.2s, v9.4s[3]
+ OP_ir v29.2s, v1.2s, v8.4s[3]
+.endm
+
+.macro SAVE2x4
+ mov pCRow1, pCRow0
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmla v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.2s, v5.2s}, [pCRow1]
+ fmla v4.2s, v20.2s, alphaV0_R
+ fmls v4.2s, v21.2s, alphaV0_I
+ fmla v5.2s, v20.2s, alphaV1_I
+ fmla v5.2s, v21.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v24.2s, alphaV0_R
+ fmls v0.2s, v25.2s, alphaV0_I
+ fmla v1.2s, v24.2s, alphaV1_I
+ fmla v1.2s, v25.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.2s, v5.2s}, [pCRow1]
+ fmla v4.2s, v28.2s, alphaV0_R
+ fmls v4.2s, v29.2s, alphaV0_I
+ fmla v5.2s, v28.2s, alphaV1_I
+ fmla v5.2s, v29.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+ fmov s24, s16
+ fmov s25, s17
+ fmov s28, s16
+ fmov s29, s17
+.endm
+
+.macro KERNEL1x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.4s[0]
+ OP_ii s16, s1, v9.4s[0]
+ OP_ri s17, s0, v9.4s[0]
+ OP_ir s17, s1, v8.4s[0]
+
+ OP_rr s20, s0, v8.4s[1]
+ OP_ii s20, s1, v9.4s[1]
+ OP_ri s21, s0, v9.4s[1]
+ OP_ir s21, s1, v8.4s[1]
+
+ OP_rr s24, s0, v8.4s[2]
+ OP_ii s24, s1, v9.4s[2]
+ OP_ri s25, s0, v9.4s[2]
+ OP_ir s25, s1, v8.4s[2]
+
+ OP_rr s28, s0, v8.4s[3]
+ OP_ii s28, s1, v9.4s[3]
+ OP_ri s29, s0, v9.4s[3]
+ OP_ir s29, s1, v8.4s[3]
+.endm
+
+.macro SAVE1x4
+ mov pCRow1, pCRow0
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmla s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.s, v5.s}[0], [pCRow1]
+ fmla s4, s20, alphaV0_R
+ fmls s4, s21, alphaV0_I
+ fmla s5, s20, alphaV1_I
+ fmla s5, s21, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s24, alphaV0_R
+ fmls s0, s25, alphaV0_I
+ fmla s1, s24, alphaV1_I
+ fmla s1, s25, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.s, v5.s}[0], [pCRow1]
+ fmla s4, s28, alphaV0_R
+ fmls s4, s29, alphaV0_I
+ fmla s5, s28, alphaV1_I
+ fmla s5, s29, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+.endm
+
+.macro KERNEL8x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.2s[0]
+ OP_ii v16.4s, v1.4s, v9.2s[0]
+ OP_ri v17.4s, v0.4s, v9.2s[0]
+ OP_ir v17.4s, v1.4s, v8.2s[0]
+
+ OP_rr v18.4s, v2.4s, v8.2s[0]
+ OP_ii v18.4s, v3.4s, v9.2s[0]
+ OP_ri v19.4s, v2.4s, v9.2s[0]
+ OP_ir v19.4s, v3.4s, v8.2s[0]
+
+ OP_rr v20.4s, v0.4s, v8.2s[1]
+ OP_ii v20.4s, v1.4s, v9.2s[1]
+ OP_ri v21.4s, v0.4s, v9.2s[1]
+ OP_ir v21.4s, v1.4s, v8.2s[1]
+
+ OP_rr v22.4s, v2.4s, v8.2s[1]
+ OP_ii v22.4s, v3.4s, v9.2s[1]
+ OP_ri v23.4s, v2.4s, v9.2s[1]
+ OP_ir v23.4s, v3.4s, v8.2s[1]
+.endm
+
+.macro SAVE8x2
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v2.4s, v3.4s}, [pCRow2]
+ fmla v2.4s, v18.4s, alphaV0_R
+ fmls v2.4s, v19.4s, alphaV0_I
+ fmla v3.4s, v18.4s, alphaV1_I
+ fmla v3.4s, v19.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmla v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v6.4s, v7.4s}, [pCRow2]
+ fmla v6.4s, v22.4s, alphaV0_R
+ fmls v6.4s, v23.4s, alphaV0_I
+ fmla v7.4s, v22.4s, alphaV1_I
+ fmla v7.4s, v23.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+.endm
+
+.macro KERNEL4x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.2s[0]
+ OP_ii v16.4s, v1.4s, v9.2s[0]
+ OP_ri v17.4s, v0.4s, v9.2s[0]
+ OP_ir v17.4s, v1.4s, v8.2s[0]
+
+ OP_rr v20.4s, v0.4s, v8.2s[1]
+ OP_ii v20.4s, v1.4s, v9.2s[1]
+ OP_ri v21.4s, v0.4s, v9.2s[1]
+ OP_ir v21.4s, v1.4s, v8.2s[1]
+.endm
+
+.macro SAVE4x2
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmla v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+.endm
+
+.macro KERNEL2x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.2s[0]
+ OP_ii v16.2s, v1.2s, v9.2s[0]
+ OP_ri v17.2s, v0.2s, v9.2s[0]
+ OP_ir v17.2s, v1.2s, v8.2s[0]
+
+ OP_rr v20.2s, v0.2s, v8.2s[1]
+ OP_ii v20.2s, v1.2s, v9.2s[1]
+ OP_ri v21.2s, v0.2s, v9.2s[1]
+ OP_ir v21.2s, v1.2s, v8.2s[1]
+.endm
+
+.macro SAVE2x2
+ mov pCRow1, pCRow0
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmla v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.2s, v5.2s}, [pCRow1]
+ fmla v4.2s, v20.2s, alphaV0_R
+ fmls v4.2s, v21.2s, alphaV0_I
+ fmla v5.2s, v20.2s, alphaV1_I
+ fmla v5.2s, v21.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, wzr
+ fmov s21, wzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.2s[0]
+ OP_ii s16, s1, v9.2s[0]
+ OP_ri s17, s0, v9.2s[0]
+ OP_ir s17, s1, v8.2s[0]
+
+ OP_rr s20, s0, v8.2s[1]
+ OP_ii s20, s1, v9.2s[1]
+ OP_ri s21, s0, v9.2s[1]
+ OP_ir s21, s1, v8.2s[1]
+.endm
+
+.macro SAVE1x2
+ mov pCRow1, pCRow0
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmla s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.s, v5.s}[0], [pCRow1]
+ fmla s4, s20, alphaV0_R
+ fmls s4, s21, alphaV0_I
+ fmla s5, s20, alphaV1_I
+ fmla s5, s21, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+.endm
+
+.macro KERNEL8x1_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v8.4s[1]
+ OP_ri v17.4s, v0.4s, v8.4s[1]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v8.4s[1]
+ OP_ri v19.4s, v2.4s, v8.4s[1]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+.endm
+
+.macro SAVE8x1
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, #32
+
+ ld2 {v2.4s, v3.4s}, [pCRow1]
+ fmla v2.4s, v18.4s, alphaV0_R
+ fmls v2.4s, v19.4s, alphaV0_I
+ fmla v3.4s, v18.4s, alphaV1_I
+ fmla v3.4s, v19.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov s16, wzr
+ fmov s17, s16
+.endm
+
+.macro KERNEL4x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.s[0]
+ OP_ii v16.4s, v1.4s, v9.s[0]
+ OP_ri v17.4s, v0.4s, v9.s[0]
+ OP_ir v17.4s, v1.4s, v8.s[0]
+.endm
+
+.macro SAVE4x1
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.s[0]
+ OP_ii v16.2s, v1.2s, v9.s[0]
+ OP_ri v17.2s, v0.2s, v9.s[0]
+ OP_ir v17.2s, v1.2s, v8.s[0]
+.endm
+
+.macro SAVE2x1
+ mov pCRow1, pCRow0
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmla v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.s[0]
+ OP_ii s16, s1, v9.s[0]
+ OP_ri s17, s0, v9.s[0]
+ OP_ir s17, s1, v8.s[0]
+.endm
+
+.macro SAVE1x1
+ mov pCRow1, pCRow0
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmla s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0_R, s0
+ fmov alpha0_I, s1
+ fmov alpha1_R, s0
+ fmov alpha1_I, s1
+
+ lsl LDC, LDC, #3 // ldc = ldc * 8
+
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #2 // J = J / 4
+ cmp counterJ, #0
+ ble cgemm_kernel_L2_BEGIN
+
+/******************************************************************************/
+
+cgemm_kernel_L4_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+ mov pA, origPA // pA = start of A array
+
+cgemm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble cgemm_kernel_L4_M4_BEGIN
+
+cgemm_kernel_L4_M8_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt cgemm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2 // subtract 2
+ ble cgemm_kernel_L4_M8_22a
+ .align 5
+
+cgemm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M8_22
+
+
+cgemm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b cgemm_kernel_L4_M8_44
+
+cgemm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble cgemm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+
+ KERNEL8x4_E
+
+ b cgemm_kernel_L4_M8_44
+
+cgemm_kernel_L4_M8_40:
+
+ INIT8x4
+
+cgemm_kernel_L4_M8_44:
+
+ ands counterL , origK, #1
+ ble cgemm_kernel_L4_M8_100
+
+cgemm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+cgemm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+cgemm_kernel_L4_M8_END:
+ subs counterI, counterI, #1
+ bne cgemm_kernel_L4_M8_20
+
+cgemm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble cgemm_kernel_L4_END
+
+ tst counterI, #4
+ ble cgemm_kernel_L4_M2_BEGIN
+
+
+cgemm_kernel_L4_M4_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt cgemm_kernel_L4_M4_32
+
+ KERNEL4x4_I // do one in the K
+ KERNEL4x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble cgemm_kernel_L4_M4_22a
+ .align 5
+
+
+cgemm_kernel_L4_M4_22:
+
+ KERNEL4x4_M1
+ KERNEL4x4_M2
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M4_22
+
+cgemm_kernel_L4_M4_22a:
+ KERNEL4x4_M1
+ KERNEL4x4_E
+ b cgemm_kernel_L4_M4_44
+cgemm_kernel_L4_M4_32:
+ tst counterL, #1
+ ble cgemm_kernel_L4_M4_40
+ KERNEL4x4_I
+ KERNEL4x4_E
+ b cgemm_kernel_L4_M4_44
+cgemm_kernel_L4_M4_40:
+
+ INIT4x4
+
+cgemm_kernel_L4_M4_44:
+ ands counterL , origK, #1
+ ble cgemm_kernel_L4_M4_100
+
+cgemm_kernel_L4_M4_46:
+ KERNEL4x4_SUB
+
+cgemm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+cgemm_kernel_L4_M4_END:
+
+cgemm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble cgemm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble cgemm_kernel_L4_M1_BEGIN
+
+cgemm_kernel_L4_M2_20:
+
+ INIT2x4
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L4_M2_40
+
+cgemm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M2_22
+
+
+cgemm_kernel_L4_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L4_M2_100
+
+cgemm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M2_42
+
+cgemm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+cgemm_kernel_L4_M2_END:
+
+
+cgemm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble cgemm_kernel_L4_END
+
+cgemm_kernel_L4_M1_20:
+
+ INIT1x4
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L4_M1_40
+
+cgemm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M1_22
+
+
+cgemm_kernel_L4_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L4_M1_100
+
+cgemm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M1_42
+
+cgemm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+
+cgemm_kernel_L4_END:
+
+ lsl temp, origK, #5
+ add origPB, origPB, temp // B = B + K * 4 * 8
+
+ subs counterJ, counterJ , #1 // j--
+ bgt cgemm_kernel_L4_BEGIN
+
+
+/******************************************************************************/
+
+cgemm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble cgemm_kernel_L999 // error, N was less than 4?
+
+ tst counterJ , #2
+ ble cgemm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+ mov pA, origPA // pA = A
+
+
+cgemm_kernel_L2_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble cgemm_kernel_L2_M4_BEGIN
+
+cgemm_kernel_L2_M8_20:
+
+ INIT8x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble cgemm_kernel_L2_M8_40
+ .align 5
+
+cgemm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M8_22
+
+
+cgemm_kernel_L2_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L2_M8_100
+
+cgemm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M8_42
+
+cgemm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+cgemm_kernel_L2_M8_END:
+
+ subs counterI, counterI, #1
+ bgt cgemm_kernel_L2_M8_20
+
+cgemm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble cgemm_kernel_L2_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble cgemm_kernel_L2_M2_BEGIN
+
+cgemm_kernel_L2_M4_20:
+
+ INIT4x2
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble cgemm_kernel_L2_M4_40
+ .align 5
+
+cgemm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M4_22
+
+
+cgemm_kernel_L2_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L2_M4_100
+
+cgemm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M4_42
+
+cgemm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+cgemm_kernel_L2_M4_END:
+
+cgemm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble cgemm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble cgemm_kernel_L2_M1_BEGIN
+
+cgemm_kernel_L2_M2_20:
+
+ INIT2x2
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble cgemm_kernel_L2_M2_40
+
+cgemm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M2_22
+
+
+cgemm_kernel_L2_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L2_M2_100
+
+cgemm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M2_42
+
+cgemm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+cgemm_kernel_L2_M2_END:
+
+
+cgemm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble cgemm_kernel_L2_END
+
+cgemm_kernel_L2_M1_20:
+
+ INIT1x2
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble cgemm_kernel_L2_M1_40
+
+cgemm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M1_22
+
+
+cgemm_kernel_L2_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L2_M1_100
+
+cgemm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M1_42
+
+cgemm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+
+cgemm_kernel_L2_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
+
+/******************************************************************************/
+
+cgemm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble cgemm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+ mov pA, origPA // pA = A
+
+
+cgemm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble cgemm_kernel_L1_M4_BEGIN
+
+cgemm_kernel_L1_M8_20:
+
+ INIT8x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L1_M8_40
+ .align 5
+
+cgemm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M8_22
+
+
+cgemm_kernel_L1_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L1_M8_100
+
+cgemm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M8_42
+
+cgemm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+cgemm_kernel_L1_M8_END:
+
+ subs counterI, counterI, #1
+ bgt cgemm_kernel_L1_M8_20
+
+cgemm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble cgemm_kernel_L1_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble cgemm_kernel_L1_M2_BEGIN
+
+
+cgemm_kernel_L1_M4_20:
+
+ INIT4x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L1_M4_40
+ .align 5
+
+cgemm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M4_22
+
+
+cgemm_kernel_L1_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L1_M4_100
+
+cgemm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M4_42
+
+cgemm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+cgemm_kernel_L1_M4_END:
+
+
+cgemm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble cgemm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble cgemm_kernel_L1_M1_BEGIN
+
+cgemm_kernel_L1_M2_20:
+
+ INIT2x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L1_M2_40
+
+cgemm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M2_22
+
+
+cgemm_kernel_L1_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L1_M2_100
+
+cgemm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M2_42
+
+cgemm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+cgemm_kernel_L1_M2_END:
+
+
+cgemm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble cgemm_kernel_L1_END
+
+cgemm_kernel_L1_M1_20:
+
+ INIT1x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L1_M1_40
+
+cgemm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M1_22
+
+
+cgemm_kernel_L1_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L1_M1_100
+
+cgemm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M1_42
+
+cgemm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+
+cgemm_kernel_L1_END:
+
+
+cgemm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 s1 X3 x4 x5 x6 x7*/
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0, FLOAT alpha1,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define offset x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+#define temp x16
+#define tempOffset x17
+#define tempK x18
+
+#define alpha0_R s10
+#define alphaV0_R v10.s[0]
+#define alpha0_I s11
+#define alphaV0_I v11.s[0]
+
+#define alpha1_R s14
+#define alphaV1_R v14.s[0]
+#define alpha1_I s15
+#define alphaV1_I v15.s[0]
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
+#define OP_rr fmla
+#define OP_ii fmls
+#define OP_ri fmla
+#define OP_ir fmla
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+#define OP_rr fmla
+#define OP_ii fmla
+#define OP_ri fmls
+#define OP_ir fmla
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+#define OP_rr fmla
+#define OP_ii fmla
+#define OP_ri fmla
+#define OP_ir fmls
+#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
+#define OP_rr fmla
+#define OP_ii fmls
+#define OP_ri fmls
+#define OP_ir fmls
+#endif
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 temp
+// 17 tempOffset
+// 18 must save tempK
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA_R -> pA0_00_R, pA0_01_R, pA0_02_R, pA0_03_R
+//v01 ALPHA_I -> pA0_00_I, pA0_01_I, pA0_02_I, pA0_03_I
+//v02 pA0_04_R, pA0_05_R, pA0_06_R, pA0_07_R
+//v03 pA0_04_I, pA0_05_I, pA0_06_I, pA0_07_I
+//v04 pA1_00_R, pA1_01_R, pA1_02_R, pA1_03_R
+//v05 pA1_00_I, pA1_01_I, pA1_02_I, pA1_03_I
+//v06 pA1_04_R, pA1_05_R, pA1_06_R, pA1_07_R
+//v07 pA1_04_I, pA1_05_I, pA1_06_I, pA1_07_I
+//v08 must save pB0_00_R, pB0_01_R, pB0_02_R, pB0_03_R
+//v09 must save pB0_00_I, pB0_01_I, pB0_02_I, pB0_03_I
+//v10 must save ALPHA0_R
+//v11 must save ALPHA0_I
+//v12 must save pB1_00_R, pB1_01_R, pB1_02_R, pB1_03_R
+//v13 must save pB1_00_I, pB1_01_I, pB1_02_I, pB1_03_I
+//v14 must save ALPHA1_R
+//v15 must save ALPHA1_I
+//v16 must save pC_00_R, pC_01_R, pC_02_R, pC_03_R
+//v17 must save pC_00_I, pC_01_I, pC_02_I, pC_03_I
+//v18 pC_04_R, pC_05_R, pC_06_R, pC_07_R
+//v19 pC_04_I, pC_05_I, pC_06_I, pC_07_I
+//v20 pC_08_R, pC_09_R, pC_10_R, pC_11_R
+//v21 pC_08_I, pC_09_I, pC_10_I, pC_11_I
+//v22 pC_12_R, pC_13_R, pC_14_R, pC_15_R
+//v23 pC_12_I, pC_13_I, pC_14_I, pC_15_I
+//v24 pC_16_R, pC_17_R, pC_18_R, pC_19_R
+//v25 pC_16_I, pC_17_I, pC_18_I, pC_19_I
+//v26 pC_20_R, pC_21_R, pC_22_R, pC_23_R
+//v27 pC_20_I, pC_21_I, pC_22_I, pC_23_I
+//v28 pC_24_R, pC_25_R, pC_26_R, pC_27_R
+//v29 pC_24_I, pC_25_I, pC_26_I, pC_27_I
+//v30 pC_28_R, pC_29_R, pC_30_R, pC_31_R
+//v31 pC_28_I, pC_29_I, pC_30_I, pC_31_I
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT8x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s26, s17
+ fmov s27, s18
+ fmov s28, wzr
+ fmov s29, s16
+ fmov s30, s17
+ fmov s31, s18
+.endm
+
+.macro KERNEL8x4_I
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ fmul v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v17.16b, v17.16b, v17.16b
+ fmls v17.4s, v0.4s, v9.4s[0]
+#else
+ fmul v17.4s, v0.4s, v9.4s[0]
+#endif
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ fmul v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v19.16b, v19.16b, v19.16b
+ fmls v19.4s, v2.4s, v9.4s[0]
+#else
+ fmul v19.4s, v2.4s, v9.4s[0]
+#endif
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ fmul v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v21.16b, v21.16b, v21.16b
+ fmls v21.4s, v0.4s, v9.4s[1]
+#else
+ fmul v21.4s, v0.4s, v9.4s[1]
+#endif
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ fmul v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v23.16b, v23.16b, v23.16b
+ fmls v23.4s, v2.4s, v9.4s[1]
+#else
+ fmul v23.4s, v2.4s, v9.4s[1]
+#endif
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ fmul v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v25.16b, v25.16b, v25.16b
+ fmls v25.4s, v0.4s, v9.4s[2]
+#else
+ fmul v25.4s, v0.4s, v9.4s[2]
+#endif
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ fmul v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v27.16b, v27.16b, v27.16b
+ fmls v27.4s, v2.4s, v9.4s[2]
+#else
+ fmul v27.4s, v2.4s, v9.4s[2]
+#endif
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ fmul v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v29.16b, v29.16b, v29.16b
+ fmls v29.4s, v0.4s, v9.4s[3]
+#else
+ fmul v29.4s, v0.4s, v9.4s[3]
+#endif
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ fmul v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v31.16b, v31.16b, v31.16b
+ fmls v31.4s, v2.4s, v9.4s[3]
+#else
+ fmul v31.4s, v2.4s, v9.4s[3]
+#endif
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+
+ ld2 {v12.4s, v13.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v4.4s, v5.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v6.4s, v7.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL8x4_M1
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+ OP_ri v19.4s, v2.4s, v9.4s[0]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ OP_rr v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+ OP_ri v23.4s, v2.4s, v9.4s[1]
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+ OP_ri v27.4s, v2.4s, v9.4s[2]
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ OP_rr v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+ OP_ri v31.4s, v2.4s, v9.4s[3]
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+
+ ld2 {v12.4s, v13.4s}, [pB] // For next round
+ add pB, pB, #32
+ ld2 {v4.4s, v5.4s}, [pA] // For next round
+ add pA, pA, #32
+ ld2 {v6.4s, v7.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL8x4_M2
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ OP_rr v18.4s, v6.4s, v12.4s[0]
+ OP_ii v18.4s, v7.4s, v13.4s[0]
+ OP_ri v19.4s, v6.4s, v13.4s[0]
+ OP_ir v19.4s, v7.4s, v12.4s[0]
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ OP_rr v22.4s, v6.4s, v12.4s[1]
+ OP_ii v22.4s, v7.4s, v13.4s[1]
+ OP_ri v23.4s, v6.4s, v13.4s[1]
+ OP_ir v23.4s, v7.4s, v12.4s[1]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ OP_rr v26.4s, v6.4s, v12.4s[2]
+ OP_ii v26.4s, v7.4s, v13.4s[2]
+ OP_ri v27.4s, v6.4s, v13.4s[2]
+ OP_ir v27.4s, v7.4s, v12.4s[2]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+
+ OP_rr v30.4s, v6.4s, v12.4s[3]
+ OP_ii v30.4s, v7.4s, v13.4s[3]
+ OP_ri v31.4s, v6.4s, v13.4s[3]
+ OP_ir v31.4s, v7.4s, v12.4s[3]
+
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL8x4_E
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ OP_rr v18.4s, v6.4s, v12.4s[0]
+ OP_ii v18.4s, v7.4s, v13.4s[0]
+ OP_ri v19.4s, v6.4s, v13.4s[0]
+ OP_ir v19.4s, v7.4s, v12.4s[0]
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ OP_rr v22.4s, v6.4s, v12.4s[1]
+ OP_ii v22.4s, v7.4s, v13.4s[1]
+ OP_ri v23.4s, v6.4s, v13.4s[1]
+ OP_ir v23.4s, v7.4s, v12.4s[1]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ OP_rr v26.4s, v6.4s, v12.4s[2]
+ OP_ii v26.4s, v7.4s, v13.4s[2]
+ OP_ri v27.4s, v6.4s, v13.4s[2]
+ OP_ir v27.4s, v7.4s, v12.4s[2]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+
+ OP_rr v30.4s, v6.4s, v12.4s[3]
+ OP_ii v30.4s, v7.4s, v13.4s[3]
+ OP_ri v31.4s, v6.4s, v13.4s[3]
+ OP_ir v31.4s, v7.4s, v12.4s[3]
+
+.endm
+
+.macro KERNEL8x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+ OP_ri v19.4s, v2.4s, v9.4s[0]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ OP_rr v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+ OP_ri v23.4s, v2.4s, v9.4s[1]
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+ OP_ri v27.4s, v2.4s, v9.4s[2]
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ OP_rr v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+ OP_ri v31.4s, v2.4s, v9.4s[3]
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+
+.endm
+
+.macro SAVE8x4
+ mov pCRow1, pCRow0
+
+
+ fmul v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmul v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+
+ fmul v2.4s, v18.4s, alphaV0_R
+ fmls v2.4s, v19.4s, alphaV0_I
+ fmul v3.4s, v18.4s, alphaV1_I
+ fmla v3.4s, v19.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmul v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+
+ fmul v6.4s, v22.4s, alphaV0_R
+ fmls v6.4s, v23.4s, alphaV0_I
+ fmul v7.4s, v22.4s, alphaV1_I
+ fmla v7.4s, v23.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v0.4s, v24.4s, alphaV0_R
+ fmls v0.4s, v25.4s, alphaV0_I
+ fmul v1.4s, v24.4s, alphaV1_I
+ fmla v1.4s, v25.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+
+ fmul v2.4s, v26.4s, alphaV0_R
+ fmls v2.4s, v27.4s, alphaV0_I
+ fmul v3.4s, v26.4s, alphaV1_I
+ fmla v3.4s, v27.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.4s, v28.4s, alphaV0_R
+ fmls v4.4s, v29.4s, alphaV0_I
+ fmul v5.4s, v28.4s, alphaV1_I
+ fmla v5.4s, v29.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+
+ fmul v6.4s, v30.4s, alphaV0_R
+ fmls v6.4s, v31.4s, alphaV0_I
+ fmul v7.4s, v30.4s, alphaV1_I
+ fmla v7.4s, v31.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s28, s17
+ fmov s29, s16
+.endm
+
+.macro KERNEL4x4_I
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ fmul v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v17.16b, v17.16b, v17.16b
+ fmls v17.4s, v0.4s, v9.4s[0]
+#else
+ fmul v17.4s, v0.4s, v9.4s[0]
+#endif
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ fmul v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v21.16b, v21.16b, v21.16b
+ fmls v21.4s, v0.4s, v9.4s[1]
+#else
+ fmul v21.4s, v0.4s, v9.4s[1]
+#endif
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ fmul v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v25.16b, v25.16b, v25.16b
+ fmls v25.4s, v0.4s, v9.4s[2]
+#else
+ fmul v25.4s, v0.4s, v9.4s[2]
+#endif
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ fmul v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ eor v29.16b, v29.16b, v29.16b
+ fmls v29.4s, v0.4s, v9.4s[3]
+#else
+ fmul v29.4s, v0.4s, v9.4s[3]
+#endif
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ ld2 {v12.4s, v13.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v4.4s, v5.4s}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL4x4_M1
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ ld2 {v12.4s, v13.4s}, [pB] // For next round
+ add pB, pB, #32
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ ld2 {v4.4s, v5.4s}, [pA] // For next round
+ add pA, pA, #32
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+.endm
+
+.macro KERNEL4x4_M2
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ ld2 {v8.4s, v9.4s}, [pB] // For next round
+ add pB, pB, #32
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ ld2 {v0.4s, v1.4s}, [pA] // For next round
+ add pA, pA, #32
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+.endm
+
+.macro KERNEL4x4_E
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+.endm
+
+.macro SAVE4x4
+ mov pCRow1, pCRow0
+
+
+ fmul v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmul v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmul v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v0.4s, v24.4s, alphaV0_R
+ fmls v0.4s, v25.4s, alphaV0_I
+ fmul v1.4s, v24.4s, alphaV1_I
+ fmla v1.4s, v25.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.4s, v28.4s, alphaV0_R
+ fmls v4.4s, v29.4s, alphaV0_I
+ fmul v5.4s, v28.4s, alphaV1_I
+ fmla v5.4s, v29.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+ fmov s24, s16
+ fmov s25, s17
+ fmov s28, s16
+ fmov s29, s17
+.endm
+
+.macro KERNEL2x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.4s[0]
+ OP_ii v16.2s, v1.2s, v9.4s[0]
+ OP_ri v17.2s, v0.2s, v9.4s[0]
+ OP_ir v17.2s, v1.2s, v8.4s[0]
+
+ OP_rr v20.2s, v0.2s, v8.4s[1]
+ OP_ii v20.2s, v1.2s, v9.4s[1]
+ OP_ri v21.2s, v0.2s, v9.4s[1]
+ OP_ir v21.2s, v1.2s, v8.4s[1]
+
+ OP_rr v24.2s, v0.2s, v8.4s[2]
+ OP_ii v24.2s, v1.2s, v9.4s[2]
+ OP_ri v25.2s, v0.2s, v9.4s[2]
+ OP_ir v25.2s, v1.2s, v8.4s[2]
+
+ OP_rr v28.2s, v0.2s, v8.4s[3]
+ OP_ii v28.2s, v1.2s, v9.4s[3]
+ OP_ri v29.2s, v0.2s, v9.4s[3]
+ OP_ir v29.2s, v1.2s, v8.4s[3]
+.endm
+
+.macro SAVE2x4
+ mov pCRow1, pCRow0
+
+
+ fmul v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmul v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.2s, v20.2s, alphaV0_R
+ fmls v4.2s, v21.2s, alphaV0_I
+ fmul v5.2s, v20.2s, alphaV1_I
+ fmla v5.2s, v21.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v0.2s, v24.2s, alphaV0_R
+ fmls v0.2s, v25.2s, alphaV0_I
+ fmul v1.2s, v24.2s, alphaV1_I
+ fmla v1.2s, v25.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.2s, v28.2s, alphaV0_R
+ fmls v4.2s, v29.2s, alphaV0_I
+ fmul v5.2s, v28.2s, alphaV1_I
+ fmla v5.2s, v29.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+ fmov s24, s16
+ fmov s25, s17
+ fmov s28, s16
+ fmov s29, s17
+.endm
+
+.macro KERNEL1x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.4s[0]
+ OP_ii s16, s1, v9.4s[0]
+ OP_ri s17, s0, v9.4s[0]
+ OP_ir s17, s1, v8.4s[0]
+
+ OP_rr s20, s0, v8.4s[1]
+ OP_ii s20, s1, v9.4s[1]
+ OP_ri s21, s0, v9.4s[1]
+ OP_ir s21, s1, v8.4s[1]
+
+ OP_rr s24, s0, v8.4s[2]
+ OP_ii s24, s1, v9.4s[2]
+ OP_ri s25, s0, v9.4s[2]
+ OP_ir s25, s1, v8.4s[2]
+
+ OP_rr s28, s0, v8.4s[3]
+ OP_ii s28, s1, v9.4s[3]
+ OP_ri s29, s0, v9.4s[3]
+ OP_ir s29, s1, v8.4s[3]
+.endm
+
+.macro SAVE1x4
+ mov pCRow1, pCRow0
+
+
+ fmul s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmul s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul s4, s20, alphaV0_R
+ fmls s4, s21, alphaV0_I
+ fmul s5, s20, alphaV1_I
+ fmla s5, s21, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul s0, s24, alphaV0_R
+ fmls s0, s25, alphaV0_I
+ fmul s1, s24, alphaV1_I
+ fmla s1, s25, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul s4, s28, alphaV0_R
+ fmls s4, s29, alphaV0_I
+ fmul s5, s28, alphaV1_I
+ fmla s5, s29, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+.endm
+
+.macro KERNEL8x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.2s[0]
+ OP_ii v16.4s, v1.4s, v9.2s[0]
+ OP_ri v17.4s, v0.4s, v9.2s[0]
+ OP_ir v17.4s, v1.4s, v8.2s[0]
+
+ OP_rr v18.4s, v2.4s, v8.2s[0]
+ OP_ii v18.4s, v3.4s, v9.2s[0]
+ OP_ri v19.4s, v2.4s, v9.2s[0]
+ OP_ir v19.4s, v3.4s, v8.2s[0]
+
+ OP_rr v20.4s, v0.4s, v8.2s[1]
+ OP_ii v20.4s, v1.4s, v9.2s[1]
+ OP_ri v21.4s, v0.4s, v9.2s[1]
+ OP_ir v21.4s, v1.4s, v8.2s[1]
+
+ OP_rr v22.4s, v2.4s, v8.2s[1]
+ OP_ii v22.4s, v3.4s, v9.2s[1]
+ OP_ri v23.4s, v2.4s, v9.2s[1]
+ OP_ir v23.4s, v3.4s, v8.2s[1]
+.endm
+
+.macro SAVE8x2
+ mov pCRow1, pCRow0
+
+
+ fmul v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmul v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+
+ fmul v2.4s, v18.4s, alphaV0_R
+ fmls v2.4s, v19.4s, alphaV0_I
+ fmul v3.4s, v18.4s, alphaV1_I
+ fmla v3.4s, v19.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmul v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow2, pCRow1, #32
+
+
+ fmul v6.4s, v22.4s, alphaV0_R
+ fmls v6.4s, v23.4s, alphaV0_I
+ fmul v7.4s, v22.4s, alphaV1_I
+ fmla v7.4s, v23.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+.endm
+
+.macro KERNEL4x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.2s[0]
+ OP_ii v16.4s, v1.4s, v9.2s[0]
+ OP_ri v17.4s, v0.4s, v9.2s[0]
+ OP_ir v17.4s, v1.4s, v8.2s[0]
+
+ OP_rr v20.4s, v0.4s, v8.2s[1]
+ OP_ii v20.4s, v1.4s, v9.2s[1]
+ OP_ri v21.4s, v0.4s, v9.2s[1]
+ OP_ir v21.4s, v1.4s, v8.2s[1]
+.endm
+
+.macro SAVE4x2
+ mov pCRow1, pCRow0
+
+
+ fmul v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmul v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmul v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+.endm
+
+.macro KERNEL2x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.2s[0]
+ OP_ii v16.2s, v1.2s, v9.2s[0]
+ OP_ri v17.2s, v0.2s, v9.2s[0]
+ OP_ir v17.2s, v1.2s, v8.2s[0]
+
+ OP_rr v20.2s, v0.2s, v8.2s[1]
+ OP_ii v20.2s, v1.2s, v9.2s[1]
+ OP_ri v21.2s, v0.2s, v9.2s[1]
+ OP_ir v21.2s, v1.2s, v8.2s[1]
+.endm
+
+.macro SAVE2x2
+ mov pCRow1, pCRow0
+
+
+ fmul v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmul v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul v4.2s, v20.2s, alphaV0_R
+ fmls v4.2s, v21.2s, alphaV0_I
+ fmul v5.2s, v20.2s, alphaV1_I
+ fmla v5.2s, v21.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, wzr
+ fmov s21, wzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.2s[0]
+ OP_ii s16, s1, v9.2s[0]
+ OP_ri s17, s0, v9.2s[0]
+ OP_ir s17, s1, v8.2s[0]
+
+ OP_rr s20, s0, v8.2s[1]
+ OP_ii s20, s1, v9.2s[1]
+ OP_ri s21, s0, v9.2s[1]
+ OP_ir s21, s1, v8.2s[1]
+.endm
+
+.macro SAVE1x2
+ mov pCRow1, pCRow0
+
+
+ fmul s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmul s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+
+ fmul s4, s20, alphaV0_R
+ fmls s4, s21, alphaV0_I
+ fmul s5, s20, alphaV1_I
+ fmla s5, s21, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+.endm
+
+.macro KERNEL8x1_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v8.4s[1]
+ OP_ri v17.4s, v0.4s, v8.4s[1]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v8.4s[1]
+ OP_ri v19.4s, v2.4s, v8.4s[1]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+.endm
+
+.macro SAVE8x1
+ mov pCRow1, pCRow0
+
+
+ fmul v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmul v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, #32
+
+
+ fmul v2.4s, v18.4s, alphaV0_R
+ fmls v2.4s, v19.4s, alphaV0_I
+ fmul v3.4s, v18.4s, alphaV1_I
+ fmla v3.4s, v19.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov s16, wzr
+ fmov s17, s16
+.endm
+
+.macro KERNEL4x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.s[0]
+ OP_ii v16.4s, v1.4s, v9.s[0]
+ OP_ri v17.4s, v0.4s, v9.s[0]
+ OP_ir v17.4s, v1.4s, v8.s[0]
+.endm
+
+.macro SAVE4x1
+ mov pCRow1, pCRow0
+
+
+ fmul v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmul v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.s[0]
+ OP_ii v16.2s, v1.2s, v9.s[0]
+ OP_ri v17.2s, v0.2s, v9.s[0]
+ OP_ir v17.2s, v1.2s, v8.s[0]
+.endm
+
+.macro SAVE2x1
+ mov pCRow1, pCRow0
+
+
+ fmul v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmul v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.s[0]
+ OP_ii s16, s1, v9.s[0]
+ OP_ri s17, s0, v9.s[0]
+ OP_ir s17, s1, v8.s[0]
+.endm
+
+.macro SAVE1x1
+ mov pCRow1, pCRow0
+
+
+ fmul s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmul s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0_R, s0
+ fmov alpha0_I, s1
+ fmov alpha1_R, s0
+ fmov alpha1_I, s1
+
+ lsl LDC, LDC, #3 // ldc = ldc * 8
+
+#if !defined(LEFT)
+ neg tempOffset, offset
+#endif
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #2 // J = J / 4
+ cmp counterJ, #0
+ ble ctrmm_kernel_L2_BEGIN
+
+/******************************************************************************/
+
+ctrmm_kernel_L4_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = start of A array
+
+ctrmm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble ctrmm_kernel_L4_M4_BEGIN
+
+ctrmm_kernel_L4_M8_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt ctrmm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2 // subtract 2
+ ble ctrmm_kernel_L4_M8_22a
+ .align 5
+
+ctrmm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L4_M8_22
+
+
+ctrmm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b ctrmm_kernel_L4_M8_44
+
+ctrmm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble ctrmm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+
+ KERNEL8x4_E
+
+ b ctrmm_kernel_L4_M8_44
+
+ctrmm_kernel_L4_M8_40:
+
+ INIT8x4
+
+ctrmm_kernel_L4_M8_44:
+
+ ands counterL , tempK, #1
+ ble ctrmm_kernel_L4_M8_100
+
+ctrmm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+ctrmm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+ctrmm_kernel_L4_M8_END:
+ subs counterI, counterI, #1
+ bne ctrmm_kernel_L4_M8_20
+
+ctrmm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble ctrmm_kernel_L4_END
+
+ tst counterI, #4
+ ble ctrmm_kernel_L4_M2_BEGIN
+
+ctrmm_kernel_L4_M4_20:
+
+ INIT4x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble ctrmm_kernel_L4_M4_40
+
+ctrmm_kernel_L4_M4_22:
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L4_M4_22
+
+
+ctrmm_kernel_L4_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L4_M4_100
+
+ctrmm_kernel_L4_M4_42:
+
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L4_M4_42
+
+ctrmm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+ctrmm_kernel_L4_M4_END:
+
+
+ctrmm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble ctrmm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble ctrmm_kernel_L4_M1_BEGIN
+
+ctrmm_kernel_L4_M2_20:
+
+ INIT2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble ctrmm_kernel_L4_M2_40
+
+ctrmm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L4_M2_22
+
+
+ctrmm_kernel_L4_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L4_M2_100
+
+ctrmm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L4_M2_42
+
+ctrmm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+ctrmm_kernel_L4_M2_END:
+
+
+ctrmm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble ctrmm_kernel_L4_END
+
+ctrmm_kernel_L4_M1_20:
+
+ INIT1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble ctrmm_kernel_L4_M1_40
+
+ctrmm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L4_M1_22
+
+
+ctrmm_kernel_L4_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L4_M1_100
+
+ctrmm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L4_M1_42
+
+ctrmm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+
+ctrmm_kernel_L4_END:
+
+ lsl temp, origK, #5
+ add origPB, origPB, temp // B = B + K * 4 * 8
+
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+ subs counterJ, counterJ , #1 // j--
+ bgt ctrmm_kernel_L4_BEGIN
+
+
+/******************************************************************************/
+
+ctrmm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble ctrmm_kernel_L999 // error, N was less than 4?
+
+ tst counterJ , #2
+ ble ctrmm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+ctrmm_kernel_L2_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble ctrmm_kernel_L2_M4_BEGIN
+
+ctrmm_kernel_L2_M8_20:
+
+ INIT8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble ctrmm_kernel_L2_M8_40
+ .align 5
+
+ctrmm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M8_22
+
+
+ctrmm_kernel_L2_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L2_M8_100
+
+ctrmm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M8_42
+
+ctrmm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+ctrmm_kernel_L2_M8_END:
+
+ subs counterI, counterI, #1
+ bgt ctrmm_kernel_L2_M8_20
+
+ctrmm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble ctrmm_kernel_L2_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble ctrmm_kernel_L2_M2_BEGIN
+
+ctrmm_kernel_L2_M4_20:
+
+ INIT4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble ctrmm_kernel_L2_M4_40
+ .align 5
+
+ctrmm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M4_22
+
+
+ctrmm_kernel_L2_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L2_M4_100
+
+ctrmm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M4_42
+
+ctrmm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+ctrmm_kernel_L2_M4_END:
+
+
+ctrmm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble ctrmm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble ctrmm_kernel_L2_M1_BEGIN
+
+ctrmm_kernel_L2_M2_20:
+
+ INIT2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble ctrmm_kernel_L2_M2_40
+
+ctrmm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M2_22
+
+
+ctrmm_kernel_L2_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L2_M2_100
+
+ctrmm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M2_42
+
+ctrmm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+ctrmm_kernel_L2_M2_END:
+
+
+ctrmm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble ctrmm_kernel_L2_END
+
+ctrmm_kernel_L2_M1_20:
+
+ INIT1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble ctrmm_kernel_L2_M1_40
+
+ctrmm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M1_22
+
+
+ctrmm_kernel_L2_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L2_M1_100
+
+ctrmm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L2_M1_42
+
+ctrmm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+
+ctrmm_kernel_L2_END:
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+ add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
+
+/******************************************************************************/
+
+ctrmm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble ctrmm_kernel_L999 // done
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+ctrmm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble ctrmm_kernel_L1_M4_BEGIN
+
+ctrmm_kernel_L1_M8_20:
+
+ INIT8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble ctrmm_kernel_L1_M8_40
+ .align 5
+
+ctrmm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M8_22
+
+
+ctrmm_kernel_L1_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L1_M8_100
+
+ctrmm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M8_42
+
+ctrmm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+ctrmm_kernel_L1_M8_END:
+
+ subs counterI, counterI, #1
+ bgt ctrmm_kernel_L1_M8_20
+
+ctrmm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble ctrmm_kernel_L1_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble ctrmm_kernel_L1_M2_BEGIN
+
+ctrmm_kernel_L1_M4_20:
+
+ INIT4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+#endif
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble ctrmm_kernel_L1_M4_40
+ .align 5
+
+ctrmm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M4_22
+
+
+ctrmm_kernel_L1_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L1_M4_100
+
+ctrmm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M4_42
+
+ctrmm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+ctrmm_kernel_L1_M4_END:
+
+ctrmm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble ctrmm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble ctrmm_kernel_L1_M1_BEGIN
+
+ctrmm_kernel_L1_M2_20:
+
+ INIT2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble ctrmm_kernel_L1_M2_40
+
+ctrmm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M2_22
+
+
+ctrmm_kernel_L1_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L1_M2_100
+
+ctrmm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M2_42
+
+ctrmm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+ctrmm_kernel_L1_M2_END:
+
+
+ctrmm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble ctrmm_kernel_L1_END
+
+ctrmm_kernel_L1_M1_20:
+
+ INIT1x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble ctrmm_kernel_L1_M1_40
+
+ctrmm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M1_22
+
+
+ctrmm_kernel_L1_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble ctrmm_kernel_L1_M1_100
+
+ctrmm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt ctrmm_kernel_L1_M1_42
+
+ctrmm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+
+ctrmm_kernel_L1_END:
+
+
+ctrmm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define temp x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+
+#define alpha0 d2
+#define alphaV0 v2.d[0]
+#define alpha1 d3
+#define alphaV1 v3.d[0]
+#define alpha2 d6
+#define alphaV2 v6.d[0]
+#define alpha3 d7
+#define alphaV3 v7.d[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 temp
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16
+// 17
+// 18 must save
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA00, pA01
+//v01 pA02, pA03
+//v02 ALPHA0
+//v03 ALPHA1
+//v04 pA10, pA11
+//v05 pA12, pA13
+//v06 ALPHA2
+//v07 ALPHA3
+//v08 must save pB0_0, pB0_1
+//v09 must save pB0_2, pB0_3
+//v10 must save pB0_4, pB0_5
+//v11 must save pB0_6, pB0_7
+//v12 must save pB1_0, pB1_1
+//v13 must save pB1_2, pB1_3
+//v14 must save pB1_4, pB1_5
+//v15 must save pB1_6, pB1_7
+//v16 must save C00, C01
+//v17 must save C02, C03
+//v18 C04, C05
+//v19 C06, C07
+//v20 C10, C11
+//v21 C12, C13
+//v22 C14, C15
+//v23 C16, C17
+//v24 C20, C21
+//v25 C22, C23
+//v26 C24, C25
+//v27 C26, C27
+//v28 C30, C31
+//v29 C32, C33
+//v30 C34, C35
+//v31 C36, C37
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT4x8
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, xzr
+ fmov d19, d16
+ fmov d20, xzr
+ fmov d21, d16
+ fmov d22, d17
+ fmov d23, d18
+ fmov d24, xzr
+ fmov d25, d16
+ fmov d26, d17
+ fmov d27, d18
+ fmov d28, xzr
+ fmov d29, d16
+ fmov d30, d17
+ fmov d31, d18
+.endm
+
+.macro KERNEL4x8_I
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmul v16.2d, v0.2d, v8.2d[0]
+ fmul v17.2d, v1.2d, v8.2d[0]
+ fmul v18.2d, v0.2d, v8.2d[1]
+ fmul v19.2d, v1.2d, v8.2d[1]
+
+ fmul v20.2d, v0.2d, v9.2d[0]
+ fmul v21.2d, v1.2d, v9.2d[0]
+ fmul v22.2d, v0.2d, v9.2d[1]
+ fmul v23.2d, v1.2d, v9.2d[1]
+
+ fmul v24.2d, v0.2d, v10.2d[0]
+ fmul v25.2d, v1.2d, v10.2d[0]
+ fmul v26.2d, v0.2d, v10.2d[1]
+ fmul v27.2d, v1.2d, v10.2d[1]
+
+ fmul v28.2d, v0.2d, v11.2d[0]
+ fmul v29.2d, v1.2d, v11.2d[0]
+ fmul v30.2d, v0.2d, v11.2d[1]
+ fmul v31.2d, v1.2d, v11.2d[1]
+
+ ld1 {v12.2d, v13.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v4.2d, v5.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v14.2d, v15.2d}, [pB]
+ add pB, pB, #32
+.endm
+
+.macro KERNEL4x8_M1
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v0.2d, v8.2d[1]
+ fmla v19.2d, v1.2d, v8.2d[1]
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v9.2d[0]
+ fmla v22.2d, v0.2d, v9.2d[1]
+ fmla v23.2d, v1.2d, v9.2d[1]
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v25.2d, v1.2d, v10.2d[0]
+ fmla v26.2d, v0.2d, v10.2d[1]
+ fmla v27.2d, v1.2d, v10.2d[1]
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v29.2d, v1.2d, v11.2d[0]
+ fmla v30.2d, v0.2d, v11.2d[1]
+ fmla v31.2d, v1.2d, v11.2d[1]
+
+ ld1 {v12.2d, v13.2d}, [pB] // For next round
+ add pB, pB, #32
+ ld1 {v4.2d, v5.2d}, [pA] // For next round
+ add pA, pA, #32
+ ld1 {v14.2d, v15.2d}, [pB]
+ add pB, pB, #32
+
+ prfm PLDL1KEEP, [pA, #512]
+.endm
+
+.macro KERNEL4x8_M2
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+ fmla v18.2d, v4.2d, v12.2d[1]
+ fmla v19.2d, v5.2d, v12.2d[1]
+
+ fmla v20.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v13.2d[0]
+ fmla v22.2d, v4.2d, v13.2d[1]
+ fmla v23.2d, v5.2d, v13.2d[1]
+
+ fmla v24.2d, v4.2d, v14.2d[0]
+ fmla v25.2d, v5.2d, v14.2d[0]
+ fmla v26.2d, v4.2d, v14.2d[1]
+ fmla v27.2d, v5.2d, v14.2d[1]
+
+ fmla v28.2d, v4.2d, v15.2d[0]
+ fmla v29.2d, v5.2d, v15.2d[0]
+ fmla v30.2d, v4.2d, v15.2d[1]
+ fmla v31.2d, v5.2d, v15.2d[1]
+
+ ld1 {v8.2d, v9.2d}, [pB] // For next round
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA] // For next round
+ add pA, pA, #32
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ prfm PLDL1KEEP, [pB, #512]
+.endm
+
+.macro KERNEL4x8_E
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+ fmla v18.2d, v4.2d, v12.2d[1]
+ fmla v19.2d, v5.2d, v12.2d[1]
+
+ fmla v20.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v13.2d[0]
+ fmla v22.2d, v4.2d, v13.2d[1]
+ fmla v23.2d, v5.2d, v13.2d[1]
+
+ fmla v24.2d, v4.2d, v14.2d[0]
+ fmla v25.2d, v5.2d, v14.2d[0]
+ fmla v26.2d, v4.2d, v14.2d[1]
+ fmla v27.2d, v5.2d, v14.2d[1]
+
+ fmla v28.2d, v4.2d, v15.2d[0]
+ fmla v29.2d, v5.2d, v15.2d[0]
+ fmla v30.2d, v4.2d, v15.2d[1]
+ fmla v31.2d, v5.2d, v15.2d[1]
+.endm
+
+.macro KERNEL4x8_SUB
+ ld1 {v8.2d, v9.2d}, [pB] // For next round
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA] // For next round
+ add pA, pA, #32
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v0.2d, v8.2d[1]
+ fmla v19.2d, v1.2d, v8.2d[1]
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v9.2d[0]
+ fmla v22.2d, v0.2d, v9.2d[1]
+ fmla v23.2d, v1.2d, v9.2d[1]
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v25.2d, v1.2d, v10.2d[0]
+ fmla v26.2d, v0.2d, v10.2d[1]
+ fmla v27.2d, v1.2d, v10.2d[1]
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v29.2d, v1.2d, v11.2d[0]
+ fmla v30.2d, v0.2d, v11.2d[1]
+ fmla v31.2d, v1.2d, v11.2d[1]
+.endm
+
+.macro SAVE4x8
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v8.2d, v9.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ fmla v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v10.2d, v11.2d}, [pCRow1]
+ fmla v10.2d, v18.2d, alphaV2
+ fmla v11.2d, v19.2d, alphaV3
+ st1 {v10.2d, v11.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow2]
+ fmla v12.2d, v20.2d, alphaV0
+ fmla v13.2d, v21.2d, alphaV1
+ st1 {v12.2d, v13.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v14.2d, v15.2d}, [pCRow1]
+ fmla v14.2d, v22.2d, alphaV2
+ fmla v15.2d, v23.2d, alphaV3
+ st1 {v14.2d, v15.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v8.2d, v9.2d}, [pCRow2]
+ fmla v8.2d, v24.2d, alphaV0
+ fmla v9.2d, v25.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v10.2d, v11.2d}, [pCRow1]
+ fmla v10.2d, v26.2d, alphaV2
+ fmla v11.2d, v27.2d, alphaV3
+ st1 {v10.2d, v11.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow2]
+ fmla v12.2d, v28.2d, alphaV0
+ fmla v13.2d, v29.2d, alphaV1
+ st1 {v12.2d, v13.2d}, [pCRow2]
+
+ ld1 {v14.2d, v15.2d}, [pCRow1]
+ fmla v14.2d, v30.2d, alphaV2
+ fmla v15.2d, v31.2d, alphaV3
+ st1 {v14.2d, v15.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x8
+ fmov d16, xzr
+ fmov d18, xzr
+ fmov d20, xzr
+ fmov d22, d16
+ fmov d24, xzr
+ fmov d26, d16
+ fmov d28, xzr
+ fmov d30, d16
+.endm
+
+.macro KERNEL2x8_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v18.2d, v0.2d, v8.2d[1]
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v22.2d, v0.2d, v9.2d[1]
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v26.2d, v0.2d, v10.2d[1]
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v30.2d, v0.2d, v11.2d[1]
+.endm
+
+.macro SAVE2x8
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v8.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v10.2d}, [pCRow1]
+ fmla v10.2d, v18.2d, alphaV2
+ st1 {v10.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d}, [pCRow2]
+ fmla v12.2d, v20.2d, alphaV0
+ st1 {v12.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v14.2d}, [pCRow1]
+ fmla v14.2d, v22.2d, alphaV2
+ st1 {v14.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v8.2d}, [pCRow2]
+ fmla v8.2d, v24.2d, alphaV0
+ st1 {v8.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v10.2d}, [pCRow1]
+ fmla v10.2d, v26.2d, alphaV2
+ st1 {v10.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d}, [pCRow2]
+ fmla v12.2d, v28.2d, alphaV0
+ st1 {v12.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v14.2d}, [pCRow1]
+ fmla v14.2d, v30.2d, alphaV2
+ st1 {v14.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x8
+ fmov d16, xzr
+ fmov d20, xzr
+ fmov d24, xzr
+ fmov d28, xzr
+.endm
+
+.macro KERNEL1x8_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ldr d0, [pA]
+ add pA, pA, #8
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v8.2d, v0.d[0]
+ fmla v20.2d, v9.2d, v0.d[0]
+ fmla v24.2d, v10.2d, v0.d[0]
+ fmla v28.2d, v11.2d, v0.d[0]
+.endm
+
+.macro SAVE1x8
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v8.d}[0], [pCRow0]
+ ld1 {v8.d}[1], [pCRow1]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v10.d}[0], [pCRow2]
+ ld1 {v10.d}[1], [pCRow1]
+ fmla v10.2d, v20.2d, alphaV1
+ st1 {v10.d}[0], [pCRow2]
+ st1 {v10.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.d}[0], [pCRow2]
+ ld1 {v12.d}[1], [pCRow1]
+ fmla v12.2d, v24.2d, alphaV2
+ st1 {v12.d}[0], [pCRow2]
+ st1 {v12.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v14.d}[0], [pCRow2]
+ ld1 {v14.d}[1], [pCRow1]
+ fmla v14.2d, v28.2d, alphaV3
+ st1 {v14.d}[0], [pCRow2]
+ st1 {v14.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+ fmov d24, d17
+ fmov d25, d16
+ fmov d28, d17
+ fmov d29, d16
+.endm
+
+.macro KERNEL4x4_I
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmul v16.2d, v0.2d, v8.2d[0]
+ fmul v29.2d, v1.2d, v9.2d[1]
+
+ fmul v20.2d, v0.2d, v8.2d[1]
+ fmul v25.2d, v1.2d, v9.2d[0]
+
+ fmul v24.2d, v0.2d, v9.2d[0]
+ fmul v21.2d, v1.2d, v8.2d[1]
+
+ fmul v28.2d, v0.2d, v9.2d[1]
+ fmul v17.2d, v1.2d, v8.2d[0]
+
+ ld1 {v12.2d, v13.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v4.2d, v5.2d}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL4x4_M1
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v29.2d, v1.2d, v9.2d[1]
+
+ ld1 {v12.2d, v13.2d}, [pB] // For next round
+ add pB, pB, #32
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v25.2d, v1.2d, v9.2d[0]
+
+ ld1 {v4.2d, v5.2d}, [pA] // For next round
+ add pA, pA, #32
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v8.2d[1]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro KERNEL4x4_M2
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v29.2d, v5.2d, v13.2d[1]
+
+ ld1 {v8.2d, v9.2d}, [pB] // For next round
+ add pB, pB, #32
+
+ fmla v20.2d, v4.2d, v12.2d[1]
+ fmla v25.2d, v5.2d, v13.2d[0]
+
+ ld1 {v0.2d, v1.2d}, [pA] // For next round
+ add pA, pA, #32
+
+ fmla v24.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v12.2d[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ fmla v28.2d, v4.2d, v13.2d[1]
+ fmla v17.2d, v5.2d, v12.2d[0]
+.endm
+
+.macro KERNEL4x4_E
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v29.2d, v5.2d, v13.2d[1]
+
+ fmla v20.2d, v4.2d, v12.2d[1]
+ fmla v25.2d, v5.2d, v13.2d[0]
+
+ fmla v24.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v12.2d[1]
+
+ fmla v28.2d, v4.2d, v13.2d[1]
+ fmla v17.2d, v5.2d, v12.2d[0]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v29.2d, v1.2d, v9.2d[1]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v25.2d, v1.2d, v9.2d[0]
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v8.2d[1]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x4
+ ld1 {v8.2d, v9.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ fmla v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV2
+ fmla v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v8.2d, v9.2d}, [pCRow2]
+ fmla v8.2d, v24.2d, alphaV0
+ fmla v9.2d, v25.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow1]
+ fmla v12.2d, v28.2d, alphaV2
+ fmla v13.2d, v29.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov d16, xzr
+ fmov d20, d16
+ fmov d24, d20
+ fmov d28, d16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v28.2d, v0.2d, v9.2d[1]
+.endm
+
+.macro SAVE2x4
+ ld1 {v8.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v12.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v8.2d}, [pCRow2]
+ fmla v8.2d, v24.2d, alphaV2
+ st1 {v8.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d}, [pCRow1]
+ fmla v12.2d, v28.2d, alphaV3
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr d0, [pA]
+ add pA, pA, #8
+
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v8.2d, v0.d[0]
+ fmla v20.2d, v9.2d, v0.d[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v8.d}[0], [pCRow0]
+ ld1 {v8.d}[1], [pCRow1]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.d}[0], [pCRow2]
+ ld1 {v12.d}[1], [pCRow1]
+ fmla v12.2d, v20.2d, alphaV1
+ st1 {v12.d}[0], [pCRow2]
+ st1 {v12.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+.endm
+
+.macro SAVE4x2
+ ld1 {v8.2d, v9.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ fmla v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV2
+ fmla v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+.endm
+
+.macro SAVE2x2
+ ld1 {v8.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+
+ ld1 {v12.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2d} , [pB]
+ add pB , pB, #16
+
+ ldr d0 , [pA]
+ add pA, pA, #8
+
+ fmla v16.2d, v8.2d, v0.2d[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+
+ ld1 {v8.d}[0], [pCRow0]
+ ld1 {v8.d}[1], [pCRow1]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov d16, xzr
+ fmov d17, d16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA , pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x1
+ ld1 {v8.2d, v9.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ fmla v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+
+
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+.endm
+
+.macro SAVE2x1
+ ld1 {v8.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ldr d0, [pA]
+ add pA , pA, #8
+
+ fmadd d16, d0, d8, d16
+.endm
+
+.macro SAVE1x1
+ ldr d8, [pCRow0]
+ fmadd d8, d16, alpha0, d8
+ str d8, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0, d0
+ fmov alpha1, d0
+ fmov alpha2, d0
+ fmov alpha3, d0
+
+ lsl LDC, LDC, #3 // ldc = ldc * 8
+
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #3 // J = J / 8
+ cmp counterJ, #0
+ ble dgemm_kernel_L4_BEGIN
+
+/******************************************************************************/
+
+dgemm_kernel_L8_BEGIN:
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #3
+
+ mov pA, origPA // pA = start of A array
+
+dgemm_kernel_L8_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI, #0
+ ble dgemm_kernel_L8_M2_BEGIN
+
+dgemm_kernel_L8_M4_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt dgemm_kernel_L8_M4_32
+
+ KERNEL4x8_I // do one in the K
+ KERNEL4x8_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble dgemm_kernel_L8_M4_22a
+ .align 5
+
+dgemm_kernel_L8_M4_22:
+
+ KERNEL4x8_M1
+ KERNEL4x8_M2
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L8_M4_22
+
+
+dgemm_kernel_L8_M4_22a:
+
+ KERNEL4x8_M1
+ KERNEL4x8_E
+
+ b dgemm_kernel_L8_M4_44
+
+dgemm_kernel_L8_M4_32:
+
+ tst counterL, #1
+ ble dgemm_kernel_L8_M4_40
+
+ KERNEL4x8_I
+
+ KERNEL4x8_E
+
+ b dgemm_kernel_L8_M4_44
+
+
+dgemm_kernel_L8_M4_40:
+
+ INIT4x8
+
+dgemm_kernel_L8_M4_44:
+
+ ands counterL , origK, #1
+ ble dgemm_kernel_L8_M4_100
+
+dgemm_kernel_L8_M4_46:
+
+ KERNEL4x8_SUB
+
+dgemm_kernel_L8_M4_100:
+
+ SAVE4x8
+
+dgemm_kernel_L8_M4_END:
+ subs counterI, counterI, #1
+ bne dgemm_kernel_L8_M4_20
+
+dgemm_kernel_L8_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dgemm_kernel_L8_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dgemm_kernel_L8_M1_BEGIN
+
+dgemm_kernel_L8_M2_20:
+
+ INIT2x8
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L8_M2_40
+
+dgemm_kernel_L8_M2_22:
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L8_M2_22
+
+
+dgemm_kernel_L8_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L8_M2_100
+
+dgemm_kernel_L8_M2_42:
+
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L8_M2_42
+
+dgemm_kernel_L8_M2_100:
+
+ SAVE2x8
+
+dgemm_kernel_L8_M2_END:
+
+
+dgemm_kernel_L8_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dgemm_kernel_L8_END
+
+dgemm_kernel_L8_M1_20:
+
+ INIT1x8
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L8_M1_40
+
+dgemm_kernel_L8_M1_22:
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L8_M1_22
+
+
+dgemm_kernel_L8_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L8_M1_100
+
+dgemm_kernel_L8_M1_42:
+
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L8_M1_42
+
+dgemm_kernel_L8_M1_100:
+
+ SAVE1x8
+
+dgemm_kernel_L8_END:
+
+ lsl temp, origK, #6
+ add origPB, origPB, temp // B = B + K * 8 * 8
+
+ subs counterJ, counterJ , #1 // j--
+ bgt dgemm_kernel_L8_BEGIN
+
+
+/******************************************************************************/
+
+dgemm_kernel_L4_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #7
+ ble dgemm_kernel_L999
+
+ tst counterJ , #4
+ ble dgemm_kernel_L2_BEGIN
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+ mov pA, origPA // pA = start of A array
+
+dgemm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI, #0
+ ble dgemm_kernel_L4_M2_BEGIN
+
+dgemm_kernel_L4_M4_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt dgemm_kernel_L4_M4_32
+
+ KERNEL4x4_I // do one in the K
+ KERNEL4x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble dgemm_kernel_L4_M4_22a
+ .align 5
+
+dgemm_kernel_L4_M4_22:
+
+ KERNEL4x4_M1
+ KERNEL4x4_M2
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M4_22
+
+
+dgemm_kernel_L4_M4_22a:
+
+ KERNEL4x4_M1
+ KERNEL4x4_E
+
+ b dgemm_kernel_L4_M4_44
+
+dgemm_kernel_L4_M4_32:
+
+ tst counterL, #1
+ ble dgemm_kernel_L4_M4_40
+
+ KERNEL4x4_I
+
+ KERNEL4x4_E
+
+ b dgemm_kernel_L4_M4_44
+
+
+dgemm_kernel_L4_M4_40:
+
+ INIT4x4
+
+dgemm_kernel_L4_M4_44:
+
+ ands counterL , origK, #1
+ ble dgemm_kernel_L4_M4_100
+
+dgemm_kernel_L4_M4_46:
+
+ KERNEL4x4_SUB
+
+dgemm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+dgemm_kernel_L4_M4_END:
+ subs counterI, counterI, #1
+ bne dgemm_kernel_L4_M4_20
+
+dgemm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dgemm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dgemm_kernel_L4_M1_BEGIN
+
+dgemm_kernel_L4_M2_20:
+
+ INIT2x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L4_M2_40
+
+dgemm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M2_22
+
+
+dgemm_kernel_L4_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L4_M2_100
+
+dgemm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M2_42
+
+dgemm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+dgemm_kernel_L4_M2_END:
+
+
+dgemm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dgemm_kernel_L4_END
+
+dgemm_kernel_L4_M1_20:
+
+ INIT1x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L4_M1_40
+
+dgemm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M1_22
+
+
+dgemm_kernel_L4_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L4_M1_100
+
+dgemm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M1_42
+
+dgemm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+dgemm_kernel_L4_END:
+
+ lsl temp, origK, #5
+ add origPB, origPB, temp // B = B + K * 4 * 8
+
+/******************************************************************************/
+
+dgemm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble dgemm_kernel_L999 // error, N was less than 4?
+
+ tst counterJ , #2
+ ble dgemm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+ mov pA, origPA // pA = A
+
+
+dgemm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI,#0
+ ble dgemm_kernel_L2_M2_BEGIN
+
+dgemm_kernel_L2_M4_20:
+
+ INIT4x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dgemm_kernel_L2_M4_40
+ .align 5
+
+dgemm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M4_22
+
+
+dgemm_kernel_L2_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L2_M4_100
+
+dgemm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M4_42
+
+dgemm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+dgemm_kernel_L2_M4_END:
+
+ subs counterI, counterI, #1
+ bgt dgemm_kernel_L2_M4_20
+
+
+dgemm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dgemm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dgemm_kernel_L2_M1_BEGIN
+
+dgemm_kernel_L2_M2_20:
+
+ INIT2x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dgemm_kernel_L2_M2_40
+
+dgemm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M2_22
+
+
+dgemm_kernel_L2_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L2_M2_100
+
+dgemm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M2_42
+
+dgemm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+dgemm_kernel_L2_M2_END:
+
+
+dgemm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dgemm_kernel_L2_END
+
+dgemm_kernel_L2_M1_20:
+
+ INIT1x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble dgemm_kernel_L2_M1_40
+
+dgemm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M1_22
+
+
+dgemm_kernel_L2_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L2_M1_100
+
+dgemm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M1_42
+
+dgemm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+dgemm_kernel_L2_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
+
+/******************************************************************************/
+
+dgemm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble dgemm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+ mov pA, origPA // pA = A
+
+dgemm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI, #0
+ ble dgemm_kernel_L1_M2_BEGIN
+
+dgemm_kernel_L1_M4_20:
+
+ INIT4x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L1_M4_40
+ .align 5
+
+dgemm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M4_22
+
+
+dgemm_kernel_L1_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L1_M4_100
+
+dgemm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M4_42
+
+dgemm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+dgemm_kernel_L1_M4_END:
+
+ subs counterI, counterI, #1
+ bgt dgemm_kernel_L1_M4_20
+
+
+dgemm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dgemm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dgemm_kernel_L1_M1_BEGIN
+
+dgemm_kernel_L1_M2_20:
+
+ INIT2x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L1_M2_40
+
+dgemm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M2_22
+
+
+dgemm_kernel_L1_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L1_M2_100
+
+dgemm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M2_42
+
+dgemm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+dgemm_kernel_L1_M2_END:
+
+
+dgemm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dgemm_kernel_L1_END
+
+dgemm_kernel_L1_M1_20:
+
+ INIT1x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L1_M1_40
+
+dgemm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M1_22
+
+
+dgemm_kernel_L1_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L1_M1_100
+
+dgemm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M1_42
+
+dgemm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+
+dgemm_kernel_L1_END:
+
+
+dgemm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define temp x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pCRow3 x15
+#define pA x16
+#define alpha x17
+
+#define alpha0 d10
+#define alphaV0 v10.d[0]
+#define alpha1 d11
+#define alphaV1 v11.d[0]
+#define alpha2 d14
+#define alphaV2 v14.d[0]
+#define alpha3 d15
+#define alphaV3 v15.d[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 temp
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16
+// 17
+// 18 must save
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA0_0, pA0_1
+//v01 pA0_2, pA0_3
+//v02 pA0_4, pA0_5
+//v03 pA0_6, pA0_7
+//v04 pA1_0, pA1_1
+//v05 pA1_2, pA1_3
+//v06 pA1_4, pA1_5
+//v07 pA1_6, pA1_7
+//v08 must save pB0_0, pB0_1
+//v09 must save pB0_2, pB0_3
+//v10 must save ALPHA0
+//v11 must save ALPHA1
+//v12 must save pB1_0, pB1_1
+//v13 must save pB1_2, pB1_3
+//v14 must save ALPHA2
+//v15 must save ALPHA3
+//v16 must save C00, C01
+//v17 must save C02, C03
+//v18 C04, C05
+//v19 C06, C07
+//v20 C10, C11
+//v21 C12, C13
+//v22 C14, C15
+//v23 C16, C17
+//v24 C20, C21
+//v25 C22, C23
+//v26 C24, C25
+//v27 C26, C27
+//v28 C30, C31
+//v29 C32, C33
+//v30 C34, C35
+//v31 C36, C37
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT8x4
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, d16
+ fmov d19, xzr
+ fmov d20, xzr
+ fmov d21, d16
+ fmov d22, d17
+ fmov d23, d18
+ fmov d24, xzr
+ fmov d25, d16
+ fmov d26, d17
+ fmov d27, d18
+ fmov d28, xzr
+ fmov d29, d16
+ fmov d30, d17
+ fmov d31, d18
+.endm
+
+.macro KERNEL8x4_I
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+ ldp d8, d9, [pB]
+ add pB, pB, #16
+ ldp d10, d11, [pB]
+ add pB, pB, #16
+
+ fmul v16.2d, v0.2d, v8.2d[0]
+ fmul v17.2d, v1.2d, v8.2d[0]
+
+ fmul v18.2d, v2.2d, v8.2d[0]
+ fmul v19.2d, v3.2d, v8.2d[0]
+
+ fmul v20.2d, v0.2d, v9.2d[0]
+ fmul v21.2d, v1.2d, v9.2d[0]
+
+ fmul v22.2d, v2.2d, v9.2d[0]
+ fmul v23.2d, v3.2d, v9.2d[0]
+
+ fmul v24.2d, v0.2d, v10.2d[0]
+ fmul v25.2d, v1.2d, v10.2d[0]
+
+ fmul v26.2d, v2.2d, v10.2d[0]
+ fmul v27.2d, v3.2d, v10.2d[0]
+
+ fmul v28.2d, v0.2d, v11.2d[0]
+ fmul v29.2d, v1.2d, v11.2d[0]
+
+ fmul v30.2d, v2.2d, v11.2d[0]
+ fmul v31.2d, v3.2d, v11.2d[0]
+
+ ld1 {v4.2d, v5.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v6.2d, v7.2d}, [pA]
+ add pA, pA, #32
+ ldp d12, d13, [pB]
+ add pB, pB, #16
+ ldp d14, d15, [pB]
+ add pB, pB, #16
+.endm
+
+.macro KERNEL8x4_M1
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v21.2d, v1.2d, v9.2d[0]
+ fmla v26.2d, v2.2d, v10.2d[0]
+ fmla v31.2d, v3.2d, v11.2d[0]
+
+ ld1 {v4.2d}, [pA], #16
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+
+ ld1 {v5.2d}, [pA], #16
+
+ fmla v30.2d, v2.2d, v11.2d[0]
+ fmla v27.2d, v3.2d, v10.2d[0]
+
+ ldp d12, d13, [pB]
+ add pB, pB, #16
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v25.2d, v1.2d, v10.2d[0]
+
+ ldp d14, d15, [pB]
+ add pB, pB, #16
+
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v23.2d, v3.2d, v9.2d[0]
+
+ ld1 {v6.2d}, [pA], #16
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v29.2d, v1.2d, v11.2d[0]
+
+ ld1 {v7.2d}, [pA], #16
+
+ fmla v22.2d, v2.2d, v9.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+
+ prfm PLDL1KEEP, [pA, #224]
+ prfm PLDL1KEEP, [pA, #224+64]
+.endm
+
+.macro KERNEL8x4_M2
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v21.2d, v5.2d, v13.2d[0]
+ fmla v26.2d, v6.2d, v14.2d[0]
+ fmla v31.2d, v7.2d, v15.2d[0]
+
+ ld1 {v0.2d}, [pA], #16
+
+ fmla v20.2d, v4.2d, v13.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+
+ ld1 {v1.2d}, [pA], #16
+
+ fmla v30.2d, v6.2d, v15.2d[0]
+ fmla v27.2d, v7.2d, v14.2d[0]
+
+ ldp d8, d9, [pB]
+ add pB, pB, #16
+
+ fmla v28.2d, v4.2d, v15.2d[0]
+ fmla v25.2d, v5.2d, v14.2d[0]
+
+ ldp d10, d11, [pB]
+ add pB, pB, #16
+
+ fmla v22.2d, v6.2d, v13.2d[0]
+ fmla v19.2d, v7.2d, v12.2d[0]
+
+ ld1 {v2.2d}, [pA], #16
+
+ fmla v24.2d, v4.2d, v14.2d[0]
+ fmla v29.2d, v5.2d, v15.2d[0]
+
+ ld1 {v3.2d}, [pA], #16
+
+ fmla v18.2d, v6.2d, v12.2d[0]
+ fmla v23.2d, v7.2d, v13.2d[0]
+
+ prfm PLDL1KEEP, [pB, #640]
+.endm
+
+.macro KERNEL8x4_E
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+ fmla v18.2d, v6.2d, v12.2d[0]
+ fmla v19.2d, v7.2d, v12.2d[0]
+ fmla v20.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v13.2d[0]
+ fmla v22.2d, v6.2d, v13.2d[0]
+ fmla v23.2d, v7.2d, v13.2d[0]
+ fmla v24.2d, v4.2d, v14.2d[0]
+ fmla v25.2d, v5.2d, v14.2d[0]
+ fmla v26.2d, v6.2d, v14.2d[0]
+ fmla v27.2d, v7.2d, v14.2d[0]
+ fmla v28.2d, v4.2d, v15.2d[0]
+ fmla v29.2d, v5.2d, v15.2d[0]
+ fmla v30.2d, v6.2d, v15.2d[0]
+ fmla v31.2d, v7.2d, v15.2d[0]
+.endm
+
+.macro KERNEL8x4_SUB
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+ ldp d8, d9, [pB]
+ add pB, pB, #16
+ ldp d10, d11, [pB]
+ add pB, pB, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v9.2d[0]
+ fmla v22.2d, v2.2d, v9.2d[0]
+ fmla v23.2d, v3.2d, v9.2d[0]
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v25.2d, v1.2d, v10.2d[0]
+ fmla v26.2d, v2.2d, v10.2d[0]
+ fmla v27.2d, v3.2d, v10.2d[0]
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v29.2d, v1.2d, v11.2d[0]
+ fmla v30.2d, v2.2d, v11.2d[0]
+ fmla v31.2d, v3.2d, v11.2d[0]
+.endm
+
+.macro SAVE8x4
+ fmov alpha0, alpha
+
+ ld1 {v0.2d, v1.2d}, [pCRow0]
+ fmla v0.2d, v16.2d, alphaV0
+ fmla v1.2d, v17.2d, alphaV0
+ st1 {v0.2d, v1.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+
+ ld1 {v2.2d, v3.2d}, [pCRow0]
+ fmla v2.2d, v18.2d, alphaV0
+ fmla v3.2d, v19.2d, alphaV0
+ st1 {v2.2d, v3.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+
+ ld1 {v4.2d, v5.2d}, [pCRow1]
+ fmla v4.2d, v20.2d, alphaV0
+ fmla v5.2d, v21.2d, alphaV0
+ st1 {v4.2d, v5.2d}, [pCRow1]
+
+ add pCRow1, pCRow1, #32
+
+ ld1 {v6.2d, v7.2d}, [pCRow1]
+ fmla v6.2d, v22.2d, alphaV0
+ fmla v7.2d, v23.2d, alphaV0
+ st1 {v6.2d, v7.2d}, [pCRow1]
+
+ add pCRow1, pCRow1, #32
+
+ ld1 {v0.2d, v1.2d}, [pCRow2]
+ fmla v0.2d, v24.2d, alphaV0
+ fmla v1.2d, v25.2d, alphaV0
+ st1 {v0.2d, v1.2d}, [pCRow2]
+
+ add pCRow2, pCRow2, #32
+ ld1 {v2.2d, v3.2d}, [pCRow2]
+ fmla v2.2d, v26.2d, alphaV0
+ fmla v3.2d, v27.2d, alphaV0
+ st1 {v2.2d, v3.2d}, [pCRow2]
+
+ add pCRow2, pCRow2, #32
+
+ ld1 {v4.2d, v5.2d}, [pCRow3]
+ fmla v4.2d, v28.2d, alphaV0
+ fmla v5.2d, v29.2d, alphaV0
+ st1 {v4.2d, v5.2d}, [pCRow3]
+
+ add pCRow3, pCRow3, #32
+
+ ld1 {v6.2d, v7.2d}, [pCRow3]
+ fmla v6.2d, v30.2d, alphaV0
+ fmla v7.2d, v31.2d, alphaV0
+ st1 {v6.2d, v7.2d}, [pCRow3]
+
+ add pCRow3, pCRow3, #32
+
+ prfm PLDL2KEEP, [pCRow0, #128]
+ prfm PLDL2KEEP, [pCRow1, #128]
+ prfm PLDL2KEEP, [pCRow2, #128]
+ prfm PLDL2KEEP, [pCRow3, #128]
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+ fmov d24, d17
+ fmov d25, d16
+ fmov d28, d17
+ fmov d29, d16
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v29.2d, v1.2d, v9.2d[1]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v25.2d, v1.2d, v9.2d[0]
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v8.2d[1]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x4
+ ld1 {v8.2d, v9.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ fmla v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV2
+ fmla v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v8.2d, v9.2d}, [pCRow2]
+ fmla v8.2d, v24.2d, alphaV0
+ fmla v9.2d, v25.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow1]
+ fmla v12.2d, v28.2d, alphaV2
+ fmla v13.2d, v29.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+
+.macro INIT2x4
+ fmov d16, xzr
+ fmov d20, d16
+ fmov d24, d20
+ fmov d28, d16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v28.2d, v0.2d, v9.2d[1]
+.endm
+
+.macro SAVE2x4
+ ld1 {v8.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v12.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v8.2d}, [pCRow2]
+ fmla v8.2d, v24.2d, alphaV2
+ st1 {v8.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.2d}, [pCRow1]
+ fmla v12.2d, v28.2d, alphaV3
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr d0, [pA]
+ add pA, pA, #8
+
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v8.2d, v0.d[0]
+ fmla v20.2d, v9.2d, v0.d[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v8.d}[0], [pCRow0]
+ ld1 {v8.d}[1], [pCRow1]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v12.d}[0], [pCRow2]
+ ld1 {v12.d}[1], [pCRow1]
+ fmla v12.2d, v20.2d, alphaV1
+ st1 {v12.d}[0], [pCRow2]
+ st1 {v12.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, d16
+ fmov d19, d17
+ fmov d20, xzr
+ fmov d21, d16
+ fmov d22, d17
+ fmov d23, d18
+.endm
+
+.macro KERNEL8x2_SUB
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+ fmla v22.2d, v2.2d, v8.2d[1]
+ fmla v23.2d, v3.2d, v8.2d[1]
+.endm
+
+.macro SAVE8x2
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
+ fmla v0.2d, v16.2d, alphaV0
+ fmla v1.2d, v17.2d, alphaV1
+ fmla v2.2d, v18.2d, alphaV2
+ fmla v3.2d, v19.2d, alphaV3
+ st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
+
+ ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
+ fmla v4.2d, v20.2d, alphaV0
+ fmla v5.2d, v21.2d, alphaV1
+ fmla v6.2d, v22.2d, alphaV2
+ fmla v7.2d, v23.2d, alphaV3
+ st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+.endm
+
+.macro SAVE4x2
+ ld1 {v8.2d, v9.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ fmla v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v12.2d, v13.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV2
+ fmla v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+.endm
+
+.macro SAVE2x2
+ ld1 {v8.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+
+ ld1 {v12.2d}, [pCRow1]
+ fmla v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2d} , [pB]
+ add pB , pB, #16
+
+ ldr d0 , [pA]
+ add pA, pA, #8
+
+ fmla v16.2d, v8.2d, v0.2d[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+
+ ld1 {v8.d}[0], [pCRow0]
+ ld1 {v8.d}[1], [pCRow1]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, d16
+ fmov d19, d17
+.endm
+
+.macro KERNEL8x1_SUB
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA , pA, #32
+
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+.endm
+
+.macro SAVE8x1
+ ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
+ fmla v0.2d, v16.2d, alphaV0
+ fmla v1.2d, v17.2d, alphaV1
+ fmla v2.2d, v18.2d, alphaV2
+ fmla v3.2d, v19.2d, alphaV3
+ st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov d16, xzr
+ fmov d17, d16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA , pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x1
+ ld1 {v8.2d, v9.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ fmla v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+.endm
+
+.macro SAVE2x1
+ ld1 {v8.2d}, [pCRow0]
+ fmla v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ldr d0, [pA]
+ add pA , pA, #8
+
+ fmadd d16, d0, d8, d16
+.endm
+
+.macro SAVE1x1
+ ldr d8, [pCRow0]
+ fmadd d8, d16, alpha0, d8
+ str d8, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha, d0
+
+ lsl LDC, LDC, #3 // ldc = ldc * 8
+
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #2 // J = J / 4
+ cmp counterJ, #0
+ ble dgemm_kernel_L2_BEGIN
+
+/******************************************************************************/
+
+dgemm_kernel_L4_BEGIN:
+ mov pCRow0, pC
+ add pCRow1, pCRow0, LDC
+ add pCRow2, pCRow1, LDC
+ add pCRow3, pCRow2, LDC
+ add pC, pCRow3, LDC
+
+ mov pA, origPA // pA = start of A array
+
+dgemm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble dgemm_kernel_L4_M4_BEGIN
+
+dgemm_kernel_L4_M8_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // L = K / 8
+ cmp counterL , #2 // is there at least 4 to do?
+ blt dgemm_kernel_L4_M8_32
+
+ KERNEL8x4_I
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #2 // subtract 2
+ ble dgemm_kernel_L4_M8_22a
+ .align 5
+
+dgemm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M8_22
+
+
+dgemm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b dgemm_kernel_L4_M8_44
+
+dgemm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble dgemm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b dgemm_kernel_L4_M8_44
+
+dgemm_kernel_L4_M8_40:
+
+ INIT8x4
+
+dgemm_kernel_L4_M8_44:
+
+ ands counterL , origK, #7
+ ble dgemm_kernel_L4_M8_100
+
+dgemm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+ subs counterL, counterL, #1
+ bne dgemm_kernel_L4_M8_46
+
+dgemm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+dgemm_kernel_L4_M8_END:
+ subs counterI, counterI, #1
+ bne dgemm_kernel_L4_M8_20
+
+dgemm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble dgemm_kernel_L4_END
+
+ tst counterI, #4
+ ble dgemm_kernel_L4_M2_BEGIN
+
+dgemm_kernel_L4_M4_20:
+
+ INIT4x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L4_M4_40
+
+dgemm_kernel_L4_M4_22:
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M4_22
+
+dgemm_kernel_L4_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L4_M4_100
+
+dgemm_kernel_L4_M4_42:
+
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M4_42
+
+dgemm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+dgemm_kernel_L4_M4_END:
+
+
+dgemm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dgemm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dgemm_kernel_L4_M1_BEGIN
+
+dgemm_kernel_L4_M2_20:
+
+ INIT2x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L4_M2_40
+
+dgemm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M2_22
+
+
+dgemm_kernel_L4_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L4_M2_100
+
+dgemm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M2_42
+
+dgemm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+dgemm_kernel_L4_M2_END:
+
+
+dgemm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dgemm_kernel_L4_END
+
+dgemm_kernel_L4_M1_20:
+
+ INIT1x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L4_M1_40
+
+dgemm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M1_22
+
+
+dgemm_kernel_L4_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L4_M1_100
+
+dgemm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L4_M1_42
+
+dgemm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+dgemm_kernel_L4_END:
+
+ lsl temp, origK, #5
+ add origPB, origPB, temp // B = B + K * 4 * 8
+
+ subs counterJ, counterJ , #1 // j--
+ bgt dgemm_kernel_L4_BEGIN
+
+
+/******************************************************************************/
+
+dgemm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble dgemm_kernel_L999 // error, N was less than 4?
+
+ tst counterJ , #2
+ ble dgemm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+ mov pA, origPA // pA = A
+
+dgemm_kernel_L2_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble dgemm_kernel_L2_M4_BEGIN
+
+dgemm_kernel_L2_M8_20:
+
+ INIT8x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dgemm_kernel_L2_M8_40
+ .align 5
+
+dgemm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M8_22
+
+
+dgemm_kernel_L2_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L2_M8_100
+
+dgemm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M8_42
+
+dgemm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+dgemm_kernel_L2_M8_END:
+
+ subs counterI, counterI, #1
+ bgt dgemm_kernel_L2_M8_20
+
+dgemm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble dgemm_kernel_L2_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble dgemm_kernel_L2_M2_BEGIN
+
+dgemm_kernel_L2_M4_20:
+
+ INIT4x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dgemm_kernel_L2_M4_40
+ .align 5
+
+dgemm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M4_22
+
+
+dgemm_kernel_L2_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L2_M4_100
+
+dgemm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M4_42
+
+dgemm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+dgemm_kernel_L2_M4_END:
+
+
+dgemm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dgemm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dgemm_kernel_L2_M1_BEGIN
+
+dgemm_kernel_L2_M2_20:
+
+ INIT2x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dgemm_kernel_L2_M2_40
+
+dgemm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M2_22
+
+
+dgemm_kernel_L2_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L2_M2_100
+
+dgemm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M2_42
+
+dgemm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+dgemm_kernel_L2_M2_END:
+
+
+dgemm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dgemm_kernel_L2_END
+
+dgemm_kernel_L2_M1_20:
+
+ INIT1x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble dgemm_kernel_L2_M1_40
+
+dgemm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M1_22
+
+
+dgemm_kernel_L2_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L2_M1_100
+
+dgemm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L2_M1_42
+
+dgemm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+dgemm_kernel_L2_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
+
+/******************************************************************************/
+
+dgemm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble dgemm_kernel_L999 // done
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+ mov pA, origPA // pA = A
+
+dgemm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble dgemm_kernel_L1_M4_BEGIN
+
+dgemm_kernel_L1_M8_20:
+
+ INIT8x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L1_M8_40
+ .align 5
+
+dgemm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M8_22
+
+
+dgemm_kernel_L1_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L1_M8_100
+
+dgemm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M8_42
+
+dgemm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+dgemm_kernel_L1_M8_END:
+
+ subs counterI, counterI, #1
+ bgt dgemm_kernel_L1_M8_20
+
+dgemm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble dgemm_kernel_L1_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble dgemm_kernel_L1_M2_BEGIN
+
+dgemm_kernel_L1_M4_20:
+
+ INIT4x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L1_M4_40
+ .align 5
+
+dgemm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M4_22
+
+
+dgemm_kernel_L1_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L1_M4_100
+
+dgemm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M4_42
+
+dgemm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+dgemm_kernel_L1_M4_END:
+
+dgemm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dgemm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dgemm_kernel_L1_M1_BEGIN
+
+dgemm_kernel_L1_M2_20:
+
+ INIT2x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L1_M2_40
+
+dgemm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M2_22
+
+
+dgemm_kernel_L1_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L1_M2_100
+
+dgemm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M2_42
+
+dgemm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+dgemm_kernel_L1_M2_END:
+
+
+dgemm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dgemm_kernel_L1_END
+
+dgemm_kernel_L1_M1_20:
+
+ INIT1x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dgemm_kernel_L1_M1_40
+
+dgemm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M1_22
+
+
+dgemm_kernel_L1_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble dgemm_kernel_L1_M1_100
+
+dgemm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dgemm_kernel_L1_M1_42
+
+dgemm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+
+dgemm_kernel_L1_END:
+
+
+dgemm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 x7*/
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define offset x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+#define temp x16
+#define tempOffset x17
+#define tempK x18
+
+#define alpha0 d2
+#define alphaV0 v2.d[0]
+#define alpha1 d3
+#define alphaV1 v3.d[0]
+#define alpha2 d6
+#define alphaV2 v6.d[0]
+#define alpha3 d7
+#define alphaV3 v7.d[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 temp
+// 17 tempOffset
+// 18 must save tempK
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA00, pA01
+//v01 pA02, pA03
+//v02 ALPHA0
+//v03 ALPHA1
+//v04 pA10, pA11
+//v05 pA12, pA13
+//v06 ALPHA2
+//v07 ALPHA3
+//v08 must save pB0_0, pB0_1
+//v09 must save pB0_2, pB0_3
+//v10 must save pB0_4, pB0_5
+//v11 must save pB0_6, pB0_7
+//v12 must save pB1_0, pB1_1
+//v13 must save pB1_2, pB1_3
+//v14 must save pB1_4, pB1_5
+//v15 must save pB1_6, pB1_7
+//v16 must save C00, C01
+//v17 must save C02, C03
+//v18 C04, C05
+//v19 C06, C07
+//v20 C10, C11
+//v21 C12, C13
+//v22 C14, C15
+//v23 C16, C17
+//v24 C20, C21
+//v25 C22, C23
+//v26 C24, C25
+//v27 C26, C27
+//v28 C30, C31
+//v29 C32, C33
+//v30 C34, C35
+//v31 C36, C37
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT4x8
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, xzr
+ fmov d19, d16
+ fmov d20, xzr
+ fmov d21, d16
+ fmov d22, d17
+ fmov d23, d18
+ fmov d24, xzr
+ fmov d25, d16
+ fmov d26, d17
+ fmov d27, d18
+ fmov d28, xzr
+ fmov d29, d16
+ fmov d30, d17
+ fmov d31, d18
+.endm
+
+.macro KERNEL4x8_I
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmul v16.2d, v0.2d, v8.2d[0]
+ fmul v17.2d, v1.2d, v8.2d[0]
+ fmul v18.2d, v0.2d, v8.2d[1]
+ fmul v19.2d, v1.2d, v8.2d[1]
+
+ fmul v20.2d, v0.2d, v9.2d[0]
+ fmul v21.2d, v1.2d, v9.2d[0]
+ fmul v22.2d, v0.2d, v9.2d[1]
+ fmul v23.2d, v1.2d, v9.2d[1]
+
+ fmul v24.2d, v0.2d, v10.2d[0]
+ fmul v25.2d, v1.2d, v10.2d[0]
+ fmul v26.2d, v0.2d, v10.2d[1]
+ fmul v27.2d, v1.2d, v10.2d[1]
+
+ fmul v28.2d, v0.2d, v11.2d[0]
+ fmul v29.2d, v1.2d, v11.2d[0]
+ fmul v30.2d, v0.2d, v11.2d[1]
+ fmul v31.2d, v1.2d, v11.2d[1]
+
+ ld1 {v12.2d, v13.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v4.2d, v5.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v14.2d, v15.2d}, [pB]
+ add pB, pB, #32
+.endm
+
+.macro KERNEL4x8_M1
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v0.2d, v8.2d[1]
+ fmla v19.2d, v1.2d, v8.2d[1]
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v9.2d[0]
+ fmla v22.2d, v0.2d, v9.2d[1]
+ fmla v23.2d, v1.2d, v9.2d[1]
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v25.2d, v1.2d, v10.2d[0]
+ fmla v26.2d, v0.2d, v10.2d[1]
+ fmla v27.2d, v1.2d, v10.2d[1]
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v29.2d, v1.2d, v11.2d[0]
+ fmla v30.2d, v0.2d, v11.2d[1]
+ fmla v31.2d, v1.2d, v11.2d[1]
+
+ ld1 {v12.2d, v13.2d}, [pB] // For next round
+ add pB, pB, #32
+ ld1 {v4.2d, v5.2d}, [pA] // For next round
+ add pA, pA, #32
+ ld1 {v14.2d, v15.2d}, [pB]
+ add pB, pB, #32
+
+ prfm PLDL1KEEP, [pA, #512]
+.endm
+
+.macro KERNEL4x8_M2
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+ fmla v18.2d, v4.2d, v12.2d[1]
+ fmla v19.2d, v5.2d, v12.2d[1]
+
+ fmla v20.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v13.2d[0]
+ fmla v22.2d, v4.2d, v13.2d[1]
+ fmla v23.2d, v5.2d, v13.2d[1]
+
+ fmla v24.2d, v4.2d, v14.2d[0]
+ fmla v25.2d, v5.2d, v14.2d[0]
+ fmla v26.2d, v4.2d, v14.2d[1]
+ fmla v27.2d, v5.2d, v14.2d[1]
+
+ fmla v28.2d, v4.2d, v15.2d[0]
+ fmla v29.2d, v5.2d, v15.2d[0]
+ fmla v30.2d, v4.2d, v15.2d[1]
+ fmla v31.2d, v5.2d, v15.2d[1]
+
+ ld1 {v8.2d, v9.2d}, [pB] // For next round
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA] // For next round
+ add pA, pA, #32
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ prfm PLDL1KEEP, [pB, #512]
+.endm
+
+.macro KERNEL4x8_E
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+ fmla v18.2d, v4.2d, v12.2d[1]
+ fmla v19.2d, v5.2d, v12.2d[1]
+
+ fmla v20.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v13.2d[0]
+ fmla v22.2d, v4.2d, v13.2d[1]
+ fmla v23.2d, v5.2d, v13.2d[1]
+
+ fmla v24.2d, v4.2d, v14.2d[0]
+ fmla v25.2d, v5.2d, v14.2d[0]
+ fmla v26.2d, v4.2d, v14.2d[1]
+ fmla v27.2d, v5.2d, v14.2d[1]
+
+ fmla v28.2d, v4.2d, v15.2d[0]
+ fmla v29.2d, v5.2d, v15.2d[0]
+ fmla v30.2d, v4.2d, v15.2d[1]
+ fmla v31.2d, v5.2d, v15.2d[1]
+.endm
+
+.macro KERNEL4x8_SUB
+ ld1 {v8.2d, v9.2d}, [pB] // For next round
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA] // For next round
+ add pA, pA, #32
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v0.2d, v8.2d[1]
+ fmla v19.2d, v1.2d, v8.2d[1]
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v9.2d[0]
+ fmla v22.2d, v0.2d, v9.2d[1]
+ fmla v23.2d, v1.2d, v9.2d[1]
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v25.2d, v1.2d, v10.2d[0]
+ fmla v26.2d, v0.2d, v10.2d[1]
+ fmla v27.2d, v1.2d, v10.2d[1]
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v29.2d, v1.2d, v11.2d[0]
+ fmla v30.2d, v0.2d, v11.2d[1]
+ fmla v31.2d, v1.2d, v11.2d[1]
+.endm
+
+.macro SAVE4x8
+ add pCRow1, pCRow0, LDC
+
+ fmul v8.2d, v16.2d, alphaV0
+ fmul v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v10.2d, v18.2d, alphaV2
+ fmul v11.2d, v19.2d, alphaV3
+ st1 {v10.2d, v11.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v20.2d, alphaV0
+ fmul v13.2d, v21.2d, alphaV1
+ st1 {v12.2d, v13.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v14.2d, v22.2d, alphaV2
+ fmul v15.2d, v23.2d, alphaV3
+ st1 {v14.2d, v15.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v8.2d, v24.2d, alphaV0
+ fmul v9.2d, v25.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v10.2d, v26.2d, alphaV2
+ fmul v11.2d, v27.2d, alphaV3
+ st1 {v10.2d, v11.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v28.2d, alphaV0
+ fmul v13.2d, v29.2d, alphaV1
+ st1 {v12.2d, v13.2d}, [pCRow2]
+
+ fmul v14.2d, v30.2d, alphaV2
+ fmul v15.2d, v31.2d, alphaV3
+ st1 {v14.2d, v15.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x8
+ fmov d16, xzr
+ fmov d18, xzr
+ fmov d20, xzr
+ fmov d22, d16
+ fmov d24, xzr
+ fmov d26, d16
+ fmov d28, xzr
+ fmov d30, d16
+.endm
+
+.macro KERNEL2x8_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v18.2d, v0.2d, v8.2d[1]
+
+ fmla v20.2d, v0.2d, v9.2d[0]
+ fmla v22.2d, v0.2d, v9.2d[1]
+
+ fmla v24.2d, v0.2d, v10.2d[0]
+ fmla v26.2d, v0.2d, v10.2d[1]
+
+ fmla v28.2d, v0.2d, v11.2d[0]
+ fmla v30.2d, v0.2d, v11.2d[1]
+.endm
+
+.macro SAVE2x8
+ add pCRow1, pCRow0, LDC
+
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v10.2d, v18.2d, alphaV2
+ st1 {v10.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v20.2d, alphaV0
+ st1 {v12.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v14.2d, v22.2d, alphaV2
+ st1 {v14.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v8.2d, v24.2d, alphaV0
+ st1 {v8.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v10.2d, v26.2d, alphaV2
+ st1 {v10.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v28.2d, alphaV0
+ st1 {v12.2d}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v14.2d, v30.2d, alphaV2
+ st1 {v14.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x8
+ fmov d16, xzr
+ fmov d20, xzr
+ fmov d24, xzr
+ fmov d28, xzr
+.endm
+
+.macro KERNEL1x8_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ldr d0, [pA]
+ add pA, pA, #8
+ ld1 {v10.2d, v11.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v8.2d, v0.d[0]
+ fmla v20.2d, v9.2d, v0.d[0]
+ fmla v24.2d, v10.2d, v0.d[0]
+ fmla v28.2d, v11.2d, v0.d[0]
+.endm
+
+.macro SAVE1x8
+ add pCRow1, pCRow0, LDC
+
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ fmul v10.2d, v20.2d, alphaV1
+ st1 {v10.d}[0], [pCRow2]
+ st1 {v10.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v24.2d, alphaV2
+ st1 {v12.d}[0], [pCRow2]
+ st1 {v12.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ fmul v14.2d, v28.2d, alphaV3
+ st1 {v14.d}[0], [pCRow2]
+ st1 {v14.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+ fmov d24, d17
+ fmov d25, d16
+ fmov d28, d17
+ fmov d29, d16
+.endm
+
+.macro KERNEL4x4_I
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmul v16.2d, v0.2d, v8.2d[0]
+ fmul v29.2d, v1.2d, v9.2d[1]
+
+ fmul v20.2d, v0.2d, v8.2d[1]
+ fmul v25.2d, v1.2d, v9.2d[0]
+
+ fmul v24.2d, v0.2d, v9.2d[0]
+ fmul v21.2d, v1.2d, v8.2d[1]
+
+ fmul v28.2d, v0.2d, v9.2d[1]
+ fmul v17.2d, v1.2d, v8.2d[0]
+
+ ld1 {v12.2d, v13.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v4.2d, v5.2d}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL4x4_M1
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v29.2d, v1.2d, v9.2d[1]
+
+ ld1 {v12.2d, v13.2d}, [pB] // For next round
+ add pB, pB, #32
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v25.2d, v1.2d, v9.2d[0]
+
+ ld1 {v4.2d, v5.2d}, [pA] // For next round
+ add pA, pA, #32
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v8.2d[1]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro KERNEL4x4_M2
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v29.2d, v5.2d, v13.2d[1]
+
+ ld1 {v8.2d, v9.2d}, [pB] // For next round
+ add pB, pB, #32
+
+ fmla v20.2d, v4.2d, v12.2d[1]
+ fmla v25.2d, v5.2d, v13.2d[0]
+
+ ld1 {v0.2d, v1.2d}, [pA] // For next round
+ add pA, pA, #32
+
+ fmla v24.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v12.2d[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ fmla v28.2d, v4.2d, v13.2d[1]
+ fmla v17.2d, v5.2d, v12.2d[0]
+.endm
+
+.macro KERNEL4x4_E
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v29.2d, v5.2d, v13.2d[1]
+
+ fmla v20.2d, v4.2d, v12.2d[1]
+ fmla v25.2d, v5.2d, v13.2d[0]
+
+ fmla v24.2d, v4.2d, v13.2d[0]
+ fmla v21.2d, v5.2d, v12.2d[1]
+
+ fmla v28.2d, v4.2d, v13.2d[1]
+ fmla v17.2d, v5.2d, v12.2d[0]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v29.2d, v1.2d, v9.2d[1]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v25.2d, v1.2d, v9.2d[0]
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v8.2d[1]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x4
+ fmul v8.2d, v16.2d, alphaV0
+ fmul v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV2
+ fmul v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v8.2d, v24.2d, alphaV0
+ fmul v9.2d, v25.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v28.2d, alphaV2
+ fmul v13.2d, v29.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov d16, xzr
+ fmov d20, d16
+ fmov d24, d20
+ fmov d28, d16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v28.2d, v0.2d, v9.2d[1]
+.endm
+
+.macro SAVE2x4
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v8.2d, v24.2d, alphaV2
+ st1 {v8.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v28.2d, alphaV3
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr d0, [pA]
+ add pA, pA, #8
+
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v8.2d, v0.d[0]
+ fmla v20.2d, v9.2d, v0.d[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v20.2d, alphaV1
+ st1 {v12.d}[0], [pCRow2]
+ st1 {v12.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+.endm
+
+.macro SAVE4x2
+ fmul v8.2d, v16.2d, alphaV0
+ fmul v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV2
+ fmul v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+.endm
+
+.macro SAVE2x2
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2d} , [pB]
+ add pB , pB, #16
+
+ ldr d0 , [pA]
+ add pA, pA, #8
+
+ fmla v16.2d, v8.2d, v0.2d[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov d16, xzr
+ fmov d17, d16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA , pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x1
+ fmul v8.2d, v16.2d, alphaV0
+ fmul v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+
+
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+.endm
+
+.macro SAVE2x1
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ldr d0, [pA]
+ add pA , pA, #8
+
+ fmadd d16, d0, d8, d16
+.endm
+
+.macro SAVE1x1
+ fmul d8, d16, alpha0
+ str d8, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0, d0
+ fmov alpha1, d0
+ fmov alpha2, d0
+ fmov alpha3, d0
+
+ lsl LDC, LDC, #3 // ldc = ldc * 8
+
+#if !defined(LEFT)
+ neg tempOffset, offset
+#endif
+
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #3 // J = J / 8
+ cmp counterJ, #0
+ ble dtrmm_kernel_L4_BEGIN
+
+/******************************************************************************/
+
+dtrmm_kernel_L8_BEGIN:
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #3
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+
+ mov pA, origPA // pA = start of A array
+
+dtrmm_kernel_L8_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI, #0
+ ble dtrmm_kernel_L8_M2_BEGIN
+
+dtrmm_kernel_L8_M4_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ lsl temp, tempOffset, #6
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #8
+#endif
+
+ asr counterL, tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt dtrmm_kernel_L8_M4_32
+
+ KERNEL4x8_I // do one in the K
+ KERNEL4x8_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble dtrmm_kernel_L8_M4_22a
+ .align 5
+
+dtrmm_kernel_L8_M4_22:
+
+ KERNEL4x8_M1
+ KERNEL4x8_M2
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L8_M4_22
+
+
+dtrmm_kernel_L8_M4_22a:
+
+ KERNEL4x8_M1
+ KERNEL4x8_E
+
+ b dtrmm_kernel_L8_M4_44
+
+dtrmm_kernel_L8_M4_32:
+
+ tst counterL, #1
+ ble dtrmm_kernel_L8_M4_40
+
+ KERNEL4x8_I
+
+ KERNEL4x8_E
+
+ b dtrmm_kernel_L8_M4_44
+
+
+dtrmm_kernel_L8_M4_40:
+
+ INIT4x8
+
+dtrmm_kernel_L8_M4_44:
+
+ ands counterL, tempK, #1
+ ble dtrmm_kernel_L8_M4_100
+
+dtrmm_kernel_L8_M4_46:
+
+ KERNEL4x8_SUB
+
+dtrmm_kernel_L8_M4_100:
+
+ SAVE4x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #8
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #6
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+dtrmm_kernel_L8_M4_END:
+ subs counterI, counterI, #1
+ bne dtrmm_kernel_L8_M4_20
+
+dtrmm_kernel_L8_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dtrmm_kernel_L8_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dtrmm_kernel_L8_M1_BEGIN
+
+dtrmm_kernel_L8_M2_20:
+
+ INIT2x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+ lsl temp, tempOffset, #6
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #8
+#endif
+
+ asr counterL, tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L8_M2_40
+
+dtrmm_kernel_L8_M2_22:
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L8_M2_22
+
+
+dtrmm_kernel_L8_M2_40:
+
+ ands counterL, tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L8_M2_100
+
+dtrmm_kernel_L8_M2_42:
+
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L8_M2_42
+
+dtrmm_kernel_L8_M2_100:
+
+ SAVE2x8
+
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #8
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #6
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+dtrmm_kernel_L8_M2_END:
+
+
+dtrmm_kernel_L8_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dtrmm_kernel_L8_END
+
+dtrmm_kernel_L8_M1_20:
+
+ INIT1x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+ lsl temp, tempOffset, #6
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #8
+#endif
+
+ asr counterL, tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L8_M1_40
+
+dtrmm_kernel_L8_M1_22:
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L8_M1_22
+
+
+dtrmm_kernel_L8_M1_40:
+
+ ands counterL, tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L8_M1_100
+
+dtrmm_kernel_L8_M1_42:
+
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L8_M1_42
+
+dtrmm_kernel_L8_M1_100:
+
+ SAVE1x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #8
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #6
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+
+dtrmm_kernel_L8_END:
+
+ lsl temp, origK, #6
+ add origPB, origPB, temp // B = B + K * 8 * 8
+
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+ subs counterJ, counterJ , #1 // j--
+ bgt dtrmm_kernel_L8_BEGIN
+
+
+/******************************************************************************/
+
+dtrmm_kernel_L4_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #7
+ ble dtrmm_kernel_L999
+
+ tst counterJ , #4
+ ble dtrmm_kernel_L2_BEGIN
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+
+ mov pA, origPA // pA = start of A array
+
+dtrmm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI, #0
+ ble dtrmm_kernel_L4_M2_BEGIN
+
+dtrmm_kernel_L4_M4_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL, tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt dtrmm_kernel_L4_M4_32
+
+ KERNEL4x4_I // do one in the K
+ KERNEL4x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble dtrmm_kernel_L4_M4_22a
+ .align 5
+
+dtrmm_kernel_L4_M4_22:
+
+ KERNEL4x4_M1
+ KERNEL4x4_M2
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M4_22
+
+
+dtrmm_kernel_L4_M4_22a:
+
+ KERNEL4x4_M1
+ KERNEL4x4_E
+
+ b dtrmm_kernel_L4_M4_44
+
+dtrmm_kernel_L4_M4_32:
+
+ tst counterL, #1
+ ble dtrmm_kernel_L4_M4_40
+
+ KERNEL4x4_I
+
+ KERNEL4x4_E
+
+ b dtrmm_kernel_L4_M4_44
+
+
+dtrmm_kernel_L4_M4_40:
+
+ INIT4x4
+
+dtrmm_kernel_L4_M4_44:
+
+ ands counterL , tempK, #1
+ ble dtrmm_kernel_L4_M4_100
+
+dtrmm_kernel_L4_M4_46:
+
+ KERNEL4x4_SUB
+
+dtrmm_kernel_L4_M4_100:
+
+ SAVE4x4
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+dtrmm_kernel_L4_M4_END:
+ subs counterI, counterI, #1
+ bne dtrmm_kernel_L4_M4_20
+
+dtrmm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dtrmm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dtrmm_kernel_L4_M1_BEGIN
+
+dtrmm_kernel_L4_M2_20:
+
+ INIT2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L4_M2_40
+
+dtrmm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M2_22
+
+
+dtrmm_kernel_L4_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L4_M2_100
+
+dtrmm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M2_42
+
+dtrmm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+dtrmm_kernel_L4_M2_END:
+
+
+dtrmm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dtrmm_kernel_L4_END
+
+dtrmm_kernel_L4_M1_20:
+
+ INIT1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L4_M1_40
+
+dtrmm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M1_22
+
+
+dtrmm_kernel_L4_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L4_M1_100
+
+dtrmm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M1_42
+
+dtrmm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+dtrmm_kernel_L4_END:
+
+ lsl temp, origK, #5
+ add origPB, origPB, temp // B = B + K * 4 * 8
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+/******************************************************************************/
+
+dtrmm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble dtrmm_kernel_L999 // error, N was less than 4?
+
+ tst counterJ , #2
+ ble dtrmm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+
+dtrmm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI,#0
+ ble dtrmm_kernel_L2_M2_BEGIN
+
+dtrmm_kernel_L2_M4_20:
+
+ INIT4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dtrmm_kernel_L2_M4_40
+ .align 5
+
+dtrmm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M4_22
+
+
+dtrmm_kernel_L2_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L2_M4_100
+
+dtrmm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M4_42
+
+dtrmm_kernel_L2_M4_100:
+
+ SAVE4x2
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+dtrmm_kernel_L2_M4_END:
+
+ subs counterI, counterI, #1
+ bgt dtrmm_kernel_L2_M4_20
+
+
+dtrmm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dtrmm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dtrmm_kernel_L2_M1_BEGIN
+
+dtrmm_kernel_L2_M2_20:
+
+ INIT2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dtrmm_kernel_L2_M2_40
+
+dtrmm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M2_22
+
+
+dtrmm_kernel_L2_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L2_M2_100
+
+dtrmm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M2_42
+
+dtrmm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+dtrmm_kernel_L2_M2_END:
+
+
+dtrmm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dtrmm_kernel_L2_END
+
+dtrmm_kernel_L2_M1_20:
+
+ INIT1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble dtrmm_kernel_L2_M1_40
+
+dtrmm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M1_22
+
+
+dtrmm_kernel_L2_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L2_M1_100
+
+dtrmm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M1_42
+
+dtrmm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+dtrmm_kernel_L2_END:
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+ add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
+
+/******************************************************************************/
+
+dtrmm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble dtrmm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+dtrmm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI, #0
+ ble dtrmm_kernel_L1_M2_BEGIN
+
+dtrmm_kernel_L1_M4_20:
+
+ INIT4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+#endif
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L1_M4_40
+ .align 5
+
+dtrmm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M4_22
+
+
+dtrmm_kernel_L1_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L1_M4_100
+
+dtrmm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M4_42
+
+dtrmm_kernel_L1_M4_100:
+
+ SAVE4x1
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+dtrmm_kernel_L1_M4_END:
+
+ subs counterI, counterI, #1
+ bgt dtrmm_kernel_L1_M4_20
+
+
+dtrmm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dtrmm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dtrmm_kernel_L1_M1_BEGIN
+
+dtrmm_kernel_L1_M2_20:
+
+ INIT2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L1_M2_40
+
+dtrmm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M2_22
+
+
+dtrmm_kernel_L1_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L1_M2_100
+
+dtrmm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M2_42
+
+dtrmm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+dtrmm_kernel_L1_M2_END:
+
+
+dtrmm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dtrmm_kernel_L1_END
+
+dtrmm_kernel_L1_M1_20:
+
+ INIT1x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L1_M1_40
+
+dtrmm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M1_22
+
+
+dtrmm_kernel_L1_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L1_M1_100
+
+dtrmm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M1_42
+
+dtrmm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+
+dtrmm_kernel_L1_END:
+
+
+dtrmm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 x7*/
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define offset x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+#define temp x16
+#define tempOffset x17
+#define tempK x18
+
+#define alpha0 d10
+#define alphaV0 v10.d[0]
+#define alpha1 d11
+#define alphaV1 v11.d[0]
+#define alpha2 d14
+#define alphaV2 v14.d[0]
+#define alpha3 d15
+#define alphaV3 v15.d[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 temp
+// 17 tempOffset
+// 18 must save tempK
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA0_0, pA0_1
+//v01 pA0_2, pA0_3
+//v02 pA0_4, pA0_5
+//v03 pA0_6, pA0_7
+//v04 pA1_0, pA1_1
+//v05 pA1_2, pA1_3
+//v06 pA1_4, pA1_5
+//v07 pA1_6, pA1_7
+//v08 must save pB0_0, pB0_1
+//v09 must save pB0_2, pB0_3
+//v10 must save ALPHA0
+//v11 must save ALPHA1
+//v12 must save pB1_0, pB1_1
+//v13 must save pB1_2, pB1_3
+//v14 must save ALPHA2
+//v15 must save ALPHA3
+//v16 must save C00, C01
+//v17 must save C02, C03
+//v18 C04, C05
+//v19 C06, C07
+//v20 C10, C11
+//v21 C12, C13
+//v22 C14, C15
+//v23 C16, C17
+//v24 C20, C21
+//v25 C22, C23
+//v26 C24, C25
+//v27 C26, C27
+//v28 C30, C31
+//v29 C32, C33
+//v30 C34, C35
+//v31 C36, C37
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT8x4
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, d16
+ fmov d19, xzr
+ fmov d20, xzr
+ fmov d21, d16
+ fmov d22, d17
+ fmov d23, d18
+ fmov d24, xzr
+ fmov d25, d16
+ fmov d26, d17
+ fmov d27, d18
+ fmov d28, xzr
+ fmov d29, d16
+ fmov d30, d17
+ fmov d31, d18
+.endm
+
+.macro KERNEL8x4_I
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+
+ fmul v16.2d, v0.2d, v8.2d[0]
+ fmul v17.2d, v1.2d, v8.2d[0]
+ fmul v18.2d, v2.2d, v8.2d[0]
+ fmul v19.2d, v3.2d, v8.2d[0]
+
+ fmul v20.2d, v0.2d, v8.2d[1]
+ fmul v21.2d, v1.2d, v8.2d[1]
+ fmul v22.2d, v2.2d, v8.2d[1]
+ fmul v23.2d, v3.2d, v8.2d[1]
+
+ fmul v24.2d, v0.2d, v9.2d[0]
+ fmul v25.2d, v1.2d, v9.2d[0]
+ fmul v26.2d, v2.2d, v9.2d[0]
+ fmul v27.2d, v3.2d, v9.2d[0]
+
+ fmul v28.2d, v0.2d, v9.2d[1]
+ fmul v29.2d, v1.2d, v9.2d[1]
+ fmul v30.2d, v2.2d, v9.2d[1]
+ fmul v31.2d, v3.2d, v9.2d[1]
+
+ ld1 {v4.2d, v5.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v12.2d, v13.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v6.2d, v7.2d}, [pA]
+ add pA, pA, #32
+.endm
+
+.macro KERNEL8x4_M1
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+ fmla v22.2d, v2.2d, v8.2d[1]
+ fmla v23.2d, v3.2d, v8.2d[1]
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v25.2d, v1.2d, v9.2d[0]
+ fmla v26.2d, v2.2d, v9.2d[0]
+ fmla v27.2d, v3.2d, v9.2d[0]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v29.2d, v1.2d, v9.2d[1]
+ fmla v30.2d, v2.2d, v9.2d[1]
+ fmla v31.2d, v3.2d, v9.2d[1]
+
+ ld1 {v4.2d, v5.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v12.2d, v13.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v6.2d, v7.2d}, [pA]
+ add pA, pA, #32
+
+ prfm PLDL1KEEP, [pA, #512]
+.endm
+
+.macro KERNEL8x4_M2
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+ fmla v18.2d, v6.2d, v12.2d[0]
+ fmla v19.2d, v7.2d, v12.2d[0]
+
+ fmla v20.2d, v4.2d, v12.2d[1]
+ fmla v21.2d, v5.2d, v12.2d[1]
+ fmla v22.2d, v6.2d, v12.2d[1]
+ fmla v23.2d, v7.2d, v12.2d[1]
+
+ fmla v24.2d, v4.2d, v13.2d[0]
+ fmla v25.2d, v5.2d, v13.2d[0]
+ fmla v26.2d, v6.2d, v13.2d[0]
+ fmla v27.2d, v7.2d, v13.2d[0]
+
+ fmla v28.2d, v4.2d, v13.2d[1]
+ fmla v29.2d, v5.2d, v13.2d[1]
+ fmla v30.2d, v6.2d, v13.2d[1]
+ fmla v31.2d, v7.2d, v13.2d[1]
+
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+
+ prfm PLDL1KEEP, [pB, #512]
+.endm
+
+.macro KERNEL8x4_E
+ fmla v16.2d, v4.2d, v12.2d[0]
+ fmla v17.2d, v5.2d, v12.2d[0]
+ fmla v18.2d, v6.2d, v12.2d[0]
+ fmla v19.2d, v7.2d, v12.2d[0]
+
+ fmla v20.2d, v4.2d, v12.2d[1]
+ fmla v21.2d, v5.2d, v12.2d[1]
+ fmla v22.2d, v6.2d, v12.2d[1]
+ fmla v23.2d, v7.2d, v12.2d[1]
+
+ fmla v24.2d, v4.2d, v13.2d[0]
+ fmla v25.2d, v5.2d, v13.2d[0]
+ fmla v26.2d, v6.2d, v13.2d[0]
+ fmla v27.2d, v7.2d, v13.2d[0]
+
+ fmla v28.2d, v4.2d, v13.2d[1]
+ fmla v29.2d, v5.2d, v13.2d[1]
+ fmla v30.2d, v6.2d, v13.2d[1]
+ fmla v31.2d, v7.2d, v13.2d[1]
+.endm
+
+.macro KERNEL8x4_SUB
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+ fmla v22.2d, v2.2d, v8.2d[1]
+ fmla v23.2d, v3.2d, v8.2d[1]
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v25.2d, v1.2d, v9.2d[0]
+ fmla v26.2d, v2.2d, v9.2d[0]
+ fmla v27.2d, v3.2d, v9.2d[0]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v29.2d, v1.2d, v9.2d[1]
+ fmla v30.2d, v2.2d, v9.2d[1]
+ fmla v31.2d, v3.2d, v9.2d[1]
+.endm
+
+.macro SAVE8x4
+ add pCRow1, pCRow0, LDC
+
+ fmul v0.2d, v16.2d, alphaV0
+ fmul v1.2d, v17.2d, alphaV1
+ fmul v2.2d, v18.2d, alphaV2
+ fmul v3.2d, v19.2d, alphaV3
+ st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v4.2d, v20.2d, alphaV0
+ fmul v5.2d, v21.2d, alphaV1
+ fmul v6.2d, v22.2d, alphaV2
+ fmul v7.2d, v23.2d, alphaV3
+ st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v0.2d, v24.2d, alphaV0
+ fmul v1.2d, v25.2d, alphaV1
+ fmul v2.2d, v26.2d, alphaV2
+ fmul v3.2d, v27.2d, alphaV3
+ st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow2]
+
+ fmul v4.2d, v28.2d, alphaV0
+ fmul v5.2d, v29.2d, alphaV1
+ fmul v6.2d, v30.2d, alphaV2
+ fmul v7.2d, v31.2d, alphaV3
+ st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+ fmov d24, d17
+ fmov d25, d16
+ fmov d28, d17
+ fmov d29, d16
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v29.2d, v1.2d, v9.2d[1]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v25.2d, v1.2d, v9.2d[0]
+
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v21.2d, v1.2d, v8.2d[1]
+
+ fmla v28.2d, v0.2d, v9.2d[1]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x4
+ fmul v8.2d, v16.2d, alphaV0
+ fmul v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV2
+ fmul v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v8.2d, v24.2d, alphaV0
+ fmul v9.2d, v25.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v28.2d, alphaV2
+ fmul v13.2d, v29.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+
+.macro INIT2x4
+ fmov d16, xzr
+ fmov d20, d16
+ fmov d24, d20
+ fmov d28, d16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v24.2d, v0.2d, v9.2d[0]
+ fmla v28.2d, v0.2d, v9.2d[1]
+.endm
+
+.macro SAVE2x4
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v8.2d, v24.2d, alphaV2
+ st1 {v8.2d}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v28.2d, alphaV3
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr d0, [pA]
+ add pA, pA, #8
+
+ ld1 {v8.2d, v9.2d}, [pB]
+ add pB, pB, #32
+
+ fmla v16.2d, v8.2d, v0.d[0]
+ fmla v20.2d, v9.2d, v0.d[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2d, v20.2d, alphaV1
+ st1 {v12.d}[0], [pCRow2]
+ st1 {v12.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, d16
+ fmov d19, d17
+ fmov d20, xzr
+ fmov d21, d16
+ fmov d22, d17
+ fmov d23, d18
+.endm
+
+.macro KERNEL8x2_SUB
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+ fmla v22.2d, v2.2d, v8.2d[1]
+ fmla v23.2d, v3.2d, v8.2d[1]
+.endm
+
+.macro SAVE8x2
+ add pCRow1, pCRow0, LDC
+
+ fmul v0.2d, v16.2d, alphaV0
+ fmul v1.2d, v17.2d, alphaV1
+ fmul v2.2d, v18.2d, alphaV2
+ fmul v3.2d, v19.2d, alphaV3
+ st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
+
+ fmul v4.2d, v20.2d, alphaV0
+ fmul v5.2d, v21.2d, alphaV1
+ fmul v6.2d, v22.2d, alphaV2
+ fmul v7.2d, v23.2d, alphaV3
+ st1 {v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov d16, xzr
+ fmov d17, d16
+ fmov d20, d17
+ fmov d21, d16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+ fmla v21.2d, v1.2d, v8.2d[1]
+.endm
+
+.macro SAVE4x2
+ fmul v8.2d, v16.2d, alphaV0
+ fmul v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV2
+ fmul v13.2d, v21.2d, alphaV3
+ st1 {v12.2d, v13.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov d16, xzr
+ fmov d20, d16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2d}, [pB]
+ add pB, pB, #16
+
+ ld1 {v0.2d}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v20.2d, v0.2d, v8.2d[1]
+.endm
+
+.macro SAVE2x2
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+
+ fmul v12.2d, v20.2d, alphaV1
+ st1 {v12.2d}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2d} , [pB]
+ add pB , pB, #16
+
+ ldr d0 , [pA]
+ add pA, pA, #8
+
+ fmla v16.2d, v8.2d, v0.2d[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.d}[0], [pCRow0]
+ st1 {v8.d}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov d16, xzr
+ fmov d17, xzr
+ fmov d18, d16
+ fmov d19, d17
+.endm
+
+.macro KERNEL8x1_SUB
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA , pA, #32
+
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v2.2d, v3.2d}, [pA]
+ add pA, pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+ fmla v18.2d, v2.2d, v8.2d[0]
+ fmla v19.2d, v3.2d, v8.2d[0]
+.endm
+
+.macro SAVE8x1
+ fmul v0.2d, v16.2d, alphaV0
+ fmul v1.2d, v17.2d, alphaV1
+ fmul v2.2d, v18.2d, alphaV2
+ fmul v3.2d, v19.2d, alphaV3
+ st1 {v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov d16, xzr
+ fmov d17, d16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d, v1.2d}, [pA]
+ add pA , pA, #32
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+ fmla v17.2d, v1.2d, v8.2d[0]
+.endm
+
+.macro SAVE4x1
+ fmul v8.2d, v16.2d, alphaV0
+ fmul v9.2d, v17.2d, alphaV1
+ st1 {v8.2d, v9.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ld1 {v0.2d}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2d, v0.2d, v8.2d[0]
+.endm
+
+.macro SAVE2x1
+ fmul v8.2d, v16.2d, alphaV0
+ st1 {v8.2d}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov d16, xzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr d8, [pB]
+ add pB , pB, #8
+
+ ldr d0, [pA]
+ add pA , pA, #8
+
+ fmadd d16, d0, d8, d16
+.endm
+
+.macro SAVE1x1
+ fmul d8, d16, alpha0
+ str d8, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0, d0
+ fmov alpha1, d0
+ fmov alpha2, d0
+ fmov alpha3, d0
+
+ lsl LDC, LDC, #3 // ldc = ldc * 8
+
+#if !defined(LEFT)
+ neg tempOffset, offset
+#endif
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #2 // J = J / 4
+ cmp counterJ, #0
+ ble dtrmm_kernel_L2_BEGIN
+
+/******************************************************************************/
+
+dtrmm_kernel_L4_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = start of A array
+
+dtrmm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble dtrmm_kernel_L4_M4_BEGIN
+
+dtrmm_kernel_L4_M8_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt dtrmm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2 // subtract 2
+ ble dtrmm_kernel_L4_M8_22a
+ .align 5
+
+dtrmm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M8_22
+
+
+dtrmm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b dtrmm_kernel_L4_M8_44
+
+dtrmm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble dtrmm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+
+ KERNEL8x4_E
+
+ b dtrmm_kernel_L4_M8_44
+
+dtrmm_kernel_L4_M8_40:
+
+ INIT8x4
+
+dtrmm_kernel_L4_M8_44:
+
+ ands counterL , tempK, #1
+ ble dtrmm_kernel_L4_M8_100
+
+dtrmm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+dtrmm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+dtrmm_kernel_L4_M8_END:
+ subs counterI, counterI, #1
+ bne dtrmm_kernel_L4_M8_20
+
+dtrmm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble dtrmm_kernel_L4_END
+
+ tst counterI, #4
+ ble dtrmm_kernel_L4_M2_BEGIN
+
+dtrmm_kernel_L4_M4_20:
+
+ INIT4x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L4_M4_40
+
+dtrmm_kernel_L4_M4_22:
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M4_22
+
+
+dtrmm_kernel_L4_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L4_M4_100
+
+dtrmm_kernel_L4_M4_42:
+
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M4_42
+
+dtrmm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+dtrmm_kernel_L4_M4_END:
+
+
+dtrmm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dtrmm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dtrmm_kernel_L4_M1_BEGIN
+
+dtrmm_kernel_L4_M2_20:
+
+ INIT2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L4_M2_40
+
+dtrmm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M2_22
+
+
+dtrmm_kernel_L4_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L4_M2_100
+
+dtrmm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M2_42
+
+dtrmm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+dtrmm_kernel_L4_M2_END:
+
+
+dtrmm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dtrmm_kernel_L4_END
+
+dtrmm_kernel_L4_M1_20:
+
+ INIT1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L4_M1_40
+
+dtrmm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M1_22
+
+
+dtrmm_kernel_L4_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L4_M1_100
+
+dtrmm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L4_M1_42
+
+dtrmm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+
+dtrmm_kernel_L4_END:
+
+ lsl temp, origK, #5
+ add origPB, origPB, temp // B = B + K * 4 * 8
+
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+ subs counterJ, counterJ , #1 // j--
+ bgt dtrmm_kernel_L4_BEGIN
+
+
+/******************************************************************************/
+
+dtrmm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble dtrmm_kernel_L999 // error, N was less than 4?
+
+ tst counterJ , #2
+ ble dtrmm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+dtrmm_kernel_L2_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble dtrmm_kernel_L2_M4_BEGIN
+
+dtrmm_kernel_L2_M8_20:
+
+ INIT8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dtrmm_kernel_L2_M8_40
+ .align 5
+
+dtrmm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M8_22
+
+
+dtrmm_kernel_L2_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L2_M8_100
+
+dtrmm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M8_42
+
+dtrmm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+dtrmm_kernel_L2_M8_END:
+
+ subs counterI, counterI, #1
+ bgt dtrmm_kernel_L2_M8_20
+
+dtrmm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble dtrmm_kernel_L2_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble dtrmm_kernel_L2_M2_BEGIN
+
+dtrmm_kernel_L2_M4_20:
+
+ INIT4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dtrmm_kernel_L2_M4_40
+ .align 5
+
+dtrmm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M4_22
+
+
+dtrmm_kernel_L2_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L2_M4_100
+
+dtrmm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M4_42
+
+dtrmm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+dtrmm_kernel_L2_M4_END:
+
+
+dtrmm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dtrmm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dtrmm_kernel_L2_M1_BEGIN
+
+dtrmm_kernel_L2_M2_20:
+
+ INIT2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble dtrmm_kernel_L2_M2_40
+
+dtrmm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M2_22
+
+
+dtrmm_kernel_L2_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L2_M2_100
+
+dtrmm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M2_42
+
+dtrmm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+dtrmm_kernel_L2_M2_END:
+
+
+dtrmm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dtrmm_kernel_L2_END
+
+dtrmm_kernel_L2_M1_20:
+
+ INIT1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble dtrmm_kernel_L2_M1_40
+
+dtrmm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M1_22
+
+
+dtrmm_kernel_L2_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L2_M1_100
+
+dtrmm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L2_M1_42
+
+dtrmm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+
+dtrmm_kernel_L2_END:
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+ add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
+
+/******************************************************************************/
+
+dtrmm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble dtrmm_kernel_L999 // done
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+dtrmm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble dtrmm_kernel_L1_M4_BEGIN
+
+dtrmm_kernel_L1_M8_20:
+
+ INIT8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L1_M8_40
+ .align 5
+
+dtrmm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M8_22
+
+
+dtrmm_kernel_L1_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L1_M8_100
+
+dtrmm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M8_42
+
+dtrmm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+dtrmm_kernel_L1_M8_END:
+
+ subs counterI, counterI, #1
+ bgt dtrmm_kernel_L1_M8_20
+
+dtrmm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble dtrmm_kernel_L1_END
+
+ tst counterI, #4 // counterI = counterI / 2
+ ble dtrmm_kernel_L1_M2_BEGIN
+
+dtrmm_kernel_L1_M4_20:
+
+ INIT4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+#endif
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L1_M4_40
+ .align 5
+
+dtrmm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M4_22
+
+
+dtrmm_kernel_L1_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L1_M4_100
+
+dtrmm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M4_42
+
+dtrmm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+dtrmm_kernel_L1_M4_END:
+
+dtrmm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble dtrmm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble dtrmm_kernel_L1_M1_BEGIN
+
+dtrmm_kernel_L1_M2_20:
+
+ INIT2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L1_M2_40
+
+dtrmm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M2_22
+
+
+dtrmm_kernel_L1_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L1_M2_100
+
+dtrmm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M2_42
+
+dtrmm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+dtrmm_kernel_L1_M2_END:
+
+
+dtrmm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble dtrmm_kernel_L1_END
+
+dtrmm_kernel_L1_M1_20:
+
+ INIT1x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble dtrmm_kernel_L1_M1_40
+
+dtrmm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M1_22
+
+
+dtrmm_kernel_L1_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble dtrmm_kernel_L1_M1_100
+
+dtrmm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt dtrmm_kernel_L1_M1_42
+
+dtrmm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+
+dtrmm_kernel_L1_END:
+
+
+dtrmm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc) */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define temp x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+
+#define alpha0 s10
+#define alphaV0 v10.s[0]
+#define alpha1 s11
+#define alphaV1 v11.s[0]
+#define alpha2 s14
+#define alphaV2 v14.s[0]
+#define alpha3 s15
+#define alphaV3 v15.s[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 temp
+// 17
+// 18 must save
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA0_00, pA0_01, pA0_02, pA0_03
+//v01 pA0_04, pA0_05, pA0_06, pA0_07
+//v02 pA0_08, pA0_09, pA0_10, pA0_11
+//v03 pA0_12, pA0_13, pA0_14, pA0_15
+//v04 pA1_00, pA1_01, pA1_02, pA1_03
+//v05 pA1_04, pA1_05, pA1_06, pA1_07
+//v06 pA1_08, pA1_09, pA1_10, pA1_11
+//v07 pA1_12, pA1_13, pA1_14, pA1_15
+//v08 must save pB00, pB01
+//v09 must save pB02, pB03
+//v10 must save ALPHA0
+//v11 must save ALPHA1
+//v12 must save pB10, pB11
+//v13 must save pB12, pB13
+//v14 must save ALPHA2
+//v15 must save ALPHA3
+//v16 must save C00, C01, C02, C03
+//v17 must save C04, C05, C06, C07
+//v18 C08, C09, C10, C11
+//v19 C12, C13, C14, C15
+//v20 C16, C17, C18, C19
+//v21 C20, C21, C22, C23
+//v22 C24, C25, C26, C27
+//v23 C28, C29, C30, C31
+//v24 C32, C33, C34, C35
+//v25 C36, C37, C38, C39
+//v26 C40, C41, C42, C43
+//v27 C44, C45, C46, C47
+//v28 C48, C49, C50, C51
+//v29 C52, C53, C54, C55
+//v30 C56, C57, C58, C59
+//v31 C60, C61, C62, C63
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT16x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, s16
+ fmov s19, s17
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s26, s17
+ fmov s27, s18
+ fmov s28, wzr
+ fmov s29, s16
+ fmov s30, s17
+ fmov s31, s18
+.endm
+
+.macro KERNEL16x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v8.2s[0]
+ fmul v17.4s, v1.4s, v8.2s[0]
+ fmul v18.4s, v2.4s, v8.2s[0]
+ fmul v19.4s, v3.4s, v8.2s[0]
+
+ fmul v20.4s, v0.4s, v8.2s[1]
+ fmul v21.4s, v1.4s, v8.2s[1]
+ fmul v22.4s, v2.4s, v8.2s[1]
+ fmul v23.4s, v3.4s, v8.2s[1]
+
+ fmul v24.4s, v0.4s, v9.2s[0]
+ fmul v25.4s, v1.4s, v9.2s[0]
+ fmul v26.4s, v2.4s, v9.2s[0]
+ fmul v27.4s, v3.4s, v9.2s[0]
+
+ fmul v28.4s, v0.4s, v9.2s[1]
+ fmul v29.4s, v1.4s, v9.2s[1]
+ fmul v30.4s, v2.4s, v9.2s[1]
+ fmul v31.4s, v3.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v6.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v7.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL16x4_M1
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v22.4s, v2.4s, v8.2s[1]
+ fmla v23.4s, v3.4s, v8.2s[1]
+
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v26.4s, v2.4s, v9.2s[0]
+ fmla v27.4s, v3.4s, v9.2s[0]
+
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+ fmla v30.4s, v2.4s, v9.2s[1]
+ fmla v31.4s, v3.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v6.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v7.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL16x4_M2
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v18.4s, v6.4s, v12.2s[0]
+ fmla v19.4s, v7.4s, v12.2s[0]
+
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v22.4s, v6.4s, v12.2s[1]
+ fmla v23.4s, v7.4s, v12.2s[1]
+
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v26.4s, v6.4s, v13.2s[0]
+ fmla v27.4s, v7.4s, v13.2s[0]
+
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+ fmla v30.4s, v6.4s, v13.2s[1]
+ fmla v31.4s, v7.4s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL16x4_E
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v18.4s, v6.4s, v12.2s[0]
+ fmla v19.4s, v7.4s, v12.2s[0]
+
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v22.4s, v6.4s, v12.2s[1]
+ fmla v23.4s, v7.4s, v12.2s[1]
+
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v26.4s, v6.4s, v13.2s[0]
+ fmla v27.4s, v7.4s, v13.2s[0]
+
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+ fmla v30.4s, v6.4s, v13.2s[1]
+ fmla v31.4s, v7.4s, v13.2s[1]
+.endm
+
+.macro KERNEL16x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v22.4s, v2.4s, v8.2s[1]
+ fmla v23.4s, v3.4s, v8.2s[1]
+
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v26.4s, v2.4s, v9.2s[0]
+ fmla v27.4s, v3.4s, v9.2s[0]
+
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+ fmla v30.4s, v2.4s, v9.2s[1]
+ fmla v31.4s, v3.4s, v9.2s[1]
+.endm
+
+.macro SAVE16x4
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ fmla v2.4s, v18.4s, alphaV2
+ fmla v3.4s, v19.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0
+ fmla v5.4s, v21.4s, alphaV1
+ fmla v6.4s, v22.4s, alphaV2
+ fmla v7.4s, v23.4s, alphaV3
+ st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow2]
+ fmla v0.4s, v24.4s, alphaV0
+ fmla v1.4s, v25.4s, alphaV1
+ fmla v2.4s, v26.4s, alphaV2
+ fmla v3.4s, v27.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow2]
+
+ ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+ fmla v4.4s, v28.4s, alphaV0
+ fmla v5.4s, v29.4s, alphaV1
+ fmla v6.4s, v30.4s, alphaV2
+ fmla v7.4s, v31.4s, alphaV3
+ st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s28, wzr
+ fmov s29, s16
+.endm
+
+.macro KERNEL8x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v8.2s[0]
+ fmul v17.4s, v1.4s, v8.2s[0]
+ fmul v20.4s, v0.4s, v8.2s[1]
+ fmul v21.4s, v1.4s, v8.2s[1]
+ fmul v24.4s, v0.4s, v9.2s[0]
+ fmul v25.4s, v1.4s, v9.2s[0]
+ fmul v28.4s, v0.4s, v9.2s[1]
+ fmul v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M1
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M2
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_E
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+.endm
+
+.macro KERNEL8x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+.endm
+
+.macro SAVE8x4
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0
+ fmla v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow2]
+ fmla v0.4s, v24.4s, alphaV0
+ fmla v1.4s, v25.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow2]
+
+ ld1 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v28.4s, alphaV0
+ fmla v5.4s, v29.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s28, s17
+ fmov s29, s16
+.endm
+
+.macro KERNEL4x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.2s, v0.2s, v8.2s[0]
+ fmul v29.2s, v1.2s, v9.2s[1]
+
+ fmul v20.2s, v0.2s, v8.2s[1]
+ fmul v25.2s, v1.2s, v9.2s[0]
+
+ fmul v24.2s, v0.2s, v9.2s[0]
+ fmul v21.2s, v1.2s, v8.2s[1]
+
+ fmul v28.2s, v0.2s, v9.2s[1]
+ fmul v17.2s, v1.2s, v8.2s[0]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.2s, v5.2s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x4_M1
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ ld1 {v4.2s, v5.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro KERNEL4x4_M2
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ ld1 {v0.2s, v1.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_E
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x4
+ ld1 {v8.2s, v9.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ fmla v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+ ld1 {v12.2s, v13.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV2
+ fmla v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ ld1 {v8.2s, v9.2s}, [pCRow2]
+ fmla v8.2s, v24.2s, alphaV0
+ fmla v9.2s, v25.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+ ld1 {v12.2s, v13.2s}, [pCRow1]
+ fmla v12.2s, v28.2s, alphaV2
+ fmla v13.2s, v29.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov s16, wzr
+ fmov s20, s16
+ fmov s24, s20
+ fmov s28, s16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v28.2s, v0.2s, v9.2s[1]
+.endm
+
+.macro SAVE2x4
+ ld1 {v8.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+ ld1 {v12.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ ld1 {v8.2s}, [pCRow2]
+ fmla v8.2s, v24.2s, alphaV2
+ st1 {v8.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+ ld1 {v12.2s}, [pCRow1]
+ fmla v12.2s, v28.2s, alphaV3
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr s0, [pA]
+ add pA, pA, #4
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+
+ fmla v16.2s, v8.2s, v0.s[0]
+ fmla v20.2s, v9.2s, v0.s[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+ ld1 {v8.s}[0], [pCRow0]
+ ld1 {v8.s}[1], [pCRow1]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+ ld1 {v12.s}[0], [pCRow2]
+ ld1 {v12.s}[1], [pCRow1]
+ fmla v12.2s, v20.2s, alphaV1
+ st1 {v12.s}[0], [pCRow2]
+ st1 {v12.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT16x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, wzr
+ fmov s23, s16
+.endm
+
+.macro KERNEL16x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v22.4s, v2.4s, v8.2s[1]
+ fmla v23.4s, v3.4s, v8.2s[1]
+.endm
+
+.macro SAVE16x2
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ fmla v2.4s, v18.4s, alphaV2
+ fmla v3.4s, v19.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+
+ ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0
+ fmla v5.4s, v21.4s, alphaV1
+ fmla v6.4s, v22.4s, alphaV2
+ fmla v7.4s, v23.4s, alphaV3
+ st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL8x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+.endm
+
+.macro SAVE8x2
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0
+ fmla v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v21.2s, v1.2s, v8.2s[1]
+.endm
+
+.macro SAVE4x2
+ ld1 {v8.2s, v9.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ fmla v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+ ld1 {v12.2s, v13.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV2
+ fmla v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+.endm
+
+.macro SAVE2x2
+ ld1 {v8.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+ ld1 {v12.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2s} , [pB]
+ add pB , pB, #8
+
+ ldr s0 , [pA]
+ add pA, pA, #4
+
+ fmla v16.2s, v8.2s, v0.2s[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+ ld1 {v8.s}[0], [pCRow0]
+ ld1 {v8.s}[1], [pCRow1]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT16x1
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+.endm
+
+.macro KERNEL16x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+.endm
+
+.macro SAVE16x1
+ ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ fmla v2.4s, v18.4s, alphaV2
+ fmla v3.4s, v19.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL8x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+.endm
+
+.macro SAVE8x1
+ ld1 {v0.4s, v1.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov s16, wzr
+ fmov s17, s16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x1
+ ld1 {v8.2s, v9.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ fmla v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s}, [pA]
+ add pA , pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+.endm
+
+.macro SAVE2x1
+ ld1 {v8.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ldr s0, [pA]
+ add pA , pA, #4
+
+ fmadd s16, s0, s8, s16
+.endm
+
+.macro SAVE1x1
+ ldr s8, [pCRow0]
+ fmla s8, s16, alphaV0
+ str s8, [pCRow0]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+sgemm_kernel_begin:
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0, s0
+ fmov alpha1, s0
+ fmov alpha2, s0
+ fmov alpha3, s0
+
+ lsl LDC, LDC, #2 // ldc = ldc * 4
+
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #2 // J = J / 4
+ cmp counterJ, #0
+ ble sgemm_kernel_L2_BEGIN
+
+/******************************************************************************/
+
+sgemm_kernel_L4_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+ mov pA, origPA // pA = start of A array
+
+sgemm_kernel_L4_M16_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #4 // counterI = counterI / 16
+ cmp counterI, #0
+ ble sgemm_kernel_L4_M8_BEGIN
+
+sgemm_kernel_L4_M16_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt sgemm_kernel_L4_M16_32
+
+ KERNEL16x4_I // do one in the K
+ KERNEL16x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble sgemm_kernel_L4_M16_22a
+ .align 5
+
+sgemm_kernel_L4_M16_22:
+
+ KERNEL16x4_M1
+ KERNEL16x4_M2
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M16_22
+
+sgemm_kernel_L4_M16_22a:
+
+ KERNEL16x4_M1
+ KERNEL16x4_E
+
+ b sgemm_kernel_L4_M16_44
+
+sgemm_kernel_L4_M16_32:
+
+ tst counterL, #1
+ ble sgemm_kernel_L4_M16_40
+
+ KERNEL16x4_I
+ KERNEL16x4_E
+
+ b sgemm_kernel_L4_M16_44
+
+sgemm_kernel_L4_M16_40:
+
+ INIT16x4
+
+sgemm_kernel_L4_M16_44:
+
+ ands counterL , origK, #1
+ ble sgemm_kernel_L4_M16_100
+
+sgemm_kernel_L4_M16_46:
+
+ KERNEL16x4_SUB
+
+sgemm_kernel_L4_M16_100:
+
+ SAVE16x4
+
+sgemm_kernel_L4_M16_END:
+ subs counterI, counterI, #1
+ bne sgemm_kernel_L4_M16_20
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #15
+ ble sgemm_kernel_L4_END
+
+ tst counterI, #8
+ ble sgemm_kernel_L4_M4_BEGIN
+
+sgemm_kernel_L4_M8_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt sgemm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble sgemm_kernel_L4_M8_22a
+ .align 5
+
+sgemm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M8_22
+
+sgemm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b sgemm_kernel_L4_M8_44
+
+sgemm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble sgemm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+ KERNEL8x4_E
+
+ b sgemm_kernel_L4_M8_44
+
+sgemm_kernel_L4_M8_40:
+
+ INIT8x4
+
+sgemm_kernel_L4_M8_44:
+
+ ands counterL , origK, #1
+ ble sgemm_kernel_L4_M8_100
+
+sgemm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+sgemm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+sgemm_kernel_L4_M8_END:
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble sgemm_kernel_L4_END
+
+ tst counterI, #4
+ ble sgemm_kernel_L4_M2_BEGIN
+
+sgemm_kernel_L4_M4_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt sgemm_kernel_L4_M4_32
+
+ KERNEL4x4_I // do one in the K
+ KERNEL4x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble sgemm_kernel_L4_M4_22a
+ .align 5
+
+sgemm_kernel_L4_M4_22:
+
+ KERNEL4x4_M1
+ KERNEL4x4_M2
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M4_22
+
+sgemm_kernel_L4_M4_22a:
+
+ KERNEL4x4_M1
+ KERNEL4x4_E
+
+ b sgemm_kernel_L4_M4_44
+
+sgemm_kernel_L4_M4_32:
+
+ tst counterL, #1
+ ble sgemm_kernel_L4_M4_40
+
+ KERNEL4x4_I
+ KERNEL4x4_E
+
+ b sgemm_kernel_L4_M4_44
+
+sgemm_kernel_L4_M4_40:
+
+ INIT4x4
+
+sgemm_kernel_L4_M4_44:
+
+ ands counterL , origK, #1
+ ble sgemm_kernel_L4_M4_100
+
+sgemm_kernel_L4_M4_46:
+
+ KERNEL4x4_SUB
+
+sgemm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+sgemm_kernel_L4_M4_END:
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble sgemm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble sgemm_kernel_L4_M1_BEGIN
+
+sgemm_kernel_L4_M2_20:
+
+ INIT2x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L4_M2_40
+
+sgemm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M2_22
+
+
+sgemm_kernel_L4_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L4_M2_100
+
+sgemm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M2_42
+
+sgemm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+sgemm_kernel_L4_M2_END:
+
+
+sgemm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble sgemm_kernel_L4_END
+
+sgemm_kernel_L4_M1_20:
+
+ INIT1x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L4_M1_40
+
+sgemm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M1_22
+
+
+sgemm_kernel_L4_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L4_M1_100
+
+sgemm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M1_42
+
+sgemm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+sgemm_kernel_L4_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4
+
+ subs counterJ, counterJ , #1 // j--
+ bgt sgemm_kernel_L4_BEGIN
+
+
+/******************************************************************************/
+
+sgemm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble sgemm_kernel_L999
+
+ tst counterJ , #2
+ ble sgemm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+ mov pA, origPA // pA = A
+
+sgemm_kernel_L2_M16_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #4 // counterI = counterI / 16
+ cmp counterI,#0
+ ble sgemm_kernel_L2_M8_BEGIN
+
+sgemm_kernel_L2_M16_20:
+
+ INIT16x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble sgemm_kernel_L2_M16_40
+ .align 5
+
+sgemm_kernel_L2_M16_22:
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M16_22
+
+
+sgemm_kernel_L2_M16_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M16_100
+
+sgemm_kernel_L2_M16_42:
+
+ KERNEL16x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M16_42
+
+sgemm_kernel_L2_M16_100:
+
+ SAVE16x2
+
+sgemm_kernel_L2_M16_END:
+
+ subs counterI, counterI, #1
+ bgt sgemm_kernel_L2_M16_20
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L2_M8_BEGIN:
+ mov counterI, origM
+ tst counterI , #15
+ ble sgemm_kernel_L2_END
+
+ tst counterI, #8
+ ble sgemm_kernel_L2_M4_BEGIN
+
+sgemm_kernel_L2_M8_20:
+
+ INIT8x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble sgemm_kernel_L2_M8_40
+ .align 5
+
+sgemm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M8_22
+
+
+sgemm_kernel_L2_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M8_100
+
+sgemm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M8_42
+
+sgemm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+sgemm_kernel_L2_M8_END:
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L2_M4_BEGIN:
+ mov counterI, origM
+ tst counterI , #7
+ ble sgemm_kernel_L2_END
+
+ tst counterI, #4
+ ble sgemm_kernel_L2_M2_BEGIN
+
+sgemm_kernel_L2_M4_20:
+
+ INIT4x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble sgemm_kernel_L2_M4_40
+ .align 5
+
+sgemm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M4_22
+
+
+sgemm_kernel_L2_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M4_100
+
+sgemm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M4_42
+
+sgemm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+sgemm_kernel_L2_M4_END:
+
+//------------------------------------------------------------------------------
+
+
+sgemm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble sgemm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble sgemm_kernel_L2_M1_BEGIN
+
+sgemm_kernel_L2_M2_20:
+
+ INIT2x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble sgemm_kernel_L2_M2_40
+
+sgemm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M2_22
+
+
+sgemm_kernel_L2_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M2_100
+
+sgemm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M2_42
+
+sgemm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+sgemm_kernel_L2_M2_END:
+
+
+sgemm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble sgemm_kernel_L2_END
+
+sgemm_kernel_L2_M1_20:
+
+ INIT1x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble sgemm_kernel_L2_M1_40
+
+sgemm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M1_22
+
+
+sgemm_kernel_L2_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M1_100
+
+sgemm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M1_42
+
+sgemm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+sgemm_kernel_L2_END:
+
+ add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4
+
+/******************************************************************************/
+
+sgemm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble sgemm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+ mov pA, origPA // pA = A
+
+sgemm_kernel_L1_M16_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #4 // counterI = counterI / 16
+ cmp counterI, #0
+ ble sgemm_kernel_L1_M8_BEGIN
+
+sgemm_kernel_L1_M16_20:
+
+ INIT16x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M16_40
+ .align 5
+
+sgemm_kernel_L1_M16_22:
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M16_22
+
+
+sgemm_kernel_L1_M16_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M16_100
+
+sgemm_kernel_L1_M16_42:
+
+ KERNEL16x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M16_42
+
+sgemm_kernel_L1_M16_100:
+
+ SAVE16x1
+
+sgemm_kernel_L1_M16_END:
+
+ subs counterI, counterI, #1
+ bgt sgemm_kernel_L1_M16_20
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #15
+ ble sgemm_kernel_L1_END
+
+ tst counterI, #8
+ ble sgemm_kernel_L1_M4_BEGIN
+
+sgemm_kernel_L1_M8_20:
+
+ INIT8x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M8_40
+ .align 5
+
+sgemm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M8_22
+
+
+sgemm_kernel_L1_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M8_100
+
+sgemm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M8_42
+
+sgemm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+sgemm_kernel_L1_M8_END:
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L1_M4_BEGIN:
+ mov counterI, origM
+ tst counterI , #7
+ ble sgemm_kernel_L1_END
+
+ tst counterI, #4
+ ble sgemm_kernel_L1_M2_BEGIN
+
+sgemm_kernel_L1_M4_20:
+
+ INIT4x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M4_40
+ .align 5
+
+sgemm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M4_22
+
+
+sgemm_kernel_L1_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M4_100
+
+sgemm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M4_42
+
+sgemm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+sgemm_kernel_L1_M4_END:
+
+//------------------------------------------------------------------------------
+
+sgemm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble sgemm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble sgemm_kernel_L1_M1_BEGIN
+
+sgemm_kernel_L1_M2_20:
+
+ INIT2x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M2_40
+
+sgemm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M2_22
+
+
+sgemm_kernel_L1_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M2_100
+
+sgemm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M2_42
+
+sgemm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+sgemm_kernel_L1_M2_END:
+
+
+sgemm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble sgemm_kernel_L1_END
+
+sgemm_kernel_L1_M1_20:
+
+ INIT1x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M1_40
+
+sgemm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M1_22
+
+
+sgemm_kernel_L1_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M1_100
+
+sgemm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M1_42
+
+sgemm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+sgemm_kernel_L1_END:
+
+sgemm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc) */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define offset x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+#define temp x16
+
+#define alpha0 s10
+#define alphaV0 v10.s[0]
+#define alpha1 s11
+#define alphaV1 v11.s[0]
+#define alpha2 s14
+#define alphaV2 v14.s[0]
+#define alpha3 s15
+#define alphaV3 v15.s[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 temp
+// 17
+// 18 must save
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA0_0, pA0_1, pA0_2, pA0_3
+//v01 pA0_4, pA0_5, pA0_6, pA0_7
+//v02 pA1_0, pA1_1, pA1_2, pA1_3
+//v03 pA1_4, pA1_5, pA1_6, pA1_7
+//v04 pB0_0, pB0_1, pB0_2, pB0_3
+//v05 pB0_4, pB0_5, pB0_6, pB0_7
+//v06 pB1_0, pB1_1, pB1_2, pB1_3
+//v07 pB1_4, pB1_5, pB1_6, pB1_7
+//v08 must save
+//v09 must save
+//v10 must save ALPHA0
+//v11 must save ALPHA1
+//v12 must save
+//v13 must save
+//v14 must save ALPHA2
+//v15 must save ALPHA3
+//v16 must save C00, C01, C02, C03
+//v17 must save C04, C05, C06, C07
+//v18 C08, C09, C10, C11
+//v19 C12, C13, C14, C15
+//v20 C16, C17, C18, C19
+//v21 C20, C21, C22, C23
+//v22 C24, C25, C26, C27
+//v23 C28, C29, C30, C31
+//v24 C32, C33, C34, C35
+//v25 C36, C37, C38, C39
+//v26 C40, C41, C42, C43
+//v27 C44, C45, C46, C47
+//v28 C48, C49, C50, C51
+//v29 C52, C53, C54, C55
+//v30 C56, C57, C58, C59
+//v31 C60, C61, C62, C63
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT8x8
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, s16
+ fmov s19, s17
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s26, s17
+ fmov s27, s18
+ fmov s28, wzr
+ fmov s29, s16
+ fmov s30, s17
+ fmov s31, s18
+.endm
+
+.macro KERNEL8x8_I
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v4.4s[0]
+ fmul v17.4s, v1.4s, v4.4s[0]
+ fmul v18.4s, v0.4s, v4.4s[1]
+ fmul v19.4s, v1.4s, v4.4s[1]
+ fmul v20.4s, v0.4s, v4.4s[2]
+ fmul v21.4s, v1.4s, v4.4s[2]
+ fmul v22.4s, v0.4s, v4.4s[3]
+ fmul v23.4s, v1.4s, v4.4s[3]
+ fmul v24.4s, v0.4s, v5.4s[0]
+ fmul v25.4s, v1.4s, v5.4s[0]
+ fmul v26.4s, v0.4s, v5.4s[1]
+ fmul v27.4s, v1.4s, v5.4s[1]
+ fmul v28.4s, v0.4s, v5.4s[2]
+ fmul v29.4s, v1.4s, v5.4s[2]
+ fmul v30.4s, v0.4s, v5.4s[3]
+ fmul v31.4s, v1.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x8_M1
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v17.4s, v1.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v19.4s, v1.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v21.4s, v1.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v23.4s, v1.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v25.4s, v1.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v27.4s, v1.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v29.4s, v1.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+ fmla v31.4s, v1.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x8_M2
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v17.4s, v3.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v19.4s, v3.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v21.4s, v3.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v23.4s, v3.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v25.4s, v3.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v27.4s, v3.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v29.4s, v3.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+ fmla v31.4s, v3.4s, v7.4s[3]
+
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x8_E
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v17.4s, v3.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v19.4s, v3.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v21.4s, v3.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v23.4s, v3.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v25.4s, v3.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v27.4s, v3.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v29.4s, v3.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+ fmla v31.4s, v3.4s, v7.4s[3]
+.endm
+
+.macro KERNEL8x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v17.4s, v1.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v19.4s, v1.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v21.4s, v1.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v23.4s, v1.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v25.4s, v1.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v27.4s, v1.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v29.4s, v1.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+ fmla v31.4s, v1.4s, v5.4s[3]
+.endm
+
+.macro SAVE8x8
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v2.4s, v3.4s}, [pCRow1]
+ fmla v2.4s, v18.4s, alphaV2
+ fmla v3.4s, v19.4s, alphaV3
+ st1 {v2.4s, v3.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v4.4s, v5.4s}, [pCRow2]
+ fmla v4.4s, v20.4s, alphaV0
+ fmla v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v6.4s, v7.4s}, [pCRow1]
+ fmla v6.4s, v22.4s, alphaV2
+ fmla v7.4s, v23.4s, alphaV3
+ st1 {v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow2]
+ fmla v0.4s, v24.4s, alphaV0
+ fmla v1.4s, v25.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v2.4s, v3.4s}, [pCRow1]
+ fmla v2.4s, v26.4s, alphaV2
+ fmla v3.4s, v27.4s, alphaV3
+ st1 {v2.4s, v3.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v4.4s, v5.4s}, [pCRow2]
+ fmla v4.4s, v28.4s, alphaV0
+ fmla v5.4s, v29.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow2]
+
+ ld1 {v6.4s, v7.4s}, [pCRow1]
+ fmla v6.4s, v30.4s, alphaV2
+ fmla v7.4s, v31.4s, alphaV3
+ st1 {v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+
+.macro INIT4x8
+ fmov s16, wzr
+ fmov s18, wzr
+ fmov s20, wzr
+ fmov s22, s16
+ fmov s24, wzr
+ fmov s26, s16
+ fmov s28, s18
+ fmov s30, s20
+.endm
+
+.macro KERNEL4x8_I
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v4.4s[0]
+ fmul v18.4s, v0.4s, v4.4s[1]
+ fmul v20.4s, v0.4s, v4.4s[2]
+ fmul v22.4s, v0.4s, v4.4s[3]
+ fmul v24.4s, v0.4s, v5.4s[0]
+ fmul v26.4s, v0.4s, v5.4s[1]
+ fmul v28.4s, v0.4s, v5.4s[2]
+ fmul v30.4s, v0.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x8_M1
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x8_M2
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x8_E
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+.endm
+
+.macro KERNEL4x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+.endm
+
+.macro SAVE4x8
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ st1 {v0.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v2.4s}, [pCRow1]
+ fmla v2.4s, v18.4s, alphaV2
+ st1 {v2.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v4.4s}, [pCRow2]
+ fmla v4.4s, v20.4s, alphaV0
+ st1 {v4.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v6.4s}, [pCRow1]
+ fmla v6.4s, v22.4s, alphaV2
+ st1 {v6.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v0.4s}, [pCRow2]
+ fmla v0.4s, v24.4s, alphaV0
+ st1 {v0.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v2.4s}, [pCRow1]
+ fmla v2.4s, v26.4s, alphaV2
+ st1 {v2.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v4.4s}, [pCRow2]
+ fmla v4.4s, v28.4s, alphaV0
+ st1 {v4.4s}, [pCRow2]
+
+ ld1 {v6.4s}, [pCRow1]
+ fmla v6.4s, v30.4s, alphaV2
+ st1 {v6.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x8
+ fmov s16, wzr
+ fmov s18, wzr
+ fmov s20, wzr
+ fmov s22, s16
+ fmov s24, wzr
+ fmov s26, s16
+ fmov s28, s18
+ fmov s30, s20
+.endm
+
+.macro KERNEL2x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v4.4s[0]
+ fmla v18.2s, v0.2s, v4.4s[1]
+ fmla v20.2s, v0.2s, v4.4s[2]
+ fmla v22.2s, v0.2s, v4.4s[3]
+ fmla v24.2s, v0.2s, v5.4s[0]
+ fmla v26.2s, v0.2s, v5.4s[1]
+ fmla v28.2s, v0.2s, v5.4s[2]
+ fmla v30.2s, v0.2s, v5.4s[3]
+.endm
+
+.macro SAVE2x8
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.2s}, [pCRow0]
+ fmla v0.2s, v16.2s, alphaV0
+ st1 {v0.2s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v2.2s}, [pCRow1]
+ fmla v2.2s, v18.2s, alphaV2
+ st1 {v2.2s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v4.2s}, [pCRow2]
+ fmla v4.2s, v20.2s, alphaV0
+ st1 {v4.2s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v6.2s}, [pCRow1]
+ fmla v6.2s, v22.2s, alphaV2
+ st1 {v6.2s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v0.2s}, [pCRow2]
+ fmla v0.2s, v24.2s, alphaV0
+ st1 {v0.2s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v2.2s}, [pCRow1]
+ fmla v2.2s, v26.2s, alphaV2
+ st1 {v2.2s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v4.2s}, [pCRow2]
+ fmla v4.2s, v28.2s, alphaV0
+ st1 {v4.2s}, [pCRow2]
+
+ ld1 {v6.2s}, [pCRow1]
+ fmla v6.2s, v30.2s, alphaV2
+ st1 {v6.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x8
+ fmov s16, wzr
+ fmov s18, wzr
+ fmov s20, wzr
+ fmov s22, s16
+ fmov s24, wzr
+ fmov s26, s16
+ fmov s28, s18
+ fmov s30, s20
+.endm
+
+.macro KERNEL1x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ldr s0, [pA]
+ add pA, pA, #4
+
+ fmla s16, s0, v4.4s[0]
+ fmla s18, s0, v4.4s[1]
+ fmla s20, s0, v4.4s[2]
+ fmla s22, s0, v4.4s[3]
+ fmla s24, s0, v5.4s[0]
+ fmla s26, s0, v5.4s[1]
+ fmla s28, s0, v5.4s[2]
+ fmla s30, s0, v5.4s[3]
+.endm
+
+.macro SAVE1x8
+ add pCRow1, pCRow0, LDC
+
+ ldr s0, [pCRow0]
+ fmla s0, s16, alphaV0
+ str s0, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ldr s2, [pCRow1]
+ fmla s2, s18, alphaV2
+ str s2, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ldr s4, [pCRow2]
+ fmla s4, s20, alphaV0
+ str s4, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ldr s6, [pCRow1]
+ fmla s6, s22, alphaV2
+ str s6, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ldr s0, [pCRow2]
+ fmla s0, s24, alphaV0
+ str s0, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ ldr s2, [pCRow1]
+ fmla s2, s26, alphaV2
+ str s2, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ldr s4, [pCRow2]
+ fmla s4, s28, alphaV0
+ str s4, [pCRow2]
+
+ ldr s6, [pCRow1]
+ fmla s6, s30, alphaV2
+ str s6, [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s28, wzr
+ fmov s29, s16
+.endm
+
+.macro KERNEL8x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v8.2s[0]
+ fmul v17.4s, v1.4s, v8.2s[0]
+ fmul v20.4s, v0.4s, v8.2s[1]
+ fmul v21.4s, v1.4s, v8.2s[1]
+ fmul v24.4s, v0.4s, v9.2s[0]
+ fmul v25.4s, v1.4s, v9.2s[0]
+ fmul v28.4s, v0.4s, v9.2s[1]
+ fmul v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M1
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M2
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_E
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+.endm
+
+.macro KERNEL8x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+.endm
+
+.macro SAVE8x4
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0
+ fmla v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow2]
+ fmla v0.4s, v24.4s, alphaV0
+ fmla v1.4s, v25.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow2]
+
+ ld1 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v28.4s, alphaV0
+ fmla v5.4s, v29.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+
+.macro INIT4x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s28, s17
+ fmov s29, s16
+.endm
+
+.macro KERNEL4x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.2s, v0.2s, v8.2s[0]
+ fmul v29.2s, v1.2s, v9.2s[1]
+
+ fmul v20.2s, v0.2s, v8.2s[1]
+ fmul v25.2s, v1.2s, v9.2s[0]
+
+ fmul v24.2s, v0.2s, v9.2s[0]
+ fmul v21.2s, v1.2s, v8.2s[1]
+
+ fmul v28.2s, v0.2s, v9.2s[1]
+ fmul v17.2s, v1.2s, v8.2s[0]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.2s, v5.2s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x4_M1
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ ld1 {v4.2s, v5.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro KERNEL4x4_M2
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ ld1 {v0.2s, v1.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_E
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x4
+ ld1 {v8.2s, v9.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ fmla v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+ ld1 {v12.2s, v13.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV2
+ fmla v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ ld1 {v8.2s, v9.2s}, [pCRow2]
+ fmla v8.2s, v24.2s, alphaV0
+ fmla v9.2s, v25.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+ ld1 {v12.2s, v13.2s}, [pCRow1]
+ fmla v12.2s, v28.2s, alphaV2
+ fmla v13.2s, v29.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov s16, wzr
+ fmov s20, s16
+ fmov s24, s20
+ fmov s28, s16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v28.2s, v0.2s, v9.2s[1]
+.endm
+
+.macro SAVE2x4
+ ld1 {v8.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+ ld1 {v12.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ ld1 {v8.2s}, [pCRow2]
+ fmla v8.2s, v24.2s, alphaV2
+ st1 {v8.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+ ld1 {v12.2s}, [pCRow1]
+ fmla v12.2s, v28.2s, alphaV3
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr s0, [pA]
+ add pA, pA, #4
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+
+ fmla v16.2s, v8.2s, v0.s[0]
+ fmla v20.2s, v9.2s, v0.s[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+ ld1 {v8.s}[0], [pCRow0]
+ ld1 {v8.s}[1], [pCRow1]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+ ld1 {v12.s}[0], [pCRow2]
+ ld1 {v12.s}[1], [pCRow1]
+ fmla v12.2s, v20.2s, alphaV1
+ st1 {v12.s}[0], [pCRow2]
+ st1 {v12.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL8x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+.endm
+
+.macro SAVE8x2
+ add pCRow1, pCRow0, LDC
+
+ ld1 {v0.4s, v1.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ ld1 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0
+ fmla v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v21.2s, v1.2s, v8.2s[1]
+.endm
+
+.macro SAVE4x2
+ ld1 {v8.2s, v9.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ fmla v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+ ld1 {v12.2s, v13.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV2
+ fmla v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+.endm
+
+.macro SAVE2x2
+ ld1 {v8.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+ ld1 {v12.2s}, [pCRow1]
+ fmla v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2s} , [pB]
+ add pB , pB, #8
+
+ ldr s0 , [pA]
+ add pA, pA, #4
+
+ fmla v16.2s, v8.2s, v0.2s[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+ ld1 {v8.s}[0], [pCRow0]
+ ld1 {v8.s}[1], [pCRow1]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL8x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+.endm
+
+.macro SAVE8x1
+ ld1 {v0.4s, v1.4s}, [pCRow0]
+ fmla v0.4s, v16.4s, alphaV0
+ fmla v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov s16, wzr
+ fmov s17, s16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x1
+ ld1 {v8.2s, v9.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ fmla v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s}, [pA]
+ add pA , pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+.endm
+
+.macro SAVE2x1
+ ld1 {v8.2s}, [pCRow0]
+ fmla v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ldr s0, [pA]
+ add pA , pA, #4
+
+ fmadd s16, s0, s8, s16
+.endm
+
+.macro SAVE1x1
+ ldr s8, [pCRow0]
+ fmla s8, s16, alphaV0
+ str s8, [pCRow0]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+sgemm_kernel_begin:
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0, s0
+ fmov alpha1, s0
+ fmov alpha2, s0
+ fmov alpha3, s0
+
+ lsl LDC, LDC, #2 // ldc = ldc * 4
+
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #3 // J = J / 8
+ cmp counterJ, #0
+ ble sgemm_kernel_L4_BEGIN
+
+/******************************************************************************/
+/******************************************************************************/
+
+sgemm_kernel_L8_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #3
+
+ mov pA, origPA // pA = start of A array
+
+/******************************************************************************/
+
+sgemm_kernel_L8_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble sgemm_kernel_L8_M4_BEGIN
+
+sgemm_kernel_L8_M8_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt sgemm_kernel_L8_M8_32
+
+ KERNEL8x8_I // do one in the K
+ KERNEL8x8_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble sgemm_kernel_L8_M8_22a
+ .align 5
+
+sgemm_kernel_L8_M8_22:
+
+ KERNEL8x8_M1
+ KERNEL8x8_M2
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L8_M8_22
+
+sgemm_kernel_L8_M8_22a:
+
+ KERNEL8x8_M1
+ KERNEL8x8_E
+
+ b sgemm_kernel_L8_M8_44
+
+sgemm_kernel_L8_M8_32:
+
+ tst counterL, #1
+ ble sgemm_kernel_L8_M8_40
+
+ KERNEL8x8_I
+ KERNEL8x8_E
+
+ b sgemm_kernel_L8_M8_44
+
+sgemm_kernel_L8_M8_40:
+
+ INIT8x8
+
+sgemm_kernel_L8_M8_44:
+
+ ands counterL , origK, #1
+ ble sgemm_kernel_L8_M8_100
+
+sgemm_kernel_L8_M8_46:
+
+ KERNEL8x8_SUB
+
+sgemm_kernel_L8_M8_100:
+
+ SAVE8x8
+
+sgemm_kernel_L8_M8_END:
+ subs counterI, counterI, #1
+ bne sgemm_kernel_L8_M8_20
+
+/******************************************************************************/
+
+sgemm_kernel_L8_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble sgemm_kernel_L8_END
+
+ tst counterI, #4
+ ble sgemm_kernel_L8_M2_BEGIN
+
+sgemm_kernel_L8_M4_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt sgemm_kernel_L8_M4_32
+
+ KERNEL4x8_I // do one in the K
+ KERNEL4x8_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble sgemm_kernel_L8_M4_22a
+ .align 5
+
+sgemm_kernel_L8_M4_22:
+
+ KERNEL4x8_M1
+ KERNEL4x8_M2
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L8_M4_22
+
+sgemm_kernel_L8_M4_22a:
+
+ KERNEL4x8_M1
+ KERNEL4x8_E
+
+ b sgemm_kernel_L8_M4_44
+
+sgemm_kernel_L8_M4_32:
+
+ tst counterL, #1
+ ble sgemm_kernel_L8_M4_40
+
+ KERNEL4x8_I
+ KERNEL4x8_E
+
+ b sgemm_kernel_L8_M4_44
+
+sgemm_kernel_L8_M4_40:
+
+ INIT4x8
+
+sgemm_kernel_L8_M4_44:
+
+ ands counterL , origK, #1
+ ble sgemm_kernel_L8_M4_100
+
+sgemm_kernel_L8_M4_46:
+
+ KERNEL4x8_SUB
+
+sgemm_kernel_L8_M4_100:
+
+ SAVE4x8
+
+sgemm_kernel_L8_M4_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L8_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble sgemm_kernel_L8_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble sgemm_kernel_L8_M1_BEGIN
+
+sgemm_kernel_L8_M2_20:
+
+ INIT2x8
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L8_M2_40
+
+sgemm_kernel_L8_M2_22:
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L8_M2_22
+
+
+sgemm_kernel_L8_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L8_M2_100
+
+sgemm_kernel_L8_M2_42:
+
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L8_M2_42
+
+sgemm_kernel_L8_M2_100:
+
+ SAVE2x8
+
+sgemm_kernel_L8_M2_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L8_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble sgemm_kernel_L8_END
+
+sgemm_kernel_L8_M1_20:
+
+ INIT1x8
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L8_M1_40
+
+sgemm_kernel_L8_M1_22:
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L8_M1_22
+
+
+sgemm_kernel_L8_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L8_M1_100
+
+sgemm_kernel_L8_M1_42:
+
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L8_M1_42
+
+sgemm_kernel_L8_M1_100:
+
+ SAVE1x8
+
+sgemm_kernel_L8_END:
+ lsl temp, origK, #5 // B = B + K * 4 * 8
+ add origPB, origPB, temp
+
+ subs counterJ, counterJ , #1 // j--
+ bgt sgemm_kernel_L8_BEGIN
+
+/******************************************************************************/
+/******************************************************************************/
+
+sgemm_kernel_L4_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #7
+ ble sgemm_kernel_L999
+
+ tst counterJ , #4
+ ble sgemm_kernel_L2_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #2
+
+ mov pA, origPA // pA = A
+
+/******************************************************************************/
+
+sgemm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble sgemm_kernel_L4_M4_BEGIN
+
+sgemm_kernel_L4_M8_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt sgemm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble sgemm_kernel_L4_M8_22a
+ .align 5
+
+sgemm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M8_22
+
+sgemm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b sgemm_kernel_L4_M8_44
+
+sgemm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble sgemm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+ KERNEL8x4_E
+
+ b sgemm_kernel_L4_M8_44
+
+sgemm_kernel_L4_M8_40:
+
+ INIT8x4
+
+sgemm_kernel_L4_M8_44:
+
+ ands counterL , origK, #1
+ ble sgemm_kernel_L4_M8_100
+
+sgemm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+sgemm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+sgemm_kernel_L4_M8_END:
+ subs counterI, counterI, #1
+ bne sgemm_kernel_L4_M8_20
+
+/******************************************************************************/
+
+sgemm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble sgemm_kernel_L4_END
+
+ tst counterI, #4
+ ble sgemm_kernel_L4_M2_BEGIN
+
+sgemm_kernel_L4_M4_20:
+
+ mov pB, origPB
+
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt sgemm_kernel_L4_M4_32
+
+ KERNEL4x4_I // do one in the K
+ KERNEL4x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble sgemm_kernel_L4_M4_22a
+ .align 5
+
+sgemm_kernel_L4_M4_22:
+
+ KERNEL4x4_M1
+ KERNEL4x4_M2
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M4_22
+
+sgemm_kernel_L4_M4_22a:
+
+ KERNEL4x4_M1
+ KERNEL4x4_E
+
+ b sgemm_kernel_L4_M4_44
+
+sgemm_kernel_L4_M4_32:
+
+ tst counterL, #1
+ ble sgemm_kernel_L4_M4_40
+
+ KERNEL4x4_I
+ KERNEL4x4_E
+
+ b sgemm_kernel_L4_M4_44
+
+sgemm_kernel_L4_M4_40:
+
+ INIT4x4
+
+sgemm_kernel_L4_M4_44:
+
+ ands counterL , origK, #1
+ ble sgemm_kernel_L4_M4_100
+
+sgemm_kernel_L4_M4_46:
+
+ KERNEL4x4_SUB
+
+sgemm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+sgemm_kernel_L4_M4_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble sgemm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble sgemm_kernel_L4_M1_BEGIN
+
+sgemm_kernel_L4_M2_20:
+
+ INIT2x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L4_M2_40
+
+sgemm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M2_22
+
+
+sgemm_kernel_L4_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L4_M2_100
+
+sgemm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M2_42
+
+sgemm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+sgemm_kernel_L4_M2_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble sgemm_kernel_L4_END
+
+sgemm_kernel_L4_M1_20:
+
+ INIT1x4
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L4_M1_40
+
+sgemm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M1_22
+
+
+sgemm_kernel_L4_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L4_M1_100
+
+sgemm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L4_M1_42
+
+sgemm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+sgemm_kernel_L4_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4
+
+/******************************************************************************/
+/******************************************************************************/
+
+sgemm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble sgemm_kernel_L999
+
+ tst counterJ , #2
+ ble sgemm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+ mov pA, origPA // pA = A
+
+/******************************************************************************/
+
+sgemm_kernel_L2_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI,#0
+ ble sgemm_kernel_L2_M4_BEGIN
+
+sgemm_kernel_L2_M8_20:
+
+ INIT8x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble sgemm_kernel_L2_M8_40
+ .align 5
+
+sgemm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M8_22
+
+
+sgemm_kernel_L2_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M8_100
+
+sgemm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M8_42
+
+sgemm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+sgemm_kernel_L2_M8_END:
+
+ subs counterI, counterI, #1
+ bgt sgemm_kernel_L2_M8_20
+
+/******************************************************************************/
+
+sgemm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble sgemm_kernel_L2_END
+
+ tst counterI, #4
+ ble sgemm_kernel_L2_M2_BEGIN
+
+sgemm_kernel_L2_M4_20:
+
+ INIT4x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble sgemm_kernel_L2_M4_40
+ .align 5
+
+sgemm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M4_22
+
+
+sgemm_kernel_L2_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M4_100
+
+sgemm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M4_42
+
+sgemm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+sgemm_kernel_L2_M4_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble sgemm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble sgemm_kernel_L2_M1_BEGIN
+
+sgemm_kernel_L2_M2_20:
+
+ INIT2x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble sgemm_kernel_L2_M2_40
+
+sgemm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M2_22
+
+
+sgemm_kernel_L2_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M2_100
+
+sgemm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M2_42
+
+sgemm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+sgemm_kernel_L2_M2_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble sgemm_kernel_L2_END
+
+sgemm_kernel_L2_M1_20:
+
+ INIT1x2
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble sgemm_kernel_L2_M1_40
+
+sgemm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M1_22
+
+
+sgemm_kernel_L2_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L2_M1_100
+
+sgemm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L2_M1_42
+
+sgemm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+sgemm_kernel_L2_END:
+
+ add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4
+
+/******************************************************************************/
+/******************************************************************************/
+
+sgemm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble sgemm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+ mov pA, origPA // pA = A
+
+/******************************************************************************/
+
+sgemm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3
+ cmp counterI, #0
+ ble sgemm_kernel_L1_M4_BEGIN
+
+sgemm_kernel_L1_M8_20:
+
+ INIT8x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M8_40
+ .align 5
+
+sgemm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M8_22
+
+
+sgemm_kernel_L1_M8_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M8_100
+
+sgemm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M8_42
+
+sgemm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+sgemm_kernel_L1_M8_END:
+
+ subs counterI, counterI, #1
+ bgt sgemm_kernel_L1_M8_20
+
+/******************************************************************************/
+
+sgemm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble sgemm_kernel_L1_END
+
+ tst counterI, #4
+ ble sgemm_kernel_L1_M2_BEGIN
+
+sgemm_kernel_L1_M4_20:
+
+ INIT4x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M4_40
+ .align 5
+
+sgemm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M4_22
+
+
+sgemm_kernel_L1_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M4_100
+
+sgemm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M4_42
+
+sgemm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+sgemm_kernel_L1_M4_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble sgemm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble sgemm_kernel_L1_M1_BEGIN
+
+sgemm_kernel_L1_M2_20:
+
+ INIT2x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M2_40
+
+sgemm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M2_22
+
+
+sgemm_kernel_L1_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M2_100
+
+sgemm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M2_42
+
+sgemm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+sgemm_kernel_L1_M2_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble sgemm_kernel_L1_END
+
+sgemm_kernel_L1_M1_20:
+
+ INIT1x1
+
+ mov pB, origPB
+
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble sgemm_kernel_L1_M1_40
+
+sgemm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M1_22
+
+
+sgemm_kernel_L1_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble sgemm_kernel_L1_M1_100
+
+sgemm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt sgemm_kernel_L1_M1_42
+
+sgemm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+sgemm_kernel_L1_END:
+
+/******************************************************************************/
+
+sgemm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 x7 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define offset x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+#define temp x16
+#define tempOffset x17
+#define tempK x18
+
+#define alpha0 s10
+#define alphaV0 v10.s[0]
+#define alpha1 s11
+#define alphaV1 v11.s[0]
+#define alpha2 s14
+#define alphaV2 v14.s[0]
+#define alpha3 s15
+#define alphaV3 v15.s[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 temp
+// 17 tempOffset
+// 18 must save tempK
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA0_00, pA0_01, pA0_02, pA0_03
+//v01 pA0_04, pA0_05, pA0_06, pA0_07
+//v02 pA0_08, pA0_09, pA0_10, pA0_11
+//v03 pA0_12, pA0_13, pA0_14, pA0_15
+//v04 pA1_00, pA1_01, pA1_02, pA1_03
+//v05 pA1_04, pA1_05, pA1_06, pA1_07
+//v06 pA1_08, pA1_09, pA1_10, pA1_11
+//v07 pA1_12, pA1_13, pA1_14, pA1_15
+//v08 must save pB00, pB01
+//v09 must save pB02, pB03
+//v10 must save ALPHA0
+//v11 must save ALPHA1
+//v12 must save pB10, pB11
+//v13 must save pB12, pB13
+//v14 must save ALPHA2
+//v15 must save ALPHA3
+//v16 must save C00, C01, C02, C03
+//v17 must save C04, C05, C06, C07
+//v18 C08, C09, C10, C11
+//v19 C12, C13, C14, C15
+//v20 C16, C17, C18, C19
+//v21 C20, C21, C22, C23
+//v22 C24, C25, C26, C27
+//v23 C28, C29, C30, C31
+//v24 C32, C33, C34, C35
+//v25 C36, C37, C38, C39
+//v26 C40, C41, C42, C43
+//v27 C44, C45, C46, C47
+//v28 C48, C49, C50, C51
+//v29 C52, C53, C54, C55
+//v30 C56, C57, C58, C59
+//v31 C60, C61, C62, C63
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT16x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, s16
+ fmov s19, s17
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s26, s17
+ fmov s27, s18
+ fmov s28, wzr
+ fmov s29, s16
+ fmov s30, s17
+ fmov s31, s18
+.endm
+
+.macro KERNEL16x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v8.2s[0]
+ fmul v17.4s, v1.4s, v8.2s[0]
+ fmul v18.4s, v2.4s, v8.2s[0]
+ fmul v19.4s, v3.4s, v8.2s[0]
+
+ fmul v20.4s, v0.4s, v8.2s[1]
+ fmul v21.4s, v1.4s, v8.2s[1]
+ fmul v22.4s, v2.4s, v8.2s[1]
+ fmul v23.4s, v3.4s, v8.2s[1]
+
+ fmul v24.4s, v0.4s, v9.2s[0]
+ fmul v25.4s, v1.4s, v9.2s[0]
+ fmul v26.4s, v2.4s, v9.2s[0]
+ fmul v27.4s, v3.4s, v9.2s[0]
+
+ fmul v28.4s, v0.4s, v9.2s[1]
+ fmul v29.4s, v1.4s, v9.2s[1]
+ fmul v30.4s, v2.4s, v9.2s[1]
+ fmul v31.4s, v3.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v6.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v7.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL16x4_M1
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v22.4s, v2.4s, v8.2s[1]
+ fmla v23.4s, v3.4s, v8.2s[1]
+
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v26.4s, v2.4s, v9.2s[0]
+ fmla v27.4s, v3.4s, v9.2s[0]
+
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+ fmla v30.4s, v2.4s, v9.2s[1]
+ fmla v31.4s, v3.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v6.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v7.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL16x4_M2
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v18.4s, v6.4s, v12.2s[0]
+ fmla v19.4s, v7.4s, v12.2s[0]
+
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v22.4s, v6.4s, v12.2s[1]
+ fmla v23.4s, v7.4s, v12.2s[1]
+
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v26.4s, v6.4s, v13.2s[0]
+ fmla v27.4s, v7.4s, v13.2s[0]
+
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+ fmla v30.4s, v6.4s, v13.2s[1]
+ fmla v31.4s, v7.4s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL16x4_E
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v18.4s, v6.4s, v12.2s[0]
+ fmla v19.4s, v7.4s, v12.2s[0]
+
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v22.4s, v6.4s, v12.2s[1]
+ fmla v23.4s, v7.4s, v12.2s[1]
+
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v26.4s, v6.4s, v13.2s[0]
+ fmla v27.4s, v7.4s, v13.2s[0]
+
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+ fmla v30.4s, v6.4s, v13.2s[1]
+ fmla v31.4s, v7.4s, v13.2s[1]
+.endm
+
+.macro KERNEL16x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v22.4s, v2.4s, v8.2s[1]
+ fmla v23.4s, v3.4s, v8.2s[1]
+
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v26.4s, v2.4s, v9.2s[0]
+ fmla v27.4s, v3.4s, v9.2s[0]
+
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+ fmla v30.4s, v2.4s, v9.2s[1]
+ fmla v31.4s, v3.4s, v9.2s[1]
+.endm
+
+.macro SAVE16x4
+ add pCRow1, pCRow0, LDC
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ fmul v2.4s, v18.4s, alphaV2
+ fmul v3.4s, v19.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v4.4s, v20.4s, alphaV0
+ fmul v5.4s, v21.4s, alphaV1
+ fmul v6.4s, v22.4s, alphaV2
+ fmul v7.4s, v23.4s, alphaV3
+ st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v0.4s, v24.4s, alphaV0
+ fmul v1.4s, v25.4s, alphaV1
+ fmul v2.4s, v26.4s, alphaV2
+ fmul v3.4s, v27.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow2]
+
+ fmul v4.4s, v28.4s, alphaV0
+ fmul v5.4s, v29.4s, alphaV1
+ fmul v6.4s, v30.4s, alphaV2
+ fmul v7.4s, v31.4s, alphaV3
+ st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s28, wzr
+ fmov s29, s16
+.endm
+
+.macro KERNEL8x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v8.2s[0]
+ fmul v17.4s, v1.4s, v8.2s[0]
+ fmul v20.4s, v0.4s, v8.2s[1]
+ fmul v21.4s, v1.4s, v8.2s[1]
+ fmul v24.4s, v0.4s, v9.2s[0]
+ fmul v25.4s, v1.4s, v9.2s[0]
+ fmul v28.4s, v0.4s, v9.2s[1]
+ fmul v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M1
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M2
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_E
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+.endm
+
+.macro KERNEL8x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+.endm
+
+.macro SAVE8x4
+ add pCRow1, pCRow0, LDC
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v4.4s, v20.4s, alphaV0
+ fmul v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v0.4s, v24.4s, alphaV0
+ fmul v1.4s, v25.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow2]
+
+ fmul v4.4s, v28.4s, alphaV0
+ fmul v5.4s, v29.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s28, s17
+ fmov s29, s16
+.endm
+
+.macro KERNEL4x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.2s, v0.2s, v8.2s[0]
+ fmul v29.2s, v1.2s, v9.2s[1]
+
+ fmul v20.2s, v0.2s, v8.2s[1]
+ fmul v25.2s, v1.2s, v9.2s[0]
+
+ fmul v24.2s, v0.2s, v9.2s[0]
+ fmul v21.2s, v1.2s, v8.2s[1]
+
+ fmul v28.2s, v0.2s, v9.2s[1]
+ fmul v17.2s, v1.2s, v8.2s[0]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.2s, v5.2s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x4_M1
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ ld1 {v4.2s, v5.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro KERNEL4x4_M2
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ ld1 {v0.2s, v1.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_E
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x4
+
+ fmul v8.2s, v16.2s, alphaV0
+ fmul v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2s, v20.2s, alphaV2
+ fmul v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v8.2s, v24.2s, alphaV0
+ fmul v9.2s, v25.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2s, v28.2s, alphaV2
+ fmul v13.2s, v29.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov s16, wzr
+ fmov s20, s16
+ fmov s24, s20
+ fmov s28, s16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v28.2s, v0.2s, v9.2s[1]
+.endm
+
+.macro SAVE2x4
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+ fmul v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ fmul v8.2s, v24.2s, alphaV2
+ st1 {v8.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+ fmul v12.2s, v28.2s, alphaV3
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr s0, [pA]
+ add pA, pA, #4
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+
+ fmla v16.2s, v8.2s, v0.s[0]
+ fmla v20.2s, v9.2s, v0.s[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2s, v20.2s, alphaV1
+ st1 {v12.s}[0], [pCRow2]
+ st1 {v12.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT16x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, wzr
+ fmov s23, s16
+.endm
+
+.macro KERNEL16x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v22.4s, v2.4s, v8.2s[1]
+ fmla v23.4s, v3.4s, v8.2s[1]
+.endm
+
+.macro SAVE16x2
+ add pCRow1, pCRow0, LDC
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ fmul v2.4s, v18.4s, alphaV2
+ fmul v3.4s, v19.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+
+ fmul v4.4s, v20.4s, alphaV0
+ fmul v5.4s, v21.4s, alphaV1
+ fmul v6.4s, v22.4s, alphaV2
+ fmul v7.4s, v23.4s, alphaV3
+ st1 {v4.4s, v5.4s, v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL8x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+.endm
+
+.macro SAVE8x2
+ add pCRow1, pCRow0, LDC
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v4.4s, v20.4s, alphaV0
+ fmul v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v21.2s, v1.2s, v8.2s[1]
+.endm
+
+.macro SAVE4x2
+
+ fmul v8.2s, v16.2s, alphaV0
+ fmul v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2s, v20.2s, alphaV2
+ fmul v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+.endm
+
+.macro SAVE2x2
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+
+ fmul v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2s} , [pB]
+ add pB , pB, #8
+
+ ldr s0 , [pA]
+ add pA, pA, #4
+
+ fmla v16.2s, v8.2s, v0.2s[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT16x1
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, wzr
+ fmov s19, s16
+.endm
+
+.macro KERNEL16x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v18.4s, v2.4s, v8.2s[0]
+ fmla v19.4s, v3.4s, v8.2s[0]
+.endm
+
+.macro SAVE16x1
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ fmul v2.4s, v18.4s, alphaV2
+ fmul v3.4s, v19.4s, alphaV3
+ st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [pCRow0]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL8x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+.endm
+
+.macro SAVE8x1
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov s16, wzr
+ fmov s17, s16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x1
+
+ fmul v8.2s, v16.2s, alphaV0
+ fmul v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s}, [pA]
+ add pA , pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+.endm
+
+.macro SAVE2x1
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ldr s0, [pA]
+ add pA , pA, #4
+
+ fmadd s16, s0, s8, s16
+.endm
+
+.macro SAVE1x1
+ fmul s8, s16, alpha0
+ str s8, [pCRow0]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+strmm_kernel_begin:
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0, s0
+ fmov alpha1, s0
+ fmov alpha2, s0
+ fmov alpha3, s0
+
+ lsl LDC, LDC, #2 // ldc = ldc * 4
+
+#if !defined(LEFT)
+ neg tempOffset, offset
+#endif
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #2 // J = J / 4
+ cmp counterJ, #0
+ ble strmm_kernel_L2_BEGIN
+
+/******************************************************************************/
+
+strmm_kernel_L4_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = start of A array
+
+strmm_kernel_L4_M16_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #4 // counterI = counterI / 16
+ cmp counterI, #0
+ ble strmm_kernel_L4_M8_BEGIN
+
+strmm_kernel_L4_M16_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #16
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt strmm_kernel_L4_M16_32
+
+ KERNEL16x4_I // do one in the K
+ KERNEL16x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble strmm_kernel_L4_M16_22a
+ .align 5
+
+strmm_kernel_L4_M16_22:
+
+ KERNEL16x4_M1
+ KERNEL16x4_M2
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M16_22
+
+strmm_kernel_L4_M16_22a:
+
+ KERNEL16x4_M1
+ KERNEL16x4_E
+
+ b strmm_kernel_L4_M16_44
+
+strmm_kernel_L4_M16_32:
+
+ tst counterL, #1
+ ble strmm_kernel_L4_M16_40
+
+ KERNEL16x4_I
+ KERNEL16x4_E
+
+ b strmm_kernel_L4_M16_44
+
+strmm_kernel_L4_M16_40:
+
+ INIT16x4
+
+strmm_kernel_L4_M16_44:
+
+ ands counterL , tempK, #1
+ ble strmm_kernel_L4_M16_100
+
+strmm_kernel_L4_M16_46:
+
+ KERNEL16x4_SUB
+
+strmm_kernel_L4_M16_100:
+
+ SAVE16x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #16
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #16
+#endif
+
+strmm_kernel_L4_M16_END:
+ subs counterI, counterI, #1
+ bne strmm_kernel_L4_M16_20
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #15
+ ble strmm_kernel_L4_END
+
+ tst counterI, #8
+ ble strmm_kernel_L4_M4_BEGIN
+
+strmm_kernel_L4_M8_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt strmm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble strmm_kernel_L4_M8_22a
+ .align 5
+
+strmm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M8_22
+
+strmm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b strmm_kernel_L4_M8_44
+
+strmm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble strmm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+ KERNEL8x4_E
+
+ b strmm_kernel_L4_M8_44
+
+strmm_kernel_L4_M8_40:
+
+ INIT8x4
+
+strmm_kernel_L4_M8_44:
+
+ ands counterL , tempK, #1
+ ble strmm_kernel_L4_M8_100
+
+strmm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+strmm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+strmm_kernel_L4_M8_END:
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble strmm_kernel_L4_END
+
+ tst counterI, #4
+ ble strmm_kernel_L4_M2_BEGIN
+
+strmm_kernel_L4_M4_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt strmm_kernel_L4_M4_32
+
+ KERNEL4x4_I // do one in the K
+ KERNEL4x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble strmm_kernel_L4_M4_22a
+ .align 5
+
+strmm_kernel_L4_M4_22:
+
+ KERNEL4x4_M1
+ KERNEL4x4_M2
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M4_22
+
+strmm_kernel_L4_M4_22a:
+
+ KERNEL4x4_M1
+ KERNEL4x4_E
+
+ b strmm_kernel_L4_M4_44
+
+strmm_kernel_L4_M4_32:
+
+ tst counterL, #1
+ ble strmm_kernel_L4_M4_40
+
+ KERNEL4x4_I
+ KERNEL4x4_E
+
+ b strmm_kernel_L4_M4_44
+
+strmm_kernel_L4_M4_40:
+
+ INIT4x4
+
+strmm_kernel_L4_M4_44:
+
+ ands counterL , tempK, #1
+ ble strmm_kernel_L4_M4_100
+
+strmm_kernel_L4_M4_46:
+
+ KERNEL4x4_SUB
+
+strmm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+strmm_kernel_L4_M4_END:
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble strmm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble strmm_kernel_L4_M1_BEGIN
+
+strmm_kernel_L4_M2_20:
+
+ INIT2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L4_M2_40
+
+strmm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M2_22
+
+
+strmm_kernel_L4_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L4_M2_100
+
+strmm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M2_42
+
+strmm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+strmm_kernel_L4_M2_END:
+
+
+strmm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble strmm_kernel_L4_END
+
+strmm_kernel_L4_M1_20:
+
+ INIT1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #2
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L4_M1_40
+
+strmm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M1_22
+
+
+strmm_kernel_L4_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L4_M1_100
+
+strmm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M1_42
+
+strmm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #2
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+strmm_kernel_L4_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+ subs counterJ, counterJ , #1 // j--
+ bgt strmm_kernel_L4_BEGIN
+
+
+/******************************************************************************/
+
+strmm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble strmm_kernel_L999
+
+ tst counterJ , #2
+ ble strmm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+strmm_kernel_L2_M16_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #4 // counterI = counterI / 16
+ cmp counterI,#0
+ ble strmm_kernel_L2_M8_BEGIN
+
+strmm_kernel_L2_M16_20:
+
+ INIT16x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #16
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble strmm_kernel_L2_M16_40
+ .align 5
+
+strmm_kernel_L2_M16_22:
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+ KERNEL16x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M16_22
+
+
+strmm_kernel_L2_M16_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M16_100
+
+strmm_kernel_L2_M16_42:
+
+ KERNEL16x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M16_42
+
+strmm_kernel_L2_M16_100:
+
+ SAVE16x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #16
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #16
+#endif
+
+strmm_kernel_L2_M16_END:
+
+ subs counterI, counterI, #1
+ bgt strmm_kernel_L2_M16_20
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L2_M8_BEGIN:
+ mov counterI, origM
+ tst counterI , #15
+ ble strmm_kernel_L2_END
+
+ tst counterI, #8
+ ble strmm_kernel_L2_M4_BEGIN
+
+strmm_kernel_L2_M8_20:
+
+ INIT8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #2
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble strmm_kernel_L2_M8_40
+ .align 5
+
+strmm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M8_22
+
+
+strmm_kernel_L2_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M8_100
+
+strmm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M8_42
+
+strmm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+strmm_kernel_L2_M8_END:
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L2_M4_BEGIN:
+ mov counterI, origM
+ tst counterI , #7
+ ble strmm_kernel_L2_END
+
+ tst counterI, #4
+ ble strmm_kernel_L2_M2_BEGIN
+
+strmm_kernel_L2_M4_20:
+
+ INIT4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble strmm_kernel_L2_M4_40
+ .align 5
+
+strmm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M4_22
+
+
+strmm_kernel_L2_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M4_100
+
+strmm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M4_42
+
+strmm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+strmm_kernel_L2_M4_END:
+
+//------------------------------------------------------------------------------
+
+
+strmm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble strmm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble strmm_kernel_L2_M1_BEGIN
+
+strmm_kernel_L2_M2_20:
+
+ INIT2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble strmm_kernel_L2_M2_40
+
+strmm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M2_22
+
+
+strmm_kernel_L2_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M2_100
+
+strmm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M2_42
+
+strmm_kernel_L2_M2_100:
+
+ SAVE2x2
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+strmm_kernel_L2_M2_END:
+
+
+strmm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble strmm_kernel_L2_END
+
+strmm_kernel_L2_M1_20:
+
+ INIT1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #2
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble strmm_kernel_L2_M1_40
+
+strmm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M1_22
+
+
+strmm_kernel_L2_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M1_100
+
+strmm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M1_42
+
+strmm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #2
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+strmm_kernel_L2_END:
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+ add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4
+
+/******************************************************************************/
+
+strmm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble strmm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+strmm_kernel_L1_M16_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #4 // counterI = counterI / 16
+ cmp counterI, #0
+ ble strmm_kernel_L1_M8_BEGIN
+
+strmm_kernel_L1_M16_20:
+
+ INIT16x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #6
+ add pA, pA, temp
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #16
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M16_40
+ .align 5
+
+strmm_kernel_L1_M16_22:
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+ KERNEL16x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M16_22
+
+
+strmm_kernel_L1_M16_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M16_100
+
+strmm_kernel_L1_M16_42:
+
+ KERNEL16x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M16_42
+
+strmm_kernel_L1_M16_100:
+
+ SAVE16x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #16
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #6
+ add pA, pA, temp
+ lsl temp, tempK, #2
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #16
+#endif
+
+strmm_kernel_L1_M16_END:
+
+ subs counterI, counterI, #1
+ bgt strmm_kernel_L1_M16_20
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #15
+ ble strmm_kernel_L1_END
+
+ tst counterI, #8
+ ble strmm_kernel_L1_M4_BEGIN
+
+strmm_kernel_L1_M8_20:
+
+ INIT8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #1
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M8_40
+ .align 5
+
+strmm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M8_22
+
+
+strmm_kernel_L1_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M8_100
+
+strmm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M8_42
+
+strmm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #2
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+strmm_kernel_L1_M8_END:
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L1_M4_BEGIN:
+ mov counterI, origM
+ tst counterI , #7
+ ble strmm_kernel_L1_END
+
+ tst counterI, #4
+ ble strmm_kernel_L1_M2_BEGIN
+
+strmm_kernel_L1_M4_20:
+
+ INIT4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M4_40
+ .align 5
+
+strmm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M4_22
+
+
+strmm_kernel_L1_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M4_100
+
+strmm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M4_42
+
+strmm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #2
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+strmm_kernel_L1_M4_END:
+
+//------------------------------------------------------------------------------
+
+strmm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble strmm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble strmm_kernel_L1_M1_BEGIN
+
+strmm_kernel_L1_M2_20:
+
+ INIT2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M2_40
+
+strmm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M2_22
+
+
+strmm_kernel_L1_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M2_100
+
+strmm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M2_42
+
+strmm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #2
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+strmm_kernel_L1_M2_END:
+
+
+strmm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble strmm_kernel_L1_END
+
+strmm_kernel_L1_M1_20:
+
+ INIT1x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+ lsl temp, tempOffset, #2
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M1_40
+
+strmm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M1_22
+
+
+strmm_kernel_L1_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M1_100
+
+strmm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M1_42
+
+strmm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+strmm_kernel_L1_END:
+
+strmm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 x7 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc, BLASLONG offset) */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define offset x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+#define temp x16
+#define tempOffset x17
+#define tempK x18
+
+#define alpha0 s10
+#define alphaV0 v10.s[0]
+#define alpha1 s11
+#define alphaV1 v11.s[0]
+#define alpha2 s14
+#define alphaV2 v14.s[0]
+#define alpha3 s15
+#define alphaV3 v15.s[0]
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 temp
+// 17 tempOffset
+// 18 must save tempK
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA -> pA0_0, pA0_1, pA0_2, pA0_3
+//v01 pA0_4, pA0_5, pA0_6, pA0_7
+//v02 pA1_0, pA1_1, pA1_2, pA1_3
+//v03 pA1_4, pA1_5, pA1_6, pA1_7
+//v04 pB0_0, pB0_1, pB0_2, pB0_3
+//v05 pB0_4, pB0_5, pB0_6, pB0_7
+//v06 pB1_0, pB1_1, pB1_2, pB1_3
+//v07 pB1_4, pB1_5, pB1_6, pB1_7
+//v08 must save
+//v09 must save
+//v10 must save ALPHA0
+//v11 must save ALPHA1
+//v12 must save
+//v13 must save
+//v14 must save ALPHA2
+//v15 must save ALPHA3
+//v16 must save C00, C01, C02, C03
+//v17 must save C04, C05, C06, C07
+//v18 C08, C09, C10, C11
+//v19 C12, C13, C14, C15
+//v20 C16, C17, C18, C19
+//v21 C20, C21, C22, C23
+//v22 C24, C25, C26, C27
+//v23 C28, C29, C30, C31
+//v24 C32, C33, C34, C35
+//v25 C36, C37, C38, C39
+//v26 C40, C41, C42, C43
+//v27 C44, C45, C46, C47
+//v28 C48, C49, C50, C51
+//v29 C52, C53, C54, C55
+//v30 C56, C57, C58, C59
+//v31 C60, C61, C62, C63
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT8x8
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s18, s16
+ fmov s19, s17
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s18
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s26, s17
+ fmov s27, s18
+ fmov s28, wzr
+ fmov s29, s16
+ fmov s30, s17
+ fmov s31, s18
+.endm
+
+.macro KERNEL8x8_I
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v4.4s[0]
+ fmul v17.4s, v1.4s, v4.4s[0]
+ fmul v18.4s, v0.4s, v4.4s[1]
+ fmul v19.4s, v1.4s, v4.4s[1]
+ fmul v20.4s, v0.4s, v4.4s[2]
+ fmul v21.4s, v1.4s, v4.4s[2]
+ fmul v22.4s, v0.4s, v4.4s[3]
+ fmul v23.4s, v1.4s, v4.4s[3]
+ fmul v24.4s, v0.4s, v5.4s[0]
+ fmul v25.4s, v1.4s, v5.4s[0]
+ fmul v26.4s, v0.4s, v5.4s[1]
+ fmul v27.4s, v1.4s, v5.4s[1]
+ fmul v28.4s, v0.4s, v5.4s[2]
+ fmul v29.4s, v1.4s, v5.4s[2]
+ fmul v30.4s, v0.4s, v5.4s[3]
+ fmul v31.4s, v1.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x8_M1
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v17.4s, v1.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v19.4s, v1.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v21.4s, v1.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v23.4s, v1.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v25.4s, v1.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v27.4s, v1.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v29.4s, v1.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+ fmla v31.4s, v1.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v3.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x8_M2
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v17.4s, v3.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v19.4s, v3.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v21.4s, v3.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v23.4s, v3.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v25.4s, v3.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v27.4s, v3.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v29.4s, v3.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+ fmla v31.4s, v3.4s, v7.4s[3]
+
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x8_E
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v17.4s, v3.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v19.4s, v3.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v21.4s, v3.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v23.4s, v3.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v25.4s, v3.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v27.4s, v3.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v29.4s, v3.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+ fmla v31.4s, v3.4s, v7.4s[3]
+.endm
+
+.macro KERNEL8x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v17.4s, v1.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v19.4s, v1.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v21.4s, v1.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v23.4s, v1.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v25.4s, v1.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v27.4s, v1.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v29.4s, v1.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+ fmla v31.4s, v1.4s, v5.4s[3]
+.endm
+
+.macro SAVE8x8
+ add pCRow1, pCRow0, LDC
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v2.4s, v18.4s, alphaV2
+ fmul v3.4s, v19.4s, alphaV3
+ st1 {v2.4s, v3.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v4.4s, v20.4s, alphaV0
+ fmul v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v6.4s, v22.4s, alphaV2
+ fmul v7.4s, v23.4s, alphaV3
+ st1 {v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v0.4s, v24.4s, alphaV0
+ fmul v1.4s, v25.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v2.4s, v26.4s, alphaV2
+ fmul v3.4s, v27.4s, alphaV3
+ st1 {v2.4s, v3.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v4.4s, v28.4s, alphaV0
+ fmul v5.4s, v29.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow2]
+
+ fmul v6.4s, v30.4s, alphaV2
+ fmul v7.4s, v31.4s, alphaV3
+ st1 {v6.4s, v7.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+
+.macro INIT4x8
+ fmov s16, wzr
+ fmov s18, wzr
+ fmov s20, wzr
+ fmov s22, s16
+ fmov s24, wzr
+ fmov s26, s16
+ fmov s28, s18
+ fmov s30, s20
+.endm
+
+.macro KERNEL4x8_I
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v4.4s[0]
+ fmul v18.4s, v0.4s, v4.4s[1]
+ fmul v20.4s, v0.4s, v4.4s[2]
+ fmul v22.4s, v0.4s, v4.4s[3]
+ fmul v24.4s, v0.4s, v5.4s[0]
+ fmul v26.4s, v0.4s, v5.4s[1]
+ fmul v28.4s, v0.4s, v5.4s[2]
+ fmul v30.4s, v0.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x8_M1
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+
+ ld1 {v6.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v7.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v2.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x8_M2
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x8_E
+ fmla v16.4s, v2.4s, v6.4s[0]
+ fmla v18.4s, v2.4s, v6.4s[1]
+ fmla v20.4s, v2.4s, v6.4s[2]
+ fmla v22.4s, v2.4s, v6.4s[3]
+ fmla v24.4s, v2.4s, v7.4s[0]
+ fmla v26.4s, v2.4s, v7.4s[1]
+ fmla v28.4s, v2.4s, v7.4s[2]
+ fmla v30.4s, v2.4s, v7.4s[3]
+.endm
+
+.macro KERNEL4x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v4.4s[0]
+ fmla v18.4s, v0.4s, v4.4s[1]
+ fmla v20.4s, v0.4s, v4.4s[2]
+ fmla v22.4s, v0.4s, v4.4s[3]
+ fmla v24.4s, v0.4s, v5.4s[0]
+ fmla v26.4s, v0.4s, v5.4s[1]
+ fmla v28.4s, v0.4s, v5.4s[2]
+ fmla v30.4s, v0.4s, v5.4s[3]
+.endm
+
+.macro SAVE4x8
+ add pCRow1, pCRow0, LDC
+
+
+ fmul v0.4s, v16.4s, alphaV0
+ st1 {v0.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v2.4s, v18.4s, alphaV2
+ st1 {v2.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v4.4s, v20.4s, alphaV0
+ st1 {v4.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v6.4s, v22.4s, alphaV2
+ st1 {v6.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v0.4s, v24.4s, alphaV0
+ st1 {v0.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v2.4s, v26.4s, alphaV2
+ st1 {v2.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v4.4s, v28.4s, alphaV0
+ st1 {v4.4s}, [pCRow2]
+
+
+ fmul v6.4s, v30.4s, alphaV2
+ st1 {v6.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x8
+ fmov s16, wzr
+ fmov s18, wzr
+ fmov s20, wzr
+ fmov s22, s16
+ fmov s24, wzr
+ fmov s26, s16
+ fmov s28, s18
+ fmov s30, s20
+.endm
+
+.macro KERNEL2x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v4.4s[0]
+ fmla v18.2s, v0.2s, v4.4s[1]
+ fmla v20.2s, v0.2s, v4.4s[2]
+ fmla v22.2s, v0.2s, v4.4s[3]
+ fmla v24.2s, v0.2s, v5.4s[0]
+ fmla v26.2s, v0.2s, v5.4s[1]
+ fmla v28.2s, v0.2s, v5.4s[2]
+ fmla v30.2s, v0.2s, v5.4s[3]
+.endm
+
+.macro SAVE2x8
+ add pCRow1, pCRow0, LDC
+
+
+ fmul v0.2s, v16.2s, alphaV0
+ st1 {v0.2s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v2.2s, v18.2s, alphaV2
+ st1 {v2.2s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v4.2s, v20.2s, alphaV0
+ st1 {v4.2s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v6.2s, v22.2s, alphaV2
+ st1 {v6.2s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v0.2s, v24.2s, alphaV0
+ st1 {v0.2s}, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v2.2s, v26.2s, alphaV2
+ st1 {v2.2s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v4.2s, v28.2s, alphaV0
+ st1 {v4.2s}, [pCRow2]
+
+
+ fmul v6.2s, v30.2s, alphaV2
+ st1 {v6.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x8
+ fmov s16, wzr
+ fmov s18, wzr
+ fmov s20, wzr
+ fmov s22, s16
+ fmov s24, wzr
+ fmov s26, s16
+ fmov s28, s18
+ fmov s30, s20
+.endm
+
+.macro KERNEL1x8_SUB
+ ld1 {v4.4s}, [pB]
+ add pB, pB, #16
+ ld1 {v5.4s}, [pB]
+ add pB, pB, #16
+ ldr s0, [pA]
+ add pA, pA, #4
+
+ fmla s16, s0, v4.4s[0]
+ fmla s18, s0, v4.4s[1]
+ fmla s20, s0, v4.4s[2]
+ fmla s22, s0, v4.4s[3]
+ fmla s24, s0, v5.4s[0]
+ fmla s26, s0, v5.4s[1]
+ fmla s28, s0, v5.4s[2]
+ fmla s30, s0, v5.4s[3]
+.endm
+
+.macro SAVE1x8
+ add pCRow1, pCRow0, LDC
+
+
+ fmul s0, s16, alphaV0
+ str s0, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul s2, s18, alphaV2
+ str s2, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul s4, s20, alphaV0
+ str s4, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul s6, s22, alphaV2
+ str s6, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul s0, s24, alphaV0
+ str s0, [pCRow2]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul s2, s26, alphaV2
+ str s2, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul s4, s28, alphaV0
+ str s4, [pCRow2]
+
+
+ fmul s6, s30, alphaV2
+ str s6, [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, wzr
+ fmov s21, s16
+ fmov s24, wzr
+ fmov s25, s16
+ fmov s28, wzr
+ fmov s29, s16
+.endm
+
+.macro KERNEL8x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.4s, v0.4s, v8.2s[0]
+ fmul v17.4s, v1.4s, v8.2s[0]
+ fmul v20.4s, v0.4s, v8.2s[1]
+ fmul v21.4s, v1.4s, v8.2s[1]
+ fmul v24.4s, v0.4s, v9.2s[0]
+ fmul v25.4s, v1.4s, v9.2s[0]
+ fmul v28.4s, v0.4s, v9.2s[1]
+ fmul v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M1
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v5.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_M2
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL8x4_E
+ fmla v16.4s, v4.4s, v12.2s[0]
+ fmla v17.4s, v5.4s, v12.2s[0]
+ fmla v20.4s, v4.4s, v12.2s[1]
+ fmla v21.4s, v5.4s, v12.2s[1]
+ fmla v24.4s, v4.4s, v13.2s[0]
+ fmla v25.4s, v5.4s, v13.2s[0]
+ fmla v28.4s, v4.4s, v13.2s[1]
+ fmla v29.4s, v5.4s, v13.2s[1]
+.endm
+
+.macro KERNEL8x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+ fmla v24.4s, v0.4s, v9.2s[0]
+ fmla v25.4s, v1.4s, v9.2s[0]
+ fmla v28.4s, v0.4s, v9.2s[1]
+ fmla v29.4s, v1.4s, v9.2s[1]
+.endm
+
+.macro SAVE8x4
+ add pCRow1, pCRow0, LDC
+
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v4.4s, v20.4s, alphaV0
+ fmul v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v0.4s, v24.4s, alphaV0
+ fmul v1.4s, v25.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow2]
+
+
+ fmul v4.4s, v28.4s, alphaV0
+ fmul v5.4s, v29.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+
+.macro INIT4x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s28, s17
+ fmov s29, s16
+.endm
+
+.macro KERNEL4x4_I
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmul v16.2s, v0.2s, v8.2s[0]
+ fmul v29.2s, v1.2s, v9.2s[1]
+
+ fmul v20.2s, v0.2s, v8.2s[1]
+ fmul v25.2s, v1.2s, v9.2s[0]
+
+ fmul v24.2s, v0.2s, v9.2s[0]
+ fmul v21.2s, v1.2s, v8.2s[1]
+
+ fmul v28.2s, v0.2s, v9.2s[1]
+ fmul v17.2s, v1.2s, v8.2s[0]
+
+ ld1 {v12.2s, v13.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v4.2s, v5.2s}, [pA]
+ add pA, pA, #16
+.endm
+
+.macro KERNEL4x4_M1
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ ld1 {v12.2s, v13.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ ld1 {v4.2s, v5.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro KERNEL4x4_M2
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ ld1 {v8.2s, v9.2s}, [pB] // For next round
+ add pB, pB, #16
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ ld1 {v0.2s, v1.2s}, [pA] // For next round
+ add pA, pA, #16
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_E
+ fmla v16.2s, v4.2s, v12.2s[0]
+ fmla v29.2s, v5.2s, v13.2s[1]
+
+ fmla v20.2s, v4.2s, v12.2s[1]
+ fmla v25.2s, v5.2s, v13.2s[0]
+
+ fmla v24.2s, v4.2s, v13.2s[0]
+ fmla v21.2s, v5.2s, v12.2s[1]
+
+ fmla v28.2s, v4.2s, v13.2s[1]
+ fmla v17.2s, v5.2s, v12.2s[0]
+.endm
+
+.macro KERNEL4x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v29.2s, v1.2s, v9.2s[1]
+
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v25.2s, v1.2s, v9.2s[0]
+
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v21.2s, v1.2s, v8.2s[1]
+
+ fmla v28.2s, v0.2s, v9.2s[1]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x4
+
+ fmul v8.2s, v16.2s, alphaV0
+ fmul v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2s, v20.2s, alphaV2
+ fmul v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v8.2s, v24.2s, alphaV0
+ fmul v9.2s, v25.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2s, v28.2s, alphaV2
+ fmul v13.2s, v29.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov s16, wzr
+ fmov s20, s16
+ fmov s24, s20
+ fmov s28, s16
+.endm
+
+.macro KERNEL2x4_SUB
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v24.2s, v0.2s, v9.2s[0]
+ fmla v28.2s, v0.2s, v9.2s[1]
+.endm
+
+.macro SAVE2x4
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+
+ fmul v8.2s, v24.2s, alphaV2
+ st1 {v8.2s}, [pCRow2]
+
+ add pCRow1, pCRow2, LDC
+
+ fmul v12.2s, v28.2s, alphaV3
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL1x4_SUB
+ ldr s0, [pA]
+ add pA, pA, #4
+
+ ld1 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+
+ fmla v16.2s, v8.2s, v0.s[0]
+ fmla v20.2s, v9.2s, v0.s[0]
+.endm
+
+.macro SAVE1x4
+ add pCRow1, pCRow0, LDC
+
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow2, pCRow1, LDC
+ add pCRow1, pCRow2, LDC
+
+
+ fmul v12.2s, v20.2s, alphaV1
+ st1 {v12.s}[0], [pCRow2]
+ st1 {v12.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL8x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+
+ fmla v20.4s, v0.4s, v8.2s[1]
+ fmla v21.4s, v1.4s, v8.2s[1]
+.endm
+
+.macro SAVE8x2
+ add pCRow1, pCRow0, LDC
+
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow2, pCRow1, LDC
+
+
+ fmul v4.4s, v20.4s, alphaV0
+ fmul v5.4s, v21.4s, alphaV1
+ st1 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+.endm
+
+.macro KERNEL4x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+ fmla v21.2s, v1.2s, v8.2s[1]
+.endm
+
+.macro SAVE4x2
+
+ fmul v8.2s, v16.2s, alphaV0
+ fmul v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow1, pCRow0, LDC
+
+ fmul v12.2s, v20.2s, alphaV2
+ fmul v13.2s, v21.2s, alphaV3
+ st1 {v12.2s, v13.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov s16, wzr
+ fmov s20, s16
+.endm
+
+.macro KERNEL2x2_SUB
+ ld1 {v8.2s}, [pB]
+ add pB, pB, #8
+
+ ld1 {v0.2s}, [pA]
+ add pA, pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v20.2s, v0.2s, v8.2s[1]
+.endm
+
+.macro SAVE2x2
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow1 , pCRow0, LDC
+
+ fmul v12.2s, v20.2s, alphaV1
+ st1 {v12.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld1 {v8.2s} , [pB]
+ add pB , pB, #8
+
+ ldr s0 , [pA]
+ add pA, pA, #4
+
+ fmla v16.2s, v8.2s, v0.2s[0]
+.endm
+
+.macro SAVE1x2
+ add pCRow1 , pCRow0, LDC
+
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.s}[0], [pCRow0]
+ st1 {v8.s}[1], [pCRow1]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/******************************************************************************/
+
+.macro INIT8x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL8x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.4s}, [pA]
+ add pA, pA, #16
+ ld1 {v1.4s}, [pA]
+ add pA, pA, #16
+
+ fmla v16.4s, v0.4s, v8.2s[0]
+ fmla v17.4s, v1.4s, v8.2s[0]
+.endm
+
+.macro SAVE8x1
+
+ fmul v0.4s, v16.4s, alphaV0
+ fmul v1.4s, v17.4s, alphaV1
+ st1 {v0.4s, v1.4s}, [pCRow0]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov s16, wzr
+ fmov s17, s16
+.endm
+
+.macro KERNEL4x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s, v1.2s}, [pA]
+ add pA , pA, #16
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+ fmla v17.2s, v1.2s, v8.2s[0]
+.endm
+
+.macro SAVE4x1
+
+ fmul v8.2s, v16.2s, alphaV0
+ fmul v9.2s, v17.2s, alphaV1
+ st1 {v8.2s, v9.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ld1 {v0.2s}, [pA]
+ add pA , pA, #8
+
+ fmla v16.2s, v0.2s, v8.2s[0]
+.endm
+
+.macro SAVE2x1
+
+ fmul v8.2s, v16.2s, alphaV0
+ st1 {v8.2s}, [pCRow0]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov s16, wzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ldr s8, [pB]
+ add pB , pB, #4
+
+ ldr s0, [pA]
+ add pA , pA, #4
+
+ fmadd s16, s0, s8, s16
+.endm
+
+.macro SAVE1x1
+
+ fmul s8, s16, alpha0
+ str s8, [pCRow0]
+
+ add pCRow0, pCRow0, #4
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+strmm_kernel_begin:
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0, s0
+ fmov alpha1, s0
+ fmov alpha2, s0
+ fmov alpha3, s0
+
+ lsl LDC, LDC, #2 // ldc = ldc * 4
+
+#if !defined(LEFT)
+ neg tempOffset, offset
+#endif
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #3 // J = J / 8
+ cmp counterJ, #0
+ ble strmm_kernel_L4_BEGIN
+
+/******************************************************************************/
+/******************************************************************************/
+
+strmm_kernel_L8_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #3
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+
+ mov pA, origPA // pA = start of A array
+
+/******************************************************************************/
+
+strmm_kernel_L8_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble strmm_kernel_L8_M4_BEGIN
+
+strmm_kernel_L8_M8_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #8
+#endif
+
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt strmm_kernel_L8_M8_32
+
+ KERNEL8x8_I // do one in the K
+ KERNEL8x8_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble strmm_kernel_L8_M8_22a
+ .align 5
+
+strmm_kernel_L8_M8_22:
+
+ KERNEL8x8_M1
+ KERNEL8x8_M2
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L8_M8_22
+
+strmm_kernel_L8_M8_22a:
+
+ KERNEL8x8_M1
+ KERNEL8x8_E
+
+ b strmm_kernel_L8_M8_44
+
+strmm_kernel_L8_M8_32:
+
+ tst counterL, #1
+ ble strmm_kernel_L8_M8_40
+
+ KERNEL8x8_I
+ KERNEL8x8_E
+
+ b strmm_kernel_L8_M8_44
+
+strmm_kernel_L8_M8_40:
+
+ INIT8x8
+
+strmm_kernel_L8_M8_44:
+
+ ands counterL , tempK, #1
+ ble strmm_kernel_L8_M8_100
+
+strmm_kernel_L8_M8_46:
+
+ KERNEL8x8_SUB
+
+strmm_kernel_L8_M8_100:
+
+ SAVE8x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #8
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+strmm_kernel_L8_M8_END:
+ subs counterI, counterI, #1
+ bne strmm_kernel_L8_M8_20
+
+/******************************************************************************/
+
+strmm_kernel_L8_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble strmm_kernel_L8_END
+
+ tst counterI, #4
+ ble strmm_kernel_L8_M2_BEGIN
+
+strmm_kernel_L8_M4_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #8
+#endif
+
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt strmm_kernel_L8_M4_32
+
+ KERNEL4x8_I // do one in the K
+ KERNEL4x8_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble strmm_kernel_L8_M4_22a
+ .align 5
+
+strmm_kernel_L8_M4_22:
+
+ KERNEL4x8_M1
+ KERNEL4x8_M2
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L8_M4_22
+
+strmm_kernel_L8_M4_22a:
+
+ KERNEL4x8_M1
+ KERNEL4x8_E
+
+ b strmm_kernel_L8_M4_44
+
+strmm_kernel_L8_M4_32:
+
+ tst counterL, #1
+ ble strmm_kernel_L8_M4_40
+
+ KERNEL4x8_I
+ KERNEL4x8_E
+
+ b strmm_kernel_L8_M4_44
+
+strmm_kernel_L8_M4_40:
+
+ INIT4x8
+
+strmm_kernel_L8_M4_44:
+
+ ands counterL , tempK, #1
+ ble strmm_kernel_L8_M4_100
+
+strmm_kernel_L8_M4_46:
+
+ KERNEL4x8_SUB
+
+strmm_kernel_L8_M4_100:
+
+ SAVE4x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #8
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+strmm_kernel_L8_M4_END:
+
+/******************************************************************************/
+
+strmm_kernel_L8_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble strmm_kernel_L8_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble strmm_kernel_L8_M1_BEGIN
+
+strmm_kernel_L8_M2_20:
+
+ INIT2x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #8
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L8_M2_40
+
+strmm_kernel_L8_M2_22:
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L8_M2_22
+
+
+strmm_kernel_L8_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L8_M2_100
+
+strmm_kernel_L8_M2_42:
+
+ KERNEL2x8_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L8_M2_42
+
+strmm_kernel_L8_M2_100:
+
+ SAVE2x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #8
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+strmm_kernel_L8_M2_END:
+
+/******************************************************************************/
+
+strmm_kernel_L8_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble strmm_kernel_L8_END
+
+strmm_kernel_L8_M1_20:
+
+ INIT1x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #2
+ add pA, pA, temp
+ lsl temp, tempOffset, #5
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #8
+#endif
+
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L8_M1_40
+
+strmm_kernel_L8_M1_22:
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L8_M1_22
+
+
+strmm_kernel_L8_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L8_M1_100
+
+strmm_kernel_L8_M1_42:
+
+ KERNEL1x8_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L8_M1_42
+
+strmm_kernel_L8_M1_100:
+
+ SAVE1x8
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #8
+#endif
+ lsl temp, tempK, #2
+ add pA, pA, temp
+ lsl temp, tempK, #5
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+
+strmm_kernel_L8_END:
+ lsl temp, origK, #5 // B = B + K * 4 * 8
+ add origPB, origPB, temp
+
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+
+ subs counterJ, counterJ , #1 // j--
+ bgt strmm_kernel_L8_BEGIN
+
+/******************************************************************************/
+/******************************************************************************/
+
+strmm_kernel_L4_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #7
+ ble strmm_kernel_L999
+
+ tst counterJ , #4
+ ble strmm_kernel_L2_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #2
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+
+ mov pA, origPA // pA = A
+
+/******************************************************************************/
+
+strmm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble strmm_kernel_L4_M4_BEGIN
+
+strmm_kernel_L4_M8_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #4
+#endif
+
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt strmm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble strmm_kernel_L4_M8_22a
+ .align 5
+
+strmm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M8_22
+
+strmm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b strmm_kernel_L4_M8_44
+
+strmm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble strmm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+ KERNEL8x4_E
+
+ b strmm_kernel_L4_M8_44
+
+strmm_kernel_L4_M8_40:
+
+ INIT8x4
+
+strmm_kernel_L4_M8_44:
+
+ ands counterL , tempK, #1
+ ble strmm_kernel_L4_M8_100
+
+strmm_kernel_L4_M8_46:
+
+ KERNEL8x4_SUB
+
+strmm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+strmm_kernel_L4_M8_END:
+ subs counterI, counterI, #1
+ bne strmm_kernel_L4_M8_20
+
+/******************************************************************************/
+
+strmm_kernel_L4_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble strmm_kernel_L4_END
+
+ tst counterI, #4
+ ble strmm_kernel_L4_M2_BEGIN
+
+strmm_kernel_L4_M4_20:
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt strmm_kernel_L4_M4_32
+
+ KERNEL4x4_I // do one in the K
+ KERNEL4x4_M2 // do another in the K
+
+ subs counterL, counterL, #2
+ ble strmm_kernel_L4_M4_22a
+ .align 5
+
+strmm_kernel_L4_M4_22:
+
+ KERNEL4x4_M1
+ KERNEL4x4_M2
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M4_22
+
+strmm_kernel_L4_M4_22a:
+
+ KERNEL4x4_M1
+ KERNEL4x4_E
+
+ b strmm_kernel_L4_M4_44
+
+strmm_kernel_L4_M4_32:
+
+ tst counterL, #1
+ ble strmm_kernel_L4_M4_40
+
+ KERNEL4x4_I
+ KERNEL4x4_E
+
+ b strmm_kernel_L4_M4_44
+
+strmm_kernel_L4_M4_40:
+
+ INIT4x4
+
+strmm_kernel_L4_M4_44:
+
+ ands counterL , tempK, #1
+ ble strmm_kernel_L4_M4_100
+
+strmm_kernel_L4_M4_46:
+
+ KERNEL4x4_SUB
+
+strmm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+strmm_kernel_L4_M4_END:
+
+/******************************************************************************/
+
+strmm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble strmm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble strmm_kernel_L4_M1_BEGIN
+
+strmm_kernel_L4_M2_20:
+
+ INIT2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L4_M2_40
+
+strmm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M2_22
+
+
+strmm_kernel_L4_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L4_M2_100
+
+strmm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M2_42
+
+strmm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+strmm_kernel_L4_M2_END:
+
+/******************************************************************************/
+
+strmm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble strmm_kernel_L4_END
+
+strmm_kernel_L4_M1_20:
+
+ INIT1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #4
+ add pB, pB, temp
+ lsl temp, tempOffset, #2
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #4
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L4_M1_40
+
+strmm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M1_22
+
+
+strmm_kernel_L4_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L4_M1_100
+
+strmm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L4_M1_42
+
+strmm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #4
+#endif
+ lsl temp, tempK, #2
+ add pA, pA, temp
+ lsl temp, tempK, #4
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+strmm_kernel_L4_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 4 * 4
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+
+/******************************************************************************/
+/******************************************************************************/
+
+strmm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble strmm_kernel_L999
+
+ tst counterJ , #2
+ ble strmm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+/******************************************************************************/
+
+strmm_kernel_L2_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI,#0
+ ble strmm_kernel_L2_M4_BEGIN
+
+strmm_kernel_L2_M8_20:
+
+ INIT8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble strmm_kernel_L2_M8_40
+ .align 5
+
+strmm_kernel_L2_M8_22:
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M8_22
+
+
+strmm_kernel_L2_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M8_100
+
+strmm_kernel_L2_M8_42:
+
+ KERNEL8x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M8_42
+
+strmm_kernel_L2_M8_100:
+
+ SAVE8x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+strmm_kernel_L2_M8_END:
+
+ subs counterI, counterI, #1
+ bgt strmm_kernel_L2_M8_20
+
+/******************************************************************************/
+
+strmm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble strmm_kernel_L2_END
+
+ tst counterI, #4
+ ble strmm_kernel_L2_M2_BEGIN
+
+strmm_kernel_L2_M4_20:
+
+ INIT4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble strmm_kernel_L2_M4_40
+ .align 5
+
+strmm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M4_22
+
+
+strmm_kernel_L2_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M4_100
+
+strmm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M4_42
+
+strmm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+strmm_kernel_L2_M4_END:
+
+/******************************************************************************/
+
+strmm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble strmm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble strmm_kernel_L2_M1_BEGIN
+
+strmm_kernel_L2_M2_20:
+
+ INIT2x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble strmm_kernel_L2_M2_40
+
+strmm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M2_22
+
+
+strmm_kernel_L2_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M2_100
+
+strmm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M2_42
+
+strmm_kernel_L2_M2_100:
+
+ SAVE2x2
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+
+strmm_kernel_L2_M2_END:
+
+/******************************************************************************/
+
+strmm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble strmm_kernel_L2_END
+
+strmm_kernel_L2_M1_20:
+
+ INIT1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #3
+ add pB, pB, temp
+ lsl temp, tempOffset, #2
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #2
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble strmm_kernel_L2_M1_40
+
+strmm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M1_22
+
+
+strmm_kernel_L2_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L2_M1_100
+
+strmm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L2_M1_42
+
+strmm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #1
+#else
+ sub tempK, tempK, #2
+#endif
+ lsl temp, tempK, #2
+ add pA, pA, temp
+ lsl temp, tempK, #3
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #1
+#endif
+strmm_kernel_L2_END:
+#if !defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+ add origPB, origPB, origK, lsl #3 // B = B + K * 2 * 4
+
+/******************************************************************************/
+/******************************************************************************/
+
+strmm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble strmm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+#if defined(LEFT)
+ mov tempOffset, offset
+#endif
+ mov pA, origPA // pA = A
+
+/******************************************************************************/
+
+strmm_kernel_L1_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3
+ cmp counterI, #0
+ ble strmm_kernel_L1_M4_BEGIN
+
+strmm_kernel_L1_M8_20:
+
+ INIT8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #5
+ add pA, pA, temp
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #8
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M8_40
+ .align 5
+
+strmm_kernel_L1_M8_22:
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M8_22
+
+
+strmm_kernel_L1_M8_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M8_100
+
+strmm_kernel_L1_M8_42:
+
+ KERNEL8x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M8_42
+
+strmm_kernel_L1_M8_100:
+
+ SAVE8x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #8
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #5
+ add pA, pA, temp
+ lsl temp, tempK, #2
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #8
+#endif
+strmm_kernel_L1_M8_END:
+
+ subs counterI, counterI, #1
+ bgt strmm_kernel_L1_M8_20
+
+/******************************************************************************/
+
+strmm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #7
+ ble strmm_kernel_L1_END
+
+ tst counterI, #4
+ ble strmm_kernel_L1_M2_BEGIN
+
+strmm_kernel_L1_M4_20:
+
+ INIT4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+ lsl temp, tempOffset, #4
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #4
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M4_40
+ .align 5
+
+strmm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M4_22
+
+
+strmm_kernel_L1_M4_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M4_100
+
+strmm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M4_42
+
+strmm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #4
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #4
+ add pA, pA, temp
+ lsl temp, tempK, #2
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #4
+#endif
+strmm_kernel_L1_M4_END:
+
+/******************************************************************************/
+
+strmm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble strmm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble strmm_kernel_L1_M1_BEGIN
+
+strmm_kernel_L1_M2_20:
+
+ INIT2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+ lsl temp, tempOffset, #3
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #2
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M2_40
+
+strmm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M2_22
+
+
+strmm_kernel_L1_M2_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M2_100
+
+strmm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M2_42
+
+strmm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ sub tempK, origK, tempOffset
+#if defined(LEFT)
+ sub tempK, tempK, #2
+#else
+ sub tempK, tempK, #1
+#endif
+ lsl temp, tempK, #3
+ add pA, pA, temp
+ lsl temp, tempK, #2
+ add pB, pB, temp
+#endif
+#if defined(LEFT)
+ add tempOffset, tempOffset, #2
+#endif
+strmm_kernel_L1_M2_END:
+
+/******************************************************************************/
+
+strmm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble strmm_kernel_L1_END
+
+strmm_kernel_L1_M1_20:
+
+ INIT1x1
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ mov pB, origPB
+#else
+ mov pB, origPB
+ lsl temp, tempOffset, #2
+ add pB, pB, temp
+ lsl temp, tempOffset, #2
+ add pA, pA, temp
+#endif
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ sub tempK, origK, tempOffset
+#elif defined(LEFT)
+ add tempK, tempOffset, #1
+#else
+ add tempK, tempOffset, #1
+#endif
+ asr counterL , tempK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble strmm_kernel_L1_M1_40
+
+strmm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M1_22
+
+
+strmm_kernel_L1_M1_40:
+
+ ands counterL , tempK, #7 // counterL = counterL % 8
+ ble strmm_kernel_L1_M1_100
+
+strmm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt strmm_kernel_L1_M1_42
+
+strmm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+strmm_kernel_L1_END:
+
+/******************************************************************************/
+
+strmm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
projects / platform / upstream / openblas.git / commitdiff