--- /dev/null
+/*******************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+/* X0 X1 X2 s0 X3 x4 x5 x6 */
+/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc */
+
+#define origM x0
+#define origN x1
+#define origK x2
+#define origPA x3
+#define origPB x4
+#define pC x5
+#define LDC x6
+#define temp x7
+#define counterL x8
+#define counterI x9
+#define counterJ x10
+#define pB x11
+#define pCRow0 x12
+#define pCRow1 x13
+#define pCRow2 x14
+#define pA x15
+#define ppC x16
+#define ppA x17
+
+#define alpha0_R s10
+#define alphaV0_R v10.s[0]
+#define alpha0_I s11
+#define alphaV0_I v11.s[0]
+
+#define alpha1_R s14
+#define alphaV1_R v14.s[0]
+#define alpha1_I s15
+#define alphaV1_I v15.s[0]
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
+#define OP_rr fmla
+#define OP_ii fmls
+#define OP_ri fmla
+#define OP_ir fmla
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+#define OP_rr fmla
+#define OP_ii fmla
+#define OP_ri fmls
+#define OP_ir fmla
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+#define OP_rr fmla
+#define OP_ii fmla
+#define OP_ri fmla
+#define OP_ir fmls
+#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
+#define OP_rr fmla
+#define OP_ii fmls
+#define OP_ri fmls
+#define OP_ir fmls
+#endif
+
+// 00 origM
+// 01 origN
+// 02 origK
+// 03 origPA
+// 04 origPB
+// 05 pC
+// 06 origLDC -> LDC
+// 07 offset -> temp
+// 08 counterL
+// 09 counterI
+// 10 counterJ
+// 11 pB
+// 12 pCRow0
+// 13 pCRow1
+// 14 pCRow2
+// 15 pA
+// 16 ppC
+// 17 ppA
+// 18 must save
+// 19 must save
+// 20 must save
+// 21 must save
+// 22 must save
+// 23 must save
+// 24 must save
+// 25 must save
+// 26 must save
+// 27 must save
+// 28 must save
+// 29 frame
+// 30 link
+// 31 sp
+
+//v00 ALPHA_R -> pA00_R, pA01_R, pA02_R, pA03_R
+//v01 ALPHA_I -> pA00_I, pA01_I, pA02_I, pA03_I
+//v02 ppA00_R, ppA01_R, ppA02_R, ppA03_R
+//v03 ppA00_I, ppA01_I, ppA02_I, ppA03_I
+//v04 pA10_R, pA11_R, pA12_R, pA13_R
+//v05 pA10_I, pA11_I, pA12_I, pA13_I
+//v06 ppA10_R, ppA11_R, ppA12_R, ppA13_R
+//v07 ppA10_I, ppA11_I, ppA12_I, ppA13_I
+//v08 must save pB00_R, pB01_R, pB02_R, pB03_R
+//v09 must save pB00_I, pB01_I, pB02_I, pB03_I
+//v10 must save ALPHA0_R
+//v11 must save ALPHA0_I
+//v12 must save pB10_R, pB11_R, pB12_R, pB13_R
+//v13 must save pB10_I, pB11_I, pB12_I, pB13_I
+//v14 must save ALPHA1_R
+//v15 must save ALPHA1_I
+//v16 must save pC00_R, pC01_R, pC02_R, pC03_R
+//v17 must save pC00_I, pC01_I, pC02_I, pC03_I
+//v18 ppC00_R, ppC01_R, ppC02_R, ppC03_R
+//v19 ppC00_I, ppC01_I, ppC02_I, ppC03_I
+//v20 pC10_R, pC11_R, pC12_R, pC13_R
+//v21 pC10_I, pC11_I, pC12_I, pC13_I
+//v22 ppC10_R, ppC11_R, ppC12_R, ppC13_R
+//v23 ppC10_I, ppC11_I, ppC12_I, ppC13_I
+//v24 pC20_R, pC21_R, pC22_R, pC23_R
+//v25 pC20_I, pC21_I, pC22_I, pC23_I
+//v26 ppC20_R, ppC21_R, ppC22_R, ppC23_R
+//v27 ppC20_I, ppC21_I, ppC22_I, ppC23_I
+//v28 pC30_R, pC31_R, pC32_R, pC33_R
+//v29 pC30_I, pC31_I, pC32_I, pC33_I
+//v30 ppC30_R, ppC31_R, ppC32_R, ppC33_R
+//v31 ppC30_I, ppC31_I, ppC32_I, ppC33_I
+
+/*******************************************************************************
+* Macro definitions
+*******************************************************************************/
+
+.macro INIT8x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s18, s17
+ fmov s19, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s22, s17
+ fmov s23, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s26, s17
+ fmov s27, s16
+ fmov s28, s17
+ fmov s29, s16
+ fmov s30, s17
+ fmov s31, s16
+.endm
+
+.macro KERNEL8x4_I
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+ ld2 {v2.4s, v3.4s}, [ppA]
+ add ppA, ppA, #32
+
+ fmul v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ fmul v17.4s, v0.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v17.4s, v17.4s
+#endif
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ fmul v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ fmul v21.4s, v0.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v21.4s, v21.4s
+#endif
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ fmul v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ fmul v25.4s, v0.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v25.4s, v25.4s
+#endif
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ fmul v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ fmul v29.4s, v0.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v29.4s, v29.4s
+#endif
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ fmul v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+ fmul v19.4s, v2.4s, v9.4s[0]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v19.4s, v19.4s
+#endif
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ fmul v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+ fmul v23.4s, v2.4s, v9.4s[1]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v23.4s, v23.4s
+#endif
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ fmul v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+ fmul v27.4s, v2.4s, v9.4s[2]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v27.4s, v27.4s
+#endif
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ fmul v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+ fmul v31.4s, v2.4s, v9.4s[3]
+#if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
+ defined(RR) || defined(RC) || defined(CR) || defined(CC)
+ fneg v31.4s, v31.4s
+#endif
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+
+ ld2 {v12.4s, v13.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v4.4s, v5.4s} , [pA]
+ add pA, pA, #32
+ ld2 {v6.4s, v7.4s} , [ppA]
+ add ppA, ppA, #32
+.endm
+
+.macro KERNEL8x4_M1
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ ld2 {v12.4s, v13.4s}, [pB] // for next round
+ add pB, pB, #32
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ ld2 {v4.4s, v5.4s} , [pA] // for next round
+ add pA, pA, #32
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+ OP_ri v19.4s, v2.4s, v9.4s[0]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ ld2 {v6.4s, v7.4s} , [ppA] // for next round
+ add ppA, ppA, #32
+
+ OP_rr v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+ OP_ri v23.4s, v2.4s, v9.4s[1]
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ prfm PLDL1KEEP, [ppA, #512]
+
+ OP_rr v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+ OP_ri v27.4s, v2.4s, v9.4s[2]
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ OP_rr v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+ OP_ri v31.4s, v2.4s, v9.4s[3]
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+.endm
+
+.macro KERNEL8x4_M2
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ ld2 {v8.4s, v9.4s}, [pB] // for next round
+ add pB, pB, #32
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ prfm PLDL1KEEP, [pA, #512]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ ld2 {v0.4s, v1.4s}, [pA] // for next round
+ add pA, pA, #32
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+
+ prfm PLDL1KEEP, [ppA, #512]
+
+ OP_rr v18.4s, v6.4s, v12.4s[0]
+ OP_ii v18.4s, v7.4s, v13.4s[0]
+ OP_ri v19.4s, v6.4s, v13.4s[0]
+ OP_ir v19.4s, v7.4s, v12.4s[0]
+
+ ld2 {v2.4s, v3.4s}, [ppA] // for next round
+ add ppA, ppA, #32
+
+ OP_rr v22.4s, v6.4s, v12.4s[1]
+ OP_ii v22.4s, v7.4s, v13.4s[1]
+ OP_ri v23.4s, v6.4s, v13.4s[1]
+ OP_ir v23.4s, v7.4s, v12.4s[1]
+
+ prfm PLDL1KEEP, [pB, #512]
+
+ OP_rr v26.4s, v6.4s, v12.4s[2]
+ OP_ii v26.4s, v7.4s, v13.4s[2]
+ OP_ri v27.4s, v6.4s, v13.4s[2]
+ OP_ir v27.4s, v7.4s, v12.4s[2]
+
+ OP_rr v30.4s, v6.4s, v12.4s[3]
+ OP_ii v30.4s, v7.4s, v13.4s[3]
+ OP_ri v31.4s, v6.4s, v13.4s[3]
+ OP_ir v31.4s, v7.4s, v12.4s[3]
+.endm
+
+.macro KERNEL8x4_E
+ OP_rr v16.4s, v4.4s, v12.4s[0]
+ OP_ii v16.4s, v5.4s, v13.4s[0]
+ OP_ri v17.4s, v4.4s, v13.4s[0]
+ OP_ir v17.4s, v5.4s, v12.4s[0]
+
+ OP_rr v20.4s, v4.4s, v12.4s[1]
+ OP_ii v20.4s, v5.4s, v13.4s[1]
+ OP_ri v21.4s, v4.4s, v13.4s[1]
+ OP_ir v21.4s, v5.4s, v12.4s[1]
+
+ OP_rr v24.4s, v4.4s, v12.4s[2]
+ OP_ii v24.4s, v5.4s, v13.4s[2]
+ OP_ri v25.4s, v4.4s, v13.4s[2]
+ OP_ir v25.4s, v5.4s, v12.4s[2]
+
+ OP_rr v28.4s, v4.4s, v12.4s[3]
+ OP_ii v28.4s, v5.4s, v13.4s[3]
+ OP_ri v29.4s, v4.4s, v13.4s[3]
+ OP_ir v29.4s, v5.4s, v12.4s[3]
+
+ OP_rr v18.4s, v6.4s, v12.4s[0]
+ OP_ii v18.4s, v7.4s, v13.4s[0]
+ OP_ri v19.4s, v6.4s, v13.4s[0]
+ OP_ir v19.4s, v7.4s, v12.4s[0]
+
+ OP_rr v22.4s, v6.4s, v12.4s[1]
+ OP_ii v22.4s, v7.4s, v13.4s[1]
+ OP_ri v23.4s, v6.4s, v13.4s[1]
+ OP_ir v23.4s, v7.4s, v12.4s[1]
+
+ OP_rr v26.4s, v6.4s, v12.4s[2]
+ OP_ii v26.4s, v7.4s, v13.4s[2]
+ OP_ri v27.4s, v6.4s, v13.4s[2]
+ OP_ir v27.4s, v7.4s, v12.4s[2]
+
+ OP_rr v30.4s, v6.4s, v12.4s[3]
+ OP_ii v30.4s, v7.4s, v13.4s[3]
+ OP_ri v31.4s, v6.4s, v13.4s[3]
+ OP_ir v31.4s, v7.4s, v12.4s[3]
+.endm
+
+.macro KERNEL8x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ ld2 {v2.4s, v3.4s}, [ppA]
+ add ppA, ppA, #32
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+
+ OP_rr v18.4s, v2.4s, v8.4s[0]
+ OP_ii v18.4s, v3.4s, v9.4s[0]
+ OP_ri v19.4s, v2.4s, v9.4s[0]
+ OP_ir v19.4s, v3.4s, v8.4s[0]
+
+ OP_rr v22.4s, v2.4s, v8.4s[1]
+ OP_ii v22.4s, v3.4s, v9.4s[1]
+ OP_ri v23.4s, v2.4s, v9.4s[1]
+ OP_ir v23.4s, v3.4s, v8.4s[1]
+
+ OP_rr v26.4s, v2.4s, v8.4s[2]
+ OP_ii v26.4s, v3.4s, v9.4s[2]
+ OP_ri v27.4s, v2.4s, v9.4s[2]
+ OP_ir v27.4s, v3.4s, v8.4s[2]
+
+ OP_rr v30.4s, v2.4s, v8.4s[3]
+ OP_ii v30.4s, v3.4s, v9.4s[3]
+ OP_ri v31.4s, v2.4s, v9.4s[3]
+ OP_ir v31.4s, v3.4s, v8.4s[3]
+.endm
+
+.macro SAVE8x4
+ mov pCRow1, pCRow0
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v2.4s, v3.4s}, [pCRow2]
+ fmla v2.4s, v18.4s, alphaV0_R
+ fmls v2.4s, v19.4s, alphaV0_I
+ fmla v3.4s, v18.4s, alphaV1_I
+ fmla v3.4s, v19.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmla v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v6.4s, v7.4s}, [pCRow2]
+ fmla v6.4s, v22.4s, alphaV0_R
+ fmls v6.4s, v23.4s, alphaV0_I
+ fmla v7.4s, v22.4s, alphaV1_I
+ fmla v7.4s, v23.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v24.4s, alphaV0_R
+ fmls v0.4s, v25.4s, alphaV0_I
+ fmla v1.4s, v24.4s, alphaV1_I
+ fmla v1.4s, v25.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v2.4s, v3.4s}, [pCRow2]
+ fmla v2.4s, v26.4s, alphaV0_R
+ fmls v2.4s, v27.4s, alphaV0_I
+ fmla v3.4s, v26.4s, alphaV1_I
+ fmla v3.4s, v27.4s, alphaV1_R
+ st2 {v2.4s, v3.4s}, [pCRow2]
+
+ add pCRow2, pCRow1, #32
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v28.4s, alphaV0_R
+ fmls v4.4s, v29.4s, alphaV0_I
+ fmla v5.4s, v28.4s, alphaV1_I
+ fmla v5.4s, v29.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v6.4s, v7.4s}, [pCRow2]
+ fmla v6.4s, v30.4s, alphaV0_R
+ fmls v6.4s, v31.4s, alphaV0_I
+ fmla v7.4s, v30.4s, alphaV1_I
+ fmla v7.4s, v31.4s, alphaV1_R
+ st2 {v6.4s, v7.4s}, [pCRow2]
+
+ add pCRow0, pCRow0, #64
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x4
+ fmov s16, wzr
+ fmov s17, s16
+ fmov s20, s17
+ fmov s21, s16
+ fmov s24, s17
+ fmov s25, s16
+ fmov s28, s17
+ fmov s29, s16
+.endm
+
+.macro KERNEL4x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.4s[0]
+ OP_ii v16.4s, v1.4s, v9.4s[0]
+ OP_ri v17.4s, v0.4s, v9.4s[0]
+ OP_ir v17.4s, v1.4s, v8.4s[0]
+
+ OP_rr v20.4s, v0.4s, v8.4s[1]
+ OP_ii v20.4s, v1.4s, v9.4s[1]
+ OP_ri v21.4s, v0.4s, v9.4s[1]
+ OP_ir v21.4s, v1.4s, v8.4s[1]
+
+ OP_rr v24.4s, v0.4s, v8.4s[2]
+ OP_ii v24.4s, v1.4s, v9.4s[2]
+ OP_ri v25.4s, v0.4s, v9.4s[2]
+ OP_ir v25.4s, v1.4s, v8.4s[2]
+
+ OP_rr v28.4s, v0.4s, v8.4s[3]
+ OP_ii v28.4s, v1.4s, v9.4s[3]
+ OP_ri v29.4s, v0.4s, v9.4s[3]
+ OP_ir v29.4s, v1.4s, v8.4s[3]
+.endm
+
+.macro SAVE4x4
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmla v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v24.4s, alphaV0_R
+ fmls v0.4s, v25.4s, alphaV0_I
+ fmla v1.4s, v24.4s, alphaV1_I
+ fmla v1.4s, v25.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v28.4s, alphaV0_R
+ fmls v4.4s, v29.4s, alphaV0_I
+ fmla v5.4s, v28.4s, alphaV1_I
+ fmla v5.4s, v29.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+ fmov s24, s16
+ fmov s25, s17
+ fmov s28, s16
+ fmov s29, s17
+.endm
+
+.macro KERNEL2x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.4s[0]
+ OP_ii v16.2s, v1.2s, v9.4s[0]
+ OP_ri v17.2s, v0.2s, v9.4s[0]
+ OP_ir v17.2s, v1.2s, v8.4s[0]
+
+ OP_rr v20.2s, v0.2s, v8.4s[1]
+ OP_ii v20.2s, v1.2s, v9.4s[1]
+ OP_ri v21.2s, v0.2s, v9.4s[1]
+ OP_ir v21.2s, v1.2s, v8.4s[1]
+
+ OP_rr v24.2s, v0.2s, v8.4s[2]
+ OP_ii v24.2s, v1.2s, v9.4s[2]
+ OP_ri v25.2s, v0.2s, v9.4s[2]
+ OP_ir v25.2s, v1.2s, v8.4s[2]
+
+ OP_rr v28.2s, v0.2s, v8.4s[3]
+ OP_ii v28.2s, v1.2s, v9.4s[3]
+ OP_ri v29.2s, v0.2s, v9.4s[3]
+ OP_ir v29.2s, v1.2s, v8.4s[3]
+.endm
+
+.macro SAVE2x4
+ mov pCRow1, pCRow0
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmla v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.2s, v5.2s}, [pCRow1]
+ fmla v4.2s, v20.2s, alphaV0_R
+ fmls v4.2s, v21.2s, alphaV0_I
+ fmla v5.2s, v20.2s, alphaV1_I
+ fmla v5.2s, v21.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v24.2s, alphaV0_R
+ fmls v0.2s, v25.2s, alphaV0_I
+ fmla v1.2s, v24.2s, alphaV1_I
+ fmla v1.2s, v25.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.2s, v5.2s}, [pCRow1]
+ fmla v4.2s, v28.2s, alphaV0_R
+ fmls v4.2s, v29.2s, alphaV0_I
+ fmla v5.2s, v28.2s, alphaV1_I
+ fmla v5.2s, v29.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x4
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+ fmov s24, s16
+ fmov s25, s17
+ fmov s28, s16
+ fmov s29, s17
+.endm
+
+.macro KERNEL1x4_SUB
+ ld2 {v8.4s, v9.4s}, [pB]
+ add pB, pB, #32
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.4s[0]
+ OP_ii s16, s1, v9.4s[0]
+ OP_ri s17, s0, v9.4s[0]
+ OP_ir s17, s1, v8.4s[0]
+
+ OP_rr s20, s0, v8.4s[1]
+ OP_ii s20, s1, v9.4s[1]
+ OP_ri s21, s0, v9.4s[1]
+ OP_ir s21, s1, v8.4s[1]
+
+ OP_rr s24, s0, v8.4s[2]
+ OP_ii s24, s1, v9.4s[2]
+ OP_ri s25, s0, v9.4s[2]
+ OP_ir s25, s1, v8.4s[2]
+
+ OP_rr s28, s0, v8.4s[3]
+ OP_ii s28, s1, v9.4s[3]
+ OP_ri s29, s0, v9.4s[3]
+ OP_ir s29, s1, v8.4s[3]
+.endm
+
+.macro SAVE1x4
+ mov pCRow1, pCRow0
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmla s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.s, v5.s}[0], [pCRow1]
+ fmla s4, s20, alphaV0_R
+ fmls s4, s21, alphaV0_I
+ fmla s5, s20, alphaV1_I
+ fmla s5, s21, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s24, alphaV0_R
+ fmls s0, s25, alphaV0_I
+ fmla s1, s24, alphaV1_I
+ fmla s1, s25, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.s, v5.s}[0], [pCRow1]
+ fmla s4, s28, alphaV0_R
+ fmls s4, s29, alphaV0_I
+ fmla s5, s28, alphaV1_I
+ fmla s5, s29, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+.endm
+
+.macro KERNEL4x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.2s[0]
+ OP_ii v16.4s, v1.4s, v9.2s[0]
+ OP_ri v17.4s, v0.4s, v9.2s[0]
+ OP_ir v17.4s, v1.4s, v8.2s[0]
+
+ OP_rr v20.4s, v0.4s, v8.2s[1]
+ OP_ii v20.4s, v1.4s, v9.2s[1]
+ OP_ri v21.4s, v0.4s, v9.2s[1]
+ OP_ir v21.4s, v1.4s, v8.2s[1]
+.endm
+
+.macro SAVE4x2
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.4s, v5.4s}, [pCRow1]
+ fmla v4.4s, v20.4s, alphaV0_R
+ fmls v4.4s, v21.4s, alphaV0_I
+ fmla v5.4s, v20.4s, alphaV1_I
+ fmla v5.4s, v21.4s, alphaV1_R
+ st2 {v4.4s, v5.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, s16
+ fmov s21, s17
+.endm
+
+.macro KERNEL2x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.2s[0]
+ OP_ii v16.2s, v1.2s, v9.2s[0]
+ OP_ri v17.2s, v0.2s, v9.2s[0]
+ OP_ir v17.2s, v1.2s, v8.2s[0]
+
+ OP_rr v20.2s, v0.2s, v8.2s[1]
+ OP_ii v20.2s, v1.2s, v9.2s[1]
+ OP_ri v21.2s, v0.2s, v9.2s[1]
+ OP_ir v21.2s, v1.2s, v8.2s[1]
+.endm
+
+.macro SAVE2x2
+ mov pCRow1, pCRow0
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmla v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.2s, v5.2s}, [pCRow1]
+ fmla v4.2s, v20.2s, alphaV0_R
+ fmls v4.2s, v21.2s, alphaV0_I
+ fmla v5.2s, v20.2s, alphaV1_I
+ fmla v5.2s, v21.2s, alphaV1_R
+ st2 {v4.2s, v5.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x2
+ fmov s16, wzr
+ fmov s17, wzr
+ fmov s20, wzr
+ fmov s21, wzr
+.endm
+
+.macro KERNEL1x2_SUB
+ ld2 {v8.2s, v9.2s}, [pB]
+ add pB, pB, #16
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.2s[0]
+ OP_ii s16, s1, v9.2s[0]
+ OP_ri s17, s0, v9.2s[0]
+ OP_ir s17, s1, v8.2s[0]
+
+ OP_rr s20, s0, v8.2s[1]
+ OP_ii s20, s1, v9.2s[1]
+ OP_ri s21, s0, v9.2s[1]
+ OP_ir s21, s1, v8.2s[1]
+.endm
+
+.macro SAVE1x2
+ mov pCRow1, pCRow0
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmla s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow1, pCRow1, LDC
+
+ ld2 {v4.s, v5.s}[0], [pCRow1]
+ fmla s4, s20, alphaV0_R
+ fmls s4, s21, alphaV0_I
+ fmla s5, s20, alphaV1_I
+ fmla s5, s21, alphaV1_R
+ st2 {v4.s, v5.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/******************************************************************************/
+
+.macro INIT4x1
+ fmov s16, wzr
+ fmov s17, s16
+.endm
+
+.macro KERNEL4x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.4s, v1.4s}, [pA]
+ add pA, pA, #32
+
+ OP_rr v16.4s, v0.4s, v8.s[0]
+ OP_ii v16.4s, v1.4s, v9.s[0]
+ OP_ri v17.4s, v0.4s, v9.s[0]
+ OP_ir v17.4s, v1.4s, v8.s[0]
+.endm
+
+.macro SAVE4x1
+ mov pCRow1, pCRow0
+
+ ld2 {v0.4s, v1.4s}, [pCRow1]
+ fmla v0.4s, v16.4s, alphaV0_R
+ fmls v0.4s, v17.4s, alphaV0_I
+ fmla v1.4s, v16.4s, alphaV1_I
+ fmla v1.4s, v17.4s, alphaV1_R
+ st2 {v0.4s, v1.4s}, [pCRow1]
+
+ add pCRow0, pCRow0, #32
+.endm
+
+/******************************************************************************/
+
+.macro INIT2x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL2x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.2s, v1.2s}, [pA]
+ add pA, pA, #16
+
+ OP_rr v16.2s, v0.2s, v8.s[0]
+ OP_ii v16.2s, v1.2s, v9.s[0]
+ OP_ri v17.2s, v0.2s, v9.s[0]
+ OP_ir v17.2s, v1.2s, v8.s[0]
+.endm
+
+.macro SAVE2x1
+ mov pCRow1, pCRow0
+
+ ld2 {v0.2s, v1.2s}, [pCRow1]
+ fmla v0.2s, v16.2s, alphaV0_R
+ fmls v0.2s, v17.2s, alphaV0_I
+ fmla v1.2s, v16.2s, alphaV1_I
+ fmla v1.2s, v17.2s, alphaV1_R
+ st2 {v0.2s, v1.2s}, [pCRow1]
+
+ add pCRow0, pCRow0, #16
+
+.endm
+
+/******************************************************************************/
+
+.macro INIT1x1
+ fmov s16, wzr
+ fmov s17, wzr
+.endm
+
+.macro KERNEL1x1_SUB
+ ld2 {v8.s, v9.s}[0], [pB]
+ add pB, pB, #8
+ ld2 {v0.s, v1.s}[0], [pA]
+ add pA, pA, #8
+
+ OP_rr s16, s0, v8.s[0]
+ OP_ii s16, s1, v9.s[0]
+ OP_ri s17, s0, v9.s[0]
+ OP_ir s17, s1, v8.s[0]
+.endm
+
+.macro SAVE1x1
+ mov pCRow1, pCRow0
+
+ ld2 {v0.s, v1.s}[0], [pCRow1]
+ fmla s0, s16, alphaV0_R
+ fmls s0, s17, alphaV0_I
+ fmla s1, s16, alphaV1_I
+ fmla s1, s17, alphaV1_R
+ st2 {v0.s, v1.s}[0], [pCRow1]
+
+ add pCRow0, pCRow0, #8
+.endm
+
+/*******************************************************************************
+* End of macro definitions
+*******************************************************************************/
+
+ PROLOGUE
+
+ .align 5
+ add sp, sp, #-(11 * 16)
+ stp d8, d9, [sp, #(0 * 16)]
+ stp d10, d11, [sp, #(1 * 16)]
+ stp d12, d13, [sp, #(2 * 16)]
+ stp d14, d15, [sp, #(3 * 16)]
+ stp d16, d17, [sp, #(4 * 16)]
+ stp x18, x19, [sp, #(5 * 16)]
+ stp x20, x21, [sp, #(6 * 16)]
+ stp x22, x23, [sp, #(7 * 16)]
+ stp x24, x25, [sp, #(8 * 16)]
+ stp x26, x27, [sp, #(9 * 16)]
+ str x28, [sp, #(10 * 16)]
+
+ fmov alpha0_R, s0
+ fmov alpha0_I, s1
+ fmov alpha1_R, s0
+ fmov alpha1_I, s1
+
+ lsl LDC, LDC, #3 // ldc = ldc * 8
+
+ mov pB, origPB
+
+ mov counterJ, origN
+ asr counterJ, counterJ, #2 // J = J / 4
+ cmp counterJ, #0
+ ble cgemm_kernel_L2_BEGIN
+
+/******************************************************************************/
+
+cgemm_kernel_L4_BEGIN:
+ mov pCRow0, pC // pCRow0 = C
+ add pC, pC, LDC, lsl #2
+
+ lsl temp, origK, #5 // k * 4 * 8
+ mov pA, origPA // pA = start of A array
+ add ppA, temp, pA
+
+cgemm_kernel_L4_M8_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #3 // counterI = counterI / 8
+ cmp counterI, #0
+ ble cgemm_kernel_L4_M4_BEGIN
+
+cgemm_kernel_L4_M8_20:
+
+ mov pB, origPB
+ asr counterL , origK, #1 // L = K / 2
+ cmp counterL , #2 // is there at least 4 to do?
+ blt cgemm_kernel_L4_M8_32
+
+ KERNEL8x4_I // do one in the K
+ KERNEL8x4_M2 // do another in the K
+
+ subs counterL, counterL, #2 // subtract 2
+ ble cgemm_kernel_L4_M8_22a
+ .align 5
+
+cgemm_kernel_L4_M8_22:
+
+ KERNEL8x4_M1
+ KERNEL8x4_M2
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M8_22
+
+
+cgemm_kernel_L4_M8_22a:
+
+ KERNEL8x4_M1
+ KERNEL8x4_E
+
+ b cgemm_kernel_L4_M8_44
+
+cgemm_kernel_L4_M8_32:
+
+ tst counterL, #1
+ ble cgemm_kernel_L4_M8_40
+
+ KERNEL8x4_I
+ KERNEL8x4_E
+
+ b cgemm_kernel_L4_M8_44
+
+
+cgemm_kernel_L4_M8_40:
+
+ INIT8x4
+
+cgemm_kernel_L4_M8_44:
+
+ ands counterL , origK, #1
+ ble cgemm_kernel_L4_M8_100
+
+cgemm_kernel_L4_M8_46:
+ KERNEL8x4_SUB
+
+cgemm_kernel_L4_M8_100:
+
+ SAVE8x4
+
+cgemm_kernel_L4_M8_END:
+ lsl temp, origK, #5 // k * 4 * 8
+ add pA, pA, temp
+ add ppA, ppA, temp
+ subs counterI, counterI, #1
+ bne cgemm_kernel_L4_M8_20
+
+
+cgemm_kernel_L4_M4_BEGIN:
+ mov counterI, origM
+ tst counterI , #7
+ ble cgemm_kernel_L4_END
+
+ tst counterI, #4
+ ble cgemm_kernel_L4_M2_BEGIN
+
+cgemm_kernel_L4_M4_20:
+
+ INIT4x4
+
+ mov pB, origPB
+ asr counterL, origK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble cgemm_kernel_L4_M4_40
+
+cgemm_kernel_L4_M4_22:
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M4_22
+
+
+cgemm_kernel_L4_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L4_M4_100
+
+cgemm_kernel_L4_M4_42:
+
+ KERNEL4x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M4_42
+
+cgemm_kernel_L4_M4_100:
+
+ SAVE4x4
+
+cgemm_kernel_L4_M4_END:
+
+
+cgemm_kernel_L4_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble cgemm_kernel_L4_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble cgemm_kernel_L4_M1_BEGIN
+
+cgemm_kernel_L4_M2_20:
+
+ INIT2x4
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L4_M2_40
+
+cgemm_kernel_L4_M2_22:
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M2_22
+
+
+cgemm_kernel_L4_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L4_M2_100
+
+cgemm_kernel_L4_M2_42:
+
+ KERNEL2x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M2_42
+
+cgemm_kernel_L4_M2_100:
+
+ SAVE2x4
+
+cgemm_kernel_L4_M2_END:
+
+
+cgemm_kernel_L4_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble cgemm_kernel_L4_END
+
+cgemm_kernel_L4_M1_20:
+
+ INIT1x4
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L4_M1_40
+
+cgemm_kernel_L4_M1_22:
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M1_22
+
+
+cgemm_kernel_L4_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L4_M1_100
+
+cgemm_kernel_L4_M1_42:
+
+ KERNEL1x4_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L4_M1_42
+
+cgemm_kernel_L4_M1_100:
+
+ SAVE1x4
+
+
+cgemm_kernel_L4_END:
+
+ lsl temp, origK, #5
+ add origPB, origPB, temp // B = B + K * 4 * 8
+
+ subs counterJ, counterJ , #1 // j--
+ bgt cgemm_kernel_L4_BEGIN
+
+
+/******************************************************************************/
+
+cgemm_kernel_L2_BEGIN: // less than 2 left in N direction
+
+ mov counterJ , origN
+ tst counterJ , #3
+ ble cgemm_kernel_L999 // error, N was less than 4?
+
+ tst counterJ , #2
+ ble cgemm_kernel_L1_BEGIN
+
+ mov pCRow0, pC // pCRow0 = pC
+
+ add pC,pC,LDC, lsl #1
+
+ mov pA, origPA // pA = A
+
+
+
+cgemm_kernel_L2_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI,#0
+ ble cgemm_kernel_L2_M2_BEGIN
+
+cgemm_kernel_L2_M4_20:
+
+ INIT4x2
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble cgemm_kernel_L2_M4_40
+ .align 5
+
+cgemm_kernel_L2_M4_22:
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M4_22
+
+
+cgemm_kernel_L2_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L2_M4_100
+
+cgemm_kernel_L2_M4_42:
+
+ KERNEL4x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M4_42
+
+cgemm_kernel_L2_M4_100:
+
+ SAVE4x2
+
+cgemm_kernel_L2_M4_END:
+
+ subs counterI, counterI, #1
+ bgt cgemm_kernel_L2_M4_20
+
+
+cgemm_kernel_L2_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble cgemm_kernel_L2_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble cgemm_kernel_L2_M1_BEGIN
+
+cgemm_kernel_L2_M2_20:
+
+ INIT2x2
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL,#0
+ ble cgemm_kernel_L2_M2_40
+
+cgemm_kernel_L2_M2_22:
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M2_22
+
+
+cgemm_kernel_L2_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L2_M2_100
+
+cgemm_kernel_L2_M2_42:
+
+ KERNEL2x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M2_42
+
+cgemm_kernel_L2_M2_100:
+
+ SAVE2x2
+
+cgemm_kernel_L2_M2_END:
+
+
+cgemm_kernel_L2_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble cgemm_kernel_L2_END
+
+cgemm_kernel_L2_M1_20:
+
+ INIT1x2
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL, #0
+ ble cgemm_kernel_L2_M1_40
+
+cgemm_kernel_L2_M1_22:
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M1_22
+
+
+cgemm_kernel_L2_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L2_M1_100
+
+cgemm_kernel_L2_M1_42:
+
+ KERNEL1x2_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L2_M1_42
+
+cgemm_kernel_L2_M1_100:
+
+ SAVE1x2
+
+
+cgemm_kernel_L2_END:
+ add origPB, origPB, origK, lsl #4 // B = B + K * 2 * 8
+
+/******************************************************************************/
+
+cgemm_kernel_L1_BEGIN:
+
+ mov counterJ , origN
+ tst counterJ , #1
+ ble cgemm_kernel_L999 // done
+
+
+ mov pCRow0, pC // pCRow0 = C
+ add pC , pC , LDC // Update pC to point to next
+
+ mov pA, origPA // pA = A
+
+
+
+cgemm_kernel_L1_M4_BEGIN:
+
+ mov counterI, origM
+ asr counterI, counterI, #2 // counterI = counterI / 4
+ cmp counterI, #0
+ ble cgemm_kernel_L1_M2_BEGIN
+
+cgemm_kernel_L1_M4_20:
+
+ INIT4x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L1_M4_40
+ .align 5
+
+cgemm_kernel_L1_M4_22:
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M4_22
+
+
+cgemm_kernel_L1_M4_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L1_M4_100
+
+cgemm_kernel_L1_M4_42:
+
+ KERNEL4x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M4_42
+
+cgemm_kernel_L1_M4_100:
+
+ SAVE4x1
+
+cgemm_kernel_L1_M4_END:
+
+ subs counterI, counterI, #1
+ bgt cgemm_kernel_L1_M4_20
+
+
+cgemm_kernel_L1_M2_BEGIN:
+
+ mov counterI, origM
+ tst counterI , #3
+ ble cgemm_kernel_L1_END
+
+ tst counterI, #2 // counterI = counterI / 2
+ ble cgemm_kernel_L1_M1_BEGIN
+
+cgemm_kernel_L1_M2_20:
+
+ INIT2x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L1_M2_40
+
+cgemm_kernel_L1_M2_22:
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M2_22
+
+
+cgemm_kernel_L1_M2_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L1_M2_100
+
+cgemm_kernel_L1_M2_42:
+
+ KERNEL2x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M2_42
+
+cgemm_kernel_L1_M2_100:
+
+ SAVE2x1
+
+cgemm_kernel_L1_M2_END:
+
+
+cgemm_kernel_L1_M1_BEGIN:
+
+ tst counterI, #1 // counterI = counterI % 2
+ ble cgemm_kernel_L1_END
+
+cgemm_kernel_L1_M1_20:
+
+ INIT1x1
+
+ mov pB, origPB
+ asr counterL , origK, #3 // counterL = counterL / 8
+ cmp counterL , #0
+ ble cgemm_kernel_L1_M1_40
+
+cgemm_kernel_L1_M1_22:
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M1_22
+
+
+cgemm_kernel_L1_M1_40:
+
+ ands counterL , origK, #7 // counterL = counterL % 8
+ ble cgemm_kernel_L1_M1_100
+
+cgemm_kernel_L1_M1_42:
+
+ KERNEL1x1_SUB
+
+ subs counterL, counterL, #1
+ bgt cgemm_kernel_L1_M1_42
+
+cgemm_kernel_L1_M1_100:
+
+ SAVE1x1
+
+
+cgemm_kernel_L1_END:
+
+
+cgemm_kernel_L999:
+ mov x0, #0 // set return value
+ ldp d8, d9, [sp, #(0 * 16)]
+ ldp d10, d11, [sp, #(1 * 16)]
+ ldp d12, d13, [sp, #(2 * 16)]
+ ldp d14, d15, [sp, #(3 * 16)]
+ ldp d16, d17, [sp, #(4 * 16)]
+ ldp x18, x19, [sp, #(5 * 16)]
+ ldp x20, x21, [sp, #(6 * 16)]
+ ldp x22, x23, [sp, #(7 * 16)]
+ ldp x24, x25, [sp, #(8 * 16)]
+ ldp x26, x27, [sp, #(9 * 16)]
+ ldr x28, [sp, #(10 * 16)]
+ add sp, sp, #(11*16)
+ ret
+
+ EPILOGUE
+
--- /dev/null
+#include "common.h"
+
+#define MADD_ALPHA_N_STORE(C, res, alpha) \
+ C[0] = res ## _r * alpha ## _r - res ## _i * alpha ## _i; \
+ C[1] = res ## _r * alpha ## _i + res ## _i * alpha ## _r;
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
+#define MADD(res, op1, op2) \
+ res ## _r += op1 ## _r * op2 ## _r; \
+ res ## _r -= op1 ## _i * op2 ## _i; \
+ res ## _i += op1 ## _r * op2 ## _i; \
+ res ## _i += op1 ## _i * op2 ## _r;
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+#define MADD(res, op1, op2) \
+ res ## _r += op1 ## _r * op2 ## _r; \
+ res ## _r += op1 ## _i * op2 ## _i; \
+ res ## _i -= op1 ## _r * op2 ## _i; \
+ res ## _i += op1 ## _i * op2 ## _r;
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+#define MADD(res, op1, op2) \
+ res ## _r += op1 ## _r * op2 ## _r; \
+ res ## _r += op1 ## _i * op2 ## _i; \
+ res ## _i += op1 ## _r * op2 ## _i; \
+ res ## _i -= op1 ## _i * op2 ## _r;
+#elif defined(RR) || defined(RC) || defined(CR) || defined(CC)
+#define MADD(res, op1, op2) \
+ res ## _r += op1 ## _r * op2 ## _r; \
+ res ## _r -= op1 ## _i * op2 ## _i; \
+ res ## _i -= op1 ## _r * op2 ## _i; \
+ res ## _i -= op1 ## _i * op2 ## _r;
+#endif
+
+int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha_r, FLOAT alpha_i,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc
+ , BLASLONG offset
+ )
+{
+
+ BLASLONG i,j,k;
+ FLOAT *C0,*C1,*C2,*C3,*ptrba,*ptrbb;
+ FLOAT res00_r, res01_r, res02_r, res03_r;
+ FLOAT res00_i, res01_i, res02_i, res03_i;
+ FLOAT res10_r, res11_r, res12_r, res13_r;
+ FLOAT res10_i, res11_i, res12_i, res13_i;
+ FLOAT res20_r, res21_r, res22_r, res23_r;
+ FLOAT res20_i, res21_i, res22_i, res23_i;
+ FLOAT res30_r, res31_r, res32_r, res33_r;
+ FLOAT res30_i, res31_i, res32_i, res33_i;
+ FLOAT a0_r, a1_r;
+ FLOAT a0_i, a1_i;
+ FLOAT b0_r, b1_r, b2_r, b3_r;
+ FLOAT b0_i, b1_i, b2_i, b3_i;
+ BLASLONG off, temp;
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+ off = -offset;
+#endif
+
+ for (j=0; j<bn/4; j+=1) // do blocks of the Mx4 loops
+ {
+ C0 = C;
+ C1 = C0+2*ldc;
+ C2 = C1+2*ldc;
+ C3 = C2+2*ldc;
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+ off = offset;
+#endif
+
+ ptrba = ba;
+
+ for (i=0; i<bm/4; i+=1) // do blocks of 4x4
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+
+ ptrba += off*4*2; // number of values in A
+ ptrbb = bb + off*4*2; // number of values in B
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+ res01_r = 0;
+ res01_i = 0;
+ res02_r = 0;
+ res02_i = 0;
+ res03_r = 0;
+ res03_i = 0;
+
+ res10_r = 0;
+ res10_i = 0;
+ res11_r = 0;
+ res11_i = 0;
+ res12_r = 0;
+ res12_i = 0;
+ res13_r = 0;
+ res13_i = 0;
+
+ res20_r = 0;
+ res20_i = 0;
+ res21_r = 0;
+ res21_i = 0;
+ res22_r = 0;
+ res22_i = 0;
+ res23_r = 0;
+ res23_i = 0;
+
+ res30_r = 0;
+ res30_i = 0;
+ res31_r = 0;
+ res31_i = 0;
+ res32_r = 0;
+ res32_i = 0;
+ res33_r = 0;
+ res33_i = 0;
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk - off;
+#elif defined(LEFT)
+ temp = off + 4;
+#else
+ temp = off + 4;
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+ b1_r = ptrbb[2*1+0]; b1_i = ptrbb[2*1+1];
+ b2_r = ptrbb[2*2+0]; b2_i = ptrbb[2*2+1];
+ b3_r = ptrbb[2*3+0]; b3_i = ptrbb[2*3+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+ MADD(res10, a0, b1);
+ MADD(res20, a0, b2);
+ MADD(res30, a0, b3);
+
+ a1_r = ptrba[2*1+0]; a1_i = ptrba[2*1+1];
+ MADD(res01, a1, b0);
+ MADD(res11, a1, b1);
+ MADD(res21, a1, b2);
+ MADD(res31, a1, b3);
+
+ a0_r = ptrba[2*2+0]; a0_i = ptrba[2*2+1];
+ MADD(res02, a0, b0);
+ MADD(res12, a0, b1);
+ MADD(res22, a0, b2);
+ MADD(res32, a0, b3);
+
+
+ a1_r = ptrba[2*3+0]; a1_i = ptrba[2*3+1];
+ MADD(res03, a1, b0);
+ MADD(res13, a1, b1);
+ MADD(res23, a1, b2);
+ MADD(res33, a1, b3);
+
+ ptrba = ptrba+8;
+ ptrbb = ptrbb+8;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res01, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res02, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res03, alpha);
+ C0 = C0 + 2;
+
+ MADD_ALPHA_N_STORE(C1, res10, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res11, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res12, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res13, alpha);
+ C1 = C1 + 2;
+
+ MADD_ALPHA_N_STORE(C2, res20, alpha);
+ C2 = C2 + 2;
+ MADD_ALPHA_N_STORE(C2, res21, alpha);
+ C2 = C2 + 2;
+ MADD_ALPHA_N_STORE(C2, res22, alpha);
+ C2 = C2 + 2;
+ MADD_ALPHA_N_STORE(C2, res23, alpha);
+ C2 = C2 + 2;
+
+ MADD_ALPHA_N_STORE(C3, res30, alpha);
+ C3 = C3 + 2;
+ MADD_ALPHA_N_STORE(C3, res31, alpha);
+ C3 = C3 + 2;
+ MADD_ALPHA_N_STORE(C3, res32, alpha);
+ C3 = C3 + 2;
+ MADD_ALPHA_N_STORE(C3, res33, alpha);
+ C3 = C3 + 2;
+
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk-off;
+#if defined(LEFT)
+ temp = temp - 4;
+#else
+ temp = temp - 4;
+#endif
+ ptrba += temp*4*2; // number of values in A
+ ptrbb += temp*4*2; // number of values in B
+#endif
+#ifdef LEFT
+ off += 4; // number of values in A
+#endif
+
+
+ }
+
+ if ( bm & 2 ) // do any 2x4 loop
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*2*2;
+ ptrbb = bb + off*4*2;
+#endif
+
+
+ res00_r = 0;
+ res00_i = 0;
+ res01_r = 0;
+ res01_i = 0;
+
+ res10_r = 0;
+ res10_i = 0;
+ res11_r = 0;
+ res11_i = 0;
+
+ res20_r = 0;
+ res20_i = 0;
+ res21_r = 0;
+ res21_i = 0;
+
+ res30_r = 0;
+ res30_i = 0;
+ res31_r = 0;
+ res31_i = 0;
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+2; // number of values in A
+#else
+ temp = off+4; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+ b1_r = ptrbb[2*1+0]; b1_i = ptrbb[2*1+1];
+ b2_r = ptrbb[2*2+0]; b2_i = ptrbb[2*2+1];
+ b3_r = ptrbb[2*3+0]; b3_i = ptrbb[2*3+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+ MADD(res10, a0, b1);
+ MADD(res20, a0, b2);
+ MADD(res30, a0, b3);
+
+ a1_r = ptrba[2*1+0]; a1_i = ptrba[2*1+1];
+ MADD(res01, a1, b0);
+ MADD(res11, a1, b1);
+ MADD(res21, a1, b2);
+ MADD(res31, a1, b3);
+
+
+ ptrba = ptrba+4;
+ ptrbb = ptrbb+8;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res01, alpha);
+ C0 = C0 + 2;
+
+ MADD_ALPHA_N_STORE(C1, res10, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res11, alpha);
+ C1 = C1 + 2;
+
+ MADD_ALPHA_N_STORE(C2, res20, alpha);
+ C2 = C2 + 2;
+ MADD_ALPHA_N_STORE(C2, res21, alpha);
+ C2 = C2 + 2;
+
+ MADD_ALPHA_N_STORE(C3, res30, alpha);
+ C3 = C3 + 2;
+ MADD_ALPHA_N_STORE(C3, res31, alpha);
+ C3 = C3 + 2;
+
+
+
+
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 2; // number of values in A
+#else
+ temp -= 4; // number of values in B
+#endif
+ ptrba += temp*2*2;
+ ptrbb += temp*4*2;
+#endif
+
+#ifdef LEFT
+ off += 2; // number of values in A
+#endif
+
+
+ }
+
+ if ( bm & 1 ) // do any 1x4 loop
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*1*2;
+ ptrbb = bb + off*4*2;
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+ res10_r = 0;
+ res10_i = 0;
+ res20_r = 0;
+ res20_i = 0;
+ res30_r = 0;
+ res30_i = 0;
+
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+1; // number of values in A
+#else
+ temp = off+4; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+ b1_r = ptrbb[2*1+0]; b1_i = ptrbb[2*1+1];
+ b2_r = ptrbb[2*2+0]; b2_i = ptrbb[2*2+1];
+ b3_r = ptrbb[2*3+0]; b3_i = ptrbb[2*3+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+ MADD(res10, a0, b1);
+ MADD(res20, a0, b2);
+ MADD(res30, a0, b3);
+
+
+ ptrba = ptrba+2;
+ ptrbb = ptrbb+8;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+
+ MADD_ALPHA_N_STORE(C1, res10, alpha);
+ C1 = C1 + 2;
+
+ MADD_ALPHA_N_STORE(C2, res20, alpha);
+ C2 = C2 + 2;
+
+ MADD_ALPHA_N_STORE(C3, res30, alpha);
+ C3 = C3 + 2;
+
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 1; // number of values in A
+#else
+ temp -= 4; // number of values in B
+#endif
+ ptrba += temp*1*2;
+ ptrbb += temp*4*2;
+#endif
+
+#ifdef LEFT
+ off += 1; // number of values in A
+#endif
+
+
+ }
+
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+ off += 4;
+#endif
+
+ k = (bk<<3);
+ bb = bb+k;
+ i = (ldc<<3);
+ C = C+i;
+ }
+
+ for (j=0; j<(bn&2); j+=2) // do the Mx2 loops
+ {
+ C0 = C;
+ C1 = C0+ldc*2;
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+ off = offset;
+#endif
+
+
+ ptrba = ba;
+
+ for (i=0; i<bm/4; i+=1) // do blocks of 4x2
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*4*2;
+ ptrbb = bb + off*2*2;
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+ res01_r = 0;
+ res01_i = 0;
+ res02_r = 0;
+ res02_i = 0;
+ res03_r = 0;
+ res03_i = 0;
+
+ res10_r = 0;
+ res10_i = 0;
+ res11_r = 0;
+ res11_i = 0;
+ res12_r = 0;
+ res12_i = 0;
+ res13_r = 0;
+ res13_i = 0;
+
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+4; // number of values in A
+#else
+ temp = off+2; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+ b1_r = ptrbb[2*1+0]; b1_i = ptrbb[2*1+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+ MADD(res10, a0, b1);
+
+ a1_r = ptrba[2*1+0]; a1_i = ptrba[2*1+1];
+ MADD(res01, a1, b0);
+ MADD(res11, a1, b1);
+
+ a0_r = ptrba[2*2+0]; a0_i = ptrba[2*2+1];
+ MADD(res02, a0, b0);
+ MADD(res12, a0, b1);
+
+ a1_r = ptrba[2*3+0]; a1_i = ptrba[2*3+1];
+ MADD(res03, a1, b0);
+ MADD(res13, a1, b1);
+
+ ptrba = ptrba+8;
+ ptrbb = ptrbb+4;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res01, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res02, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res03, alpha);
+ C0 = C0 + 2;
+
+ MADD_ALPHA_N_STORE(C1, res10, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res11, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res12, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res13, alpha);
+ C1 = C1 + 2;
+
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 4; // number of values in A
+#else
+ temp -= 2; // number of values in B
+#endif
+ ptrba += temp*4*2;
+ ptrbb += temp*2*2;
+#endif
+
+#ifdef LEFT
+ off += 4; // number of values in A
+#endif
+
+ }
+
+ if ( bm & 2 ) // do any 2x2 loop
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*2*2;
+ ptrbb = bb + off*2*2;
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+ res01_r = 0;
+ res01_i = 0;
+
+ res10_r = 0;
+ res10_i = 0;
+ res11_r = 0;
+ res11_i = 0;
+
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+2; // number of values in A
+#else
+ temp = off+2; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+ b1_r = ptrbb[2*1+0]; b1_i = ptrbb[2*1+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+ MADD(res10, a0, b1);
+
+ a1_r = ptrba[2*1+0]; a1_i = ptrba[2*1+1];
+ MADD(res01, a1, b0);
+ MADD(res11, a1, b1);
+
+
+ ptrba = ptrba+4;
+ ptrbb = ptrbb+4;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res01, alpha);
+ C0 = C0 + 2;
+
+ MADD_ALPHA_N_STORE(C1, res10, alpha);
+ C1 = C1 + 2;
+ MADD_ALPHA_N_STORE(C1, res11, alpha);
+ C1 = C1 + 2;
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 2; // number of values in A
+#else
+ temp -= 2; // number of values in B
+#endif
+ ptrba += temp*2*2;
+ ptrbb += temp*2*2;
+#endif
+
+#ifdef LEFT
+ off += 2; // number of values in A
+#endif
+
+ }
+
+ if ( bm & 1 ) // do any 1x2 loop
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*1*2;
+ ptrbb = bb + off*2*2;
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+
+ res10_r = 0;
+ res10_i = 0;
+
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+1; // number of values in A
+#else
+ temp = off+2; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+ b1_r = ptrbb[2*1+0]; b1_i = ptrbb[2*1+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+ MADD(res10, a0, b1);
+
+ ptrba = ptrba+2;
+ ptrbb = ptrbb+4;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+
+ MADD_ALPHA_N_STORE(C1, res10, alpha);
+ C1 = C1 + 2;
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 1; // number of values in A
+#else
+ temp -= 2; // number of values in B
+#endif
+ ptrba += temp*1*2;
+ ptrbb += temp*2*2;
+#endif
+
+#ifdef LEFT
+ off += 1; // number of values in A
+#endif
+
+ }
+
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+ off += 2;
+#endif
+
+ k = (bk<<2);
+ bb = bb+k;
+ i = (ldc<<2);
+ C = C+i;
+ }
+
+
+
+
+
+
+
+ for (j=0; j<(bn&1); j+=1) // do the Mx1 loops
+ {
+ C0 = C;
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+ off = offset;
+#endif
+
+ ptrba = ba;
+
+ for (i=0; i<bm/4; i+=1) // do blocks of 4x1 loops
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*4*2;
+ ptrbb = bb + off*1*2;
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+ res01_r = 0;
+ res01_i = 0;
+ res02_r = 0;
+ res02_i = 0;
+ res03_r = 0;
+ res03_i = 0;
+
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+4; // number of values in A
+#else
+ temp = off+1; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+
+ a1_r = ptrba[2*1+0]; a1_i = ptrba[2*1+1];
+ MADD(res01, a1, b0);
+
+ a0_r = ptrba[2*2+0]; a0_i = ptrba[2*2+1];
+ MADD(res02, a0, b0);
+
+ a1_r = ptrba[2*3+0]; a1_i = ptrba[2*3+1];
+ MADD(res03, a1, b0);
+
+ ptrba = ptrba+8;
+ ptrbb = ptrbb+2;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res01, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res02, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res03, alpha);
+ C0 = C0 + 2;
+
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 4; // number of values in A
+#else
+ temp -= 1; // number of values in B
+#endif
+ ptrba += temp*4*2;
+ ptrbb += temp*1*2;
+#endif
+
+#ifdef LEFT
+ off += 4; // number of values in A
+#endif
+
+ }
+
+ if ( bm & 2 ) // do any 2x1 loop
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*2*2;
+ ptrbb = bb + off*1*2;
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+ res01_r = 0;
+ res01_i = 0;
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+2; // number of values in A
+#else
+ temp = off+1; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+
+ a1_r = ptrba[2*1+0]; a1_i = ptrba[2*1+1];
+ MADD(res01, a1, b0);
+
+
+ ptrba = ptrba+4;
+ ptrbb = ptrbb+2;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+ MADD_ALPHA_N_STORE(C0, res01, alpha);
+ C0 = C0 + 2;
+
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 2; // number of values in A
+#else
+ temp -= 1; // number of values in B
+#endif
+ ptrba += temp*2*2;
+ ptrbb += temp*1*2;
+#endif
+
+#ifdef LEFT
+ off += 2; // number of values in A
+#endif
+
+ }
+
+ if ( bm & 1 ) // do any 1x1 loop
+ {
+
+#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ ptrbb = bb;
+#else
+ ptrba += off*1*2;
+ ptrbb = bb + off*1*2;
+#endif
+
+ res00_r = 0;
+ res00_i = 0;
+
+
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+ temp = bk-off;
+#elif defined(LEFT)
+ temp = off+1; // number of values in A
+#else
+ temp = off+1; // number of values in B
+#endif
+
+ for (k=0; k<temp; k++)
+ {
+ b0_r = ptrbb[2*0+0]; b0_i = ptrbb[2*0+1];
+
+ a0_r = ptrba[2*0+0]; a0_i = ptrba[2*0+1];
+ MADD(res00, a0, b0);
+
+ ptrba = ptrba+2;
+ ptrbb = ptrbb+2;
+ }
+
+ MADD_ALPHA_N_STORE(C0, res00, alpha);
+ C0 = C0 + 2;
+
+#if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
+ temp = bk - off;
+#ifdef LEFT
+ temp -= 1; // number of values in A
+#else
+ temp -= 1; // number of values in B
+#endif
+ ptrba += temp*1*2;
+ ptrbb += temp*1*2;
+#endif
+
+#ifdef LEFT
+ off += 1; // number of values in A
+#endif
+
+ }
+
+
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+ off += 1;
+#endif
+
+ k = (bk<<1);
+ bb = bb+k;
+ i = (ldc<<1);
+ C = C+i;
+ }
+ return 0;
+}
projects / platform / upstream / openblas.git / commitdiff