From 08786c4b95ae1337ba49c1e8eda9d9151e338420 Mon Sep 17 00:00:00 2001 From: Abdurrauf Date: Mon, 13 Mar 2017 01:23:16 +0400 Subject: [PATCH] strmm and ctrmm --- CONTRIBUTORS.md | 1 + README.md | 2 +- kernel/zarch/KERNEL.Z13 | 8 +- kernel/zarch/ckernelMacrosV.S | 1484 +++++++++++++++++++++++++++++ kernel/zarch/{dtrmm8x4V.S => ctrmm4x4V.S} | 422 +++----- kernel/zarch/dkernelMacros.S | 903 ------------------ kernel/zarch/skernelMacros.S | 4 +- kernel/zarch/strmm8x4V.S | 13 +- param.h | 14 +- 9 files changed, 1653 insertions(+), 1198 deletions(-) create mode 100644 kernel/zarch/ckernelMacrosV.S rename kernel/zarch/{dtrmm8x4V.S => ctrmm4x4V.S} (65%) delete mode 100644 kernel/zarch/dkernelMacros.S diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ad59c31..a960708 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -165,5 +165,6 @@ In chronological order: * Abdelrauf * [2017-01-01] dgemm and dtrmm kernels for IBM z13 * [2017-02-26] ztrmm kernel for IBM z13 + * [2017-03-13] strmm and ctrmm kernel for IBM z13 diff --git a/README.md b/README.md index 1c3255f..c933f62 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Please read GotoBLAS_01Readme.txt - **ARM Cortex-A57**: Experimental #### IBM zEnterprise System: -- **Z13**: blas3 for double +- **Z13**: Optimized Level-3 BLAS ### Support OS: diff --git a/kernel/zarch/KERNEL.Z13 b/kernel/zarch/KERNEL.Z13 index c25d047..9e3650b 100644 --- a/kernel/zarch/KERNEL.Z13 +++ b/kernel/zarch/KERNEL.Z13 @@ -82,7 +82,7 @@ ZGEMVTKERNEL = ../arm/zgemv_t.c STRMMKERNEL = strmm8x4V.S DTRMMKERNEL = trmm8x4V.S -CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CTRMMKERNEL = ctrmm4x4V.S ZTRMMKERNEL = ztrmm4x4V.S SGEMMKERNEL = strmm8x4V.S @@ -107,9 +107,9 @@ DGEMMITCOPYOBJ = dgemm_itcopy.o DGEMMONCOPYOBJ = dgemm_oncopy.o DGEMMOTCOPYOBJ = dgemm_otcopy.o -CGEMMKERNEL = ../generic/zgemmkernel_2x2.c -CGEMMONCOPY = ../generic/zgemm_ncopy_2.c -CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMKERNEL = ctrmm4x4V.S +CGEMMONCOPY = ../generic/zgemm_ncopy_4.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_4.c CGEMMONCOPYOBJ = cgemm_oncopy.o CGEMMOTCOPYOBJ = cgemm_otcopy.o diff --git a/kernel/zarch/ckernelMacrosV.S b/kernel/zarch/ckernelMacrosV.S new file mode 100644 index 0000000..87d5410 --- /dev/null +++ b/kernel/zarch/ckernelMacrosV.S @@ -0,0 +1,1484 @@ +/****************************************Implementation**Details**********************************************/ +/* */ +/* Lets denote (a,a1i) complex which is mathematically a+a1*i */ +/* Complex number multiplication: (a,a1i)*(b,b1i) */ +/* As i*i=-1 .The multiplication result will be: */ +/* (a+a1*i)(b+b1*i)=a*b+a1*i*b1*i+ a1*i*b+a*b1*i=a*b-a1*b1 + (a1*b+a*b1)*i which is (ab-a1b1,a1b+ab1) */ +/* so let c= ab-a1b1 , ci=a1b+ab1 then */ +/* c=c+a*b-a1*b1 => c=a*b-( a1*b1-c) => c= a1*b1-c then c=a*b-c two mseb */ +/* ci=ci+a1*b+a*b1 => ci= a1*b+ci then ci= a*b1+ci */ +/* For simd real and imaginary parts will be grouped together */ +/* such (realA,realK) and (imageA ,imageK) */ +/* Simd(0,1)=(a*b,k*b)-((ai*bi,ki*bi)-Simd(0,1)) */ +/* SimdI(0,1)=SimdI(0,1)+(a*bi,k*bi)+(ai*b,ki*b) */ +/* */ +/* */ +/* for defined(NR) || defined(NC) || defined(TR) || defined(TC) */ +/* (a+a1*I)(b-b1*I)=ab+a1*b1+I(a1b-ab1) */ +/* */ +/* c=c+ab+a1b1 => c=a1b1+c;c=ab+c */ +/* ci=ci+a1b-ab1 => ci=a1*b-(ab1-ci) => ci=ab1-ci; ci=a1*b-ci */ +/* */ +/* */ +/* for defined(RN) || defined(RT) || defined(CN) || defined(CT) */ +/* (a-a1*I)(b+b1*I)=ab+a1*b1+I(-a1b+ab1) */ +/* */ +/* c=c+ab+a1b1 => c=a1b1+c;c=ab+c */ +/* ci=ci+a1b-ab1 => ci=a*b1-(a1b-ci) => ci=a1b-ci; ci=a*b1-ci */ +/* */ +/* */ +/* for defined(RR) || defined(RC) || defined(CR) || defined(CC) */ +/* (a-a1*I)(b-b1*I)=ab-a1*b1+I(-a1b-ab1) */ +/* */ +/* c= a1*b1-c then c=a*b-c */ +/* ci = ci-a1*b -a*b1; */ +/* as ibm z13 only has x*z-m x*z+m instructions implementation will be changed a bit */ +/* Assuming ci=0; and cix=cix+a1b+ab1 ; ci=ci-cix will work */ +/* cix= a*b1+cix ; cix= a1*b+cix (two madb) ci=ci-cix (sign change if ci=0) */ +/* As c=0 then */ +/* c=a*b-c then c=a1*b1-c => c=(a1*b1-(a*b-c)) which is -1*( a*b -(a1*b1-c)) */ +/* */ +/* Values will be equal to (-c) and (-ci) */ +/* To change sign it'll be multiplied by -1*(alpha+alpha_i) */ +/* This is done once: */ +/* lcdbr ALPHA_I,ALPHA_I */ +/* lcdbr ALPHA ,ALPHA */ +/*************************************************************************************************************/ + +/*************************Zero vectors***************************************/ +/*zero vectors for 4x4 */ +.macro ZERO_ZCVEC_4x4 + vzero %v16 + vzero %v17 + vzero %v18 + vzero %v19 + vzero %v20 + vzero %v21 + vzero %v22 + vzero %v23 + vzero %v24 + vzero %v25 + vzero %v26 + vzero %v27 + vzero %v28 + vzero %v29 + vzero %v30 + vzero %v31 +.endm + +/*zero vectors for */ +.macro ZERO_ZCVEC_2x4 + vzero %v16 + vzero %v17 + vzero %v18 + vzero %v19 + vzero %v20 + vzero %v21 + vzero %v22 + vzero %v23 +.endm + +/*zero vectors for */ +.macro ZERO_ZCVEC_1x4 + vzero %v16 + vzero %v17 + vzero %v18 + vzero %v19 +.endm + +/*zero vectors for */ +.macro ZERO_ZCVEC_4x2 + ZERO_ZCVEC_2x4 +.endm + +.macro ZERO_ZCVEC_4x1 + ZERO_ZCVEC_1x4 +.endm + +/*zero vectors for */ +.macro ZERO_ZCVEC_2x2 + vzero %v16 + vzero %v17 + vzero %v20 + vzero %v21 +.endm + +/*zero vectors for */ +.macro ZERO_ZCVEC_1x2 + vzero %v16 + vzero %v17 +.endm + +/*zero vectors for */ +.macro ZERO_ZCVEC_2x1 + vzero %v16 + vzero %v17 +.endm + +/*zero vectors for 1x1*/ +.macro ZERO_ZCVEC_1x1 + lzer %f6 + lzer %f7 +.endm + + +/* + Calculate for 4x2 inner +*/ +.macro CalcComplex_4x2 vResR1, vResI1, vResR2, vResI2, vResR3, vResI3, vResR4, vResI4, vr1, vi1, vr2, vi2, vrB, viB,vrB2, viB2 + + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + vfmsdb \vResR1, \vi1, \viB, \vResR1 + vfmadb \vResI1, \vr1, \viB, \vResI1 + vfmsdb \vResR2, \vi2, \viB, \vResR2 + vfmadb \vResI2, \vr2, \viB, \vResI2 + + vfmsdb \vResR3, \vi1, \viB2, \vResR3 + vfmadb \vResI3, \vr1, \viB2, \vResI3 + vfmsdb \vResR4, \vi2, \viB2, \vResR4 + vfmadb \vResI4, \vr2, \viB2, \vResI4 + + vfmsdb \vResR1, \vr1, \vrB, \vResR1 + vfmadb \vResI1, \vi1, \vrB, \vResI1 + vfmsdb \vResR2, \vr2, \vrB, \vResR2 + vfmadb \vResI2, \vi2, \vrB, \vResI2 + + vfmsdb \vResR3, \vr1, \vrB2, \vResR3 + vfmadb \vResI3, \vi1, \vrB2, \vResI3 + vfmsdb \vResR4, \vr2, \vrB2, \vResR4 + vfmadb \vResI4, \vi2, \vrB2, \vResI4 + + #endif + + #if defined(NR) || defined(NC) || defined(TR) || defined(TC) + vfmadb \vResR1, \vi1, \viB, \vResR1 + vfmsdb \vResI1, \vr1, \viB, \vResI1 + vfmadb \vResR2, \vi2, \viB, \vResR2 + vfmsdb \vResI2, \vr2, \viB, \vResI2 + + vfmadb \vResR3, \vi1, \viB2, \vResR3 + vfmsdb \vResI3, \vr1, \viB2, \vResI3 + vfmadb \vResR4, \vi2, \viB2, \vResR4 + vfmsdb \vResI4, \vr2, \viB2, \vResI4 + + vfmadb \vResR1, \vr1, \vrB, \vResR1 + vfmsdb \vResI1, \vi1, \vrB, \vResI1 + vfmadb \vResR2, \vr2, \vrB, \vResR2 + vfmsdb \vResI2, \vi2, \vrB, \vResI2 + + vfmadb \vResR3, \vr1, \vrB2, \vResR3 + vfmsdb \vResI3, \vi1, \vrB2, \vResI3 + vfmadb \vResR4, \vr2, \vrB2, \vResR4 + vfmsdb \vResI4, \vi2, \vrB2, \vResI4 + + #endif + + #if defined(RN) || defined(RT) || defined(CN) || defined(CT) + vfmadb \vResR1, \vi1, \viB, \vResR1 + vfmsdb \vResI1, \vi1, \vrB, \vResI1 + vfmadb \vResR2, \vi2, \viB, \vResR2 + vfmsdb \vResI2, \vi2, \vrB, \vResI2 + + vfmadb \vResR3, \vi1, \viB2, \vResR3 + vfmsdb \vResI3, \vi1, \vrB2, \vResI3 + vfmadb \vResR4, \vi2, \viB2, \vResR4 + vfmsdb \vResI4, \vi2, \vrB2, \vResI4 + + vfmadb \vResR1, \vr1, \vrB, \vResR1 + vfmsdb \vResI1, \vr1, \viB, \vResI1 + vfmadb \vResR2, \vr2, \vrB, \vResR2 + vfmsdb \vResI2, \vr2, \viB, \vResI2 + + vfmadb \vResR3, \vr1, \vrB2, \vResR3 + vfmsdb \vResI3, \vr1, \viB2, \vResI3 + vfmadb \vResR4, \vr2, \vrB2, \vResR4 + vfmsdb \vResI4, \vr2, \viB2, \vResI4 + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + + vfmsdb \vResR1, \vr1, \vrB, \vResR1 + vfmadb \vResI1, \vi1, \vrB, \vResI1 + vfmsdb \vResR2, \vr2, \vrB, \vResR2 + vfmadb \vResI2, \vi2, \vrB, \vResI2 + + vfmsdb \vResR3, \vr1, \vrB2, \vResR3 + vfmadb \vResI3, \vi1, \vrB2, \vResI3 + vfmsdb \vResR4, \vr2, \vrB2, \vResR4 + vfmadb \vResI4, \vi2, \vrB2, \vResI4 + + vfmsdb \vResR1, \vi1, \viB, \vResR1 + vfmadb \vResI1, \vr1, \viB, \vResI1 + vfmsdb \vResR2, \vi2, \viB, \vResR2 + vfmadb \vResI2, \vr2, \viB, \vResI2 + + vfmsdb \vResR3, \vi1, \viB2, \vResR3 + vfmadb \vResI3, \vr1, \viB2, \vResI3 + vfmsdb \vResR4, \vi2, \viB2, \vResR4 + vfmadb \vResI4, \vr2, \viB2, \vResI4 + + + #endif + +.endm + +/* + Calculate for 2x4 inner +*/ +.macro CalcComplex_2x4 vResR1, vResI1, vResR2, vResI2, vResR3, vResI3, vResR4, vResI4, vr1, vi1, vr2, vi2, vrB, viB,vrB2, viB2 + + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + vfmsdb \vResR1, \vi1, \viB, \vResR1 + vfmadb \vResI1, \vr1, \viB, \vResI1 + vfmsdb \vResR2, \vi2, \viB, \vResR2 + vfmadb \vResI2, \vr2, \viB, \vResI2 + + vfmsdb \vResR3, \vi1, \viB2, \vResR3 + vfmadb \vResI3, \vr1, \viB2, \vResI3 + vfmsdb \vResR4, \vi2, \viB2, \vResR4 + vfmadb \vResI4, \vr2, \viB2, \vResI4 + + vfmsdb \vResR1, \vr1, \vrB, \vResR1 + vfmadb \vResI1, \vi1, \vrB, \vResI1 + vfmsdb \vResR2, \vr2, \vrB, \vResR2 + vfmadb \vResI2, \vi2, \vrB, \vResI2 + + vfmsdb \vResR3, \vr1, \vrB2, \vResR3 + vfmadb \vResI3, \vi1, \vrB2, \vResI3 + vfmsdb \vResR4, \vr2, \vrB2, \vResR4 + vfmadb \vResI4, \vi2, \vrB2, \vResI4 + + #endif + + #if defined(RN) || defined(RT) || defined(CN) || defined(CT) + vfmadb \vResR1, \vi1, \viB, \vResR1 + vfmsdb \vResI1, \vr1, \viB, \vResI1 + vfmadb \vResR2, \vi2, \viB, \vResR2 + vfmsdb \vResI2, \vr2, \viB, \vResI2 + + vfmadb \vResR3, \vi1, \viB2, \vResR3 + vfmsdb \vResI3, \vr1, \viB2, \vResI3 + vfmadb \vResR4, \vi2, \viB2, \vResR4 + vfmsdb \vResI4, \vr2, \viB2, \vResI4 + + vfmadb \vResR1, \vr1, \vrB, \vResR1 + vfmsdb \vResI1, \vi1, \vrB, \vResI1 + vfmadb \vResR2, \vr2, \vrB, \vResR2 + vfmsdb \vResI2, \vi2, \vrB, \vResI2 + + vfmadb \vResR3, \vr1, \vrB2, \vResR3 + vfmsdb \vResI3, \vi1, \vrB2, \vResI3 + vfmadb \vResR4, \vr2, \vrB2, \vResR4 + vfmsdb \vResI4, \vi2, \vrB2, \vResI4 + + #endif + + #if defined(NR) || defined(NC) || defined(TR) || defined(TC) + vfmadb \vResR1, \vi1, \viB, \vResR1 + vfmsdb \vResI1, \vi1, \vrB, \vResI1 + vfmadb \vResR2, \vi2, \viB, \vResR2 + vfmsdb \vResI2, \vi2, \vrB, \vResI2 + + vfmadb \vResR3, \vi1, \viB2, \vResR3 + vfmsdb \vResI3, \vi1, \vrB2, \vResI3 + vfmadb \vResR4, \vi2, \viB2, \vResR4 + vfmsdb \vResI4, \vi2, \vrB2, \vResI4 + + vfmadb \vResR1, \vr1, \vrB, \vResR1 + vfmsdb \vResI1, \vr1, \viB, \vResI1 + vfmadb \vResR2, \vr2, \vrB, \vResR2 + vfmsdb \vResI2, \vr2, \viB, \vResI2 + + vfmadb \vResR3, \vr1, \vrB2, \vResR3 + vfmsdb \vResI3, \vr1, \viB2, \vResI3 + vfmadb \vResR4, \vr2, \vrB2, \vResR4 + vfmsdb \vResI4, \vr2, \viB2, \vResI4 + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + + vfmsdb \vResR1, \vr1, \vrB, \vResR1 + vfmadb \vResI1, \vi1, \vrB, \vResI1 + vfmsdb \vResR2, \vr2, \vrB, \vResR2 + vfmadb \vResI2, \vi2, \vrB, \vResI2 + + vfmsdb \vResR3, \vr1, \vrB2, \vResR3 + vfmadb \vResI3, \vi1, \vrB2, \vResI3 + vfmsdb \vResR4, \vr2, \vrB2, \vResR4 + vfmadb \vResI4, \vi2, \vrB2, \vResI4 + + vfmsdb \vResR1, \vi1, \viB, \vResR1 + vfmadb \vResI1, \vr1, \viB, \vResI1 + vfmsdb \vResR2, \vi2, \viB, \vResR2 + vfmadb \vResI2, \vr2, \viB, \vResI2 + + vfmsdb \vResR3, \vi1, \viB2, \vResR3 + vfmadb \vResI3, \vr1, \viB2, \vResI3 + vfmsdb \vResR4, \vi2, \viB2, \vResR4 + vfmadb \vResI4, \vr2, \viB2, \vResI4 + + + #endif + +.endm + +/* + Calculate for 2x2 inner +*/ +.macro CalcComplex_2x2 vResR1, vResI1,vResR2, vResI2, vR1, vI1, vRB, vIB, vRB2, vIB2 + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + vfmsdb \vResR1, \vI1, \vIB, \vResR1 + vfmadb \vResI1, \vR1, \vIB, \vResI1 + + vfmsdb \vResR2, \vI1, \vIB2, \vResR2 + vfmadb \vResI2, \vR1, \vIB2, \vResI2 + + vfmsdb \vResR1, \vR1, \vRB, \vResR1 + vfmadb \vResI1, \vI1, \vRB, \vResI1 + + vfmsdb \vResR2, \vR1, \vRB2, \vResR2 + vfmadb \vResI2, \vI1, \vRB2, \vResI2 + #endif + + #if defined(NR) || defined(NC) || defined(TR) || defined(TC) + vfmadb \vResR1, \vI1, \vIB, \vResR1 + vfmsdb \vResI1, \vR1, \vIB, \vResI1 + + vfmadb \vResR2, \vI1, \vIB2, \vResR2 + vfmsdb \vResI2, \vR1, \vIB2, \vResI2 + + vfmadb \vResR1, \vR1, \vRB, \vResR1 + vfmsdb \vResI1, \vI1, \vRB, \vResI1 + + vfmadb \vResR2, \vR1, \vRB2, \vResR2 + vfmsdb \vResI2, \vI1, \vRB2, \vResI2 + #endif + + #if defined(RN) || defined(RT) || defined(CN) || defined(CT) + vfmadb \vResR1, \vI1, \vIB, \vResR1 + vfmsdb \vResI1, \vI1, \vRB, \vResI1 + + vfmadb \vResR2, \vI1, \vIB2, \vResR2 + vfmsdb \vResI2, \vI1, \vRB2, \vResI2 + + vfmadb \vResR1, \vR1, \vRB, \vResR1 + vfmsdb \vResI1, \vR1, \vIB, \vResI1 + + vfmadb \vResR2, \vR1, \vRB2, \vResR2 + vfmsdb \vResI2, \vR1, \vIB2, \vResI2 + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + vfmsdb \vResR1, \vR1, \vRB, \vResR1 + vfmadb \vResI1, \vI1, \vRB, \vResI1 + + vfmsdb \vResR2, \vR1, \vRB2, \vResR2 + vfmadb \vResI2, \vI1, \vRB2, \vResI2 + + vfmsdb \vResR1, \vI1, \vIB, \vResR1 + vfmadb \vResI1, \vR1, \vIB, \vResI1 + + vfmsdb \vResR2, \vI1, \vIB2, \vResR2 + vfmadb \vResI2, \vR1, \vIB2, \vResI2 + #endif +.endm + +/* + Calculate for 2x1 inner +*/ +.macro CalcComplex_2x1 vRealResult1, vImageResult1, vReal1, vImage1, vecRealB, vecImageB + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + #endif + + #if defined(NR) || defined(NC) || defined(TR) || defined(TC) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + #endif + + #if defined(RN) || defined(RT) || defined(CN) || defined(CT) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + #endif +.endm + +/* + Calculate for 1x2 inner +*/ +.macro CalcComplex_1x2 vRealResult1, vImageResult1, vReal1, vImage1, vecRealB, vecImageB + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + #endif + + #if defined(RN) || defined(CN) || defined(RT) || defined(CT) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + #endif + + #if defined(NR) || defined(TR) || defined(NC) || defined(TC) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + #endif +.endm + + +/* + Calculate for 4x1 inner +*/ +.macro CalcComplex_4x1 vRealResult1, vImageResult1, vRealResult2, vImageResult2, vReal1, vImage1, vReal2, vImage2, vecRealB, vecImageB + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmsdb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmadb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmsdb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmadb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + #endif + + #if defined(NR) || defined(NC) || defined(TR) || defined(TC) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmadb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmsdb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmadb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmsdb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + #endif + + #if defined(RN) || defined(RT) || defined(CN) || defined(CT) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmadb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmsdb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmadb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmsdb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmsdb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmadb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmsdb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmadb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + #endif + +.endm + +/* + Calculate for 1x4 inner +*/ +.macro CalcComplex_1x4 vRealResult1, vImageResult1, vRealResult2, vImageResult2, vReal1, vImage1, vReal2, vImage2, vecRealB, vecImageB + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmsdb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmadb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmsdb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmadb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + #endif + + #if defined(RN) || defined(CN) || defined(RT) || defined(CT) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmadb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmsdb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmadb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmsdb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + #endif + + #if defined(NR) || defined(TR) || defined(NC) || defined(TC) + vfmadb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmsdb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmadb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmsdb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + vfmadb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmsdb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmadb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmsdb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmsdb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmadb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmsdb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmadb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 + #endif + +.endm + +.macro CalcComplex_1x1 RealResult1, ImageResult1, Real1, Image1, RealB, ImageB + #if defined(NN) || defined(NT) || defined(TN) || defined(TT) + msebr \RealResult1, \Image1, \ImageB + maebr \ImageResult1, \Real1, \ImageB + msebr \RealResult1, \Real1, \RealB + maebr \ImageResult1, \Image1, \RealB + #endif + + #if defined(NR) || defined(NC) || defined(TR) || defined(TC) + maebr \RealResult1, \Image1, \ImageB + msebr \ImageResult1, \Real1, \ImageB + maebr \RealResult1, \Real1, \RealB + msebr \ImageResult1, \Image1, \RealB + #endif + + #if defined(RN) || defined(RT) || defined(CN) || defined(CT) + maebr \RealResult1, \Image1, \ImageB + msebr \ImageResult1, \Image1, \RealB + maebr \RealResult1, \Real1, \RealB + msebr \ImageResult1, \Real1, \ImageB + #endif + #if defined(RR) || defined(RC) || defined(CR) || defined(CC) + msebr \RealResult1, \Real1, \RealB + maebr \ImageResult1, \Image1, \RealB + msebr \RealResult1, \Image1, \ImageB + maebr \ImageResult1, \Real1, \ImageB + #endif +.endm + +#define DISP(ind,stride,disp) (ind*stride+disp) +#define DISP64(ind,disp) (ind*32+disp) +#define DISP32(ind,disp) (ind*16+disp) +#define DISP16(ind,disp) (ind*8+disp) + +#define unit_size 8 +#define DISP(ind,stride,disp) (ind*stride+disp) +#define DISP8(ind,disp) (ind*unit_size*8+disp) +#define DISP4(ind,disp) (ind*unit_size*4+disp) +#define DISP2(ind,disp) (ind*unit_size*2+disp) +#define DISP1(ind,disp) (ind*unit_size+disp) +#define N8 (8*unit_size) +#define N4 (4*unit_size) +#define N2 (2*unit_size) +#define N1 (1*unit_size) + + + +.macro ZCALC_4x4_I PTR_A_REG,PTR_B_REG,Index,IsLast + + vlef %v1, DISP4(\Index ,0) (\PTR_A_REG),0 + vlef %v5, DISP4(\Index ,4) (\PTR_A_REG),0 + vlef %v1, DISP4(\Index ,8) (\PTR_A_REG),2 + vlef %v5, DISP4(\Index ,12) (\PTR_A_REG),2 + vlef %v3, DISP4(\Index ,16) (\PTR_A_REG),0 + vlef %v7, DISP4(\Index ,20) (\PTR_A_REG),0 + vlef %v3, DISP4(\Index ,24) (\PTR_A_REG),2 + vlef %v7, DISP4(\Index ,28) (\PTR_A_REG),2 + vlrepf %v9, DISP4(\Index ,0)(\PTR_B_REG) + vlrepf %v10 , DISP4(\Index ,4)(\PTR_B_REG) + vlrepf %v11, DISP4(\Index ,8)(\PTR_B_REG) + vlrepf %v12 , DISP4(\Index ,12)(\PTR_B_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v3,%v3 + vldeb %v7,%v7 + vldeb %v9,%v9 + vldeb %v10,%v10 + vldeb %v11,%v11 + vldeb %v12,%v12 + + CalcComplex_4x2 %v16,%v17,%v18,%v19,%v20,%v21,%v22,%v23,%v1,%v5,%v3,%v7,%v9,%v10,%v11,%v12 + + vlrepf %v9, DISP4(\Index ,16)(\PTR_B_REG) + vlrepf %v10 , DISP4(\Index ,20)(\PTR_B_REG) + vlrepf %v11, DISP4(\Index ,24)(\PTR_B_REG) + vlrepf %v12 , DISP4(\Index ,28)(\PTR_B_REG) + vldeb %v9,%v9 + vldeb %v10,%v10 + vldeb %v11,%v11 + vldeb %v12,%v12 + + .if \IsLast==1 + la \PTR_A_REG, DISP4(\Index ,32)(\PTR_A_REG) + .endif + CalcComplex_4x2 %v24,%v25,%v26,%v27,%v28,%v29,%v30,%v31,%v1,%v5,%v3,%v7,%v9,%v10,%v11,%v12 + + .if \IsLast==1 + la \PTR_B_REG, DISP4(\Index ,32)(\PTR_B_REG) + .endif +.endm + +.macro ZCALC_4x2_I PTR_A_REG,PTR_B_REG,Index,IsLast + + vlef %v1, DISP4(\Index ,0) (\PTR_A_REG),0 + vlef %v5, DISP4(\Index ,4) (\PTR_A_REG),0 + vlef %v1, DISP4(\Index ,8) (\PTR_A_REG),2 + vlef %v5, DISP4(\Index ,12) (\PTR_A_REG),2 + vlef %v3, DISP4(\Index ,16) (\PTR_A_REG),0 + vlef %v7, DISP4(\Index ,20) (\PTR_A_REG),0 + vlef %v3, DISP4(\Index ,24) (\PTR_A_REG),2 + vlef %v7, DISP4(\Index ,28) (\PTR_A_REG),2 + vlrepf %v9, DISP2(\Index ,0)(\PTR_B_REG) + vlrepf %v10 , DISP2(\Index ,4)(\PTR_B_REG) + vlrepf %v11, DISP2(\Index ,8)(\PTR_B_REG) + vlrepf %v12 , DISP2(\Index ,12)(\PTR_B_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v3,%v3 + vldeb %v7,%v7 + vldeb %v9,%v9 + vldeb %v10,%v10 + vldeb %v11,%v11 + vldeb %v12,%v12 + .if \IsLast==1 + la \PTR_A_REG, DISP4(\Index ,32)(\PTR_A_REG) + .endif + CalcComplex_4x2 %v16,%v17,%v18,%v19,%v20,%v21,%v22,%v23,%v1,%v5,%v3,%v7,%v9,%v10,%v11,%v12 + + .if \IsLast==1 + la \PTR_B_REG, DISP2(\Index ,16)(\PTR_B_REG) + .endif +.endm + +.macro ZCALC_2x4_I PTR_A_REG,PTR_B_REG,Index,IsLast + vlef %v1, DISP4(\Index ,0) (\PTR_B_REG),0 + vlef %v5, DISP4(\Index ,4) (\PTR_B_REG),0 + vlef %v1, DISP4(\Index ,8) (\PTR_B_REG),2 + vlef %v5, DISP4(\Index ,12) (\PTR_B_REG),2 + vlef %v3, DISP4(\Index ,16) (\PTR_B_REG),0 + vlef %v7, DISP4(\Index ,20) (\PTR_B_REG),0 + vlef %v3, DISP4(\Index ,24) (\PTR_B_REG),2 + vlef %v7, DISP4(\Index ,28) (\PTR_B_REG),2 + vlrepf %v9, DISP2(\Index ,0)(\PTR_A_REG) + vlrepf %v10 , DISP2(\Index ,4)(\PTR_A_REG) + vlrepf %v11, DISP2(\Index ,8)(\PTR_A_REG) + vlrepf %v12 , DISP2(\Index ,12)(\PTR_A_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v3,%v3 + vldeb %v7,%v7 + vldeb %v9,%v9 + vldeb %v10,%v10 + vldeb %v11,%v11 + vldeb %v12,%v12 + .if \IsLast==1 + la \PTR_B_REG, DISP4(\Index ,32)(\PTR_B_REG) + .endif + CalcComplex_2x4 %v16,%v17,%v18,%v19,%v20,%v21,%v22,%v23,%v1,%v5,%v3,%v7,%v9,%v10,%v11,%v12 + + .if \IsLast==1 + la \PTR_A_REG, DISP2(\Index ,16)(\PTR_A_REG) + .endif +.endm + +.macro ZCALC_4x1_I PTR_A_REG,PTR_B_REG,Index,IsLast + vlef %v1, DISP4(\Index ,0) (\PTR_A_REG),0 + vlef %v5, DISP4(\Index ,4) (\PTR_A_REG),0 + vlef %v1, DISP4(\Index ,8) (\PTR_A_REG),2 + vlef %v5, DISP4(\Index ,12) (\PTR_A_REG),2 + vlef %v3, DISP4(\Index ,16) (\PTR_A_REG),0 + vlef %v7, DISP4(\Index ,20) (\PTR_A_REG),0 + vlef %v3, DISP4(\Index ,24) (\PTR_A_REG),2 + vlef %v7, DISP4(\Index ,28) (\PTR_A_REG),2 + vlrepf %v9, DISP1(\Index ,0)(\PTR_B_REG) + vlrepf %v10 , DISP1(\Index ,4)(\PTR_B_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v3,%v3 + vldeb %v7,%v7 + vldeb %v9,%v9 + vldeb %v10,%v10 + .if \IsLast==1 + la \PTR_A_REG, DISP4(\Index ,32)(\PTR_A_REG) + .endif + CalcComplex_4x1 %v16,%v17,%v18,%v19,%v1,%v5,%v3,%v7,%v9,%v10 + + .if \IsLast==1 + la \PTR_B_REG, DISP1(\Index ,8)(\PTR_B_REG) + .endif + +.endm + +.macro ZCALC_1x4_I PTR_A_REG,PTR_B_REG,Index,IsLast + vlef %v1, DISP4(\Index ,0) (\PTR_B_REG),0 + vlef %v5, DISP4(\Index ,4) (\PTR_B_REG),0 + vlef %v1, DISP4(\Index ,8) (\PTR_B_REG),2 + vlef %v5, DISP4(\Index ,12) (\PTR_B_REG),2 + vlef %v3, DISP4(\Index ,16) (\PTR_B_REG),0 + vlef %v7, DISP4(\Index ,20) (\PTR_B_REG),0 + vlef %v3, DISP4(\Index ,24) (\PTR_B_REG),2 + vlef %v7, DISP4(\Index ,28) (\PTR_B_REG),2 + vlrepf %v9, DISP1(\Index ,0)(\PTR_A_REG) + vlrepf %v10 , DISP1(\Index ,4)(\PTR_A_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v3,%v3 + vldeb %v7,%v7 + vldeb %v9,%v9 + vldeb %v10,%v10 + .if \IsLast==1 + la \PTR_B_REG, DISP4(\Index ,32)(\PTR_B_REG) + .endif + CalcComplex_1x4 %v16,%v17,%v18,%v19,%v1,%v5,%v3,%v7,%v9,%v10 + + .if \IsLast==1 + la \PTR_A_REG, DISP1(\Index ,8)(\PTR_A_REG) + .endif +.endm + +.macro ZCALC_2x2_I PTR_A_REG,PTR_B_REG ,Index,IsLast + vlef %v1, DISP2(\Index ,0) (\PTR_A_REG),0 + vlef %v5, DISP2(\Index ,4) (\PTR_A_REG),0 + vlef %v1, DISP2(\Index ,8) (\PTR_A_REG),2 + vlef %v5, DISP2(\Index ,12) (\PTR_A_REG),2 + vlrepf %v9, DISP2(\Index ,0)(\PTR_B_REG) + vlrepf %v10 , DISP2(\Index ,4)(\PTR_B_REG) + vlrepf %v11, DISP2(\Index ,8)(\PTR_B_REG) + vlrepf %v12 , DISP2(\Index ,12)(\PTR_B_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v9,%v9 + vldeb %v10,%v10 + vldeb %v11,%v11 + vldeb %v12,%v12 + .if \IsLast==1 + la \PTR_A_REG, DISP2(\Index ,16)(\PTR_A_REG) + .endif + CalcComplex_2x2 %v16,%v17,%v20,%v21,%v1,%v5, %v9,%v10,%v11,%v12 + .if \IsLast==1 + la \PTR_B_REG, DISP2(\Index ,16)(\PTR_B_REG) + .endif +.endm + +.macro ZCALC_2x1_I PTR_A_REG,PTR_B_REG ,Index,IsLast + vlef %v1, DISP2(\Index ,0) (\PTR_A_REG),0 + vlef %v5, DISP2(\Index ,4) (\PTR_A_REG),0 + vlef %v1, DISP2(\Index ,8) (\PTR_A_REG),2 + vlef %v5, DISP2(\Index ,12) (\PTR_A_REG),2 + vlrepf %v9, DISP1(\Index ,0)(\PTR_B_REG) + vlrepf %v10 , DISP1(\Index ,4)(\PTR_B_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v9,%v9 + vldeb %v10,%v10 + .if \IsLast==1 + la \PTR_A_REG, DISP2(\Index ,16)(\PTR_A_REG) + .endif + CalcComplex_2x1 %v16,%v17, %v1,%v5, %v9,%v10 + .if \IsLast==1 + la \PTR_B_REG, DISP1(\Index ,8)(\PTR_B_REG) + .endif +.endm + +.macro ZCALC_1x2_I PTR_A_REG,PTR_B_REG ,Index,IsLast + vlef %v1, DISP2(\Index ,0) (\PTR_B_REG),0 + vlef %v5, DISP2(\Index ,4) (\PTR_B_REG),0 + vlef %v1, DISP2(\Index ,8) (\PTR_B_REG),2 + vlef %v5, DISP2(\Index ,12) (\PTR_B_REG),2 + vlrepf %v9, DISP1(\Index ,0)(\PTR_A_REG) + vlrepf %v10 , DISP1(\Index ,4)(\PTR_A_REG) + vldeb %v1,%v1 + vldeb %v5,%v5 + vldeb %v9,%v9 + vldeb %v10,%v10 + .if \IsLast==1 + la \PTR_B_REG, DISP2(\Index ,16)(\PTR_B_REG) + .endif + CalcComplex_1x2 %v16,%v17, %v1,%v5, %v9,%v10 + .if \IsLast==1 + la \PTR_A_REG, DISP1(\Index ,8)(\PTR_A_REG) + .endif +.endm + +.macro ZCALC_1x1_I PTR_A_REG,PTR_B_REG ,Index,IsLast + le %f1 , DISP1(\Index ,0)(\PTR_A_REG) + le %f3 , DISP1(\Index ,4)(\PTR_A_REG) + le %f4 , DISP1(\Index ,0)(\PTR_B_REG) + le %f5 , DISP1(\Index ,4)(\PTR_B_REG) + .if \IsLast==1 + la \PTR_A_REG, DISP1(\Index ,8)(\PTR_A_REG) + .endif + CalcComplex_1x1 %f6,%f7,%f1,%f3,%f4,%f5 + .if \IsLast==1 + la \PTR_B_REG, DISP1(\Index ,8)(\PTR_B_REG) + .endif +.endm + +.macro ZCALC_4x4 PTR_A_REG,PTR_B_REG + ZCALC_4x4_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm +.macro ZCALC_4x2 PTR_A_REG,PTR_B_REG + ZCALC_4x2_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm +.macro ZCALC_4x1 PTR_A_REG,PTR_B_REG + ZCALC_4x1_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm + +.macro ZCALC_4x4_4 PTR_A_REG,PTR_B_REG + ZCALC_4x4_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_4x4_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_4x4_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_4x4_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm +.macro ZCALC_4x2_4 PTR_A_REG,PTR_B_REG + ZCALC_4x2_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_4x2_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_4x2_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_4x2_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm +.macro ZCALC_4x1_4 PTR_A_REG,PTR_B_REG + ZCALC_4x1_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_4x1_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_4x1_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_4x1_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm + +.macro ZCALC_2x4_4 PTR_A_REG,PTR_B_REG + ZCALC_2x4_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_2x4_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_2x4_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_2x4_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm + +.macro ZCALC_2x4 PTR_A_REG,PTR_B_REG + ZCALC_2x4_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm + +.macro ZCALC_1x4_4 PTR_A_REG,PTR_B_REG + ZCALC_1x4_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_1x4_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_1x4_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_1x4_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm + +.macro ZCALC_1x4 PTR_A_REG,PTR_B_REG + ZCALC_1x4_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm +.macro ZCALC_2x2 PTR_A_REG,PTR_B_REG + ZCALC_2x2_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm + +.macro ZCALC_2x2_4 PTR_A_REG,PTR_B_REG + ZCALC_2x2_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_2x2_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_2x2_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_2x2_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm + +.macro ZCALC_2x1 PTR_A_REG,PTR_B_REG + ZCALC_2x1_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm + +.macro ZCALC_2x1_4 PTR_A_REG,PTR_B_REG + ZCALC_2x1_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_2x1_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_2x1_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_2x1_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm + + +.macro ZCALC_1x2_4 PTR_A_REG,PTR_B_REG + ZCALC_1x2_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_1x2_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_1x2_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_1x2_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm + +.macro ZCALC_1x2 PTR_A_REG,PTR_B_REG + ZCALC_1x2_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm + +.macro ZCALC_1x1_4 PTR_A_REG,PTR_B_REG + ZCALC_1x1_I \PTR_A_REG,\PTR_B_REG,0,0 + ZCALC_1x1_I \PTR_A_REG,\PTR_B_REG,1,0 + ZCALC_1x1_I \PTR_A_REG,\PTR_B_REG,2,0 + ZCALC_1x1_I \PTR_A_REG,\PTR_B_REG,3,1 +.endm + +.macro ZCALC_1x1 PTR_A_REG,PTR_B_REG + ZCALC_1x1_I \PTR_A_REG,\PTR_B_REG,0,1 +.endm + + + +/*****************************STORE RESULTS************************************/ +.macro CalcMultAlpha_4x1 vRealResult1, vImageResult1, vRealResult2, vImageResult2, vReal1, vImage1, vReal2, vImage2, vecRealB, vecImageB + #if defined (TRMMKERNEL) + vfmdb \vRealResult1, \vImage1, \vecImageB + vfmdb \vImageResult1, \vReal1, \vecImageB + vfmdb \vRealResult2, \vImage2, \vecImageB + vfmdb \vImageResult2, \vReal2, \vecImageB + #else + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 + vfmsdb \vRealResult2, \vImage2, \vecImageB, \vRealResult2 + vfmadb \vImageResult2, \vReal2, \vecImageB, \vImageResult2 +#endif + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 + vfmsdb \vRealResult2, \vReal2, \vecRealB, \vRealResult2 + vfmadb \vImageResult2, \vImage2, \vecRealB, \vImageResult2 + +.endm + +.macro CalcMultAlpha_2x1 vRealResult1, vImageResult1, vReal1, vImage1, vecRealB, vecImageB + #if defined (TRMMKERNEL) + vfmdb \vRealResult1, \vImage1, \vecImageB + vfmdb \vImageResult1, \vReal1, \vecImageB +#else + vfmsdb \vRealResult1, \vImage1, \vecImageB, \vRealResult1 + vfmadb \vImageResult1, \vReal1, \vecImageB, \vImageResult1 +#endif + vfmsdb \vRealResult1, \vReal1, \vecRealB, \vRealResult1 + vfmadb \vImageResult1, \vImage1, \vecRealB, \vImageResult1 +.endm + +.macro CalcMultAlpha_1x1 RealResult1, ImageResult1, Real1, Image1, RealB, ImageB + + msebr \RealResult1, \Image1, \ImageB + maebr \ImageResult1, \Real1, \ImageB + msebr \RealResult1, \Real1, \RealB + maebr \ImageResult1, \Image1, \RealB +.endm + +.macro ZSTORE_4x4 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL ,LC1,LC2 + #if !defined(TRMMKERNEL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v3, 8(\CIJ_REG),2 + vlef %v4, 12(\CIJ_REG),2 + vlef %v5, 16(\CIJ_REG),0 + vlef %v6, 20(\CIJ_REG),0 + vlef %v5, 24(\CIJ_REG),2 + vlef %v6, 28(\CIJ_REG),2 + vldeb %v3,%v3 + vldeb %v4,%v4 + vldeb %v5,%v5 + vldeb %v6,%v6 +#endif + la \LC1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) + CalcMultAlpha_4x1 %v3,%v4,%v5,%v6,%v16,%v17,%v18,%v19,\ALPHA_VECREG,\ALPHA_VECI + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vledb %v5, %v5,0,0 + vledb %v6, %v6,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v3, 8(\CIJ_REG),2 + vstef %v4, 12(\CIJ_REG),2 + vstef %v5, 16(\CIJ_REG),0 + vstef %v6, 20(\CIJ_REG),0 + vstef %v5, 24(\CIJ_REG),2 + vstef %v6, 28(\CIJ_REG),2 + + la \LC2,0(\LC1,\LDC_BYTE_ORIGINAL ) + + #if !defined(TRMMKERNEL) + vlef %v16, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v17, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v16, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v17, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v18, 16(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v19, 20(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v18, 24(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v19, 28(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vldeb %v16,%v16 + vldeb %v17,%v17 + vldeb %v18,%v18 + vldeb %v19,%v19 +#endif + CalcMultAlpha_4x1 %v16,%v17,%v18,%v19,%v20,%v21,%v22,%v23,\ALPHA_VECREG,\ALPHA_VECI + vledb %v16, %v16,0,0 + vledb %v17, %v17,0,0 + vledb %v18, %v18,0,0 + vledb %v19, %v19,0,0 + vstef %v16, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v17, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v16, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v17, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v18, 16(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v19, 20(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v18, 24(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v19, 28(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + +#if !defined(TRMMKERNEL) + vlef %v3, 0(\CIJ_REG, \LC1),0 + vlef %v4, 4(\CIJ_REG, \LC1),0 + vlef %v3, 8(\CIJ_REG, \LC1),2 + vlef %v4, 12(\CIJ_REG, \LC1),2 + vlef %v5, 16(\CIJ_REG, \LC1),0 + vlef %v6, 20(\CIJ_REG, \LC1),0 + vlef %v5, 24(\CIJ_REG, \LC1),2 + vlef %v6, 28(\CIJ_REG, \LC1),2 + vldeb %v3,%v3 + vldeb %v4,%v4 + vldeb %v5,%v5 + vldeb %v6,%v6 +#endif + CalcMultAlpha_4x1 %v3,%v4,%v5,%v6,%v24,%v25,%v26,%v27,\ALPHA_VECREG,\ALPHA_VECI + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vledb %v5, %v5,0,0 + vledb %v6, %v6,0,0 + vstef %v3, 0(\CIJ_REG,\LC1),0 + vstef %v4, 4(\CIJ_REG,\LC1),0 + vstef %v3, 8(\CIJ_REG,\LC1),2 + vstef %v4, 12(\CIJ_REG,\LC1),2 + vstef %v5, 16(\CIJ_REG,\LC1),0 + vstef %v6, 20(\CIJ_REG,\LC1),0 + vstef %v5, 24(\CIJ_REG,\LC1),2 + vstef %v6, 28(\CIJ_REG,\LC1),2 + + #if !defined(TRMMKERNEL) + vlef %v16, 0(\CIJ_REG,\LC2),0 + vlef %v17, 4(\CIJ_REG,\LC2),0 + vlef %v16, 8(\CIJ_REG,\LC2),2 + vlef %v17, 12(\CIJ_REG,\LC2),2 + vlef %v18, 16(\CIJ_REG,\LC2),0 + vlef %v19, 20(\CIJ_REG,\LC2),0 + vlef %v18, 24(\CIJ_REG,\LC2),2 + vlef %v19, 28(\CIJ_REG,\LC2),2 + vldeb %v16,%v16 + vldeb %v17,%v17 + vldeb %v18,%v18 + vldeb %v19,%v19 +#endif + CalcMultAlpha_4x1 %v16,%v17,%v18,%v19,%v28,%v29,%v30,%v31,\ALPHA_VECREG,\ALPHA_VECI + vledb %v16, %v16,0,0 + vledb %v17, %v17,0,0 + vledb %v18, %v18,0,0 + vledb %v19, %v19,0,0 + vstef %v16, 0(\CIJ_REG,\LC2),0 + vstef %v17, 4(\CIJ_REG,\LC2),0 + vstef %v16, 8(\CIJ_REG,\LC2),2 + vstef %v17, 12(\CIJ_REG,\LC2),2 + vstef %v18, 16(\CIJ_REG,\LC2),0 + vstef %v19, 20(\CIJ_REG,\LC2),0 + vstef %v18, 24(\CIJ_REG,\LC2),2 + vstef %v19, 28(\CIJ_REG,\LC2),2 + + la \CIJ_REG,32(\CIJ_REG) +.endm + +.macro ZSTORE_4x2 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL + #if !defined(TRMMKERNEL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v3, 8(\CIJ_REG),2 + vlef %v4, 12(\CIJ_REG),2 + vlef %v5, 16(\CIJ_REG),0 + vlef %v6, 20(\CIJ_REG),0 + vlef %v5, 24(\CIJ_REG),2 + vlef %v6, 28(\CIJ_REG),2 + vldeb %v3,%v3 + vldeb %v4,%v4 + vldeb %v5,%v5 + vldeb %v6,%v6 +#endif + CalcMultAlpha_4x1 %v3,%v4,%v5,%v6,%v16,%v17,%v18,%v19,\ALPHA_VECREG,\ALPHA_VECI + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vledb %v5, %v5,0,0 + vledb %v6, %v6,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v3, 8(\CIJ_REG),2 + vstef %v4, 12(\CIJ_REG),2 + vstef %v5, 16(\CIJ_REG),0 + vstef %v6, 20(\CIJ_REG),0 + vstef %v5, 24(\CIJ_REG),2 + vstef %v6, 28(\CIJ_REG),2 + + #if !defined(TRMMKERNEL) + vlef %v16, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v17, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v16, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v17, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v18, 16(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v19, 20(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v18, 24(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v19, 28(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vldeb %v16,%v16 + vldeb %v17,%v17 + vldeb %v18,%v18 + vldeb %v19,%v19 +#endif + CalcMultAlpha_4x1 %v16,%v17,%v18,%v19,%v20,%v21,%v22,%v23,\ALPHA_VECREG,\ALPHA_VECI + vledb %v16, %v16,0,0 + vledb %v17, %v17,0,0 + vledb %v18, %v18,0,0 + vledb %v19, %v19,0,0 + vstef %v16, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v17, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v16, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v17, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v18, 16(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v19, 20(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v18, 24(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v19, 28(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + + la \CIJ_REG,32(\CIJ_REG) +.endm +.macro ZSTORE_4x1 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL + #if !defined(TRMMKERNEL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v3, 8(\CIJ_REG),2 + vlef %v4, 12(\CIJ_REG),2 + vlef %v5, 16(\CIJ_REG),0 + vlef %v6, 20(\CIJ_REG),0 + vlef %v5, 24(\CIJ_REG),2 + vlef %v6, 28(\CIJ_REG),2 + vldeb %v3,%v3 + vldeb %v4,%v4 + vldeb %v5,%v5 + vldeb %v6,%v6 +#endif + CalcMultAlpha_4x1 %v3,%v4,%v5,%v6,%v16,%v17,%v18,%v19,\ALPHA_VECREG,\ALPHA_VECI + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vledb %v5, %v5,0,0 + vledb %v6, %v6,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v3, 8(\CIJ_REG),2 + vstef %v4, 12(\CIJ_REG),2 + vstef %v5, 16(\CIJ_REG),0 + vstef %v6, 20(\CIJ_REG),0 + vstef %v5, 24(\CIJ_REG),2 + vstef %v6, 28(\CIJ_REG),2 + la \CIJ_REG,32(\CIJ_REG) +.endm + +.macro ZSTORE_1x4 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL,LC1,LC2 + #if !defined(TRMMKERNEL) + la \LC1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v3, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v4, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + la \LC2,0(\LC1,\LDC_BYTE_ORIGINAL ) + vlef %v5, 0(\CIJ_REG,\LC1),0 + vlef %v6, 4(\CIJ_REG,\LC1),0 + vlef %v5, 0(\CIJ_REG,\LC2),2 + vlef %v6, 4(\CIJ_REG,\LC2),2 + vldeb %v3,%v3 + vldeb %v4,%v4 + vldeb %v5,%v5 + vldeb %v6,%v6 +#else + la \LC1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) +#endif + CalcMultAlpha_4x1 %v3,%v4,%v5,%v6,%v16,%v17,%v18,%v19,\ALPHA_VECREG,\ALPHA_VECI +#if defined(TRMMKERNEL) + la \LC2,0(\LC1,\LDC_BYTE_ORIGINAL ) +#endif + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vledb %v5, %v5,0,0 + vledb %v6, %v6,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v3, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v4, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v5, 0(\CIJ_REG,\LC1),0 + vstef %v6, 4(\CIJ_REG,\LC1),0 + vstef %v5, 0(\CIJ_REG,\LC2),2 + vstef %v6, 4(\CIJ_REG,\LC2),2 + la \CIJ_REG,8(\CIJ_REG) +.endm +.macro ZSTORE_2x4 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL,LC1,LC2 + #if !defined(TRMMKERNEL) + la \LC1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v24, 8(\CIJ_REG),0 + vlef %v25, 12(\CIJ_REG),0 + vlef %v3, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v4, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v24, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v25, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + la \LC2,0(\LC1,\LDC_BYTE_ORIGINAL ) + vlef %v5, 0(\CIJ_REG,\LC1),0 + vlef %v6, 4(\CIJ_REG,\LC1),0 + vlef %v26, 8(\CIJ_REG,\LC1),0 + vlef %v27, 12(\CIJ_REG,\LC1),0 + vlef %v5, 0(\CIJ_REG,\LC2),2 + vlef %v6, 4(\CIJ_REG,\LC2),2 + vlef %v26, 8(\CIJ_REG,\LC2),2 + vlef %v27, 12(\CIJ_REG,\LC2),2 + + vldeb %v3,%v3 + vldeb %v4,%v4 + vldeb %v5,%v5 + vldeb %v6,%v6 + vldeb %v24,%v24 + vldeb %v25,%v25 + vldeb %v26,%v26 + vldeb %v27,%v27 +#else + la \LC1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) +#endif + CalcMultAlpha_4x1 %v3,%v4,%v5,%v6,%v16,%v17,%v18,%v19,\ALPHA_VECREG,\ALPHA_VECI + CalcMultAlpha_4x1 %v24,%v25,%v26,%v27,%v20,%v21,%v22,%v23,\ALPHA_VECREG,\ALPHA_VECI +#if defined(TRMMKERNEL) + la \LC2,0(\LC1,\LDC_BYTE_ORIGINAL ) +#endif + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vledb %v5, %v5,0,0 + vledb %v6, %v6,0,0 + vledb %v24, %v24,0,0 + vledb %v25, %v25,0,0 + vledb %v26, %v26,0,0 + vledb %v27, %v27,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v24, 8(\CIJ_REG),0 + vstef %v25, 12(\CIJ_REG),0 + vstef %v3, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v4, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v24, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v25, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v5, 0(\CIJ_REG,\LC1),0 + vstef %v6, 4(\CIJ_REG,\LC1),0 + vstef %v26, 8(\CIJ_REG,\LC1),0 + vstef %v27, 12(\CIJ_REG,\LC1),0 + vstef %v5, 0(\CIJ_REG,\LC2),2 + vstef %v6, 4(\CIJ_REG,\LC2),2 + vstef %v26, 8(\CIJ_REG,\LC2),2 + vstef %v27, 12(\CIJ_REG,\LC2),2 + + la \CIJ_REG,16(\CIJ_REG) + +.endm + +.macro ZSTORE_2x2 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL +#if !defined(TRMMKERNEL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v3, 8(\CIJ_REG),2 + vlef %v4, 12(\CIJ_REG),2 + vlef %v5, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v6, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vlef %v5, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v6, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vldeb %v3,%v3 + vldeb %v4,%v4 + vldeb %v5,%v5 + vldeb %v6,%v6 +#endif + CalcMultAlpha_2x1 %v3,%v4, %v16,%v17,\ALPHA_VECREG,\ALPHA_VECI + CalcMultAlpha_2x1 %v5,%v6, %v20,%v21 ,\ALPHA_VECREG,\ALPHA_VECI + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vledb %v5, %v5,0,0 + vledb %v6, %v6,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v3, 8(\CIJ_REG),2 + vstef %v4, 12(\CIJ_REG),2 + vstef %v5, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v6, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),0 + vstef %v5, 8(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v6, 12(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + la \CIJ_REG,16(\CIJ_REG) +.endm + +.macro ZSTORE_2x1 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL +#if !defined(TRMMKERNEL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v3, 8(\CIJ_REG),2 + vlef %v4, 12(\CIJ_REG),2 + vldeb %v3,%v3 + vldeb %v4,%v4 +#endif + CalcMultAlpha_2x1 %v3,%v4, %v16,%v17,\ALPHA_VECREG,\ALPHA_VECI + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v3, 8(\CIJ_REG),2 + vstef %v4, 12(\CIJ_REG),2 + la \CIJ_REG,16(\CIJ_REG) +.endm + +.macro ZSTORE_1x2 ALPHA_VECREG,ALPHA_VECI,CIJ_REG , LDC_BYTE_ORIGINAL + #if !defined(TRMMKERNEL) + vlef %v3, 0(\CIJ_REG),0 + vlef %v4, 4(\CIJ_REG),0 + vlef %v3, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vlef %v4, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vldeb %v3,%v3 + vldeb %v4,%v4 + +#endif + CalcMultAlpha_2x1 %v3,%v4, %v16,%v17,\ALPHA_VECREG,\ALPHA_VECI + vledb %v3, %v3,0,0 + vledb %v4, %v4,0,0 + vstef %v3, 0(\CIJ_REG),0 + vstef %v4, 4(\CIJ_REG),0 + vstef %v3, 0(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + vstef %v4, 4(\CIJ_REG,\LDC_BYTE_ORIGINAL),2 + la \CIJ_REG,8(\CIJ_REG) +.endm + +.macro ZSTORE_1x1 ALPHA_RR,ALPHA_RI ,CIJ_REG +#if defined (TRMMKERNEL) + lzer %f1 + lzer %f3 +#else + le %f1 , 0(\CIJ_REG) + le %f3 , 4(\CIJ_REG ) +#endif + ledbr %f4,\ALPHA_RR + ledbr %f5,\ALPHA_RI + CalcMultAlpha_1x1 %f1,%f3, %f6,%f7,%f4,%f5 + ste %f1,0(\CIJ_REG) + ste %f3,4(\CIJ_REG) + la \CIJ_REG,8(\CIJ_REG) +.endm + +/****************************TRMM POINTER REFRESH MACROSES*************************/ + +.macro RefreshPointers PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B + #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + /* ptrbb = bb;*/ + lgr \PTR_B,\B_VAL /*refresh BPOINT*/ + + #else + /* ptrba =ptrba+ off*C_A; + ptrbb = bb + off*C_B;*/ +.if \C_B==4 + .if \C_A==4 + sllg \PTR_B, \OFF_VAL,5 + agr \PTR_A,\PTR_B /*ptrba+off*4**/ + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + .elseif \C_A==2 + sllg \PTR_B, \OFF_VAL,4 + la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*2**/ + agr \PTR_B, \PTR_B + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + + .elseif \C_A==1 + sllg \PTR_B, \OFF_VAL,3 + agr \PTR_A,\PTR_B /*ptrba+off*4**/ + sllg \PTR_B, \OFF_VAL,5 + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + .endif + +.elseif \C_B==2 + .if \C_A==4 + sllg \PTR_B, \OFF_VAL,4 + la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*2**/ + agr \PTR_A,\PTR_B /*ptrba+off*2**/ + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + .elseif \C_A==2 + sllg \PTR_B, \OFF_VAL,4 + agr \PTR_A,\PTR_B /*ptrba+off*2**/ + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + .elseif \C_A==1 + sllg \PTR_B, \OFF_VAL,3 + la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*1**/ + agr \PTR_B,\PTR_B /* off+off**/ + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + .endif + +.elseif \C_B==1 + .if \C_A==4 + sllg \PTR_B, \OFF_VAL,5 + agr \PTR_A,\PTR_B /*ptrba+off*4**/ + sllg \PTR_B, \OFF_VAL,3 + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + .elseif \C_A==2 + sllg \PTR_B, \OFF_VAL,3 + la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*1**/ + agr \PTR_A,\PTR_B /*ptrba+off*1**/ + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + + .elseif \C_A==1 + sllg \PTR_B, \OFF_VAL,3 + agr \PTR_A,\PTR_B /*ptrba+off*1**/ + la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ + .endif +.endif + + #endif +.endm + +/**/ +.macro RefreshTempBk TEMP_VAL,BK_VAL,OFF_VAL,INCR_A,INCR_B + #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) + /* temp = bk-off;*/ + sgrk \TEMP_VAL,\BK_VAL,\OFF_VAL + + #elif defined(LEFT) + /* temp = off+INCR_A; // number of values in A */ + la \TEMP_VAL,\INCR_A(\OFF_VAL) + #else + /* temp = off+INCR_B // number of values in B*/ + la \TEMP_VAL,\INCR_B(\OFF_VAL) + #endif + +.endm + +.macro RefreshPointersAndOFF TEMP_VAL,BK_VAL,OFF_VAL,PTR_A,C_A,C_B + + #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) + /*temp = bk - off;*/ + sgrk \TEMP_VAL,\BK_VAL,\OFF_VAL + #ifdef LEFT + /*temp -= 8; // number of values in A*/ + lay \TEMP_VAL,-\C_A(\TEMP_VAL) + #else + /*temp -= 4; // number of values in B*/ + lay \TEMP_VAL,-\C_B(\TEMP_VAL) + #endif + /*ptrba += temp*C_A; + ptrbb += temp*C_B;*/ + + .if \C_A==4 + sllg \TEMP_VAL, \TEMP_VAL,5 /*temp*4*/ + .elseif \C_A==2 + sllg \TEMP_VAL, \TEMP_VAL,4 /*temp*2*/ + .elseif \C_A==1 + sllg \TEMP_VAL, \TEMP_VAL,3 /*temp*1*/ + .endif + la \PTR_A,0(\PTR_A,\TEMP_VAL) /*ptrba+temp*C_A*/ + #endif + + #ifdef LEFT + /*off += \c_A; // number of values in A*/ + aghi \OFF_VAL,\C_A + #endif +.endm + diff --git a/kernel/zarch/dtrmm8x4V.S b/kernel/zarch/ctrmm4x4V.S similarity index 65% rename from kernel/zarch/dtrmm8x4V.S rename to kernel/zarch/ctrmm4x4V.S index 790c798..c0e4df1 100644 --- a/kernel/zarch/dtrmm8x4V.S +++ b/kernel/zarch/ctrmm4x4V.S @@ -26,10 +26,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ /************************************************************************************** -* 2017/03/05 AbdelRauf (quickwritereader@gmail.com) -* BLASTEST : passed -* CTEST : passed -* TEST : passed +* 2017/03/12 AbdelRauf (quickwritereader@gmail.com) +* BLASTEST : passed +* CTEST : passed +* TEST : passed **************************************************************************************/ /*********************************************************************/ @@ -73,14 +73,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ASSEMBLER #include "common.h" +/* -/* -#BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc - ##bm=r2,bn=r3, bk=r4, alpha=f0,ba=r5,bb=r6,stack[160] ,ldc=stack[168] -offset=stack[176] +BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alphar,FLOAT alphai,FLOAT* ba,FLOAT* bb, + FLOAT* C,BLASLONG ldc, BLASLONG offset) + ##bm=r2,bn=r3, bk=r4, alpha=f0,aplhai=f2, ba=r5,bb=r6,stack[160] ,ldc=stack[168] +offset=stack[176] + **********************************************************************************************/ -/*Note: r0 can not be used as address disp register */ +/*Note: r0 can not be used as address disp register */ #define BM %r2 #define BM_CUR %r0 @@ -89,7 +91,9 @@ offset=stack[176] #define BK %r4 #define LDC_BYTE %r8 #define ALPHA %f0 +#define ALPHA_I %f2 #define ALPHA_VECT %v0 +#define ALPHA_VECT_I %v2 #define LOCAL_VAR1 %r9 #define LOCAL_VAR2 %r1 #define LOCAL_VAR3 %r11 @@ -99,34 +103,48 @@ offset=stack[176] #define CIJ_LOCAL %r12 #define OFF %r13 #define OFFSET %f8 -#define ALIGN_4 .align 16 -#define ALIGN_2 .align 8 +#define ALIGN_4 .align 32 +#define ALIGN_2 .align 16 #define PREFETCH_INS 1 /**************************Include kernel helper macrosses**********************************/ -#include "dkernelMacros.S" +#include "ckernelMacrosV.S" -/***********************************DGEMM***********************************************************/ +/***********************************CGEMM**4x4*******************************************************/ PROLOGUE #if defined(TRMMKERNEL) - std OFFSET,40(%r15) - stmg %r6,%r13,48(%r15) + std OFFSET ,40(%r15) + stmg %r6,%r13,48(%r15) #else - stmg %r6,%r12,48(%r15) + stmg %r6,%r12,48(%r15) #endif +std %f9, 128(%r15) +std %f10,136(%r15) +std %f11,144(%r15) +std %f12,152(%r15) + lg CIJ, 160(%r15) lg LOCAL_VAR1, 168(%r15) #if defined(TRMMKERNEL) -lg OFF,176(%r15) -ldgr OFFSET ,OFF + lg OFF,176(%r15) + ldgr OFFSET ,OFF #endif srlg BN_CUR,BN,2 +#if defined(RR) || defined(RC) || defined(CR) || defined(CC) + lcdbr ALPHA_I,ALPHA_I + lcdbr ALPHA ,ALPHA +#endif + vrepg ALPHA_VECT,ALPHA_VECT,0 /*replicate alpha which in f0*/ -sllg LDC_BYTE, LOCAL_VAR1,3 /*calculate lcd stride with bytes double=8 x<<3 */ +sllg LDC_BYTE, LOCAL_VAR1,3 /*calculate lcd stride with complex=8 x<<4 */ +vrepg ALPHA_VECT_I,ALPHA_VECT_I,0 /*replicate alpha which in f0*/ + +vldeb ALPHA_VECT,ALPHA_VECT +vldeb ALPHA_VECT_I,ALPHA_VECT_I #if defined(TRMMKERNEL) && !defined(LEFT) /*off = -offset;*/ lgdr LOCAL_VAR1,OFFSET @@ -137,83 +155,19 @@ cijle BN_CUR,0,.LX2 ALIGN_4 .LX4_BN: #if defined(PREFETCH_INS) - pfd 1, 0(A) - pfd 1, 256(A) - pfd 1, 0(B) - pfd 1, 256(B) + pfd 1, 0(A) + pfd 1, 0(B) #endif #if defined(TRMMKERNEL) && defined(LEFT) /*off = offset;*/ lgdr OFF,OFFSET #endif -srlg BM_CUR,BM,3 +srlg BM_CUR,BM,2 lgr LOCAL_VAR3,A lgr CIJ_LOCAL,CIJ -cijle BM_CUR,0,.L4x4 -ALIGN_4 -.L8x4_BM: /*BM_CUR LOOP */ - -#if defined(TRMMKERNEL) - - /* RefreshPointers PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B */ - RefreshPointers LOCAL_VAR3,LOCAL_VAR2,OFF,B,8,4 - - RefreshTempBk LOCAL_VAR1,BK,OFF,8,4 - srl LOCAL_VAR1,2 - -#else - srlg LOCAL_VAR1,BK,2 /*refresh BK*/ - lgr LOCAL_VAR2,B /*refresh BPOINT*/ -#endif - -ZERO_CVEC_8x4 -cijle LOCAL_VAR1,0,.L8x4_mod - +cijle BM_CUR,0,.L2x4 ALIGN_4 -.L8x4_4_BK: /*BK_CUR LOOP */ -#if defined(PREFETCH_INS) - pfd 1, 512(LOCAL_VAR3) -#endif - CALC_8x4_4 LOCAL_VAR3,LOCAL_VAR2 -#if defined(PREFETCH_INS) - pfd 1, 512(LOCAL_VAR2) -#endif -brctg LOCAL_VAR1,.L8x4_4_BK - -ALIGN_4 -.L8x4_mod: -#if defined(TRMMKERNEL) - RefreshTempBk LOCAL_VAR1,BK,OFF,8,4 - nill LOCAL_VAR1,3 -#else -la LOCAL_VAR1,3(0,0) -NGR LOCAL_VAR1,BK /*refresh BK*/ -#endif -jz .L8x4_BK_Store - -ALIGN_4 -.L8x4_BK: /*BK_CUR LOOP */ - CALC_8x4 LOCAL_VAR3,LOCAL_VAR2 -brctg LOCAL_VAR1,.L8x4_BK - -ALIGN_4 -.L8x4_BK_Store: -/*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_8x4 ALPHA_VECT,CIJ_LOCAL, LDC_BYTE , LOCAL_VAR1 ,LOCAL_VAR2 -#if defined(TRMMKERNEL) - /*RefreshPointersAndOFF TEMP_VAL,BK_VAL,OFF_VAL,L_VAR,PTR_A,C_A*/ - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,8,4 -#endif -brctg BM_CUR,.L8x4_BM - -ALIGN_4 -.L4x4: - -tmll BM,4 -jz .L2x4 - -ALIGN_4 .L4x4_BM: /*BM start*/ #if defined(TRMMKERNEL) @@ -226,12 +180,16 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_4x4 +ZERO_ZCVEC_4x4 cijle LOCAL_VAR1,0,.L4x4_mod ALIGN_4 -.L4x4_4_BK: /*BK_CUR LOOP */ - CALC_4x4_4 LOCAL_VAR3,LOCAL_VAR2 +.L4x4_4_BK: /*BK_CUR LOOP */ + ZCALC_4x4_4 LOCAL_VAR3,LOCAL_VAR2 +#if defined(PREFETCH_INS) + pfd 1, 128(LOCAL_VAR3) /*256-128*/ + pfd 1, 128(LOCAL_VAR2 ) +#endif brctg LOCAL_VAR1,.L4x4_4_BK ALIGN_4 @@ -247,16 +205,19 @@ jz .L4x4_BK_Store ALIGN_4 .L4x4_BK: /*BK_CUR LOOP */ - CALC_4x4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_4x4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L4x4_BK ALIGN_4 .L4x4_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_4x4 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE , LOCAL_VAR1 ,LOCAL_VAR2 +ZSTORE_4x4 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE,LOCAL_VAR1,LOCAL_VAR2 #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,4,4 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,4,4 #endif + +brctg BM_CUR,.L4x4_BM + ALIGN_2 .L2x4: @@ -268,8 +229,7 @@ ALIGN_4 #if defined(TRMMKERNEL) /* RefreshPointers PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B */ - RefreshPointers LOCAL_VAR3,LOCAL_VAR2,OFF,B,2,4 - + RefreshPointers LOCAL_VAR3,LOCAL_VAR2,OFF,B,2,4 RefreshTempBk LOCAL_VAR1,BK,OFF,2,4 srl LOCAL_VAR1,2 @@ -277,12 +237,15 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_2x4 +ZERO_ZCVEC_2x4 cijle LOCAL_VAR1,0,.L2x4_mod ALIGN_4 .L2x4_4_BK: /*BK_CUR LOOP */ - CALC_2x4_4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_2x4_4 LOCAL_VAR3,LOCAL_VAR2 +#if defined(PREFETCH_INS) + pfd 1, 128(LOCAL_VAR2) +#endif brctg LOCAL_VAR1,.L2x4_4_BK ALIGN_4 @@ -298,15 +261,15 @@ jz .L2x4_BK_Store ALIGN_4 .L2x4_BK: /*BK_CUR LOOP */ - CALC_2x4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_2x4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L2x4_BK ALIGN_4 .L2x4_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_2x4 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE , LOCAL_VAR1 ,LOCAL_VAR2 +ZSTORE_2x4 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE ,LOCAL_VAR1,LOCAL_VAR2 #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,2,4 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,2,4 #endif ALIGN_4 @@ -328,12 +291,12 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_1x4 +ZERO_ZCVEC_1x4 cijle LOCAL_VAR1,0,.L1x4_mod ALIGN_4 -.L1x4_4_BK: /*BK_CUR LOOP */ - CALC_1x4_4 LOCAL_VAR3,LOCAL_VAR2 +.L1x4_4_BK: /*BK_CUR LOOP */ + ZCALC_1x4_4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L1x4_4_BK ALIGN_4 @@ -349,28 +312,28 @@ jz .L1x4_BK_Store ALIGN_4 .L1x4_BK: /*BK_CUR LOOP */ - CALC_1x4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_1x4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L1x4_BK ALIGN_4 .L1x4_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_1x4 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE , LOCAL_VAR1 ,LOCAL_VAR2 +ZSTORE_1x4 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE,LOCAL_VAR1,LOCAL_VAR2 #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,1,4 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,1,4 #endif ALIGN_2 .Lx4_INNER_END: /*add LDC_BYTE_COPY to new*/ -sllg LOCAL_VAR1,LDC_BYTE,2 /*op*4 */ +sllg LOCAL_VAR1,LDC_BYTE,2 /*multiply*4 */ #if defined(TRMMKERNEL) && !defined(LEFT) aghi OFF,4 #endif -sllg LOCAL_VAR2,BK,5 /*op*4*sizeof(double) =op*32* 2**5 */ +sllg LOCAL_VAR2,BK,5 /*multiply*4*sizeof(complex) =multiply*4*8* 2**5 */ la CIJ,0(CIJ,LOCAL_VAR1) /*refresh CIJ=CIJ+LDC_BYTE*4*/ -la B,0(B,LOCAL_VAR2) /*refresh B=B+Bk*4*sizeof(double) */ +la B,0(B,LOCAL_VAR2) /*refresh B=B+Bk*4*sizeof(complex) */ brctg BN_CUR,.LX4_BN @@ -388,68 +351,10 @@ ALIGN_4 lgdr OFF,OFFSET #endif -srlg BM_CUR,BM,3 +srlg BM_CUR,BM,2 lgr LOCAL_VAR3,A lgr CIJ_LOCAL,CIJ -cijle BM_CUR,0,.L4x2 - - -ALIGN_4 -.L8x2_BM: /*BM_CUR LOOP */ -#if defined(TRMMKERNEL) - - /* RefreshPointers PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B */ - RefreshPointers LOCAL_VAR3,LOCAL_VAR2,OFF,B,8,2 - RefreshTempBk LOCAL_VAR1,BK,OFF,8,2 - srl LOCAL_VAR1,2 - -#else -srlg LOCAL_VAR1,BK,2 /*refresh BK*/ -lgr LOCAL_VAR2,B /*refresh BPOINT*/ -#endif -ZERO_CVEC_8x2 -cijle LOCAL_VAR1,0,.L8x2_mod - -ALIGN_4 -.L8x2_4_BK: /*BK_CUR LOOP */ -#if defined(PREFETCH_INS) - pfd 1, 256(LOCAL_VAR3) - pfd 1,64(LOCAL_VAR2) -#endif - CALC_8x2_4 LOCAL_VAR3,LOCAL_VAR2 -brctg LOCAL_VAR1,.L8x2_4_BK - -ALIGN_4 -.L8x2_mod: -#if defined(TRMMKERNEL) - RefreshTempBk LOCAL_VAR1,BK,OFF,8,2 - nill LOCAL_VAR1,3 -#else -la LOCAL_VAR1,3(0,0) -NGR LOCAL_VAR1,BK /*refresh BK*/ -#endif -jz .L8x2_BK_Store - -ALIGN_4 -.L8x2_BK: /*BK_CUR LOOP */ - CALC_8x2 LOCAL_VAR3,LOCAL_VAR2 -brctg LOCAL_VAR1,.L8x2_BK - -ALIGN_4 -.L8x2_BK_Store: -/*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_8x2 ALPHA_VECT,CIJ_LOCAL, LDC_BYTE -#if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,8,2 -#endif -ALIGN_4 -brctg BM_CUR,.L8x2_BM - -ALIGN_2 -.L4x2: - -tmll BM,4 -jz .L2x2 +cijle BM_CUR,0,.L2x2 ALIGN_4 .L4x2_BM: /*BM start*/ @@ -460,15 +365,18 @@ ALIGN_4 srl LOCAL_VAR1,2 #else -srlg LOCAL_VAR1,BK,2 /*refresh BK*/ -lgr LOCAL_VAR2,B /*refresh BPOINT*/ + srlg LOCAL_VAR1,BK,2 /*refresh BK*/ + lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_4x2 +ZERO_ZCVEC_4x2 cijle LOCAL_VAR1,0,.L4x2_mod ALIGN_4 -.L4x2_4_BK: /*BK_CUR LOOP */ - CALC_4x2_4 LOCAL_VAR3,LOCAL_VAR2 +.L4x2_4_BK: /*BK_CUR LOOP */ + ZCALC_4x2_4 LOCAL_VAR3,LOCAL_VAR2 +#if defined(PREFETCH_INS) + pfd 1, 128(LOCAL_VAR3) +#endif brctg LOCAL_VAR1,.L4x2_4_BK ALIGN_4 @@ -484,16 +392,19 @@ jz .L4x2_BK_Store ALIGN_4 .L4x2_BK: /*BK_CUR LOOP */ - CALC_4x2 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_4x2 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L4x2_BK ALIGN_4 .L4x2_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_4x2 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE +ZSTORE_4x2 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,4,2 -#endif + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,4,2 +#endif +ALIGN_4 +brctg BM_CUR,.L4x2_BM + ALIGN_2 .L2x2: @@ -513,12 +424,16 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_2x2 +ZERO_ZCVEC_2x2 cijle LOCAL_VAR1,0,.L2x2_mod ALIGN_4 .L2x2_4_BK: /*BK_CUR LOOP */ - CALC_2x2_4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_2x2_4 LOCAL_VAR3,LOCAL_VAR2 +#if defined(PREFETCH_INS) + pfd 1, 256(LOCAL_VAR3) + pfd 1, 256(LOCAL_VAR2) +#endif brctg LOCAL_VAR1,.L2x2_4_BK ALIGN_4 @@ -534,15 +449,15 @@ jz .L2x2_BK_Store ALIGN_4 .L2x2_BK: /*BK_CUR LOOP */ - CALC_2x2 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_2x2 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L2x2_BK ALIGN_4 .L2x2_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_2x2 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE +ZSTORE_2x2 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,2,2 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,2,2 #endif ALIGN_2 @@ -563,12 +478,12 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_1x2 +ZERO_ZCVEC_1x2 cijle LOCAL_VAR1,0,.L1x2_mod ALIGN_4 .L1x2_4_BK: /*BK_CUR LOOP */ - CALC_1x2_4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_1x2_4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L1x2_4_BK ALIGN_4 @@ -584,26 +499,26 @@ jz .L1x2_BK_Store ALIGN_4 .L1x2_BK: /*BK_CUR LOOP */ - CALC_1x2 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_1x2 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L1x2_BK ALIGN_4 .L1x2_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_1x2 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE +ZSTORE_1x2 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,1,2 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,1,2 #endif ALIGN_2 .Lx2_INNER_END: /*add LDC_BYTE_COPY to new*/ -la LOCAL_VAR1,0(LDC_BYTE,LDC_BYTE) /*op*2 */ -sllg LOCAL_VAR2,BK,4 /*op*2*sizeof(double) =op*16* 2**4 */ -la CIJ,0(CIJ,LOCAL_VAR1) /*refresh CIJ=CIJ+LDC_BYTE*4*/ +la LOCAL_VAR1,0(LDC_BYTE,LDC_BYTE) /*multiply*2 */ +sllg LOCAL_VAR2,BK,4 /*multiply*2*sizeof(complex) =multiply*2*8 2^4 */ +la CIJ,0(CIJ,LOCAL_VAR1) /*refresh CIJ=CIJ+LDC_BYTE*2*/ #if defined(TRMMKERNEL) && !defined(LEFT) aghi OFF,2 #endif -la B,0(B,LOCAL_VAR2) /*refresh B=B+Bk*4*sizeof(double) */ +la B,0(B,LOCAL_VAR2) /*refresh B=B+Bk*2*sizeof(complex) */ @@ -621,69 +536,12 @@ ALIGN_4 /*off = offset;*/ lgdr OFF,OFFSET #endif -srlg BM_CUR,BM,3 +srlg BM_CUR,BM,2 lgr LOCAL_VAR3,A lgr CIJ_LOCAL,CIJ -cijle BM_CUR,0,.L4x1 - +cijle BM_CUR,0,.L2x1 ALIGN_4 -.L8x1_BM: /*BM_CUR LOOP */ -#if defined(TRMMKERNEL) - - /* RefreshPointers PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B */ - RefreshPointers LOCAL_VAR3,LOCAL_VAR2,OFF,B,8,1 - RefreshTempBk LOCAL_VAR1,BK,OFF,8,1 - srl LOCAL_VAR1,2 - -#else -srlg LOCAL_VAR1,BK,2 /*refresh BK*/ -lgr LOCAL_VAR2,B /*refresh BPOINT*/ -#endif -ZERO_CVEC_8x1 -cijle LOCAL_VAR1,0,.L8x1_mod - -ALIGN_4 -.L8x1_4_BK: /*BK_CUR LOOP */ -#if defined(PREFETCH_INS) - pfd 1, 256(LOCAL_VAR3) -#endif - CALC_8x1_4 LOCAL_VAR3,LOCAL_VAR2 -brctg LOCAL_VAR1,.L8x1_4_BK - -ALIGN_4 -.L8x1_mod: -#if defined(TRMMKERNEL) - RefreshTempBk LOCAL_VAR1,BK,OFF,8,1 - nill LOCAL_VAR1,3 -#else -la LOCAL_VAR1,3(0,0) -NGR LOCAL_VAR1,BK /*refresh BK*/ -#endif -jz .L8x1_BK_Store - -ALIGN_4 -.L8x1_BK: /*BK_CUR LOOP */ - CALC_8x1 LOCAL_VAR3,LOCAL_VAR2 -brctg LOCAL_VAR1,.L8x1_BK - -ALIGN_4 -.L8x1_BK_Store: -/*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_8x1 ALPHA_VECT,CIJ_LOCAL, LDC_BYTE - #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,8,1 -#endif -ALIGN_4 -brctg BM_CUR,.L8x1_BM - -ALIGN_2 -.L4x1: - -tmll BM,4 -jz .L2x1 - -ALIGN_4 .L4x1_BM: /*BM start*/ #if defined(TRMMKERNEL) @@ -696,12 +554,12 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_4x1 +ZERO_ZCVEC_4x1 cijle LOCAL_VAR1,0,.L4x1_mod ALIGN_4 -.L4x1_4_BK: /*BK_CUR LOOP */ - CALC_4x1_4 LOCAL_VAR3,LOCAL_VAR2 +.L4x1_4_BK: /*BK_CUR LOOP */ + ZCALC_4x1_4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L4x1_4_BK ALIGN_4 @@ -717,16 +575,19 @@ jz .L4x1_BK_Store ALIGN_4 .L4x1_BK: /*BK_CUR LOOP */ - CALC_4x1 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_4x1 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L4x1_BK ALIGN_4 .L4x1_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_4x1 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE +ZSTORE_4x1 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,4,1 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,4,1 #endif +ALIGN_4 +brctg BM_CUR , .L4x1_BM + ALIGN_2 .L2x1: @@ -745,12 +606,12 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_2x1 +ZERO_ZCVEC_2x1 cijle LOCAL_VAR1,0,.L2x1_mod ALIGN_4 .L2x1_4_BK: /*BK_CUR LOOP */ - CALC_2x1_4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_2x1_4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L2x1_4_BK ALIGN_4 @@ -766,15 +627,15 @@ jz .L2x1_BK_Store ALIGN_4 .L2x1_BK: /*BK_CUR LOOP */ - CALC_2x1 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_2x1 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L2x1_BK ALIGN_4 .L2x1_BK_Store: /*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_2x1 ALPHA_VECT ,CIJ_LOCAL, LDC_BYTE +ZSTORE_2x1 ALPHA_VECT,ALPHA_VECT_I ,CIJ_LOCAL, LDC_BYTE #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,2,1 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,2,1 #endif ALIGN_2 @@ -795,12 +656,12 @@ ALIGN_4 srlg LOCAL_VAR1,BK,2 /*refresh BK*/ lgr LOCAL_VAR2,B /*refresh BPOINT*/ #endif -ZERO_CVEC_1x1 +ZERO_ZCVEC_1x1 cijle LOCAL_VAR1,0,.L1x1_mod ALIGN_4 .L1x1_4_BK: /*BK_CUR LOOP */ - CALC_1x1_4 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_1x1_4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L1x1_4_BK ALIGN_4 @@ -816,36 +677,42 @@ jz .L1x1_BK_Store ALIGN_4 .L1x1_BK: /*BK_CUR LOOP */ - CALC_1x1 LOCAL_VAR3,LOCAL_VAR2 + ZCALC_1x1 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L1x1_BK ALIGN_4 .L1x1_BK_Store: -/*store C and use LDC_BYTE AND CIJ_COPY for mem storing*/ -STORE_1x1 ALPHA ,CIJ_LOCAL, LDC_BYTE +/*store C and use CIJ_COPY for mem storing*/ +ZSTORE_1x1 ALPHA,ALPHA_I ,CIJ_LOCAL #if defined(TRMMKERNEL) - RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR2,LOCAL_VAR3,1,1 + RefreshPointersAndOFF LOCAL_VAR1,BK,OFF,LOCAL_VAR3,1,1 #endif ALIGN_2 .Lx1_INNER_END: /*add LDC_BYTE_COPY to new*/ -sllg LOCAL_VAR2,BK,3 /*op*2*sizeof(double) =op*8* 2**3 */ +sllg LOCAL_VAR2,BK,3 /*multiply*1*sizeof(complex) =multiply*1*8* 2^3 */ la CIJ,0(CIJ,LDC_BYTE) /*refresh CIJ=CIJ+LDC_BYTE */ #if defined(TRMMKERNEL) && !defined(LEFT) aghi OFF,1 #endif -la B,0(B,LOCAL_VAR2) /*refresh B=B+Bk*1*sizeof(double) */ +la B,0(B,LOCAL_VAR2) /*refresh B=B+Bk*1*sizeof(complex) */ ALIGN_2 .L_FUNC_END: /*end*/ + + #if defined(TRMMKERNEL) - ld OFFSET,40(%r15) - lmg %r6,%r13,48(%r15) +ld OFFSET,40(%r15) +lmg %r6,%r13,48(%r15) #else - lmg %r6,%r12,48(%r15) +lmg %r6,%r12,48(%r15) #endif +ld %f9, 128(%r15) +ld %f10,136(%r15) +ld %f11,144(%r15) +ld %f12,152(%r15) br %r14 .end @@ -855,3 +722,12 @@ br %r14 + + + + + + + + + diff --git a/kernel/zarch/dkernelMacros.S b/kernel/zarch/dkernelMacros.S deleted file mode 100644 index c7b9ac5..0000000 --- a/kernel/zarch/dkernelMacros.S +++ /dev/null @@ -1,903 +0,0 @@ -/**********************************Zero Vectors**************************************************/ - -.macro ZERO_CVEC_8x4 - vzero %v16 - vzero %v17 - vzero %v18 - vzero %v19 - vzero %v20 - vzero %v21 - vzero %v22 - vzero %v23 - vzero %v24 - vzero %v25 - vzero %v26 - vzero %v27 - vzero %v28 - vzero %v29 - vzero %v30 - vzero %v31 -.endm - - -.macro ZERO_CVEC_8x2 - vzero %v16 - vzero %v17 - vzero %v18 - vzero %v19 - vzero %v20 - vzero %v21 - vzero %v22 - vzero %v23 - -.endm - -.macro ZERO_CVEC_8x1 - vzero %v16 - vzero %v17 - vzero %v18 - vzero %v19 -.endm - -.macro ZERO_CVEC_4x4 - vzero %v16 - vzero %v17 - vzero %v20 - vzero %v21 - vzero %v24 - vzero %v25 - vzero %v28 - vzero %v29 -.endm - -.macro ZERO_CVEC_4x2 - vzero %v16 - vzero %v17 - vzero %v20 - vzero %v21 - -.endm - -.macro ZERO_CVEC_4x1 - vzero %v16 - vzero %v17 -.endm - -.macro ZERO_CVEC_2x4 - vzero %v16 - vzero %v17 - vzero %v20 - vzero %v21 - -.endm - -.macro ZERO_CVEC_2x2 - vzero %v16 - vzero %v20 - -.endm - -.macro ZERO_CVEC_2x1 - vzero %v16 -.endm - -.macro ZERO_CVEC_1x4 - vzero %v16 - vzero %v17 -.endm - -.macro ZERO_CVEC_1x2 - vzero %v16 -.endm - -.macro ZERO_CVEC_1x1 - LZDR %f1 -.endm - - -/***********************************Helper Calculations*************************************/ -#define unit_size 8 -#define DISP(ind,stride,disp) (ind*stride+disp) -#define DISP8(ind,disp) (ind*unit_size*8+disp) -#define DISP4(ind,disp) (ind*unit_size*4+disp) -#define DISP2(ind,disp) (ind*unit_size*2+disp) -#define DISP1(ind,disp) (ind*unit_size+disp) -#define N8 (8*unit_size) -#define N4 (4*unit_size) -#define N2 (2*unit_size) -#define N1 (1*unit_size) - -.macro Calculate_8x4_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP4(\Index ,0)(\PTR_B_REG) - vlrepg %v1, DISP4(\Index ,8)(\PTR_B_REG) - vl %v2, DISP8(\Index , 0)(\PTR_A_REG) - vl %v3, DISP8(\Index ,16)(\PTR_A_REG) - vl %v4, DISP8(\Index ,32)(\PTR_A_REG) - vl %v5, DISP8(\Index ,48)(\PTR_A_REG) - vfmadb %v16,%v2,%v7,%v16 - vfmadb %v17,%v3,%v7,%v17 - vfmadb %v18,%v4,%v7,%v18 - vfmadb %v19,%v5,%v7,%v19 - vfmadb %v20,%v2,%v1,%v20 - vfmadb %v21,%v3,%v1,%v21 - vfmadb %v22,%v4,%v1,%v22 - vfmadb %v23,%v5,%v1,%v23 - vlrepg %v7, DISP4(\Index ,16)(\PTR_B_REG) - vlrepg %v1, DISP4(\Index ,24)(\PTR_B_REG) - .if \IsLast==1 - la \PTR_A_REG, DISP8(\Index ,64)(\PTR_A_REG) - .endif - vfmadb %v24,%v2,%v7,%v24 - vfmadb %v25,%v3,%v7,%v25 - vfmadb %v26,%v4,%v7,%v26 - vfmadb %v27,%v5,%v7,%v27 - vfmadb %v28,%v2,%v1,%v28 - vfmadb %v29,%v3,%v1,%v29 - vfmadb %v30,%v4,%v1,%v30 - vfmadb %v31,%v5,%v1,%v31 - .if \IsLast==1 - la \PTR_B_REG, DISP4(\Index ,32)(\PTR_B_REG) - .endif -.endm - -.macro Calculate_8x2_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP2(\Index ,0)(\PTR_B_REG) - vlrepg %v1, DISP2(\Index ,8)(\PTR_B_REG) - vl %v2, DISP8(\Index ,0)(\PTR_A_REG) - vl %v3, DISP8(\Index ,16)(\PTR_A_REG) - vl %v4, DISP8(\Index ,32)(\PTR_A_REG) - vl %v5, DISP8(\Index ,48)(\PTR_A_REG) - vfmadb %v16,%v2,%v7,%v16 - vfmadb %v17,%v3,%v7,%v17 - vfmadb %v18,%v4,%v7,%v18 - vfmadb %v19,%v5,%v7,%v19 - vfmadb %v20,%v2,%v1,%v20 - vfmadb %v21,%v3,%v1,%v21 - .if \IsLast==1 - la \PTR_A_REG, DISP8(\Index ,64)(\PTR_A_REG) - .endif - vfmadb %v22,%v4,%v1,%v22 - vfmadb %v23,%v5,%v1,%v23 - .if \IsLast==1 - la \PTR_B_REG, DISP2(\Index ,16)(\PTR_B_REG) - .endif -.endm - -.macro Calculate_8x1_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP1(\Index ,0)(\PTR_B_REG) - vl %v2, DISP8(\Index ,0)(\PTR_A_REG) - vl %v3, DISP8(\Index ,16)(\PTR_A_REG) - vl %v4, DISP8(\Index ,32)(\PTR_A_REG) - vl %v5, DISP8(\Index ,48)(\PTR_A_REG) - vfmadb %v16,%v2,%v7,%v16 - .if \IsLast==1 - la \PTR_B_REG, DISP1(\Index ,8)(\PTR_B_REG) - .endif - vfmadb %v17,%v3,%v7,%v17 - vfmadb %v18,%v4,%v7,%v18 - vfmadb %v19,%v5,%v7,%v19 - .if \IsLast==1 - la \PTR_A_REG, DISP8(\Index ,64)(\PTR_A_REG) - .endif -.endm - -.macro Calculate_4x4_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP4(\Index ,0)(\PTR_B_REG) - vlrepg %v1, DISP4(\Index ,8)(\PTR_B_REG) - vl %v2, DISP4(\Index ,0)(\PTR_A_REG) - vl %v3, DISP4(\Index ,16)(\PTR_A_REG) - vfmadb %v16,%v2,%v7,%v16 - vfmadb %v17,%v3,%v7,%v17 - vfmadb %v20,%v2,%v1,%v20 - vfmadb %v21,%v3,%v1,%v21 - vlrepg %v7, DISP4(\Index ,16)(\PTR_B_REG) - vlrepg %v1, DISP4(\Index ,24)(\PTR_B_REG) - .if \IsLast==1 - la \PTR_A_REG, DISP4(\Index ,32)(\PTR_A_REG) - .endif - vfmadb %v24,%v2,%v7,%v24 - vfmadb %v25,%v3,%v7,%v25 - vfmadb %v28,%v2,%v1,%v28 - vfmadb %v29,%v3,%v1,%v29 - .if \IsLast==1 - la \PTR_B_REG, DISP4(\Index ,32)(\PTR_B_REG) - .endif -.endm - -.macro Calculate_4x2_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP2(\Index ,0)(\PTR_B_REG) - vlrepg %v1, DISP2(\Index ,8)(\PTR_B_REG) - vl %v2, DISP4(\Index ,0)(\PTR_A_REG) - vl %v3, DISP4(\Index ,16)(\PTR_A_REG) - vfmadb %v16,%v2,%v7,%v16 - vfmadb %v17,%v3,%v7,%v17 - .if \IsLast==1 - la \PTR_B_REG, DISP2(\Index ,16)(\PTR_B_REG) - .endif - vfmadb %v20,%v2,%v1,%v20 - vfmadb %v21,%v3,%v1,%v21 - .if \IsLast==1 - la \PTR_A_REG, DISP4(\Index ,32)(\PTR_A_REG) - .endif -.endm - -.macro Calculate_4x1_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP1(\Index ,0)(\PTR_B_REG) - vl %v2, DISP4(\Index ,0)(\PTR_A_REG) - vl %v3, DISP4(\Index ,16)(\PTR_A_REG) - .if \IsLast==1 - la \PTR_B_REG, DISP1(\Index ,8)(\PTR_B_REG) - .endif - vfmadb %v16,%v2,%v7,%v16 - vfmadb %v17,%v3,%v7,%v17 - .if \IsLast==1 - la \PTR_A_REG, DISP4(\Index ,32)(\PTR_A_REG) - .endif -.endm - -.macro Calculate_2x2_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP2(\Index ,0)(\PTR_B_REG) - vlrepg %v1, DISP2(\Index ,8)(\PTR_B_REG) - vl %v2, DISP2(\Index ,0)(\PTR_A_REG) - vfmadb %v16,%v2,%v7,%v16 - .if \IsLast==1 - la \PTR_A_REG, DISP2(\Index ,16)(\PTR_A_REG) - .endif - vfmadb %v20,%v2,%v1,%v20 - .if \IsLast==1 - la \PTR_B_REG, DISP2(\Index ,16)(\PTR_B_REG) - .endif -.endm - - - -.macro Calculate_2x1_I PTR_A_REG,PTR_B_REG,Index,IsLast - vlrepg %v7, DISP1(\Index ,0)(\PTR_B_REG) - vl %v2, DISP2(\Index ,0)(\PTR_A_REG) - .if \IsLast==1 - la \PTR_B_REG, DISP1(\Index ,8)(\PTR_B_REG) - .endif - vfmadb %v16,%v2,%v7,%v16 - .if \IsLast==1 - la \PTR_A_REG, DISP2(\Index ,16)(\PTR_A_REG) - .endif -.endm - -.macro Calculate_1x1_I PTR_A_REG,PTR_B_REG,Index,IsLast - ld %f2,DISP1(\Index ,0)(\PTR_A_REG) /**a*/ - .if \IsLast==1 - la \PTR_A_REG,DISP1(\Index ,8)(\PTR_A_REG) - .endif - madb %f1,%f2,DISP1(\Index ,0)(\PTR_B_REG) - .if \IsLast==1 - la \PTR_B_REG,DISP1(\Index ,8)(\PTR_B_REG) - .endif -.endm - -.macro CALC_8x4 PTR_A_REG,PTR_B_REG - Calculate_8x4_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_8x4_4 PTR_A_REG,PTR_B_REG - Calculate_8x4_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_8x4_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_8x4_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_8x4_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_8x2 PTR_A_REG,PTR_B_REG - Calculate_8x2_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_8x2_4 PTR_A_REG,PTR_B_REG - Calculate_8x2_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_8x2_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_8x2_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_8x2_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_8x1 PTR_A_REG,PTR_B_REG - Calculate_8x1_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_8x1_4 PTR_A_REG,PTR_B_REG - Calculate_8x1_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_8x1_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_8x1_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_8x1_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_4x4 PTR_A_REG,PTR_B_REG - Calculate_4x4_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_4x4_4 PTR_A_REG,PTR_B_REG - Calculate_4x4_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_4x4_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_4x4_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_4x4_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_4x2 PTR_A_REG,PTR_B_REG - Calculate_4x2_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_4x2_4 PTR_A_REG,PTR_B_REG - Calculate_4x2_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_4x2_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_4x2_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_4x2_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_4x1 PTR_A_REG,PTR_B_REG - Calculate_4x1_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_4x1_4 PTR_A_REG,PTR_B_REG - Calculate_4x1_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_4x1_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_4x1_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_4x1_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_2x4 PTR_A_REG,PTR_B_REG - Calculate_4x2_I \PTR_B_REG,\PTR_A_REG,0,1 -.endm - -.macro CALC_2x4_4 PTR_A_REG,PTR_B_REG - Calculate_4x2_I \PTR_B_REG,\PTR_A_REG,0,0 - Calculate_4x2_I \PTR_B_REG,\PTR_A_REG,1,0 - Calculate_4x2_I \PTR_B_REG,\PTR_A_REG,2,0 - Calculate_4x2_I \PTR_B_REG,\PTR_A_REG,3,1 -.endm - -.macro CALC_2x2 PTR_A_REG,PTR_B_REG - Calculate_2x2_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_2x2_4 PTR_A_REG,PTR_B_REG - Calculate_2x2_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_2x2_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_2x2_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_2x2_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_2x1 PTR_A_REG,PTR_B_REG - Calculate_2x1_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_2x1_4 PTR_A_REG,PTR_B_REG - Calculate_2x1_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_2x1_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_2x1_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_2x1_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - -.macro CALC_1x4 PTR_A_REG,PTR_B_REG - Calculate_4x1_I \PTR_B_REG,\PTR_A_REG,0,1 -.endm - -.macro CALC_1x4_4 PTR_A_REG,PTR_B_REG - Calculate_4x1_I \PTR_B_REG,\PTR_A_REG,0,0 - Calculate_4x1_I \PTR_B_REG,\PTR_A_REG,1,0 - Calculate_4x1_I \PTR_B_REG,\PTR_A_REG,2,0 - Calculate_4x1_I \PTR_B_REG,\PTR_A_REG,3,1 -.endm - -.macro CALC_1x2 PTR_A_REG,PTR_B_REG - Calculate_2x1_I \PTR_B_REG,\PTR_A_REG,0,1 -.endm - -.macro CALC_1x2_4 PTR_A_REG,PTR_B_REG - Calculate_2x1_I \PTR_B_REG,\PTR_A_REG,0,0 - Calculate_2x1_I \PTR_B_REG,\PTR_A_REG,1,0 - Calculate_2x1_I \PTR_B_REG,\PTR_A_REG,2,0 - Calculate_2x1_I \PTR_B_REG,\PTR_A_REG,3,1 -.endm - -.macro CALC_1x1 PTR_A_REG,PTR_B_REG - Calculate_1x1_I \PTR_A_REG,\PTR_B_REG,0,1 -.endm - -.macro CALC_1x1_4 PTR_A_REG,PTR_B_REG - Calculate_1x1_I \PTR_A_REG,\PTR_B_REG,0,0 - Calculate_1x1_I \PTR_A_REG,\PTR_B_REG,1,0 - Calculate_1x1_I \PTR_A_REG,\PTR_B_REG,2,0 - Calculate_1x1_I \PTR_A_REG,\PTR_B_REG,3,1 -.endm - - -/**************************************STORAGE*************************************************/ - - -.macro Multiply_8x1 vr1,vr2,vr3,vr4,va1,va2,va3,va4,vb1 - #if defined(TRMMKERNEL) - vfmdb \vr1,\va1,\vb1 - vfmdb \vr2,\va2,\vb1 - vfmdb \vr3,\va3,\vb1 - vfmdb \vr4,\va4,\vb1 - #else - vfmadb \vr1,\va1,\vb1,\vr1 - vfmadb \vr2,\va2,\vb1,\vr2 - vfmadb \vr3,\va3,\vb1,\vr3 - vfmadb \vr4,\va4,\vb1,\vr4 - #endif -.endm - -.macro Multiply_4x1 vr1,vr2, va1,va2, vb1 - #if defined(TRMMKERNEL) - vfmdb \vr1,\va1,\vb1 - vfmdb \vr2,\va2,\vb1 - #else - vfmadb \vr1,\va1,\vb1,\vr1 - vfmadb \vr2,\va2,\vb1,\vr2 - #endif -.endm - -.macro Multiply_2x1 vr1, va1,vb1 - #if defined(TRMMKERNEL) - vfmdb \vr1,\va1,\vb1 - #else - vfmadb \vr1,\va1,\vb1,\vr1 - #endif -.endm - - -.macro STORE_8x4 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL , LV1 ,LV2 - la \LV1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) - vl %v2,16(\CIJ_REG) - vl %v3,32(\CIJ_REG) - vl %v4,48(\CIJ_REG) -#endif - Multiply_8x1 %v1,%v2,%v3,%v4, %v16,%v17,%v18,%v19 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - vst %v2,16(\CIJ_REG) - vst %v3,32(\CIJ_REG) - vst %v4,48(\CIJ_REG) - - la \LV2,0(\LV1,\LDC_BYTE_ORIGINAL ) -#if !defined(TRMMKERNEL) - vl %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v18,32(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v19,48(\CIJ_REG,\LDC_BYTE_ORIGINAL) -#endif - Multiply_8x1 %v16,%v17,%v18,%v19, %v20,%v21,%v22,%v23 ,\ALPHA_VECREG - vst %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v18,32(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v19,48(\CIJ_REG,\LDC_BYTE_ORIGINAL) - - #if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG,\LV1) - vl %v2,16(\CIJ_REG,\LV1) - vl %v3,32(\CIJ_REG,\LV1) - vl %v4,48(\CIJ_REG,\LV1) -#endif - Multiply_8x1 %v1,%v2,%v3,%v4, %v24,%v25,%v26,%v27 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG,\LV1) - vst %v2,16(\CIJ_REG,\LV1) - vst %v3,32(\CIJ_REG,\LV1) - vst %v4,48(\CIJ_REG,\LV1) - -#if !defined(TRMMKERNEL) - vl %v16,0(\CIJ_REG,\LV2) - vl %v17,16(\CIJ_REG,\LV2) - vl %v18,32(\CIJ_REG,\LV2) - vl %v19,48(\CIJ_REG,\LV2) -#endif - Multiply_8x1 %v16,%v17,%v18,%v19, %v28,%v29,%v30,%v31 ,\ALPHA_VECREG - vst %v16,0(\CIJ_REG,\LV2) - vst %v17,16(\CIJ_REG,\LV2) - vst %v18,32(\CIJ_REG,\LV2) - vst %v19,48(\CIJ_REG,\LV2) - - la \CIJ_REG,64(\CIJ_REG) - -.endm - -.macro STORE_8x2 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) - vl %v2,16(\CIJ_REG) - vl %v3,32(\CIJ_REG) - vl %v4,48(\CIJ_REG) -#endif - Multiply_8x1 %v1,%v2,%v3,%v4, %v16,%v17,%v18,%v19 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - vst %v2,16(\CIJ_REG) - vst %v3,32(\CIJ_REG) - vst %v4,48(\CIJ_REG) - -#if !defined(TRMMKERNEL) - vl %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v18,32(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v19,48(\CIJ_REG,\LDC_BYTE_ORIGINAL) -#endif - Multiply_8x1 %v16,%v17,%v18,%v19, %v20,%v21,%v22,%v23 ,\ALPHA_VECREG - vst %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v18,32(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v19,48(\CIJ_REG,\LDC_BYTE_ORIGINAL) - - la \CIJ_REG,64(\CIJ_REG) - -.endm - -.macro STORE_8x1 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL - -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) - vl %v2,16(\CIJ_REG) - vl %v3,32(\CIJ_REG) - vl %v4,48(\CIJ_REG) -#endif - Multiply_8x1 %v1,%v2,%v3,%v4, %v16,%v17,%v18,%v19 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - vst %v2,16(\CIJ_REG) - vst %v3,32(\CIJ_REG) - vst %v4,48(\CIJ_REG) - - la \CIJ_REG,64(\CIJ_REG) -.endm - - -.macro STORE_4x4 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL, LV1 ,LV2 - la \LV1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) - vl %v2,16(\CIJ_REG) -#endif - Multiply_4x1 %v1,%v2 , %v16,%v17 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - vst %v2,16(\CIJ_REG) - - la \LV2,0(\LV1,\LDC_BYTE_ORIGINAL ) -#if !defined(TRMMKERNEL) - vl %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) -#endif - Multiply_4x1 %v16,%v17 , %v20,%v21 ,\ALPHA_VECREG - vst %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) - - #if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG,\LV1) - vl %v2,16(\CIJ_REG,\LV1) -#endif - Multiply_4x1 %v1,%v2 , %v24,%v25 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG,\LV1) - vst %v2,16(\CIJ_REG,\LV1) - -#if !defined(TRMMKERNEL) - vl %v16,0(\CIJ_REG,\LV2) - vl %v17,16(\CIJ_REG,\LV2) -#endif - Multiply_4x1 %v16,%v17, %v28,%v29 ,\ALPHA_VECREG - vst %v16,0(\CIJ_REG,\LV2) - vst %v17,16(\CIJ_REG,\LV2) - - la \CIJ_REG,32(\CIJ_REG) - -.endm - - -.macro STORE_4x2 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL - -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) - vl %v2,16(\CIJ_REG) -#endif - Multiply_4x1 %v1,%v2 , %v16,%v17 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - vst %v2,16(\CIJ_REG) - -#if !defined(TRMMKERNEL) - vl %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vl %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) -#endif - Multiply_4x1 %v16,%v17 , %v20,%v21 ,\ALPHA_VECREG - vst %v16,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - vst %v17,16(\CIJ_REG,\LDC_BYTE_ORIGINAL) - - la \CIJ_REG,32(\CIJ_REG) - -.endm -.macro STORE_4x1 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL - -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) - vl %v2,16(\CIJ_REG) -#endif - Multiply_4x1 %v1,%v2 , %v16,%v17 ,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - vst %v2,16(\CIJ_REG) - - la \CIJ_REG,32(\CIJ_REG) - -.endm - -.macro STORE_2x2 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL - -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) -#endif - Multiply_2x1 %v1,%v16,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - -#if !defined(TRMMKERNEL) - vl %v2,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) -#endif - Multiply_2x1 %v2,%v20,\ALPHA_VECREG - vst %v2,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - - la \CIJ_REG,16(\CIJ_REG) - -.endm - - -.macro STORE_2x1 ALPHA_VECREG,CIJ_REG , LDC_BYTE_ORIGINAL - -#if !defined(TRMMKERNEL) - vl %v1,0(\CIJ_REG) -#endif - Multiply_2x1 %v1,%v16,\ALPHA_VECREG - vst %v1,0(\CIJ_REG) - - la \CIJ_REG,16(\CIJ_REG) -.endm - - -/*STORE C1X1*/ -.macro STORE_1x1 ALPHA_FLOAT,CIJ_REG,LDC_BYTE_ORIGINAL - -#if defined(TRMMKERNEL) - mdbr %f1,\ALPHA_FLOAT - std %f1,0(CIJ_LOCAL) -#else - ld %f2,0(CIJ_LOCAL) - madbr %f2,%f1,\ALPHA_FLOAT - std %f2,0(CIJ_LOCAL) -#endif - la \CIJ_REG,8(\CIJ_REG) -.endm - -/*reversed ones*/ - -.macro STORE_2x4 ALPHA_REG,CIJ_REG , LDC_BYTE_ORIGINAL , LV1 ,LV2 -/**/ - vfmdb %v1,%v16,\ALPHA_REG - vfmdb %v2,%v17,\ALPHA_REG - vfmdb %v6,%v20,\ALPHA_REG - vfmdb %v7,%v21,\ALPHA_REG - vrepg %v4,%v1,1 - vrepg %v5,%v6,1 - la \LV1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) - #if !defined(TRMMKERNEL) - adb %f1, 0(\CIJ_REG) - #endif - std %f1,0(\CIJ_REG) - #if !defined(TRMMKERNEL) - adb %f6, 8(\CIJ_REG) - #endif - std %f6,8(\CIJ_REG) - #if !defined(TRMMKERNEL) - adb %f4,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - #endif - std %f4,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - #if !defined(TRMMKERNEL) - adb %f5,8(\CIJ_REG,\LDC_BYTE_ORIGINAL) - #endif - std %f5,8(\CIJ_REG,\LDC_BYTE_ORIGINAL) - - /*add LDC_BYTE */ - la \LV2,0(\LV1,\LDC_BYTE_ORIGINAL ) - vrepg %v4,%v2,1 - vrepg %v5,%v7,1 - #if !defined(TRMMKERNEL) - adb %f2,0(\CIJ_REG,\LV1) - #endif - std %f2,0(\CIJ_REG,\LV1) - #if !defined(TRMMKERNEL) - adb %f7,8(\CIJ_REG,\LV1) - #endif - std %f7,8(\CIJ_REG,\LV1) - #if !defined(TRMMKERNEL) - adb %f4,0(\CIJ_REG,\LV2) - #endif - std %f4,0(\CIJ_REG,\LV2) - #if !defined(TRMMKERNEL) - adb %f5,8(\CIJ_REG,\LV2) - #endif - std %f5,8(\CIJ_REG,\LV2) - - la \CIJ_REG,16(\CIJ_REG) - -.endm - -.macro STORE_1x4 ALPHA_REG,CIJ_REG , LDC_BYTE_ORIGINAL , LV1 ,LV2 - - vfmdb %v1,%v16,\ALPHA_REG - vfmdb %v2,%v17,\ALPHA_REG - vrepg %v4,%v1,1 - vrepg %v5,%v2,1 - la \LV1,0(\LDC_BYTE_ORIGINAL, \LDC_BYTE_ORIGINAL) - #if !defined(TRMMKERNEL) - adb %f1, 0(\CIJ_REG) - #endif - std %f1,0(\CIJ_REG) - #if !defined(TRMMKERNEL) - adb %f4,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - #endif - std %f4,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - /*add LDC_BYTE */ - la \LV2,0(\LV1,\LDC_BYTE_ORIGINAL ) - #if !defined(TRMMKERNEL) - adb %f2,0(\CIJ_REG,\LV1) - #endif - std %f2,0(\CIJ_REG,\LV1) - #if !defined(TRMMKERNEL) - adb %f5,0(\CIJ_REG,\LV2) - #endif - std %f5,0(\CIJ_REG,\LV2) - - la \CIJ_REG,8(\CIJ_REG) - -.endm - - .macro STORE_1x2 ALPHA_REG,CIJ_REG , LDC_BYTE_ORIGINAL -/**/ - vfmdb %v1,%v16,\ALPHA_REG - vrepg %v4,%v1,1 - #if !defined(TRMMKERNEL) - adb %f1, 0(\CIJ_REG) - #endif - std %f1,0(\CIJ_REG) - #if !defined(TRMMKERNEL) - adb %f4,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - #endif - std %f4,0(\CIJ_REG,\LDC_BYTE_ORIGINAL) - - la \CIJ_REG,8(\CIJ_REG) - -.endm - - - - -/****************************TRMM POINTER REFRESH MACROSES*************************/ - -.macro RefreshPointers PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B - #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) - /* ptrbb = bb;*/ - lgr \PTR_B,\B_VAL /*refresh BPOINT*/ - - #else - /* ptrba =ptrba+ off*C_A; - ptrbb = bb + off*C_B;*/ -.if \C_B==4 - .if \C_A==8 - sllg \PTR_B, \OFF_VAL,5 - la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*4*/ - agr \PTR_A,\PTR_B /*ptrba+off*4**/ - la \PTR_B,0(\B_VAL,\PTR_B) - .elseif \C_A==4 - sllg \PTR_B, \OFF_VAL,5 - agr \PTR_A,\PTR_B /*ptrba+off*4**/ - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .elseif \C_A==2 - sllg \PTR_B, \OFF_VAL,4 - la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*2**/ - agr \PTR_B, \PTR_B - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - - .elseif \C_A==1 - sllg \PTR_B, \OFF_VAL,3 - agr \PTR_A,\PTR_B /*ptrba+off*4**/ - sllg \PTR_B, \OFF_VAL,5 - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .endif - -.elseif \C_B==2 - .if \C_A==8 - sllg \PTR_B, \OFF_VAL,6 - agr \PTR_A,\PTR_B /*ptrba+off*8**/ - sllg \PTR_B, \OFF_VAL,4 - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .elseif \C_A==4 - sllg \PTR_B, \OFF_VAL,4 - la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*2**/ - agr \PTR_A,\PTR_B /*ptrba+off*2**/ - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .elseif \C_A==2 - sllg \PTR_B, \OFF_VAL,4 - agr \PTR_A,\PTR_B /*ptrba+off*2**/ - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .elseif \C_A==1 - sllg \PTR_B, \OFF_VAL,3 - la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*1**/ - agr \PTR_B,\PTR_B /* off+off**/ - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .endif - -.elseif \C_B==1 - .if \C_A==8 - sllg \PTR_B, \OFF_VAL,6 - agr \PTR_A,\PTR_B /*ptrba+off*8**/ - sllg \PTR_B, \OFF_VAL,3 - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .elseif \C_A==4 - sllg \PTR_B, \OFF_VAL,5 - agr \PTR_A,\PTR_B /*ptrba+off*4**/ - sllg \PTR_B, \OFF_VAL,3 - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .elseif \C_A==2 - sllg \PTR_B, \OFF_VAL,3 - la \PTR_A,0(\PTR_A,\PTR_B) /*ptrba+off*1**/ - agr \PTR_A,\PTR_B /*ptrba+off*1**/ - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - - .elseif \C_A==1 - sllg \PTR_B, \OFF_VAL,3 - agr \PTR_A,\PTR_B /*ptrba+off*1**/ - la \PTR_B,0(\B_VAL,\PTR_B) /*refresh BPOINT*/ - .endif -.endif - - - #endif -.endm - -/**/ -.macro RefreshTempBk TEMP_VAL,BK_VAL,OFF_VAL,INCR_A,INCR_B - #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) - /* temp = bk-off;*/ - sgrk \TEMP_VAL,\BK_VAL,\OFF_VAL - - #elif defined(LEFT) - /* temp = off+INCR_A; // number of values in A */ - la \TEMP_VAL,\INCR_A(\OFF_VAL) - #else - /* temp = off+INCR_B // number of values in B*/ - la \TEMP_VAL,\INCR_B(\OFF_VAL) - #endif - -.endm - - -.macro RefreshPointersAndOFF TEMP_VAL,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B - - #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) - /*temp = bk - off;*/ - sgrk \TEMP_VAL,\BK_VAL,\OFF_VAL - #ifdef LEFT - /*temp -= 8; // number of values in A*/ - lay \TEMP_VAL,-\C_A(\TEMP_VAL) - #else - /*temp -= 4; // number of values in B*/ - lay \TEMP_VAL,-\C_B(\TEMP_VAL) - #endif - /*ptrba += temp*C_A; - ptrbb += temp*C_B;*/ - .if \C_A==8 - sllg \TEMP_VAL, \TEMP_VAL,6 - .elseif \C_A==4 - sllg \TEMP_VAL, \TEMP_VAL,5 /*temp*4*/ - .elseif \C_A==2 - sllg \TEMP_VAL, \TEMP_VAL,4 /*temp*2*/ - .elseif \C_A==1 - sllg \TEMP_VAL, \TEMP_VAL,3 /*temp*1*/ - .endif - la \PTR_A,0(\PTR_A,\TEMP_VAL) /*ptrba+temp*C_A*/ - /*we do not need to refresh ptrbb. so lets ignore it*/ - - #endif - - #ifdef LEFT - /*off += 8; // number of values in A*/ - aghi \OFF_VAL,\C_A - #endif -.endm \ No newline at end of file diff --git a/kernel/zarch/skernelMacros.S b/kernel/zarch/skernelMacros.S index 8e1cb69..6f74f2b 100644 --- a/kernel/zarch/skernelMacros.S +++ b/kernel/zarch/skernelMacros.S @@ -120,11 +120,11 @@ vmrlf %v4,%v1,%v1 vmrlf %v6,%v2,%v2 vldeb %v3, %v3 - vldeb %v4, %v4 + vldeb %v4, %v4 vldeb %v5, %v5 vlrepf %v7, DISP4(\Index ,0)(\PTR_B_REG) vlrepf %v1, DISP4(\Index ,4)(\PTR_B_REG) - vldeb %v6, %v6 + vldeb %v6, %v6 vldeb %v7, %v7 vldeb %v1, %v1 diff --git a/kernel/zarch/strmm8x4V.S b/kernel/zarch/strmm8x4V.S index 0d0d6a3..f8e7481 100644 --- a/kernel/zarch/strmm8x4V.S +++ b/kernel/zarch/strmm8x4V.S @@ -26,7 +26,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ /************************************************************************************** -* 2017/03/06 AbdelRauf (quickwritereader@gmail.com) +* 2017/03/01 AbdelRauf (quickwritereader@gmail.com) * BLASTEST : passed * CTEST : passed * TEST : passed @@ -138,10 +138,8 @@ cijle BN_CUR,0,.LX2 ALIGN_4 .LX4_BN: #if defined(PREFETCH_INS) - pfd 1, 0(A) - pfd 1, 256(A) - pfd 1, 0(B) - pfd 1, 256(B) + pfd 1, 0(A) + pfd 1, 0(B) #endif #if defined(TRMMKERNEL) && defined(LEFT) /*off = offset;*/ @@ -178,7 +176,7 @@ ALIGN_4 #endif CALC_8x4_4 LOCAL_VAR3,LOCAL_VAR2 #if defined(PREFETCH_INS) - pfd 1, 256(LOCAL_VAR2) + pfd 1, 128(LOCAL_VAR2) #endif brctg LOCAL_VAR1,.L8x4_4_BK @@ -414,8 +412,7 @@ cijle LOCAL_VAR1,0,.L8x2_mod ALIGN_4 .L8x2_4_BK: /*BK_CUR LOOP */ #if defined(PREFETCH_INS) - pfd 1, 256(LOCAL_VAR3) - pfd 1,64(LOCAL_VAR2) + pfd 1, 256(LOCAL_VAR3) #endif CALC_8x2_4 LOCAL_VAR3,LOCAL_VAR2 brctg LOCAL_VAR1,.L8x2_4_BK diff --git a/param.h b/param.h index 8587cfa..d697174 100644 --- a/param.h +++ b/param.h @@ -2638,20 +2638,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define DGEMM_DEFAULT_UNROLL_M 8 #define DGEMM_DEFAULT_UNROLL_N 4 -#define CGEMM_DEFAULT_UNROLL_M 2 -#define CGEMM_DEFAULT_UNROLL_N 2 +#define CGEMM_DEFAULT_UNROLL_M 4 +#define CGEMM_DEFAULT_UNROLL_N 4 #define ZGEMM_DEFAULT_UNROLL_M 4 #define ZGEMM_DEFAULT_UNROLL_N 4 -#define SGEMM_DEFAULT_P 320 - #define DGEMM_DEFAULT_P 320 -#define CGEMM_DEFAULT_P 96 +#define SGEMM_DEFAULT_P 456 +#define DGEMM_DEFAULT_P 320 +#define CGEMM_DEFAULT_P 480 #define ZGEMM_DEFAULT_P 224 -#define SGEMM_DEFAULT_Q 384 +#define SGEMM_DEFAULT_Q 488 #define DGEMM_DEFAULT_Q 384 -#define CGEMM_DEFAULT_Q 120 +#define CGEMM_DEFAULT_Q 128 #define ZGEMM_DEFAULT_Q 352 #define SGEMM_DEFAULT_R 8192 -- 2.7.4