small optimization on dgemm_kernel for N=1

diff --git a/kernel/x86_64/dgemm_kernel_4x4_haswell.S b/kernel/x86_64/dgemm_kernel_4x4_haswell.S
index a49a51e..0a2ca7a 100644 (file)
--- a/kernel/x86_64/dgemm_kernel_4x4_haswell.S
+++ b/kernel/x86_64/dgemm_kernel_4x4_haswell.S
@@ -1092,18 +1092,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 .macro INIT4x1\r
 \r
-       vxorpd          %xmm4 , %xmm4 , %xmm4\r
-       vxorpd          %xmm5 , %xmm5 , %xmm5\r
+       vxorpd          %ymm4 , %ymm4 , %ymm4\r
+       vxorpd          %ymm5 , %ymm5 , %ymm5\r
+       vxorpd          %ymm6 , %ymm6 , %ymm6\r
+       vxorpd          %ymm7 , %ymm7 , %ymm7\r
+\r
+.endm\r
+\r
+\r
+.macro KERNEL4x1\r
+\r
+       vbroadcastsd    -12 * SIZE(BO), %ymm0\r
+       vbroadcastsd    -11 * SIZE(BO), %ymm1\r
+       vbroadcastsd    -10 * SIZE(BO), %ymm2\r
+       vbroadcastsd    -9  * SIZE(BO), %ymm3\r
+\r
+       vfmadd231pd     -16 * SIZE(AO) ,%ymm0  , %ymm4\r
+       vfmadd231pd     -12 * SIZE(AO) ,%ymm1  , %ymm5\r
+\r
+       vbroadcastsd    -8  * SIZE(BO), %ymm0\r
+       vbroadcastsd    -7  * SIZE(BO), %ymm1\r
+\r
+       vfmadd231pd     -8  * SIZE(AO) ,%ymm2  , %ymm6\r
+       vfmadd231pd     -4  * SIZE(AO) ,%ymm3  , %ymm7\r
+\r
+       vbroadcastsd    -6  * SIZE(BO), %ymm2\r
+       vbroadcastsd    -5  * SIZE(BO), %ymm3\r
+\r
+       vfmadd231pd      0  * SIZE(AO) ,%ymm0  , %ymm4\r
+       vfmadd231pd      4  * SIZE(AO) ,%ymm1  , %ymm5\r
+       vfmadd231pd      8  * SIZE(AO) ,%ymm2  , %ymm6\r
+       vfmadd231pd      12 * SIZE(AO) ,%ymm3  , %ymm7\r
+\r
+       addq            $ 8 *SIZE, BO\r
+       addq            $ 32*SIZE, AO\r
 \r
 .endm\r
 \r
 \r
 .macro KERNEL4x1_SUB\r
-       vmovddup        -12 * SIZE(BO), %xmm2\r
-       vmovups         -16 * SIZE(AO), %xmm0\r
-       vmovups         -14 * SIZE(AO), %xmm1\r
-       vfmadd231pd     %xmm0 ,%xmm2  , %xmm4\r
-       vfmadd231pd     %xmm1 ,%xmm2  , %xmm5\r
+       vbroadcastsd    -12 * SIZE(BO), %ymm2\r
+       vmovups         -16 * SIZE(AO), %ymm0\r
+       vfmadd231pd     %ymm0 ,%ymm2  , %ymm4\r
        addq            $ 1*SIZE, BO\r
        addq            $ 4*SIZE, AO\r
 \r
@@ -1112,21 +1142,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 .macro SAVE4x1\r
 \r
-       vmovddup        ALPHA, %xmm0\r
+       vbroadcastsd    ALPHA, %ymm0\r
 \r
-       vmulpd  %xmm0 , %xmm4 , %xmm4\r
-       vmulpd  %xmm0 , %xmm5 , %xmm5\r
+       vaddpd  %ymm4,%ymm5, %ymm4 \r
+       vaddpd  %ymm6,%ymm7, %ymm6 \r
+       vaddpd  %ymm4,%ymm6, %ymm4 \r
+\r
+       vmulpd  %ymm0 , %ymm4 , %ymm4\r
 \r
 \r
 #if !defined(TRMMKERNEL)\r
 \r
-       vaddpd                  (CO1)     , %xmm4, %xmm4\r
-       vaddpd          2 * SIZE(CO1)     , %xmm5, %xmm5\r
+       vaddpd                  (CO1)     , %ymm4, %ymm4\r
 \r
 #endif\r
 \r
-       vmovups %xmm4 ,         (CO1)\r
-       vmovups %xmm5 , 2 * SIZE(CO1)\r
+       vmovups %ymm4 ,         (CO1)\r
 \r
        addq    $ 4*SIZE, CO1\r
 .endm\r
@@ -2112,15 +2143,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 .L1_12:\r
 \r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
+       KERNEL4x1\r
 \r
        dec     %rax\r
        jne     .L1_12\r
@@ -3180,15 +3203,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 \r
 .L1_12:\r
 \r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
-       KERNEL4x1_SUB\r
+       KERNEL4x1\r
 \r
        dec     %rax\r
        jne     .L1_12\r