From: wernsaar <wernsaar@googlemail.com>
Date: Mon, 17 Jun 2013 12:19:09 +0000 (+0200)
Subject: added dgemm_tcopy_8_bulldozer.S
X-Git-Tag: v0.2.9.rc1~81^2~2
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=d0b6299b136b43ea87a61b3e6836b0b82275229f;p=platform%2Fupstream%2Fopenblas.git

added dgemm_tcopy_8_bulldozer.S
---

diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER
index 1a0395a..1bd8073 100644
--- a/kernel/x86_64/KERNEL.BULLDOZER
+++ b/kernel/x86_64/KERNEL.BULLDOZER
@@ -14,7 +14,7 @@ SGEMMONCOPYOBJ =  sgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
 DGEMMKERNEL    =  dgemm_kernel_8x2_bulldozer.S
 DGEMMINCOPY    =  ../generic/gemm_ncopy_8.c
-DGEMMITCOPY    =  ../generic/gemm_tcopy_8.c
+DGEMMITCOPY    =  dgemm_tcopy_8_bulldozer.S
 DGEMMONCOPY    =  gemm_ncopy_2_bulldozer.S
 DGEMMOTCOPY    =  ../generic/gemm_tcopy_2.c
 DGEMMINCOPYOBJ =  dgemm_incopy$(TSUFFIX).$(SUFFIX)
diff --git a/kernel/x86_64/dgemm_tcopy_8_bulldozer.S b/kernel/x86_64/dgemm_tcopy_8_bulldozer.S
new file mode 100644
index 0000000..e62b9da
--- /dev/null
+++ b/kernel/x86_64/dgemm_tcopy_8_bulldozer.S
@@ -0,0 +1,667 @@
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+#define VMOVUPS_A1(OFF, ADDR, REGS) vmovups     OFF(ADDR), REGS
+#define VMOVUPS_A2(OFF, ADDR, BASE, SCALE, REGS) vmovups        OFF(ADDR, BASE, SCALE), REGS
+
+#define A_PRE 256
+
+#ifndef WINDOWS_ABI
+
+#define N	ARG1	/* rsi */
+#define M	ARG2	/* rdi */
+#define A	ARG3	/* rdx */
+#define LDA	ARG4	/* rcx */
+#define B	ARG5	/* r8  */
+
+#define AO1	%r9
+#define AO2	%r10
+#define LDA3	%r11
+#define M8	%r12
+
+#else
+
+#define N	ARG1	/* rdx */
+#define M	ARG2	/* rcx */
+#define A	ARG3	/* r8  */
+#define LDA	ARG4	/* r9  */
+#define OLD_B		40 + 56(%rsp)
+
+#define B	%r12
+
+#define AO1	%rsi
+#define AO2	%rdi
+#define LDA3	%r10
+#define M8	%r11
+#endif
+
+#define I	%rax
+
+#define B0	%rbp
+#define	B1	%r13
+#define	B2	%r14
+#define	B3	%r15
+
+	PROLOGUE
+	PROFCODE
+	
+#ifdef WINDOWS_ABI
+	pushq	%rdi
+	pushq	%rsi
+#endif
+
+	pushq	%r15
+	pushq	%r14
+	pushq	%r13
+	pushq	%r12
+	pushq	%rbp
+
+#ifdef WINDOWS_ABI
+	movq	OLD_B,     B
+#endif
+
+	subq	$-16 * SIZE, B
+
+	movq	M,    B1
+	movq	M,    B2
+	movq	M,    B3
+
+	andq	$-8,  B1
+	andq	$-4,  B2
+	andq	$-2,  B3
+
+	imulq	N,    B1
+	imulq	N,    B2
+	imulq	N,    B3
+
+	leaq	(B, B1, SIZE), B1
+	leaq	(B, B2, SIZE), B2
+	leaq	(B, B3, SIZE), B3
+
+	leaq	(,LDA, SIZE), LDA
+	leaq	(LDA, LDA, 2), LDA3
+
+	leaq	(, N, SIZE), M8
+
+	cmpq	$8, N
+	jl	.L20
+	ALIGN_4
+
+.L11:
+	subq	$8, N
+
+	movq	A, AO1
+	leaq	(A, LDA, 4), AO2
+	leaq	(A, LDA, 8), A
+
+	movq	B, B0
+	addq	$64 * SIZE, B
+
+	movq	M, I
+	sarq	$3, I
+	jle	.L14
+	ALIGN_4
+
+.L13:
+
+	prefetchnta	A_PRE(AO1)
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)
+	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B0)
+	vmovups	%xmm1,  -14 * SIZE(B0)
+	vmovups	%xmm2,  -12 * SIZE(B0)
+	vmovups	%xmm3,  -10 * SIZE(B0)
+
+
+	prefetchnta	A_PRE(AO1, LDA, 1)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO1, LDA, 1, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO1, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,   -8 * SIZE(B0)
+	vmovups	%xmm1,   -6 * SIZE(B0)
+	vmovups	%xmm2,   -4 * SIZE(B0)
+	vmovups	%xmm3,   -2 * SIZE(B0)
+
+
+	prefetchnta	A_PRE(AO1, LDA, 2)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA, 2, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA, 2, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO1, LDA, 2, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO1, LDA, 2, %xmm3)
+
+
+	vmovups	%xmm0,    0 * SIZE(B0)
+	vmovups	%xmm1,    2 * SIZE(B0)
+	vmovups	%xmm2,    4 * SIZE(B0)
+	vmovups	%xmm3,    6 * SIZE(B0)
+
+
+	prefetchnta	A_PRE(AO1, LDA3, 1)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA3, 1, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA3, 1, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO1, LDA3, 1, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO1, LDA3, 1, %xmm3)
+
+	vmovups	%xmm0,    8 * SIZE(B0)
+	vmovups	%xmm1,   10 * SIZE(B0)
+	vmovups	%xmm2,   12 * SIZE(B0)
+	vmovups	%xmm3,   14 * SIZE(B0)
+
+	prefetchnta	A_PRE(AO2)
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)
+	VMOVUPS_A1(4 * SIZE, AO2, %xmm2)
+	VMOVUPS_A1(6 * SIZE, AO2, %xmm3)
+
+	vmovups	%xmm0,   16 * SIZE(B0)
+	vmovups	%xmm1,   18 * SIZE(B0)
+	vmovups	%xmm2,   20 * SIZE(B0)
+	vmovups	%xmm3,   22 * SIZE(B0)
+
+	prefetchnta	A_PRE(AO2, LDA, 1)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO2, LDA, 1, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO2, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,   24 * SIZE(B0)
+	vmovups	%xmm1,   26 * SIZE(B0)
+	vmovups	%xmm2,   28 * SIZE(B0)
+	vmovups	%xmm3,   30 * SIZE(B0)
+
+	prefetchnta	A_PRE(AO2, LDA, 2)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA, 2, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA, 2, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO2, LDA, 2, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO2, LDA, 2, %xmm3)
+
+	vmovups	%xmm0,   32 * SIZE(B0)
+	vmovups	%xmm1,   34 * SIZE(B0)
+	vmovups	%xmm2,   36 * SIZE(B0)
+	vmovups	%xmm3,   38 * SIZE(B0)
+
+	prefetchnta	A_PRE(AO2, LDA3, 1)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA3, 1, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA3, 1, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO2, LDA3, 1, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO2, LDA3, 1, %xmm3)
+
+	vmovups	%xmm0,   40 * SIZE(B0)
+	vmovups	%xmm1,   42 * SIZE(B0)
+	vmovups	%xmm2,   44 * SIZE(B0)
+	vmovups	%xmm3,   46 * SIZE(B0)
+
+	addq	$8 * SIZE, AO1
+	addq	$8 * SIZE, AO2
+	leaq	(B0, M8, 8), B0
+
+	decq	I
+	jg	.L13
+	ALIGN_4
+
+.L14:
+	testq	$4, M
+	jle	.L16
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm2)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B1)
+	vmovups	%xmm1,  -14 * SIZE(B1)
+	vmovups	%xmm2,  -12 * SIZE(B1)
+	vmovups	%xmm3,  -10 * SIZE(B1)
+
+	VMOVUPS_A2(0 * SIZE, AO1, LDA,  2, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA,  2, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA3, 1, %xmm2)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA3, 1, %xmm3)
+
+	vmovups	%xmm0,   -8 * SIZE(B1)
+	vmovups	%xmm1,   -6 * SIZE(B1)
+	vmovups	%xmm2,   -4 * SIZE(B1)
+	vmovups	%xmm3,   -2 * SIZE(B1)
+
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm2)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,    0 * SIZE(B1)
+	vmovups	%xmm1,    2 * SIZE(B1)
+	vmovups	%xmm2,    4 * SIZE(B1)
+	vmovups	%xmm3,    6 * SIZE(B1)
+
+	VMOVUPS_A2(0 * SIZE, AO2, LDA,  2, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA,  2, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA3, 1, %xmm2)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA3, 1, %xmm3)
+
+	vmovups	%xmm0,    8 * SIZE(B1)
+	vmovups	%xmm1,   10 * SIZE(B1)
+	vmovups	%xmm2,   12 * SIZE(B1)
+	vmovups	%xmm3,   14 * SIZE(B1)
+
+	addq	$4 * SIZE, AO1
+	addq	$4 * SIZE, AO2
+	subq	$-32 * SIZE, B1
+	ALIGN_4
+
+.L16:
+	testq	$2, M
+	jle	.L18
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA,  1, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA,  2, %xmm2)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA3, 1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B2)
+	vmovups	%xmm1,  -14 * SIZE(B2)
+	vmovups	%xmm2,  -12 * SIZE(B2)
+	vmovups	%xmm3,  -10 * SIZE(B2)
+
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA,  1, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA,  2, %xmm2)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA3, 1, %xmm3)
+
+	vmovups	%xmm0,   -8 * SIZE(B2)
+	vmovups	%xmm1,   -6 * SIZE(B2)
+	vmovups	%xmm2,   -4 * SIZE(B2)
+	vmovups	%xmm3,   -2 * SIZE(B2)
+
+	addq	$2 * SIZE, AO1
+	addq	$2 * SIZE, AO2
+	subq	$-16 * SIZE, B2
+	ALIGN_4
+
+.L18:
+	testq	$1, M
+	jle	.L19
+
+	vmovsd	0 * SIZE(AO1),         %xmm0
+	vmovsd	0 * SIZE(AO1, LDA),    %xmm1
+	vmovsd	0 * SIZE(AO1, LDA, 2), %xmm2
+	vmovsd	0 * SIZE(AO1, LDA3),   %xmm3
+
+	vunpcklpd %xmm1, %xmm0 , %xmm0
+	vunpcklpd %xmm3, %xmm2 , %xmm2
+
+	vmovups	%xmm0,  -16 * SIZE(B3)
+	vmovups	%xmm2,  -14 * SIZE(B3)
+
+	vmovsd	0 * SIZE(AO2),         %xmm0
+	vmovsd	0 * SIZE(AO2, LDA),    %xmm1
+	vmovsd	0 * SIZE(AO2, LDA, 2), %xmm2
+	vmovsd	0 * SIZE(AO2, LDA3),   %xmm3
+
+	vunpcklpd %xmm1, %xmm0 , %xmm0
+	vunpcklpd %xmm3, %xmm2 , %xmm2
+
+	vmovups	%xmm0,  -12 * SIZE(B3)
+	vmovups	%xmm2,  -10 * SIZE(B3)
+
+	subq	$-8 * SIZE, B3
+	ALIGN_4
+
+.L19:
+	cmpq	$8, N
+	jge	.L11
+	ALIGN_4
+
+.L20:
+	cmpq	$4, N
+	jl	.L30
+
+	subq	$4, N
+
+	movq	A, AO1
+	leaq	(A, LDA, 2), AO2
+	leaq	(A, LDA, 4), A
+
+	movq	B, B0
+	addq	$32 * SIZE, B
+
+	movq	M, I
+	sarq	$3, I
+	jle	.L24
+	ALIGN_4
+
+.L23:
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)
+	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B0)
+	vmovups	%xmm1,  -14 * SIZE(B0)
+	vmovups	%xmm2,  -12 * SIZE(B0)
+	vmovups	%xmm3,  -10 * SIZE(B0)
+
+
+	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO1, LDA, 1, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO1, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,   -8 * SIZE(B0)
+	vmovups	%xmm1,   -6 * SIZE(B0)
+	vmovups	%xmm2,   -4 * SIZE(B0)
+	vmovups	%xmm3,   -2 * SIZE(B0)
+
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)
+	VMOVUPS_A1(4 * SIZE, AO2, %xmm2)
+	VMOVUPS_A1(6 * SIZE, AO2, %xmm3)
+
+	vmovups	%xmm0,    0 * SIZE(B0)
+	vmovups	%xmm1,    2 * SIZE(B0)
+	vmovups	%xmm2,    4 * SIZE(B0)
+	vmovups	%xmm3,    6 * SIZE(B0)
+
+	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm0)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm1)
+	VMOVUPS_A2(4 * SIZE, AO2, LDA, 1, %xmm2)
+	VMOVUPS_A2(6 * SIZE, AO2, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,    8 * SIZE(B0)
+	vmovups	%xmm1,   10 * SIZE(B0)
+	vmovups	%xmm2,   12 * SIZE(B0)
+	vmovups	%xmm3,   14 * SIZE(B0)
+
+	addq	$8 * SIZE, AO1
+	addq	$8 * SIZE, AO2
+	leaq	(B0, M8, 8), B0
+
+	decq	I
+	jg	.L23
+	ALIGN_4
+
+.L24:
+	testq	$4, M
+	jle	.L26
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm2)
+	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B1)
+	vmovups	%xmm1,  -14 * SIZE(B1)
+	vmovups	%xmm2,  -12 * SIZE(B1)
+	vmovups	%xmm3,  -10 * SIZE(B1)
+
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm2)
+	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm3)
+
+	vmovups	%xmm0,   -8 * SIZE(B1)
+	vmovups	%xmm1,   -6 * SIZE(B1)
+	vmovups	%xmm2,   -4 * SIZE(B1)
+	vmovups	%xmm3,   -2 * SIZE(B1)
+
+	addq	$4 * SIZE, AO1
+	addq	$4 * SIZE, AO2
+	subq	$-16 * SIZE, B1
+	ALIGN_4
+
+.L26:
+	testq	$2, M
+	jle	.L28
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A2(0 * SIZE, AO1, LDA,  1, %xmm1)
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm2)
+	VMOVUPS_A2(0 * SIZE, AO2, LDA,  1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B2)
+	vmovups	%xmm1,  -14 * SIZE(B2)
+	vmovups	%xmm2,  -12 * SIZE(B2)
+	vmovups	%xmm3,  -10 * SIZE(B2)
+
+	addq	$2 * SIZE, AO1
+	addq	$2 * SIZE, AO2
+	subq	$-8 * SIZE, B2
+	ALIGN_4
+
+.L28:
+	testq	$1, M
+	jle	.L30
+
+	vmovsd	0 * SIZE(AO1),      %xmm0
+	vmovsd	0 * SIZE(AO1, LDA), %xmm1
+	vmovsd	0 * SIZE(AO2),      %xmm2
+	vmovsd	0 * SIZE(AO2, LDA), %xmm3
+
+	vunpcklpd %xmm1, %xmm0, %xmm0
+	vunpcklpd %xmm3, %xmm2, %xmm2
+
+	vmovups	%xmm0,  -16 * SIZE(B3)
+	vmovups	%xmm2,  -14 * SIZE(B3)
+	subq	$-4 * SIZE, B3
+	ALIGN_4
+
+.L30:
+	cmpq	$2, N
+	jl	.L40
+
+	subq	$2, N
+
+	movq	A, AO1
+	leaq	(A, LDA), AO2
+	leaq	(A, LDA, 2), A
+
+	movq	B, B0
+	addq	$16 * SIZE, B
+
+	movq	M, I
+	sarq	$3, I
+	jle	.L34
+	ALIGN_4
+
+.L33:
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)
+	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B0)
+	vmovups	%xmm1,  -14 * SIZE(B0)
+	vmovups	%xmm2,  -12 * SIZE(B0)
+	vmovups	%xmm3,  -10 * SIZE(B0)
+
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)
+	VMOVUPS_A1(4 * SIZE, AO2, %xmm2)
+	VMOVUPS_A1(6 * SIZE, AO2, %xmm3)
+
+	vmovups	%xmm0,   -8 * SIZE(B0)
+	vmovups	%xmm1,   -6 * SIZE(B0)
+	vmovups	%xmm2,   -4 * SIZE(B0)
+	vmovups	%xmm3,   -2 * SIZE(B0)
+
+	addq	$8 * SIZE, AO1
+	addq	$8 * SIZE, AO2
+	leaq	(B0, M8, 8), B0
+
+	decq	I
+	jg	.L33
+	ALIGN_4
+
+.L34:
+	testq	$4, M
+	jle	.L36
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm2)
+	VMOVUPS_A1(2 * SIZE, AO2, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B1)
+	vmovups	%xmm1,  -14 * SIZE(B1)
+	vmovups	%xmm2,  -12 * SIZE(B1)
+	vmovups	%xmm3,  -10 * SIZE(B1)
+
+	addq	$4 * SIZE, AO1
+	addq	$4 * SIZE, AO2
+	subq	$-8 * SIZE, B1
+	ALIGN_4
+
+.L36:
+	testq	$2, M
+	jle	.L38
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(0 * SIZE, AO2, %xmm1)
+
+	vmovups	%xmm0,  -16 * SIZE(B2)
+	vmovups	%xmm1,  -14 * SIZE(B2)
+
+	addq	$2 * SIZE, AO1
+	addq	$2 * SIZE, AO2
+	subq	$-4 * SIZE, B2
+	ALIGN_4
+
+.L38:
+	testq	$1, M
+	jle	.L40
+
+	vmovsd	0 * SIZE(AO1),      %xmm0
+	vmovsd	0 * SIZE(AO2),      %xmm1
+
+	vunpcklpd %xmm1, %xmm0, %xmm0
+
+	vmovups	%xmm0,  -16 * SIZE(B3)
+	subq	$-2 * SIZE, B3
+	ALIGN_4
+
+.L40:
+	cmpq	$1, N
+	jl	.L999
+
+	movq	A, AO1
+
+	movq	B, B0
+
+	movq	M, I
+	sarq	$3, I
+	jle	.L44
+	ALIGN_4
+
+.L43:
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)
+	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)
+
+	vmovups	%xmm0,  -16 * SIZE(B0)
+	vmovups	%xmm1,  -14 * SIZE(B0)
+	vmovups	%xmm2,  -12 * SIZE(B0)
+	vmovups	%xmm3,  -10 * SIZE(B0)
+
+	addq	$8 * SIZE, AO1
+	leaq	(B0, M8, 8), B0
+
+	decq	I
+	jg	.L43
+	ALIGN_4
+
+.L44:
+	testq	$4, M
+	jle	.L45
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)
+
+	vmovups	%xmm0, -16 * SIZE(B1)
+	vmovups	%xmm1, -14 * SIZE(B1)
+
+	addq	$4 * SIZE, AO1
+	subq	$-4 * SIZE, B1
+	ALIGN_4
+
+.L45:
+	testq	$2, M
+	jle	.L46
+
+	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)
+
+	vmovups	%xmm0,  -16 * SIZE(B2)
+
+	addq	$2 * SIZE, AO1
+	subq	$-2 * SIZE, B2
+	ALIGN_4
+
+.L46:
+	testq	$1, M
+	jle	.L999
+
+	vmovsd	0 * SIZE(AO1),      %xmm0
+
+	movlpd	%xmm0,  -16 * SIZE(B3)
+	jmp	.L999
+	ALIGN_4
+	
+.L999:
+	popq	%rbp
+	popq	%r12
+	popq	%r13
+	popq	%r14
+	popq	%r15
+
+#ifdef WINDOWS_ABI
+	popq	%rsi
+	popq	%rdi
+#endif
+	ret
+
+	EPILOGUE