From: wernsaar Date: Fri, 21 Jun 2013 14:06:51 +0000 (+0200) Subject: added dcopy_bulldozer.S X-Git-Tag: v0.2.9.rc1~80^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=16012767f4f3b7595fb6039e56c85b325774b426;p=platform%2Fupstream%2Fopenblas.git added dcopy_bulldozer.S --- diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER index 5dd6f6d..8ebd422 100644 --- a/kernel/x86_64/KERNEL.BULLDOZER +++ b/kernel/x86_64/KERNEL.BULLDOZER @@ -3,8 +3,9 @@ ZGEMVTKERNEL = zgemv_t_dup.S DGEMVNKERNEL = dgemv_n_bulldozer.S DGEMVTKERNEL = dgemv_t_bulldozer.S -DAXPYKERNEL = daxpy_bulldozer.S -DDOTKERNEL = ddot_bulldozer.S +DAXPYKERNEL = daxpy_bulldozer.S +DDOTKERNEL = ddot_bulldozer.S +DCOPYKERNEL = dcopy_bulldozer.S SGEMMKERNEL = sgemm_kernel_16x2_bulldozer.S SGEMMINCOPY = ../generic/gemm_ncopy_16.c diff --git a/kernel/x86_64/dcopy_bulldozer.S b/kernel/x86_64/dcopy_bulldozer.S new file mode 100644 index 0000000..87f1a4e --- /dev/null +++ b/kernel/x86_64/dcopy_bulldozer.S @@ -0,0 +1,291 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#define M ARG1 /* rdi */ +#define X ARG2 /* rsi */ +#define INCX ARG3 /* rdx */ +#define Y ARG4 /* rcx */ +#ifndef WINDOWS_ABI +#define INCY ARG5 /* r8 */ +#else +#define INCY %r10 +#endif + +#include "l1param.h" + +#define VLOAD(OFFSET, ADDR, REG) vmovups OFFSET(ADDR), REG +#define VSHUFPD_1(REG1 , REG2) vshufpd $0x01, REG1, REG2, REG2 +#define A_PRE 640 +#define B_PRE 640 + + PROLOGUE + PROFCODE + +#ifdef WINDOWS_ABI + movq 40(%rsp), INCY +#endif + + SAVEREGISTERS + + leaq (, INCX, SIZE), INCX + leaq (, INCY, SIZE), INCY + + cmpq $SIZE, INCX + jne .L40 + cmpq $SIZE, INCY + jne .L40 + + testq $SIZE, X + je .L10 + + vmovsd (X), %xmm0 + vmovsd %xmm0, (Y) + addq $1 * SIZE, X + addq $1 * SIZE, Y + decq M + jle .L19 + ALIGN_4 + +.L10: + subq $-16 * SIZE, X + subq $-16 * SIZE, Y + + + movq M, %rax + sarq $4, %rax + jle .L13 + + vmovups -16 * SIZE(X), %xmm0 + vmovups -14 * SIZE(X), %xmm1 + vmovups -12 * SIZE(X), %xmm2 + vmovups -10 * SIZE(X), %xmm3 + vmovups -8 * SIZE(X), %xmm4 + vmovups -6 * SIZE(X), %xmm5 + vmovups -4 * SIZE(X), %xmm6 + vmovups -2 * SIZE(X), %xmm7 + + decq %rax + jle .L12 + ALIGN_4 + +.L11: + + prefetchnta A_PRE(X) + nop + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + prefetchnta B_PRE(Y) + nop + vmovups %xmm2, -12 * SIZE(Y) + vmovups %xmm3, -10 * SIZE(Y) + + VLOAD( 0 * SIZE, X, %xmm0) + VLOAD( 2 * SIZE, X, %xmm1) + VLOAD( 4 * SIZE, X, %xmm2) + VLOAD( 6 * SIZE, X, %xmm3) + + prefetchnta A_PRE+64(X) + nop + vmovups %xmm4, -8 * SIZE(Y) + vmovups %xmm5, -6 * SIZE(Y) + prefetchnta B_PRE+64(Y) + nop + vmovups %xmm6, -4 * SIZE(Y) + vmovups %xmm7, -2 * SIZE(Y) + + VLOAD( 8 * SIZE, X, %xmm4) + VLOAD(10 * SIZE, X, %xmm5) + subq $-16 * SIZE, Y + VLOAD(12 * SIZE, X, %xmm6) + VLOAD(14 * SIZE, X, %xmm7) + + subq $-16 * SIZE, X + decq %rax + jg .L11 + ALIGN_3 + +.L12: + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + vmovups %xmm2, -12 * SIZE(Y) + vmovups %xmm3, -10 * SIZE(Y) + vmovups %xmm4, -8 * SIZE(Y) + vmovups %xmm5, -6 * SIZE(Y) + vmovups %xmm6, -4 * SIZE(Y) + vmovups %xmm7, -2 * SIZE(Y) + + subq $-16 * SIZE, Y + subq $-16 * SIZE, X + ALIGN_3 + +.L13: + testq $8, M + jle .L14 + ALIGN_3 + + vmovups -16 * SIZE(X), %xmm0 + vmovups -14 * SIZE(X), %xmm1 + vmovups -12 * SIZE(X), %xmm2 + vmovups -10 * SIZE(X), %xmm3 + + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + vmovups %xmm2, -12 * SIZE(Y) + vmovups %xmm3, -10 * SIZE(Y) + + addq $8 * SIZE, X + addq $8 * SIZE, Y + ALIGN_3 + +.L14: + testq $4, M + jle .L15 + ALIGN_3 + + vmovups -16 * SIZE(X), %xmm0 + vmovups -14 * SIZE(X), %xmm1 + + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + + addq $4 * SIZE, X + addq $4 * SIZE, Y + ALIGN_3 + +.L15: + testq $2, M + jle .L16 + ALIGN_3 + + vmovups -16 * SIZE(X), %xmm0 + vmovups %xmm0, -16 * SIZE(Y) + + addq $2 * SIZE, X + addq $2 * SIZE, Y + ALIGN_3 + +.L16: + testq $1, M + jle .L19 + ALIGN_3 + + vmovsd -16 * SIZE(X), %xmm0 + vmovsd %xmm0, -16 * SIZE(Y) + ALIGN_3 + +.L19: + xorq %rax,%rax + + RESTOREREGISTERS + + ret + ALIGN_3 + + + +.L40: + movq M, %rax + sarq $3, %rax + jle .L45 + ALIGN_3 + +.L41: + vmovsd (X), %xmm0 + addq INCX, X + vmovsd (X), %xmm4 + addq INCX, X + vmovsd (X), %xmm1 + addq INCX, X + vmovsd (X), %xmm5 + addq INCX, X + vmovsd (X), %xmm2 + addq INCX, X + vmovsd (X), %xmm6 + addq INCX, X + vmovsd (X), %xmm3 + addq INCX, X + vmovsd (X), %xmm7 + addq INCX, X + + vmovsd %xmm0, (Y) + addq INCY, Y + vmovsd %xmm4, (Y) + addq INCY, Y + vmovsd %xmm1, (Y) + addq INCY, Y + vmovsd %xmm5, (Y) + addq INCY, Y + vmovsd %xmm2, (Y) + addq INCY, Y + vmovsd %xmm6, (Y) + addq INCY, Y + vmovsd %xmm3, (Y) + addq INCY, Y + vmovsd %xmm7, (Y) + addq INCY, Y + + decq %rax + jg .L41 + ALIGN_3 + +.L45: + movq M, %rax + andq $7, %rax + jle .L47 + ALIGN_3 + +.L46: + vmovsd (X), %xmm0 + addq INCX, X + vmovsd %xmm0, (Y) + addq INCY, Y + decq %rax + jg .L46 + ALIGN_3 + +.L47: + xorq %rax, %rax + + RESTOREREGISTERS + + ret + + EPILOGUE