From 8dc0c72583eb60ec5e8272049e35fd34af409a13 Mon Sep 17 00:00:00 2001 From: wernsaar Date: Thu, 20 Jun 2013 14:07:54 +0200 Subject: [PATCH] added daxpy_bulldozer.S --- kernel/x86_64/KERNEL.BULLDOZER | 1 + kernel/x86_64/daxpy_bulldozer.S | 408 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 409 insertions(+) create mode 100644 kernel/x86_64/daxpy_bulldozer.S diff --git a/kernel/x86_64/KERNEL.BULLDOZER b/kernel/x86_64/KERNEL.BULLDOZER index f481f33..2afeec0 100644 --- a/kernel/x86_64/KERNEL.BULLDOZER +++ b/kernel/x86_64/KERNEL.BULLDOZER @@ -3,6 +3,7 @@ ZGEMVTKERNEL = zgemv_t_dup.S DGEMVNKERNEL = dgemv_n_bulldozer.S DGEMVTKERNEL = dgemv_t_bulldozer.S +DAXPYKERNEL = daxpy_bulldozer.S SGEMMKERNEL = sgemm_kernel_16x2_bulldozer.S SGEMMINCOPY = ../generic/gemm_ncopy_16.c diff --git a/kernel/x86_64/daxpy_bulldozer.S b/kernel/x86_64/daxpy_bulldozer.S new file mode 100644 index 0000000..dfc10e8 --- /dev/null +++ b/kernel/x86_64/daxpy_bulldozer.S @@ -0,0 +1,408 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#define ASSEMBLER +#include "common.h" + +#ifndef WINDOWS_ABI +#define M ARG1 +#define X ARG4 +#define INCX ARG5 +#define Y ARG6 +#define INCY ARG2 +#else +#define M ARG1 +#define X ARG2 +#define INCX ARG3 +#define Y ARG4 +#define INCY %r10 +#endif + +#define YY %r11 +#define ALPHA %xmm15 + +#define A_PRE 640 + +#include "l1param.h" + + PROLOGUE + PROFCODE + +#ifndef WINDOWS_ABI +#ifndef XDOUBLE + movq 8(%rsp), INCY +#else + movq 24(%rsp), INCY +#endif + vmovups %xmm0, ALPHA +#else + vmovups %xmm3, ALPHA + + movq 40(%rsp), X + movq 48(%rsp), INCX + movq 56(%rsp), Y + movq 64(%rsp), INCY +#endif + + SAVEREGISTERS + + unpcklpd ALPHA, ALPHA + + leaq (, INCX, SIZE), INCX + leaq (, INCY, SIZE), INCY + + testq M, M + jle .L47 + + cmpq $SIZE, INCX + jne .L40 + cmpq $SIZE, INCY + jne .L40 + + testq $SIZE, Y + je .L10 + + movsd (X), %xmm0 + mulsd ALPHA, %xmm0 + addsd (Y), %xmm0 + movsd %xmm0, (Y) + addq $1 * SIZE, X + addq $1 * SIZE, Y + decq M + jle .L19 + ALIGN_4 + +.L10: + subq $-16 * SIZE, X + subq $-16 * SIZE, Y + + movq M, %rax + sarq $4, %rax + jle .L13 + + vmovups -16 * SIZE(X), %xmm0 + vmovups -14 * SIZE(X), %xmm1 + vmovups -12 * SIZE(X), %xmm2 + vmovups -10 * SIZE(X), %xmm3 + + decq %rax + jle .L12 + ALIGN_3 + +.L11: + + prefetchnta A_PRE(Y) + + vmovups -8 * SIZE(X), %xmm4 + vfmaddpd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 + vfmaddpd -14 * SIZE(Y), ALPHA, %xmm1 , %xmm1 + vmovups -6 * SIZE(X), %xmm5 + vmovups -4 * SIZE(X), %xmm6 + vfmaddpd -12 * SIZE(Y), ALPHA, %xmm2 , %xmm2 + vfmaddpd -10 * SIZE(Y), ALPHA, %xmm3 , %xmm3 + vmovups -2 * SIZE(X), %xmm7 + + + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + prefetchnta A_PRE(X) + nop + vmovups %xmm2, -12 * SIZE(Y) + vmovups %xmm3, -10 * SIZE(Y) + + prefetchnta A_PRE+64(Y) + + vmovups 0 * SIZE(X), %xmm0 + vfmaddpd -8 * SIZE(Y), ALPHA, %xmm4 , %xmm4 + vfmaddpd -6 * SIZE(Y), ALPHA, %xmm5 , %xmm5 + vmovups 2 * SIZE(X), %xmm1 + vmovups 4 * SIZE(X), %xmm2 + vfmaddpd -4 * SIZE(Y), ALPHA, %xmm6 , %xmm6 + vfmaddpd -2 * SIZE(Y), ALPHA, %xmm7 , %xmm7 + vmovups 6 * SIZE(X), %xmm3 + + + vmovups %xmm4, -8 * SIZE(Y) + vmovups %xmm5, -6 * SIZE(Y) + prefetchnta A_PRE+64(X) + nop + vmovups %xmm6, -4 * SIZE(Y) + vmovups %xmm7, -2 * SIZE(Y) + + subq $-16 * SIZE, Y + subq $-16 * SIZE, X + decq %rax + jg .L11 + ALIGN_3 + +.L12: + + vmovups -8 * SIZE(X), %xmm4 + vfmaddpd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 + vfmaddpd -14 * SIZE(Y), ALPHA, %xmm1 , %xmm1 + vmovups -6 * SIZE(X), %xmm5 + vmovups -4 * SIZE(X), %xmm6 + vfmaddpd -12 * SIZE(Y), ALPHA, %xmm2 , %xmm2 + vfmaddpd -10 * SIZE(Y), ALPHA, %xmm3 , %xmm3 + vmovups -2 * SIZE(X), %xmm7 + + + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + vmovups %xmm2, -12 * SIZE(Y) + vmovups %xmm3, -10 * SIZE(Y) + + vfmaddpd -8 * SIZE(Y), ALPHA, %xmm4 , %xmm4 + vfmaddpd -6 * SIZE(Y), ALPHA, %xmm5 , %xmm5 + vfmaddpd -4 * SIZE(Y), ALPHA, %xmm6 , %xmm6 + vfmaddpd -2 * SIZE(Y), ALPHA, %xmm7 , %xmm7 + + vmovups %xmm4, -8 * SIZE(Y) + vmovups %xmm5, -6 * SIZE(Y) + vmovups %xmm6, -4 * SIZE(Y) + vmovups %xmm7, -2 * SIZE(Y) + + subq $-16 * SIZE, Y + subq $-16 * SIZE, X + ALIGN_3 + +.L13: + + + movq M, %rax + andq $8, %rax + jle .L14 + ALIGN_3 + + vmovups -16 * SIZE(X), %xmm0 + vmovups -14 * SIZE(X), %xmm1 + vmovups -12 * SIZE(X), %xmm2 + vmovups -10 * SIZE(X), %xmm3 + + vfmaddpd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 + vfmaddpd -14 * SIZE(Y), ALPHA, %xmm1 , %xmm1 + vfmaddpd -12 * SIZE(Y), ALPHA, %xmm2 , %xmm2 + vfmaddpd -10 * SIZE(Y), ALPHA, %xmm3 , %xmm3 + + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + vmovups %xmm2, -12 * SIZE(Y) + vmovups %xmm3, -10 * SIZE(Y) + + addq $8 * SIZE, X + addq $8 * SIZE, Y + ALIGN_3 + +.L14: + movq M, %rax + andq $4, %rax + jle .L15 + ALIGN_3 + + vmovups -16 * SIZE(X), %xmm0 + vmovups -14 * SIZE(X), %xmm1 + + vfmaddpd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 + vfmaddpd -14 * SIZE(Y), ALPHA, %xmm1 , %xmm1 + + vmovups %xmm0, -16 * SIZE(Y) + vmovups %xmm1, -14 * SIZE(Y) + + addq $4 * SIZE, X + addq $4 * SIZE, Y + ALIGN_3 + +.L15: + movq M, %rax + andq $2, %rax + jle .L16 + ALIGN_3 + + vmovups -16 * SIZE(X), %xmm0 + vfmaddpd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 + vmovups %xmm0, -16 * SIZE(Y) + + addq $2 * SIZE, X + addq $2 * SIZE, Y + ALIGN_3 + +.L16: + movq M, %rax + andq $1, %rax + jle .L19 + ALIGN_3 + + vmovsd -16 * SIZE(X), %xmm0 + vfmaddsd -16 * SIZE(Y), ALPHA, %xmm0 , %xmm0 + + vmovsd %xmm0, -16 * SIZE(Y) + ALIGN_3 + +.L19: + xorq %rax,%rax + + RESTOREREGISTERS + + ret + ALIGN_3 + + +.L40: + movq Y, YY + movq M, %rax +//If incx==0 || incy==0, avoid unloop. + cmpq $0, INCX + je .L46 + cmpq $0, INCY + je .L46 + + sarq $3, %rax + jle .L45 + + prefetchnta 512(X) + prefetchnta 512+64(X) + prefetchnta 512+128(X) + prefetchnta 512+192(X) + + prefetchnta 512(Y) + prefetchnta 512+64(Y) + prefetchnta 512+128(Y) + prefetchnta 512+192(Y) + ALIGN_3 + +.L41: + + vmovsd 0 * SIZE(X), %xmm0 + addq INCX, X + vmovhpd 0 * SIZE(X), %xmm0 , %xmm0 + addq INCX, X + + vmovsd 0 * SIZE(YY), %xmm6 + addq INCY, YY + vmovhpd 0 * SIZE(YY), %xmm6 , %xmm6 + addq INCY, YY + + + vmovsd 0 * SIZE(X), %xmm1 + addq INCX, X + vmovhpd 0 * SIZE(X), %xmm1 , %xmm1 + addq INCX, X + + vmovsd 0 * SIZE(YY), %xmm7 + addq INCY, YY + vmovhpd 0 * SIZE(YY), %xmm7 , %xmm7 + addq INCY, YY + + vfmaddpd %xmm6 , ALPHA , %xmm0 , %xmm0 + + vmovsd 0 * SIZE(X), %xmm2 + addq INCX, X + vmovhpd 0 * SIZE(X), %xmm2 , %xmm2 + addq INCX, X + + vmovsd 0 * SIZE(YY), %xmm8 + addq INCY, YY + vmovhpd 0 * SIZE(YY), %xmm8 , %xmm8 + addq INCY, YY + + vfmaddpd %xmm7 , ALPHA , %xmm1 , %xmm1 + + vmovsd 0 * SIZE(X), %xmm3 + addq INCX, X + vmovhpd 0 * SIZE(X), %xmm3 , %xmm3 + addq INCX, X + + vfmaddpd %xmm8 , ALPHA , %xmm2 , %xmm2 + + vmovsd 0 * SIZE(YY), %xmm9 + addq INCY, YY + vmovhpd 0 * SIZE(YY), %xmm9 , %xmm9 + addq INCY, YY + + + vmovsd %xmm0, 0 * SIZE(Y) + addq INCY, Y + vmovhpd %xmm0, 0 * SIZE(Y) + addq INCY, Y + vmovsd %xmm1, 0 * SIZE(Y) + addq INCY, Y + vmovhpd %xmm1, 0 * SIZE(Y) + addq INCY, Y + vmovsd %xmm2, 0 * SIZE(Y) + addq INCY, Y + vmovhpd %xmm2, 0 * SIZE(Y) + addq INCY, Y + + vfmaddpd %xmm9 , ALPHA , %xmm3 , %xmm3 + + vmovsd %xmm3, 0 * SIZE(Y) + addq INCY, Y + vmovhpd %xmm3, 0 * SIZE(Y) + addq INCY, Y + + decq %rax + jg .L41 + ALIGN_3 + +.L45: + movq M, %rax + andq $7, %rax + jle .L47 + ALIGN_3 + +.L46: + vmovsd (X), %xmm0 + addq INCX, X + + vfmaddsd (Y) , ALPHA , %xmm0 , %xmm0 + + vmovsd %xmm0, (Y) + addq INCY, Y + + decq %rax + jg .L46 + ALIGN_3 + +.L47: + xorq %rax, %rax + + RESTOREREGISTERS + + ret + + EPILOGUE -- 2.7.4