--- /dev/null
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+#include "common.h"
+
+#if defined(BULLDOZER) || defined(PILEDRIVER)
+#include "sgemv_t_microk_bulldozer.c"
+#endif
+
+static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src)
+{
+ BLASLONG i;
+ for ( i=0; i<n; i++ )
+ {
+ *dest = *src;
+ dest++;
+ src += inc_src;
+ }
+}
+
+static void sgemv_kernel_1( BLASLONG n, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, FLOAT *y)
+{
+
+ FLOAT register temp0 = 0.0;
+ BLASLONG i;
+ for ( i=0; i<n ; i++)
+ {
+ temp0 += a[i] * x[i];
+ }
+ temp0 *= alpha ;
+ *y += temp0;
+}
+
+
+
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
+{
+ BLASLONG i;
+ BLASLONG j;
+ FLOAT *a_ptr;
+ FLOAT *x_ptr;
+ FLOAT *y_ptr;
+ FLOAT *a_ptrl;
+ BLASLONG m1;
+ BLASLONG register m2;
+ FLOAT *xbuffer;
+ xbuffer = buffer;
+ BLASLONG register Mblock;
+
+ m1 = m / 1024 ;
+ m2 = m % 1024 ;
+
+ x_ptr = x;
+ a_ptr = a;
+
+ for (j=0; j<m1; j++)
+ {
+
+ if ( inc_x == 1 )
+ xbuffer = x_ptr;
+ else
+ copy_x(1024,x_ptr,xbuffer,inc_x);
+
+ y_ptr = y;
+ a_ptrl = a_ptr;
+
+ for(i = 0; i<n; i++ )
+ {
+ sgemv_kernel_16(1024,alpha,a_ptrl,lda,xbuffer,y_ptr);
+ y_ptr += inc_y;
+ a_ptrl += lda;
+ }
+ a_ptr += 1024;
+ x_ptr += 1024 * inc_x;
+ }
+
+ if ( m2 == 0 ) return(0);
+
+ Mblock = 512;
+ while ( Mblock >= 16 )
+ {
+ if ( m2 & Mblock)
+ {
+
+ if ( inc_x == 1 )
+ xbuffer = x_ptr;
+ else
+ copy_x(Mblock,x_ptr,xbuffer,inc_x);
+
+ y_ptr = y;
+ a_ptrl = a_ptr;
+
+ for(i = 0; i<n; i++ )
+ {
+ sgemv_kernel_16(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
+ y_ptr += inc_y;
+ a_ptrl += lda;
+ }
+ a_ptr += Mblock;
+ x_ptr += Mblock * inc_x;
+
+
+ }
+ Mblock /= 2;
+
+ }
+
+ if ( m2 & Mblock)
+ {
+
+ if ( inc_x == 1 )
+ xbuffer = x_ptr;
+ else
+ copy_x(Mblock,x_ptr,xbuffer,inc_x);
+
+ y_ptr = y;
+ a_ptrl = a_ptr;
+
+ for(i = 0; i<n; i++ )
+ {
+ sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
+ y_ptr += inc_y;
+ a_ptrl += lda;
+ }
+ a_ptr += Mblock;
+ x_ptr += Mblock * inc_x;
+
+
+ }
+ Mblock /= 2;
+
+
+ if ( m2 & Mblock)
+ {
+
+ if ( inc_x == 1 )
+ xbuffer = x_ptr;
+ else
+ copy_x(Mblock,x_ptr,xbuffer,inc_x);
+
+ y_ptr = y;
+ a_ptrl = a_ptr;
+
+ for(i = 0; i<n; i++ )
+ {
+ sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
+ y_ptr += inc_y;
+ a_ptrl += lda;
+ }
+ a_ptr += Mblock;
+ x_ptr += Mblock * inc_x;
+
+
+ }
+ Mblock /= 2;
+
+ if ( m2 & Mblock)
+ {
+
+ if ( inc_x == 1 )
+ xbuffer = x_ptr;
+ else
+ copy_x(Mblock,x_ptr,xbuffer,inc_x);
+
+ y_ptr = y;
+ a_ptrl = a_ptr;
+
+ for(i = 0; i<n; i++ )
+ {
+ sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
+ y_ptr += inc_y;
+ a_ptrl += lda;
+ }
+ a_ptr += Mblock;
+ x_ptr += Mblock * inc_x;
+
+
+ }
+ Mblock /= 2;
+
+ if ( m2 & Mblock)
+ {
+
+ xbuffer = x_ptr;
+
+ y_ptr = y;
+ a_ptrl = a_ptr;
+
+ for(i = 0; i<n; i++ )
+ {
+ sgemv_kernel_1(Mblock,alpha,a_ptrl,lda,xbuffer,y_ptr);
+ y_ptr += inc_y;
+ a_ptrl += lda;
+ }
+
+
+ }
+
+ return(0);
+}
+
+
--- /dev/null
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+static void sgemv_kernel_16( long n, float alpha, float *a, long lda, float *x, float *y)
+{
+
+ //n = n / 16;
+
+ __asm__ __volatile__
+ (
+ "movq %0, %%rax\n\t" // n -> rax
+ "vmovss %1, %%xmm1\n\t" // alpha -> xmm1
+ "movq %2, %%rsi\n\t" // adress of a -> rsi
+ "movq %3, %%rcx\n\t" // value of lda > rcx
+ "movq %4, %%rdi\n\t" // adress of x -> rdi
+ "movq %5, %%rdx\n\t" // adress of y -> rdx
+
+ "leaq (, %%rcx,4), %%rcx \n\t" // scale lda by size of float
+ "leaq (%%rsi,%%rcx,1), %%r8 \n\t" // pointer to next line
+
+ "vxorps %%xmm12, %%xmm12, %%xmm12\n\t" // set to zero
+ "vxorps %%xmm13, %%xmm13, %%xmm13\n\t" // set to zero
+ "vxorps %%xmm14, %%xmm14, %%xmm14\n\t" // set to zero
+ "vxorps %%xmm15, %%xmm15, %%xmm15\n\t" // set to zero
+
+ "sarq $4, %%rax \n\t" // n = n / 16
+
+ ".align 16 \n\t"
+ ".L01LOOP%=: \n\t"
+ // "prefetcht0 512(%%rsi) \n\t"
+ "prefetcht0 (%%r8) \n\t" //prefetch next line of a
+ "vmovups (%%rsi), %%xmm4 \n\t"
+ "vmovups 4*4(%%rsi), %%xmm5 \n\t"
+ "vmovups 8*4(%%rsi), %%xmm6 \n\t"
+ "vmovups 12*4(%%rsi), %%xmm7 \n\t"
+
+ "vfmaddps %%xmm12, 0*4(%%rdi), %%xmm4, %%xmm12\n\t" // multiply a and c and add to temp
+ "vfmaddps %%xmm13, 4*4(%%rdi), %%xmm5, %%xmm13\n\t" // multiply a and c and add to temp
+ "vfmaddps %%xmm14, 8*4(%%rdi), %%xmm6, %%xmm14\n\t" // multiply a and c and add to temp
+ "vfmaddps %%xmm15, 12*4(%%rdi), %%xmm7, %%xmm15\n\t" // multiply a and c and add to temp
+
+ "addq $16*4 , %%r8 \n\t" // increment prefetch pointer
+ "addq $16*4 , %%rsi \n\t" // increment pointer of a
+ "addq $16*4 , %%rdi \n\t" // increment pointer of c
+ "dec %%rax \n\t" // n = n -1
+ "jnz .L01LOOP%= \n\t"
+
+ "vaddps %%xmm12, %%xmm14, %%xmm12\n\t"
+ "vaddps %%xmm13, %%xmm15, %%xmm13\n\t"
+ "vaddps %%xmm12, %%xmm13, %%xmm12\n\t"
+ "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
+ "vhaddps %%xmm12, %%xmm12, %%xmm12\n\t"
+
+ "vfmaddss (%%rdx), %%xmm12, %%xmm1, %%xmm12\n\t"
+ "vmovss %%xmm12, (%%rdx) \n\t" // store temp -> y
+
+ :
+ :
+ "m" (n), // 0
+ "m" (alpha), // 1
+ "m" (a), // 2
+ "m" (lda), // 3
+ "m" (x), // 4
+ "m" (y) // 5
+ : "%rax", "%rcx", "%rdx", "%rsi", "%rdi", "%r8",
+ "%xmm0", "%xmm1",
+ "%xmm4", "%xmm5", "%xmm6", "%xmm7",
+ "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+ "memory"
+ );
+
+}
+
+
+
projects / platform / upstream / openblas.git / commitdiff