--- /dev/null
+/***************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+#include "common.h"
+
+#if defined(BULLDOZER)
+#include "dsymv_L_microk_bulldozer-2.c"
+#elif defined(NEHALEM)
+#include "dsymv_U_microk_nehalem-2.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_4x4
+
+static void dsymv_kernel_4x4(BLASLONG from, BLASLONG to, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *tmp1, FLOAT *temp2)
+{
+ FLOAT tmp2[4] = { 0.0, 0.0, 0.0, 0.0 };
+ BLASLONG i;
+
+ for (i=from; i<to; i+=4)
+ {
+
+ y[i] += tmp1[0] * ap[0][i];
+ tmp2[0] += ap[0][i] * x[i];
+ y[i] += tmp1[1] * ap[1][i];
+ tmp2[1] += ap[1][i] * x[i];
+ y[i] += tmp1[2] * ap[2][i];
+ tmp2[2] += ap[2][i] * x[i];
+ y[i] += tmp1[3] * ap[3][i];
+ tmp2[3] += ap[3][i] * x[i];
+
+ y[i+1] += tmp1[0] * ap[0][i+1];
+ tmp2[0] += ap[0][i+1] * x[i+1];
+ y[i+1] += tmp1[1] * ap[1][i+1];
+ tmp2[1] += ap[1][i+1] * x[i+1];
+ y[i+1] += tmp1[2] * ap[2][i+1];
+ tmp2[2] += ap[2][i+1] * x[i+1];
+ y[i+1] += tmp1[3] * ap[3][i+1];
+ tmp2[3] += ap[3][i+1] * x[i+1];
+
+ y[i+2] += tmp1[0] * ap[0][i+2];
+ tmp2[0] += ap[0][i+2] * x[i+2];
+ y[i+2] += tmp1[1] * ap[1][i+2];
+ tmp2[1] += ap[1][i+2] * x[i+2];
+ y[i+2] += tmp1[2] * ap[2][i+2];
+ tmp2[2] += ap[2][i+2] * x[i+2];
+ y[i+2] += tmp1[3] * ap[3][i+2];
+ tmp2[3] += ap[3][i+2] * x[i+2];
+
+ y[i+3] += tmp1[0] * ap[0][i+3];
+ tmp2[0] += ap[0][i+3] * x[i+3];
+ y[i+3] += tmp1[1] * ap[1][i+3];
+ tmp2[1] += ap[1][i+3] * x[i+3];
+ y[i+3] += tmp1[2] * ap[2][i+3];
+ tmp2[2] += ap[2][i+3] * x[i+3];
+ y[i+3] += tmp1[3] * ap[3][i+3];
+ tmp2[3] += ap[3][i+3] * x[i+3];
+
+ }
+
+ temp2[0] += tmp2[0];
+ temp2[1] += tmp2[1];
+ temp2[2] += tmp2[2];
+ temp2[3] += tmp2[3];
+}
+
+#endif
+
+
+
+
+int CNAME(BLASLONG m, BLASLONG offset, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
+{
+ BLASLONG i;
+ BLASLONG ix,iy;
+ BLASLONG jx,jy;
+ BLASLONG j;
+ FLOAT temp1;
+ FLOAT temp2;
+ FLOAT tmp1[4];
+ FLOAT tmp2[4];
+ FLOAT *ap[4];
+
+#if 0
+ if ( m != offset )
+ printf("Symv_L: m=%d offset=%d\n",m,offset);
+#endif
+
+
+ if ( (inc_x != 1) || (inc_y != 1) )
+ {
+
+ jx = 0;
+ jy = 0;
+
+ for (j=0; j<offset; j++)
+ {
+ temp1 = alpha * x[jx];
+ temp2 = 0.0;
+ y[jy] += temp1 * a[j*lda+j];
+ iy = jy;
+ ix = jx;
+ for (i=j+1; i<m; i++)
+ {
+ ix += inc_x;
+ iy += inc_y;
+ y[iy] += temp1 * a[j*lda+i];
+ temp2 += a[j*lda+i] * x[ix];
+
+ }
+ y[jy] += alpha * temp2;
+ jx += inc_x;
+ jy += inc_y;
+ }
+ return(0);
+ }
+
+ BLASLONG offset1 = (offset/4)*4;
+
+ for (j=0; j<offset1; j+=4)
+ {
+ tmp1[0] = alpha * x[j];
+ tmp1[1] = alpha * x[j+1];
+ tmp1[2] = alpha * x[j+2];
+ tmp1[3] = alpha * x[j+3];
+ tmp2[0] = 0.0;
+ tmp2[1] = 0.0;
+ tmp2[2] = 0.0;
+ tmp2[3] = 0.0;
+ ap[0] = &a[j*lda];
+ ap[1] = ap[0] + lda;
+ ap[2] = ap[1] + lda;
+ ap[3] = ap[2] + lda;
+ y[j] += tmp1[0] * ap[0][j];
+ y[j+1] += tmp1[1] * ap[1][j+1];
+ y[j+2] += tmp1[2] * ap[2][j+2];
+ y[j+3] += tmp1[3] * ap[3][j+3];
+ BLASLONG from = j+1;
+ if ( m - from >=12 )
+ {
+ BLASLONG m2 = (m/4)*4;
+ for (i=j+1; i<j+4; i++)
+ {
+ y[i] += tmp1[0] * ap[0][i];
+ tmp2[0] += ap[0][i] * x[i];
+ }
+
+ for (i=j+2; i<j+4; i++)
+ {
+ y[i] += tmp1[1] * ap[1][i];
+ tmp2[1] += ap[1][i] * x[i];
+ }
+
+ for (i=j+3; i<j+4; i++)
+ {
+ y[i] += tmp1[2] * ap[2][i];
+ tmp2[2] += ap[2][i] * x[i];
+ }
+
+ if ( m2 > j+4 )
+ dsymv_kernel_4x4(j+4,m2,ap,x,y,tmp1,tmp2);
+
+
+ for (i=m2; i<m; i++)
+ {
+ y[i] += tmp1[0] * ap[0][i];
+ tmp2[0] += ap[0][i] * x[i];
+
+ y[i] += tmp1[1] * ap[1][i];
+ tmp2[1] += ap[1][i] * x[i];
+
+ y[i] += tmp1[2] * ap[2][i];
+ tmp2[2] += ap[2][i] * x[i];
+
+ y[i] += tmp1[3] * ap[3][i];
+ tmp2[3] += ap[3][i] * x[i];
+
+ }
+
+
+ }
+ else
+ {
+
+ for (i=j+1; i<j+4; i++)
+ {
+ y[i] += tmp1[0] * ap[0][i];
+ tmp2[0] += ap[0][i] * x[i];
+ }
+
+ for (i=j+2; i<j+4; i++)
+ {
+ y[i] += tmp1[1] * ap[1][i];
+ tmp2[1] += ap[1][i] * x[i];
+ }
+
+ for (i=j+3; i<j+4; i++)
+ {
+ y[i] += tmp1[2] * ap[2][i];
+ tmp2[2] += ap[2][i] * x[i];
+ }
+
+ for (i=j+4; i<m; i++)
+ {
+ y[i] += tmp1[0] * ap[0][i];
+ tmp2[0] += ap[0][i] * x[i];
+
+ y[i] += tmp1[1] * ap[1][i];
+ tmp2[1] += ap[1][i] * x[i];
+
+ y[i] += tmp1[2] * ap[2][i];
+ tmp2[2] += ap[2][i] * x[i];
+
+ y[i] += tmp1[3] * ap[3][i];
+ tmp2[3] += ap[3][i] * x[i];
+
+ }
+
+ }
+ y[j] += alpha * tmp2[0];
+ y[j+1] += alpha * tmp2[1];
+ y[j+2] += alpha * tmp2[2];
+ y[j+3] += alpha * tmp2[3];
+ }
+
+
+ for (j=offset1; j<offset; j++)
+ {
+ temp1 = alpha * x[j];
+ temp2 = 0.0;
+ y[j] += temp1 * a[j*lda+j];
+ BLASLONG from = j+1;
+ if ( m - from >=8 )
+ {
+ BLASLONG j1 = ((from + 4)/4)*4;
+ BLASLONG j2 = (m/4)*4;
+ for (i=from; i<j1; i++)
+ {
+ y[i] += temp1 * a[j*lda+i];
+ temp2 += a[j*lda+i] * x[i];
+
+ }
+
+ for (i=j1; i<j2; i++)
+ {
+ y[i] += temp1 * a[j*lda+i];
+ temp2 += a[j*lda+i] * x[i];
+
+ }
+
+ for (i=j2; i<m; i++)
+ {
+ y[i] += temp1 * a[j*lda+i];
+ temp2 += a[j*lda+i] * x[i];
+
+ }
+
+ }
+ else
+ {
+ for (i=from; i<m; i++)
+ {
+ y[i] += temp1 * a[j*lda+i];
+ temp2 += a[j*lda+i] * x[i];
+
+ }
+
+ }
+ y[j] += alpha * temp2;
+ }
+ return(0);
+}
+
+
--- /dev/null
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define HAVE_KERNEL_4x4 1
+static void dsymv_kernel_4x4( BLASLONG from, BLASLONG to, FLOAT **a, FLOAT *x, FLOAT *y, FLOAT *temp1, FLOAT *temp2) __attribute__ ((noinline));
+
+static void dsymv_kernel_4x4(BLASLONG from, BLASLONG to, FLOAT **a, FLOAT *x, FLOAT *y, FLOAT *temp1, FLOAT *temp2)
+{
+
+
+ __asm__ __volatile__
+ (
+ "vxorpd %%xmm0 , %%xmm0 , %%xmm0 \n\t" // temp2[0]
+ "vxorpd %%xmm1 , %%xmm1 , %%xmm1 \n\t" // temp2[1]
+ "vxorpd %%xmm2 , %%xmm2 , %%xmm2 \n\t" // temp2[2]
+ "vxorpd %%xmm3 , %%xmm3 , %%xmm3 \n\t" // temp2[3]
+ "vmovddup (%8), %%xmm4 \n\t" // temp1[0]
+ "vmovddup 8(%8), %%xmm5 \n\t" // temp1[1]
+ "vmovddup 16(%8), %%xmm6 \n\t" // temp1[1]
+ "vmovddup 24(%8), %%xmm7 \n\t" // temp1[1]
+
+ ".align 16 \n\t"
+ ".L01LOOP%=: \n\t"
+
+ "vmovups (%4,%0,8), %%xmm12 \n\t" // 2 * a
+ "vmovups (%2,%0,8), %%xmm8 \n\t" // 2 * x
+ "vmovups (%3,%0,8), %%xmm9 \n\t" // 2 * y
+
+ "vmovups (%5,%0,8), %%xmm13 \n\t" // 2 * a
+
+ "vfmaddpd %%xmm0 , %%xmm8, %%xmm12 , %%xmm0 \n\t" // temp2 += x * a
+ "vfmaddpd %%xmm9 , %%xmm4, %%xmm12 , %%xmm9 \n\t" // y += temp1 * a
+ "vmovups (%6,%0,8), %%xmm14 \n\t" // 2 * a
+
+ "vfmaddpd %%xmm1 , %%xmm8, %%xmm13 , %%xmm1 \n\t" // temp2 += x * a
+ "vfmaddpd %%xmm9 , %%xmm5, %%xmm13 , %%xmm9 \n\t" // y += temp1 * a
+ "vmovups (%7,%0,8), %%xmm15 \n\t" // 2 * a
+
+ "vmovups 16(%3,%0,8), %%xmm11 \n\t" // 2 * y
+ "vfmaddpd %%xmm2 , %%xmm8, %%xmm14 , %%xmm2 \n\t" // temp2 += x * a
+ "vmovups 16(%4,%0,8), %%xmm12 \n\t" // 2 * a
+ "vfmaddpd %%xmm9 , %%xmm6, %%xmm14 , %%xmm9 \n\t" // y += temp1 * a
+ "vmovups 16(%2,%0,8), %%xmm10 \n\t" // 2 * x
+
+ "vfmaddpd %%xmm3 , %%xmm8, %%xmm15 , %%xmm3 \n\t" // temp2 += x * a
+ "vfmaddpd %%xmm9 , %%xmm7, %%xmm15 , %%xmm9 \n\t" // y += temp1 * a
+
+ "vmovups 16(%5,%0,8), %%xmm13 \n\t" // 2 * a
+ "vmovups 16(%6,%0,8), %%xmm14 \n\t" // 2 * a
+
+ "vfmaddpd %%xmm0 , %%xmm10, %%xmm12 , %%xmm0 \n\t" // temp2 += x * a
+ "vfmaddpd %%xmm11 , %%xmm4, %%xmm12 , %%xmm11 \n\t" // y += temp1 * a
+
+ "vmovups 16(%7,%0,8), %%xmm15 \n\t" // 2 * a
+ "vfmaddpd %%xmm1 , %%xmm10, %%xmm13 , %%xmm1 \n\t" // temp2 += x * a
+ "vfmaddpd %%xmm11 , %%xmm5, %%xmm13 , %%xmm11 \n\t" // y += temp1 * a
+
+ "vfmaddpd %%xmm2 , %%xmm10, %%xmm14 , %%xmm2 \n\t" // temp2 += x * a
+ "vfmaddpd %%xmm11 , %%xmm6, %%xmm14 , %%xmm11 \n\t" // y += temp1 * a
+
+ "vfmaddpd %%xmm3 , %%xmm10, %%xmm15 , %%xmm3 \n\t" // temp2 += x * a
+ "vfmaddpd %%xmm11 , %%xmm7, %%xmm15 , %%xmm11 \n\t" // y += temp1 * a
+ "addq $4 , %0 \n\t"
+
+ "vmovups %%xmm9 , -32(%3,%0,8) \n\t"
+ "vmovups %%xmm11 , -16(%3,%0,8) \n\t"
+
+ "cmpq %0 , %1 \n\t"
+ "jnz .L01LOOP%= \n\t"
+
+ "vmovsd (%9), %%xmm4 \n\t"
+ "vmovsd 8(%9), %%xmm5 \n\t"
+ "vmovsd 16(%9), %%xmm6 \n\t"
+ "vmovsd 24(%9), %%xmm7 \n\t"
+
+ "vhaddpd %%xmm0, %%xmm0, %%xmm0 \n\t"
+ "vhaddpd %%xmm1, %%xmm1, %%xmm1 \n\t"
+ "vhaddpd %%xmm2, %%xmm2, %%xmm2 \n\t"
+ "vhaddpd %%xmm3, %%xmm3, %%xmm3 \n\t"
+
+ "vaddsd %%xmm4, %%xmm0, %%xmm0 \n\t"
+ "vaddsd %%xmm5, %%xmm1, %%xmm1 \n\t"
+ "vaddsd %%xmm6, %%xmm2, %%xmm2 \n\t"
+ "vaddsd %%xmm7, %%xmm3, %%xmm3 \n\t"
+
+ "vmovsd %%xmm0 , (%9) \n\t" // save temp2
+ "vmovsd %%xmm1 , 8(%9) \n\t" // save temp2
+ "vmovsd %%xmm2 ,16(%9) \n\t" // save temp2
+ "vmovsd %%xmm3 ,24(%9) \n\t" // save temp2
+
+ :
+ :
+ "r" (from), // 0
+ "r" (to), // 1
+ "r" (x), // 2
+ "r" (y), // 3
+ "r" (a[0]), // 4
+ "r" (a[1]), // 5
+ "r" (a[2]), // 6
+ "r" (a[3]), // 8
+ "r" (temp1), // 8
+ "r" (temp2) // 9
+ : "cc",
+ "%xmm0", "%xmm1", "%xmm2", "%xmm3",
+ "%xmm4", "%xmm5", "%xmm6", "%xmm7",
+ "%xmm8", "%xmm9", "%xmm10", "%xmm11",
+ "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+ "memory"
+ );
+
+}
+
+
projects / platform / upstream / openblas.git / commitdiff