#endif
-#if defined(POWER8) || defined(POWER9) || defined(POWER10)
#if defined(__VEC__) || defined(__ALTIVEC__)
+#if defined(POWER8) || defined(POWER9)
#include "dasum_microk_power8.c"
+#elif defined(POWER10)
+#include "dasum_microk_power10.c"
#endif
#endif
if ( inc_x == 1 )
{
+#if defined(POWER10)
+ if ( n >= 16 )
+ {
+ BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3;
+ for (i = 0; i < align; i++) {
+ sumf += ABS(x[i]);
+ }
+ }
+ n1 = (n-i) & -16;
+ if ( n1 > 0 )
+ {
+ sumf += dasum_kernel_16(n1, &x[i]);
+ i+=n1;
+ }
+#else
n1 = n & -16;
if ( n1 > 0 )
{
sumf = dasum_kernel_16(n1, x);
i=n1;
}
+#endif
while(i < n)
{
--- /dev/null
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define HAVE_KERNEL_16 1
+
+static double dasum_kernel_16 (long n, double *x)
+{
+ double sum;
+ __vector double t0;
+ __vector double t1;
+ __vector double t2;
+ __vector double t3;
+
+ __asm__
+ (
+ "dcbt 0, %2 \n\t"
+
+ "xxlxor 32, 32, 32 \n\t"
+ "xxlxor 33, 33, 33 \n\t"
+ "xxlxor 34, 34, 34 \n\t"
+ "xxlxor 35, 35, 35 \n\t"
+ "xxlxor 36, 36, 36 \n\t"
+ "xxlxor 37, 37, 37 \n\t"
+ "xxlxor 38, 38, 38 \n\t"
+ "xxlxor 39, 39, 39 \n\t"
+
+ "lxvp 40, 0(%2) \n\t"
+ "lxvp 42, 32(%2) \n\t"
+ "lxvp 44, 64(%2) \n\t"
+ "lxvp 46, 96(%2) \n\t"
+
+ "addi %2, %2, 128 \n\t"
+
+ "addic. %1, %1, -16 \n\t"
+ "ble two%= \n\t"
+
+ ".align 5 \n"
+ "one%=: \n\t"
+
+ "xvabsdp 48, 40 \n\t"
+ "xvabsdp 49, 41 \n\t"
+ "xvabsdp 50, 42 \n\t"
+ "xvabsdp 51, 43 \n\t"
+ "lxvp 40, 0(%2) \n\t"
+
+
+ "xvabsdp %x3, 44 \n\t"
+ "xvabsdp %x4, 45 \n\t"
+ "lxvp 42, 32(%2) \n\t"
+
+
+ "xvabsdp %x5, 46 \n\t"
+ "xvabsdp %x6, 47 \n\t"
+ "lxvp 44, 64(%2) \n\t"
+
+
+ "xvadddp 32, 32, 48 \n\t"
+ "xvadddp 33, 33, 49 \n\t"
+
+ "lxvp 46, 96(%2) \n\t"
+
+ "xvadddp 34, 34, 50 \n\t"
+ "xvadddp 35, 35, 51 \n\t"
+ "addi %2, %2, 128 \n\t"
+ "xvadddp 36, 36, %x3 \n\t"
+ "xvadddp 37, 37, %x4 \n\t"
+ "addic. %1, %1, -16 \n\t"
+ "xvadddp 38, 38, %x5 \n\t"
+ "xvadddp 39, 39, %x6 \n\t"
+
+ "bgt one%= \n"
+
+ "two%=: \n\t"
+
+ "xvabsdp 48, 40 \n\t"
+ "xvabsdp 49, 41 \n\t"
+ "xvabsdp 50, 42 \n\t"
+ "xvabsdp 51, 43 \n\t"
+ "xvabsdp %x3, 44 \n\t"
+ "xvabsdp %x4, 45 \n\t"
+ "xvabsdp %x5, 46 \n\t"
+ "xvabsdp %x6, 47 \n\t"
+
+ "xvadddp 32, 32, 48 \n\t"
+ "xvadddp 33, 33, 49 \n\t"
+ "xvadddp 34, 34, 50 \n\t"
+ "xvadddp 35, 35, 51 \n\t"
+ "xvadddp 36, 36, %x3 \n\t"
+ "xvadddp 37, 37, %x4 \n\t"
+ "xvadddp 38, 38, %x5 \n\t"
+ "xvadddp 39, 39, %x6 \n\t"
+
+ "xvadddp 32, 32, 33 \n\t"
+ "xvadddp 34, 34, 35 \n\t"
+ "xvadddp 36, 36, 37 \n\t"
+ "xvadddp 38, 38, 39 \n\t"
+
+ "xvadddp 32, 32, 34 \n\t"
+ "xvadddp 36, 36, 38 \n\t"
+
+ "xvadddp 32, 32, 36 \n\t"
+
+ XXSWAPD_S(33,32)
+ "xsadddp %x0, 32, 33 \n"
+
+ "#n=%1 x=%3=%2 sum=%0\n"
+ "#t0=%x3 t1=%x4 t2=%x5 t3=%x6"
+ :
+ "=d" (sum), // 0
+ "+r" (n), // 1
+ "+b" (x), // 2
+ "=wa" (t0), // 3
+ "=wa" (t1), // 4
+ "=wa" (t2), // 5
+ "=wa" (t3) // 6
+ :
+ "m" (*x)
+ :
+ "cr0",
+ "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",
+ "vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47",
+ "vs48","vs49","vs50","vs51"
+ );
+
+ return sum;
+}
+
+
#endif
-#if defined(POWER8) || defined(POWER9) || defined(POWER10)
#if defined(__VEC__) || defined(__ALTIVEC__)
+#if defined(POWER8) || defined(POWER9)
#include "sasum_microk_power8.c"
+#elif defined(POWER10)
+#include "sasum_microk_power10.c"
#endif
#endif
if ( inc_x == 1 )
{
+#if defined(POWER10)
+ if ( n >= 32 )
+ {
+ BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 2) & 0x7;
+ for (i = 0; i < align; i++) {
+ sumf += ABS(x[i]);
+ }
+ }
+ n1 = (n-i) & -32;
+ if ( n1 > 0 )
+ {
+ sumf += sasum_kernel_32(n1, &x[i]);
+ i+=n1;
+ }
+#else
n1 = n & -32;
if ( n1 > 0 )
{
sumf = sasum_kernel_32(n1, x);
i=n1;
}
+#endif
while(i < n)
{
--- /dev/null
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+#define HAVE_KERNEL_32 1
+
+static float sasum_kernel_32 (long n, float *x)
+{
+ float sum;
+ __vector float t0;
+ __vector float t1;
+ __vector float t2;
+ __vector float t3;
+
+ __asm__
+ (
+ "dcbt 0, %2 \n\t"
+
+ "xxlxor 32, 32, 32 \n\t"
+ "xxlxor 33, 33, 33 \n\t"
+ "xxlxor 34, 34, 34 \n\t"
+ "xxlxor 35, 35, 35 \n\t"
+ "xxlxor 36, 36, 36 \n\t"
+ "xxlxor 37, 37, 37 \n\t"
+ "xxlxor 38, 38, 38 \n\t"
+ "xxlxor 39, 39, 39 \n\t"
+
+ "lxvp 40, 0(%2) \n\t"
+ "lxvp 42, 32(%2) \n\t"
+ "lxvp 44, 64(%2) \n\t"
+ "lxvp 46, 96(%2) \n\t"
+
+ "addi %2, %2, 128 \n\t"
+
+ "addic. %1, %1, -32 \n\t"
+ "ble two%= \n\t"
+
+ ".align 5 \n"
+ "one%=: \n\t"
+
+ "xvabssp 48, 40 \n\t"
+ "xvabssp 49, 41 \n\t"
+ "xvabssp 50, 42 \n\t"
+ "xvabssp 51, 43 \n\t"
+ "lxvp 40, 0(%2) \n\t"
+
+ "xvabssp %x3, 44 \n\t"
+ "xvabssp %x4, 45 \n\t"
+ "lxvp 42, 32(%2) \n\t"
+
+ "xvabssp %x5, 46 \n\t"
+ "xvabssp %x6, 47 \n\t"
+ "lxvp 44, 64(%2) \n\t"
+
+ "xvaddsp 32, 32, 48 \n\t"
+ "xvaddsp 33, 33, 49 \n\t"
+
+ "lxvp 46, 96(%2) \n\t"
+
+ "xvaddsp 34, 34, 50 \n\t"
+ "xvaddsp 35, 35, 51 \n\t"
+ "addi %2, %2, 128 \n\t"
+ "xvaddsp 36, 36, %x3 \n\t"
+ "xvaddsp 37, 37, %x4 \n\t"
+ "addic. %1, %1, -32 \n\t"
+ "xvaddsp 38, 38, %x5 \n\t"
+ "xvaddsp 39, 39, %x6 \n\t"
+
+ "bgt one%= \n"
+
+ "two%=: \n\t"
+
+ "xvabssp 48, 40 \n\t"
+ "xvabssp 49, 41 \n\t"
+ "xvabssp 50, 42 \n\t"
+ "xvabssp 51, 43 \n\t"
+ "xvabssp %x3, 44 \n\t"
+ "xvabssp %x4, 45 \n\t"
+ "xvabssp %x5, 46 \n\t"
+ "xvabssp %x6, 47 \n\t"
+
+ "xvaddsp 32, 32, 48 \n\t"
+ "xvaddsp 33, 33, 49 \n\t"
+ "xvaddsp 34, 34, 50 \n\t"
+ "xvaddsp 35, 35, 51 \n\t"
+ "xvaddsp 36, 36, %x3 \n\t"
+ "xvaddsp 37, 37, %x4 \n\t"
+ "xvaddsp 38, 38, %x5 \n\t"
+ "xvaddsp 39, 39, %x6 \n\t"
+
+ "xvaddsp 32, 32, 33 \n\t"
+ "xvaddsp 34, 34, 35 \n\t"
+ "xvaddsp 36, 36, 37 \n\t"
+ "xvaddsp 38, 38, 39 \n\t"
+
+ "xvaddsp 32, 32, 34 \n\t"
+ "xvaddsp 36, 36, 38 \n\t"
+
+ "xvaddsp 32, 32, 36 \n\t"
+
+ "xxsldwi 33, 32, 32, 2 \n\t"
+ "xvaddsp 32, 32, 33 \n\t"
+
+ "xxsldwi 33, 32, 32, 1 \n\t"
+ "xvaddsp 32, 32, 33 \n\t"
+
+ "xscvspdp %x0, 32 \n"
+
+ "#n=%1 x=%3=%2 sum=%0\n"
+ "#t0=%x3 t1=%x4 t2=%x5 t3=%x6"
+ :
+ "=f" (sum), // 0
+ "+r" (n), // 1
+ "+b" (x), // 2
+ "=wa" (t0), // 3
+ "=wa" (t1), // 4
+ "=wa" (t2), // 5
+ "=wa" (t3) // 6
+ :
+ "m" (*x)
+ :
+ "cr0",
+ "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",
+ "vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47",
+ "vs48","vs49","vs50","vs51"
+ );
+
+ return sum;
+}
projects / platform / upstream / openblas.git / commitdiff