#include "common.h"
-#if defined(POWER8) || defined(POWER9) || defined(POWER10)
#if defined(__VEC__) || defined(__ALTIVEC__)
+#if defined(POWER8) || defined(POWER9)
#include "dscal_microk_power8.c"
+#elif defined(POWER10)
+#include "dscal_microk_power10.c"
#endif
#endif
if ( da == 0.0 )
{
+#if defined(POWER10)
+ if ( n >= 16 )
+ {
+ BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3;
+ for (j = 0; j < align; j++) {
+ x[j] = 0.0;
+ }
+ }
+ BLASLONG n1 = (n-j) & -16;
+ if ( n1 > 0 )
+ {
+ dscal_kernel_8_zero(n1, &x[j]);
+ j+=n1;
+ }
+#else
BLASLONG n1 = n & -16;
if ( n1 > 0 )
{
dscal_kernel_8_zero(n1, x);
j=n1;
}
+#endif
while(j < n)
{
else
{
+#if defined(POWER10)
+ if ( n >= 16 )
+ {
+ BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3;
+ for (j = 0; j < align; j++) {
+ x[j] = da * x[j];
+ }
+ }
+ BLASLONG n1 = (n-j) & -16;
+ if ( n1 > 0 )
+ {
+ dscal_kernel_8(n1, &x[j], da);
+ j+=n1;
+ }
+#else
BLASLONG n1 = n & -16;
if ( n1 > 0 )
{
dscal_kernel_8(n1, x, da);
j=n1;
}
+#endif
while(j < n)
{
--- /dev/null
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define HAVE_KERNEL_8 1
+
+static void dscal_kernel_8 (long n, double *x, double alpha)
+{
+ __asm__
+ (
+ "dcbt 0, %2 \n\t"
+
+ XXSPLTD_S(48,%x3,0)
+
+ "lxvp 32, 0(%2) \n\t"
+ "lxvp 34, 32(%2) \n\t"
+ "lxvp 36, 64(%2) \n\t"
+ "lxvp 38, 96(%2) \n\t"
+
+ "addic. %1, %1, -16 \n\t"
+ "ble two%= \n\t"
+
+ ".align 5 \n"
+ "one%=: \n\t"
+
+ "xvmuldp 40, 32, 48 \n\t"
+ "xvmuldp 41, 33, 48 \n\t"
+ "xvmuldp 42, 34, 48 \n\t"
+ "xvmuldp 43, 35, 48 \n\t"
+ "lxvp 32, 128(%2) \n\t"
+ "lxvp 34, 160(%2) \n\t"
+ "xvmuldp 44, 36, 48 \n\t"
+ "xvmuldp 45, 37, 48 \n\t"
+ "xvmuldp 46, 38, 48 \n\t"
+ "xvmuldp 47, 39, 48 \n\t"
+ "lxvp 36, 192(%2) \n\t"
+ "lxvp 38, 224(%2) \n\t"
+
+ "stxvp 40, 0(%2) \n\t"
+ "stxvp 42, 32(%2) \n\t"
+ "stxvp 44, 64(%2) \n\t"
+ "stxvp 46, 96(%2) \n\t"
+
+ "addi %2, %2, 128 \n\t"
+
+ "addic. %1, %1, -16 \n\t"
+ "bgt one%= \n"
+
+ "two%=: \n\t"
+
+ "xvmuldp 40, 32, 48 \n\t"
+ "xvmuldp 41, 33, 48 \n\t"
+ "xvmuldp 42, 34, 48 \n\t"
+ "xvmuldp 43, 35, 48 \n\t"
+
+ "xvmuldp 44, 36, 48 \n\t"
+ "xvmuldp 45, 37, 48 \n\t"
+ "xvmuldp 46, 38, 48 \n\t"
+ "xvmuldp 47, 39, 48 \n\t"
+
+ "stxvp 40, 0(%2) \n\t"
+ "stxvp 42, 32(%2) \n\t"
+ "stxvp 44, 64(%2) \n\t"
+ "stxvp 46, 96(%2) \n\t"
+
+ "#n=%1 alpha=%3 x=%0=%2"
+ :
+ "+m" (*x),
+ "+r" (n), // 1
+ "+b" (x) // 2
+ :
+ "d" (alpha) // 3
+ :
+ "cr0",
+ "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",
+ "vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47","vs48"
+ );
+}
+
+
+static void dscal_kernel_8_zero (long n, double *x)
+{
+
+ __asm__
+ (
+ "xxlxor 32, 32, 32 \n\t"
+ "xxlxor 33, 33, 33 \n\t"
+
+ ".align 5 \n"
+ "one%=: \n\t"
+
+ "stxvp 32, 0(%2) \n\t"
+ "stxvp 32, 32(%2) \n\t"
+ "stxvp 32, 64(%2) \n\t"
+ "stxvp 32, 96(%2) \n\t"
+
+ "addi %2, %2, 128 \n\t"
+
+ "addic. %1, %1, -16 \n\t"
+ "bgt one%= \n"
+
+ "#n=%1 x=%0=%2 "
+ :
+ "=m" (*x),
+ "+r" (n), // 1
+ "+b" (x) // 2
+ :
+ :
+ "cr0","vs32","vs33"
+ );
+}
#include "common.h"
-#if defined(POWER8) || defined(POWER9) || defined(POWER10)
#if defined(__VEC__) || defined(__ALTIVEC__)
+#if defined(POWER8) || defined(POWER9)
#include "sscal_microk_power8.c"
+#elif defined(POWER10)
+#include "sscal_microk_power10.c"
#endif
#endif
if ( da == 0.0 )
{
+#if defined(POWER10)
+ if ( n >= 32 )
+ {
+ BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 2) & 0x7;
+ for (j = 0; j < align; j++) {
+ x[j] = 0.0;
+ }
+ }
+ BLASLONG n1 = (n-j) & -32;
+ if ( n1 > 0 )
+ {
+ sscal_kernel_16_zero(n1, &x[j]);
+ j+=n1;
+ }
+#else
BLASLONG n1 = n & -32;
if ( n1 > 0 )
{
sscal_kernel_16_zero(n1, x);
j=n1;
}
+#endif
while(j < n)
{
else
{
+#if defined(POWER10)
+ if ( n >= 32 )
+ {
+ BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 2) & 0x7;
+ for (j = 0; j < align; j++) {
+ x[j] = da * x[j];
+ }
+ }
+ BLASLONG n1 = (n-j) & -32;
+ if ( n1 > 0 )
+ {
+ sscal_kernel_16(n1, &x[j], da);
+ j+=n1;
+ }
+#else
BLASLONG n1 = n & -32;
if ( n1 > 0 )
{
sscal_kernel_16(n1, x, da);
j=n1;
}
+#endif
while(j < n)
{
--- /dev/null
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define HAVE_KERNEL_16 1
+
+static void sscal_kernel_16 (long n, float *x, float alpha)
+{
+ __asm__
+ (
+ "dcbt 0, %2 \n\t"
+
+ "xscvdpspn 48, %x3 \n\t"
+ "xxspltw 48, 48, 0 \n\t"
+
+ "lxvp 32, 0(%2) \n\t"
+ "lxvp 34, 32(%2) \n\t"
+ "lxvp 36, 64(%2) \n\t"
+ "lxvp 38, 96(%2) \n\t"
+
+ "addic. %1, %1, -32 \n\t"
+ "ble two%= \n\t"
+
+ ".align 5 \n"
+ "one%=: \n\t"
+
+ "xvmulsp 40, 32, 48 \n\t"
+ "xvmulsp 41, 33, 48 \n\t"
+ "xvmulsp 42, 34, 48 \n\t"
+ "xvmulsp 43, 35, 48 \n\t"
+ "lxvp 32, 128(%2) \n\t"
+ "lxvp 34, 160(%2) \n\t"
+ "xvmulsp 44, 36, 48 \n\t"
+ "xvmulsp 45, 37, 48 \n\t"
+ "xvmulsp 46, 38, 48 \n\t"
+ "xvmulsp 47, 39, 48 \n\t"
+ "lxvp 36, 192(%2) \n\t"
+ "lxvp 38, 224(%2) \n\t"
+
+ "stxvp 40, 0(%2) \n\t"
+ "stxvp 42, 32(%2) \n\t"
+ "stxvp 44, 64(%2) \n\t"
+ "stxvp 46, 96(%2) \n\t"
+
+ "addi %2, %2, 128 \n\t"
+
+ "addic. %1, %1, -32 \n\t"
+ "bgt one%= \n"
+
+ "two%=: \n\t"
+
+ "xvmulsp 40, 32, 48 \n\t"
+ "xvmulsp 41, 33, 48 \n\t"
+ "xvmulsp 42, 34, 48 \n\t"
+ "xvmulsp 43, 35, 48 \n\t"
+
+ "xvmulsp 44, 36, 48 \n\t"
+ "xvmulsp 45, 37, 48 \n\t"
+ "xvmulsp 46, 38, 48 \n\t"
+ "xvmulsp 47, 39, 48 \n\t"
+
+ "stxvp 40, 0(%2) \n\t"
+ "stxvp 42, 32(%2) \n\t"
+ "stxvp 44, 64(%2) \n\t"
+ "stxvp 46, 96(%2) \n\t"
+
+ "#n=%1 alpha=%3 x=%0=%2"
+ :
+ "+m" (*x),
+ "+r" (n), // 1
+ "+b" (x) // 2
+ :
+ "f" (alpha) // 3
+ :
+ "cr0",
+ "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",
+ "vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47","vs48"
+ );
+}
+
+
+static void sscal_kernel_16_zero (long n, float *x)
+{
+
+ __asm__
+ (
+ "xxlxor 32, 32, 32 \n\t"
+ "xxlxor 33, 33, 33 \n\t"
+
+ ".align 5 \n"
+ "one%=: \n\t"
+
+ "stxvp 32, 0(%2) \n\t"
+ "stxvp 32, 32(%2) \n\t"
+ "stxvp 32, 64(%2) \n\t"
+ "stxvp 32, 96(%2) \n\t"
+
+ "addi %2, %2, 128 \n\t"
+
+ "addic. %1, %1, -32 \n\t"
+ "bgt one%= \n"
+
+ "#n=%1 x=%0=%2 "
+ :
+ "=m" (*x),
+ "+r" (n), // 1
+ "+b" (x) // 2
+ :
+ :
+ "cr0","vs32","vs33"
+ );
+}
projects / platform / upstream / openblas.git / commitdiff