sbgemm: cooperlake: enable SBGEMM by small matrix path

diff --git a/kernel/x86_64/KERNEL.COOPERLAKE b/kernel/x86_64/KERNEL.COOPERLAKE
index 0b2f3c0..151c02d 100644 (file)
--- a/kernel/x86_64/KERNEL.COOPERLAKE
+++ b/kernel/x86_64/KERNEL.COOPERLAKE
@@ -1 +1,11 @@
 include $(KERNELDIR)/KERNEL.SKYLAKEX
+
+SBGEMM_SMALL_M_PERMIT = sbgemm_small_kernel_permit_cooperlake.c
+SBGEMM_SMALL_K_NN = sbgemm_small_kernel_nn_cooperlake.c
+SBGEMM_SMALL_K_B0_NN = sbgemm_small_kernel_b0_nn_cooperlake.c
+SBGEMM_SMALL_K_NT = sbgemm_small_kernel_nt_cooperlake.c
+SBGEMM_SMALL_K_B0_NT = sbgemm_small_kernel_b0_nt_cooperlake.c
+SBGEMM_SMALL_K_TN = sbgemm_small_kernel_tn_cooperlake.c
+SBGEMM_SMALL_K_B0_TN = sbgemm_small_kernel_b0_tn_cooperlake.c
+SBGEMM_SMALL_K_TT = sbgemm_small_kernel_tt_cooperlake.c
+SBGEMM_SMALL_K_B0_TT = sbgemm_small_kernel_b0_tt_cooperlake.c

diff --git a/kernel/x86_64/sbgemm_block_microk_cooperlake.c b/kernel/x86_64/sbgemm_block_microk_cooperlake.c
index 147c5eb..2c27221 100644 (file)
--- a/kernel/x86_64/sbgemm_block_microk_cooperlake.c
+++ b/kernel/x86_64/sbgemm_block_microk_cooperlake.c
@@ -1,6 +1,5 @@
-//#include "sbgemm.h"
-
 #include <immintrin.h>
+
 // Walk around those intrinsics that missed by compiler
 #define MM256_LOADU_EPI16(addr)   \
             _mm256_maskz_loadu_epi16(~0, (addr))
@@ -1747,7 +1746,7 @@ void COL_MAJOR_OTCOPY_KERNEL_Kx8m(BLASLONG k, BLASLONG n, bfloat16 * B, BLASLONG
 }
 
 // Scale matrix C when beta is not ZERO or ONE
-void sbgemm_scal_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc)
+void sbgemm_scal_operation(BLASLONG M, BLASLONG N, float beta, float *C, BLASLONG ldc)
 {
     float * C_addr0 = C;
     float * C_addr1 = C + ldc;
@@ -1759,12 +1758,6 @@ void sbgemm_scal_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST
     __m512 array_512_0, array_512_1, array_512_2, array_512_3;
     __m512 BETAVECTOR  = _mm512_set1_ps(beta);
 
-    if (Order == CblasRowMajor) {
-        blasint tmp = M;
-        M = N;
-        N = tmp;
-    }
-
     BLASLONG tag_n_Nx = N & (~3);
     BLASLONG tag_n_Mx = M & (~15);
     unsigned short tail_mask = (((unsigned short)0xffff) >> (16-M+tag_n_Mx));
@@ -1828,7 +1821,7 @@ void sbgemm_scal_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST
 }
 
 // Zero C matrix when Beta is 0
-void sbgemm_zero_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, float *C, OPENBLAS_CONST blasint ldc)
+void sbgemm_zero_operation(BLASLONG M, BLASLONG N, float *C, BLASLONG ldc)
 {
     float * C_addr0 = C;
     float * C_addr1 = C + ldc;
@@ -1839,12 +1832,6 @@ void sbgemm_zero_operation(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST
 
     __m512  ZEROVECTOR  = _mm512_setzero_ps();
 
-    if (Order == CblasRowMajor) {
-        blasint tmp = M;
-        M = N;
-        N = tmp;
-    }
-
     BLASLONG tag_n_Nx = N & (~3);
     BLASLONG tag_n_Mx = M & (~15);
     unsigned short tail_mask = (((unsigned short)0xffff) >> (16-M+tag_n_Mx));

diff --git a/kernel/x86_64/sbgemm_microk_cooperlake_template.c b/kernel/x86_64/sbgemm_microk_cooperlake_template.c
index c715958..b8ed983 100644 (file)
--- a/kernel/x86_64/sbgemm_microk_cooperlake_template.c
+++ b/kernel/x86_64/sbgemm_microk_cooperlake_template.c
@@ -1,8 +1,6 @@
-#include "sbgemm.h"
 #include "bf16_common_macros.h"
 #include <immintrin.h>
 
-/*  These macros are needed and should be placed at the right place
 #define BF16_BLOCK_STEP_N 8
 #define BF16_BLOCK_THRES_K 1024
 #define BF16_BLOCK_THRES_M 32
@@ -14,7 +12,6 @@
 
 #define ONE  1.e0f
 #define ZERO  0.e0f
-*/
 
 #undef STORE16_COMPLETE_RESULT
 #undef STORE16_MASK_COMPLETE_RESULT
@@ -1798,6 +1795,7 @@ void sbgemm_blocking_kernel_tt_one(blasint M, blasint N, blasint K, float alpha,
 }
 /* ----------------------------------------- End of TT kernels --------------------------------------- */
 
+/*
 #ifndef ONE_ALPHA      // ALPHA is not ONE
 void sbgemm_internal_kernel_alpha(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K,
                 OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, float *C, OPENBLAS_CONST blasint ldc)
@@ -1836,3 +1834,4 @@ void sbgemm_internal_kernel_one(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_
         }
     }
 }
+*/

diff --git a/kernel/x86_64/sbgemm_small_kernel_b0_nn_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_b0_nn_cooperlake.c
new file mode 100644 (file)
index 0000000..373457f
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_b0_nn_cooperlake.c
@@ -0,0 +1,2 @@
+#define B0 1
+#include "./sbgemm_small_kernel_nn_cooperlake.c"

diff --git a/kernel/x86_64/sbgemm_small_kernel_b0_nt_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_b0_nt_cooperlake.c
new file mode 100644 (file)
index 0000000..0b840c2
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_b0_nt_cooperlake.c
@@ -0,0 +1,2 @@
+#define B0 1
+#include "./sbgemm_small_kernel_nt_cooperlake.c"

diff --git a/kernel/x86_64/sbgemm_small_kernel_b0_tn_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_b0_tn_cooperlake.c
new file mode 100644 (file)
index 0000000..67542b6
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_b0_tn_cooperlake.c
@@ -0,0 +1,2 @@
+#define B0 1
+#include "./sbgemm_small_kernel_tn_cooperlake.c"

diff --git a/kernel/x86_64/sbgemm_small_kernel_b0_tt_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_b0_tt_cooperlake.c
new file mode 100644 (file)
index 0000000..17b5b41
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_b0_tt_cooperlake.c
@@ -0,0 +1,2 @@
+#define B0 1
+#include "./sbgemm_small_kernel_tt_cooperlake.c"

diff --git a/kernel/x86_64/sbgemm_small_kernel_nn_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_nn_cooperlake.c
new file mode 100644 (file)
index 0000000..ec40a50
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_nn_cooperlake.c
@@ -0,0 +1,2 @@
+#define TRANS_NN
+#include "sbgemm_small_kernel_template_cooperlake.c"

diff --git a/kernel/x86_64/sbgemm_small_kernel_nt_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_nt_cooperlake.c
new file mode 100644 (file)
index 0000000..1cdfd29
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_nt_cooperlake.c
@@ -0,0 +1,2 @@
+#define TRANS_NT
+#include "sbgemm_small_kernel_template_cooperlake.c"

diff --git a/kernel/x86_64/sbgemm_small_kernel_permit_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_permit_cooperlake.c
new file mode 100644 (file)
index 0000000..823aafb
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_permit_cooperlake.c
@@ -0,0 +1,42 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#include "sbgemm_block_microk_cooperlake.c"
+// Define micro kernels for ALPHA not ONE scenarios
+#undef  ONE_ALPHA
+#include "sbgemm_microk_cooperlake_template.c"
+
+// Define micro kernels for ALPHA as ONE scenarios
+#define ONE_ALPHA 1
+#include "sbgemm_microk_cooperlake_template.c"
+
+int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta)
+{
+       return 1;
+}

diff --git a/kernel/x86_64/sbgemm_small_kernel_template_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_template_cooperlake.c
new file mode 100644 (file)
index 0000000..d328b09
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_template_cooperlake.c
@@ -0,0 +1,89 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+#include <memory.h>
+
+extern void sbgemm_scal_operation(BLASLONG M, BLASLONG N, float beta, float *C, BLASLONG ldc);
+extern void sbgemm_zero_operation(BLASLONG M, BLASLONG N, float *C, BLASLONG ldc);
+
+extern void sbgemm_blocking_kernel_nn_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+extern void sbgemm_blocking_kernel_nn_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+extern void sbgemm_blocking_kernel_nt_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+extern void sbgemm_blocking_kernel_nt_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+extern void sbgemm_blocking_kernel_tn_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+extern void sbgemm_blocking_kernel_tn_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+extern void sbgemm_blocking_kernel_tt_alpha(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+extern void sbgemm_blocking_kernel_tt_one(blasint M, blasint N, blasint K, float alpha, bfloat16 *A, blasint lda, bfloat16 *B, blasint ldb, float *C, blasint ldc, bfloat16 * block_A, bfloat16 * block_B);
+
+#if defined(TRANS_NN)
+#define SBGEMM_BLOCKING_KERNEL_ONE     sbgemm_blocking_kernel_nn_one
+#define SBGEMM_BLOCKING_KERNEL_ALPHA   sbgemm_blocking_kernel_nn_alpha
+#elif defined(TRANS_NT)
+#define SBGEMM_BLOCKING_KERNEL_ONE     sbgemm_blocking_kernel_nt_one
+#define SBGEMM_BLOCKING_KERNEL_ALPHA   sbgemm_blocking_kernel_nt_alpha
+#elif defined(TRANS_TN)
+#define SBGEMM_BLOCKING_KERNEL_ONE     sbgemm_blocking_kernel_tn_one
+#define SBGEMM_BLOCKING_KERNEL_ALPHA   sbgemm_blocking_kernel_tn_alpha
+#elif defined(TRANS_TT)
+#define SBGEMM_BLOCKING_KERNEL_ONE     sbgemm_blocking_kernel_tt_one
+#define SBGEMM_BLOCKING_KERNEL_ALPHA   sbgemm_blocking_kernel_tt_alpha
+#endif
+
+#define BF16_BLOCK_THRES_K 1024
+// If we want to adjust this to be bigger, need to change COL_MAJOR_INCOPY_KERNEL_Kx32 kernel to be bigger also
+#define BF16_BLOCK_THRES_M 32
+#define BF16_BLOCK_THRES_N 1024
+
+
+#if defined(B0)
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+       bfloat16 * block_A;
+       bfloat16 * block_B;
+
+       block_A = (bfloat16 *) malloc(sizeof(bfloat16) * BF16_BLOCK_THRES_K * BF16_BLOCK_THRES_M);
+       block_B = (bfloat16 *) malloc(sizeof(bfloat16) * BF16_BLOCK_THRES_N * BF16_BLOCK_THRES_K);
+
+#if defined(B0)
+       sbgemm_zero_operation(M, N, C, ldc);
+#else
+       sbgemm_scal_operation(M, N, beta, C, ldc);
+#endif
+
+       if (alpha == ONE) {
+               SBGEMM_BLOCKING_KERNEL_ONE(M, N, K, alpha, A, lda, B, ldb, C, ldc, block_A, block_B);
+       } else {
+               SBGEMM_BLOCKING_KERNEL_ALPHA(M, N, K, alpha, A, lda, B, ldb, C, ldc, block_A, block_B);
+       }
+       free(block_A);
+       free(block_B);
+       return 0;
+}

diff --git a/kernel/x86_64/sbgemm_small_kernel_tn_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_tn_cooperlake.c
new file mode 100644 (file)
index 0000000..f1a0d0d
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_tn_cooperlake.c
@@ -0,0 +1,2 @@
+#define TRANS_TN
+#include "sbgemm_small_kernel_template_cooperlake.c"

diff --git a/kernel/x86_64/sbgemm_small_kernel_tt_cooperlake.c b/kernel/x86_64/sbgemm_small_kernel_tt_cooperlake.c
new file mode 100644 (file)
index 0000000..8a2a597
--- /dev/null
+++ b/kernel/x86_64/sbgemm_small_kernel_tt_cooperlake.c
@@ -0,0 +1,2 @@
+#define TRANS_TT
+#include "sbgemm_small_kernel_template_cooperlake.c"