neoverse n2 sbgemm:

	implement ncopy tcopy kernel_8x4

diff --git a/Makefile.arm64 b/Makefile.arm64
index 9844d20..c88728e 100644 (file)
--- a/Makefile.arm64
+++ b/Makefile.arm64
@@ -121,7 +121,7 @@ ifeq ($(CORE), NEOVERSEN2)
 ifeq (1, $(filter 1,$(GCCVERSIONGTEQ7) $(ISCLANG)))
 ifeq ($(GCCVERSIONGTEQ9), 1)
 ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ4) $(GCCVERSIONGTEQ10)))
-CCOMMON_OPT += -march=armv8.5-a -mtune=neoverse-n2
+CCOMMON_OPT += -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2
 ifneq ($(F_COMPILER), NAG)
 FCOMMON_OPT += -march=armv8.5-a -mtune=neoverse-n2
 endif

diff --git a/kernel/arm64/KERNEL.NEOVERSEN2 b/kernel/arm64/KERNEL.NEOVERSEN2
index f880f96..07a94a0 100644 (file)
--- a/kernel/arm64/KERNEL.NEOVERSEN2
+++ b/kernel/arm64/KERNEL.NEOVERSEN2
@@ -189,7 +189,7 @@ ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
 ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 SBGEMM_BETA    =  sbgemm_beta_neoversen2.c
-SBGEMMKERNEL    = sbgemm_kernel_neoversen2.c
+SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversen2.c
 SBGEMMINCOPY    = sbgemm_ncopy_neoversen2.c
 SBGEMMITCOPY    = sbgemm_tcopy_neoversen2.c
 SBGEMMONCOPY    = sbgemm_ncopy_neoversen2.c

diff --git a/kernel/arm64/sbgemm_kernel_8x4_neoversen2.c b/kernel/arm64/sbgemm_kernel_8x4_neoversen2.c
new file mode 100644 (file)
index 0000000..01a96cd
--- /dev/null
+++ b/kernel/arm64/sbgemm_kernel_8x4_neoversen2.c
@@ -0,0 +1,314 @@
+/***************************************************************************
+ * Copyright (c) 2022, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include <arm_sve.h>
+#include "common.h"
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
+          FLOAT *C, BLASLONG ldc) {
+    // printf("m: %d, n: %d, k: %d\n", m, n, k);
+    BLASLONG padk = (k + 3) & ~3;
+    BLASLONG padm = (m + 1) & ~1;
+    BLASLONG padn = (n + 1) & ~1;
+    FLOAT *RC = (FLOAT *) calloc(padm * padn, sizeof(float));
+    BLASLONG nldc = padm;
+
+    IFLOAT *ptr_a = A;
+    IFLOAT *ptr_b = B;
+    FLOAT *ptr_c = RC;
+
+    IFLOAT *ptr_a0, *ptr_a1, *ptr_a2, *ptr_a3;
+    IFLOAT *ptr_b0, *ptr_b1;
+    FLOAT *ptr_c00, *ptr_c10, *ptr_c20, *ptr_c30, *ptr_c01, *ptr_c11, *ptr_c21, *ptr_c31;
+
+    svbfloat16_t ma0, ma1, ma2, ma3, mb0, mb1;
+    svfloat32_t mc00, mc01, mc10, mc11, mc20, mc21, mc30, mc31;
+    svbool_t pg16 = svptrue_b16();
+    svbool_t pg32 = svptrue_b32();
+    svfloat32_t svalpha = svdup_f32(alpha);
+
+    uint32_t off_c[] = {0, (uint32_t) nldc, 1, (uint32_t) nldc + 1}; // 00 01 10 11
+    svuint32_t off_vc = svld1_u32(pg32, off_c);
+
+    for (BLASLONG j = 0; j < padn/4; j++) {
+        ptr_c00 = ptr_c;
+        ptr_c10 = ptr_c00 + 2;
+        ptr_c20 = ptr_c10 + 2;
+        ptr_c30 = ptr_c20 + 2;
+        ptr_c01 = ptr_c + 2 * nldc;
+        ptr_c11 = ptr_c01 + 2;
+        ptr_c21 = ptr_c11 + 2;
+        ptr_c31 = ptr_c21 + 2;
+        ptr_c += 4 * nldc;
+
+        ptr_a = A;
+
+        for (BLASLONG i = 0; i < padm/8; i++) {
+            ptr_a0 = ptr_a;
+            ptr_a1 = ptr_a0 + 2 * padk;
+            ptr_a2 = ptr_a1 + 2 * padk;
+            ptr_a3 = ptr_a2 + 2 * padk;
+            ptr_a += 8 * padk;
+
+            ptr_b0 = ptr_b;
+            ptr_b1 = ptr_b0 + 2 * padk;
+
+            mc00 = svdup_f32(0); mc01 = svdup_f32(0); 
+            mc10 = svdup_f32(0); mc11 = svdup_f32(0);
+            mc20 = svdup_f32(0); mc21 = svdup_f32(0);
+            mc30 = svdup_f32(0); mc31 = svdup_f32(0);
+            
+            for (BLASLONG p = 0; p < padk/4; p++) {
+                ma0 = svld1_bf16(pg16, (bfloat16_t *) ptr_a0);
+                ma1 = svld1_bf16(pg16, (bfloat16_t *) ptr_a1);
+                ma2 = svld1_bf16(pg16, (bfloat16_t *) ptr_a2);
+                ma3 = svld1_bf16(pg16, (bfloat16_t *) ptr_a3);
+                mb0 = svld1_bf16(pg16, (bfloat16_t *) ptr_b0);
+                mb1 = svld1_bf16(pg16, (bfloat16_t *) ptr_b1);
+
+                mc00 = svbfmmla(mc00, ma0, mb0);
+                mc10 = svbfmmla(mc10, ma1, mb0);
+                mc20 = svbfmmla(mc20, ma2, mb0);
+                mc30 = svbfmmla(mc30, ma3, mb0);
+                mc01 = svbfmmla(mc01, ma0, mb1);
+                mc11 = svbfmmla(mc11, ma1, mb1);
+                mc21 = svbfmmla(mc21, ma2, mb1);
+                mc31 = svbfmmla(mc31, ma3, mb1);
+
+                ptr_a0 += 8;
+                ptr_a1 += 8;
+                ptr_a2 += 8;
+                ptr_a3 += 8;
+                ptr_b0 += 8;
+                ptr_b1 += 8;
+            }
+            svst1_scatter_index(pg32, ptr_c00, off_vc, mc00);
+            svst1_scatter_index(pg32, ptr_c10, off_vc, mc10);
+            svst1_scatter_index(pg32, ptr_c20, off_vc, mc20);
+            svst1_scatter_index(pg32, ptr_c30, off_vc, mc30);
+            svst1_scatter_index(pg32, ptr_c01, off_vc, mc01);
+            svst1_scatter_index(pg32, ptr_c11, off_vc, mc11);
+            svst1_scatter_index(pg32, ptr_c21, off_vc, mc21);
+            svst1_scatter_index(pg32, ptr_c31, off_vc, mc31);
+
+            ptr_c00 += 8;
+            ptr_c10 += 8;
+            ptr_c20 += 8;
+            ptr_c30 += 8;
+            ptr_c01 += 8;
+            ptr_c11 += 8;
+            ptr_c21 += 8;
+            ptr_c31 += 8;
+        }
+
+        if (padm & 4) {
+            // rest 4 or 6
+            ptr_a0 = ptr_a;
+            ptr_a1 = ptr_a0 + 2 * padk;
+            ptr_a += 4 * padk;
+
+            ptr_b0 = ptr_b;
+            ptr_b1 = ptr_b0 + 2 * padk;
+
+            mc00 = svdup_f32(0); mc01 = svdup_f32(0);
+            mc10 = svdup_f32(0); mc11 = svdup_f32(0);
+            for (BLASLONG p = 0; p < padk/4; p++) {
+                ma0 = svld1_bf16(pg16, (bfloat16_t *) ptr_a0);
+                ma1 = svld1_bf16(pg16, (bfloat16_t *) ptr_a1);
+                mb0 = svld1_bf16(pg16, (bfloat16_t *) ptr_b0);
+                mb1 = svld1_bf16(pg16, (bfloat16_t *) ptr_b1);
+
+                mc00 = svbfmmla(mc00, ma0, mb0);
+                mc10 = svbfmmla(mc10, ma1, mb0);
+                mc01 = svbfmmla(mc01, ma0, mb1);
+                mc11 = svbfmmla(mc11, ma1, mb1);
+
+                ptr_a0 += 8;
+                ptr_a1 += 8;
+                ptr_b0 += 8;
+                ptr_b1 += 8;
+            }
+            svst1_scatter_index(pg32, ptr_c00, off_vc, mc00);
+            svst1_scatter_index(pg32, ptr_c10, off_vc, mc10);
+            svst1_scatter_index(pg32, ptr_c01, off_vc, mc01);
+            svst1_scatter_index(pg32, ptr_c11, off_vc, mc11);
+
+            ptr_c00 += 4;
+            ptr_c10 += 4;
+            ptr_c01 += 4;
+            ptr_c11 += 4;
+        }
+
+        if (padm & 2) {
+            // rest 2
+            ptr_a0 = ptr_a;
+
+            ptr_b0 = ptr_b;
+            ptr_b1 = ptr_b0 + 2 * padk;
+
+            mc00 = svdup_f32(0); mc01 = svdup_f32(0);
+            for (BLASLONG p = 0; p < padk/4; p++) {
+                ma0 = svld1_bf16(pg16, (bfloat16_t *) ptr_a0);
+                mb0 = svld1_bf16(pg16, (bfloat16_t *) ptr_b0);
+                mb1 = svld1_bf16(pg16, (bfloat16_t *) ptr_b1);
+                mc00 = svbfmmla(mc00, ma0, mb0);
+                mc01 = svbfmmla(mc01, ma0, mb1);
+                ptr_a0 += 8;
+                ptr_b0 += 8;
+                ptr_b1 += 8;
+            }
+            svst1_scatter_index(pg32, ptr_c00, off_vc, mc00);
+            svst1_scatter_index(pg32, ptr_c01, off_vc, mc01);
+            ptr_c00 += 2;
+            ptr_c01 += 2;
+        }
+
+        ptr_b += 4 * padk;
+
+    }
+
+    if (padn & 2) {
+        // rest 2
+        ptr_c00 = ptr_c;
+        ptr_c10 = ptr_c00 + 2;
+        ptr_c20 = ptr_c10 + 2;
+        ptr_c30 = ptr_c20 + 2;
+        ptr_c += 2 * nldc;
+
+        ptr_a = A;
+
+        for (BLASLONG i = 0; i < padm/8; i++) {
+            ptr_a0 = ptr_a;
+            ptr_a1 = ptr_a0 + 2 * padk;
+            ptr_a2 = ptr_a1 + 2 * padk;
+            ptr_a3 = ptr_a2 + 2 * padk;
+            ptr_a += 8 * padk;
+
+            ptr_b0 = ptr_b;
+
+            mc00 = svdup_f32(0);
+            mc10 = svdup_f32(0);
+            mc20 = svdup_f32(0);
+            mc30 = svdup_f32(0);
+
+            for (BLASLONG p = 0; p < padk/4; p++) {
+                ma0 = svld1_bf16(pg16, (bfloat16_t *) ptr_a0);
+                ma1 = svld1_bf16(pg16, (bfloat16_t *) ptr_a1);
+                ma2 = svld1_bf16(pg16, (bfloat16_t *) ptr_a2);
+                ma3 = svld1_bf16(pg16, (bfloat16_t *) ptr_a3);
+                mb0 = svld1_bf16(pg16, (bfloat16_t *) ptr_b0);
+                mc00 = svbfmmla(mc00, ma0, mb0);
+                mc10 = svbfmmla(mc10, ma1, mb0);
+                mc20 = svbfmmla(mc20, ma2, mb0);
+                mc30 = svbfmmla(mc30, ma3, mb0);
+                ptr_a0 += 8;
+                ptr_a1 += 8;
+                ptr_a2 += 8;
+                ptr_a3 += 8;
+                ptr_b0 += 8;
+            }
+            svst1_scatter_index(pg32, ptr_c00, off_vc, mc00);
+            svst1_scatter_index(pg32, ptr_c10, off_vc, mc10);
+            svst1_scatter_index(pg32, ptr_c20, off_vc, mc20);
+            svst1_scatter_index(pg32, ptr_c30, off_vc, mc30);
+            ptr_c00 += 8;
+            ptr_c10 += 8;
+            ptr_c20 += 8;
+            ptr_c30 += 8;
+        }
+
+        if (padm & 4) {
+            ptr_a0 = ptr_a;
+            ptr_a1 = ptr_a0 + 2 * padk;
+            ptr_a += 4 * padk;
+
+            ptr_b0 = ptr_b;
+
+            mc00 = svdup_f32(0);
+            mc10 = svdup_f32(0);
+            for (BLASLONG p = 0; p < padk/4; p++) {
+                ma0 = svld1_bf16(pg16, (bfloat16_t *) ptr_a0);
+                ma1 = svld1_bf16(pg16, (bfloat16_t *) ptr_a1);
+                mb0 = svld1_bf16(pg16, (bfloat16_t *) ptr_b0);
+                mc00 = svbfmmla(mc00, ma0, mb0);
+                mc10 = svbfmmla(mc10, ma1, mb0);
+                ptr_a0 += 8;
+                ptr_a1 += 8;
+                ptr_b0 += 8;
+            }
+            svst1_scatter_index(pg32, ptr_c00, off_vc, mc00);
+            svst1_scatter_index(pg32, ptr_c10, off_vc, mc10);
+            ptr_c00 += 4;
+            ptr_c10 += 4;
+        }
+
+        if (padm & 2) {
+            ptr_a0 = ptr_a;
+            ptr_a += 2 * padk;
+            ptr_b0 = ptr_b;
+            mc00 = svdup_f32(0);
+            for (BLASLONG p = 0; p < padk/4; p++) {
+                ma0 = svld1_bf16(pg16, (bfloat16_t *) ptr_a0);
+                mb0 = svld1_bf16(pg16, (bfloat16_t *) ptr_b0);
+                mc00 = svbfmmla(mc00, ma0, mb0);
+                ptr_a0 += 8;
+                ptr_b0 += 8;
+            }
+            svst1_scatter_index(pg32, ptr_c00, off_vc, mc00);
+            ptr_c00 += 2;
+        }
+
+        ptr_b += 2 * padk;
+    }
+
+    FLOAT *org_c = C;
+    FLOAT *raw_c = RC;
+    FLOAT *org_c0, *raw_c0;
+    svfloat32_t org_vc0, raw_vc0;
+    for (BLASLONG j = 0; j < n; j++) {
+        org_c0 = org_c;
+        raw_c0 = raw_c;
+        org_c += ldc;
+        raw_c += nldc;
+        BLASLONG i;
+        for (i = 0; i < m/4; i++) {
+            org_vc0 = svld1_f32(pg32, org_c0);
+            raw_vc0 = svld1_f32(pg32, raw_c0);
+            org_vc0 = svmad_z(pg32, svalpha, raw_vc0, org_vc0); // alpha * raw + org, raw -> a * b
+            svst1_f32(pg32, org_c0, org_vc0);
+            org_c0 += 4;
+            raw_c0 += 4;
+        }
+        for (i = 0; i < (m & 3); i++) {
+            *org_c0 += alpha * (*raw_c0);
+            org_c0++;
+            raw_c0++;
+        }
+    }
+    return 0;
+}

diff --git a/kernel/arm64/sbgemm_kernel_neoversen2.c b/kernel/arm64/sbgemm_kernel_neoversen2.c
deleted file mode 100644 (file)
index f1022c9..0000000
--- a/kernel/arm64/sbgemm_kernel_neoversen2.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/***************************************************************************
- * Copyright (c) 2022, The OpenBLAS Project
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * 3. Neither the name of the OpenBLAS project nor the names of
- * its contributors may be used to endorse or promote products
- * derived from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- * *****************************************************************************/
-
-#include "common.h"
-
-int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
-          FLOAT *C, BLASLONG ldc) {
-    return 0;
-}

diff --git a/kernel/arm64/sbgemm_ncopy_neoversen2.c b/kernel/arm64/sbgemm_ncopy_neoversen2.c
index 608b889..183106f 100644 (file)
--- a/kernel/arm64/sbgemm_ncopy_neoversen2.c
+++ b/kernel/arm64/sbgemm_ncopy_neoversen2.c
@@ -29,5 +29,87 @@
 #include "common.h"
 
 int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
+    IFLOAT *a_offset, *a_offset1, *a_offset2;
+    IFLOAT *b_offset;
+
+    a_offset = a;
+    b_offset = b;
+
+    BLASLONG m4 = m & ~3;
+    BLASLONG n2 = n & ~1;
+
+    BLASLONG j = 0;
+    for (; j < n2; j += 2) {
+        a_offset1 = a_offset;
+        a_offset2 = a_offset1 + lda;
+        a_offset += 2 * lda;
+
+        BLASLONG i = 0;
+        for (; i < m4; i += 4) {
+            *(b_offset + 0) = *(a_offset1 + 0);
+            *(b_offset + 1) = *(a_offset1 + 1);
+            *(b_offset + 2) = *(a_offset1 + 2);
+            *(b_offset + 3) = *(a_offset1 + 3);
+            *(b_offset + 4) = *(a_offset2 + 0);
+            *(b_offset + 5) = *(a_offset2 + 1);
+            *(b_offset + 6) = *(a_offset2 + 2);
+            *(b_offset + 7) = *(a_offset2 + 3);
+
+            a_offset1 += 4;
+            a_offset2 += 4;
+            b_offset += 8;
+        }
+        if (i < m) {
+            *(b_offset + 0) = *(a_offset1 + 0);
+            *(b_offset + 4) = *(a_offset2 + 0);
+
+            if (i + 1 < m) {
+                *(b_offset + 1) = *(a_offset1 + 1);
+                *(b_offset + 5) = *(a_offset2 + 1);
+            } else {
+                *(b_offset + 1) = 0;
+                *(b_offset + 5) = 0;
+            }
+
+            if (i + 2 < m) {
+                *(b_offset + 2) = *(a_offset1 + 2);
+                *(b_offset + 6) = *(a_offset2 + 2);
+            } else {
+                *(b_offset + 2) = 0;
+                *(b_offset + 6) = 0;
+            }
+
+            *(b_offset + 3) = 0;
+            *(b_offset + 7) = 0;
+
+            b_offset += 8;
+        }
+    }
+    if (j < n) {
+        BLASLONG i = 0;
+        for (; i < m4; i += 4) {
+            *(b_offset + 0) = *(a_offset + 0);
+            *(b_offset + 1) = *(a_offset + 1);
+            *(b_offset + 2) = *(a_offset + 2);
+            *(b_offset + 3) = *(a_offset + 3);
+            *(b_offset + 4) = 0;
+            *(b_offset + 5) = 0;
+            *(b_offset + 6) = 0;
+            *(b_offset + 7) = 0;
+            a_offset += 4;
+            b_offset += 8;
+        }
+        if (i < m) {
+            *(b_offset + 4) = 0;
+            *(b_offset + 5) = 0;
+            *(b_offset + 6) = 0;
+            *(b_offset + 7) = 0;
+
+            *(b_offset + 0) = *(a_offset + 0);
+            *(b_offset + 1) = (i + 1 < m) ? *(a_offset + 1) : 0;
+            *(b_offset + 2) = (i + 2 < m) ? *(a_offset + 2) : 0;
+            *(b_offset + 3) = 0;
+        }
+    }
     return 0;
 }

diff --git a/kernel/arm64/sbgemm_tcopy_neoversen2.c b/kernel/arm64/sbgemm_tcopy_neoversen2.c
index 608b889..60e6855 100644 (file)
--- a/kernel/arm64/sbgemm_tcopy_neoversen2.c
+++ b/kernel/arm64/sbgemm_tcopy_neoversen2.c
@@ -29,5 +29,89 @@
 #include "common.h"
 
 int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
+    IFLOAT *a_offset, *a_offset1, *a_offset2, *a_offset3, *a_offset4;
+    IFLOAT *b_offset;
+    a_offset = a;
+    b_offset = b;
+
+    BLASLONG m4 = m & ~3;
+    BLASLONG n2 = n & ~1;
+
+    BLASLONG j = 0;
+    for (; j < n2; j += 2) {
+        a_offset1 = a_offset;
+        a_offset2 = a_offset1 + lda;
+        a_offset3 = a_offset2 + lda;
+        a_offset4 = a_offset3 + lda;
+        a_offset += 2;
+
+        BLASLONG i = 0;
+        for (; i < m4; i += 4) {
+            *(b_offset + 0) = *(a_offset1 + 0);
+            *(b_offset + 1) = *(a_offset2 + 0);
+            *(b_offset + 2) = *(a_offset3 + 0);
+            *(b_offset + 3) = *(a_offset4 + 0);
+            *(b_offset + 4) = *(a_offset1 + 1);
+            *(b_offset + 5) = *(a_offset2 + 1);
+            *(b_offset + 6) = *(a_offset3 + 1);
+            *(b_offset + 7) = *(a_offset4 + 1);
+
+            b_offset += 8;
+            a_offset1 += 4 * lda;
+            a_offset2 += 4 * lda;
+            a_offset3 += 4 * lda;
+            a_offset4 += 4 * lda;
+        }
+        if (i < m) {  // padding 4
+            *(b_offset + 0) = *(a_offset1 + 0);
+            *(b_offset + 4) = *(a_offset1 + 1);
+
+            if (i + 1 < m) {
+                *(b_offset + 1) = *(a_offset2 + 0);
+                *(b_offset + 5) = *(a_offset2 + 1);
+            } else {
+                *(b_offset + 1) = 0;
+                *(b_offset + 5) = 0;
+            }
+
+            if (i + 2 < m) {
+                *(b_offset + 2) = *(a_offset3 + 0);
+                *(b_offset + 6) = *(a_offset3 + 1);
+            } else {
+                *(b_offset + 2) = 0;
+                *(b_offset + 6) = 0;
+            }
+
+            *(b_offset + 3) = 0;
+            *(b_offset + 7) = 0;
+            b_offset += 8;
+        }
+    }
+    if (j < n) { // padding 2
+        BLASLONG i = 0;
+        for (; i < m4; i += 4) {
+            *(b_offset + 0) = *(a_offset + 0);
+            *(b_offset + 1) = *(a_offset + 1 * lda);
+            *(b_offset + 2) = *(a_offset + 2 * lda);
+            *(b_offset + 3) = *(a_offset + 3 * lda);
+            *(b_offset + 4) = 0;
+            *(b_offset + 5) = 0;
+            *(b_offset + 6) = 0;
+            *(b_offset + 7) = 0;
+            b_offset += 8;
+            a_offset += 4 * lda;
+        }
+        if (i < m) {
+            *(b_offset + 4) = 0;
+            *(b_offset + 5) = 0;
+            *(b_offset + 6) = 0;
+            *(b_offset + 7) = 0;
+
+            *(b_offset + 0) = *(a_offset + 0);
+            *(b_offset + 1) = (i + 1 < m) ? *(a_offset + 1 * lda) : 0;
+            *(b_offset + 2) = (i + 2 < m) ? *(a_offset + 2 * lda) : 0;
+            *(b_offset + 3) = 0;
+        }
+    }
     return 0;
 }

diff --git a/param.h b/param.h
index 4e2497b..5fa8299 100644 (file)
--- a/param.h
+++ b/param.h
@@ -3330,6 +3330,9 @@ is a big desktop or server with abundant cache rather than a phone or embedded d
 
 #elif defined(NEOVERSEN2)
 
+#define SBGEMM_DEFAULT_UNROOL_M 8
+#define SBGEMM_DEFAULT_UNROOL_N 4
+
 #define SGEMM_DEFAULT_UNROLL_M  16
 #define SGEMM_DEFAULT_UNROLL_N  4