Tweak scheduling use of SQDMULH in quantized AVG pooling

Resolves COMPMID-4195

Change-Id: Ie5116c1ddddccafba40432fd4b5245bb27890a88
Signed-off-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4997
Reviewed-by: TeresaARM <teresa.charlinreyes@arm.com>
Reviewed-by: Manuel Bottini <manuel.bottini@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
index f11bb68c24051e1f85415e7c45cd55611ea583f6..89dbf5ce02e22c980a821f995c79ef2fa2bcc457 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
@@ -84,13 +84,12 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x4, [%x[args], %[offsetof_n_channels]]\n"
     "mov x5, #0x0\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x6, #0x0\n"
-    "ldr d8, [%x[args], %[offsetof_rescale]]\n"
-    "ldp x7, x8, [x19, #0x0]\n"
-    "cmp x4, #0x8\n"
-    "ldp x17, x16, [x19, #0x10]\n"
     "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "cmp x4, #0x8\n"
+    "ldp x7, x8, [x20, #0x0]\n"
+    "ldp x17, x16, [x20, #0x10]\n"
     "ldp x15, x14, [x19, #0x0]\n"
     "ldp x13, x12, [x19, #0x10]\n"
     "ldp x11, x10, [x19, #0x20]\n"
@@ -99,12 +98,14 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldp x25, x24, [x19, #0x50]\n"
     "ldp x23, x22, [x19, #0x60]\n"
     "ldp x21, x20, [x19, #0x70]\n"
+    "ldr d8, [%x[args], %[offsetof_rescale]]\n"
     "blt 3f\n"
-    "lsr x19, x4, #0x3\n"
-    "sub x4, x4, x19, LSL #3\n"
     "ldr q7, [x10, x5]\n"
+    "lsr x19, x4, #0x3\n"
     "ldr q6, [x9, x5]\n"
+    "sub x4, x4, x19, LSL #3\n"
     "ldr q5, [x26, x5]\n"
+    "subs x19, x19, #0x1\n"
     "ldr q4, [x25, x5]\n"
     "ldr q3, [x14, x5]\n"
     "ldr q2, [x13, x5]\n"
@@ -119,26 +120,26 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldr q25, [x23, x5]\n"
     "ldr q24, [x20, x5]\n"
     "add x5, x5, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "beq 2f\n"
     "1:"  // Vector: Loop
     "fadd v17.8h, v7.8h, v6.8h\n"
     "ldr q7, [x10, x5]\n"
+    "subs x19, x19, #0x1\n"
     "fadd v16.8h, v5.8h, v4.8h\n"
     "ldr q6, [x9, x5]\n"
     "fadd v18.8h, v3.8h, v2.8h\n"
     "ldr q5, [x26, x5]\n"
     "fadd v23.8h, v1.8h, v0.8h\n"
     "ldr q4, [x25, x5]\n"
-    "fadd v17.8h, v17.8h, v16.8h\n"
-    "ldr q3, [x14, x5]\n"
     "fadd v22.8h, v31.8h, v30.8h\n"
+    "ldr q3, [x14, x5]\n"
+    "fadd v17.8h, v17.8h, v16.8h\n"
     "ldr q2, [x13, x5]\n"
     "fadd v16.8h, v29.8h, v28.8h\n"
     "ldr q1, [x11, x5]\n"
-    "fadd v21.8h, v18.8h, v17.8h\n"
-    "ldr q0, [x27, x5]\n"
     "fadd v19.8h, v27.8h, v23.8h\n"
+    "ldr q0, [x27, x5]\n"
+    "fadd v21.8h, v18.8h, v17.8h\n"
     "ldr q31, [x28, x5]\n"
     "fadd v20.8h, v16.8h, v17.8h\n"
     "ldr q30, [x24, x5]\n"
@@ -148,21 +149,20 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldr q28, [x21, x5]\n"
     "fadd v16.8h, v24.8h, v22.8h\n"
     "ldr q27, [x15, x5]\n"
-    "fadd v19.8h, v19.8h, v21.8h\n"
+    "fadd v19.8h, v21.8h, v19.8h\n"
     "ldr q26, [x12, x5]\n"
     "fadd v18.8h, v21.8h, v18.8h\n"
     "ldr q25, [x23, x5]\n"
     "fadd v17.8h, v17.8h, v20.8h\n"
     "ldr q24, [x20, x5]\n"
-    "fadd v16.8h, v20.8h, v16.8h\n"
     "add x5, x5, #0x10\n"
+    "fadd v16.8h, v20.8h, v16.8h\n"
     "fmul v19.8h, v19.8h, v8.h[0]\n"
-    "subs x19, x19, #0x1\n"
-    "fmul v18.8h, v18.8h, v8.h[1]\n"
     "str q19, [x7, x6]\n"
+    "fmul v18.8h, v18.8h, v8.h[1]\n"
     "fmul v17.8h, v17.8h, v8.h[2]\n"
-    "fmul v16.8h, v16.8h, v8.h[3]\n"
     "str q18, [x8, x6]\n"
+    "fmul v16.8h, v16.8h, v8.h[3]\n"
     "str q17, [x17, x6]\n"
     "str q16, [x16, x6]\n"
     "add x6, x6, #0x10\n"
@@ -181,7 +181,7 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "fadd v18.8h, v26.8h, v22.8h\n"
     "fadd v17.8h, v25.8h, v23.8h\n"
     "fadd v16.8h, v24.8h, v22.8h\n"
-    "fadd v19.8h, v19.8h, v21.8h\n"
+    "fadd v19.8h, v21.8h, v19.8h\n"
     "fadd v18.8h, v21.8h, v18.8h\n"
     "fadd v17.8h, v17.8h, v20.8h\n"
     "fadd v16.8h, v20.8h, v16.8h\n"
@@ -197,6 +197,7 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "cbz x4, 4f\n"
     "3:"  // Oddments
     "ldr h7, [x10, x5]\n"
+    "subs x4, x4, #0x1\n"
     "ldr h6, [x9, x5]\n"
     "fadd v17.8h, v7.8h, v6.8h\n"
     "ldr h5, [x26, x5]\n"
@@ -209,33 +210,32 @@ void a64_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldr h0, [x27, x5]\n"
     "fadd v18.8h, v3.8h, v2.8h\n"
     "ldr h31, [x28, x5]\n"
-    "ldr h30, [x24, x5]\n"
     "fadd v23.8h, v1.8h, v0.8h\n"
-    "ldr h29, [x22, x5]\n"
+    "ldr h30, [x24, x5]\n"
     "fadd v21.8h, v18.8h, v17.8h\n"
+    "ldr h29, [x22, x5]\n"
     "ldr h28, [x21, x5]\n"
-    "ldr h27, [x15, x5]\n"
     "fadd v22.8h, v31.8h, v30.8h\n"
+    "ldr h27, [x15, x5]\n"
     "ldr h26, [x12, x5]\n"
     "fadd v16.8h, v29.8h, v28.8h\n"
     "ldr h25, [x23, x5]\n"
-    "fadd v19.8h, v27.8h, v23.8h\n"
+    "fadd v20.8h, v16.8h, v17.8h\n"
     "ldr h24, [x20, x5]\n"
-    "fadd v18.8h, v26.8h, v22.8h\n"
     "add x5, x5, #0x2\n"
-    "subs x4, x4, #0x1\n"
-    "fadd v20.8h, v16.8h, v17.8h\n"
-    "fadd v19.8h, v19.8h, v21.8h\n"
-    "fadd v18.8h, v21.8h, v18.8h\n"
+    "fadd v19.8h, v27.8h, v23.8h\n"
+    "fadd v18.8h, v26.8h, v22.8h\n"
     "fadd v17.8h, v25.8h, v23.8h\n"
     "fadd v16.8h, v24.8h, v22.8h\n"
-    "fmul v19.8h, v19.8h, v8.h[0]\n"
-    "str h19, [x7, x6]\n"
+    "fadd v19.8h, v21.8h, v19.8h\n"
+    "fadd v18.8h, v21.8h, v18.8h\n"
     "fadd v17.8h, v17.8h, v20.8h\n"
     "fadd v16.8h, v20.8h, v16.8h\n"
+    "fmul v19.8h, v19.8h, v8.h[0]\n"
+    "str h19, [x7, x6]\n"
     "fmul v18.8h, v18.8h, v8.h[1]\n"
-    "str h18, [x8, x6]\n"
     "fmul v17.8h, v17.8h, v8.h[2]\n"
+    "str h18, [x8, x6]\n"
     "fmul v16.8h, v16.8h, v8.h[3]\n"
     "str h17, [x17, x6]\n"
     "str h16, [x16, x6]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst.hpp
index 7bf1f4327e69010a3974fc8afcc9145d5665bbe8..9dc153a7641a9d7f556d0c13420b527a1098fd79 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_fp16_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = a64_fp16_nhwc_avg_generic_depthfirst_impl;
 
   a64_fp16_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp
index 420616b38b0513a173dd5cd0ee491cc2fd33c1af..5bef7f2bf40756ca2e6089003c7a67563ce17fe2 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_avg_generic_depthfirst/generic.cpp
@@ -41,7 +41,7 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl(
   const auto rescale_value = static_cast<__fp16>(1.0f / static_cast<float>(window_cells));
 
   __asm__ __volatile__(
-    "ld1r { v7.8h }, [%x[rescale_ptr]]\n"
+    "ld1r { v8.8h }, [%x[rescale_ptr]]\n"
     "mov x28, #0x0\n"
     "mov x27, #0x10\n" // cntb _, ALL, #1
     "mov x26, #0x20\n" // cntb _, ALL, #2
@@ -49,23 +49,23 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x20\n"
     "blt 7f\n"
     "1:"  // 4-vectors of channels
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "mov x19, %x[inptrs]\n"
-    "movi v5.16b, #0x0\n"
+    "movi v6.16b, #0x0\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "movi v5.16b, #0x0\n"
     "movi v4.16b, #0x0\n"
-    "movi v3.16b, #0x0\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -78,47 +78,47 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "fadd v23.8h, v2.8h, v1.8h\n"
+    "fadd v23.8h, v3.8h, v2.8h\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd v19.8h, v0.8h, v31.8h\n"
+    "fadd v19.8h, v1.8h, v0.8h\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd v22.8h, v30.8h, v22.8h\n"
     "add x19, x19, #0x20\n"
+    "fadd v22.8h, v31.8h, v30.8h\n"
+    "ldr q3, [x23, x28]\n"
     "fadd v18.8h, v29.8h, v28.8h\n"
-    "ldr q2, [x23, x28]\n"
     "fadd v21.8h, v27.8h, v21.8h\n"
+    "ldr q2, [x22, x28]\n"
     "fadd v17.8h, v26.8h, v17.8h\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "fadd v20.8h, v25.8h, v20.8h\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "fadd v16.8h, v24.8h, v16.8h\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "fadd v19.8h, v23.8h, v19.8h\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "fadd v18.8h, v22.8h, v18.8h\n"
-    "ldr q22, [x22, x27]\n"
-    "fadd v17.8h, v21.8h, v17.8h\n"
     "ldr q29, [x21, x27]\n"
-    "fadd v16.8h, v20.8h, v16.8h\n"
+    "fadd v17.8h, v21.8h, v17.8h\n"
     "ldr q28, [x20, x27]\n"
-    "fadd v6.8h, v6.8h, v19.8h\n"
+    "fadd v16.8h, v20.8h, v16.8h\n"
     "ldr q27, [x23, x26]\n"
-    "fadd v5.8h, v5.8h, v18.8h\n"
+    "fadd v7.8h, v7.8h, v19.8h\n"
     "ldr q21, [x22, x26]\n"
-    "fadd v4.8h, v4.8h, v17.8h\n"
+    "fadd v6.8h, v6.8h, v18.8h\n"
     "ldr q26, [x21, x26]\n"
-    "fadd v3.8h, v3.8h, v16.8h\n"
+    "fadd v5.8h, v5.8h, v17.8h\n"
     "ldr q17, [x20, x26]\n"
+    "fadd v4.8h, v4.8h, v16.8h\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "fadd v23.8h, v2.8h, v1.8h\n"
-    "fadd v19.8h, v0.8h, v31.8h\n"
-    "fadd v22.8h, v30.8h, v22.8h\n"
+    "fadd v23.8h, v3.8h, v2.8h\n"
+    "fadd v19.8h, v1.8h, v0.8h\n"
+    "fadd v22.8h, v31.8h, v30.8h\n"
     "fadd v18.8h, v29.8h, v28.8h\n"
     "fadd v21.8h, v27.8h, v21.8h\n"
     "fadd v17.8h, v26.8h, v17.8h\n"
@@ -128,221 +128,221 @@ void a64_fp16_nhwc_avg_generic_depthfirst_impl(
     "fadd v18.8h, v22.8h, v18.8h\n"
     "fadd v17.8h, v21.8h, v17.8h\n"
     "fadd v16.8h, v20.8h, v16.8h\n"
-    "fadd v6.8h, v6.8h, v19.8h\n"
-    "fadd v5.8h, v5.8h, v18.8h\n"
-    "fadd v4.8h, v4.8h, v17.8h\n"
-    "fadd v3.8h, v3.8h, v16.8h\n"
+    "fadd v7.8h, v7.8h, v19.8h\n"
+    "fadd v6.8h, v6.8h, v18.8h\n"
+    "fadd v5.8h, v5.8h, v17.8h\n"
+    "fadd v4.8h, v4.8h, v16.8h\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fadd v6.8h, v6.8h, v2.8h\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "fadd v7.8h, v7.8h, v3.8h\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "fadd v5.8h, v5.8h, v30.8h\n"
+    "fadd v6.8h, v6.8h, v31.8h\n"
     "ldr q25, [x23, x25]\n"
-    "fadd v4.8h, v4.8h, v27.8h\n"
-    "fadd v3.8h, v3.8h, v25.8h\n"
+    "fadd v5.8h, v5.8h, v27.8h\n"
+    "fadd v4.8h, v4.8h, v25.8h\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "fmul v6.8h, v6.8h, v7.8h\n"
-    "sub %x[n_channels], %x[n_channels], #0x20\n"
-    "fmul v5.8h, v5.8h, v7.8h\n"
-    "cmp %x[n_channels], #0x20\n"
-    "fmul v4.8h, v4.8h, v7.8h\n"
-    "str q6, [%x[outptr], x28]\n"
-    "fmul v3.8h, v3.8h, v7.8h\n"
-    "str q5, [%x[outptr], x27]\n"
-    "str q4, [%x[outptr], x26]\n"
+    "fmul v7.8h, v7.8h, v8.8h\n"
+    "str q7, [%x[outptr], x28]\n"
+    "fmul v6.8h, v6.8h, v8.8h\n"
     "add x28, x28, #0x40\n"
+    "fmul v5.8h, v5.8h, v8.8h\n"
+    "str q6, [%x[outptr], x27]\n"
+    "fmul v4.8h, v4.8h, v8.8h\n"
     "add x27, x27, #0x40\n"
-    "str q3, [%x[outptr], x25]\n"
+    "str q5, [%x[outptr], x26]\n"
     "add x26, x26, #0x40\n"
+    "sub %x[n_channels], %x[n_channels], #0x20\n"
+    "str q4, [%x[outptr], x25]\n"
     "add x25, x25, #0x40\n"
+    "cmp %x[n_channels], #0x20\n"
     "bge 1b\n"
     "cbz %x[n_channels], 31f\n"
     "7:"  // Single vector of channels
     "cmp %x[n_channels], #0x8\n"
     "blt 14f\n"
     "8:"  // Single vector of channels: Loop
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "fadd v23.8h, v2.8h, v1.8h\n"
+    "fadd v23.8h, v3.8h, v2.8h\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd v19.8h, v0.8h, v31.8h\n"
+    "fadd v19.8h, v1.8h, v0.8h\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd v19.8h, v23.8h, v19.8h\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "fadd v6.8h, v6.8h, v19.8h\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "fadd v19.8h, v23.8h, v19.8h\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "fadd v7.8h, v7.8h, v19.8h\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "fadd v23.8h, v2.8h, v1.8h\n"
-    "fadd v19.8h, v0.8h, v31.8h\n"
+    "fadd v23.8h, v3.8h, v2.8h\n"
+    "fadd v19.8h, v1.8h, v0.8h\n"
     "fadd v19.8h, v23.8h, v19.8h\n"
-    "fadd v6.8h, v6.8h, v19.8h\n"
+    "fadd v7.8h, v7.8h, v19.8h\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fadd v6.8h, v6.8h, v2.8h\n"
+    "ldr q3, [x23, x28]\n"
+    "fadd v7.8h, v7.8h, v3.8h\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "fmul v6.8h, v6.8h, v7.8h\n"
+    "fmul v7.8h, v7.8h, v8.8h\n"
+    "str q7, [%x[outptr], x28]\n"
+    "add x28, x28, #0x10\n"
     "sub %x[n_channels], %x[n_channels], #0x8\n"
     "cmp %x[n_channels], #0x8\n"
-    "str q6, [%x[outptr], x28]\n"
-    "add x28, x28, #0x10\n"
     "bge 8b\n"
     "cbz %x[n_channels], 31f\n"
     "14:"  // Oddments
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "add %x[outptr], %x[outptr], x28\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 20f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #2, 17f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "tbz %x[n_channels], #0, 19f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
-    "ld1 { v1.h }[6], [x22], #0x2\n"
-    "ld1 { v0.h }[6], [x21], #0x2\n"
-    "ld1 { v31.h }[6], [x20], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
+    "ld1 { v2.h }[6], [x22], #0x2\n"
+    "ld1 { v1.h }[6], [x21], #0x2\n"
+    "ld1 { v0.h }[6], [x20], #0x2\n"
     "b 19f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 19f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
-    "ld1 { v1.h }[4], [x22], #0x2\n"
-    "ld1 { v0.h }[4], [x21], #0x2\n"
-    "ld1 { v31.h }[4], [x20], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
+    "ld1 { v2.h }[4], [x22], #0x2\n"
+    "ld1 { v1.h }[4], [x21], #0x2\n"
+    "ld1 { v0.h }[4], [x20], #0x2\n"
     "b 19f\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 2: Unset
     "tbz %x[n_channels], #1, 18f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "tbz %x[n_channels], #0, 19f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
-    "ld1 { v1.h }[2], [x22], #0x2\n"
-    "ld1 { v0.h }[2], [x21], #0x2\n"
-    "ld1 { v31.h }[2], [x20], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
+    "ld1 { v2.h }[2], [x22], #0x2\n"
+    "ld1 { v1.h }[2], [x21], #0x2\n"
+    "ld1 { v0.h }[2], [x20], #0x2\n"
     "b 19f\n"
     "18:"  // Oddments: 4 inputs loop: Load: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 19f\n"
-    "ldr h2, [x23], #0x2\n"
-    "ldr h1, [x22], #0x2\n"
-    "ldr h0, [x21], #0x2\n"
-    "ldr h31, [x20], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
+    "ldr h2, [x22], #0x2\n"
+    "ldr h1, [x21], #0x2\n"
+    "ldr h0, [x20], #0x2\n"
     "19:"  // Oddments: 4 inputs loop: Load: Bit 2: End
-    "fadd v23.8h, v2.8h, v1.8h\n"
+    "fadd v23.8h, v3.8h, v2.8h\n"
     "subs x24, x24, #0x1\n"
-    "fadd v19.8h, v0.8h, v31.8h\n"
+    "fadd v19.8h, v1.8h, v0.8h\n"
     "fadd v19.8h, v23.8h, v19.8h\n"
-    "fadd v6.8h, v6.8h, v19.8h\n"
+    "fadd v7.8h, v7.8h, v19.8h\n"
     "bgt 15b\n"
     "20:"  // Oddments: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 26f\n"
     "21:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #2, 23f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #1, 22f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "tbz %x[n_channels], #0, 25f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
     "b 25f\n"
     "22:"  // Oddments: Single input loop: Load: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 25f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
     "b 25f\n"
     "23:"  // Oddments: Single input loop: Load: Bit 2: Unset
     "tbz %x[n_channels], #1, 24f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "tbz %x[n_channels], #0, 25f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
     "b 25f\n"
     "24:"  // Oddments: Single input loop: Load: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 25f\n"
-    "ldr h2, [x23], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
     "25:"  // Oddments: Single input loop: Load: Bit 2: End
-    "fadd v6.8h, v6.8h, v2.8h\n"
+    "fadd v7.8h, v7.8h, v3.8h\n"
     "subs x20, x20, #0x1\n"
     "bgt 21b\n"
     "26:"  // Oddments: Single input loop: End
-    "fmul v6.8h, v6.8h, v7.8h\n"
+    "fmul v7.8h, v7.8h, v8.8h\n"
     "tbz %x[n_channels], #2, 28f\n"
-    "st1 { v6.d }[0], [%x[outptr]], #0x8\n"
+    "st1 { v7.d }[0], [%x[outptr]], #0x8\n"
     "tbz %x[n_channels], #1, 27f\n"
-    "st1 { v6.s }[2], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[2], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[6], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[6], [%x[outptr]], #0x2\n"
     "b 30f\n"
     "27:"  // Oddments: Store: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[4], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[4], [%x[outptr]], #0x2\n"
     "b 30f\n"
     "28:"  // Oddments: Store: Bit 2: Unset
     "tbz %x[n_channels], #1, 29f\n"
-    "st1 { v6.s }[0], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[0], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[2], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[2], [%x[outptr]], #0x2\n"
     "b 30f\n"
     "29:"  // Oddments: Store: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[0], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[0], [%x[outptr]], #0x2\n"
     "30:"  // Oddments: Store: Bit 2: End
 
     "31:"  // End
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [rescale_ptr] "r" (&rescale_value)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index 6e69ca0ada8c0ad519132c7deb6923cd6ca13cec..1c461ee163c2b9ee37b8a65964839a9bab017f19 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -65,23 +65,24 @@ void a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x15, [%x[args], %[offsetof_n_channels]]\n"
     "mov x14, #0x0\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
-    "ldp x12, x11, [x19, #0x0]\n"
-    "cmp x15, #0x8\n"
-    "ldp x10, x9, [x19, #0x10]\n"
     "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "cmp x15, #0x8\n"
+    "ldp x12, x11, [x20, #0x0]\n"
+    "ldp x10, x9, [x20, #0x10]\n"
     "ldp x28, x27, [x19, #0x0]\n"
     "ldp x26, x25, [x19, #0x10]\n"
     "ldp x24, x23, [x19, #0x20]\n"
     "ldp x22, x21, [x19, #0x30]\n"
     "ldr x20, [x19, #0x40]\n"
     "blt 3f\n"
-    "lsr x19, x15, #0x3\n"
-    "sub x15, x15, x19, LSL #3\n"
     "ldr q30, [x27, x14]\n"
+    "lsr x19, x15, #0x3\n"
     "ldr q29, [x24, x14]\n"
+    "sub x15, x15, x19, LSL #3\n"
     "ldr q28, [x21, x14]\n"
+    "subs x19, x19, #0x1\n"
     "ldr q27, [x25, x14]\n"
     "ldr q26, [x28, x14]\n"
     "ldr q25, [x23, x14]\n"
@@ -89,11 +90,11 @@ void a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "ldr q22, [x20, x14]\n"
     "add x14, x14, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "beq 2f\n"
     "1:"  // Vector: Loop
     "fmax v21.8h, v30.8h, v29.8h\n"
     "ldr q30, [x27, x14]\n"
+    "subs x19, x19, #0x1\n"
     "fmax v20.8h, v29.8h, v28.8h\n"
     "ldr q29, [x24, x14]\n"
     "fmax v19.8h, v27.8h, v26.8h\n"
@@ -110,14 +111,13 @@ void a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "fmax v17.8h, v20.8h, v17.8h\n"
     "ldr q22, [x20, x14]\n"
-    "fmax v16.8h, v20.8h, v16.8h\n"
     "add x14, x14, #0x10\n"
+    "fmax v16.8h, v20.8h, v16.8h\n"
     "str q19, [x12, x13]\n"
     "str q18, [x11, x13]\n"
     "str q17, [x10, x13]\n"
     "str q16, [x9, x13]\n"
     "add x13, x13, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "bgt 1b\n"
     "2:"  // Vector: Tail
     "fmax v21.8h, v30.8h, v29.8h\n"
@@ -138,6 +138,7 @@ void a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "cbz x15, 4f\n"
     "3:"  // Oddments
     "ldr h30, [x27, x14]\n"
+    "subs x15, x15, #0x1\n"
     "ldr h29, [x24, x14]\n"
     "fmax v21.8h, v30.8h, v29.8h\n"
     "ldr h28, [x21, x14]\n"
@@ -152,9 +153,8 @@ void a64_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr h22, [x20, x14]\n"
     "add x14, x14, #0x2\n"
     "fmax v18.8h, v25.8h, v24.8h\n"
-    "subs x15, x15, #0x1\n"
-    "fmax v17.8h, v23.8h, v27.8h\n"
     "str h19, [x12, x13]\n"
+    "fmax v17.8h, v23.8h, v27.8h\n"
     "fmax v16.8h, v25.8h, v22.8h\n"
     "fmax v18.8h, v21.8h, v18.8h\n"
     "str h18, [x11, x13]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst.hpp
index c90378532477af0392edb5b1fe8400bf7c670e8f..8bea0bf5df1e5564612bd28adc07b2d3eca24bcb 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_fp16_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = a64_fp16_nhwc_max_generic_depthfirst_impl;
 
   a64_fp16_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp
index 9901b204c0225b307d87ff091adcf2226627ac02..e5f7ee3c72c128b40dc3a70a28fd1600cfc31b2d 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp16_nhwc_max_generic_depthfirst/generic.cpp
@@ -47,23 +47,23 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl(
     "blt 7f\n"
     "1:"  // 4-vectors of channels
     "mov w20, #0xfc00\n"
-    "dup v6.8h, w20\n"
+    "dup v7.8h, w20\n"
     "mov x19, %x[inptrs]\n"
-    "dup v5.8h, w20\n"
+    "dup v6.8h, w20\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "dup v5.8h, w20\n"
     "dup v4.8h, w20\n"
-    "dup v3.8h, w20\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -76,47 +76,47 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "fmax v23.8h, v2.8h, v1.8h\n"
+    "fmax v23.8h, v3.8h, v2.8h\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fmax v19.8h, v0.8h, v31.8h\n"
+    "fmax v19.8h, v1.8h, v0.8h\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fmax v22.8h, v30.8h, v22.8h\n"
     "add x19, x19, #0x20\n"
+    "fmax v22.8h, v31.8h, v30.8h\n"
+    "ldr q3, [x23, x28]\n"
     "fmax v18.8h, v29.8h, v28.8h\n"
-    "ldr q2, [x23, x28]\n"
     "fmax v21.8h, v27.8h, v21.8h\n"
+    "ldr q2, [x22, x28]\n"
     "fmax v17.8h, v26.8h, v17.8h\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "fmax v20.8h, v25.8h, v20.8h\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "fmax v16.8h, v24.8h, v16.8h\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "fmax v19.8h, v23.8h, v19.8h\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "fmax v18.8h, v22.8h, v18.8h\n"
-    "ldr q22, [x22, x27]\n"
-    "fmax v17.8h, v21.8h, v17.8h\n"
     "ldr q29, [x21, x27]\n"
-    "fmax v16.8h, v20.8h, v16.8h\n"
+    "fmax v17.8h, v21.8h, v17.8h\n"
     "ldr q28, [x20, x27]\n"
-    "fmax v6.8h, v6.8h, v19.8h\n"
+    "fmax v16.8h, v20.8h, v16.8h\n"
     "ldr q27, [x23, x26]\n"
-    "fmax v5.8h, v5.8h, v18.8h\n"
+    "fmax v7.8h, v7.8h, v19.8h\n"
     "ldr q21, [x22, x26]\n"
-    "fmax v4.8h, v4.8h, v17.8h\n"
+    "fmax v6.8h, v6.8h, v18.8h\n"
     "ldr q26, [x21, x26]\n"
-    "fmax v3.8h, v3.8h, v16.8h\n"
+    "fmax v5.8h, v5.8h, v17.8h\n"
     "ldr q17, [x20, x26]\n"
+    "fmax v4.8h, v4.8h, v16.8h\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "fmax v23.8h, v2.8h, v1.8h\n"
-    "fmax v19.8h, v0.8h, v31.8h\n"
-    "fmax v22.8h, v30.8h, v22.8h\n"
+    "fmax v23.8h, v3.8h, v2.8h\n"
+    "fmax v19.8h, v1.8h, v0.8h\n"
+    "fmax v22.8h, v31.8h, v30.8h\n"
     "fmax v18.8h, v29.8h, v28.8h\n"
     "fmax v21.8h, v27.8h, v21.8h\n"
     "fmax v17.8h, v26.8h, v17.8h\n"
@@ -126,33 +126,33 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl(
     "fmax v18.8h, v22.8h, v18.8h\n"
     "fmax v17.8h, v21.8h, v17.8h\n"
     "fmax v16.8h, v20.8h, v16.8h\n"
-    "fmax v6.8h, v6.8h, v19.8h\n"
-    "fmax v5.8h, v5.8h, v18.8h\n"
-    "fmax v4.8h, v4.8h, v17.8h\n"
-    "fmax v3.8h, v3.8h, v16.8h\n"
+    "fmax v7.8h, v7.8h, v19.8h\n"
+    "fmax v6.8h, v6.8h, v18.8h\n"
+    "fmax v5.8h, v5.8h, v17.8h\n"
+    "fmax v4.8h, v4.8h, v16.8h\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fmax v6.8h, v6.8h, v2.8h\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "fmax v7.8h, v7.8h, v3.8h\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "fmax v5.8h, v5.8h, v30.8h\n"
+    "fmax v6.8h, v6.8h, v31.8h\n"
     "ldr q25, [x23, x25]\n"
-    "fmax v4.8h, v4.8h, v27.8h\n"
-    "fmax v3.8h, v3.8h, v25.8h\n"
+    "fmax v5.8h, v5.8h, v27.8h\n"
+    "fmax v4.8h, v4.8h, v25.8h\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
-    "str q5, [%x[outptr], x27]\n"
-    "str q4, [%x[outptr], x26]\n"
-    "str q3, [%x[outptr], x25]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x40\n"
+    "str q6, [%x[outptr], x27]\n"
     "add x27, x27, #0x40\n"
+    "str q5, [%x[outptr], x26]\n"
     "add x26, x26, #0x40\n"
+    "str q4, [%x[outptr], x25]\n"
     "add x25, x25, #0x40\n"
     "sub %x[n_channels], %x[n_channels], #0x20\n"
     "cmp %x[n_channels], #0x20\n"
@@ -163,49 +163,49 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl(
     "blt 14f\n"
     "8:"  // Single vector of channels: Loop
     "mov w19, #0xfc00\n"
-    "dup v6.8h, w19\n"
+    "dup v7.8h, w19\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "fmax v23.8h, v2.8h, v1.8h\n"
+    "fmax v23.8h, v3.8h, v2.8h\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fmax v19.8h, v0.8h, v31.8h\n"
+    "fmax v19.8h, v1.8h, v0.8h\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fmax v19.8h, v23.8h, v19.8h\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "fmax v6.8h, v6.8h, v19.8h\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "fmax v19.8h, v23.8h, v19.8h\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "fmax v7.8h, v7.8h, v19.8h\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "fmax v23.8h, v2.8h, v1.8h\n"
-    "fmax v19.8h, v0.8h, v31.8h\n"
+    "fmax v23.8h, v3.8h, v2.8h\n"
+    "fmax v19.8h, v1.8h, v0.8h\n"
     "fmax v19.8h, v23.8h, v19.8h\n"
-    "fmax v6.8h, v6.8h, v19.8h\n"
+    "fmax v7.8h, v7.8h, v19.8h\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fmax v6.8h, v6.8h, v2.8h\n"
+    "ldr q3, [x23, x28]\n"
+    "fmax v7.8h, v7.8h, v3.8h\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x10\n"
     "sub %x[n_channels], %x[n_channels], #0x8\n"
     "cmp %x[n_channels], #0x8\n"
@@ -214,129 +214,129 @@ void a64_fp16_nhwc_max_generic_depthfirst_impl(
     "14:"  // Oddments
     "add %x[outptr], %x[outptr], x28\n"
     "mov w19, #0xfc00\n"
-    "dup v6.8h, w19\n"
+    "dup v7.8h, w19\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 20f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #2, 17f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "tbz %x[n_channels], #0, 19f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
-    "ld1 { v1.h }[6], [x22], #0x2\n"
-    "ld1 { v0.h }[6], [x21], #0x2\n"
-    "ld1 { v31.h }[6], [x20], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
+    "ld1 { v2.h }[6], [x22], #0x2\n"
+    "ld1 { v1.h }[6], [x21], #0x2\n"
+    "ld1 { v0.h }[6], [x20], #0x2\n"
     "b 19f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 19f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
-    "ld1 { v1.h }[4], [x22], #0x2\n"
-    "ld1 { v0.h }[4], [x21], #0x2\n"
-    "ld1 { v31.h }[4], [x20], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
+    "ld1 { v2.h }[4], [x22], #0x2\n"
+    "ld1 { v1.h }[4], [x21], #0x2\n"
+    "ld1 { v0.h }[4], [x20], #0x2\n"
     "b 19f\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 2: Unset
     "tbz %x[n_channels], #1, 18f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "tbz %x[n_channels], #0, 19f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
-    "ld1 { v1.h }[2], [x22], #0x2\n"
-    "ld1 { v0.h }[2], [x21], #0x2\n"
-    "ld1 { v31.h }[2], [x20], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
+    "ld1 { v2.h }[2], [x22], #0x2\n"
+    "ld1 { v1.h }[2], [x21], #0x2\n"
+    "ld1 { v0.h }[2], [x20], #0x2\n"
     "b 19f\n"
     "18:"  // Oddments: 4 inputs loop: Load: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 19f\n"
-    "ldr h2, [x23], #0x2\n"
-    "ldr h1, [x22], #0x2\n"
-    "ldr h0, [x21], #0x2\n"
-    "ldr h31, [x20], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
+    "ldr h2, [x22], #0x2\n"
+    "ldr h1, [x21], #0x2\n"
+    "ldr h0, [x20], #0x2\n"
     "19:"  // Oddments: 4 inputs loop: Load: Bit 2: End
-    "fmax v23.8h, v2.8h, v1.8h\n"
+    "fmax v23.8h, v3.8h, v2.8h\n"
     "subs x24, x24, #0x1\n"
-    "fmax v19.8h, v0.8h, v31.8h\n"
+    "fmax v19.8h, v1.8h, v0.8h\n"
     "fmax v19.8h, v23.8h, v19.8h\n"
-    "fmax v6.8h, v6.8h, v19.8h\n"
+    "fmax v7.8h, v7.8h, v19.8h\n"
     "bgt 15b\n"
     "20:"  // Oddments: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 26f\n"
     "21:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #2, 23f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #1, 22f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "tbz %x[n_channels], #0, 25f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
     "b 25f\n"
     "22:"  // Oddments: Single input loop: Load: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 25f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
     "b 25f\n"
     "23:"  // Oddments: Single input loop: Load: Bit 2: Unset
     "tbz %x[n_channels], #1, 24f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "tbz %x[n_channels], #0, 25f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
     "b 25f\n"
     "24:"  // Oddments: Single input loop: Load: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 25f\n"
-    "ldr h2, [x23], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
     "25:"  // Oddments: Single input loop: Load: Bit 2: End
-    "fmax v6.8h, v6.8h, v2.8h\n"
+    "fmax v7.8h, v7.8h, v3.8h\n"
     "subs x20, x20, #0x1\n"
     "bgt 21b\n"
     "26:"  // Oddments: Single input loop: End
     "tbz %x[n_channels], #2, 28f\n"
-    "st1 { v6.d }[0], [%x[outptr]], #0x8\n"
+    "st1 { v7.d }[0], [%x[outptr]], #0x8\n"
     "tbz %x[n_channels], #1, 27f\n"
-    "st1 { v6.s }[2], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[2], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[6], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[6], [%x[outptr]], #0x2\n"
     "b 30f\n"
     "27:"  // Oddments: Store: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[4], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[4], [%x[outptr]], #0x2\n"
     "b 30f\n"
     "28:"  // Oddments: Store: Bit 2: Unset
     "tbz %x[n_channels], #1, 29f\n"
-    "st1 { v6.s }[0], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[0], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[2], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[2], [%x[outptr]], #0x2\n"
     "b 30f\n"
     "29:"  // Oddments: Store: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 30f\n"
-    "st1 { v6.h }[0], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[0], [%x[outptr]], #0x2\n"
     "30:"  // Oddments: Store: Bit 2: End
 
     "31:"  // End
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
index bed484854b08d5b98fa16e0bdbcdee285514142a..ff8d7d8ba15ecae42472c439ff3f8c71424d9a67 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
@@ -82,13 +82,12 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x4, [%x[args], %[offsetof_n_channels]]\n"
     "mov x5, #0x0\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x6, #0x0\n"
-    "ldr q8, [%x[args], %[offsetof_rescale]]\n"
-    "ldp x7, x8, [x19, #0x0]\n"
-    "cmp x4, #0x4\n"
-    "ldp x17, x16, [x19, #0x10]\n"
     "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "cmp x4, #0x4\n"
+    "ldp x7, x8, [x20, #0x0]\n"
+    "ldp x17, x16, [x20, #0x10]\n"
     "ldp x15, x14, [x19, #0x0]\n"
     "ldp x13, x12, [x19, #0x10]\n"
     "ldp x11, x10, [x19, #0x20]\n"
@@ -97,12 +96,14 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldp x25, x24, [x19, #0x50]\n"
     "ldp x23, x22, [x19, #0x60]\n"
     "ldp x21, x20, [x19, #0x70]\n"
+    "ldr q8, [%x[args], %[offsetof_rescale]]\n"
     "blt 3f\n"
-    "lsr x19, x4, #0x2\n"
-    "sub x4, x4, x19, LSL #2\n"
     "ldr q7, [x10, x5]\n"
+    "lsr x19, x4, #0x2\n"
     "ldr q6, [x9, x5]\n"
+    "sub x4, x4, x19, LSL #2\n"
     "ldr q5, [x26, x5]\n"
+    "subs x19, x19, #0x1\n"
     "ldr q4, [x25, x5]\n"
     "ldr q3, [x14, x5]\n"
     "ldr q2, [x13, x5]\n"
@@ -117,26 +118,26 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldr q25, [x23, x5]\n"
     "ldr q24, [x20, x5]\n"
     "add x5, x5, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "beq 2f\n"
     "1:"  // Vector: Loop
     "fadd v17.4s, v7.4s, v6.4s\n"
     "ldr q7, [x10, x5]\n"
+    "subs x19, x19, #0x1\n"
     "fadd v16.4s, v5.4s, v4.4s\n"
     "ldr q6, [x9, x5]\n"
     "fadd v18.4s, v3.4s, v2.4s\n"
     "ldr q5, [x26, x5]\n"
     "fadd v23.4s, v1.4s, v0.4s\n"
     "ldr q4, [x25, x5]\n"
-    "fadd v17.4s, v17.4s, v16.4s\n"
-    "ldr q3, [x14, x5]\n"
     "fadd v22.4s, v31.4s, v30.4s\n"
+    "ldr q3, [x14, x5]\n"
+    "fadd v17.4s, v17.4s, v16.4s\n"
     "ldr q2, [x13, x5]\n"
     "fadd v16.4s, v29.4s, v28.4s\n"
     "ldr q1, [x11, x5]\n"
-    "fadd v21.4s, v18.4s, v17.4s\n"
-    "ldr q0, [x27, x5]\n"
     "fadd v19.4s, v27.4s, v23.4s\n"
+    "ldr q0, [x27, x5]\n"
+    "fadd v21.4s, v18.4s, v17.4s\n"
     "ldr q31, [x28, x5]\n"
     "fadd v20.4s, v16.4s, v17.4s\n"
     "ldr q30, [x24, x5]\n"
@@ -146,21 +147,20 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldr q28, [x21, x5]\n"
     "fadd v16.4s, v24.4s, v22.4s\n"
     "ldr q27, [x15, x5]\n"
-    "fadd v19.4s, v19.4s, v21.4s\n"
+    "fadd v19.4s, v21.4s, v19.4s\n"
     "ldr q26, [x12, x5]\n"
     "fadd v18.4s, v21.4s, v18.4s\n"
     "ldr q25, [x23, x5]\n"
     "fadd v17.4s, v17.4s, v20.4s\n"
     "ldr q24, [x20, x5]\n"
-    "fadd v16.4s, v20.4s, v16.4s\n"
     "add x5, x5, #0x10\n"
+    "fadd v16.4s, v20.4s, v16.4s\n"
     "fmul v19.4s, v19.4s, v8.s[0]\n"
-    "subs x19, x19, #0x1\n"
-    "fmul v18.4s, v18.4s, v8.s[1]\n"
     "str q19, [x7, x6]\n"
+    "fmul v18.4s, v18.4s, v8.s[1]\n"
     "fmul v17.4s, v17.4s, v8.s[2]\n"
-    "fmul v16.4s, v16.4s, v8.s[3]\n"
     "str q18, [x8, x6]\n"
+    "fmul v16.4s, v16.4s, v8.s[3]\n"
     "str q17, [x17, x6]\n"
     "str q16, [x16, x6]\n"
     "add x6, x6, #0x10\n"
@@ -179,7 +179,7 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "fadd v18.4s, v26.4s, v22.4s\n"
     "fadd v17.4s, v25.4s, v23.4s\n"
     "fadd v16.4s, v24.4s, v22.4s\n"
-    "fadd v19.4s, v19.4s, v21.4s\n"
+    "fadd v19.4s, v21.4s, v19.4s\n"
     "fadd v18.4s, v21.4s, v18.4s\n"
     "fadd v17.4s, v17.4s, v20.4s\n"
     "fadd v16.4s, v20.4s, v16.4s\n"
@@ -195,6 +195,7 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "cbz x4, 4f\n"
     "3:"  // Oddments
     "ldr s7, [x10, x5]\n"
+    "subs x4, x4, #0x1\n"
     "ldr s6, [x9, x5]\n"
     "fadd v17.4s, v7.4s, v6.4s\n"
     "ldr s5, [x26, x5]\n"
@@ -207,33 +208,32 @@ void a64_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ldr s0, [x27, x5]\n"
     "fadd v18.4s, v3.4s, v2.4s\n"
     "ldr s31, [x28, x5]\n"
-    "ldr s30, [x24, x5]\n"
     "fadd v23.4s, v1.4s, v0.4s\n"
-    "ldr s29, [x22, x5]\n"
+    "ldr s30, [x24, x5]\n"
     "fadd v21.4s, v18.4s, v17.4s\n"
+    "ldr s29, [x22, x5]\n"
     "ldr s28, [x21, x5]\n"
-    "ldr s27, [x15, x5]\n"
     "fadd v22.4s, v31.4s, v30.4s\n"
+    "ldr s27, [x15, x5]\n"
     "ldr s26, [x12, x5]\n"
     "fadd v16.4s, v29.4s, v28.4s\n"
     "ldr s25, [x23, x5]\n"
-    "fadd v19.4s, v27.4s, v23.4s\n"
+    "fadd v20.4s, v16.4s, v17.4s\n"
     "ldr s24, [x20, x5]\n"
-    "fadd v18.4s, v26.4s, v22.4s\n"
     "add x5, x5, #0x4\n"
-    "subs x4, x4, #0x1\n"
-    "fadd v20.4s, v16.4s, v17.4s\n"
-    "fadd v19.4s, v19.4s, v21.4s\n"
-    "fadd v18.4s, v21.4s, v18.4s\n"
+    "fadd v19.4s, v27.4s, v23.4s\n"
+    "fadd v18.4s, v26.4s, v22.4s\n"
     "fadd v17.4s, v25.4s, v23.4s\n"
     "fadd v16.4s, v24.4s, v22.4s\n"
-    "fmul v19.4s, v19.4s, v8.s[0]\n"
-    "str s19, [x7, x6]\n"
+    "fadd v19.4s, v21.4s, v19.4s\n"
+    "fadd v18.4s, v21.4s, v18.4s\n"
     "fadd v17.4s, v17.4s, v20.4s\n"
     "fadd v16.4s, v20.4s, v16.4s\n"
+    "fmul v19.4s, v19.4s, v8.s[0]\n"
+    "str s19, [x7, x6]\n"
     "fmul v18.4s, v18.4s, v8.s[1]\n"
-    "str s18, [x8, x6]\n"
     "fmul v17.4s, v17.4s, v8.s[2]\n"
+    "str s18, [x8, x6]\n"
     "fmul v16.4s, v16.4s, v8.s[3]\n"
     "str s17, [x17, x6]\n"
     "str s16, [x16, x6]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst.hpp
index e5a465ed75e574201ddb9d74ffd1e20fd1f3b9a5..4ef26318d499eae46b094690bd00271587cf5ae2 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_fp32_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = a64_fp32_nhwc_avg_generic_depthfirst_impl;
 
   a64_fp32_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp
index f607518c26b3d07c0d47034fd7af25a146384fb3..21f705451addc3dd00a9b32d48bed9c489595c4a 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_avg_generic_depthfirst/generic.cpp
@@ -41,7 +41,7 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl(
   const auto rescale_value = static_cast<float>(1.0f / static_cast<float>(window_cells));
 
   __asm__ __volatile__(
-    "ld1r { v7.4s }, [%x[rescale_ptr]]\n"
+    "ld1r { v8.4s }, [%x[rescale_ptr]]\n"
     "mov x28, #0x0\n"
     "mov x27, #0x10\n" // cntb _, ALL, #1
     "mov x26, #0x20\n" // cntb _, ALL, #2
@@ -49,23 +49,23 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x10\n"
     "blt 7f\n"
     "1:"  // 4-vectors of channels
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "mov x19, %x[inptrs]\n"
-    "movi v5.16b, #0x0\n"
+    "movi v6.16b, #0x0\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "movi v5.16b, #0x0\n"
     "movi v4.16b, #0x0\n"
-    "movi v3.16b, #0x0\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -78,47 +78,47 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "fadd v23.4s, v2.4s, v1.4s\n"
+    "fadd v23.4s, v3.4s, v2.4s\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd v19.4s, v0.4s, v31.4s\n"
+    "fadd v19.4s, v1.4s, v0.4s\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd v22.4s, v30.4s, v22.4s\n"
     "add x19, x19, #0x20\n"
+    "fadd v22.4s, v31.4s, v30.4s\n"
+    "ldr q3, [x23, x28]\n"
     "fadd v18.4s, v29.4s, v28.4s\n"
-    "ldr q2, [x23, x28]\n"
     "fadd v21.4s, v27.4s, v21.4s\n"
+    "ldr q2, [x22, x28]\n"
     "fadd v17.4s, v26.4s, v17.4s\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "fadd v20.4s, v25.4s, v20.4s\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "fadd v16.4s, v24.4s, v16.4s\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "fadd v19.4s, v23.4s, v19.4s\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "fadd v18.4s, v22.4s, v18.4s\n"
-    "ldr q22, [x22, x27]\n"
-    "fadd v17.4s, v21.4s, v17.4s\n"
     "ldr q29, [x21, x27]\n"
-    "fadd v16.4s, v20.4s, v16.4s\n"
+    "fadd v17.4s, v21.4s, v17.4s\n"
     "ldr q28, [x20, x27]\n"
-    "fadd v6.4s, v6.4s, v19.4s\n"
+    "fadd v16.4s, v20.4s, v16.4s\n"
     "ldr q27, [x23, x26]\n"
-    "fadd v5.4s, v5.4s, v18.4s\n"
+    "fadd v7.4s, v7.4s, v19.4s\n"
     "ldr q21, [x22, x26]\n"
-    "fadd v4.4s, v4.4s, v17.4s\n"
+    "fadd v6.4s, v6.4s, v18.4s\n"
     "ldr q26, [x21, x26]\n"
-    "fadd v3.4s, v3.4s, v16.4s\n"
+    "fadd v5.4s, v5.4s, v17.4s\n"
     "ldr q17, [x20, x26]\n"
+    "fadd v4.4s, v4.4s, v16.4s\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "fadd v23.4s, v2.4s, v1.4s\n"
-    "fadd v19.4s, v0.4s, v31.4s\n"
-    "fadd v22.4s, v30.4s, v22.4s\n"
+    "fadd v23.4s, v3.4s, v2.4s\n"
+    "fadd v19.4s, v1.4s, v0.4s\n"
+    "fadd v22.4s, v31.4s, v30.4s\n"
     "fadd v18.4s, v29.4s, v28.4s\n"
     "fadd v21.4s, v27.4s, v21.4s\n"
     "fadd v17.4s, v26.4s, v17.4s\n"
@@ -128,173 +128,173 @@ void a64_fp32_nhwc_avg_generic_depthfirst_impl(
     "fadd v18.4s, v22.4s, v18.4s\n"
     "fadd v17.4s, v21.4s, v17.4s\n"
     "fadd v16.4s, v20.4s, v16.4s\n"
-    "fadd v6.4s, v6.4s, v19.4s\n"
-    "fadd v5.4s, v5.4s, v18.4s\n"
-    "fadd v4.4s, v4.4s, v17.4s\n"
-    "fadd v3.4s, v3.4s, v16.4s\n"
+    "fadd v7.4s, v7.4s, v19.4s\n"
+    "fadd v6.4s, v6.4s, v18.4s\n"
+    "fadd v5.4s, v5.4s, v17.4s\n"
+    "fadd v4.4s, v4.4s, v16.4s\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fadd v6.4s, v6.4s, v2.4s\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "fadd v7.4s, v7.4s, v3.4s\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "fadd v5.4s, v5.4s, v30.4s\n"
+    "fadd v6.4s, v6.4s, v31.4s\n"
     "ldr q25, [x23, x25]\n"
-    "fadd v4.4s, v4.4s, v27.4s\n"
-    "fadd v3.4s, v3.4s, v25.4s\n"
+    "fadd v5.4s, v5.4s, v27.4s\n"
+    "fadd v4.4s, v4.4s, v25.4s\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "fmul v6.4s, v6.4s, v7.4s\n"
-    "sub %x[n_channels], %x[n_channels], #0x10\n"
-    "fmul v5.4s, v5.4s, v7.4s\n"
-    "cmp %x[n_channels], #0x10\n"
-    "fmul v4.4s, v4.4s, v7.4s\n"
-    "str q6, [%x[outptr], x28]\n"
-    "fmul v3.4s, v3.4s, v7.4s\n"
-    "str q5, [%x[outptr], x27]\n"
-    "str q4, [%x[outptr], x26]\n"
+    "fmul v7.4s, v7.4s, v8.4s\n"
+    "str q7, [%x[outptr], x28]\n"
+    "fmul v6.4s, v6.4s, v8.4s\n"
     "add x28, x28, #0x40\n"
+    "fmul v5.4s, v5.4s, v8.4s\n"
+    "str q6, [%x[outptr], x27]\n"
+    "fmul v4.4s, v4.4s, v8.4s\n"
     "add x27, x27, #0x40\n"
-    "str q3, [%x[outptr], x25]\n"
+    "str q5, [%x[outptr], x26]\n"
     "add x26, x26, #0x40\n"
+    "sub %x[n_channels], %x[n_channels], #0x10\n"
+    "str q4, [%x[outptr], x25]\n"
     "add x25, x25, #0x40\n"
+    "cmp %x[n_channels], #0x10\n"
     "bge 1b\n"
     "cbz %x[n_channels], 25f\n"
     "7:"  // Single vector of channels
     "cmp %x[n_channels], #0x4\n"
     "blt 14f\n"
     "8:"  // Single vector of channels: Loop
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "fadd v23.4s, v2.4s, v1.4s\n"
+    "fadd v23.4s, v3.4s, v2.4s\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd v19.4s, v0.4s, v31.4s\n"
+    "fadd v19.4s, v1.4s, v0.4s\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd v19.4s, v23.4s, v19.4s\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "fadd v6.4s, v6.4s, v19.4s\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "fadd v19.4s, v23.4s, v19.4s\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "fadd v7.4s, v7.4s, v19.4s\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "fadd v23.4s, v2.4s, v1.4s\n"
-    "fadd v19.4s, v0.4s, v31.4s\n"
+    "fadd v23.4s, v3.4s, v2.4s\n"
+    "fadd v19.4s, v1.4s, v0.4s\n"
     "fadd v19.4s, v23.4s, v19.4s\n"
-    "fadd v6.4s, v6.4s, v19.4s\n"
+    "fadd v7.4s, v7.4s, v19.4s\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fadd v6.4s, v6.4s, v2.4s\n"
+    "ldr q3, [x23, x28]\n"
+    "fadd v7.4s, v7.4s, v3.4s\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "fmul v6.4s, v6.4s, v7.4s\n"
+    "fmul v7.4s, v7.4s, v8.4s\n"
+    "str q7, [%x[outptr], x28]\n"
+    "add x28, x28, #0x10\n"
     "sub %x[n_channels], %x[n_channels], #0x4\n"
     "cmp %x[n_channels], #0x4\n"
-    "str q6, [%x[outptr], x28]\n"
-    "add x28, x28, #0x10\n"
     "bge 8b\n"
     "cbz %x[n_channels], 25f\n"
     "14:"  // Oddments
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "add %x[outptr], %x[outptr], x28\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 18f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #0, 17f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "b 17f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 1: Unset
     "tbz %x[n_channels], #0, 17f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 1: End
-    "fadd v23.4s, v2.4s, v1.4s\n"
+    "fadd v23.4s, v3.4s, v2.4s\n"
     "subs x24, x24, #0x1\n"
-    "fadd v19.4s, v0.4s, v31.4s\n"
+    "fadd v19.4s, v1.4s, v0.4s\n"
     "fadd v19.4s, v23.4s, v19.4s\n"
-    "fadd v6.4s, v6.4s, v19.4s\n"
+    "fadd v7.4s, v7.4s, v19.4s\n"
     "bgt 15b\n"
     "18:"  // Oddments: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 22f\n"
     "19:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #1, 20f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #0, 21f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "b 21f\n"
     "20:"  // Oddments: Single input loop: Load: Bit 1: Unset
     "tbz %x[n_channels], #0, 21f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "21:"  // Oddments: Single input loop: Load: Bit 1: End
-    "fadd v6.4s, v6.4s, v2.4s\n"
+    "fadd v7.4s, v7.4s, v3.4s\n"
     "subs x20, x20, #0x1\n"
     "bgt 19b\n"
     "22:"  // Oddments: Single input loop: End
-    "fmul v6.4s, v6.4s, v7.4s\n"
+    "fmul v7.4s, v7.4s, v8.4s\n"
     "tbz %x[n_channels], #1, 23f\n"
-    "st1 { v6.d }[0], [%x[outptr]], #0x8\n"
+    "st1 { v7.d }[0], [%x[outptr]], #0x8\n"
     "tbz %x[n_channels], #0, 24f\n"
-    "st1 { v6.s }[2], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[2], [%x[outptr]], #0x4\n"
     "b 24f\n"
     "23:"  // Oddments: Store: Bit 1: Unset
     "tbz %x[n_channels], #0, 24f\n"
-    "st1 { v6.s }[0], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[0], [%x[outptr]], #0x4\n"
     "24:"  // Oddments: Store: Bit 1: End
 
     "25:"  // End
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [rescale_ptr] "r" (&rescale_value)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index 9ad4a39a835eaed859b64b163129464942f48220..ea7e2195d1112d914c1ebcb01920fb4855ee4934 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -63,23 +63,24 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x15, [%x[args], %[offsetof_n_channels]]\n"
     "mov x14, #0x0\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
-    "ldp x12, x11, [x19, #0x0]\n"
-    "cmp x15, #0x4\n"
-    "ldp x10, x9, [x19, #0x10]\n"
     "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "cmp x15, #0x4\n"
+    "ldp x12, x11, [x20, #0x0]\n"
+    "ldp x10, x9, [x20, #0x10]\n"
     "ldp x28, x27, [x19, #0x0]\n"
     "ldp x26, x25, [x19, #0x10]\n"
     "ldp x24, x23, [x19, #0x20]\n"
     "ldp x22, x21, [x19, #0x30]\n"
     "ldr x20, [x19, #0x40]\n"
     "blt 3f\n"
-    "lsr x19, x15, #0x2\n"
-    "sub x15, x15, x19, LSL #2\n"
     "ldr q30, [x27, x14]\n"
+    "lsr x19, x15, #0x2\n"
     "ldr q29, [x24, x14]\n"
+    "sub x15, x15, x19, LSL #2\n"
     "ldr q28, [x21, x14]\n"
+    "subs x19, x19, #0x1\n"
     "ldr q27, [x25, x14]\n"
     "ldr q26, [x28, x14]\n"
     "ldr q25, [x23, x14]\n"
@@ -87,11 +88,11 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "ldr q22, [x20, x14]\n"
     "add x14, x14, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "beq 2f\n"
     "1:"  // Vector: Loop
     "fmax v21.4s, v30.4s, v29.4s\n"
     "ldr q30, [x27, x14]\n"
+    "subs x19, x19, #0x1\n"
     "fmax v20.4s, v29.4s, v28.4s\n"
     "ldr q29, [x24, x14]\n"
     "fmax v19.4s, v27.4s, v26.4s\n"
@@ -108,14 +109,13 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "fmax v17.4s, v20.4s, v17.4s\n"
     "ldr q22, [x20, x14]\n"
-    "fmax v16.4s, v20.4s, v16.4s\n"
     "add x14, x14, #0x10\n"
+    "fmax v16.4s, v20.4s, v16.4s\n"
     "str q19, [x12, x13]\n"
     "str q18, [x11, x13]\n"
     "str q17, [x10, x13]\n"
     "str q16, [x9, x13]\n"
     "add x13, x13, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "bgt 1b\n"
     "2:"  // Vector: Tail
     "fmax v21.4s, v30.4s, v29.4s\n"
@@ -136,6 +136,7 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "cbz x15, 4f\n"
     "3:"  // Oddments
     "ldr s30, [x27, x14]\n"
+    "subs x15, x15, #0x1\n"
     "ldr s29, [x24, x14]\n"
     "fmax v21.4s, v30.4s, v29.4s\n"
     "ldr s28, [x21, x14]\n"
@@ -150,9 +151,8 @@ void a64_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr s22, [x20, x14]\n"
     "add x14, x14, #0x4\n"
     "fmax v18.4s, v25.4s, v24.4s\n"
-    "subs x15, x15, #0x1\n"
-    "fmax v17.4s, v23.4s, v27.4s\n"
     "str s19, [x12, x13]\n"
+    "fmax v17.4s, v23.4s, v27.4s\n"
     "fmax v16.4s, v25.4s, v22.4s\n"
     "fmax v18.4s, v21.4s, v18.4s\n"
     "str s18, [x11, x13]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst.hpp
index 4b39237a35cae162b2455e83280369d930b277c6..b20ffc20cff67fcda445ed89f8a5972585cf6794 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_fp32_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = a64_fp32_nhwc_max_generic_depthfirst_impl;
 
   a64_fp32_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp
index f9619b9eb2376a7e1e5d5550157d489720f5418d..e0acb7ac02d068c9cc1ed73b9c4e05586788d36c 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_fp32_nhwc_max_generic_depthfirst/generic.cpp
@@ -47,23 +47,23 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl(
     "blt 7f\n"
     "1:"  // 4-vectors of channels
     "mov w20, #0xff800000\n"
-    "dup v6.4s, w20\n"
+    "dup v7.4s, w20\n"
     "mov x19, %x[inptrs]\n"
-    "dup v5.4s, w20\n"
+    "dup v6.4s, w20\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "dup v5.4s, w20\n"
     "dup v4.4s, w20\n"
-    "dup v3.4s, w20\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -76,47 +76,47 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "fmax v23.4s, v2.4s, v1.4s\n"
+    "fmax v23.4s, v3.4s, v2.4s\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fmax v19.4s, v0.4s, v31.4s\n"
+    "fmax v19.4s, v1.4s, v0.4s\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fmax v22.4s, v30.4s, v22.4s\n"
     "add x19, x19, #0x20\n"
+    "fmax v22.4s, v31.4s, v30.4s\n"
+    "ldr q3, [x23, x28]\n"
     "fmax v18.4s, v29.4s, v28.4s\n"
-    "ldr q2, [x23, x28]\n"
     "fmax v21.4s, v27.4s, v21.4s\n"
+    "ldr q2, [x22, x28]\n"
     "fmax v17.4s, v26.4s, v17.4s\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "fmax v20.4s, v25.4s, v20.4s\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "fmax v16.4s, v24.4s, v16.4s\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "fmax v19.4s, v23.4s, v19.4s\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "fmax v18.4s, v22.4s, v18.4s\n"
-    "ldr q22, [x22, x27]\n"
-    "fmax v17.4s, v21.4s, v17.4s\n"
     "ldr q29, [x21, x27]\n"
-    "fmax v16.4s, v20.4s, v16.4s\n"
+    "fmax v17.4s, v21.4s, v17.4s\n"
     "ldr q28, [x20, x27]\n"
-    "fmax v6.4s, v6.4s, v19.4s\n"
+    "fmax v16.4s, v20.4s, v16.4s\n"
     "ldr q27, [x23, x26]\n"
-    "fmax v5.4s, v5.4s, v18.4s\n"
+    "fmax v7.4s, v7.4s, v19.4s\n"
     "ldr q21, [x22, x26]\n"
-    "fmax v4.4s, v4.4s, v17.4s\n"
+    "fmax v6.4s, v6.4s, v18.4s\n"
     "ldr q26, [x21, x26]\n"
-    "fmax v3.4s, v3.4s, v16.4s\n"
+    "fmax v5.4s, v5.4s, v17.4s\n"
     "ldr q17, [x20, x26]\n"
+    "fmax v4.4s, v4.4s, v16.4s\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "fmax v23.4s, v2.4s, v1.4s\n"
-    "fmax v19.4s, v0.4s, v31.4s\n"
-    "fmax v22.4s, v30.4s, v22.4s\n"
+    "fmax v23.4s, v3.4s, v2.4s\n"
+    "fmax v19.4s, v1.4s, v0.4s\n"
+    "fmax v22.4s, v31.4s, v30.4s\n"
     "fmax v18.4s, v29.4s, v28.4s\n"
     "fmax v21.4s, v27.4s, v21.4s\n"
     "fmax v17.4s, v26.4s, v17.4s\n"
@@ -126,33 +126,33 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl(
     "fmax v18.4s, v22.4s, v18.4s\n"
     "fmax v17.4s, v21.4s, v17.4s\n"
     "fmax v16.4s, v20.4s, v16.4s\n"
-    "fmax v6.4s, v6.4s, v19.4s\n"
-    "fmax v5.4s, v5.4s, v18.4s\n"
-    "fmax v4.4s, v4.4s, v17.4s\n"
-    "fmax v3.4s, v3.4s, v16.4s\n"
+    "fmax v7.4s, v7.4s, v19.4s\n"
+    "fmax v6.4s, v6.4s, v18.4s\n"
+    "fmax v5.4s, v5.4s, v17.4s\n"
+    "fmax v4.4s, v4.4s, v16.4s\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fmax v6.4s, v6.4s, v2.4s\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "fmax v7.4s, v7.4s, v3.4s\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "fmax v5.4s, v5.4s, v30.4s\n"
+    "fmax v6.4s, v6.4s, v31.4s\n"
     "ldr q25, [x23, x25]\n"
-    "fmax v4.4s, v4.4s, v27.4s\n"
-    "fmax v3.4s, v3.4s, v25.4s\n"
+    "fmax v5.4s, v5.4s, v27.4s\n"
+    "fmax v4.4s, v4.4s, v25.4s\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
-    "str q5, [%x[outptr], x27]\n"
-    "str q4, [%x[outptr], x26]\n"
-    "str q3, [%x[outptr], x25]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x40\n"
+    "str q6, [%x[outptr], x27]\n"
     "add x27, x27, #0x40\n"
+    "str q5, [%x[outptr], x26]\n"
     "add x26, x26, #0x40\n"
+    "str q4, [%x[outptr], x25]\n"
     "add x25, x25, #0x40\n"
     "sub %x[n_channels], %x[n_channels], #0x10\n"
     "cmp %x[n_channels], #0x10\n"
@@ -163,49 +163,49 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl(
     "blt 14f\n"
     "8:"  // Single vector of channels: Loop
     "mov w19, #0xff800000\n"
-    "dup v6.4s, w19\n"
+    "dup v7.4s, w19\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "fmax v23.4s, v2.4s, v1.4s\n"
+    "fmax v23.4s, v3.4s, v2.4s\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fmax v19.4s, v0.4s, v31.4s\n"
+    "fmax v19.4s, v1.4s, v0.4s\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fmax v19.4s, v23.4s, v19.4s\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "fmax v6.4s, v6.4s, v19.4s\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "fmax v19.4s, v23.4s, v19.4s\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "fmax v7.4s, v7.4s, v19.4s\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "fmax v23.4s, v2.4s, v1.4s\n"
-    "fmax v19.4s, v0.4s, v31.4s\n"
+    "fmax v23.4s, v3.4s, v2.4s\n"
+    "fmax v19.4s, v1.4s, v0.4s\n"
     "fmax v19.4s, v23.4s, v19.4s\n"
-    "fmax v6.4s, v6.4s, v19.4s\n"
+    "fmax v7.4s, v7.4s, v19.4s\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "fmax v6.4s, v6.4s, v2.4s\n"
+    "ldr q3, [x23, x28]\n"
+    "fmax v7.4s, v7.4s, v3.4s\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x10\n"
     "sub %x[n_channels], %x[n_channels], #0x4\n"
     "cmp %x[n_channels], #0x4\n"
@@ -214,81 +214,81 @@ void a64_fp32_nhwc_max_generic_depthfirst_impl(
     "14:"  // Oddments
     "add %x[outptr], %x[outptr], x28\n"
     "mov w19, #0xff800000\n"
-    "dup v6.4s, w19\n"
+    "dup v7.4s, w19\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 18f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #0, 17f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "b 17f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 1: Unset
     "tbz %x[n_channels], #0, 17f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 1: End
-    "fmax v23.4s, v2.4s, v1.4s\n"
+    "fmax v23.4s, v3.4s, v2.4s\n"
     "subs x24, x24, #0x1\n"
-    "fmax v19.4s, v0.4s, v31.4s\n"
+    "fmax v19.4s, v1.4s, v0.4s\n"
     "fmax v19.4s, v23.4s, v19.4s\n"
-    "fmax v6.4s, v6.4s, v19.4s\n"
+    "fmax v7.4s, v7.4s, v19.4s\n"
     "bgt 15b\n"
     "18:"  // Oddments: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 22f\n"
     "19:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #1, 20f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #0, 21f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "b 21f\n"
     "20:"  // Oddments: Single input loop: Load: Bit 1: Unset
     "tbz %x[n_channels], #0, 21f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "21:"  // Oddments: Single input loop: Load: Bit 1: End
-    "fmax v6.4s, v6.4s, v2.4s\n"
+    "fmax v7.4s, v7.4s, v3.4s\n"
     "subs x20, x20, #0x1\n"
     "bgt 19b\n"
     "22:"  // Oddments: Single input loop: End
     "tbz %x[n_channels], #1, 23f\n"
-    "st1 { v6.d }[0], [%x[outptr]], #0x8\n"
+    "st1 { v7.d }[0], [%x[outptr]], #0x8\n"
     "tbz %x[n_channels], #0, 24f\n"
-    "st1 { v6.s }[2], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[2], [%x[outptr]], #0x4\n"
     "b 24f\n"
     "23:"  // Oddments: Store: Bit 1: Unset
     "tbz %x[n_channels], #0, 24f\n"
-    "st1 { v6.s }[0], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[0], [%x[outptr]], #0x4\n"
     "24:"  // Oddments: Store: Bit 1: End
 
     "25:"  // End
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst.hpp
index da97f773dc32166153d72302fa0655c412501634..df66ab7a2c3f0668975d6181ed2054e1e851cad1 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_s8_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = a64_s8_nhwc_avg_generic_depthfirst_impl;
 
   a64_s8_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp
index 4b1f988a78b2b0a7a73a70161d88b12350217147..5d7391dc261879d9944b0b2f853855b9ac0dff51 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_avg_generic_depthfirst/generic.cpp
@@ -40,10 +40,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -120,10 +120,10 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "movi v0.4s, #0x0\n"
     "cbz x22, 4f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "ldr q29, [x21, x25]\n"
     "ldr q28, [x20, x25]\n"
     "ldr q27, [x21, x24]\n"
@@ -136,24 +136,24 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "saddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "saddl v21.8h, v29.8b, v28.8b\n"
     "ldr q31, [x21, x26]\n"
+    "saddl v21.8h, v29.8b, v28.8b\n"
+    "subs x22, x22, #0x1\n"
     "saddl2 v20.8h, v29.16b, v28.16b\n"
-    "saddl v19.8h, v27.8b, v26.8b\n"
     "ldr q30, [x20, x26]\n"
-    "saddl2 v18.8h, v27.16b, v26.16b\n"
+    "saddl v19.8h, v27.8b, v26.8b\n"
     "ldr q29, [x21, x25]\n"
-    "saddl v17.8h, v25.8b, v24.8b\n"
+    "saddl2 v18.8h, v27.16b, v26.16b\n"
     "ldr q28, [x20, x25]\n"
-    "saddl2 v16.8h, v25.16b, v24.16b\n"
+    "saddl v17.8h, v25.8b, v24.8b\n"
     "ldr q27, [x21, x24]\n"
-    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddl2 v16.8h, v25.16b, v24.16b\n"
     "ldr q26, [x20, x24]\n"
-    "saddw2 v14.4s, v14.4s, v23.8h\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
     "ldr q25, [x21, x23]\n"
-    "saddw v13.4s, v13.4s, v22.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q24, [x20, x23]\n"
+    "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "saddw v11.4s, v11.4s, v21.4h\n"
     "saddw2 v10.4s, v10.4s, v21.8h\n"
@@ -200,19 +200,19 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "sxtl v16.8h, v31.8b\n"
+    "sxtl v23.8h, v31.8b\n"
     "ldr q29, [x21, x25]\n"
     "sxtl2 v22.8h, v31.16b\n"
     "ldr q27, [x21, x24]\n"
     "ldr q25, [x21, x23]\n"
-    "saddw v15.4s, v15.4s, v16.4h\n"
-    "saddw2 v14.4s, v14.4s, v16.8h\n"
     "sxtl v21.8h, v29.8b\n"
     "sxtl2 v20.8h, v29.16b\n"
     "sxtl v19.8h, v27.8b\n"
     "sxtl2 v18.8h, v27.16b\n"
     "sxtl v17.8h, v25.8b\n"
     "sxtl2 v16.8h, v25.16b\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "saddw v11.4s, v11.4s, v21.4h\n"
@@ -232,16 +232,16 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "movi v19.4s, #0x7f\n"
     "ld1r { v18.4s }, [%x[rescale_ptr]]\n"
     "sub %x[n_channels], %x[n_channels], #0x40\n"
-    "sqrdmulh v15.4s, v15.4s, v18.4s\n"
+    "sqdmulh v15.4s, v15.4s, v18.4s\n"
     "ld1r { v17.4s }, [%x[shift_ptr]]\n"
     "not v16.16b, v19.16b\n"
-    "sqrdmulh v14.4s, v14.4s, v18.4s\n"
+    "sqdmulh v14.4s, v14.4s, v18.4s\n"
     "cmp %x[n_channels], #0x40\n"
-    "sqrdmulh v13.4s, v13.4s, v18.4s\n"
-    "sqrdmulh v12.4s, v12.4s, v18.4s\n"
-    "sqrdmulh v11.4s, v11.4s, v18.4s\n"
-    "sqrdmulh v10.4s, v10.4s, v18.4s\n"
-    "sqrdmulh v9.4s, v9.4s, v18.4s\n"
+    "sqdmulh v13.4s, v13.4s, v18.4s\n"
+    "sqdmulh v12.4s, v12.4s, v18.4s\n"
+    "sqdmulh v11.4s, v11.4s, v18.4s\n"
+    "sqdmulh v10.4s, v10.4s, v18.4s\n"
+    "sqdmulh v9.4s, v9.4s, v18.4s\n"
     "srshl v15.4s, v15.4s, v17.4s\n"
     "srshl v14.4s, v14.4s, v17.4s\n"
     "srshl v13.4s, v13.4s, v17.4s\n"
@@ -249,23 +249,23 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "srshl v11.4s, v11.4s, v17.4s\n"
     "srshl v10.4s, v10.4s, v17.4s\n"
     "srshl v9.4s, v9.4s, v17.4s\n"
-    "sqrdmulh v8.4s, v8.4s, v18.4s\n"
-    "sqrdmulh v7.4s, v7.4s, v18.4s\n"
-    "sqrdmulh v6.4s, v6.4s, v18.4s\n"
-    "sqrdmulh v5.4s, v5.4s, v18.4s\n"
+    "sqdmulh v8.4s, v8.4s, v18.4s\n"
+    "sqdmulh v7.4s, v7.4s, v18.4s\n"
+    "sqdmulh v6.4s, v6.4s, v18.4s\n"
+    "sqdmulh v5.4s, v5.4s, v18.4s\n"
     "srshl v8.4s, v8.4s, v17.4s\n"
     "srshl v7.4s, v7.4s, v17.4s\n"
     "srshl v6.4s, v6.4s, v17.4s\n"
     "srshl v5.4s, v5.4s, v17.4s\n"
-    "sqrdmulh v4.4s, v4.4s, v18.4s\n"
-    "sqrdmulh v3.4s, v3.4s, v18.4s\n"
-    "sqrdmulh v2.4s, v2.4s, v18.4s\n"
-    "sqrdmulh v1.4s, v1.4s, v18.4s\n"
+    "sqdmulh v4.4s, v4.4s, v18.4s\n"
+    "sqdmulh v3.4s, v3.4s, v18.4s\n"
+    "sqdmulh v2.4s, v2.4s, v18.4s\n"
+    "sqdmulh v1.4s, v1.4s, v18.4s\n"
     "srshl v4.4s, v4.4s, v17.4s\n"
     "srshl v3.4s, v3.4s, v17.4s\n"
     "srshl v2.4s, v2.4s, v17.4s\n"
     "srshl v1.4s, v1.4s, v17.4s\n"
-    "sqrdmulh v0.4s, v0.4s, v18.4s\n"
+    "sqdmulh v0.4s, v0.4s, v18.4s\n"
     "smax v15.4s, v15.4s, v16.4s\n"
     "smax v14.4s, v14.4s, v16.4s\n"
     "smax v13.4s, v13.4s, v16.4s\n"
@@ -302,22 +302,22 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "uzp1 v16.16b, v13.16b, v12.16b\n"
     "smin v0.4s, v0.4s, v19.4s\n"
     "uzp1 v22.16b, v11.16b, v10.16b\n"
-    "uzp1 v18.16b, v9.16b, v8.16b\n"
-    "uzp1 v21.16b, v7.16b, v6.16b\n"
+    "uzp1 v21.16b, v9.16b, v8.16b\n"
+    "uzp1 v20.16b, v7.16b, v6.16b\n"
     "uzp1 v17.16b, v5.16b, v4.16b\n"
-    "uzp1 v20.16b, v3.16b, v2.16b\n"
-    "uzp1 v19.16b, v1.16b, v0.16b\n"
+    "uzp1 v19.16b, v3.16b, v2.16b\n"
+    "uzp1 v18.16b, v1.16b, v0.16b\n"
     "uzp1 v16.16b, v23.16b, v16.16b\n"
     "str q16, [%x[outptr], x26]\n"
-    "uzp1 v18.16b, v22.16b, v18.16b\n"
-    "uzp1 v17.16b, v21.16b, v17.16b\n"
+    "uzp1 v16.16b, v22.16b, v21.16b\n"
     "add x26, x26, #0x40\n"
-    "uzp1 v16.16b, v20.16b, v19.16b\n"
-    "str q18, [%x[outptr], x25]\n"
-    "str q17, [%x[outptr], x24]\n"
-    "str q16, [%x[outptr], x23]\n"
+    "uzp1 v17.16b, v20.16b, v17.16b\n"
+    "str q16, [%x[outptr], x25]\n"
+    "uzp1 v16.16b, v19.16b, v18.16b\n"
     "add x25, x25, #0x40\n"
+    "str q17, [%x[outptr], x24]\n"
     "add x24, x24, #0x40\n"
+    "str q16, [%x[outptr], x23]\n"
     "add x23, x23, #0x40\n"
     "bge 1b\n"
     "cbz %x[n_channels], 43f\n"
@@ -333,19 +333,19 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "movi v12.4s, #0x0\n"
     "cbz x22, 11f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 2 inputs loop
     "saddl v23.8h, v31.8b, v30.8b\n"
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "saddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "saddw v15.4s, v15.4s, v23.4h\n"
     "ldr q31, [x21, x26]\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "subs x22, x22, #0x1\n"
     "saddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q30, [x20, x26]\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
@@ -365,10 +365,10 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "sxtl v16.8h, v31.8b\n"
+    "sxtl v23.8h, v31.8b\n"
     "sxtl2 v22.8h, v31.16b\n"
-    "saddw v15.4s, v15.4s, v16.4h\n"
-    "saddw2 v14.4s, v14.4s, v16.8h\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 12b\n"
@@ -376,13 +376,13 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "movi v19.4s, #0x7f\n"
     "ld1r { v18.4s }, [%x[rescale_ptr]]\n"
     "sub %x[n_channels], %x[n_channels], #0x10\n"
-    "sqrdmulh v15.4s, v15.4s, v18.4s\n"
+    "sqdmulh v15.4s, v15.4s, v18.4s\n"
     "ld1r { v17.4s }, [%x[shift_ptr]]\n"
     "not v16.16b, v19.16b\n"
-    "sqrdmulh v14.4s, v14.4s, v18.4s\n"
+    "sqdmulh v14.4s, v14.4s, v18.4s\n"
     "cmp %x[n_channels], #0x10\n"
-    "sqrdmulh v13.4s, v13.4s, v18.4s\n"
-    "sqrdmulh v12.4s, v12.4s, v18.4s\n"
+    "sqdmulh v13.4s, v13.4s, v18.4s\n"
+    "sqdmulh v12.4s, v12.4s, v18.4s\n"
     "srshl v15.4s, v15.4s, v17.4s\n"
     "srshl v14.4s, v14.4s, v17.4s\n"
     "srshl v13.4s, v13.4s, v17.4s\n"
@@ -538,11 +538,11 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "tbz %x[n_channels], #0, 33f\n"
     "ldr b31, [x21], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "sxtl v16.8h, v31.8b\n"
+    "sxtl v23.8h, v31.8b\n"
     "subs x20, x20, #0x1\n"
     "sxtl2 v22.8h, v31.16b\n"
-    "saddw v15.4s, v15.4s, v16.4h\n"
-    "saddw2 v14.4s, v14.4s, v16.8h\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 25b\n"
@@ -550,11 +550,11 @@ void a64_s8_nhwc_avg_generic_depthfirst_impl(
     "movi v19.4s, #0x7f\n"
     "ld1r { v18.4s }, [%x[rescale_ptr]]\n"
     "not v16.16b, v19.16b\n"
-    "sqrdmulh v15.4s, v15.4s, v18.4s\n"
+    "sqdmulh v15.4s, v15.4s, v18.4s\n"
     "ld1r { v17.4s }, [%x[shift_ptr]]\n"
-    "sqrdmulh v14.4s, v14.4s, v18.4s\n"
-    "sqrdmulh v13.4s, v13.4s, v18.4s\n"
-    "sqrdmulh v12.4s, v12.4s, v18.4s\n"
+    "sqdmulh v14.4s, v14.4s, v18.4s\n"
+    "sqdmulh v13.4s, v13.4s, v18.4s\n"
+    "sqdmulh v12.4s, v12.4s, v18.4s\n"
     "srshl v15.4s, v15.4s, v17.4s\n"
     "srshl v14.4s, v14.4s, v17.4s\n"
     "srshl v13.4s, v13.4s, v17.4s\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index 0bf6a66cc121d2387cf350c176bc740696835b43..298db968616331aed4ed971c403a28e363b571f9 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -63,23 +63,24 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x15, [%x[args], %[offsetof_n_channels]]\n"
     "mov x14, #0x0\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
-    "ldp x12, x11, [x19, #0x0]\n"
-    "cmp x15, #0x10\n"
-    "ldp x10, x9, [x19, #0x10]\n"
     "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "cmp x15, #0x10\n"
+    "ldp x12, x11, [x20, #0x0]\n"
+    "ldp x10, x9, [x20, #0x10]\n"
     "ldp x28, x27, [x19, #0x0]\n"
     "ldp x26, x25, [x19, #0x10]\n"
     "ldp x24, x23, [x19, #0x20]\n"
     "ldp x22, x21, [x19, #0x30]\n"
     "ldr x20, [x19, #0x40]\n"
     "blt 3f\n"
-    "lsr x19, x15, #0x4\n"
-    "sub x15, x15, x19, LSL #4\n"
     "ldr q30, [x27, x14]\n"
+    "lsr x19, x15, #0x4\n"
     "ldr q29, [x24, x14]\n"
+    "sub x15, x15, x19, LSL #4\n"
     "ldr q28, [x21, x14]\n"
+    "subs x19, x19, #0x1\n"
     "ldr q27, [x25, x14]\n"
     "ldr q26, [x28, x14]\n"
     "ldr q25, [x23, x14]\n"
@@ -87,11 +88,11 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "ldr q22, [x20, x14]\n"
     "add x14, x14, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "beq 2f\n"
     "1:"  // Vector: Loop
     "smax v21.16b, v30.16b, v29.16b\n"
     "ldr q30, [x27, x14]\n"
+    "subs x19, x19, #0x1\n"
     "smax v20.16b, v29.16b, v28.16b\n"
     "ldr q29, [x24, x14]\n"
     "smax v19.16b, v27.16b, v26.16b\n"
@@ -108,14 +109,13 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "smax v17.16b, v20.16b, v17.16b\n"
     "ldr q22, [x20, x14]\n"
-    "smax v16.16b, v20.16b, v16.16b\n"
     "add x14, x14, #0x10\n"
+    "smax v16.16b, v20.16b, v16.16b\n"
     "str q19, [x12, x13]\n"
     "str q18, [x11, x13]\n"
     "str q17, [x10, x13]\n"
     "str q16, [x9, x13]\n"
     "add x13, x13, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "bgt 1b\n"
     "2:"  // Vector: Tail
     "smax v21.16b, v30.16b, v29.16b\n"
@@ -136,6 +136,7 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "cbz x15, 4f\n"
     "3:"  // Oddments
     "ldr b30, [x27, x14]\n"
+    "subs x15, x15, #0x1\n"
     "ldr b29, [x24, x14]\n"
     "smax v21.16b, v30.16b, v29.16b\n"
     "ldr b28, [x21, x14]\n"
@@ -150,9 +151,8 @@ void a64_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr b22, [x20, x14]\n"
     "add x14, x14, #0x1\n"
     "smax v18.16b, v25.16b, v24.16b\n"
-    "subs x15, x15, #0x1\n"
-    "smax v17.16b, v23.16b, v27.16b\n"
     "str b19, [x12, x13]\n"
+    "smax v17.16b, v23.16b, v27.16b\n"
     "smax v16.16b, v25.16b, v22.16b\n"
     "smax v18.16b, v21.16b, v18.16b\n"
     "str b18, [x11, x13]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst.hpp
index bc54992f2e02fd19131f73e1bd371cf6d5e1c6ea..6c4cd1467f6000112a324de83d995846b1c6b518 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_s8_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = a64_s8_nhwc_max_generic_depthfirst_impl;
 
   a64_s8_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp
index 0b7e6dfba9da63911cacc0b3a6580b2687dfd451..5e4c84d23e559d5b1fcf6214b641a2f262c11584 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8_nhwc_max_generic_depthfirst/generic.cpp
@@ -46,23 +46,23 @@ void a64_s8_nhwc_max_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x40\n"
     "blt 7f\n"
     "1:"  // 4-vectors of channels
-    "movi v6.16b, #0x80\n"
+    "movi v7.16b, #0x80\n"
     "mov x19, %x[inptrs]\n"
-    "movi v5.16b, #0x80\n"
+    "movi v6.16b, #0x80\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "movi v5.16b, #0x80\n"
     "movi v4.16b, #0x80\n"
-    "movi v3.16b, #0x80\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -75,47 +75,47 @@ void a64_s8_nhwc_max_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "smax v23.16b, v2.16b, v1.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "smax v22.16b, v30.16b, v22.16b\n"
     "add x19, x19, #0x20\n"
+    "smax v22.16b, v31.16b, v30.16b\n"
+    "ldr q3, [x23, x28]\n"
     "smax v18.16b, v29.16b, v28.16b\n"
-    "ldr q2, [x23, x28]\n"
     "smax v21.16b, v27.16b, v21.16b\n"
+    "ldr q2, [x22, x28]\n"
     "smax v17.16b, v26.16b, v17.16b\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "smax v20.16b, v25.16b, v20.16b\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "smax v16.16b, v24.16b, v16.16b\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "smax v19.16b, v23.16b, v19.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "smax v18.16b, v22.16b, v18.16b\n"
-    "ldr q22, [x22, x27]\n"
-    "smax v17.16b, v21.16b, v17.16b\n"
     "ldr q29, [x21, x27]\n"
-    "smax v16.16b, v20.16b, v16.16b\n"
+    "smax v17.16b, v21.16b, v17.16b\n"
     "ldr q28, [x20, x27]\n"
-    "smax v6.16b, v6.16b, v19.16b\n"
+    "smax v16.16b, v20.16b, v16.16b\n"
     "ldr q27, [x23, x26]\n"
-    "smax v5.16b, v5.16b, v18.16b\n"
+    "smax v7.16b, v7.16b, v19.16b\n"
     "ldr q21, [x22, x26]\n"
-    "smax v4.16b, v4.16b, v17.16b\n"
+    "smax v6.16b, v6.16b, v18.16b\n"
     "ldr q26, [x21, x26]\n"
-    "smax v3.16b, v3.16b, v16.16b\n"
+    "smax v5.16b, v5.16b, v17.16b\n"
     "ldr q17, [x20, x26]\n"
+    "smax v4.16b, v4.16b, v16.16b\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "smax v23.16b, v2.16b, v1.16b\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
-    "smax v22.16b, v30.16b, v22.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
+    "smax v22.16b, v31.16b, v30.16b\n"
     "smax v18.16b, v29.16b, v28.16b\n"
     "smax v21.16b, v27.16b, v21.16b\n"
     "smax v17.16b, v26.16b, v17.16b\n"
@@ -125,33 +125,33 @@ void a64_s8_nhwc_max_generic_depthfirst_impl(
     "smax v18.16b, v22.16b, v18.16b\n"
     "smax v17.16b, v21.16b, v17.16b\n"
     "smax v16.16b, v20.16b, v16.16b\n"
-    "smax v6.16b, v6.16b, v19.16b\n"
-    "smax v5.16b, v5.16b, v18.16b\n"
-    "smax v4.16b, v4.16b, v17.16b\n"
-    "smax v3.16b, v3.16b, v16.16b\n"
+    "smax v7.16b, v7.16b, v19.16b\n"
+    "smax v6.16b, v6.16b, v18.16b\n"
+    "smax v5.16b, v5.16b, v17.16b\n"
+    "smax v4.16b, v4.16b, v16.16b\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "smax v6.16b, v6.16b, v2.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "smax v7.16b, v7.16b, v3.16b\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "smax v5.16b, v5.16b, v30.16b\n"
+    "smax v6.16b, v6.16b, v31.16b\n"
     "ldr q25, [x23, x25]\n"
-    "smax v4.16b, v4.16b, v27.16b\n"
-    "smax v3.16b, v3.16b, v25.16b\n"
+    "smax v5.16b, v5.16b, v27.16b\n"
+    "smax v4.16b, v4.16b, v25.16b\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
-    "str q5, [%x[outptr], x27]\n"
-    "str q4, [%x[outptr], x26]\n"
-    "str q3, [%x[outptr], x25]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x40\n"
+    "str q6, [%x[outptr], x27]\n"
     "add x27, x27, #0x40\n"
+    "str q5, [%x[outptr], x26]\n"
     "add x26, x26, #0x40\n"
+    "str q4, [%x[outptr], x25]\n"
     "add x25, x25, #0x40\n"
     "sub %x[n_channels], %x[n_channels], #0x40\n"
     "cmp %x[n_channels], #0x40\n"
@@ -161,275 +161,275 @@ void a64_s8_nhwc_max_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x10\n"
     "blt 14f\n"
     "8:"  // Single vector of channels: Loop
-    "movi v6.16b, #0x80\n"
+    "movi v7.16b, #0x80\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "smax v23.16b, v2.16b, v1.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "smax v19.16b, v23.16b, v19.16b\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "smax v6.16b, v6.16b, v19.16b\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "smax v19.16b, v23.16b, v19.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "smax v7.16b, v7.16b, v19.16b\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "smax v23.16b, v2.16b, v1.16b\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "smax v19.16b, v23.16b, v19.16b\n"
-    "smax v6.16b, v6.16b, v19.16b\n"
+    "smax v7.16b, v7.16b, v19.16b\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "smax v6.16b, v6.16b, v2.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "smax v7.16b, v7.16b, v3.16b\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x10\n"
     "sub %x[n_channels], %x[n_channels], #0x10\n"
     "cmp %x[n_channels], #0x10\n"
     "bge 8b\n"
     "cbz %x[n_channels], 43f\n"
     "14:"  // Oddments
-    "movi v6.16b, #0x80\n"
+    "movi v7.16b, #0x80\n"
     "add %x[outptr], %x[outptr], x28\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 24f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #3, 19f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #2, 17f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
-    "ld1 { v1.h }[6], [x22], #0x2\n"
-    "ld1 { v0.h }[6], [x21], #0x2\n"
-    "ld1 { v31.h }[6], [x20], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
+    "ld1 { v2.h }[6], [x22], #0x2\n"
+    "ld1 { v1.h }[6], [x21], #0x2\n"
+    "ld1 { v0.h }[6], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
-    "ld1 { v1.b }[14], [x22], #0x1\n"
-    "ld1 { v0.b }[14], [x21], #0x1\n"
-    "ld1 { v31.b }[14], [x20], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
+    "ld1 { v2.b }[14], [x22], #0x1\n"
+    "ld1 { v1.b }[14], [x21], #0x1\n"
+    "ld1 { v0.b }[14], [x20], #0x1\n"
     "b 23f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
-    "ld1 { v1.b }[12], [x22], #0x1\n"
-    "ld1 { v0.b }[12], [x21], #0x1\n"
-    "ld1 { v31.b }[12], [x20], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
+    "ld1 { v2.b }[12], [x22], #0x1\n"
+    "ld1 { v1.b }[12], [x21], #0x1\n"
+    "ld1 { v0.b }[12], [x20], #0x1\n"
     "b 23f\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 18f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
-    "ld1 { v1.h }[4], [x22], #0x2\n"
-    "ld1 { v0.h }[4], [x21], #0x2\n"
-    "ld1 { v31.h }[4], [x20], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
+    "ld1 { v2.h }[4], [x22], #0x2\n"
+    "ld1 { v1.h }[4], [x21], #0x2\n"
+    "ld1 { v0.h }[4], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
-    "ld1 { v1.b }[10], [x22], #0x1\n"
-    "ld1 { v0.b }[10], [x21], #0x1\n"
-    "ld1 { v31.b }[10], [x20], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
+    "ld1 { v2.b }[10], [x22], #0x1\n"
+    "ld1 { v1.b }[10], [x21], #0x1\n"
+    "ld1 { v0.b }[10], [x20], #0x1\n"
     "b 23f\n"
     "18:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
-    "ld1 { v1.b }[8], [x22], #0x1\n"
-    "ld1 { v0.b }[8], [x21], #0x1\n"
-    "ld1 { v31.b }[8], [x20], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
+    "ld1 { v2.b }[8], [x22], #0x1\n"
+    "ld1 { v1.b }[8], [x21], #0x1\n"
+    "ld1 { v0.b }[8], [x20], #0x1\n"
     "b 23f\n"
     "19:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 21f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "tbz %x[n_channels], #1, 20f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
-    "ld1 { v1.h }[2], [x22], #0x2\n"
-    "ld1 { v0.h }[2], [x21], #0x2\n"
-    "ld1 { v31.h }[2], [x20], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
+    "ld1 { v2.h }[2], [x22], #0x2\n"
+    "ld1 { v1.h }[2], [x21], #0x2\n"
+    "ld1 { v0.h }[2], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
-    "ld1 { v1.b }[6], [x22], #0x1\n"
-    "ld1 { v0.b }[6], [x21], #0x1\n"
-    "ld1 { v31.b }[6], [x20], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
+    "ld1 { v2.b }[6], [x22], #0x1\n"
+    "ld1 { v1.b }[6], [x21], #0x1\n"
+    "ld1 { v0.b }[6], [x20], #0x1\n"
     "b 23f\n"
     "20:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
-    "ld1 { v1.b }[4], [x22], #0x1\n"
-    "ld1 { v0.b }[4], [x21], #0x1\n"
-    "ld1 { v31.b }[4], [x20], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
+    "ld1 { v2.b }[4], [x22], #0x1\n"
+    "ld1 { v1.b }[4], [x21], #0x1\n"
+    "ld1 { v0.b }[4], [x20], #0x1\n"
     "b 23f\n"
     "21:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 22f\n"
-    "ldr h2, [x23], #0x2\n"
-    "ldr h1, [x22], #0x2\n"
-    "ldr h0, [x21], #0x2\n"
-    "ldr h31, [x20], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
+    "ldr h2, [x22], #0x2\n"
+    "ldr h1, [x21], #0x2\n"
+    "ldr h0, [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
-    "ld1 { v1.b }[2], [x22], #0x1\n"
-    "ld1 { v0.b }[2], [x21], #0x1\n"
-    "ld1 { v31.b }[2], [x20], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
+    "ld1 { v2.b }[2], [x22], #0x1\n"
+    "ld1 { v1.b }[2], [x21], #0x1\n"
+    "ld1 { v0.b }[2], [x20], #0x1\n"
     "b 23f\n"
     "22:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ldr b2, [x23], #0x1\n"
-    "ldr b1, [x22], #0x1\n"
-    "ldr b0, [x21], #0x1\n"
-    "ldr b31, [x20], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
+    "ldr b2, [x22], #0x1\n"
+    "ldr b1, [x21], #0x1\n"
+    "ldr b0, [x20], #0x1\n"
     "23:"  // Oddments: 4 inputs loop: Load: Bit 3: End
-    "smax v23.16b, v2.16b, v1.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
     "subs x24, x24, #0x1\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "smax v19.16b, v23.16b, v19.16b\n"
-    "smax v6.16b, v6.16b, v19.16b\n"
+    "smax v7.16b, v7.16b, v19.16b\n"
     "bgt 15b\n"
     "24:"  // Oddments: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 34f\n"
     "25:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #3, 29f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #2, 27f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "tbz %x[n_channels], #1, 26f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
     "b 33f\n"
     "26:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
     "b 33f\n"
     "27:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 28f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
     "b 33f\n"
     "28:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
     "b 33f\n"
     "29:"  // Oddments: Single input loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 31f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "tbz %x[n_channels], #1, 30f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
     "b 33f\n"
     "30:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
     "b 33f\n"
     "31:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 32f\n"
-    "ldr h2, [x23], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
     "b 33f\n"
     "32:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ldr b2, [x23], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "smax v6.16b, v6.16b, v2.16b\n"
+    "smax v7.16b, v7.16b, v3.16b\n"
     "subs x20, x20, #0x1\n"
     "bgt 25b\n"
     "34:"  // Oddments: Single input loop: End
     "tbz %x[n_channels], #3, 38f\n"
-    "st1 { v6.d }[0], [%x[outptr]], #0x8\n"
+    "st1 { v7.d }[0], [%x[outptr]], #0x8\n"
     "tbz %x[n_channels], #2, 36f\n"
-    "st1 { v6.s }[2], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[2], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #1, 35f\n"
-    "st1 { v6.h }[6], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[6], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[14], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[14], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "35:"  // Oddments: Store: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[12], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[12], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "36:"  // Oddments: Store: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 37f\n"
-    "st1 { v6.h }[4], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[4], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[10], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[10], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "37:"  // Oddments: Store: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[8], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[8], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "38:"  // Oddments: Store: Bit 3: Unset
     "tbz %x[n_channels], #2, 40f\n"
-    "st1 { v6.s }[0], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[0], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #1, 39f\n"
-    "st1 { v6.h }[2], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[2], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[6], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[6], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "39:"  // Oddments: Store: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[4], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[4], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "40:"  // Oddments: Store: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 41f\n"
-    "st1 { v6.h }[0], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[0], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[2], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[2], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "41:"  // Oddments: Store: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[0], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[0], [%x[outptr]], #0x1\n"
     "42:"  // Oddments: Store: Bit 3: End
 
     "43:"  // End
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst.hpp
index e5354ca25501096009188133ebcde0bce7f055ce..a50e99a0094a6193bbda569f987d9dd998b6e1c2 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_s8q_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = a64_s8q_nhwc_avg_generic_depthfirst_impl;
 
   a64_s8q_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp
index 7246b69f069364e9d4adc8ed6adc88947152086b..f288a4119cbf7d14f968a31d9cd4260d1c19a40e 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_avg_generic_depthfirst/generic.cpp
@@ -23,7 +23,6 @@
  */
 
 #include "pooling.hpp"
-#include <cstddef>
 #include <cstdint>
 #include <cstring>
 #include <cmath>
@@ -42,10 +41,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -140,10 +139,10 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl(
     "movi v0.4s, #0x0\n"
     "cbz x22, 4f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "ldr q29, [x21, x25]\n"
     "ldr q28, [x20, x25]\n"
     "ldr q27, [x21, x24]\n"
@@ -156,24 +155,24 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl(
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "saddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "saddl v21.8h, v29.8b, v28.8b\n"
     "ldr q31, [x21, x26]\n"
+    "saddl v21.8h, v29.8b, v28.8b\n"
+    "subs x22, x22, #0x1\n"
     "saddl2 v20.8h, v29.16b, v28.16b\n"
-    "saddl v19.8h, v27.8b, v26.8b\n"
     "ldr q30, [x20, x26]\n"
-    "saddl2 v18.8h, v27.16b, v26.16b\n"
+    "saddl v19.8h, v27.8b, v26.8b\n"
     "ldr q29, [x21, x25]\n"
-    "saddl v17.8h, v25.8b, v24.8b\n"
+    "saddl2 v18.8h, v27.16b, v26.16b\n"
     "ldr q28, [x20, x25]\n"
-    "saddl2 v16.8h, v25.16b, v24.16b\n"
+    "saddl v17.8h, v25.8b, v24.8b\n"
     "ldr q27, [x21, x24]\n"
-    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddl2 v16.8h, v25.16b, v24.16b\n"
     "ldr q26, [x20, x24]\n"
-    "saddw2 v14.4s, v14.4s, v23.8h\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
     "ldr q25, [x21, x23]\n"
-    "saddw v13.4s, v13.4s, v22.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q24, [x20, x23]\n"
+    "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "saddw v11.4s, v11.4s, v21.4h\n"
     "saddw2 v10.4s, v10.4s, v21.8h\n"
@@ -220,19 +219,19 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "sxtl v16.8h, v31.8b\n"
+    "sxtl v23.8h, v31.8b\n"
     "ldr q29, [x21, x25]\n"
     "sxtl2 v22.8h, v31.16b\n"
     "ldr q27, [x21, x24]\n"
     "ldr q25, [x21, x23]\n"
-    "saddw v15.4s, v15.4s, v16.4h\n"
-    "saddw2 v14.4s, v14.4s, v16.8h\n"
     "sxtl v21.8h, v29.8b\n"
     "sxtl2 v20.8h, v29.16b\n"
     "sxtl v19.8h, v27.8b\n"
     "sxtl2 v18.8h, v27.16b\n"
     "sxtl v17.8h, v25.8b\n"
     "sxtl2 v16.8h, v25.16b\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "saddw v11.4s, v11.4s, v21.4h\n"
@@ -339,22 +338,22 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl(
     "uzp1 v23.16b, v15.16b, v14.16b\n"
     "uzp1 v16.16b, v13.16b, v12.16b\n"
     "uzp1 v22.16b, v11.16b, v10.16b\n"
-    "uzp1 v18.16b, v9.16b, v8.16b\n"
-    "uzp1 v21.16b, v7.16b, v6.16b\n"
+    "uzp1 v21.16b, v9.16b, v8.16b\n"
+    "uzp1 v20.16b, v7.16b, v6.16b\n"
     "uzp1 v17.16b, v5.16b, v4.16b\n"
-    "uzp1 v20.16b, v3.16b, v2.16b\n"
-    "uzp1 v19.16b, v1.16b, v0.16b\n"
+    "uzp1 v19.16b, v3.16b, v2.16b\n"
+    "uzp1 v18.16b, v1.16b, v0.16b\n"
     "uzp1 v16.16b, v23.16b, v16.16b\n"
     "str q16, [%x[outptr], x26]\n"
-    "uzp1 v18.16b, v22.16b, v18.16b\n"
-    "uzp1 v17.16b, v21.16b, v17.16b\n"
+    "uzp1 v16.16b, v22.16b, v21.16b\n"
     "add x26, x26, #0x40\n"
-    "uzp1 v16.16b, v20.16b, v19.16b\n"
-    "str q18, [%x[outptr], x25]\n"
-    "str q17, [%x[outptr], x24]\n"
-    "str q16, [%x[outptr], x23]\n"
+    "uzp1 v17.16b, v20.16b, v17.16b\n"
+    "str q16, [%x[outptr], x25]\n"
+    "uzp1 v16.16b, v19.16b, v18.16b\n"
     "add x25, x25, #0x40\n"
+    "str q17, [%x[outptr], x24]\n"
     "add x24, x24, #0x40\n"
+    "str q16, [%x[outptr], x23]\n"
     "add x23, x23, #0x40\n"
     "bge 1b\n"
     "cbz %x[n_channels], 43f\n"
@@ -370,19 +369,19 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl(
     "movi v12.4s, #0x0\n"
     "cbz x22, 11f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 2 inputs loop
     "saddl v23.8h, v31.8b, v30.8b\n"
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "saddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "saddw v15.4s, v15.4s, v23.4h\n"
     "ldr q31, [x21, x26]\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "subs x22, x22, #0x1\n"
     "saddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q30, [x20, x26]\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
@@ -402,10 +401,10 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "sxtl v16.8h, v31.8b\n"
+    "sxtl v23.8h, v31.8b\n"
     "sxtl2 v22.8h, v31.16b\n"
-    "saddw v15.4s, v15.4s, v16.4h\n"
-    "saddw2 v14.4s, v14.4s, v16.8h\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 12b\n"
@@ -580,11 +579,11 @@ void a64_s8q_nhwc_avg_generic_depthfirst_impl(
     "tbz %x[n_channels], #0, 33f\n"
     "ldr b31, [x21], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "sxtl v16.8h, v31.8b\n"
+    "sxtl v23.8h, v31.8b\n"
     "subs x20, x20, #0x1\n"
     "sxtl2 v22.8h, v31.16b\n"
-    "saddw v15.4s, v15.4s, v16.4h\n"
-    "saddw2 v14.4s, v14.4s, v16.8h\n"
+    "saddw v15.4s, v15.4s, v23.4h\n"
+    "saddw2 v14.4s, v14.4s, v23.8h\n"
     "saddw v13.4s, v13.4s, v22.4h\n"
     "saddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 25b\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst.hpp
index 6abbcd089e39d0b68cb64bd6ea4d206ba944379a..ea7f7f89fe92859dbc1821098d10f53bf10d8667 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_s8q_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = a64_s8q_nhwc_max_generic_depthfirst_impl;
 
   a64_s8q_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp
index 33cf634be5f1951734d66486683e35a2c2fe7593..a077121991bcd407d54f3c6ef87877aa425f2857 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp
@@ -49,7 +49,7 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x40\n"
     "blt 7f\n"
     "1:"  // 4-vectors of channels
-    "movi v3.16b, #0x80\n"
+    "movi v8.16b, #0x80\n"
     "mov x19, %x[inptrs]\n"
     "movi v7.16b, #0x80\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
@@ -60,12 +60,12 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -78,47 +78,47 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "smax v23.16b, v2.16b, v1.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "smax v22.16b, v30.16b, v22.16b\n"
     "add x19, x19, #0x20\n"
+    "smax v22.16b, v31.16b, v30.16b\n"
+    "ldr q3, [x23, x28]\n"
     "smax v18.16b, v29.16b, v28.16b\n"
-    "ldr q2, [x23, x28]\n"
     "smax v21.16b, v27.16b, v21.16b\n"
+    "ldr q2, [x22, x28]\n"
     "smax v17.16b, v26.16b, v17.16b\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "smax v20.16b, v25.16b, v20.16b\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "smax v16.16b, v24.16b, v16.16b\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "smax v19.16b, v23.16b, v19.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "smax v18.16b, v22.16b, v18.16b\n"
-    "ldr q22, [x22, x27]\n"
-    "smax v17.16b, v21.16b, v17.16b\n"
     "ldr q29, [x21, x27]\n"
-    "smax v16.16b, v20.16b, v16.16b\n"
+    "smax v17.16b, v21.16b, v17.16b\n"
     "ldr q28, [x20, x27]\n"
-    "smax v3.16b, v3.16b, v19.16b\n"
+    "smax v16.16b, v20.16b, v16.16b\n"
     "ldr q27, [x23, x26]\n"
-    "smax v7.16b, v7.16b, v18.16b\n"
+    "smax v8.16b, v8.16b, v19.16b\n"
     "ldr q21, [x22, x26]\n"
-    "smax v6.16b, v6.16b, v17.16b\n"
+    "smax v7.16b, v7.16b, v18.16b\n"
     "ldr q26, [x21, x26]\n"
-    "smax v5.16b, v5.16b, v16.16b\n"
+    "smax v6.16b, v6.16b, v17.16b\n"
     "ldr q17, [x20, x26]\n"
+    "smax v5.16b, v5.16b, v16.16b\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "smax v23.16b, v2.16b, v1.16b\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
-    "smax v22.16b, v30.16b, v22.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
+    "smax v22.16b, v31.16b, v30.16b\n"
     "smax v18.16b, v29.16b, v28.16b\n"
     "smax v21.16b, v27.16b, v21.16b\n"
     "smax v17.16b, v26.16b, v17.16b\n"
@@ -128,7 +128,7 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "smax v18.16b, v22.16b, v18.16b\n"
     "smax v17.16b, v21.16b, v17.16b\n"
     "smax v16.16b, v20.16b, v16.16b\n"
-    "smax v3.16b, v3.16b, v19.16b\n"
+    "smax v8.16b, v8.16b, v19.16b\n"
     "smax v7.16b, v7.16b, v18.16b\n"
     "smax v6.16b, v6.16b, v17.16b\n"
     "smax v5.16b, v5.16b, v16.16b\n"
@@ -138,30 +138,30 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "smax v3.16b, v3.16b, v2.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "smax v8.16b, v8.16b, v3.16b\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "smax v7.16b, v7.16b, v30.16b\n"
+    "smax v7.16b, v7.16b, v31.16b\n"
     "ldr q25, [x23, x25]\n"
     "smax v6.16b, v6.16b, v27.16b\n"
     "smax v5.16b, v5.16b, v25.16b\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "sxtl v23.8h, v3.8b\n"
+    "sxtl v23.8h, v8.8b\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_mul]\n"
     "ld1r { v4.4s }, [x19]\n"
-    "sxtl2 v22.8h, v3.16b\n"
+    "sxtl2 v22.8h, v8.16b\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n"
     "sxtl v21.8h, v7.8b\n"
     "ld1r { v3.4s }, [x19]\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n"
-    "sxtl2 v18.8h, v7.16b\n"
+    "sxtl2 v20.8h, v7.16b\n"
     "ld1r { v2.4s }, [x19]\n"
     "sub %x[n_channels], %x[n_channels], #0x40\n"
-    "sxtl v20.8h, v6.8b\n"
+    "sxtl v19.8h, v6.8b\n"
     "cmp %x[n_channels], #0x40\n"
-    "sxtl2 v19.8h, v6.16b\n"
+    "sxtl2 v18.8h, v6.16b\n"
     "sxtl v17.8h, v5.8b\n"
     "sxtl2 v16.8h, v5.16b\n"
     "sxtl v1.4s, v23.4h\n"
@@ -170,16 +170,16 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "sxtl2 v31.4s, v22.8h\n"
     "sxtl v30.4s, v21.4h\n"
     "sxtl2 v22.4s, v21.8h\n"
-    "sxtl v29.4s, v18.4h\n"
-    "sxtl2 v18.4s, v18.8h\n"
-    "sxtl v28.4s, v20.4h\n"
+    "sxtl v29.4s, v20.4h\n"
     "sxtl2 v21.4s, v20.8h\n"
-    "sxtl v27.4s, v19.4h\n"
-    "sxtl2 v26.4s, v19.8h\n"
+    "sxtl v28.4s, v19.4h\n"
+    "sxtl2 v20.4s, v19.8h\n"
+    "sxtl v27.4s, v18.4h\n"
+    "sxtl2 v26.4s, v18.8h\n"
     "sxtl v25.4s, v17.4h\n"
-    "sxtl2 v20.4s, v17.8h\n"
+    "sxtl2 v19.4s, v17.8h\n"
     "sxtl v24.4s, v16.4h\n"
-    "sxtl2 v19.4s, v16.8h\n"
+    "sxtl2 v18.4s, v16.8h\n"
     "srshl v1.4s, v1.4s, v3.4s\n"
     "srshl v23.4s, v23.4s, v3.4s\n"
     "srshl v0.4s, v0.4s, v3.4s\n"
@@ -195,39 +195,39 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "srshl v30.4s, v30.4s, v3.4s\n"
     "srshl v22.4s, v22.4s, v3.4s\n"
     "srshl v29.4s, v29.4s, v3.4s\n"
-    "srshl v18.4s, v18.4s, v3.4s\n"
+    "srshl v21.4s, v21.4s, v3.4s\n"
     "sqrdmulh v30.4s, v30.4s, v4.4s\n"
     "sqrdmulh v22.4s, v22.4s, v4.4s\n"
     "sqrdmulh v29.4s, v29.4s, v4.4s\n"
-    "sqrdmulh v18.4s, v18.4s, v4.4s\n"
+    "sqrdmulh v21.4s, v21.4s, v4.4s\n"
     "srshl v30.4s, v30.4s, v2.4s\n"
     "srshl v22.4s, v22.4s, v2.4s\n"
     "srshl v29.4s, v29.4s, v2.4s\n"
-    "srshl v18.4s, v18.4s, v2.4s\n"
+    "srshl v21.4s, v21.4s, v2.4s\n"
     "srshl v28.4s, v28.4s, v3.4s\n"
-    "srshl v21.4s, v21.4s, v3.4s\n"
+    "srshl v20.4s, v20.4s, v3.4s\n"
     "srshl v27.4s, v27.4s, v3.4s\n"
     "srshl v26.4s, v26.4s, v3.4s\n"
     "sqrdmulh v28.4s, v28.4s, v4.4s\n"
-    "sqrdmulh v21.4s, v21.4s, v4.4s\n"
+    "sqrdmulh v20.4s, v20.4s, v4.4s\n"
     "sqrdmulh v27.4s, v27.4s, v4.4s\n"
     "sqrdmulh v26.4s, v26.4s, v4.4s\n"
     "srshl v28.4s, v28.4s, v2.4s\n"
-    "srshl v21.4s, v21.4s, v2.4s\n"
+    "srshl v20.4s, v20.4s, v2.4s\n"
     "srshl v27.4s, v27.4s, v2.4s\n"
     "srshl v26.4s, v26.4s, v2.4s\n"
     "srshl v25.4s, v25.4s, v3.4s\n"
-    "srshl v20.4s, v20.4s, v3.4s\n"
-    "srshl v24.4s, v24.4s, v3.4s\n"
     "srshl v19.4s, v19.4s, v3.4s\n"
+    "srshl v24.4s, v24.4s, v3.4s\n"
+    "srshl v18.4s, v18.4s, v3.4s\n"
     "sqrdmulh v25.4s, v25.4s, v4.4s\n"
-    "sqrdmulh v20.4s, v20.4s, v4.4s\n"
-    "sqrdmulh v24.4s, v24.4s, v4.4s\n"
     "sqrdmulh v19.4s, v19.4s, v4.4s\n"
+    "sqrdmulh v24.4s, v24.4s, v4.4s\n"
+    "sqrdmulh v18.4s, v18.4s, v4.4s\n"
     "srshl v25.4s, v25.4s, v2.4s\n"
-    "srshl v20.4s, v20.4s, v2.4s\n"
-    "srshl v24.4s, v24.4s, v2.4s\n"
     "srshl v19.4s, v19.4s, v2.4s\n"
+    "srshl v24.4s, v24.4s, v2.4s\n"
+    "srshl v18.4s, v18.4s, v2.4s\n"
     "movi v17.4s, #0x7f\n"
     "not v16.16b, v17.16b\n"
     "smax v1.4s, v1.4s, v16.4s\n"
@@ -244,43 +244,43 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "smin v30.4s, v30.4s, v17.4s\n"
     "smin v22.4s, v22.4s, v17.4s\n"
     "smin v29.4s, v29.4s, v17.4s\n"
-    "smax v18.4s, v18.4s, v16.4s\n"
-    "smax v28.4s, v28.4s, v16.4s\n"
     "smax v21.4s, v21.4s, v16.4s\n"
-    "smin v18.4s, v18.4s, v17.4s\n"
-    "smin v28.4s, v28.4s, v17.4s\n"
+    "smax v28.4s, v28.4s, v16.4s\n"
+    "smax v20.4s, v20.4s, v16.4s\n"
     "smin v21.4s, v21.4s, v17.4s\n"
+    "smin v28.4s, v28.4s, v17.4s\n"
+    "smin v20.4s, v20.4s, v17.4s\n"
     "smax v27.4s, v27.4s, v16.4s\n"
     "smax v26.4s, v26.4s, v16.4s\n"
     "smax v25.4s, v25.4s, v16.4s\n"
     "smin v27.4s, v27.4s, v17.4s\n"
     "smin v26.4s, v26.4s, v17.4s\n"
     "smin v25.4s, v25.4s, v17.4s\n"
-    "smax v20.4s, v20.4s, v16.4s\n"
-    "smax v24.4s, v24.4s, v16.4s\n"
     "smax v19.4s, v19.4s, v16.4s\n"
-    "smin v20.4s, v20.4s, v17.4s\n"
-    "smin v24.4s, v24.4s, v17.4s\n"
+    "smax v24.4s, v24.4s, v16.4s\n"
+    "smax v18.4s, v18.4s, v16.4s\n"
     "smin v19.4s, v19.4s, v17.4s\n"
+    "smin v24.4s, v24.4s, v17.4s\n"
+    "smin v18.4s, v18.4s, v17.4s\n"
     "uzp1 v23.16b, v1.16b, v23.16b\n"
     "uzp1 v16.16b, v0.16b, v31.16b\n"
     "uzp1 v22.16b, v30.16b, v22.16b\n"
-    "uzp1 v18.16b, v29.16b, v18.16b\n"
-    "uzp1 v21.16b, v28.16b, v21.16b\n"
+    "uzp1 v21.16b, v29.16b, v21.16b\n"
+    "uzp1 v20.16b, v28.16b, v20.16b\n"
     "uzp1 v17.16b, v27.16b, v26.16b\n"
-    "uzp1 v20.16b, v25.16b, v20.16b\n"
-    "uzp1 v19.16b, v24.16b, v19.16b\n"
+    "uzp1 v19.16b, v25.16b, v19.16b\n"
+    "uzp1 v18.16b, v24.16b, v18.16b\n"
     "uzp1 v16.16b, v23.16b, v16.16b\n"
     "str q16, [%x[outptr], x28]\n"
-    "uzp1 v18.16b, v22.16b, v18.16b\n"
-    "uzp1 v17.16b, v21.16b, v17.16b\n"
+    "uzp1 v16.16b, v22.16b, v21.16b\n"
     "add x28, x28, #0x40\n"
-    "uzp1 v16.16b, v20.16b, v19.16b\n"
-    "str q18, [%x[outptr], x27]\n"
-    "str q17, [%x[outptr], x26]\n"
-    "str q16, [%x[outptr], x25]\n"
+    "uzp1 v17.16b, v20.16b, v17.16b\n"
+    "str q16, [%x[outptr], x27]\n"
+    "uzp1 v16.16b, v19.16b, v18.16b\n"
     "add x27, x27, #0x40\n"
+    "str q17, [%x[outptr], x26]\n"
     "add x26, x26, #0x40\n"
+    "str q16, [%x[outptr], x25]\n"
     "add x25, x25, #0x40\n"
     "bge 1b\n"
     "cbz %x[n_channels], 43f\n"
@@ -288,52 +288,52 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x10\n"
     "blt 14f\n"
     "8:"  // Single vector of channels: Loop
-    "movi v3.16b, #0x80\n"
+    "movi v8.16b, #0x80\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "smax v23.16b, v2.16b, v1.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "smax v19.16b, v23.16b, v19.16b\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "smax v3.16b, v3.16b, v19.16b\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "smax v19.16b, v23.16b, v19.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "smax v8.16b, v8.16b, v19.16b\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "smax v23.16b, v2.16b, v1.16b\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "smax v19.16b, v23.16b, v19.16b\n"
-    "smax v3.16b, v3.16b, v19.16b\n"
+    "smax v8.16b, v8.16b, v19.16b\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "smax v3.16b, v3.16b, v2.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "smax v8.16b, v8.16b, v3.16b\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "sxtl v23.8h, v3.8b\n"
+    "sxtl v23.8h, v8.8b\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_mul]\n"
     "ld1r { v4.4s }, [x19]\n"
-    "sxtl2 v22.8h, v3.16b\n"
+    "sxtl2 v22.8h, v8.16b\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n"
     "movi v17.4s, #0x7f\n"
     "ld1r { v3.4s }, [x19]\n"
@@ -374,179 +374,179 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
     "bge 8b\n"
     "cbz %x[n_channels], 43f\n"
     "14:"  // Oddments
-    "movi v3.16b, #0x80\n"
+    "movi v8.16b, #0x80\n"
     "add %x[outptr], %x[outptr], x28\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 24f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #3, 19f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #2, 17f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
-    "ld1 { v1.h }[6], [x22], #0x2\n"
-    "ld1 { v0.h }[6], [x21], #0x2\n"
-    "ld1 { v31.h }[6], [x20], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
+    "ld1 { v2.h }[6], [x22], #0x2\n"
+    "ld1 { v1.h }[6], [x21], #0x2\n"
+    "ld1 { v0.h }[6], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
-    "ld1 { v1.b }[14], [x22], #0x1\n"
-    "ld1 { v0.b }[14], [x21], #0x1\n"
-    "ld1 { v31.b }[14], [x20], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
+    "ld1 { v2.b }[14], [x22], #0x1\n"
+    "ld1 { v1.b }[14], [x21], #0x1\n"
+    "ld1 { v0.b }[14], [x20], #0x1\n"
     "b 23f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
-    "ld1 { v1.b }[12], [x22], #0x1\n"
-    "ld1 { v0.b }[12], [x21], #0x1\n"
-    "ld1 { v31.b }[12], [x20], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
+    "ld1 { v2.b }[12], [x22], #0x1\n"
+    "ld1 { v1.b }[12], [x21], #0x1\n"
+    "ld1 { v0.b }[12], [x20], #0x1\n"
     "b 23f\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 18f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
-    "ld1 { v1.h }[4], [x22], #0x2\n"
-    "ld1 { v0.h }[4], [x21], #0x2\n"
-    "ld1 { v31.h }[4], [x20], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
+    "ld1 { v2.h }[4], [x22], #0x2\n"
+    "ld1 { v1.h }[4], [x21], #0x2\n"
+    "ld1 { v0.h }[4], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
-    "ld1 { v1.b }[10], [x22], #0x1\n"
-    "ld1 { v0.b }[10], [x21], #0x1\n"
-    "ld1 { v31.b }[10], [x20], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
+    "ld1 { v2.b }[10], [x22], #0x1\n"
+    "ld1 { v1.b }[10], [x21], #0x1\n"
+    "ld1 { v0.b }[10], [x20], #0x1\n"
     "b 23f\n"
     "18:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
-    "ld1 { v1.b }[8], [x22], #0x1\n"
-    "ld1 { v0.b }[8], [x21], #0x1\n"
-    "ld1 { v31.b }[8], [x20], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
+    "ld1 { v2.b }[8], [x22], #0x1\n"
+    "ld1 { v1.b }[8], [x21], #0x1\n"
+    "ld1 { v0.b }[8], [x20], #0x1\n"
     "b 23f\n"
     "19:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 21f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "tbz %x[n_channels], #1, 20f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
-    "ld1 { v1.h }[2], [x22], #0x2\n"
-    "ld1 { v0.h }[2], [x21], #0x2\n"
-    "ld1 { v31.h }[2], [x20], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
+    "ld1 { v2.h }[2], [x22], #0x2\n"
+    "ld1 { v1.h }[2], [x21], #0x2\n"
+    "ld1 { v0.h }[2], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
-    "ld1 { v1.b }[6], [x22], #0x1\n"
-    "ld1 { v0.b }[6], [x21], #0x1\n"
-    "ld1 { v31.b }[6], [x20], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
+    "ld1 { v2.b }[6], [x22], #0x1\n"
+    "ld1 { v1.b }[6], [x21], #0x1\n"
+    "ld1 { v0.b }[6], [x20], #0x1\n"
     "b 23f\n"
     "20:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
-    "ld1 { v1.b }[4], [x22], #0x1\n"
-    "ld1 { v0.b }[4], [x21], #0x1\n"
-    "ld1 { v31.b }[4], [x20], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
+    "ld1 { v2.b }[4], [x22], #0x1\n"
+    "ld1 { v1.b }[4], [x21], #0x1\n"
+    "ld1 { v0.b }[4], [x20], #0x1\n"
     "b 23f\n"
     "21:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 22f\n"
-    "ldr h2, [x23], #0x2\n"
-    "ldr h1, [x22], #0x2\n"
-    "ldr h0, [x21], #0x2\n"
-    "ldr h31, [x20], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
+    "ldr h2, [x22], #0x2\n"
+    "ldr h1, [x21], #0x2\n"
+    "ldr h0, [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
-    "ld1 { v1.b }[2], [x22], #0x1\n"
-    "ld1 { v0.b }[2], [x21], #0x1\n"
-    "ld1 { v31.b }[2], [x20], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
+    "ld1 { v2.b }[2], [x22], #0x1\n"
+    "ld1 { v1.b }[2], [x21], #0x1\n"
+    "ld1 { v0.b }[2], [x20], #0x1\n"
     "b 23f\n"
     "22:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ldr b2, [x23], #0x1\n"
-    "ldr b1, [x22], #0x1\n"
-    "ldr b0, [x21], #0x1\n"
-    "ldr b31, [x20], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
+    "ldr b2, [x22], #0x1\n"
+    "ldr b1, [x21], #0x1\n"
+    "ldr b0, [x20], #0x1\n"
     "23:"  // Oddments: 4 inputs loop: Load: Bit 3: End
-    "smax v23.16b, v2.16b, v1.16b\n"
+    "smax v23.16b, v3.16b, v2.16b\n"
     "subs x24, x24, #0x1\n"
-    "smax v19.16b, v0.16b, v31.16b\n"
+    "smax v19.16b, v1.16b, v0.16b\n"
     "smax v19.16b, v23.16b, v19.16b\n"
-    "smax v3.16b, v3.16b, v19.16b\n"
+    "smax v8.16b, v8.16b, v19.16b\n"
     "bgt 15b\n"
     "24:"  // Oddments: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 34f\n"
     "25:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #3, 29f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #2, 27f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "tbz %x[n_channels], #1, 26f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
     "b 33f\n"
     "26:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
     "b 33f\n"
     "27:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 28f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
     "b 33f\n"
     "28:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
     "b 33f\n"
     "29:"  // Oddments: Single input loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 31f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "tbz %x[n_channels], #1, 30f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
     "b 33f\n"
     "30:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
     "b 33f\n"
     "31:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 32f\n"
-    "ldr h2, [x23], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
     "b 33f\n"
     "32:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ldr b2, [x23], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "smax v3.16b, v3.16b, v2.16b\n"
+    "smax v8.16b, v8.16b, v3.16b\n"
     "subs x20, x20, #0x1\n"
     "bgt 25b\n"
     "34:"  // Oddments: Single input loop: End
-    "sxtl v23.8h, v3.8b\n"
+    "sxtl v23.8h, v8.8b\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_mul]\n"
     "ld1r { v4.4s }, [x19]\n"
-    "sxtl2 v22.8h, v3.16b\n"
+    "sxtl2 v22.8h, v8.16b\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n"
     "movi v17.4s, #0x7f\n"
     "ld1r { v3.4s }, [x19]\n"
@@ -630,7 +630,7 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl(
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [quant_params] "r" (&qp)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst.hpp
index 943928615a337764e5408d3005738b0320fb52dc..230952452b3baa5e2be18e3aff0908c5dd61bdbf 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_u8_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = a64_u8_nhwc_avg_generic_depthfirst_impl;
 
   a64_u8_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp
index 1d210cb41921f5bb51bd38143e9281caf6502cc6..2c8a29248d1031814581b213ea91ca2a49ce9304 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_avg_generic_depthfirst/generic.cpp
@@ -40,10 +40,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -120,10 +120,10 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "movi v0.4s, #0x0\n"
     "cbz x22, 4f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "ldr q29, [x21, x25]\n"
     "ldr q28, [x20, x25]\n"
     "ldr q27, [x21, x24]\n"
@@ -136,24 +136,24 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "uaddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "uaddl v21.8h, v29.8b, v28.8b\n"
     "ldr q31, [x21, x26]\n"
+    "uaddl v21.8h, v29.8b, v28.8b\n"
+    "subs x22, x22, #0x1\n"
     "uaddl2 v20.8h, v29.16b, v28.16b\n"
-    "uaddl v19.8h, v27.8b, v26.8b\n"
     "ldr q30, [x20, x26]\n"
-    "uaddl2 v18.8h, v27.16b, v26.16b\n"
+    "uaddl v19.8h, v27.8b, v26.8b\n"
     "ldr q29, [x21, x25]\n"
-    "uaddl v17.8h, v25.8b, v24.8b\n"
+    "uaddl2 v18.8h, v27.16b, v26.16b\n"
     "ldr q28, [x20, x25]\n"
-    "uaddl2 v16.8h, v25.16b, v24.16b\n"
+    "uaddl v17.8h, v25.8b, v24.8b\n"
     "ldr q27, [x21, x24]\n"
-    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddl2 v16.8h, v25.16b, v24.16b\n"
     "ldr q26, [x20, x24]\n"
-    "uaddw2 v14.4s, v14.4s, v23.8h\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
     "ldr q25, [x21, x23]\n"
-    "uaddw v13.4s, v13.4s, v22.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q24, [x20, x23]\n"
+    "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "uaddw v11.4s, v11.4s, v21.4h\n"
     "uaddw2 v10.4s, v10.4s, v21.8h\n"
@@ -200,19 +200,19 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "uxtl v16.8h, v31.8b\n"
+    "uxtl v23.8h, v31.8b\n"
     "ldr q29, [x21, x25]\n"
     "uxtl2 v22.8h, v31.16b\n"
     "ldr q27, [x21, x24]\n"
     "ldr q25, [x21, x23]\n"
-    "uaddw v15.4s, v15.4s, v16.4h\n"
-    "uaddw2 v14.4s, v14.4s, v16.8h\n"
     "uxtl v21.8h, v29.8b\n"
     "uxtl2 v20.8h, v29.16b\n"
     "uxtl v19.8h, v27.8b\n"
     "uxtl2 v18.8h, v27.16b\n"
     "uxtl v17.8h, v25.8b\n"
     "uxtl2 v16.8h, v25.16b\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "uaddw v11.4s, v11.4s, v21.4h\n"
@@ -235,35 +235,35 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "movi v17.4s, #0xff\n"
     "ld1r { v16.4s }, [%x[shift_ptr]]\n"
     "cmp %x[n_channels], #0x40\n"
-    "sqrdmulh v15.4s, v15.4s, v18.4s\n"
-    "sqrdmulh v14.4s, v14.4s, v18.4s\n"
-    "sqrdmulh v13.4s, v13.4s, v18.4s\n"
-    "sqrdmulh v12.4s, v12.4s, v18.4s\n"
-    "sqrdmulh v11.4s, v11.4s, v18.4s\n"
+    "sqdmulh v15.4s, v15.4s, v18.4s\n"
+    "sqdmulh v14.4s, v14.4s, v18.4s\n"
+    "sqdmulh v13.4s, v13.4s, v18.4s\n"
+    "sqdmulh v12.4s, v12.4s, v18.4s\n"
+    "sqdmulh v11.4s, v11.4s, v18.4s\n"
     "srshl v15.4s, v15.4s, v16.4s\n"
     "srshl v14.4s, v14.4s, v16.4s\n"
     "srshl v13.4s, v13.4s, v16.4s\n"
     "srshl v12.4s, v12.4s, v16.4s\n"
     "srshl v11.4s, v11.4s, v16.4s\n"
-    "sqrdmulh v10.4s, v10.4s, v18.4s\n"
-    "sqrdmulh v9.4s, v9.4s, v18.4s\n"
-    "sqrdmulh v8.4s, v8.4s, v18.4s\n"
-    "sqrdmulh v7.4s, v7.4s, v18.4s\n"
+    "sqdmulh v10.4s, v10.4s, v18.4s\n"
+    "sqdmulh v9.4s, v9.4s, v18.4s\n"
+    "sqdmulh v8.4s, v8.4s, v18.4s\n"
+    "sqdmulh v7.4s, v7.4s, v18.4s\n"
     "srshl v10.4s, v10.4s, v16.4s\n"
     "srshl v9.4s, v9.4s, v16.4s\n"
     "srshl v8.4s, v8.4s, v16.4s\n"
     "srshl v7.4s, v7.4s, v16.4s\n"
-    "sqrdmulh v6.4s, v6.4s, v18.4s\n"
-    "sqrdmulh v5.4s, v5.4s, v18.4s\n"
-    "sqrdmulh v4.4s, v4.4s, v18.4s\n"
-    "sqrdmulh v3.4s, v3.4s, v18.4s\n"
+    "sqdmulh v6.4s, v6.4s, v18.4s\n"
+    "sqdmulh v5.4s, v5.4s, v18.4s\n"
+    "sqdmulh v4.4s, v4.4s, v18.4s\n"
+    "sqdmulh v3.4s, v3.4s, v18.4s\n"
     "srshl v6.4s, v6.4s, v16.4s\n"
     "srshl v5.4s, v5.4s, v16.4s\n"
     "srshl v4.4s, v4.4s, v16.4s\n"
     "srshl v3.4s, v3.4s, v16.4s\n"
-    "sqrdmulh v2.4s, v2.4s, v18.4s\n"
-    "sqrdmulh v1.4s, v1.4s, v18.4s\n"
-    "sqrdmulh v0.4s, v0.4s, v18.4s\n"
+    "sqdmulh v2.4s, v2.4s, v18.4s\n"
+    "sqdmulh v1.4s, v1.4s, v18.4s\n"
+    "sqdmulh v0.4s, v0.4s, v18.4s\n"
     "smax v15.4s, v15.4s, v19.4s\n"
     "srshl v2.4s, v2.4s, v16.4s\n"
     "srshl v1.4s, v1.4s, v16.4s\n"
@@ -302,22 +302,22 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "uzp1 v23.16b, v15.16b, v14.16b\n"
     "uzp1 v16.16b, v13.16b, v12.16b\n"
     "uzp1 v22.16b, v11.16b, v10.16b\n"
-    "uzp1 v18.16b, v9.16b, v8.16b\n"
-    "uzp1 v21.16b, v7.16b, v6.16b\n"
+    "uzp1 v21.16b, v9.16b, v8.16b\n"
+    "uzp1 v20.16b, v7.16b, v6.16b\n"
     "uzp1 v17.16b, v5.16b, v4.16b\n"
-    "uzp1 v20.16b, v3.16b, v2.16b\n"
-    "uzp1 v19.16b, v1.16b, v0.16b\n"
+    "uzp1 v19.16b, v3.16b, v2.16b\n"
+    "uzp1 v18.16b, v1.16b, v0.16b\n"
     "uzp1 v16.16b, v23.16b, v16.16b\n"
     "str q16, [%x[outptr], x26]\n"
-    "uzp1 v18.16b, v22.16b, v18.16b\n"
-    "uzp1 v17.16b, v21.16b, v17.16b\n"
+    "uzp1 v16.16b, v22.16b, v21.16b\n"
     "add x26, x26, #0x40\n"
-    "uzp1 v16.16b, v20.16b, v19.16b\n"
-    "str q18, [%x[outptr], x25]\n"
-    "str q17, [%x[outptr], x24]\n"
-    "str q16, [%x[outptr], x23]\n"
+    "uzp1 v17.16b, v20.16b, v17.16b\n"
+    "str q16, [%x[outptr], x25]\n"
+    "uzp1 v16.16b, v19.16b, v18.16b\n"
     "add x25, x25, #0x40\n"
+    "str q17, [%x[outptr], x24]\n"
     "add x24, x24, #0x40\n"
+    "str q16, [%x[outptr], x23]\n"
     "add x23, x23, #0x40\n"
     "bge 1b\n"
     "cbz %x[n_channels], 43f\n"
@@ -333,19 +333,19 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "movi v12.4s, #0x0\n"
     "cbz x22, 11f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 2 inputs loop
     "uaddl v23.8h, v31.8b, v30.8b\n"
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "uaddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "uaddw v15.4s, v15.4s, v23.4h\n"
     "ldr q31, [x21, x26]\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "subs x22, x22, #0x1\n"
     "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q30, [x20, x26]\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
@@ -365,10 +365,10 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "uxtl v16.8h, v31.8b\n"
+    "uxtl v23.8h, v31.8b\n"
     "uxtl2 v22.8h, v31.16b\n"
-    "uaddw v15.4s, v15.4s, v16.4h\n"
-    "uaddw2 v14.4s, v14.4s, v16.8h\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 12b\n"
@@ -379,10 +379,10 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "movi v17.4s, #0xff\n"
     "ld1r { v16.4s }, [%x[shift_ptr]]\n"
     "cmp %x[n_channels], #0x10\n"
-    "sqrdmulh v15.4s, v15.4s, v18.4s\n"
-    "sqrdmulh v14.4s, v14.4s, v18.4s\n"
-    "sqrdmulh v13.4s, v13.4s, v18.4s\n"
-    "sqrdmulh v12.4s, v12.4s, v18.4s\n"
+    "sqdmulh v15.4s, v15.4s, v18.4s\n"
+    "sqdmulh v14.4s, v14.4s, v18.4s\n"
+    "sqdmulh v13.4s, v13.4s, v18.4s\n"
+    "sqdmulh v12.4s, v12.4s, v18.4s\n"
     "srshl v15.4s, v15.4s, v16.4s\n"
     "srshl v14.4s, v14.4s, v16.4s\n"
     "srshl v13.4s, v13.4s, v16.4s\n"
@@ -538,11 +538,11 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "tbz %x[n_channels], #0, 33f\n"
     "ldr b31, [x21], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "uxtl v16.8h, v31.8b\n"
+    "uxtl v23.8h, v31.8b\n"
     "subs x20, x20, #0x1\n"
     "uxtl2 v22.8h, v31.16b\n"
-    "uaddw v15.4s, v15.4s, v16.4h\n"
-    "uaddw2 v14.4s, v14.4s, v16.8h\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 25b\n"
@@ -551,10 +551,10 @@ void a64_u8_nhwc_avg_generic_depthfirst_impl(
     "ld1r { v18.4s }, [%x[rescale_ptr]]\n"
     "movi v17.4s, #0xff\n"
     "ld1r { v16.4s }, [%x[shift_ptr]]\n"
-    "sqrdmulh v15.4s, v15.4s, v18.4s\n"
-    "sqrdmulh v14.4s, v14.4s, v18.4s\n"
-    "sqrdmulh v13.4s, v13.4s, v18.4s\n"
-    "sqrdmulh v12.4s, v12.4s, v18.4s\n"
+    "sqdmulh v15.4s, v15.4s, v18.4s\n"
+    "sqdmulh v14.4s, v14.4s, v18.4s\n"
+    "sqdmulh v13.4s, v13.4s, v18.4s\n"
+    "sqdmulh v12.4s, v12.4s, v18.4s\n"
     "srshl v15.4s, v15.4s, v16.4s\n"
     "srshl v14.4s, v14.4s, v16.4s\n"
     "srshl v13.4s, v13.4s, v16.4s\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index eac1f2dfdbd2846b6dc7637c62161957ce012027..02c43ccaba3de932716af43a86fc5b7ef19bfe11 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -63,23 +63,24 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x15, [%x[args], %[offsetof_n_channels]]\n"
     "mov x14, #0x0\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
-    "ldp x12, x11, [x19, #0x0]\n"
-    "cmp x15, #0x10\n"
-    "ldp x10, x9, [x19, #0x10]\n"
     "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "cmp x15, #0x10\n"
+    "ldp x12, x11, [x20, #0x0]\n"
+    "ldp x10, x9, [x20, #0x10]\n"
     "ldp x28, x27, [x19, #0x0]\n"
     "ldp x26, x25, [x19, #0x10]\n"
     "ldp x24, x23, [x19, #0x20]\n"
     "ldp x22, x21, [x19, #0x30]\n"
     "ldr x20, [x19, #0x40]\n"
     "blt 3f\n"
-    "lsr x19, x15, #0x4\n"
-    "sub x15, x15, x19, LSL #4\n"
     "ldr q30, [x27, x14]\n"
+    "lsr x19, x15, #0x4\n"
     "ldr q29, [x24, x14]\n"
+    "sub x15, x15, x19, LSL #4\n"
     "ldr q28, [x21, x14]\n"
+    "subs x19, x19, #0x1\n"
     "ldr q27, [x25, x14]\n"
     "ldr q26, [x28, x14]\n"
     "ldr q25, [x23, x14]\n"
@@ -87,11 +88,11 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "ldr q22, [x20, x14]\n"
     "add x14, x14, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "beq 2f\n"
     "1:"  // Vector: Loop
     "umax v21.16b, v30.16b, v29.16b\n"
     "ldr q30, [x27, x14]\n"
+    "subs x19, x19, #0x1\n"
     "umax v20.16b, v29.16b, v28.16b\n"
     "ldr q29, [x24, x14]\n"
     "umax v19.16b, v27.16b, v26.16b\n"
@@ -108,14 +109,13 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr q23, [x22, x14]\n"
     "umax v17.16b, v20.16b, v17.16b\n"
     "ldr q22, [x20, x14]\n"
-    "umax v16.16b, v20.16b, v16.16b\n"
     "add x14, x14, #0x10\n"
+    "umax v16.16b, v20.16b, v16.16b\n"
     "str q19, [x12, x13]\n"
     "str q18, [x11, x13]\n"
     "str q17, [x10, x13]\n"
     "str q16, [x9, x13]\n"
     "add x13, x13, #0x10\n"
-    "subs x19, x19, #0x1\n"
     "bgt 1b\n"
     "2:"  // Vector: Tail
     "umax v21.16b, v30.16b, v29.16b\n"
@@ -136,6 +136,7 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "cbz x15, 4f\n"
     "3:"  // Oddments
     "ldr b30, [x27, x14]\n"
+    "subs x15, x15, #0x1\n"
     "ldr b29, [x24, x14]\n"
     "umax v21.16b, v30.16b, v29.16b\n"
     "ldr b28, [x21, x14]\n"
@@ -150,9 +151,8 @@ void a64_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ldr b22, [x20, x14]\n"
     "add x14, x14, #0x1\n"
     "umax v18.16b, v25.16b, v24.16b\n"
-    "subs x15, x15, #0x1\n"
-    "umax v17.16b, v23.16b, v27.16b\n"
     "str b19, [x12, x13]\n"
+    "umax v17.16b, v23.16b, v27.16b\n"
     "umax v16.16b, v25.16b, v22.16b\n"
     "umax v18.16b, v21.16b, v18.16b\n"
     "str b18, [x11, x13]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst.hpp
index f018ecace9a88ed7a544caad8336d8674bd4c2b5..391af31d03e336ba9de9dbec9c5f093c945d3c7c 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_u8_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = a64_u8_nhwc_max_generic_depthfirst_impl;
 
   a64_u8_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp
index eacca15e355a5bfbc95b83377108d19b4dc732b6..f9bbfd8b907cf71ca7b5c3f426ca5164097c6efd 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8_nhwc_max_generic_depthfirst/generic.cpp
@@ -46,23 +46,23 @@ void a64_u8_nhwc_max_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x40\n"
     "blt 7f\n"
     "1:"  // 4-vectors of channels
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "mov x19, %x[inptrs]\n"
-    "movi v5.16b, #0x0\n"
+    "movi v6.16b, #0x0\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "movi v5.16b, #0x0\n"
     "movi v4.16b, #0x0\n"
-    "movi v3.16b, #0x0\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -75,47 +75,47 @@ void a64_u8_nhwc_max_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "umax v23.16b, v2.16b, v1.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "umax v22.16b, v30.16b, v22.16b\n"
     "add x19, x19, #0x20\n"
+    "umax v22.16b, v31.16b, v30.16b\n"
+    "ldr q3, [x23, x28]\n"
     "umax v18.16b, v29.16b, v28.16b\n"
-    "ldr q2, [x23, x28]\n"
     "umax v21.16b, v27.16b, v21.16b\n"
+    "ldr q2, [x22, x28]\n"
     "umax v17.16b, v26.16b, v17.16b\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "umax v20.16b, v25.16b, v20.16b\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "umax v16.16b, v24.16b, v16.16b\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "umax v19.16b, v23.16b, v19.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "umax v18.16b, v22.16b, v18.16b\n"
-    "ldr q22, [x22, x27]\n"
-    "umax v17.16b, v21.16b, v17.16b\n"
     "ldr q29, [x21, x27]\n"
-    "umax v16.16b, v20.16b, v16.16b\n"
+    "umax v17.16b, v21.16b, v17.16b\n"
     "ldr q28, [x20, x27]\n"
-    "umax v6.16b, v6.16b, v19.16b\n"
+    "umax v16.16b, v20.16b, v16.16b\n"
     "ldr q27, [x23, x26]\n"
-    "umax v5.16b, v5.16b, v18.16b\n"
+    "umax v7.16b, v7.16b, v19.16b\n"
     "ldr q21, [x22, x26]\n"
-    "umax v4.16b, v4.16b, v17.16b\n"
+    "umax v6.16b, v6.16b, v18.16b\n"
     "ldr q26, [x21, x26]\n"
-    "umax v3.16b, v3.16b, v16.16b\n"
+    "umax v5.16b, v5.16b, v17.16b\n"
     "ldr q17, [x20, x26]\n"
+    "umax v4.16b, v4.16b, v16.16b\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "umax v23.16b, v2.16b, v1.16b\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
-    "umax v22.16b, v30.16b, v22.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
+    "umax v22.16b, v31.16b, v30.16b\n"
     "umax v18.16b, v29.16b, v28.16b\n"
     "umax v21.16b, v27.16b, v21.16b\n"
     "umax v17.16b, v26.16b, v17.16b\n"
@@ -125,33 +125,33 @@ void a64_u8_nhwc_max_generic_depthfirst_impl(
     "umax v18.16b, v22.16b, v18.16b\n"
     "umax v17.16b, v21.16b, v17.16b\n"
     "umax v16.16b, v20.16b, v16.16b\n"
-    "umax v6.16b, v6.16b, v19.16b\n"
-    "umax v5.16b, v5.16b, v18.16b\n"
-    "umax v4.16b, v4.16b, v17.16b\n"
-    "umax v3.16b, v3.16b, v16.16b\n"
+    "umax v7.16b, v7.16b, v19.16b\n"
+    "umax v6.16b, v6.16b, v18.16b\n"
+    "umax v5.16b, v5.16b, v17.16b\n"
+    "umax v4.16b, v4.16b, v16.16b\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "umax v6.16b, v6.16b, v2.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "umax v7.16b, v7.16b, v3.16b\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "umax v5.16b, v5.16b, v30.16b\n"
+    "umax v6.16b, v6.16b, v31.16b\n"
     "ldr q25, [x23, x25]\n"
-    "umax v4.16b, v4.16b, v27.16b\n"
-    "umax v3.16b, v3.16b, v25.16b\n"
+    "umax v5.16b, v5.16b, v27.16b\n"
+    "umax v4.16b, v4.16b, v25.16b\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
-    "str q5, [%x[outptr], x27]\n"
-    "str q4, [%x[outptr], x26]\n"
-    "str q3, [%x[outptr], x25]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x40\n"
+    "str q6, [%x[outptr], x27]\n"
     "add x27, x27, #0x40\n"
+    "str q5, [%x[outptr], x26]\n"
     "add x26, x26, #0x40\n"
+    "str q4, [%x[outptr], x25]\n"
     "add x25, x25, #0x40\n"
     "sub %x[n_channels], %x[n_channels], #0x40\n"
     "cmp %x[n_channels], #0x40\n"
@@ -161,275 +161,275 @@ void a64_u8_nhwc_max_generic_depthfirst_impl(
     "cmp %x[n_channels], #0x10\n"
     "blt 14f\n"
     "8:"  // Single vector of channels: Loop
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "umax v23.16b, v2.16b, v1.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "umax v19.16b, v23.16b, v19.16b\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "umax v6.16b, v6.16b, v19.16b\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "umax v19.16b, v23.16b, v19.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "umax v7.16b, v7.16b, v19.16b\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "umax v23.16b, v2.16b, v1.16b\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "umax v19.16b, v23.16b, v19.16b\n"
-    "umax v6.16b, v6.16b, v19.16b\n"
+    "umax v7.16b, v7.16b, v19.16b\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "umax v6.16b, v6.16b, v2.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "umax v7.16b, v7.16b, v3.16b\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "str q6, [%x[outptr], x28]\n"
+    "str q7, [%x[outptr], x28]\n"
     "add x28, x28, #0x10\n"
     "sub %x[n_channels], %x[n_channels], #0x10\n"
     "cmp %x[n_channels], #0x10\n"
     "bge 8b\n"
     "cbz %x[n_channels], 43f\n"
     "14:"  // Oddments
-    "movi v6.16b, #0x0\n"
+    "movi v7.16b, #0x0\n"
     "add %x[outptr], %x[outptr], x28\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 24f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #3, 19f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #2, 17f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
-    "ld1 { v1.h }[6], [x22], #0x2\n"
-    "ld1 { v0.h }[6], [x21], #0x2\n"
-    "ld1 { v31.h }[6], [x20], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
+    "ld1 { v2.h }[6], [x22], #0x2\n"
+    "ld1 { v1.h }[6], [x21], #0x2\n"
+    "ld1 { v0.h }[6], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
-    "ld1 { v1.b }[14], [x22], #0x1\n"
-    "ld1 { v0.b }[14], [x21], #0x1\n"
-    "ld1 { v31.b }[14], [x20], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
+    "ld1 { v2.b }[14], [x22], #0x1\n"
+    "ld1 { v1.b }[14], [x21], #0x1\n"
+    "ld1 { v0.b }[14], [x20], #0x1\n"
     "b 23f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
-    "ld1 { v1.b }[12], [x22], #0x1\n"
-    "ld1 { v0.b }[12], [x21], #0x1\n"
-    "ld1 { v31.b }[12], [x20], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
+    "ld1 { v2.b }[12], [x22], #0x1\n"
+    "ld1 { v1.b }[12], [x21], #0x1\n"
+    "ld1 { v0.b }[12], [x20], #0x1\n"
     "b 23f\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 18f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
-    "ld1 { v1.h }[4], [x22], #0x2\n"
-    "ld1 { v0.h }[4], [x21], #0x2\n"
-    "ld1 { v31.h }[4], [x20], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
+    "ld1 { v2.h }[4], [x22], #0x2\n"
+    "ld1 { v1.h }[4], [x21], #0x2\n"
+    "ld1 { v0.h }[4], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
-    "ld1 { v1.b }[10], [x22], #0x1\n"
-    "ld1 { v0.b }[10], [x21], #0x1\n"
-    "ld1 { v31.b }[10], [x20], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
+    "ld1 { v2.b }[10], [x22], #0x1\n"
+    "ld1 { v1.b }[10], [x21], #0x1\n"
+    "ld1 { v0.b }[10], [x20], #0x1\n"
     "b 23f\n"
     "18:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
-    "ld1 { v1.b }[8], [x22], #0x1\n"
-    "ld1 { v0.b }[8], [x21], #0x1\n"
-    "ld1 { v31.b }[8], [x20], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
+    "ld1 { v2.b }[8], [x22], #0x1\n"
+    "ld1 { v1.b }[8], [x21], #0x1\n"
+    "ld1 { v0.b }[8], [x20], #0x1\n"
     "b 23f\n"
     "19:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 21f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "tbz %x[n_channels], #1, 20f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
-    "ld1 { v1.h }[2], [x22], #0x2\n"
-    "ld1 { v0.h }[2], [x21], #0x2\n"
-    "ld1 { v31.h }[2], [x20], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
+    "ld1 { v2.h }[2], [x22], #0x2\n"
+    "ld1 { v1.h }[2], [x21], #0x2\n"
+    "ld1 { v0.h }[2], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
-    "ld1 { v1.b }[6], [x22], #0x1\n"
-    "ld1 { v0.b }[6], [x21], #0x1\n"
-    "ld1 { v31.b }[6], [x20], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
+    "ld1 { v2.b }[6], [x22], #0x1\n"
+    "ld1 { v1.b }[6], [x21], #0x1\n"
+    "ld1 { v0.b }[6], [x20], #0x1\n"
     "b 23f\n"
     "20:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
-    "ld1 { v1.b }[4], [x22], #0x1\n"
-    "ld1 { v0.b }[4], [x21], #0x1\n"
-    "ld1 { v31.b }[4], [x20], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
+    "ld1 { v2.b }[4], [x22], #0x1\n"
+    "ld1 { v1.b }[4], [x21], #0x1\n"
+    "ld1 { v0.b }[4], [x20], #0x1\n"
     "b 23f\n"
     "21:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 22f\n"
-    "ldr h2, [x23], #0x2\n"
-    "ldr h1, [x22], #0x2\n"
-    "ldr h0, [x21], #0x2\n"
-    "ldr h31, [x20], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
+    "ldr h2, [x22], #0x2\n"
+    "ldr h1, [x21], #0x2\n"
+    "ldr h0, [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
-    "ld1 { v1.b }[2], [x22], #0x1\n"
-    "ld1 { v0.b }[2], [x21], #0x1\n"
-    "ld1 { v31.b }[2], [x20], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
+    "ld1 { v2.b }[2], [x22], #0x1\n"
+    "ld1 { v1.b }[2], [x21], #0x1\n"
+    "ld1 { v0.b }[2], [x20], #0x1\n"
     "b 23f\n"
     "22:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ldr b2, [x23], #0x1\n"
-    "ldr b1, [x22], #0x1\n"
-    "ldr b0, [x21], #0x1\n"
-    "ldr b31, [x20], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
+    "ldr b2, [x22], #0x1\n"
+    "ldr b1, [x21], #0x1\n"
+    "ldr b0, [x20], #0x1\n"
     "23:"  // Oddments: 4 inputs loop: Load: Bit 3: End
-    "umax v23.16b, v2.16b, v1.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
     "subs x24, x24, #0x1\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "umax v19.16b, v23.16b, v19.16b\n"
-    "umax v6.16b, v6.16b, v19.16b\n"
+    "umax v7.16b, v7.16b, v19.16b\n"
     "bgt 15b\n"
     "24:"  // Oddments: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 34f\n"
     "25:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #3, 29f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #2, 27f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "tbz %x[n_channels], #1, 26f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
     "b 33f\n"
     "26:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
     "b 33f\n"
     "27:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 28f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
     "b 33f\n"
     "28:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
     "b 33f\n"
     "29:"  // Oddments: Single input loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 31f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "tbz %x[n_channels], #1, 30f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
     "b 33f\n"
     "30:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
     "b 33f\n"
     "31:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 32f\n"
-    "ldr h2, [x23], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
     "b 33f\n"
     "32:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ldr b2, [x23], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "umax v6.16b, v6.16b, v2.16b\n"
+    "umax v7.16b, v7.16b, v3.16b\n"
     "subs x20, x20, #0x1\n"
     "bgt 25b\n"
     "34:"  // Oddments: Single input loop: End
     "tbz %x[n_channels], #3, 38f\n"
-    "st1 { v6.d }[0], [%x[outptr]], #0x8\n"
+    "st1 { v7.d }[0], [%x[outptr]], #0x8\n"
     "tbz %x[n_channels], #2, 36f\n"
-    "st1 { v6.s }[2], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[2], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #1, 35f\n"
-    "st1 { v6.h }[6], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[6], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[14], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[14], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "35:"  // Oddments: Store: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[12], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[12], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "36:"  // Oddments: Store: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 37f\n"
-    "st1 { v6.h }[4], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[4], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[10], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[10], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "37:"  // Oddments: Store: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[8], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[8], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "38:"  // Oddments: Store: Bit 3: Unset
     "tbz %x[n_channels], #2, 40f\n"
-    "st1 { v6.s }[0], [%x[outptr]], #0x4\n"
+    "st1 { v7.s }[0], [%x[outptr]], #0x4\n"
     "tbz %x[n_channels], #1, 39f\n"
-    "st1 { v6.h }[2], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[2], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[6], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[6], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "39:"  // Oddments: Store: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[4], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[4], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "40:"  // Oddments: Store: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 41f\n"
-    "st1 { v6.h }[0], [%x[outptr]], #0x2\n"
+    "st1 { v7.h }[0], [%x[outptr]], #0x2\n"
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[2], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[2], [%x[outptr]], #0x1\n"
     "b 42f\n"
     "41:"  // Oddments: Store: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 42f\n"
-    "st1 { v6.b }[0], [%x[outptr]], #0x1\n"
+    "st1 { v7.b }[0], [%x[outptr]], #0x1\n"
     "42:"  // Oddments: Store: Bit 3: End
 
     "43:"  // End
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst.hpp
index 114eacf450899049c488c55f63cc74fc208bdd6a..d46658f08026fb039e5fdd801f5bfe775966159f 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_u8q_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = a64_u8q_nhwc_avg_generic_depthfirst_impl;
 
   a64_u8q_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp
index e2cb9d7320195a8682f43f6bd158a239544ce77a..a57fe6df68a47a3425cbe7b77b69c033531ba086 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_avg_generic_depthfirst/generic.cpp
@@ -42,10 +42,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -145,10 +145,10 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl(
     "mov v0.16b, v15.16b\n"
     "cbz x22, 4f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "ldr q29, [x21, x25]\n"
     "ldr q28, [x20, x25]\n"
     "ldr q27, [x21, x24]\n"
@@ -161,24 +161,24 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl(
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "uaddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "uaddl v21.8h, v29.8b, v28.8b\n"
     "ldr q31, [x21, x26]\n"
+    "uaddl v21.8h, v29.8b, v28.8b\n"
+    "subs x22, x22, #0x1\n"
     "uaddl2 v20.8h, v29.16b, v28.16b\n"
-    "uaddl v19.8h, v27.8b, v26.8b\n"
     "ldr q30, [x20, x26]\n"
-    "uaddl2 v18.8h, v27.16b, v26.16b\n"
+    "uaddl v19.8h, v27.8b, v26.8b\n"
     "ldr q29, [x21, x25]\n"
-    "uaddl v17.8h, v25.8b, v24.8b\n"
+    "uaddl2 v18.8h, v27.16b, v26.16b\n"
     "ldr q28, [x20, x25]\n"
-    "uaddl2 v16.8h, v25.16b, v24.16b\n"
+    "uaddl v17.8h, v25.8b, v24.8b\n"
     "ldr q27, [x21, x24]\n"
-    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddl2 v16.8h, v25.16b, v24.16b\n"
     "ldr q26, [x20, x24]\n"
-    "uaddw2 v14.4s, v14.4s, v23.8h\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
     "ldr q25, [x21, x23]\n"
-    "uaddw v13.4s, v13.4s, v22.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q24, [x20, x23]\n"
+    "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "uaddw v11.4s, v11.4s, v21.4h\n"
     "uaddw2 v10.4s, v10.4s, v21.8h\n"
@@ -225,19 +225,19 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "uxtl v16.8h, v31.8b\n"
+    "uxtl v23.8h, v31.8b\n"
     "ldr q29, [x21, x25]\n"
     "uxtl2 v22.8h, v31.16b\n"
     "ldr q27, [x21, x24]\n"
     "ldr q25, [x21, x23]\n"
-    "uaddw v15.4s, v15.4s, v16.4h\n"
-    "uaddw2 v14.4s, v14.4s, v16.8h\n"
     "uxtl v21.8h, v29.8b\n"
     "uxtl2 v20.8h, v29.16b\n"
     "uxtl v19.8h, v27.8b\n"
     "uxtl2 v18.8h, v27.16b\n"
     "uxtl v17.8h, v25.8b\n"
     "uxtl2 v16.8h, v25.16b\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "uaddw v11.4s, v11.4s, v21.4h\n"
@@ -362,22 +362,22 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl(
     "uzp1 v16.16b, v13.16b, v12.16b\n"
     "smin v0.4s, v0.4s, v19.4s\n"
     "uzp1 v22.16b, v11.16b, v10.16b\n"
-    "uzp1 v18.16b, v9.16b, v8.16b\n"
-    "uzp1 v21.16b, v7.16b, v6.16b\n"
+    "uzp1 v21.16b, v9.16b, v8.16b\n"
+    "uzp1 v20.16b, v7.16b, v6.16b\n"
     "uzp1 v17.16b, v5.16b, v4.16b\n"
-    "uzp1 v20.16b, v3.16b, v2.16b\n"
-    "uzp1 v19.16b, v1.16b, v0.16b\n"
+    "uzp1 v19.16b, v3.16b, v2.16b\n"
+    "uzp1 v18.16b, v1.16b, v0.16b\n"
     "uzp1 v16.16b, v23.16b, v16.16b\n"
     "str q16, [%x[outptr], x26]\n"
-    "uzp1 v18.16b, v22.16b, v18.16b\n"
-    "uzp1 v17.16b, v21.16b, v17.16b\n"
+    "uzp1 v16.16b, v22.16b, v21.16b\n"
     "add x26, x26, #0x40\n"
-    "uzp1 v16.16b, v20.16b, v19.16b\n"
-    "str q18, [%x[outptr], x25]\n"
-    "str q17, [%x[outptr], x24]\n"
-    "str q16, [%x[outptr], x23]\n"
+    "uzp1 v17.16b, v20.16b, v17.16b\n"
+    "str q16, [%x[outptr], x25]\n"
+    "uzp1 v16.16b, v19.16b, v18.16b\n"
     "add x25, x25, #0x40\n"
+    "str q17, [%x[outptr], x24]\n"
     "add x24, x24, #0x40\n"
+    "str q16, [%x[outptr], x23]\n"
     "add x23, x23, #0x40\n"
     "bge 1b\n"
     "cbz %x[n_channels], 43f\n"
@@ -393,19 +393,19 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl(
     "mov v12.16b, v15.16b\n"
     "cbz x22, 11f\n"
     "ldp x21, x20, [x19, #0x0]\n"
-    "add x19, x19, #0x10\n"
-    "subs x22, x22, #0x1\n"
     "ldr q31, [x21, x26]\n"
+    "add x19, x19, #0x10\n"
     "ldr q30, [x20, x26]\n"
+    "subs x22, x22, #0x1\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 2 inputs loop
     "uaddl v23.8h, v31.8b, v30.8b\n"
     "ldp x21, x20, [x19, #0x0]\n"
     "add x19, x19, #0x10\n"
     "uaddl2 v22.8h, v31.16b, v30.16b\n"
-    "subs x22, x22, #0x1\n"
-    "uaddw v15.4s, v15.4s, v23.4h\n"
     "ldr q31, [x21, x26]\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "subs x22, x22, #0x1\n"
     "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "ldr q30, [x20, x26]\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
@@ -425,10 +425,10 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl(
     "ldr x21, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
     "ldr q31, [x21, x26]\n"
-    "uxtl v16.8h, v31.8b\n"
+    "uxtl v23.8h, v31.8b\n"
     "uxtl2 v22.8h, v31.16b\n"
-    "uaddw v15.4s, v15.4s, v16.4h\n"
-    "uaddw2 v14.4s, v14.4s, v16.8h\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 12b\n"
@@ -609,11 +609,11 @@ void a64_u8q_nhwc_avg_generic_depthfirst_impl(
     "tbz %x[n_channels], #0, 33f\n"
     "ldr b31, [x21], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "uxtl v16.8h, v31.8b\n"
+    "uxtl v23.8h, v31.8b\n"
     "subs x20, x20, #0x1\n"
     "uxtl2 v22.8h, v31.16b\n"
-    "uaddw v15.4s, v15.4s, v16.4h\n"
-    "uaddw2 v14.4s, v14.4s, v16.8h\n"
+    "uaddw v15.4s, v15.4s, v23.4h\n"
+    "uaddw2 v14.4s, v14.4s, v23.8h\n"
     "uaddw v13.4s, v13.4s, v22.4h\n"
     "uaddw2 v12.4s, v12.4s, v22.8h\n"
     "bgt 25b\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst.hpp
index 166f3fac031c45ebd8cc05de562b44c747419d31..1b97b458c00bcad9916e724219b085fe140c2308 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct a64_u8q_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = a64_u8q_nhwc_max_generic_depthfirst_impl;
 
   a64_u8q_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp
index b056be20bcb34b3bf8326c6d13990c7077bf182e..0d196e097efd69124c31f9a5b04cc28be754ea81 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_u8q_nhwc_max_generic_depthfirst/generic.cpp
@@ -51,7 +51,7 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "1:"  // 4-vectors of channels
     "movi v4.16b, #0x0\n"
     "mov x19, %x[inptrs]\n"
-    "movi v3.16b, #0x0\n"
+    "movi v8.16b, #0x0\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "movi v7.16b, #0x0\n"
     "movi v6.16b, #0x0\n"
@@ -60,12 +60,12 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
-    "ldr q30, [x23, x27]\n"
-    "ldr q22, [x22, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "ldr q29, [x21, x27]\n"
     "ldr q28, [x20, x27]\n"
     "ldr q27, [x23, x26]\n"
@@ -78,47 +78,47 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "ldr q16, [x20, x25]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "umax v23.16b, v2.16b, v1.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "umax v22.16b, v30.16b, v22.16b\n"
     "add x19, x19, #0x20\n"
+    "umax v22.16b, v31.16b, v30.16b\n"
+    "ldr q3, [x23, x28]\n"
     "umax v18.16b, v29.16b, v28.16b\n"
-    "ldr q2, [x23, x28]\n"
     "umax v21.16b, v27.16b, v21.16b\n"
+    "ldr q2, [x22, x28]\n"
     "umax v17.16b, v26.16b, v17.16b\n"
-    "ldr q1, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
     "umax v20.16b, v25.16b, v20.16b\n"
-    "ldr q0, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "umax v16.16b, v24.16b, v16.16b\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q31, [x23, x27]\n"
     "umax v19.16b, v23.16b, v19.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q30, [x22, x27]\n"
     "umax v18.16b, v22.16b, v18.16b\n"
-    "ldr q22, [x22, x27]\n"
-    "umax v17.16b, v21.16b, v17.16b\n"
     "ldr q29, [x21, x27]\n"
-    "umax v16.16b, v20.16b, v16.16b\n"
+    "umax v17.16b, v21.16b, v17.16b\n"
     "ldr q28, [x20, x27]\n"
-    "umax v4.16b, v4.16b, v19.16b\n"
+    "umax v16.16b, v20.16b, v16.16b\n"
     "ldr q27, [x23, x26]\n"
-    "umax v3.16b, v3.16b, v18.16b\n"
+    "umax v4.16b, v4.16b, v19.16b\n"
     "ldr q21, [x22, x26]\n"
-    "umax v7.16b, v7.16b, v17.16b\n"
+    "umax v8.16b, v8.16b, v18.16b\n"
     "ldr q26, [x21, x26]\n"
-    "umax v6.16b, v6.16b, v16.16b\n"
+    "umax v7.16b, v7.16b, v17.16b\n"
     "ldr q17, [x20, x26]\n"
+    "umax v6.16b, v6.16b, v16.16b\n"
     "ldr q25, [x23, x25]\n"
     "ldr q20, [x22, x25]\n"
     "ldr q24, [x21, x25]\n"
     "ldr q16, [x20, x25]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "umax v23.16b, v2.16b, v1.16b\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
-    "umax v22.16b, v30.16b, v22.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
+    "umax v22.16b, v31.16b, v30.16b\n"
     "umax v18.16b, v29.16b, v28.16b\n"
     "umax v21.16b, v27.16b, v21.16b\n"
     "umax v17.16b, v26.16b, v17.16b\n"
@@ -129,7 +129,7 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "umax v17.16b, v21.16b, v17.16b\n"
     "umax v16.16b, v20.16b, v16.16b\n"
     "umax v4.16b, v4.16b, v19.16b\n"
-    "umax v3.16b, v3.16b, v18.16b\n"
+    "umax v8.16b, v8.16b, v18.16b\n"
     "umax v7.16b, v7.16b, v17.16b\n"
     "umax v6.16b, v6.16b, v16.16b\n"
     "4:"  // 4-vectors of channels: After loop
@@ -138,11 +138,11 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "umax v4.16b, v4.16b, v2.16b\n"
-    "ldr q30, [x23, x27]\n"
+    "ldr q3, [x23, x28]\n"
+    "umax v4.16b, v4.16b, v3.16b\n"
+    "ldr q31, [x23, x27]\n"
     "ldr q27, [x23, x26]\n"
-    "umax v3.16b, v3.16b, v30.16b\n"
+    "umax v8.16b, v8.16b, v31.16b\n"
     "ldr q25, [x23, x25]\n"
     "umax v7.16b, v7.16b, v27.16b\n"
     "umax v6.16b, v6.16b, v25.16b\n"
@@ -153,10 +153,10 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "ld1r { v5.4s }, [x19]\n"
     "uxtl2 v16.8h, v4.16b\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_mul]\n"
-    "uxtl v21.8h, v3.8b\n"
+    "uxtl v21.8h, v8.8b\n"
     "ld1r { v4.4s }, [x19]\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n"
-    "uxtl2 v20.8h, v3.16b\n"
+    "uxtl2 v20.8h, v8.16b\n"
     "ld1r { v3.4s }, [x19]\n"
     "add x19, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n"
     "uxtl v19.8h, v7.8b\n"
@@ -287,18 +287,18 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "uzp1 v16.16b, v26.16b, v24.16b\n"
     "str q16, [%x[outptr], x28]\n"
     "add v22.4s, v22.4s, v1.4s\n"
-    "add v21.4s, v21.4s, v1.4s\n"
     "add x28, x28, #0x40\n"
+    "add v21.4s, v21.4s, v1.4s\n"
     "uzp1 v16.16b, v23.16b, v20.16b\n"
     "str q16, [%x[outptr], x27]\n"
     "smax v22.4s, v22.4s, v30.4s\n"
-    "smax v21.4s, v21.4s, v30.4s\n"
     "add x27, x27, #0x40\n"
+    "smax v21.4s, v21.4s, v30.4s\n"
     "uzp1 v16.16b, v19.16b, v18.16b\n"
     "str q16, [%x[outptr], x26]\n"
     "smin v22.4s, v22.4s, v29.4s\n"
-    "smin v21.4s, v21.4s, v29.4s\n"
     "add x26, x26, #0x40\n"
+    "smin v21.4s, v21.4s, v29.4s\n"
     "uzp1 v16.16b, v22.16b, v21.16b\n"
     "uzp1 v16.16b, v17.16b, v16.16b\n"
     "str q16, [%x[outptr], x25]\n"
@@ -314,31 +314,31 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "subs x24, x24, #0x1\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "umax v23.16b, v2.16b, v1.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "umax v19.16b, v23.16b, v19.16b\n"
     "add x19, x19, #0x20\n"
-    "ldr q2, [x23, x28]\n"
+    "umax v19.16b, v23.16b, v19.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "ldr q2, [x22, x28]\n"
     "umax v4.16b, v4.16b, v19.16b\n"
-    "ldr q1, [x22, x28]\n"
-    "ldr q0, [x21, x28]\n"
-    "ldr q31, [x20, x28]\n"
+    "ldr q1, [x21, x28]\n"
+    "ldr q0, [x20, x28]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "umax v23.16b, v2.16b, v1.16b\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "umax v19.16b, v23.16b, v19.16b\n"
     "umax v4.16b, v4.16b, v19.16b\n"
     "11:"  // Single vector of channels: Loop: After loop
@@ -347,8 +347,8 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ldr q2, [x23, x28]\n"
-    "umax v4.16b, v4.16b, v2.16b\n"
+    "ldr q3, [x23, x28]\n"
+    "umax v4.16b, v4.16b, v3.16b\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
     "uxtl v17.8h, v4.8b\n"
@@ -410,110 +410,110 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 24f\n"
     "15:"  // Oddments: 4 inputs loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "add x23, x23, x28\n"
-    "movi v1.16b, #0x0\n"
+    "movi v2.16b, #0x0\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movi v0.16b, #0x0\n"
+    "movi v1.16b, #0x0\n"
     "add x19, x19, #0x20\n"
-    "movi v31.16b, #0x0\n"
+    "movi v0.16b, #0x0\n"
     "add x22, x22, x28\n"
     "add x21, x21, x28\n"
     "add x20, x20, x28\n"
     "tbz %x[n_channels], #3, 19f\n"
-    "ldr d2, [x23], #0x8\n"
-    "ldr d1, [x22], #0x8\n"
-    "ldr d0, [x21], #0x8\n"
-    "ldr d31, [x20], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
+    "ldr d2, [x22], #0x8\n"
+    "ldr d1, [x21], #0x8\n"
+    "ldr d0, [x20], #0x8\n"
     "tbz %x[n_channels], #2, 17f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
-    "ld1 { v1.s }[2], [x22], #0x4\n"
-    "ld1 { v0.s }[2], [x21], #0x4\n"
-    "ld1 { v31.s }[2], [x20], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
+    "ld1 { v2.s }[2], [x22], #0x4\n"
+    "ld1 { v1.s }[2], [x21], #0x4\n"
+    "ld1 { v0.s }[2], [x20], #0x4\n"
     "tbz %x[n_channels], #1, 16f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
-    "ld1 { v1.h }[6], [x22], #0x2\n"
-    "ld1 { v0.h }[6], [x21], #0x2\n"
-    "ld1 { v31.h }[6], [x20], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
+    "ld1 { v2.h }[6], [x22], #0x2\n"
+    "ld1 { v1.h }[6], [x21], #0x2\n"
+    "ld1 { v0.h }[6], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
-    "ld1 { v1.b }[14], [x22], #0x1\n"
-    "ld1 { v0.b }[14], [x21], #0x1\n"
-    "ld1 { v31.b }[14], [x20], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
+    "ld1 { v2.b }[14], [x22], #0x1\n"
+    "ld1 { v1.b }[14], [x21], #0x1\n"
+    "ld1 { v0.b }[14], [x20], #0x1\n"
     "b 23f\n"
     "16:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
-    "ld1 { v1.b }[12], [x22], #0x1\n"
-    "ld1 { v0.b }[12], [x21], #0x1\n"
-    "ld1 { v31.b }[12], [x20], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
+    "ld1 { v2.b }[12], [x22], #0x1\n"
+    "ld1 { v1.b }[12], [x21], #0x1\n"
+    "ld1 { v0.b }[12], [x20], #0x1\n"
     "b 23f\n"
     "17:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 18f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
-    "ld1 { v1.h }[4], [x22], #0x2\n"
-    "ld1 { v0.h }[4], [x21], #0x2\n"
-    "ld1 { v31.h }[4], [x20], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
+    "ld1 { v2.h }[4], [x22], #0x2\n"
+    "ld1 { v1.h }[4], [x21], #0x2\n"
+    "ld1 { v0.h }[4], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
-    "ld1 { v1.b }[10], [x22], #0x1\n"
-    "ld1 { v0.b }[10], [x21], #0x1\n"
-    "ld1 { v31.b }[10], [x20], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
+    "ld1 { v2.b }[10], [x22], #0x1\n"
+    "ld1 { v1.b }[10], [x21], #0x1\n"
+    "ld1 { v0.b }[10], [x20], #0x1\n"
     "b 23f\n"
     "18:"  // Oddments: 4 inputs loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
-    "ld1 { v1.b }[8], [x22], #0x1\n"
-    "ld1 { v0.b }[8], [x21], #0x1\n"
-    "ld1 { v31.b }[8], [x20], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
+    "ld1 { v2.b }[8], [x22], #0x1\n"
+    "ld1 { v1.b }[8], [x21], #0x1\n"
+    "ld1 { v0.b }[8], [x20], #0x1\n"
     "b 23f\n"
     "19:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 21f\n"
-    "ldr s2, [x23], #0x4\n"
-    "ldr s1, [x22], #0x4\n"
-    "ldr s0, [x21], #0x4\n"
-    "ldr s31, [x20], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
+    "ldr s2, [x22], #0x4\n"
+    "ldr s1, [x21], #0x4\n"
+    "ldr s0, [x20], #0x4\n"
     "tbz %x[n_channels], #1, 20f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
-    "ld1 { v1.h }[2], [x22], #0x2\n"
-    "ld1 { v0.h }[2], [x21], #0x2\n"
-    "ld1 { v31.h }[2], [x20], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
+    "ld1 { v2.h }[2], [x22], #0x2\n"
+    "ld1 { v1.h }[2], [x21], #0x2\n"
+    "ld1 { v0.h }[2], [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
-    "ld1 { v1.b }[6], [x22], #0x1\n"
-    "ld1 { v0.b }[6], [x21], #0x1\n"
-    "ld1 { v31.b }[6], [x20], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
+    "ld1 { v2.b }[6], [x22], #0x1\n"
+    "ld1 { v1.b }[6], [x21], #0x1\n"
+    "ld1 { v0.b }[6], [x20], #0x1\n"
     "b 23f\n"
     "20:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
-    "ld1 { v1.b }[4], [x22], #0x1\n"
-    "ld1 { v0.b }[4], [x21], #0x1\n"
-    "ld1 { v31.b }[4], [x20], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
+    "ld1 { v2.b }[4], [x22], #0x1\n"
+    "ld1 { v1.b }[4], [x21], #0x1\n"
+    "ld1 { v0.b }[4], [x20], #0x1\n"
     "b 23f\n"
     "21:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 22f\n"
-    "ldr h2, [x23], #0x2\n"
-    "ldr h1, [x22], #0x2\n"
-    "ldr h0, [x21], #0x2\n"
-    "ldr h31, [x20], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
+    "ldr h2, [x22], #0x2\n"
+    "ldr h1, [x21], #0x2\n"
+    "ldr h0, [x20], #0x2\n"
     "tbz %x[n_channels], #0, 23f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
-    "ld1 { v1.b }[2], [x22], #0x1\n"
-    "ld1 { v0.b }[2], [x21], #0x1\n"
-    "ld1 { v31.b }[2], [x20], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
+    "ld1 { v2.b }[2], [x22], #0x1\n"
+    "ld1 { v1.b }[2], [x21], #0x1\n"
+    "ld1 { v0.b }[2], [x20], #0x1\n"
     "b 23f\n"
     "22:"  // Oddments: 4 inputs loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 23f\n"
-    "ldr b2, [x23], #0x1\n"
-    "ldr b1, [x22], #0x1\n"
-    "ldr b0, [x21], #0x1\n"
-    "ldr b31, [x20], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
+    "ldr b2, [x22], #0x1\n"
+    "ldr b1, [x21], #0x1\n"
+    "ldr b0, [x20], #0x1\n"
     "23:"  // Oddments: 4 inputs loop: Load: Bit 3: End
-    "umax v23.16b, v2.16b, v1.16b\n"
+    "umax v23.16b, v3.16b, v2.16b\n"
     "subs x24, x24, #0x1\n"
-    "umax v19.16b, v0.16b, v31.16b\n"
+    "umax v19.16b, v1.16b, v0.16b\n"
     "umax v19.16b, v23.16b, v19.16b\n"
     "umax v4.16b, v4.16b, v19.16b\n"
     "bgt 15b\n"
@@ -521,55 +521,55 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 34f\n"
     "25:"  // Oddments: Single input loop
-    "movi v2.16b, #0x0\n"
+    "movi v3.16b, #0x0\n"
     "ldr x23, [x19], #0x8\n"
     "add x23, x23, x28\n"
     "tbz %x[n_channels], #3, 29f\n"
-    "ldr d2, [x23], #0x8\n"
+    "ldr d3, [x23], #0x8\n"
     "tbz %x[n_channels], #2, 27f\n"
-    "ld1 { v2.s }[2], [x23], #0x4\n"
+    "ld1 { v3.s }[2], [x23], #0x4\n"
     "tbz %x[n_channels], #1, 26f\n"
-    "ld1 { v2.h }[6], [x23], #0x2\n"
+    "ld1 { v3.h }[6], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[14], [x23], #0x1\n"
+    "ld1 { v3.b }[14], [x23], #0x1\n"
     "b 33f\n"
     "26:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[12], [x23], #0x1\n"
+    "ld1 { v3.b }[12], [x23], #0x1\n"
     "b 33f\n"
     "27:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset
     "tbz %x[n_channels], #1, 28f\n"
-    "ld1 { v2.h }[4], [x23], #0x2\n"
+    "ld1 { v3.h }[4], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[10], [x23], #0x1\n"
+    "ld1 { v3.b }[10], [x23], #0x1\n"
     "b 33f\n"
     "28:"  // Oddments: Single input loop: Load: Bit 3: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[8], [x23], #0x1\n"
+    "ld1 { v3.b }[8], [x23], #0x1\n"
     "b 33f\n"
     "29:"  // Oddments: Single input loop: Load: Bit 3: Unset
     "tbz %x[n_channels], #2, 31f\n"
-    "ldr s2, [x23], #0x4\n"
+    "ldr s3, [x23], #0x4\n"
     "tbz %x[n_channels], #1, 30f\n"
-    "ld1 { v2.h }[2], [x23], #0x2\n"
+    "ld1 { v3.h }[2], [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[6], [x23], #0x1\n"
+    "ld1 { v3.b }[6], [x23], #0x1\n"
     "b 33f\n"
     "30:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[4], [x23], #0x1\n"
+    "ld1 { v3.b }[4], [x23], #0x1\n"
     "b 33f\n"
     "31:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset
     "tbz %x[n_channels], #1, 32f\n"
-    "ldr h2, [x23], #0x2\n"
+    "ldr h3, [x23], #0x2\n"
     "tbz %x[n_channels], #0, 33f\n"
-    "ld1 { v2.b }[2], [x23], #0x1\n"
+    "ld1 { v3.b }[2], [x23], #0x1\n"
     "b 33f\n"
     "32:"  // Oddments: Single input loop: Load: Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
     "tbz %x[n_channels], #0, 33f\n"
-    "ldr b2, [x23], #0x1\n"
+    "ldr b3, [x23], #0x1\n"
     "33:"  // Oddments: Single input loop: Load: Bit 3: End
-    "umax v4.16b, v4.16b, v2.16b\n"
+    "umax v4.16b, v4.16b, v3.16b\n"
     "subs x20, x20, #0x1\n"
     "bgt 25b\n"
     "34:"  // Oddments: Single input loop: End
@@ -669,7 +669,7 @@ void a64_u8q_nhwc_max_generic_depthfirst_impl(
 
     : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr)
     : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_input_offset] "I" (offsetof(Requantize32, input_offset)), [offsetof_qp_output_offset] "I" (offsetof(Requantize32, output_offset)), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [quant_params] "r" (&qp)
-    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
+    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst.hpp
index 38c70b26ab1eeb3210726c18f86eaf3fa289820e..6dffdcf01c79dff50e311beb86c976f054c92212 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp
index f2df7235ffe57bac29d983505c143bba1094d6ff..2bb22131f74edc3b7d85068d9d4f0238e8d83443 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/cpp_nhwc_1x1_stride_any_depthfirst/generic.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
index 7464349f9e9f9624b94297b8f74ac1baa525fd05..3c1858633b319e6ca3bdcb306661e79c217598e5 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
@@ -86,22 +86,22 @@ void sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "mov x4, #0x0\n"
     "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x5, #0x0\n"
+    "ldr x6, [%x[args], %[offsetof_inptrs]]\n"
     "mov x19, #0x4\n"
-    "ldp x6, x7, [x20, #0x0]\n"
+    "add x7, %x[args], %[offsetof_rescale]\n"
+    "ldp x8, x17, [x20, #0x0]\n"
+    "ldp x16, x15, [x20, #0x10]\n"
     "whilelt p0.h, XZR, x19\n"
-    "add x8, %x[args], %[offsetof_rescale]\n"
-    "ldp x17, x16, [x20, #0x10]\n"
+    "ldp x14, x13, [x6, #0x0]\n"
     "whilelt p1.h, x4, x3\n"
-    "ldr x15, [%x[args], %[offsetof_inptrs]]\n"
-    "ldp x14, x13, [x15, #0x0]\n"
-    "ldp x12, x11, [x15, #0x10]\n"
-    "ldp x10, x9, [x15, #0x20]\n"
-    "ldp x28, x27, [x15, #0x30]\n"
-    "ldp x26, x25, [x15, #0x40]\n"
-    "ldp x24, x23, [x15, #0x50]\n"
-    "ldp x22, x21, [x15, #0x60]\n"
-    "ldp x20, x19, [x15, #0x70]\n"
-    "ld1rqh { z7.h }, p0/Z, [x8]\n"
+    "ldp x12, x11, [x6, #0x10]\n"
+    "ldp x10, x9, [x6, #0x20]\n"
+    "ldp x28, x27, [x6, #0x30]\n"
+    "ldp x26, x25, [x6, #0x40]\n"
+    "ldp x24, x23, [x6, #0x50]\n"
+    "ldp x22, x21, [x6, #0x60]\n"
+    "ldp x20, x19, [x6, #0x70]\n"
+    "ld1rqh { z7.h }, p0/Z, [x7]\n"
     "ld1h { z8.h }, p1/Z, [x9, x4, LSL #1]\n"
     "ld1h { z6.h }, p1/Z, [x28, x4, LSL #1]\n"
     "ld1h { z5.h }, p1/Z, [x25, x4, LSL #1]\n"
@@ -149,9 +149,9 @@ void sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ld1h { z28.h }, p1/Z, [x20, x4, LSL #1]\n"
     "fadd z16.h, z24.h, z22.h\n"
     "ld1h { z27.h }, p1/Z, [x14, x4, LSL #1]\n"
-    "fadd z19.h, z19.h, z21.h\n"
+    "fadd z19.h, z21.h, z19.h\n"
     "ld1h { z26.h }, p1/Z, [x11, x4, LSL #1]\n"
-    "fadd z18.h, z18.h, z21.h\n"
+    "fadd z18.h, z21.h, z18.h\n"
     "ld1h { z25.h }, p1/Z, [x22, x4, LSL #1]\n"
     "fadd z17.h, z17.h, z20.h\n"
     "ld1h { z24.h }, p1/Z, [x19, x4, LSL #1]\n"
@@ -159,13 +159,13 @@ void sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "fadd z16.h, z20.h, z16.h\n"
     "whilelt p1.h, x4, x3\n"
     "fmul z19.h, z19.h, z7.h[0]\n"
-    "st1h { z19.h }, p0, [x6, x5, LSL #1]\n"
+    "st1h { z19.h }, p0, [x8, x5, LSL #1]\n"
     "fmul z18.h, z18.h, z7.h[1]\n"
     "fmul z17.h, z17.h, z7.h[2]\n"
-    "st1h { z18.h }, p0, [x7, x5, LSL #1]\n"
+    "st1h { z18.h }, p0, [x17, x5, LSL #1]\n"
     "fmul z16.h, z16.h, z7.h[3]\n"
-    "st1h { z17.h }, p0, [x17, x5, LSL #1]\n"
-    "st1h { z16.h }, p0, [x16, x5, LSL #1]\n"
+    "st1h { z17.h }, p0, [x16, x5, LSL #1]\n"
+    "st1h { z16.h }, p0, [x15, x5, LSL #1]\n"
     "incw x5\n"
     "b.any 1b\n"
     "2:"  // Vector: Tail
@@ -183,18 +183,18 @@ void sve_fp16_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "fadd z18.h, z26.h, z22.h\n"
     "fadd z17.h, z25.h, z23.h\n"
     "fadd z16.h, z24.h, z22.h\n"
-    "fadd z19.h, z19.h, z21.h\n"
-    "fadd z18.h, z18.h, z21.h\n"
+    "fadd z19.h, z21.h, z19.h\n"
+    "fadd z18.h, z21.h, z18.h\n"
     "fadd z17.h, z17.h, z20.h\n"
     "fadd z16.h, z20.h, z16.h\n"
     "fmul z19.h, z19.h, z7.h[0]\n"
-    "st1h { z19.h }, p0, [x6, x5, LSL #1]\n"
+    "st1h { z19.h }, p0, [x8, x5, LSL #1]\n"
     "fmul z18.h, z18.h, z7.h[1]\n"
     "fmul z17.h, z17.h, z7.h[2]\n"
-    "st1h { z18.h }, p0, [x7, x5, LSL #1]\n"
+    "st1h { z18.h }, p0, [x17, x5, LSL #1]\n"
     "fmul z16.h, z16.h, z7.h[3]\n"
-    "st1h { z17.h }, p0, [x17, x5, LSL #1]\n"
-    "st1h { z16.h }, p0, [x16, x5, LSL #1]\n"
+    "st1h { z17.h }, p0, [x16, x5, LSL #1]\n"
+    "st1h { z16.h }, p0, [x15, x5, LSL #1]\n"
     :
     : [args] "r" (&args), [offsetof_inptrs] "I" (offsetof(KernelArgs, inptrs)), [offsetof_n_channels] "I" (offsetof(KernelArgs, n_channels)), [offsetof_outptrs] "I" (offsetof(KernelArgs, outptrs)), [offsetof_rescale] "I" (offsetof(KernelArgs, rescale_vals))
     : "cc", "memory", "p0", "p1", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst.hpp
index 33ee25cd3049a30871173f3beddd2418a7ef81fd..391d47cf41a27b2b57d44bfa5f6c543c1674ac15 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_fp16_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = sve_fp16_nhwc_avg_generic_depthfirst_impl;
 
   sve_fp16_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp
index 20293c0e0e24f99fc6760d3d31edeaef4f0ef864..84a6acf80d1afe24b9a22f443ab7cd2bdad0d33d 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_avg_generic_depthfirst/generic.cpp
@@ -42,7 +42,7 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl(
 
   __asm__ __volatile__(
     "ptrue p0.b\n"
-    "ld1rh { z7.h }, p0/Z, [%x[rescale_ptr]]\n"
+    "ld1rh { z8.h }, p0/Z, [%x[rescale_ptr]]\n"
     "mov x28, #0x0\n"
     "cnth x27\n"
     "cnth x26, ALL, MUL #2\n"
@@ -53,23 +53,23 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl(
     "whilelt p0.h, x25, %x[n_channels]\n"
     "b.none 7f\n"
     "1:"  // 4-vectors of channels
-    "mov z6.b, #0x0\n"
+    "mov z7.b, #0x0\n"
     "mov x19, %x[inptrs]\n"
-    "mov z5.b, #0x0\n"
+    "mov z6.b, #0x0\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "mov z5.b, #0x0\n"
     "mov z4.b, #0x0\n"
-    "mov z3.b, #0x0\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ld1h { z2.h }, p3/Z, [x23, x28, LSL #1]\n"
-    "ld1h { z1.h }, p3/Z, [x22, x28, LSL #1]\n"
-    "ld1h { z0.h }, p3/Z, [x21, x28, LSL #1]\n"
-    "ld1h { z31.h }, p3/Z, [x20, x28, LSL #1]\n"
-    "ld1h { z30.h }, p2/Z, [x23, x27, LSL #1]\n"
-    "ld1h { z22.h }, p2/Z, [x22, x27, LSL #1]\n"
+    "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
+    "ld1h { z2.h }, p3/Z, [x22, x28, LSL #1]\n"
+    "ld1h { z1.h }, p3/Z, [x21, x28, LSL #1]\n"
+    "ld1h { z0.h }, p3/Z, [x20, x28, LSL #1]\n"
+    "ld1h { z31.h }, p2/Z, [x23, x27, LSL #1]\n"
+    "ld1h { z30.h }, p2/Z, [x22, x27, LSL #1]\n"
     "ld1h { z29.h }, p2/Z, [x21, x27, LSL #1]\n"
     "ld1h { z28.h }, p2/Z, [x20, x27, LSL #1]\n"
     "ld1h { z27.h }, p1/Z, [x23, x26, LSL #1]\n"
@@ -82,47 +82,47 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl(
     "ld1h { z16.h }, p0/Z, [x20, x25, LSL #1]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "fadd z23.h, z2.h, z1.h\n"
+    "fadd z23.h, z3.h, z2.h\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd z19.h, z0.h, z31.h\n"
+    "fadd z19.h, z1.h, z0.h\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd z22.h, z30.h, z22.h\n"
     "add x19, x19, #0x20\n"
+    "fadd z22.h, z31.h, z30.h\n"
+    "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
     "fadd z18.h, z29.h, z28.h\n"
-    "ld1h { z2.h }, p3/Z, [x23, x28, LSL #1]\n"
     "fadd z21.h, z27.h, z21.h\n"
+    "ld1h { z2.h }, p3/Z, [x22, x28, LSL #1]\n"
     "fadd z17.h, z26.h, z17.h\n"
-    "ld1h { z1.h }, p3/Z, [x22, x28, LSL #1]\n"
+    "ld1h { z1.h }, p3/Z, [x21, x28, LSL #1]\n"
     "fadd z20.h, z25.h, z20.h\n"
-    "ld1h { z0.h }, p3/Z, [x21, x28, LSL #1]\n"
+    "ld1h { z0.h }, p3/Z, [x20, x28, LSL #1]\n"
     "fadd z16.h, z24.h, z16.h\n"
-    "ld1h { z31.h }, p3/Z, [x20, x28, LSL #1]\n"
+    "ld1h { z31.h }, p2/Z, [x23, x27, LSL #1]\n"
     "fadd z19.h, z23.h, z19.h\n"
-    "ld1h { z30.h }, p2/Z, [x23, x27, LSL #1]\n"
+    "ld1h { z30.h }, p2/Z, [x22, x27, LSL #1]\n"
     "fadd z18.h, z22.h, z18.h\n"
-    "ld1h { z22.h }, p2/Z, [x22, x27, LSL #1]\n"
-    "fadd z17.h, z21.h, z17.h\n"
     "ld1h { z29.h }, p2/Z, [x21, x27, LSL #1]\n"
-    "fadd z16.h, z20.h, z16.h\n"
+    "fadd z17.h, z21.h, z17.h\n"
     "ld1h { z28.h }, p2/Z, [x20, x27, LSL #1]\n"
-    "fadd z6.h, z6.h, z19.h\n"
+    "fadd z16.h, z20.h, z16.h\n"
     "ld1h { z27.h }, p1/Z, [x23, x26, LSL #1]\n"
-    "fadd z5.h, z5.h, z18.h\n"
+    "fadd z7.h, z7.h, z19.h\n"
     "ld1h { z21.h }, p1/Z, [x22, x26, LSL #1]\n"
-    "fadd z4.h, z4.h, z17.h\n"
+    "fadd z6.h, z6.h, z18.h\n"
     "ld1h { z26.h }, p1/Z, [x21, x26, LSL #1]\n"
-    "fadd z3.h, z3.h, z16.h\n"
+    "fadd z5.h, z5.h, z17.h\n"
     "ld1h { z17.h }, p1/Z, [x20, x26, LSL #1]\n"
+    "fadd z4.h, z4.h, z16.h\n"
     "ld1h { z25.h }, p0/Z, [x23, x25, LSL #1]\n"
     "ld1h { z20.h }, p0/Z, [x22, x25, LSL #1]\n"
     "ld1h { z24.h }, p0/Z, [x21, x25, LSL #1]\n"
     "ld1h { z16.h }, p0/Z, [x20, x25, LSL #1]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "fadd z23.h, z2.h, z1.h\n"
-    "fadd z19.h, z0.h, z31.h\n"
-    "fadd z22.h, z30.h, z22.h\n"
+    "fadd z23.h, z3.h, z2.h\n"
+    "fadd z19.h, z1.h, z0.h\n"
+    "fadd z22.h, z31.h, z30.h\n"
     "fadd z18.h, z29.h, z28.h\n"
     "fadd z21.h, z27.h, z21.h\n"
     "fadd z17.h, z26.h, z17.h\n"
@@ -132,37 +132,37 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl(
     "fadd z18.h, z22.h, z18.h\n"
     "fadd z17.h, z21.h, z17.h\n"
     "fadd z16.h, z20.h, z16.h\n"
-    "fadd z6.h, z6.h, z19.h\n"
-    "fadd z5.h, z5.h, z18.h\n"
-    "fadd z4.h, z4.h, z17.h\n"
-    "fadd z3.h, z3.h, z16.h\n"
+    "fadd z7.h, z7.h, z19.h\n"
+    "fadd z6.h, z6.h, z18.h\n"
+    "fadd z5.h, z5.h, z17.h\n"
+    "fadd z4.h, z4.h, z16.h\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ld1h { z2.h }, p3/Z, [x23, x28, LSL #1]\n"
-    "fadd z6.h, z6.h, z2.h\n"
-    "ld1h { z30.h }, p2/Z, [x23, x27, LSL #1]\n"
+    "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
+    "fadd z7.h, z7.h, z3.h\n"
+    "ld1h { z31.h }, p2/Z, [x23, x27, LSL #1]\n"
     "ld1h { z27.h }, p1/Z, [x23, x26, LSL #1]\n"
-    "fadd z5.h, z5.h, z30.h\n"
+    "fadd z6.h, z6.h, z31.h\n"
     "ld1h { z25.h }, p0/Z, [x23, x25, LSL #1]\n"
-    "fadd z4.h, z4.h, z27.h\n"
-    "fadd z3.h, z3.h, z25.h\n"
+    "fadd z5.h, z5.h, z27.h\n"
+    "fadd z4.h, z4.h, z25.h\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "fmul z6.h, z6.h, z7.h\n"
-    "st1h { z6.h }, p3, [%x[outptr], x28, LSL #1]\n"
-    "fmul z5.h, z5.h, z7.h\n"
+    "fmul z7.h, z7.h, z8.h\n"
+    "st1h { z7.h }, p3, [%x[outptr], x28, LSL #1]\n"
+    "fmul z6.h, z6.h, z8.h\n"
     "inch x28, ALL, MUL #4\n"
-    "fmul z4.h, z4.h, z7.h\n"
-    "st1h { z5.h }, p2, [%x[outptr], x27, LSL #1]\n"
-    "fmul z3.h, z3.h, z7.h\n"
+    "fmul z5.h, z5.h, z8.h\n"
+    "st1h { z6.h }, p2, [%x[outptr], x27, LSL #1]\n"
+    "fmul z4.h, z4.h, z8.h\n"
     "inch x27, ALL, MUL #4\n"
-    "st1h { z4.h }, p1, [%x[outptr], x26, LSL #1]\n"
+    "st1h { z5.h }, p1, [%x[outptr], x26, LSL #1]\n"
     "inch x26, ALL, MUL #4\n"
-    "st1h { z3.h }, p0, [%x[outptr], x25, LSL #1]\n"
+    "st1h { z4.h }, p0, [%x[outptr], x25, LSL #1]\n"
     "inch x25, ALL, MUL #4\n"
     "whilelt p0.h, x25, %x[n_channels]\n"
     "b.any 1b\n"
@@ -170,50 +170,50 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl(
     "whilelt p3.h, x28, %x[n_channels]\n"
     "b.none 14f\n"
     "8:"  // Single vector of channels: Loop
-    "mov z6.b, #0x0\n"
+    "mov z7.b, #0x0\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ld1h { z2.h }, p3/Z, [x23, x28, LSL #1]\n"
-    "ld1h { z1.h }, p3/Z, [x22, x28, LSL #1]\n"
-    "ld1h { z0.h }, p3/Z, [x21, x28, LSL #1]\n"
-    "ld1h { z31.h }, p3/Z, [x20, x28, LSL #1]\n"
+    "subs x24, x24, #0x1\n"
+    "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
+    "ld1h { z2.h }, p3/Z, [x22, x28, LSL #1]\n"
+    "ld1h { z1.h }, p3/Z, [x21, x28, LSL #1]\n"
+    "ld1h { z0.h }, p3/Z, [x20, x28, LSL #1]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "fadd z23.h, z2.h, z1.h\n"
+    "fadd z23.h, z3.h, z2.h\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd z19.h, z0.h, z31.h\n"
+    "fadd z19.h, z1.h, z0.h\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd z19.h, z23.h, z19.h\n"
-    "ld1h { z2.h }, p3/Z, [x23, x28, LSL #1]\n"
-    "fadd z6.h, z6.h, z19.h\n"
-    "ld1h { z1.h }, p3/Z, [x22, x28, LSL #1]\n"
     "add x19, x19, #0x20\n"
-    "ld1h { z0.h }, p3/Z, [x21, x28, LSL #1]\n"
-    "ld1h { z31.h }, p3/Z, [x20, x28, LSL #1]\n"
+    "fadd z19.h, z23.h, z19.h\n"
+    "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
+    "ld1h { z2.h }, p3/Z, [x22, x28, LSL #1]\n"
+    "fadd z7.h, z7.h, z19.h\n"
+    "ld1h { z1.h }, p3/Z, [x21, x28, LSL #1]\n"
+    "ld1h { z0.h }, p3/Z, [x20, x28, LSL #1]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "fadd z23.h, z2.h, z1.h\n"
-    "fadd z19.h, z0.h, z31.h\n"
+    "fadd z23.h, z3.h, z2.h\n"
+    "fadd z19.h, z1.h, z0.h\n"
     "fadd z19.h, z23.h, z19.h\n"
-    "fadd z6.h, z6.h, z19.h\n"
+    "fadd z7.h, z7.h, z19.h\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ld1h { z2.h }, p3/Z, [x23, x28, LSL #1]\n"
-    "fadd z6.h, z6.h, z2.h\n"
+    "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
+    "fadd z7.h, z7.h, z3.h\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "fmul z6.h, z6.h, z7.h\n"
-    "st1h { z6.h }, p3, [%x[outptr], x28, LSL #1]\n"
+    "fmul z7.h, z7.h, z8.h\n"
+    "st1h { z7.h }, p3, [%x[outptr], x28, LSL #1]\n"
     "inch x28\n"
     "whilelt p3.h, x28, %x[n_channels]\n"
     "b.any 8b\n"
@@ -221,7 +221,7 @@ void sve_fp16_nhwc_avg_generic_depthfirst_impl(
 
     :
     : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [n_valid_cells] "r" (n_valid_cells), [outptr] "r" (outptr), [rescale_ptr] "r" (&rescale_value)
-    : "cc", "memory", "p0", "p1", "p2", "p3", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+    : "cc", "memory", "p0", "p1", "p2", "p3", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index 0f377d90a069e62bf6e703ecbc270b9227730fbd..f6e23215b842b51c3733459f89f4632dcc243a29 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -65,13 +65,13 @@ void sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x14, [%x[args], %[offsetof_n_channels]]\n"
     "ptrue p2.b\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
+    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
     "mov x12, #0x0\n"
-    "ldp x11, x10, [x19, #0x0]\n"
+    "ldp x11, x10, [x20, #0x0]\n"
     "whilelt p1.h, x13, x14\n"
-    "ldp x9, x28, [x19, #0x10]\n"
-    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "ldp x9, x28, [x20, #0x10]\n"
     "ldp x27, x26, [x19, #0x0]\n"
     "ldp x25, x24, [x19, #0x10]\n"
     "ldp x23, x22, [x19, #0x20]\n"
@@ -105,9 +105,9 @@ void sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ld1h { z26.h }, p1/Z, [x22, x13, LSL #1]\n"
     "movprfx z19, z22\n fmax z19.h, p2/M, z19.h, z18.h\n"
     "ld1h { z25.h }, p1/Z, [x25, x13, LSL #1]\n"
-    "movprfx z18, z17\n fmax z18.h, p2/M, z18.h, z22.h\n"
+    "movprfx z18, z22\n fmax z18.h, p2/M, z18.h, z17.h\n"
     "ld1h { z24.h }, p1/Z, [x21, x13, LSL #1]\n"
-    "movprfx z17, z16\n fmax z17.h, p2/M, z17.h, z21.h\n"
+    "movprfx z17, z21\n fmax z17.h, p2/M, z17.h, z16.h\n"
     "ld1h { z23.h }, p1/Z, [x19, x13, LSL #1]\n"
     "incw x13\n"
     "movprfx z16, z21\n fmax z16.h, p2/M, z16.h, z20.h\n"
@@ -128,8 +128,8 @@ void sve_fp16_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "movprfx z20, z26\n fmax z20.h, p2/M, z20.h, z23.h\n"
     "movprfx z19, z22\n fmax z19.h, p2/M, z19.h, z18.h\n"
     "st1h { z19.h }, p0, [x11, x12, LSL #1]\n"
-    "movprfx z18, z17\n fmax z18.h, p2/M, z18.h, z22.h\n"
-    "movprfx z17, z16\n fmax z17.h, p2/M, z17.h, z21.h\n"
+    "movprfx z18, z22\n fmax z18.h, p2/M, z18.h, z17.h\n"
+    "movprfx z17, z21\n fmax z17.h, p2/M, z17.h, z16.h\n"
     "st1h { z18.h }, p0, [x10, x12, LSL #1]\n"
     "movprfx z16, z21\n fmax z16.h, p2/M, z16.h, z20.h\n"
     "st1h { z17.h }, p0, [x9, x12, LSL #1]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst.hpp
index 92cccd56e0802614591edea78e988e86ba077b56..1c17c2761982686f11537acefbfc21484440a45d 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_fp16_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = sve_fp16_nhwc_max_generic_depthfirst_impl;
 
   sve_fp16_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp
index bbd32137c588ef845382db43e4c0d3c31917ae04..58ab91560587e8fbbf64fad97495fb428a6e7688 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp16_nhwc_max_generic_depthfirst/generic.cpp
@@ -84,33 +84,33 @@ void sve_fp16_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n fmax z23.h, p4/M, z23.h, z0.h\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movprfx z18, z31\n fmax z18.h, p4/M, z18.h, z30.h\n"
     "add x19, x19, #0x20\n"
-    "fmax z22.h, p4/M, z22.h, z29.h\n"
+    "movprfx z18, z31\n fmax z18.h, p4/M, z18.h, z30.h\n"
     "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
+    "fmax z22.h, p4/M, z22.h, z29.h\n"
     "movprfx z17, z28\n fmax z17.h, p4/M, z17.h, z27.h\n"
-    "fmax z21.h, p4/M, z21.h, z26.h\n"
     "ld1h { z2.h }, p3/Z, [x22, x28, LSL #1]\n"
-    "fmax z16.h, p4/M, z16.h, z25.h\n"
+    "fmax z21.h, p4/M, z21.h, z26.h\n"
     "ld1h { z1.h }, p3/Z, [x21, x28, LSL #1]\n"
-    "fmax z20.h, p4/M, z20.h, z24.h\n"
+    "fmax z16.h, p4/M, z16.h, z25.h\n"
     "ld1h { z0.h }, p3/Z, [x20, x28, LSL #1]\n"
-    "fmax z19.h, p4/M, z19.h, z23.h\n"
+    "fmax z20.h, p4/M, z20.h, z24.h\n"
     "ld1h { z31.h }, p2/Z, [x23, x27, LSL #1]\n"
-    "fmax z18.h, p4/M, z18.h, z22.h\n"
+    "fmax z19.h, p4/M, z19.h, z23.h\n"
     "ld1h { z30.h }, p2/Z, [x22, x27, LSL #1]\n"
-    "fmax z17.h, p4/M, z17.h, z21.h\n"
+    "fmax z18.h, p4/M, z18.h, z22.h\n"
     "ld1h { z22.h }, p2/Z, [x21, x27, LSL #1]\n"
-    "fmax z16.h, p4/M, z16.h, z20.h\n"
+    "fmax z17.h, p4/M, z17.h, z21.h\n"
     "ld1h { z29.h }, p2/Z, [x20, x27, LSL #1]\n"
-    "fmax z7.h, p4/M, z7.h, z19.h\n"
+    "fmax z16.h, p4/M, z16.h, z20.h\n"
     "ld1h { z28.h }, p1/Z, [x23, x26, LSL #1]\n"
-    "fmax z6.h, p4/M, z6.h, z18.h\n"
+    "fmax z7.h, p4/M, z7.h, z19.h\n"
     "ld1h { z27.h }, p1/Z, [x22, x26, LSL #1]\n"
-    "fmax z5.h, p4/M, z5.h, z17.h\n"
+    "fmax z6.h, p4/M, z6.h, z18.h\n"
     "ld1h { z21.h }, p1/Z, [x21, x26, LSL #1]\n"
-    "fmax z4.h, p4/M, z4.h, z16.h\n"
+    "fmax z5.h, p4/M, z5.h, z17.h\n"
     "ld1h { z26.h }, p1/Z, [x20, x26, LSL #1]\n"
+    "fmax z4.h, p4/M, z4.h, z16.h\n"
     "ld1h { z16.h }, p0/Z, [x23, x25, LSL #1]\n"
     "ld1h { z25.h }, p0/Z, [x22, x25, LSL #1]\n"
     "ld1h { z20.h }, p0/Z, [x21, x25, LSL #1]\n"
@@ -168,9 +168,9 @@ void sve_fp16_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
+    "subs x24, x24, #0x1\n"
     "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
     "ld1h { z2.h }, p3/Z, [x22, x28, LSL #1]\n"
     "ld1h { z1.h }, p3/Z, [x21, x28, LSL #1]\n"
@@ -182,11 +182,11 @@ void sve_fp16_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n fmax z23.h, p4/M, z23.h, z0.h\n"
     "ldp x21, x20, [x19, #0x10]\n"
+    "add x19, x19, #0x20\n"
     "fmax z19.h, p4/M, z19.h, z23.h\n"
     "ld1h { z3.h }, p3/Z, [x23, x28, LSL #1]\n"
-    "fmax z7.h, p4/M, z7.h, z19.h\n"
     "ld1h { z2.h }, p3/Z, [x22, x28, LSL #1]\n"
-    "add x19, x19, #0x20\n"
+    "fmax z7.h, p4/M, z7.h, z19.h\n"
     "ld1h { z1.h }, p3/Z, [x21, x28, LSL #1]\n"
     "ld1h { z0.h }, p3/Z, [x20, x28, LSL #1]\n"
     "bgt 9b\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
index 03f17364016c4c956baf79777a0621ebde81377a..50f5da4c3d571b313cd65465f78014804f205b14 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst/generic.cpp
@@ -86,22 +86,22 @@ void sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "mov x4, #0x0\n"
     "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x5, #0x0\n"
+    "ldr x6, [%x[args], %[offsetof_inptrs]]\n"
     "mov x19, #0x4\n"
-    "ldp x6, x7, [x20, #0x0]\n"
+    "add x7, %x[args], %[offsetof_rescale]\n"
+    "ldp x8, x17, [x20, #0x0]\n"
+    "ldp x16, x15, [x20, #0x10]\n"
     "whilelt p0.s, XZR, x19\n"
-    "add x8, %x[args], %[offsetof_rescale]\n"
-    "ldp x17, x16, [x20, #0x10]\n"
+    "ldp x14, x13, [x6, #0x0]\n"
     "whilelt p1.s, x4, x3\n"
-    "ldr x15, [%x[args], %[offsetof_inptrs]]\n"
-    "ldp x14, x13, [x15, #0x0]\n"
-    "ldp x12, x11, [x15, #0x10]\n"
-    "ldp x10, x9, [x15, #0x20]\n"
-    "ldp x28, x27, [x15, #0x30]\n"
-    "ldp x26, x25, [x15, #0x40]\n"
-    "ldp x24, x23, [x15, #0x50]\n"
-    "ldp x22, x21, [x15, #0x60]\n"
-    "ldp x20, x19, [x15, #0x70]\n"
-    "ld1rqw { z7.s }, p0/Z, [x8]\n"
+    "ldp x12, x11, [x6, #0x10]\n"
+    "ldp x10, x9, [x6, #0x20]\n"
+    "ldp x28, x27, [x6, #0x30]\n"
+    "ldp x26, x25, [x6, #0x40]\n"
+    "ldp x24, x23, [x6, #0x50]\n"
+    "ldp x22, x21, [x6, #0x60]\n"
+    "ldp x20, x19, [x6, #0x70]\n"
+    "ld1rqw { z7.s }, p0/Z, [x7]\n"
     "ld1w { z8.s }, p1/Z, [x9, x4, LSL #2]\n"
     "ld1w { z6.s }, p1/Z, [x28, x4, LSL #2]\n"
     "ld1w { z5.s }, p1/Z, [x25, x4, LSL #2]\n"
@@ -149,9 +149,9 @@ void sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "ld1w { z28.s }, p1/Z, [x20, x4, LSL #2]\n"
     "fadd z16.s, z24.s, z22.s\n"
     "ld1w { z27.s }, p1/Z, [x14, x4, LSL #2]\n"
-    "fadd z19.s, z19.s, z21.s\n"
+    "fadd z19.s, z21.s, z19.s\n"
     "ld1w { z26.s }, p1/Z, [x11, x4, LSL #2]\n"
-    "fadd z18.s, z18.s, z21.s\n"
+    "fadd z18.s, z21.s, z18.s\n"
     "ld1w { z25.s }, p1/Z, [x22, x4, LSL #2]\n"
     "fadd z17.s, z17.s, z20.s\n"
     "ld1w { z24.s }, p1/Z, [x19, x4, LSL #2]\n"
@@ -159,13 +159,13 @@ void sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "fadd z16.s, z20.s, z16.s\n"
     "whilelt p1.s, x4, x3\n"
     "fmul z19.s, z19.s, z7.s[0]\n"
-    "st1w { z19.s }, p0, [x6, x5, LSL #2]\n"
+    "st1w { z19.s }, p0, [x8, x5, LSL #2]\n"
     "fmul z18.s, z18.s, z7.s[1]\n"
     "fmul z17.s, z17.s, z7.s[2]\n"
-    "st1w { z18.s }, p0, [x7, x5, LSL #2]\n"
+    "st1w { z18.s }, p0, [x17, x5, LSL #2]\n"
     "fmul z16.s, z16.s, z7.s[3]\n"
-    "st1w { z17.s }, p0, [x17, x5, LSL #2]\n"
-    "st1w { z16.s }, p0, [x16, x5, LSL #2]\n"
+    "st1w { z17.s }, p0, [x16, x5, LSL #2]\n"
+    "st1w { z16.s }, p0, [x15, x5, LSL #2]\n"
     "incw x5\n"
     "b.any 1b\n"
     "2:"  // Vector: Tail
@@ -183,18 +183,18 @@ void sve_fp32_nhwc_avg_3x3_s1_output2x2_depthfirst_impl(
     "fadd z18.s, z26.s, z22.s\n"
     "fadd z17.s, z25.s, z23.s\n"
     "fadd z16.s, z24.s, z22.s\n"
-    "fadd z19.s, z19.s, z21.s\n"
-    "fadd z18.s, z18.s, z21.s\n"
+    "fadd z19.s, z21.s, z19.s\n"
+    "fadd z18.s, z21.s, z18.s\n"
     "fadd z17.s, z17.s, z20.s\n"
     "fadd z16.s, z20.s, z16.s\n"
     "fmul z19.s, z19.s, z7.s[0]\n"
-    "st1w { z19.s }, p0, [x6, x5, LSL #2]\n"
+    "st1w { z19.s }, p0, [x8, x5, LSL #2]\n"
     "fmul z18.s, z18.s, z7.s[1]\n"
     "fmul z17.s, z17.s, z7.s[2]\n"
-    "st1w { z18.s }, p0, [x7, x5, LSL #2]\n"
+    "st1w { z18.s }, p0, [x17, x5, LSL #2]\n"
     "fmul z16.s, z16.s, z7.s[3]\n"
-    "st1w { z17.s }, p0, [x17, x5, LSL #2]\n"
-    "st1w { z16.s }, p0, [x16, x5, LSL #2]\n"
+    "st1w { z17.s }, p0, [x16, x5, LSL #2]\n"
+    "st1w { z16.s }, p0, [x15, x5, LSL #2]\n"
     :
     : [args] "r" (&args), [offsetof_inptrs] "I" (offsetof(KernelArgs, inptrs)), [offsetof_n_channels] "I" (offsetof(KernelArgs, n_channels)), [offsetof_outptrs] "I" (offsetof(KernelArgs, outptrs)), [offsetof_rescale] "I" (offsetof(KernelArgs, rescale_vals))
     : "cc", "memory", "p0", "p1", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst.hpp
index de315d21dc6ab1616da93c3ef2930316c6f31560..0daa046a02ac211929bf3b0b4070399c18cc3440 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_fp32_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = sve_fp32_nhwc_avg_generic_depthfirst_impl;
 
   sve_fp32_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp
index 218c1f9df799183fee2d1df1ed62da533256598d..c2f5745adc5985d0ca9bcddac721bf5fb5b749fd 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_avg_generic_depthfirst/generic.cpp
@@ -42,7 +42,7 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl(
 
   __asm__ __volatile__(
     "ptrue p0.b\n"
-    "ld1rw { z7.s }, p0/Z, [%x[rescale_ptr]]\n"
+    "ld1rw { z8.s }, p0/Z, [%x[rescale_ptr]]\n"
     "mov x28, #0x0\n"
     "cntw x27\n"
     "cntw x26, ALL, MUL #2\n"
@@ -53,23 +53,23 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl(
     "whilelt p0.s, x25, %x[n_channels]\n"
     "b.none 7f\n"
     "1:"  // 4-vectors of channels
-    "mov z6.b, #0x0\n"
+    "mov z7.b, #0x0\n"
     "mov x19, %x[inptrs]\n"
-    "mov z5.b, #0x0\n"
+    "mov z6.b, #0x0\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
+    "mov z5.b, #0x0\n"
     "mov z4.b, #0x0\n"
-    "mov z3.b, #0x0\n"
     "cbz x24, 4f\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
     "subs x24, x24, #0x1\n"
-    "ld1w { z2.s }, p3/Z, [x23, x28, LSL #2]\n"
-    "ld1w { z1.s }, p3/Z, [x22, x28, LSL #2]\n"
-    "ld1w { z0.s }, p3/Z, [x21, x28, LSL #2]\n"
-    "ld1w { z31.s }, p3/Z, [x20, x28, LSL #2]\n"
-    "ld1w { z30.s }, p2/Z, [x23, x27, LSL #2]\n"
-    "ld1w { z22.s }, p2/Z, [x22, x27, LSL #2]\n"
+    "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
+    "ld1w { z2.s }, p3/Z, [x22, x28, LSL #2]\n"
+    "ld1w { z1.s }, p3/Z, [x21, x28, LSL #2]\n"
+    "ld1w { z0.s }, p3/Z, [x20, x28, LSL #2]\n"
+    "ld1w { z31.s }, p2/Z, [x23, x27, LSL #2]\n"
+    "ld1w { z30.s }, p2/Z, [x22, x27, LSL #2]\n"
     "ld1w { z29.s }, p2/Z, [x21, x27, LSL #2]\n"
     "ld1w { z28.s }, p2/Z, [x20, x27, LSL #2]\n"
     "ld1w { z27.s }, p1/Z, [x23, x26, LSL #2]\n"
@@ -82,47 +82,47 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl(
     "ld1w { z16.s }, p0/Z, [x20, x25, LSL #2]\n"
     "beq 3f\n"
     "2:"  // 4-vectors of channels: 4 inputs loop
-    "fadd z23.s, z2.s, z1.s\n"
+    "fadd z23.s, z3.s, z2.s\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd z19.s, z0.s, z31.s\n"
+    "fadd z19.s, z1.s, z0.s\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd z22.s, z30.s, z22.s\n"
     "add x19, x19, #0x20\n"
+    "fadd z22.s, z31.s, z30.s\n"
+    "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
     "fadd z18.s, z29.s, z28.s\n"
-    "ld1w { z2.s }, p3/Z, [x23, x28, LSL #2]\n"
     "fadd z21.s, z27.s, z21.s\n"
+    "ld1w { z2.s }, p3/Z, [x22, x28, LSL #2]\n"
     "fadd z17.s, z26.s, z17.s\n"
-    "ld1w { z1.s }, p3/Z, [x22, x28, LSL #2]\n"
+    "ld1w { z1.s }, p3/Z, [x21, x28, LSL #2]\n"
     "fadd z20.s, z25.s, z20.s\n"
-    "ld1w { z0.s }, p3/Z, [x21, x28, LSL #2]\n"
+    "ld1w { z0.s }, p3/Z, [x20, x28, LSL #2]\n"
     "fadd z16.s, z24.s, z16.s\n"
-    "ld1w { z31.s }, p3/Z, [x20, x28, LSL #2]\n"
+    "ld1w { z31.s }, p2/Z, [x23, x27, LSL #2]\n"
     "fadd z19.s, z23.s, z19.s\n"
-    "ld1w { z30.s }, p2/Z, [x23, x27, LSL #2]\n"
+    "ld1w { z30.s }, p2/Z, [x22, x27, LSL #2]\n"
     "fadd z18.s, z22.s, z18.s\n"
-    "ld1w { z22.s }, p2/Z, [x22, x27, LSL #2]\n"
-    "fadd z17.s, z21.s, z17.s\n"
     "ld1w { z29.s }, p2/Z, [x21, x27, LSL #2]\n"
-    "fadd z16.s, z20.s, z16.s\n"
+    "fadd z17.s, z21.s, z17.s\n"
     "ld1w { z28.s }, p2/Z, [x20, x27, LSL #2]\n"
-    "fadd z6.s, z6.s, z19.s\n"
+    "fadd z16.s, z20.s, z16.s\n"
     "ld1w { z27.s }, p1/Z, [x23, x26, LSL #2]\n"
-    "fadd z5.s, z5.s, z18.s\n"
+    "fadd z7.s, z7.s, z19.s\n"
     "ld1w { z21.s }, p1/Z, [x22, x26, LSL #2]\n"
-    "fadd z4.s, z4.s, z17.s\n"
+    "fadd z6.s, z6.s, z18.s\n"
     "ld1w { z26.s }, p1/Z, [x21, x26, LSL #2]\n"
-    "fadd z3.s, z3.s, z16.s\n"
+    "fadd z5.s, z5.s, z17.s\n"
     "ld1w { z17.s }, p1/Z, [x20, x26, LSL #2]\n"
+    "fadd z4.s, z4.s, z16.s\n"
     "ld1w { z25.s }, p0/Z, [x23, x25, LSL #2]\n"
     "ld1w { z20.s }, p0/Z, [x22, x25, LSL #2]\n"
     "ld1w { z24.s }, p0/Z, [x21, x25, LSL #2]\n"
     "ld1w { z16.s }, p0/Z, [x20, x25, LSL #2]\n"
     "bgt 2b\n"
     "3:"  // 4-vectors of channels: 4 inputs tail
-    "fadd z23.s, z2.s, z1.s\n"
-    "fadd z19.s, z0.s, z31.s\n"
-    "fadd z22.s, z30.s, z22.s\n"
+    "fadd z23.s, z3.s, z2.s\n"
+    "fadd z19.s, z1.s, z0.s\n"
+    "fadd z22.s, z31.s, z30.s\n"
     "fadd z18.s, z29.s, z28.s\n"
     "fadd z21.s, z27.s, z21.s\n"
     "fadd z17.s, z26.s, z17.s\n"
@@ -132,37 +132,37 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl(
     "fadd z18.s, z22.s, z18.s\n"
     "fadd z17.s, z21.s, z17.s\n"
     "fadd z16.s, z20.s, z16.s\n"
-    "fadd z6.s, z6.s, z19.s\n"
-    "fadd z5.s, z5.s, z18.s\n"
-    "fadd z4.s, z4.s, z17.s\n"
-    "fadd z3.s, z3.s, z16.s\n"
+    "fadd z7.s, z7.s, z19.s\n"
+    "fadd z6.s, z6.s, z18.s\n"
+    "fadd z5.s, z5.s, z17.s\n"
+    "fadd z4.s, z4.s, z16.s\n"
     "4:"  // 4-vectors of channels: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 6f\n"
     "5:"  // 4-vectors of channels: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ld1w { z2.s }, p3/Z, [x23, x28, LSL #2]\n"
-    "fadd z6.s, z6.s, z2.s\n"
-    "ld1w { z30.s }, p2/Z, [x23, x27, LSL #2]\n"
+    "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
+    "fadd z7.s, z7.s, z3.s\n"
+    "ld1w { z31.s }, p2/Z, [x23, x27, LSL #2]\n"
     "ld1w { z27.s }, p1/Z, [x23, x26, LSL #2]\n"
-    "fadd z5.s, z5.s, z30.s\n"
+    "fadd z6.s, z6.s, z31.s\n"
     "ld1w { z25.s }, p0/Z, [x23, x25, LSL #2]\n"
-    "fadd z4.s, z4.s, z27.s\n"
-    "fadd z3.s, z3.s, z25.s\n"
+    "fadd z5.s, z5.s, z27.s\n"
+    "fadd z4.s, z4.s, z25.s\n"
     "bgt 5b\n"
     "6:"  // 4-vectors of channels: Single input loop: End
-    "fmul z6.s, z6.s, z7.s\n"
-    "st1w { z6.s }, p3, [%x[outptr], x28, LSL #2]\n"
-    "fmul z5.s, z5.s, z7.s\n"
+    "fmul z7.s, z7.s, z8.s\n"
+    "st1w { z7.s }, p3, [%x[outptr], x28, LSL #2]\n"
+    "fmul z6.s, z6.s, z8.s\n"
     "incw x28, ALL, MUL #4\n"
-    "fmul z4.s, z4.s, z7.s\n"
-    "st1w { z5.s }, p2, [%x[outptr], x27, LSL #2]\n"
-    "fmul z3.s, z3.s, z7.s\n"
+    "fmul z5.s, z5.s, z8.s\n"
+    "st1w { z6.s }, p2, [%x[outptr], x27, LSL #2]\n"
+    "fmul z4.s, z4.s, z8.s\n"
     "incw x27, ALL, MUL #4\n"
-    "st1w { z4.s }, p1, [%x[outptr], x26, LSL #2]\n"
+    "st1w { z5.s }, p1, [%x[outptr], x26, LSL #2]\n"
     "incw x26, ALL, MUL #4\n"
-    "st1w { z3.s }, p0, [%x[outptr], x25, LSL #2]\n"
+    "st1w { z4.s }, p0, [%x[outptr], x25, LSL #2]\n"
     "incw x25, ALL, MUL #4\n"
     "whilelt p0.s, x25, %x[n_channels]\n"
     "b.any 1b\n"
@@ -170,50 +170,50 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl(
     "whilelt p3.s, x28, %x[n_channels]\n"
     "b.none 14f\n"
     "8:"  // Single vector of channels: Loop
-    "mov z6.b, #0x0\n"
+    "mov z7.b, #0x0\n"
     "mov x19, %x[inptrs]\n"
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
-    "ld1w { z2.s }, p3/Z, [x23, x28, LSL #2]\n"
-    "ld1w { z1.s }, p3/Z, [x22, x28, LSL #2]\n"
-    "ld1w { z0.s }, p3/Z, [x21, x28, LSL #2]\n"
-    "ld1w { z31.s }, p3/Z, [x20, x28, LSL #2]\n"
+    "subs x24, x24, #0x1\n"
+    "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
+    "ld1w { z2.s }, p3/Z, [x22, x28, LSL #2]\n"
+    "ld1w { z1.s }, p3/Z, [x21, x28, LSL #2]\n"
+    "ld1w { z0.s }, p3/Z, [x20, x28, LSL #2]\n"
     "beq 10f\n"
     "9:"  // Single vector of channels: Loop: 4 inputs loop
-    "fadd z23.s, z2.s, z1.s\n"
+    "fadd z23.s, z3.s, z2.s\n"
     "ldp x23, x22, [x19, #0x0]\n"
     "subs x24, x24, #0x1\n"
-    "fadd z19.s, z0.s, z31.s\n"
+    "fadd z19.s, z1.s, z0.s\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "fadd z19.s, z23.s, z19.s\n"
-    "ld1w { z2.s }, p3/Z, [x23, x28, LSL #2]\n"
-    "fadd z6.s, z6.s, z19.s\n"
-    "ld1w { z1.s }, p3/Z, [x22, x28, LSL #2]\n"
     "add x19, x19, #0x20\n"
-    "ld1w { z0.s }, p3/Z, [x21, x28, LSL #2]\n"
-    "ld1w { z31.s }, p3/Z, [x20, x28, LSL #2]\n"
+    "fadd z19.s, z23.s, z19.s\n"
+    "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
+    "ld1w { z2.s }, p3/Z, [x22, x28, LSL #2]\n"
+    "fadd z7.s, z7.s, z19.s\n"
+    "ld1w { z1.s }, p3/Z, [x21, x28, LSL #2]\n"
+    "ld1w { z0.s }, p3/Z, [x20, x28, LSL #2]\n"
     "bgt 9b\n"
     "10:"  // Single vector of channels: Loop: 4 inputs tail
-    "fadd z23.s, z2.s, z1.s\n"
-    "fadd z19.s, z0.s, z31.s\n"
+    "fadd z23.s, z3.s, z2.s\n"
+    "fadd z19.s, z1.s, z0.s\n"
     "fadd z19.s, z23.s, z19.s\n"
-    "fadd z6.s, z6.s, z19.s\n"
+    "fadd z7.s, z7.s, z19.s\n"
     "11:"  // Single vector of channels: Loop: After loop
     "ands x20, %x[n_valid_cells], #0x3\n"
     "beq 13f\n"
     "12:"  // Single vector of channels: Loop: Single input loop
     "ldr x23, [x19], #0x8\n"
     "subs x20, x20, #0x1\n"
-    "ld1w { z2.s }, p3/Z, [x23, x28, LSL #2]\n"
-    "fadd z6.s, z6.s, z2.s\n"
+    "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
+    "fadd z7.s, z7.s, z3.s\n"
     "bgt 12b\n"
     "13:"  // Single vector of channels: Loop: Single input loop: End
-    "fmul z6.s, z6.s, z7.s\n"
-    "st1w { z6.s }, p3, [%x[outptr], x28, LSL #2]\n"
+    "fmul z7.s, z7.s, z8.s\n"
+    "st1w { z7.s }, p3, [%x[outptr], x28, LSL #2]\n"
     "incw x28\n"
     "whilelt p3.s, x28, %x[n_channels]\n"
     "b.any 8b\n"
@@ -221,7 +221,7 @@ void sve_fp32_nhwc_avg_generic_depthfirst_impl(
 
     :
     : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [n_valid_cells] "r" (n_valid_cells), [outptr] "r" (outptr), [rescale_ptr] "r" (&rescale_value)
-    : "cc", "memory", "p0", "p1", "p2", "p3", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
+    : "cc", "memory", "p0", "p1", "p2", "p3", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
   );
 }
 

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index 279c690df6965aad871dcee09362a543a5458a47..250cc2422601231299cc7ff47737c5dd61932f89 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -65,13 +65,13 @@ void sve_fp32_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x14, [%x[args], %[offsetof_n_channels]]\n"
     "ptrue p2.b\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
+    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
     "mov x12, #0x0\n"
-    "ldp x11, x10, [x19, #0x0]\n"
+    "ldp x11, x10, [x20, #0x0]\n"
     "whilelt p1.s, x13, x14\n"
-    "ldp x9, x28, [x19, #0x10]\n"
-    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "ldp x9, x28, [x20, #0x10]\n"
     "ldp x27, x26, [x19, #0x0]\n"
     "ldp x25, x24, [x19, #0x10]\n"
     "ldp x23, x22, [x19, #0x20]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst.hpp
index ba5138dd1a1021d28e134cb3a99fb4041d3b9d07..17e3e5f0ba3b7ef0ea724bc38d36c34b99550e9c 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_fp32_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = sve_fp32_nhwc_max_generic_depthfirst_impl;
 
   sve_fp32_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp
index 775595f899ff0ec8182b2bc56d9a2bcc5c320aa2..8166379ce402c42f3b88dfcb3d2744fb9ee93632 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_fp32_nhwc_max_generic_depthfirst/generic.cpp
@@ -84,33 +84,33 @@ void sve_fp32_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n fmax z23.s, p4/M, z23.s, z0.s\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movprfx z18, z31\n fmax z18.s, p4/M, z18.s, z30.s\n"
     "add x19, x19, #0x20\n"
-    "fmax z22.s, p4/M, z22.s, z29.s\n"
+    "movprfx z18, z31\n fmax z18.s, p4/M, z18.s, z30.s\n"
     "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
+    "fmax z22.s, p4/M, z22.s, z29.s\n"
     "movprfx z17, z28\n fmax z17.s, p4/M, z17.s, z27.s\n"
-    "fmax z21.s, p4/M, z21.s, z26.s\n"
     "ld1w { z2.s }, p3/Z, [x22, x28, LSL #2]\n"
-    "fmax z16.s, p4/M, z16.s, z25.s\n"
+    "fmax z21.s, p4/M, z21.s, z26.s\n"
     "ld1w { z1.s }, p3/Z, [x21, x28, LSL #2]\n"
-    "fmax z20.s, p4/M, z20.s, z24.s\n"
+    "fmax z16.s, p4/M, z16.s, z25.s\n"
     "ld1w { z0.s }, p3/Z, [x20, x28, LSL #2]\n"
-    "fmax z19.s, p4/M, z19.s, z23.s\n"
+    "fmax z20.s, p4/M, z20.s, z24.s\n"
     "ld1w { z31.s }, p2/Z, [x23, x27, LSL #2]\n"
-    "fmax z18.s, p4/M, z18.s, z22.s\n"
+    "fmax z19.s, p4/M, z19.s, z23.s\n"
     "ld1w { z30.s }, p2/Z, [x22, x27, LSL #2]\n"
-    "fmax z17.s, p4/M, z17.s, z21.s\n"
+    "fmax z18.s, p4/M, z18.s, z22.s\n"
     "ld1w { z22.s }, p2/Z, [x21, x27, LSL #2]\n"
-    "fmax z16.s, p4/M, z16.s, z20.s\n"
+    "fmax z17.s, p4/M, z17.s, z21.s\n"
     "ld1w { z29.s }, p2/Z, [x20, x27, LSL #2]\n"
-    "fmax z7.s, p4/M, z7.s, z19.s\n"
+    "fmax z16.s, p4/M, z16.s, z20.s\n"
     "ld1w { z28.s }, p1/Z, [x23, x26, LSL #2]\n"
-    "fmax z6.s, p4/M, z6.s, z18.s\n"
+    "fmax z7.s, p4/M, z7.s, z19.s\n"
     "ld1w { z27.s }, p1/Z, [x22, x26, LSL #2]\n"
-    "fmax z5.s, p4/M, z5.s, z17.s\n"
+    "fmax z6.s, p4/M, z6.s, z18.s\n"
     "ld1w { z21.s }, p1/Z, [x21, x26, LSL #2]\n"
-    "fmax z4.s, p4/M, z4.s, z16.s\n"
+    "fmax z5.s, p4/M, z5.s, z17.s\n"
     "ld1w { z26.s }, p1/Z, [x20, x26, LSL #2]\n"
+    "fmax z4.s, p4/M, z4.s, z16.s\n"
     "ld1w { z16.s }, p0/Z, [x23, x25, LSL #2]\n"
     "ld1w { z25.s }, p0/Z, [x22, x25, LSL #2]\n"
     "ld1w { z20.s }, p0/Z, [x21, x25, LSL #2]\n"
@@ -168,9 +168,9 @@ void sve_fp32_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
+    "subs x24, x24, #0x1\n"
     "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
     "ld1w { z2.s }, p3/Z, [x22, x28, LSL #2]\n"
     "ld1w { z1.s }, p3/Z, [x21, x28, LSL #2]\n"
@@ -182,11 +182,11 @@ void sve_fp32_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n fmax z23.s, p4/M, z23.s, z0.s\n"
     "ldp x21, x20, [x19, #0x10]\n"
+    "add x19, x19, #0x20\n"
     "fmax z19.s, p4/M, z19.s, z23.s\n"
     "ld1w { z3.s }, p3/Z, [x23, x28, LSL #2]\n"
-    "fmax z7.s, p4/M, z7.s, z19.s\n"
     "ld1w { z2.s }, p3/Z, [x22, x28, LSL #2]\n"
-    "add x19, x19, #0x20\n"
+    "fmax z7.s, p4/M, z7.s, z19.s\n"
     "ld1w { z1.s }, p3/Z, [x21, x28, LSL #2]\n"
     "ld1w { z0.s }, p3/Z, [x20, x28, LSL #2]\n"
     "bgt 9b\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst.hpp
index 575977d9a9e111e7dba0221cd015ab19b54f66f5..2ae38b5b2f0bf3cb185a3f11fadf1450f21ded98 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_s8_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = sve_s8_nhwc_avg_generic_depthfirst_impl;
 
   sve_s8_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp
index 99321eba615e6f870b26383f4bf1fbf8591c31f9..2ea5b90561238b0155ea9eaa32afe1ea38338f0b 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_avg_generic_depthfirst/generic.cpp
@@ -40,10 +40,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -237,22 +237,22 @@ void sve_s8_nhwc_avg_generic_depthfirst_impl(
     "ld1rw { z17.s }, p4/Z, [%x[rescale_ptr]]\n"
     "ld1rw { z16.s }, p4/Z, [%x[shift_ptr]]\n"
     "not z19.s, p4/M, z20.s\n"
-    ".inst 0x04b175ef  // sqrdmulh z15.s, z15.s, z17.s\n"
-    ".inst 0x04b175ce  // sqrdmulh z14.s, z14.s, z17.s\n"
-    ".inst 0x04b175ad  // sqrdmulh z13.s, z13.s, z17.s\n"
-    ".inst 0x04b1758c  // sqrdmulh z12.s, z12.s, z17.s\n"
-    ".inst 0x04b1756b  // sqrdmulh z11.s, z11.s, z17.s\n"
-    ".inst 0x04b1754a  // sqrdmulh z10.s, z10.s, z17.s\n"
-    ".inst 0x04b17529  // sqrdmulh z9.s, z9.s, z17.s\n"
-    ".inst 0x04b17508  // sqrdmulh z8.s, z8.s, z17.s\n"
-    ".inst 0x04b174e7  // sqrdmulh z7.s, z7.s, z17.s\n"
-    ".inst 0x04b174c6  // sqrdmulh z6.s, z6.s, z17.s\n"
-    ".inst 0x04b174a5  // sqrdmulh z5.s, z5.s, z17.s\n"
-    ".inst 0x04b17484  // sqrdmulh z4.s, z4.s, z17.s\n"
-    ".inst 0x04b17463  // sqrdmulh z3.s, z3.s, z17.s\n"
-    ".inst 0x04b17442  // sqrdmulh z2.s, z2.s, z17.s\n"
-    ".inst 0x04b17421  // sqrdmulh z1.s, z1.s, z17.s\n"
-    ".inst 0x04b17400  // sqrdmulh z0.s, z0.s, z17.s\n"
+    ".inst 0x04b175ef  // sqdmulh z15.s, z15.s, z17.s\n"
+    ".inst 0x04b175ce  // sqdmulh z14.s, z14.s, z17.s\n"
+    ".inst 0x04b175ad  // sqdmulh z13.s, z13.s, z17.s\n"
+    ".inst 0x04b1758c  // sqdmulh z12.s, z12.s, z17.s\n"
+    ".inst 0x04b1756b  // sqdmulh z11.s, z11.s, z17.s\n"
+    ".inst 0x04b1754a  // sqdmulh z10.s, z10.s, z17.s\n"
+    ".inst 0x04b17529  // sqdmulh z9.s, z9.s, z17.s\n"
+    ".inst 0x04b17508  // sqdmulh z8.s, z8.s, z17.s\n"
+    ".inst 0x04b174e7  // sqdmulh z7.s, z7.s, z17.s\n"
+    ".inst 0x04b174c6  // sqdmulh z6.s, z6.s, z17.s\n"
+    ".inst 0x04b174a5  // sqdmulh z5.s, z5.s, z17.s\n"
+    ".inst 0x04b17484  // sqdmulh z4.s, z4.s, z17.s\n"
+    ".inst 0x04b17463  // sqdmulh z3.s, z3.s, z17.s\n"
+    ".inst 0x04b17442  // sqdmulh z2.s, z2.s, z17.s\n"
+    ".inst 0x04b17421  // sqdmulh z1.s, z1.s, z17.s\n"
+    ".inst 0x04b17400  // sqdmulh z0.s, z0.s, z17.s\n"
     ".inst 0x4482920f  // srshl z15.s, p4/M, z15.s, z16.s\n"
     ".inst 0x4482920e  // srshl z14.s, p4/M, z14.s, z16.s\n"
     ".inst 0x4482920d  // srshl z13.s, p4/M, z13.s, z16.s\n"
@@ -379,10 +379,10 @@ void sve_s8_nhwc_avg_generic_depthfirst_impl(
     "ld1rw { z17.s }, p4/Z, [%x[rescale_ptr]]\n"
     "ld1rw { z16.s }, p4/Z, [%x[shift_ptr]]\n"
     "not z19.s, p4/M, z20.s\n"
-    ".inst 0x04b175ef  // sqrdmulh z15.s, z15.s, z17.s\n"
-    ".inst 0x04b175ce  // sqrdmulh z14.s, z14.s, z17.s\n"
-    ".inst 0x04b175ad  // sqrdmulh z13.s, z13.s, z17.s\n"
-    ".inst 0x04b1758c  // sqrdmulh z12.s, z12.s, z17.s\n"
+    ".inst 0x04b175ef  // sqdmulh z15.s, z15.s, z17.s\n"
+    ".inst 0x04b175ce  // sqdmulh z14.s, z14.s, z17.s\n"
+    ".inst 0x04b175ad  // sqdmulh z13.s, z13.s, z17.s\n"
+    ".inst 0x04b1758c  // sqdmulh z12.s, z12.s, z17.s\n"
     ".inst 0x4482920f  // srshl z15.s, p4/M, z15.s, z16.s\n"
     ".inst 0x4482920e  // srshl z14.s, p4/M, z14.s, z16.s\n"
     ".inst 0x4482920d  // srshl z13.s, p4/M, z13.s, z16.s\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index 06c777b8edddccc194f6776b27662d0c042dc030..bdf3f532921dfb035be6cd1a4723e157e46b37eb 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -65,13 +65,13 @@ void sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x14, [%x[args], %[offsetof_n_channels]]\n"
     "ptrue p2.b\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
+    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
     "mov x12, #0x0\n"
-    "ldp x11, x10, [x19, #0x0]\n"
+    "ldp x11, x10, [x20, #0x0]\n"
     "whilelt p1.b, x13, x14\n"
-    "ldp x9, x28, [x19, #0x10]\n"
-    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "ldp x9, x28, [x20, #0x10]\n"
     "ldp x27, x26, [x19, #0x0]\n"
     "ldp x25, x24, [x19, #0x10]\n"
     "ldp x23, x22, [x19, #0x20]\n"
@@ -97,20 +97,20 @@ void sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "ld1b { z30.b }, p1/Z, [x23, x13]\n"
     "movprfx z18, z28\n smax z18.b, p2/M, z18.b, z27.b\n"
     "ld1b { z29.b }, p1/Z, [x20, x13]\n"
-    "movprfx z20, z26\n smax z20.b, p2/M, z20.b, z25.b\n"
+    "movprfx z17, z26\n smax z17.b, p2/M, z17.b, z25.b\n"
     "ld1b { z27.b }, p1/Z, [x27, x13]\n"
-    "movprfx z17, z24\n smax z17.b, p2/M, z17.b, z28.b\n"
+    "movprfx z16, z24\n smax z16.b, p2/M, z16.b, z28.b\n"
     "ld1b { z28.b }, p1/Z, [x24, x13]\n"
-    "movprfx z16, z26\n smax z16.b, p2/M, z16.b, z23.b\n"
+    "movprfx z20, z26\n smax z20.b, p2/M, z20.b, z23.b\n"
     "ld1b { z26.b }, p1/Z, [x22, x13]\n"
     "movprfx z19, z22\n smax z19.b, p2/M, z19.b, z18.b\n"
     "ld1b { z25.b }, p1/Z, [x25, x13]\n"
-    "movprfx z18, z22\n smax z18.b, p2/M, z18.b, z20.b\n"
+    "movprfx z18, z22\n smax z18.b, p2/M, z18.b, z17.b\n"
     "ld1b { z24.b }, p1/Z, [x21, x13]\n"
-    "smax z17.b, p2/M, z17.b, z21.b\n"
+    "movprfx z17, z21\n smax z17.b, p2/M, z17.b, z16.b\n"
     "ld1b { z23.b }, p1/Z, [x19, x13]\n"
     "incw x13\n"
-    "smax z16.b, p2/M, z16.b, z21.b\n"
+    "movprfx z16, z21\n smax z16.b, p2/M, z16.b, z20.b\n"
     "st1b { z19.b }, p0, [x11, x12]\n"
     "whilelt p1.b, x13, x14\n"
     "st1b { z18.b }, p0, [x10, x12]\n"
@@ -123,15 +123,15 @@ void sve_s8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
     "whilelt p0.b, x12, x14\n"
     "movprfx z21, z30\n smax z21.b, p2/M, z21.b, z29.b\n"
     "movprfx z18, z28\n smax z18.b, p2/M, z18.b, z27.b\n"
-    "movprfx z20, z26\n smax z20.b, p2/M, z20.b, z25.b\n"
-    "movprfx z17, z24\n smax z17.b, p2/M, z17.b, z28.b\n"
-    "movprfx z16, z26\n smax z16.b, p2/M, z16.b, z23.b\n"
+    "movprfx z17, z26\n smax z17.b, p2/M, z17.b, z25.b\n"
+    "movprfx z16, z24\n smax z16.b, p2/M, z16.b, z28.b\n"
+    "movprfx z20, z26\n smax z20.b, p2/M, z20.b, z23.b\n"
     "movprfx z19, z22\n smax z19.b, p2/M, z19.b, z18.b\n"
     "st1b { z19.b }, p0, [x11, x12]\n"
-    "movprfx z18, z22\n smax z18.b, p2/M, z18.b, z20.b\n"
-    "smax z17.b, p2/M, z17.b, z21.b\n"
+    "movprfx z18, z22\n smax z18.b, p2/M, z18.b, z17.b\n"
+    "movprfx z17, z21\n smax z17.b, p2/M, z17.b, z16.b\n"
     "st1b { z18.b }, p0, [x10, x12]\n"
-    "smax z16.b, p2/M, z16.b, z21.b\n"
+    "movprfx z16, z21\n smax z16.b, p2/M, z16.b, z20.b\n"
     "st1b { z17.b }, p0, [x9, x12]\n"
     "st1b { z16.b }, p0, [x28, x12]\n"
     :

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst.hpp
index 7490a92578c6e85612e70efb572b86485a5b95d8..428902ad61ab5a5dc2d4338e7beba0b205c34c2c 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_s8_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = sve_s8_nhwc_max_generic_depthfirst_impl;
 
   sve_s8_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp
index 5c4c18bfa6493c58f34e00c15b11cd6e473fc169..3e88c8729c10b225d2b8c4d032b9192a5a48a551 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8_nhwc_max_generic_depthfirst/generic.cpp
@@ -84,33 +84,33 @@ void sve_s8_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n smax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movprfx z18, z31\n smax z18.b, p4/M, z18.b, z30.b\n"
     "add x19, x19, #0x20\n"
-    "smax z22.b, p4/M, z22.b, z29.b\n"
+    "movprfx z18, z31\n smax z18.b, p4/M, z18.b, z30.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
+    "smax z22.b, p4/M, z22.b, z29.b\n"
     "movprfx z17, z28\n smax z17.b, p4/M, z17.b, z27.b\n"
-    "smax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "smax z16.b, p4/M, z16.b, z25.b\n"
+    "smax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
-    "smax z20.b, p4/M, z20.b, z24.b\n"
+    "smax z16.b, p4/M, z16.b, z25.b\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
-    "smax z19.b, p4/M, z19.b, z23.b\n"
+    "smax z20.b, p4/M, z20.b, z24.b\n"
     "ld1b { z31.b }, p2/Z, [x23, x27]\n"
-    "smax z18.b, p4/M, z18.b, z22.b\n"
+    "smax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z30.b }, p2/Z, [x22, x27]\n"
-    "smax z17.b, p4/M, z17.b, z21.b\n"
+    "smax z18.b, p4/M, z18.b, z22.b\n"
     "ld1b { z22.b }, p2/Z, [x21, x27]\n"
-    "smax z16.b, p4/M, z16.b, z20.b\n"
+    "smax z17.b, p4/M, z17.b, z21.b\n"
     "ld1b { z29.b }, p2/Z, [x20, x27]\n"
-    "smax z7.b, p4/M, z7.b, z19.b\n"
+    "smax z16.b, p4/M, z16.b, z20.b\n"
     "ld1b { z28.b }, p1/Z, [x23, x26]\n"
-    "smax z6.b, p4/M, z6.b, z18.b\n"
+    "smax z7.b, p4/M, z7.b, z19.b\n"
     "ld1b { z27.b }, p1/Z, [x22, x26]\n"
-    "smax z5.b, p4/M, z5.b, z17.b\n"
+    "smax z6.b, p4/M, z6.b, z18.b\n"
     "ld1b { z21.b }, p1/Z, [x21, x26]\n"
-    "smax z4.b, p4/M, z4.b, z16.b\n"
+    "smax z5.b, p4/M, z5.b, z17.b\n"
     "ld1b { z26.b }, p1/Z, [x20, x26]\n"
+    "smax z4.b, p4/M, z4.b, z16.b\n"
     "ld1b { z16.b }, p0/Z, [x23, x25]\n"
     "ld1b { z25.b }, p0/Z, [x22, x25]\n"
     "ld1b { z20.b }, p0/Z, [x21, x25]\n"
@@ -168,9 +168,9 @@ void sve_s8_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
+    "subs x24, x24, #0x1\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
@@ -182,11 +182,11 @@ void sve_s8_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n smax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
+    "add x19, x19, #0x20\n"
     "smax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
-    "smax z7.b, p4/M, z7.b, z19.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "add x19, x19, #0x20\n"
+    "smax z7.b, p4/M, z7.b, z19.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
     "bgt 9b\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst.hpp
index 8eb7a39170bce98f8895c8384d2815c90e5b3554..1242eaf53096431603a3c8e56feac149a314becb 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_s8q_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = sve_s8q_nhwc_avg_generic_depthfirst_impl;
 
   sve_s8q_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp
index 51d2973ce14df1420f30e53dcd2b3f61cccf2bff..928eb412b5f320b6ac0fc2c722a28548c96c7cf1 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_avg_generic_depthfirst/generic.cpp
@@ -41,10 +41,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -108,7 +108,7 @@ void sve_s8q_nhwc_avg_generic_depthfirst_impl(
       "mov %w[combined_rescale_value], v18.s[0]\n"
     : [combined_rescale_value] "=r" (combined_rescale_value)
     : [per_layer_mul] "r" (qp.per_layer_mul), [rescale_value] "r" (rescale_value)
-    : "q16", "q17", "q18"
+    : "v16", "v17", "v18"
   );
 
   __asm__ __volatile__(

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst.hpp
index fd8b2f89ccb0d53812b77e6a8ee42b3061da98b5..84aa0d3d6bcfde5c97c0011e3a16270a54e5addf 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_s8q_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = sve_s8q_nhwc_max_generic_depthfirst_impl;
 
   sve_s8q_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp
index 54f694c7383ae7b7c382bad4f33f506ea426b067..3717f8cb302a8519579bc57f6ed5899c23009762 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp
@@ -86,33 +86,33 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n smax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movprfx z18, z31\n smax z18.b, p4/M, z18.b, z30.b\n"
     "add x19, x19, #0x20\n"
-    "smax z22.b, p4/M, z22.b, z29.b\n"
+    "movprfx z18, z31\n smax z18.b, p4/M, z18.b, z30.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
+    "smax z22.b, p4/M, z22.b, z29.b\n"
     "movprfx z17, z28\n smax z17.b, p4/M, z17.b, z27.b\n"
-    "smax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "smax z16.b, p4/M, z16.b, z25.b\n"
+    "smax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
-    "smax z20.b, p4/M, z20.b, z24.b\n"
+    "smax z16.b, p4/M, z16.b, z25.b\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
-    "smax z19.b, p4/M, z19.b, z23.b\n"
+    "smax z20.b, p4/M, z20.b, z24.b\n"
     "ld1b { z31.b }, p2/Z, [x23, x27]\n"
-    "smax z18.b, p4/M, z18.b, z22.b\n"
+    "smax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z30.b }, p2/Z, [x22, x27]\n"
-    "smax z17.b, p4/M, z17.b, z21.b\n"
+    "smax z18.b, p4/M, z18.b, z22.b\n"
     "ld1b { z22.b }, p2/Z, [x21, x27]\n"
-    "smax z16.b, p4/M, z16.b, z20.b\n"
+    "smax z17.b, p4/M, z17.b, z21.b\n"
     "ld1b { z29.b }, p2/Z, [x20, x27]\n"
-    "smax z8.b, p4/M, z8.b, z19.b\n"
+    "smax z16.b, p4/M, z16.b, z20.b\n"
     "ld1b { z28.b }, p1/Z, [x23, x26]\n"
-    "smax z7.b, p4/M, z7.b, z18.b\n"
+    "smax z8.b, p4/M, z8.b, z19.b\n"
     "ld1b { z27.b }, p1/Z, [x22, x26]\n"
-    "smax z6.b, p4/M, z6.b, z17.b\n"
+    "smax z7.b, p4/M, z7.b, z18.b\n"
     "ld1b { z21.b }, p1/Z, [x21, x26]\n"
-    "smax z5.b, p4/M, z5.b, z16.b\n"
+    "smax z6.b, p4/M, z6.b, z17.b\n"
     "ld1b { z26.b }, p1/Z, [x20, x26]\n"
+    "smax z5.b, p4/M, z5.b, z16.b\n"
     "ld1b { z16.b }, p0/Z, [x23, x25]\n"
     "ld1b { z25.b }, p0/Z, [x22, x25]\n"
     "ld1b { z20.b }, p0/Z, [x21, x25]\n"
@@ -294,9 +294,9 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
+    "subs x24, x24, #0x1\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
@@ -308,11 +308,11 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n smax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
+    "add x19, x19, #0x20\n"
     "smax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
-    "smax z8.b, p4/M, z8.b, z19.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "add x19, x19, #0x20\n"
+    "smax z8.b, p4/M, z8.b, z19.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
     "bgt 9b\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst.hpp
index e9b3625e5386ee89d0cf0014b84e7a56de6b1b72..299e55c9be27a7eb621a774a210ce9c65e502a8c 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_u8_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = sve_u8_nhwc_avg_generic_depthfirst_impl;
 
   sve_u8_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp
index 85d714547d73c190461078356baa945659f9cbe8..51a69a42be541e8266d85697c837d3a9a0ecec37 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_avg_generic_depthfirst/generic.cpp
@@ -40,10 +40,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -237,22 +237,22 @@ void sve_u8_nhwc_avg_generic_depthfirst_impl(
     "ld1rw { z17.s }, p4/Z, [%x[rescale_ptr]]\n"
     "mov z19.s, #0xff\n"
     "ld1rw { z16.s }, p4/Z, [%x[shift_ptr]]\n"
-    ".inst 0x04b175ef  // sqrdmulh z15.s, z15.s, z17.s\n"
-    ".inst 0x04b175ce  // sqrdmulh z14.s, z14.s, z17.s\n"
-    ".inst 0x04b175ad  // sqrdmulh z13.s, z13.s, z17.s\n"
-    ".inst 0x04b1758c  // sqrdmulh z12.s, z12.s, z17.s\n"
-    ".inst 0x04b1756b  // sqrdmulh z11.s, z11.s, z17.s\n"
-    ".inst 0x04b1754a  // sqrdmulh z10.s, z10.s, z17.s\n"
-    ".inst 0x04b17529  // sqrdmulh z9.s, z9.s, z17.s\n"
-    ".inst 0x04b17508  // sqrdmulh z8.s, z8.s, z17.s\n"
-    ".inst 0x04b174e7  // sqrdmulh z7.s, z7.s, z17.s\n"
-    ".inst 0x04b174c6  // sqrdmulh z6.s, z6.s, z17.s\n"
-    ".inst 0x04b174a5  // sqrdmulh z5.s, z5.s, z17.s\n"
-    ".inst 0x04b17484  // sqrdmulh z4.s, z4.s, z17.s\n"
-    ".inst 0x04b17463  // sqrdmulh z3.s, z3.s, z17.s\n"
-    ".inst 0x04b17442  // sqrdmulh z2.s, z2.s, z17.s\n"
-    ".inst 0x04b17421  // sqrdmulh z1.s, z1.s, z17.s\n"
-    ".inst 0x04b17400  // sqrdmulh z0.s, z0.s, z17.s\n"
+    ".inst 0x04b175ef  // sqdmulh z15.s, z15.s, z17.s\n"
+    ".inst 0x04b175ce  // sqdmulh z14.s, z14.s, z17.s\n"
+    ".inst 0x04b175ad  // sqdmulh z13.s, z13.s, z17.s\n"
+    ".inst 0x04b1758c  // sqdmulh z12.s, z12.s, z17.s\n"
+    ".inst 0x04b1756b  // sqdmulh z11.s, z11.s, z17.s\n"
+    ".inst 0x04b1754a  // sqdmulh z10.s, z10.s, z17.s\n"
+    ".inst 0x04b17529  // sqdmulh z9.s, z9.s, z17.s\n"
+    ".inst 0x04b17508  // sqdmulh z8.s, z8.s, z17.s\n"
+    ".inst 0x04b174e7  // sqdmulh z7.s, z7.s, z17.s\n"
+    ".inst 0x04b174c6  // sqdmulh z6.s, z6.s, z17.s\n"
+    ".inst 0x04b174a5  // sqdmulh z5.s, z5.s, z17.s\n"
+    ".inst 0x04b17484  // sqdmulh z4.s, z4.s, z17.s\n"
+    ".inst 0x04b17463  // sqdmulh z3.s, z3.s, z17.s\n"
+    ".inst 0x04b17442  // sqdmulh z2.s, z2.s, z17.s\n"
+    ".inst 0x04b17421  // sqdmulh z1.s, z1.s, z17.s\n"
+    ".inst 0x04b17400  // sqdmulh z0.s, z0.s, z17.s\n"
     ".inst 0x4482920f  // srshl z15.s, p4/M, z15.s, z16.s\n"
     ".inst 0x4482920e  // srshl z14.s, p4/M, z14.s, z16.s\n"
     ".inst 0x4482920d  // srshl z13.s, p4/M, z13.s, z16.s\n"
@@ -379,10 +379,10 @@ void sve_u8_nhwc_avg_generic_depthfirst_impl(
     "ld1rw { z17.s }, p4/Z, [%x[rescale_ptr]]\n"
     "mov z19.s, #0xff\n"
     "ld1rw { z16.s }, p4/Z, [%x[shift_ptr]]\n"
-    ".inst 0x04b175ef  // sqrdmulh z15.s, z15.s, z17.s\n"
-    ".inst 0x04b175ce  // sqrdmulh z14.s, z14.s, z17.s\n"
-    ".inst 0x04b175ad  // sqrdmulh z13.s, z13.s, z17.s\n"
-    ".inst 0x04b1758c  // sqrdmulh z12.s, z12.s, z17.s\n"
+    ".inst 0x04b175ef  // sqdmulh z15.s, z15.s, z17.s\n"
+    ".inst 0x04b175ce  // sqdmulh z14.s, z14.s, z17.s\n"
+    ".inst 0x04b175ad  // sqdmulh z13.s, z13.s, z17.s\n"
+    ".inst 0x04b1758c  // sqdmulh z12.s, z12.s, z17.s\n"
     ".inst 0x4482920f  // srshl z15.s, p4/M, z15.s, z16.s\n"
     ".inst 0x4482920e  // srshl z14.s, p4/M, z14.s, z16.s\n"
     ".inst 0x4482920d  // srshl z13.s, p4/M, z13.s, z16.s\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
index 22e95a68eb65185e316b18e3c7a2d5f5b008d6da..e921f345d5f4a3544e91f458ed33bd91a17e5b63 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst/generic.cpp
@@ -65,13 +65,13 @@ void sve_u8_nhwc_max_2x2_s1_output2x2_depthfirst_impl(
   __asm__ __volatile__(
     "ldr x14, [%x[args], %[offsetof_n_channels]]\n"
     "ptrue p2.b\n"
-    "ldr x19, [%x[args], %[offsetof_outptrs]]\n"
+    "ldr x20, [%x[args], %[offsetof_outptrs]]\n"
     "mov x13, #0x0\n"
+    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
     "mov x12, #0x0\n"
-    "ldp x11, x10, [x19, #0x0]\n"
+    "ldp x11, x10, [x20, #0x0]\n"
     "whilelt p1.b, x13, x14\n"
-    "ldp x9, x28, [x19, #0x10]\n"
-    "ldr x19, [%x[args], %[offsetof_inptrs]]\n"
+    "ldp x9, x28, [x20, #0x10]\n"
     "ldp x27, x26, [x19, #0x0]\n"
     "ldp x25, x24, [x19, #0x10]\n"
     "ldp x23, x22, [x19, #0x20]\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst.hpp
index 80d3599d5e6b7ba956d282c5b246574c06b1c8f1..59cd4b9c78ee6f476162828031f07253c51653d0 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_u8_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = sve_u8_nhwc_max_generic_depthfirst_impl;
 
   sve_u8_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp
index 7990a3d9fcda34e4ed6badcd36a235a91b60167e..164847480b0b48adcb30c147b41a9b0d12ceec0d 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp
@@ -84,33 +84,33 @@ void sve_u8_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n umax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movprfx z18, z31\n umax z18.b, p4/M, z18.b, z30.b\n"
     "add x19, x19, #0x20\n"
-    "umax z22.b, p4/M, z22.b, z29.b\n"
+    "movprfx z18, z31\n umax z18.b, p4/M, z18.b, z30.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
+    "umax z22.b, p4/M, z22.b, z29.b\n"
     "movprfx z17, z28\n umax z17.b, p4/M, z17.b, z27.b\n"
-    "umax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "umax z16.b, p4/M, z16.b, z25.b\n"
+    "umax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
-    "umax z20.b, p4/M, z20.b, z24.b\n"
+    "umax z16.b, p4/M, z16.b, z25.b\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
-    "umax z19.b, p4/M, z19.b, z23.b\n"
+    "umax z20.b, p4/M, z20.b, z24.b\n"
     "ld1b { z31.b }, p2/Z, [x23, x27]\n"
-    "umax z18.b, p4/M, z18.b, z22.b\n"
+    "umax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z30.b }, p2/Z, [x22, x27]\n"
-    "umax z17.b, p4/M, z17.b, z21.b\n"
+    "umax z18.b, p4/M, z18.b, z22.b\n"
     "ld1b { z22.b }, p2/Z, [x21, x27]\n"
-    "umax z16.b, p4/M, z16.b, z20.b\n"
+    "umax z17.b, p4/M, z17.b, z21.b\n"
     "ld1b { z29.b }, p2/Z, [x20, x27]\n"
-    "umax z7.b, p4/M, z7.b, z19.b\n"
+    "umax z16.b, p4/M, z16.b, z20.b\n"
     "ld1b { z28.b }, p1/Z, [x23, x26]\n"
-    "umax z6.b, p4/M, z6.b, z18.b\n"
+    "umax z7.b, p4/M, z7.b, z19.b\n"
     "ld1b { z27.b }, p1/Z, [x22, x26]\n"
-    "umax z5.b, p4/M, z5.b, z17.b\n"
+    "umax z6.b, p4/M, z6.b, z18.b\n"
     "ld1b { z21.b }, p1/Z, [x21, x26]\n"
-    "umax z4.b, p4/M, z4.b, z16.b\n"
+    "umax z5.b, p4/M, z5.b, z17.b\n"
     "ld1b { z26.b }, p1/Z, [x20, x26]\n"
+    "umax z4.b, p4/M, z4.b, z16.b\n"
     "ld1b { z16.b }, p0/Z, [x23, x25]\n"
     "ld1b { z25.b }, p0/Z, [x22, x25]\n"
     "ld1b { z20.b }, p0/Z, [x21, x25]\n"
@@ -168,9 +168,9 @@ void sve_u8_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
+    "subs x24, x24, #0x1\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
@@ -182,11 +182,11 @@ void sve_u8_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n umax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
+    "add x19, x19, #0x20\n"
     "umax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
-    "umax z7.b, p4/M, z7.b, z19.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "add x19, x19, #0x20\n"
+    "umax z7.b, p4/M, z7.b, z19.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
     "bgt 9b\n"

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst.hpp
index 098896df71c310203bb3b744413798c2ea98363a..f6fc1a58c15ba842d6e0fd4f73b77143277d38e7 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_u8q_nhwc_avg_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::AVERAGE; }
 
+
   kern_type kernel = sve_u8q_nhwc_avg_generic_depthfirst_impl;
 
   sve_u8q_nhwc_avg_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp
index 368577c6b1a5550c0d72274d01060bf8829506ad..373848ad2b9997e86f14b476743285e02f4489c1 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp
@@ -41,10 +41,10 @@ namespace {
 
   constexpr RescaleParams rescale_params[8] = {
     {0x40000000, -0},  // 1/2
-    {0x55555555, -1},  // 1/3
+    {0x55555556, -1},  // 1/3
     {0x40000000, -1},  // 1/4
     {0x66666666, -2},  // 1/5
-    {0x55555555, -2},  // 1/6
+    {0x55555556, -2},  // 1/6
     {0x49249249, -2},  // 1/7
     {0x40000000, -2},  // 1/8
     {0x71c71c72, -3},  // 1/9
@@ -113,7 +113,7 @@ void sve_u8q_nhwc_avg_generic_depthfirst_impl(
       "mov %w[combined_rescale_value], v18.s[0]\n"
     : [combined_rescale_value] "=r" (combined_rescale_value)
     : [per_layer_mul] "r" (qp.per_layer_mul), [rescale_value] "r" (rescale_value)
-    : "q16", "q17", "q18"
+    : "v16", "v17", "v18"
   );
 
   __asm__ __volatile__(

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst.hpp
index 7633db15085edc7f44e7e0abc832c735c0e175b5..c3c0edd0d589c81f01aca0470e4455d80d8fc0be 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst.hpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst.hpp
@@ -42,6 +42,7 @@ struct sve_u8q_nhwc_max_generic_depthfirst
 
   constexpr static PoolingType pooling_type(void) { return PoolingType::MAX; }
 
+
   kern_type kernel = sve_u8q_nhwc_max_generic_depthfirst_impl;
 
   sve_u8q_nhwc_max_generic_depthfirst(const CPUInfo *) {}

diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp
index c10408850416997e75ef9fb30223eb9e1b937cc7..c1c1d296136405496aa5b1a3b827ac79fffaa759 100644 (file)
--- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp
+++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp
@@ -86,33 +86,33 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n umax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
-    "movprfx z18, z31\n umax z18.b, p4/M, z18.b, z30.b\n"
     "add x19, x19, #0x20\n"
-    "umax z22.b, p4/M, z22.b, z29.b\n"
+    "movprfx z18, z31\n umax z18.b, p4/M, z18.b, z30.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
+    "umax z22.b, p4/M, z22.b, z29.b\n"
     "movprfx z17, z28\n umax z17.b, p4/M, z17.b, z27.b\n"
-    "umax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "umax z16.b, p4/M, z16.b, z25.b\n"
+    "umax z21.b, p4/M, z21.b, z26.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
-    "umax z20.b, p4/M, z20.b, z24.b\n"
+    "umax z16.b, p4/M, z16.b, z25.b\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
-    "umax z19.b, p4/M, z19.b, z23.b\n"
+    "umax z20.b, p4/M, z20.b, z24.b\n"
     "ld1b { z31.b }, p2/Z, [x23, x27]\n"
-    "umax z18.b, p4/M, z18.b, z22.b\n"
+    "umax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z30.b }, p2/Z, [x22, x27]\n"
-    "umax z17.b, p4/M, z17.b, z21.b\n"
+    "umax z18.b, p4/M, z18.b, z22.b\n"
     "ld1b { z22.b }, p2/Z, [x21, x27]\n"
-    "umax z16.b, p4/M, z16.b, z20.b\n"
+    "umax z17.b, p4/M, z17.b, z21.b\n"
     "ld1b { z29.b }, p2/Z, [x20, x27]\n"
-    "umax z10.b, p4/M, z10.b, z19.b\n"
+    "umax z16.b, p4/M, z16.b, z20.b\n"
     "ld1b { z28.b }, p1/Z, [x23, x26]\n"
-    "umax z9.b, p4/M, z9.b, z18.b\n"
+    "umax z10.b, p4/M, z10.b, z19.b\n"
     "ld1b { z27.b }, p1/Z, [x22, x26]\n"
-    "umax z8.b, p4/M, z8.b, z17.b\n"
+    "umax z9.b, p4/M, z9.b, z18.b\n"
     "ld1b { z21.b }, p1/Z, [x21, x26]\n"
-    "umax z7.b, p4/M, z7.b, z16.b\n"
+    "umax z8.b, p4/M, z8.b, z17.b\n"
     "ld1b { z26.b }, p1/Z, [x20, x26]\n"
+    "umax z7.b, p4/M, z7.b, z16.b\n"
     "ld1b { z16.b }, p0/Z, [x23, x25]\n"
     "ld1b { z25.b }, p0/Z, [x22, x25]\n"
     "ld1b { z20.b }, p0/Z, [x21, x25]\n"
@@ -315,9 +315,9 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl(
     "lsr x24, %x[n_valid_cells], #0x2\n"
     "cbz x24, 11f\n"
     "ldp x23, x22, [x19, #0x0]\n"
-    "subs x24, x24, #0x1\n"
     "ldp x21, x20, [x19, #0x10]\n"
     "add x19, x19, #0x20\n"
+    "subs x24, x24, #0x1\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
@@ -329,11 +329,11 @@ void sve_u8q_nhwc_max_generic_depthfirst_impl(
     "subs x24, x24, #0x1\n"
     "movprfx z23, z1\n umax z23.b, p4/M, z23.b, z0.b\n"
     "ldp x21, x20, [x19, #0x10]\n"
+    "add x19, x19, #0x20\n"
     "umax z19.b, p4/M, z19.b, z23.b\n"
     "ld1b { z3.b }, p3/Z, [x23, x28]\n"
-    "umax z10.b, p4/M, z10.b, z19.b\n"
     "ld1b { z2.b }, p3/Z, [x22, x28]\n"
-    "add x19, x19, #0x20\n"
+    "umax z10.b, p4/M, z10.b, z19.b\n"
     "ld1b { z1.b }, p3/Z, [x21, x28]\n"
     "ld1b { z0.b }, p3/Z, [x20, x28]\n"
     "bgt 9b\n"