COMPMID-3975: Update CLGEMM heuristic for Mali-G52 F16

- Update the GEMM heuristic for Mali-G52
- Fix the GEMV heuristic on Mali-G76

Change-Id: I895153d02d09e73c51ecf158e2ae43f81f6d7b05
Signed-off-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4425
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>

diff --git a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp
index 70992974a3a88e289c4636fccc6b2e746d9f5980..e5c89f89af5a0560a0c6ad72e8339adef19282fb 100644 (file)
--- a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp
+++ b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp
@@ -64,7 +64,7 @@ std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedKernelConfiguratio
     static std::map<DataType, ConfigurationFunctionExecutorPtr> gemm_configs_G52 =
     {
         { DataType::F32, &CLGEMMReshapedKernelConfigurationBifrost::configure_G52_f32 },
-        { DataType::F16, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_f16 },
+        { DataType::F16, &CLGEMMReshapedKernelConfigurationBifrost::configure_G52_f16 },
         { DataType::QASYMM8, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_u8 },
         { DataType::QSYMM8, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_u8 },
         { DataType::QASYMM8_SIGNED, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_u8 },
@@ -263,6 +263,22 @@ std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedKernelConfiguratio
     }
 }
 
+std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedKernelConfigurationBifrost::configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b)
+{
+    ARM_COMPUTE_UNUSED(k);
+
+    const float workload = (static_cast<float>(m) * static_cast<float>(n) * static_cast<float>(b)) / 20.0f;
+
+    if(workload <= 232.8000f)
+    {
+        return configure_lhs_rhs_info(m, n, 2, 4, 4, 4, 4, true, true, true, false, false);
+    }
+    else
+    {
+        return configure_lhs_rhs_info(m, n, 4, 4, 4, 4, 4, true, true, true, false, false);
+    }
+}
+
 std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedKernelConfigurationBifrost::configure_G76_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b)
 {
     ARM_COMPUTE_UNUSED(k);

diff --git a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h
index 6c67b70962545bc1387510019db57eac164622fe..715e7abfa3cea0b17af7d30f53c5d88d4e474fbf 100644 (file)
--- a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h
+++ b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h
@@ -48,6 +48,7 @@ private:
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G52_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G76_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G7x_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
+    std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G7x_u8(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G76_u8(unsigned int m, unsigned int n, unsigned int k, unsigned int b);

diff --git a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp
index 188ba4d9c513d411435d9f13faa03fcb88f5893a..a2c1ed2c8e80e3b749ee0028c1a15d33dc5a04bd 100644 (file)
--- a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp
+++ b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp
@@ -65,7 +65,7 @@ std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedOnlyRHSKernelConfi
     static std::map<DataType, ConfigurationFunctionExecutorPtr> gemm_configs_G52 =
     {
         { DataType::F32, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G52_f32 },
-        { DataType::F16, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_f16 },
+        { DataType::F16, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G52_f16 },
         { DataType::QASYMM8, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_u8 },
         { DataType::QSYMM8, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_u8 },
         { DataType::QASYMM8_SIGNED, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_u8 },
@@ -168,16 +168,25 @@ std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedOnlyRHSKernelConfi
     GEMMRHSMatrixInfo rhs_info_img;
 
     const bool is_workload_big = ((m * n * b) / 16) >= 2048;
-    // Get lhs_info/rhs_info in case of OpenCL buffer
+
     if(m == 1)
     {
-        if(n <= 204.0)
+        if(n >= 8192)
         {
-            return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, 16, false, true, false, true, false);
+            const unsigned int h0 = std::max(n / 4, 1U);
+            return configure_lhs_rhs_info(m, n, 1, 4, 8, 1, h0, false, true, false, true, false);
         }
         else
         {
-            return configure_lhs_rhs_info(m, n, 1, 2, 8, 1, 32, false, true, false, true, false);
+            const unsigned int h0 = std::max(n / 2, 1U);
+            if(n <= 204)
+            {
+                return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, h0, false, true, false, true, false);
+            }
+            else
+            {
+                return configure_lhs_rhs_info(m, n, 1, 2, 8, 1, h0, false, true, false, true, false);
+            }
         }
     }
     else
@@ -306,6 +315,72 @@ std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedOnlyRHSKernelConfi
     }
 }
 
+std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b)
+{
+    const float r_mn     = static_cast<float>(m) / static_cast<float>(n);
+    const float workload = (static_cast<float>(m) * static_cast<float>(n) * static_cast<float>(b)) / 20.0f;
+    const float r_mk = static_cast<float>(m) / static_cast<float>(k);
+    const float r_nk = static_cast<float>(n) / static_cast<float>(k);
+
+    if(m == 1)
+    {
+        if(r_mk <= 0.0026f)
+        {
+            if(r_nk <= 0.4664f)
+            {
+                return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, 32, false, true, false, true, false);
+            }
+            else
+            {
+                return configure_lhs_rhs_info(m, n, 1, 4, 16, 1, 16, false, true, false, false, true);
+            }
+        }
+        else
+        {
+            if(r_mk <= 0.0148f)
+            {
+                return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, 32, false, true, false, true, false);
+            }
+            else
+            {
+                return configure_lhs_rhs_info(m, n, 1, 4, 16, 1, 16, false, true, false, false, true);
+            }
+        }
+    }
+    else
+    {
+        if(workload <= 362.6000f)
+        {
+            return configure_lhs_rhs_info(m, n, 2, 2, 8, 1, 16, false, false, false, true, false);
+        }
+        else
+        {
+            if(r_mn <= 22.6067f)
+            {
+                if(workload <= 708.8000f)
+                {
+                    return configure_lhs_rhs_info(m, n, 5, 4, 4, 1, 2, false, false, false, false, true);
+                }
+                else
+                {
+                    return configure_lhs_rhs_info(m, n, 5, 8, 2, 1, 16, false, false, false, false, false);
+                }
+            }
+            else
+            {
+                if(r_nk <= 0.0917f)
+                {
+                    return configure_lhs_rhs_info(m, n, 2, 2, 8, 1, 16, false, false, false, true, false);
+                }
+                else
+                {
+                    return configure_lhs_rhs_info(m, n, 5, 4, 4, 1, 2, false, false, false, false, true);
+                }
+            }
+        }
+    }
+}
+
 std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b)
 {
     ARM_COMPUTE_UNUSED(k);

diff --git a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h
index 3dfd96a822538fcdb83f09db559c4eb3a8641dc6..4d284ed3e82de20ee40b66c78fd42a8eb1832936 100644 (file)
--- a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h
+++ b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h
@@ -49,6 +49,7 @@ private:
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G52_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G51_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G7x_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
+    std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G51_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b);
     std::pair<GEMMLHSMatrixInfo, GEMMRHSMatrixInfo> configure_G7x_u8(unsigned int m, unsigned int n, unsigned int k, unsigned int b);

diff --git a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp
index c77746a04472c12986610a917c9a8fb616f11de2..46d07fffba3810eeecbb563e264719c19383b8a7 100644 (file)
--- a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp
+++ b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp
@@ -72,7 +72,7 @@ CLGEMMKernelType CLGEMMKernelSelectionBifrost::select_kernel(const CLGEMMKernelS
     static std::map<DataType, FunctionExecutorPtr> gemm_g52_configs =
     {
         { DataType::F32, &CLGEMMKernelSelectionBifrost::g52_f32 },
-        { DataType::F16, &CLGEMMKernelSelectionBifrost::default_f16 },
+        { DataType::F16, &CLGEMMKernelSelectionBifrost::g52_f16 },
         { DataType::QASYMM8, &CLGEMMKernelSelectionBifrost::default_q8 },
         { DataType::QASYMM8_SIGNED, &CLGEMMKernelSelectionBifrost::default_q8 },
         { DataType::QSYMM8, &CLGEMMKernelSelectionBifrost::default_q8 },
@@ -443,6 +443,126 @@ CLGEMMKernelType CLGEMMKernelSelectionBifrost::g76_f16(unsigned int m, unsigned
     }
 }
 
+CLGEMMKernelType CLGEMMKernelSelectionBifrost::g52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant)
+{
+    ARM_COMPUTE_UNUSED(b);
+
+    if (!is_rhs_constant)
+    {
+        return CLGEMMKernelType::NATIVE_V1;
+    }
+
+    if (m == 1)
+    {
+        return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+    }
+
+    const float r_mn  = static_cast<float>(m) / static_cast<float>(n);
+    const float r_mk  = static_cast<float>(m) / static_cast<float>(k);
+    const float r_nk  = static_cast<float>(n) / static_cast<float>(k);
+    const float r_mnk = static_cast<float>(m) / (static_cast<float>(n) * static_cast<float>(k));
+
+    if(r_mn <= 22.9200f)
+    {
+        if(r_mk <= 0.0157f)
+        {
+            return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+        }
+        else
+        {
+            if(r_mnk <= 7809.3750f)
+            {
+                if(r_mnk <= 101.7937f)
+                {
+                    if(r_mn <= 0.4594f)
+                    {
+                        if(r_mk <= 0.0557f)
+                        {
+                            return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+                        }
+                        else
+                        {
+                            return CLGEMMKernelType::RESHAPED;
+                        }
+                    }
+                    else
+                    {
+                        return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+                    }
+                }
+                else
+                {
+                    if(r_nk <= 0.4396f)
+                    {
+                        if(r_mn <= 1.5182f)
+                        {
+                            if(r_mnk <= 1709.9167f)
+                            {
+                                return CLGEMMKernelType::RESHAPED;
+                            }
+                            else
+                            {
+                                return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+                            }
+                        }
+                        else
+                        {
+                            if(r_mnk <= 1330.6000f)
+                            {
+                                return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+                            }
+                            else
+                            {
+                                return CLGEMMKernelType::RESHAPED;
+                            }
+                        }
+                    }
+                    else
+                    {
+                        if(r_mn <= 2.5896f)
+                        {
+                            return CLGEMMKernelType::RESHAPED;
+                        }
+                        else
+                        {
+                            if(r_mnk <= 326.6667f)
+                            {
+                                return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+                            }
+                            else
+                            {
+                                return CLGEMMKernelType::RESHAPED;
+                            }
+                        }
+                    }
+                }
+            }
+            else
+            {
+                return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+            }
+        }
+    }
+    else
+    {
+        if(r_mn <= 86.7578f)
+        {
+            if(r_mnk <= 11231.6406f)
+            {
+                return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+            }
+            else
+            {
+                return CLGEMMKernelType::RESHAPED;
+            }
+        }
+        else
+        {
+            return CLGEMMKernelType::RESHAPED_ONLY_RHS;
+        }
+    }
+}
+
 CLGEMMKernelType CLGEMMKernelSelectionBifrost::g71_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant)
 {
     ARM_COMPUTE_UNUSED(b);

diff --git a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h
index fbafc531f5fda106dcac6b1d34b38da58e2ab698..6831a12aec80e9b579da3fbecdc29909f94cd666 100644 (file)
--- a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h
+++ b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h
@@ -47,6 +47,7 @@ private:
     CLGEMMKernelType g52_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant);
     CLGEMMKernelType g76_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant);
     CLGEMMKernelType g76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant);
+    CLGEMMKernelType g52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant);
     CLGEMMKernelType g71_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant);
     CLGEMMKernelType default_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant);
     CLGEMMKernelType default_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant);