From 0001260f4bc4c8bb5509ec29e002ad6f42e84345 Mon Sep 17 00:00:00 2001 From: Werner Saar Date: Wed, 20 Apr 2016 13:06:38 +0200 Subject: [PATCH] optimized sgemm --- kernel/power/sgemm_logic_16x8_power8.S | 119 ++++++++----------- kernel/power/sgemm_macros_16x8_power8.S | 177 +++++++++++++++++++++++------ kernel/power/sgemm_tcopy_16_power8.S | 2 +- kernel/power/sgemm_tcopy_logic_16_power8.S | 20 ++++ 4 files changed, 210 insertions(+), 108 deletions(-) diff --git a/kernel/power/sgemm_logic_16x8_power8.S b/kernel/power/sgemm_logic_16x8_power8.S index 06bb79e..6ba9990 100644 --- a/kernel/power/sgemm_logic_16x8_power8.S +++ b/kernel/power/sgemm_logic_16x8_power8.S @@ -1,38 +1,3 @@ -/*************************************************************************** -Copyright (c) 2013-2016, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -/************************************************************************************** -* 2016/04/02 Werner Saar (wernsaar@googlemail.com) -* BLASTEST : OK -* CTEST : OK -* TEST : OK -* LAPACK-TEST : OK -**************************************************************************************/ - srawi. J, N, 3 ble SGEMM_L8_END @@ -40,35 +5,48 @@ SGEMM_L8_BEGIN: mr BO, B mr BBO, BBUFFER - slwi T1, K, 3 + srawi. T1, K, 2 + ble SGEMM_L8_COPYB1 + + +SGEMM_L8_COPYB4: -SGEMM_L8_COPYB: + dcbt BO, PRE dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + ble SGEMM_L8_COPYB1 - lxvw4x vs3, o0, BO - lxvw4x vs11, o16, BO - xxspltw vs4, vs3, 0 - xxspltw vs5, vs3, 1 - xxspltw vs6, vs3, 2 - xxspltw vs7, vs3, 3 - xxspltw vs12, vs11, 0 - xxspltw vs13, vs11, 1 - xxspltw vs14, vs11, 2 - xxspltw vs15, vs11, 3 - stxvw4x vs4, o0, BBO - stxvw4x vs5, o16, BBO - stxvw4x vs6, o32, BBO - stxvw4x vs7, o48, BBO - addi BO, BO, 32 - addi BBO, BBO, 64 - stxvw4x vs12, o0, BBO - stxvw4x vs13, o16, BBO - stxvw4x vs14, o32, BBO - stxvw4x vs15, o48, BBO - addic. T1, T1, -8 - addi BBO, BBO, 64 + dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + ble SGEMM_L8_COPYB1 + + dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + ble SGEMM_L8_COPYB1 - bge SGEMM_L8_COPYB + dcbtst BBO, PRE + COPYB_4x8 + addic. T1, T1, -1 + + bgt SGEMM_L8_COPYB4 + +SGEMM_L8_COPYB1: + + andi. T1, K, 3 + ble SGEMM_L8_COPYB_END + +SGEMM_L8_COPYB1_LOOP: + + + COPYB_1x8 + addic. T1, T1, -1 + + bgt SGEMM_L8_COPYB1_LOOP + +SGEMM_L8_COPYB_END: mr CO, C mr AO, A @@ -93,24 +71,24 @@ SGEMM_L8x16_LOOP_START: LOAD8x16_1 dcbt BO, PRE KERNEL8x16_I1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 addic. L, L, -2 @@ -122,24 +100,24 @@ SGEMM_L8x16_LOOP: dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 addic. L, L, -1 @@ -149,18 +127,15 @@ SGEMM_L8x16_LOOP_END: dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE + dcbt BO, PRE KERNEL8x16_2 dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE KERNEL8x16_2 - dcbt BO, PRE KERNEL8x16_1 - dcbt BO, PRE dcbt AO, PRE KERNEL8x16_2 KERNEL8x16_1 diff --git a/kernel/power/sgemm_macros_16x8_power8.S b/kernel/power/sgemm_macros_16x8_power8.S index 71dc529..94e7d7d 100644 --- a/kernel/power/sgemm_macros_16x8_power8.S +++ b/kernel/power/sgemm_macros_16x8_power8.S @@ -1,38 +1,3 @@ -/*************************************************************************** -Copyright (c) 2013-2016, The OpenBLAS Project -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -1. Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in -the documentation and/or other materials provided with the -distribution. -3. Neither the name of the OpenBLAS project nor the names of -its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*****************************************************************************/ - -/************************************************************************************** -* 2016/04/02 Werner Saar (wernsaar@googlemail.com) -* BLASTEST : OK -* CTEST : OK -* TEST : OK -* LAPACK-TEST : OK -**************************************************************************************/ - /********************************************************************************************** * Macros for N=8 and M=16 @@ -5886,3 +5851,145 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .endm + + + + +.macro COPYB_4x8 + + + lxvw4x vs5, o0, BO + xxspltw vs6, vs5, 0 + xxspltw vs7, vs5, 1 + xxspltw vs8, vs5, 2 + xxspltw vs9, vs5, 3 + + lxvw4x vs10, o16, BO + xxspltw vs11, vs10, 0 + xxspltw vs12, vs10, 1 + xxspltw vs13, vs10, 2 + xxspltw vs14, vs10, 3 + + lxvw4x vs15, o32, BO + xxspltw vs16, vs15, 0 + xxspltw vs17, vs15, 1 + xxspltw vs18, vs15, 2 + xxspltw vs19, vs15, 3 + + lxvw4x vs20, o48, BO + xxspltw vs21, vs20, 0 + xxspltw vs22, vs20, 1 + xxspltw vs23, vs20, 2 + xxspltw vs24, vs20, 3 + + addi BO, BO, 64 + lxvw4x vs35, o0, BO + xxspltw vs36, vs35, 0 + xxspltw vs37, vs35, 1 + xxspltw vs38, vs35, 2 + xxspltw vs39, vs35, 3 + + lxvw4x vs40, o16, BO + xxspltw vs41, vs40, 0 + xxspltw vs42, vs40, 1 + xxspltw vs43, vs40, 2 + xxspltw vs44, vs40, 3 + + lxvw4x vs45, o32, BO + xxspltw vs46, vs45, 0 + xxspltw vs47, vs45, 1 + xxspltw vs48, vs45, 2 + xxspltw vs49, vs45, 3 + + lxvw4x vs50, o48, BO + xxspltw vs51, vs50, 0 + xxspltw vs52, vs50, 1 + xxspltw vs53, vs50, 2 + xxspltw vs54, vs50, 3 + + addi BO, BO, 64 + + + stxvw4x vs6, o0, BBO + stxvw4x vs7, o16, BBO + stxvw4x vs8, o32, BBO + stxvw4x vs9, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs11, o0, BBO + stxvw4x vs12, o16, BBO + stxvw4x vs13, o32, BBO + stxvw4x vs14, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs16, o0, BBO + stxvw4x vs17, o16, BBO + stxvw4x vs18, o32, BBO + stxvw4x vs19, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs21, o0, BBO + stxvw4x vs22, o16, BBO + stxvw4x vs23, o32, BBO + stxvw4x vs24, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs36, o0, BBO + stxvw4x vs37, o16, BBO + stxvw4x vs38, o32, BBO + stxvw4x vs39, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs41, o0, BBO + stxvw4x vs42, o16, BBO + stxvw4x vs43, o32, BBO + stxvw4x vs44, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs46, o0, BBO + stxvw4x vs47, o16, BBO + stxvw4x vs48, o32, BBO + stxvw4x vs49, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs51, o0, BBO + stxvw4x vs52, o16, BBO + stxvw4x vs53, o32, BBO + stxvw4x vs54, o48, BBO + + addi BBO, BBO, 64 +.endm + + +.macro COPYB_1x8 + + + lxvw4x vs5, o0, BO + xxspltw vs6, vs5, 0 + xxspltw vs7, vs5, 1 + xxspltw vs8, vs5, 2 + xxspltw vs9, vs5, 3 + + lxvw4x vs10, o16, BO + xxspltw vs11, vs10, 0 + xxspltw vs12, vs10, 1 + xxspltw vs13, vs10, 2 + xxspltw vs14, vs10, 3 + + + addi BO, BO, 32 + + stxvw4x vs6, o0, BBO + stxvw4x vs7, o16, BBO + stxvw4x vs8, o32, BBO + stxvw4x vs9, o48, BBO + + addi BBO, BBO, 64 + stxvw4x vs11, o0, BBO + stxvw4x vs12, o16, BBO + stxvw4x vs13, o32, BBO + stxvw4x vs14, o48, BBO + + addi BBO, BBO, 64 +.endm + diff --git a/kernel/power/sgemm_tcopy_16_power8.S b/kernel/power/sgemm_tcopy_16_power8.S index d690aab..c31784d 100644 --- a/kernel/power/sgemm_tcopy_16_power8.S +++ b/kernel/power/sgemm_tcopy_16_power8.S @@ -136,7 +136,7 @@ add B1, B1, B li PREA, 384 - li PREB, 384 + addi PREB, M16, 128 li o4, 4 li o16, 16 diff --git a/kernel/power/sgemm_tcopy_logic_16_power8.S b/kernel/power/sgemm_tcopy_logic_16_power8.S index c0969ee..5a715c8 100644 --- a/kernel/power/sgemm_tcopy_logic_16_power8.S +++ b/kernel/power/sgemm_tcopy_logic_16_power8.S @@ -19,6 +19,26 @@ SCOPYT_L4_BEGIN: SCOPYT_L4x16_LOOP: + dcbtst BO, M16 + dcbtst BO, PREB + dcbt A0, PREA + dcbt A1, PREA + dcbt A2, PREA + dcbt A3, PREA + COPY_4x16 + + addi A0, A0, 16*SIZE + addi A1, A1, 16*SIZE + addi A2, A2, 16*SIZE + addi A3, A3, 16*SIZE + add BO, BO, M16 + + addic. J, J, -1 + ble SCOPYT_L4x8_BEGIN + + + dcbtst BO, M16 + dcbtst BO, PREB COPY_4x16 addi A0, A0, 16*SIZE -- 2.7.4