From a229966c9c76afe0cf18c566a3c13ddde3878288 Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Wed, 14 Aug 2019 08:58:40 +0000 Subject: [PATCH] [AArch64] Use SVE ADR to optimise shift-add sequences This patch uses SVE ADR to optimise shift-and-add and uxtw-and-add sequences. 2019-08-14 Richard Sandiford gcc/ * config/aarch64/predicates.md (const_1_to_3_operand): New predicate. * config/aarch64/aarch64-sve.md (*aarch64_adr_uxtw) (*aarch64_adr_shift, *aarch64_adr_shift_uxtw): New patterns. gcc/testsuite/ * gcc.target/aarch64/sve/adr_1.c: New test. * gcc.target/aarch64/sve/adr_1_run.c: Likewise. * gcc.target/aarch64/sve/adr_2.c: Likewise. * gcc.target/aarch64/sve/adr_2_run.c: Likewise. * gcc.target/aarch64/sve/adr_3.c: Likewise. * gcc.target/aarch64/sve/adr_3_run.c: Likewise. * gcc.target/aarch64/sve/adr_4.c: Likewise. * gcc.target/aarch64/sve/adr_4_run.c: Likewise. * gcc.target/aarch64/sve/adr_5.c: Likewise. * gcc.target/aarch64/sve/adr_5_run.c: Likewise. From-SVN: r274436 --- gcc/ChangeLog | 6 +++ gcc/config/aarch64/aarch64-sve.md | 60 ++++++++++++++++++++++++ gcc/config/aarch64/predicates.md | 12 +++++ gcc/testsuite/ChangeLog | 13 +++++ gcc/testsuite/gcc.target/aarch64/sve/adr_1.c | 46 ++++++++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/adr_1_run.c | 31 ++++++++++++ gcc/testsuite/gcc.target/aarch64/sve/adr_2.c | 21 +++++++++ gcc/testsuite/gcc.target/aarch64/sve/adr_2_run.c | 5 ++ gcc/testsuite/gcc.target/aarch64/sve/adr_3.c | 21 +++++++++ gcc/testsuite/gcc.target/aarch64/sve/adr_3_run.c | 5 ++ gcc/testsuite/gcc.target/aarch64/sve/adr_4.c | 9 ++++ gcc/testsuite/gcc.target/aarch64/sve/adr_4_run.c | 5 ++ gcc/testsuite/gcc.target/aarch64/sve/adr_5.c | 27 +++++++++++ gcc/testsuite/gcc.target/aarch64/sve/adr_5_run.c | 32 +++++++++++++ 14 files changed, 293 insertions(+) create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_1.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_1_run.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_2.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_2_run.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_3.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_3_run.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_4.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_4_run.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_5.c create mode 100644 gcc/testsuite/gcc.target/aarch64/sve/adr_5_run.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 95da8b5..c325192 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,5 +1,11 @@ 2019-08-14 Richard Sandiford + * config/aarch64/predicates.md (const_1_to_3_operand): New predicate. + * config/aarch64/aarch64-sve.md (*aarch64_adr_uxtw) + (*aarch64_adr_shift, *aarch64_adr_shift_uxtw): New patterns. + +2019-08-14 Richard Sandiford + * config/aarch64/aarch64.c (aarch64_expand_sve_const_pred_eor) (aarch64_expand_sve_const_pred_trn): New functions. (aarch64_expand_sve_const_pred_1): Add a recurse_p parameter and diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md index 9893660..6fed4ef 100644 --- a/gcc/config/aarch64/aarch64-sve.md +++ b/gcc/config/aarch64/aarch64-sve.md @@ -61,6 +61,7 @@ ;; ---- [INT] General binary arithmetic corresponding to rtx codes ;; ---- [INT] Addition ;; ---- [INT] Subtraction +;; ---- [INT] Take address ;; ---- [INT] Absolute difference ;; ---- [INT] Multiplication ;; ---- [INT] Highpart multiplication @@ -1672,6 +1673,65 @@ ;; Merging forms are handled through SVE_INT_BINARY. ;; ------------------------------------------------------------------------- +;; ---- [INT] Take address +;; ------------------------------------------------------------------------- +;; Includes: +;; - ADR +;; ------------------------------------------------------------------------- + +;; Unshifted ADR, with the offset being zero-extended from the low 32 bits. +(define_insn "*aarch64_adr_uxtw" + [(set (match_operand:VNx2DI 0 "register_operand" "=w") + (plus:VNx2DI + (and:VNx2DI + (match_operand:VNx2DI 2 "register_operand" "w") + (match_operand:VNx2DI 3 "aarch64_sve_uxtw_immediate")) + (match_operand:VNx2DI 1 "register_operand" "w")))] + "TARGET_SVE" + "adr\t%0.d, [%1.d, %2.d, uxtw]" +) + +;; ADR with a nonzero shift. +(define_insn_and_rewrite "*aarch64_adr_shift" + [(set (match_operand:SVE_SDI 0 "register_operand" "=w") + (plus:SVE_SDI + (unspec:SVE_SDI + [(match_operand 4) + (ashift:SVE_SDI + (match_operand:SVE_SDI 2 "register_operand" "w") + (match_operand:SVE_SDI 3 "const_1_to_3_operand"))] + UNSPEC_PRED_X) + (match_operand:SVE_SDI 1 "register_operand" "w")))] + "TARGET_SVE" + "adr\t%0., [%1., %2., lsl %3]" + "&& !CONSTANT_P (operands[4])" + { + operands[4] = CONSTM1_RTX (mode); + } +) + +;; Same, but with the index being zero-extended from the low 32 bits. +(define_insn_and_rewrite "*aarch64_adr_shift_uxtw" + [(set (match_operand:VNx2DI 0 "register_operand" "=w") + (plus:VNx2DI + (unspec:VNx2DI + [(match_operand 5) + (ashift:VNx2DI + (and:VNx2DI + (match_operand:VNx2DI 2 "register_operand" "w") + (match_operand:VNx2DI 4 "aarch64_sve_uxtw_immediate")) + (match_operand:VNx2DI 3 "const_1_to_3_operand"))] + UNSPEC_PRED_X) + (match_operand:VNx2DI 1 "register_operand" "w")))] + "TARGET_SVE" + "adr\t%0.d, [%1.d, %2.d, uxtw %3]" + "&& !CONSTANT_P (operands[5])" + { + operands[5] = CONSTM1_RTX (VNx2BImode); + } +) + +;; ------------------------------------------------------------------------- ;; ---- [INT] Absolute difference ;; ------------------------------------------------------------------------- ;; Includes: diff --git a/gcc/config/aarch64/predicates.md b/gcc/config/aarch64/predicates.md index bfbe97d..466d56d 100644 --- a/gcc/config/aarch64/predicates.md +++ b/gcc/config/aarch64/predicates.md @@ -39,6 +39,13 @@ (and (match_code "const_int") (match_test "op == CONST0_RTX (mode)"))) +(define_predicate "const_1_to_3_operand" + (match_code "const_int,const_vector") +{ + op = unwrap_const_vec_duplicate (op); + return CONST_INT_P (op) && IN_RANGE (INTVAL (op), 1, 3); +}) + (define_special_predicate "subreg_lowpart_operator" (and (match_code "subreg") (match_test "subreg_lowpart_p (op)"))) @@ -595,6 +602,11 @@ (and (match_code "const,const_vector") (match_test "aarch64_sve_inc_dec_immediate_p (op)"))) +(define_predicate "aarch64_sve_uxtw_immediate" + (and (match_code "const_vector") + (match_test "GET_MODE_UNIT_BITSIZE (GET_MODE (op)) > 32") + (match_test "aarch64_const_vec_all_same_int_p (op, 0xffffffff)"))) + (define_predicate "aarch64_sve_logical_immediate" (and (match_code "const,const_vector") (match_test "aarch64_sve_bitmask_immediate_p (op)"))) diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 7820207..c33667e 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,16 @@ +2019-08-14 Richard Sandiford + + * gcc.target/aarch64/sve/adr_1.c: New test. + * gcc.target/aarch64/sve/adr_1_run.c: Likewise. + * gcc.target/aarch64/sve/adr_2.c: Likewise. + * gcc.target/aarch64/sve/adr_2_run.c: Likewise. + * gcc.target/aarch64/sve/adr_3.c: Likewise. + * gcc.target/aarch64/sve/adr_3_run.c: Likewise. + * gcc.target/aarch64/sve/adr_4.c: Likewise. + * gcc.target/aarch64/sve/adr_4_run.c: Likewise. + * gcc.target/aarch64/sve/adr_5.c: Likewise. + * gcc.target/aarch64/sve/adr_5_run.c: Likewise. + 2019-08-14 Paolo Carlini * g++.dg/conversion/simd4.C: Test locations. diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_1.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_1.c new file mode 100644 index 0000000..223351c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_1.c @@ -0,0 +1,46 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include + +#ifndef FACTOR +#define FACTOR 2 +#endif + +#define LOOP(TYPE) \ + __attribute__ ((noipa)) \ + void \ + test_##TYPE (TYPE *restrict dst, TYPE *restrict src, \ + int count) \ + { \ + for (int i = 0; i < count; ++i) \ + dst[i] += src[i] * FACTOR; \ + } + +#define TEST_ALL(T) \ + T (int8_t) \ + T (int16_t) \ + T (int32_t) \ + T (int64_t) \ + T (uint8_t) \ + T (uint16_t) \ + T (uint32_t) \ + T (uint64_t) + +TEST_ALL (LOOP) + +/* { dg-final { scan-assembler-times {\tadd\tz[0-9]\.b,} 2 } } */ +/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]\.b,} 2 } } */ +/* { dg-final { scan-assembler-not {\tadr\tz[0-9]\.b,} } } */ + +/* { dg-final { scan-assembler-times {\tadd\tz[0-9]\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]\.h,} 2 } } */ +/* { dg-final { scan-assembler-not {\tadr\tz[0-9]\.h,} } } */ + +/* { dg-final { scan-assembler-not {\tadd\tz[0-9]\.s,} } } */ +/* { dg-final { scan-assembler-not {\tlsl\tz[0-9]\.s,} } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.s, \[z[0-9]\.s, z[0-9]\.s, lsl 1\]} 2 } } */ + +/* { dg-final { scan-assembler-not {\tadd\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-not {\tlsl\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.d, \[z[0-9]\.d, z[0-9]\.d, lsl 1\]} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_1_run.c new file mode 100644 index 0000000..383a90c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_1_run.c @@ -0,0 +1,31 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include "adr_1.c" + +#define N 131 + +#define TEST_LOOP(TYPE) \ + { \ + TYPE a[N], b[N]; \ + for (int i = 0; i < N; ++i) \ + { \ + a[i] = (TYPE) i * i + i % 5; \ + b[i] = (TYPE) i * 3 + i % 7; \ + asm volatile ("" ::: "memory"); \ + } \ + test_##TYPE (a, b, N); \ + for (int i = 0; i < N; ++i) \ + { \ + TYPE expected = ((TYPE) (i * i + i % 5) \ + + ((TYPE) i * 3 + i % 7) * FACTOR); \ + if (a[i] != expected) \ + __builtin_abort (); \ + } \ + } + +int __attribute__ ((optimize (1))) +main (void) +{ + TEST_ALL (TEST_LOOP) +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_2.c new file mode 100644 index 0000000..dc20ddb --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_2.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define FACTOR 4 +#include "adr_1.c" + +/* { dg-final { scan-assembler-times {\tadd\tz[0-9]\.b,} 2 } } */ +/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]\.b,} 2 } } */ +/* { dg-final { scan-assembler-not {\tadr\tz[0-9]\.b,} } } */ + +/* { dg-final { scan-assembler-times {\tadd\tz[0-9]\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]\.h,} 2 } } */ +/* { dg-final { scan-assembler-not {\tadr\tz[0-9]\.h,} } } */ + +/* { dg-final { scan-assembler-not {\tadd\tz[0-9]\.s,} } } */ +/* { dg-final { scan-assembler-not {\tlsl\tz[0-9]\.s,} } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.s, \[z[0-9]\.s, z[0-9]\.s, lsl 2\]} 2 } } */ + +/* { dg-final { scan-assembler-not {\tadd\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-not {\tlsl\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.d, \[z[0-9]\.d, z[0-9]\.d, lsl 2\]} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_2_run.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_2_run.c new file mode 100644 index 0000000..e823d3d --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_2_run.c @@ -0,0 +1,5 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define FACTOR 4 +#include "adr_1_run.c" diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_3.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_3.c new file mode 100644 index 0000000..b0cb180 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_3.c @@ -0,0 +1,21 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define FACTOR 8 +#include "adr_1.c" + +/* { dg-final { scan-assembler-times {\tadd\tz[0-9]\.b,} 2 } } */ +/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]\.b,} 2 } } */ +/* { dg-final { scan-assembler-not {\tadr\tz[0-9]\.b,} } } */ + +/* { dg-final { scan-assembler-times {\tadd\tz[0-9]\.h,} 2 } } */ +/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]\.h,} 2 } } */ +/* { dg-final { scan-assembler-not {\tadr\tz[0-9]\.h,} } } */ + +/* { dg-final { scan-assembler-not {\tadd\tz[0-9]\.s,} } } */ +/* { dg-final { scan-assembler-not {\tlsl\tz[0-9]\.s,} } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.s, \[z[0-9]\.s, z[0-9]\.s, lsl 3\]} 2 } } */ + +/* { dg-final { scan-assembler-not {\tadd\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-not {\tlsl\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.d, \[z[0-9]\.d, z[0-9]\.d, lsl 3\]} 2 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_3_run.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_3_run.c new file mode 100644 index 0000000..721dd68 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_3_run.c @@ -0,0 +1,5 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define FACTOR 8 +#include "adr_1_run.c" diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_4.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_4.c new file mode 100644 index 0000000..7c039ba --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_4.c @@ -0,0 +1,9 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define FACTOR 16 +#include "adr_1.c" + +/* { dg-final { scan-assembler-times {\tadd\tz[0-9]\.[bhsd],} 8 } } */ +/* { dg-final { scan-assembler-times {\tlsl\tz[0-9]\.[bhsd],} 8 } } */ +/* { dg-final { scan-assembler-not {\tadr\tz[0-9]\.[bhsd],} } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_4_run.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_4_run.c new file mode 100644 index 0000000..3fb9099 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_4_run.c @@ -0,0 +1,5 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#define FACTOR 16 +#include "adr_1_run.c" diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_5.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_5.c new file mode 100644 index 0000000..ce3991c --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_5.c @@ -0,0 +1,27 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include + +#define LOOP(FACTOR) \ + __attribute__ ((noipa)) \ + void \ + test_##FACTOR (uint64_t *restrict dst, \ + uint64_t *restrict src, int count) \ + { \ + for (int i = 0; i < count; ++i) \ + dst[i] += (src[i] & 0xffffffff) * FACTOR; \ + } + +#define TEST_ALL(T) T (1) T (2) T (4) T (8) + +TEST_ALL (LOOP) + +/* { dg-final { scan-assembler-not {\tadd\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-not {\tlsl\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-not {\tand\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-not {\tuxtw\tz[0-9]\.d,} } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.d, \[z[0-9]\.d, z[0-9]\.d, uxtw\]} 1 } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.d, \[z[0-9]\.d, z[0-9]\.d, uxtw 1\]} 1 } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.d, \[z[0-9]\.d, z[0-9]\.d, uxtw 2\]} 1 } } */ +/* { dg-final { scan-assembler-times {\tadr\tz[0-9]\.d, \[z[0-9]\.d, z[0-9]\.d, uxtw 3\]} 1 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/adr_5_run.c b/gcc/testsuite/gcc.target/aarch64/sve/adr_5_run.c new file mode 100644 index 0000000..025c38d --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/adr_5_run.c @@ -0,0 +1,32 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ + +#include "adr_5.c" + +#define N 131 + +#define TEST_LOOP(FACTOR) \ + { \ + uint64_t a[N], b[N]; \ + for (int i = 0; i < N; ++i) \ + { \ + a[i] = (uint64_t) i * i + i % 5; \ + b[i] = (uint64_t) (i * 3) << ((i & 7) * 8); \ + asm volatile ("" ::: "memory"); \ + } \ + test_##FACTOR (a, b, N); \ + for (int i = 0; i < N; ++i) \ + { \ + uint64_t expected = ((uint64_t) (i * i + i % 5) \ + + (((uint64_t) (i * 3) << ((i & 7) * 8)) \ + & 0xffffffff) * FACTOR); \ + if (a[i] != expected) \ + __builtin_abort (); \ + } \ + } + +int __attribute__ ((optimize (1))) +main (void) +{ + TEST_ALL (TEST_LOOP) +} -- 2.7.4