From 26502ad609222321f6384e3317fc03165c844be1 Mon Sep 17 00:00:00 2001 From: Cullen Rhodes Date: Thu, 18 Jun 2020 17:25:18 +0000 Subject: [PATCH] [AArch64][SVE] Add bfloat16 support to perm and select intrinsics Summary: Added for following intrinsics: * zip1, zip2, zip1q, zip2q * trn1, trn2, trn1q, trn2q * uzp1, uzp2, uzp1q, uzp2q * splice * rev * sel Reviewed By: david-arm Differential Revision: https://reviews.llvm.org/D82182 --- clang/include/clang/Basic/arm_sve.td | 21 ++++++ .../aarch64-sve-intrinsics/acle_sve_rev-bfloat.c | 21 ++++++ .../aarch64-sve-intrinsics/acle_sve_sel-bfloat.c | 27 ++++++++ .../acle_sve_splice-bfloat.c | 27 ++++++++ .../aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c | 21 ++++++ .../acle_sve_trn1-fp64-bfloat.c | 20 ++++++ .../aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c | 21 ++++++ .../acle_sve_trn2-fp64-bfloat.c | 20 ++++++ .../aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c | 21 ++++++ .../acle_sve_uzp1-fp64-bfloat.c | 20 ++++++ .../aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c | 21 ++++++ .../acle_sve_uzp2-fp64-bfloat.c | 20 ++++++ .../aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c | 21 ++++++ .../acle_sve_zip1-fp64-bfloat.c | 20 ++++++ .../aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c | 21 ++++++ .../acle_sve_zip2-fp64-bfloat.c | 20 ++++++ llvm/lib/Target/AArch64/SVEInstrFormats.td | 32 +++++---- .../CodeGen/AArch64/sve-intrinsics-perm-select.ll | 80 ++++++++++++++++++++++ llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll | 11 +++ 19 files changed, 451 insertions(+), 14 deletions(-) create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c create mode 100644 clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64-bfloat.c diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 091c9e23..b6bedac 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1188,6 +1188,18 @@ def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNo def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1">; def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2">; +let ArchGuard = "defined(__ARM_FEATURE_SVE_BF16)" in { +def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev">; +def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel">; +def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice">; +def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1">; +def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2">; +def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1">; +def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2">; +def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1">; +def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2">; +} + def SVREV_B : SInst<"svrev_{d}", "PP", "PcPsPiPl", MergeNone, "aarch64_sve_rev">; def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel">; def SVTRN1_B : SInst<"svtrn1_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_trn1">; @@ -1359,6 +1371,15 @@ def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNo def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q">; } +let ArchGuard = "defined(__ARM_FEATURE_SVE_MATMUL_FP64) && defined(__ARM_FEATURE_SVE_BF16)" in { +def SVTRN1Q_BF16 : SInst<"svtrn1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1q">; +def SVTRN2Q_BF16 : SInst<"svtrn2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2q">; +def SVUZP1Q_BF16 : SInst<"svuzp1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1q">; +def SVUZP2Q_BF16 : SInst<"svuzp2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2q">; +def SVZIP1Q_BF16 : SInst<"svzip1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1q">; +def SVZIP2Q_BF16 : SInst<"svzip2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2q">; +} + //////////////////////////////////////////////////////////////////////////////// // Vector creation def SVUNDEF_1 : SInst<"svundef_{d}", "d", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef]>; diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev-bfloat.c new file mode 100644 index 0000000..b21474d --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rev-bfloat.c @@ -0,0 +1,21 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svrev_bf16(svbfloat16_t op) +{ + // CHECK-LABEL: test_svrev_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.rev.nxv8bf16( %op) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svrev_bf16'}} + return SVE_ACLE_FUNC(svrev,_bf16,,)(op); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel-bfloat.c new file mode 100644 index 0000000..3754d2d --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_sel-bfloat.c @@ -0,0 +1,27 @@ +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t +// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it. +// ASM-NOT: warning +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svsel_bf16(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svsel_bf16 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.sel.nxv8bf16( %[[PG]], %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svsel_bf16'}} + return SVE_ACLE_FUNC(svsel,_bf16,,)(pg, op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice-bfloat.c new file mode 100644 index 0000000..f6fac70 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_splice-bfloat.c @@ -0,0 +1,27 @@ +// REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null 2>%t +// RUN: FileCheck --check-prefix=ASM --allow-empty %s <%t +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +// If this check fails please read test/CodeGen/aarch64-sve-intrinsics/README for instructions on how to resolve it. +// ASM-NOT: warning +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svsplice_bf16(svbool_t pg, svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svsplice_bf16 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.splice.nxv8bf16( %[[PG]], %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svsplice_bf16'}} + return SVE_ACLE_FUNC(svsplice,_bf16,,)(pg, op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c new file mode 100644 index 0000000..ab30870 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-bfloat.c @@ -0,0 +1,21 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svtrn1_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svtrn1_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.trn1.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svtrn1_bf16'}} + return SVE_ACLE_FUNC(svtrn1,_bf16,,)(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64-bfloat.c new file mode 100644 index 0000000..36189f7 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn1-fp64-bfloat.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svtrn1_bf16(svbfloat16_t op1, svbfloat16_t op2) { + // CHECK-LABEL: test_svtrn1_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.trn1q.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svtrn1q_bf16'}} + return SVE_ACLE_FUNC(svtrn1q, _bf16, , )(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c new file mode 100644 index 0000000..cf00930 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-bfloat.c @@ -0,0 +1,21 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svtrn2_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svtrn2_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.trn2.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svtrn2_bf16'}} + return SVE_ACLE_FUNC(svtrn2,_bf16,,)(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64-bfloat.c new file mode 100644 index 0000000..34bda89 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_trn2-fp64-bfloat.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svtrn2_bf16(svbfloat16_t op1, svbfloat16_t op2) { + // CHECK-LABEL: test_svtrn2_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.trn2q.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svtrn2q_bf16'}} + return SVE_ACLE_FUNC(svtrn2q, _bf16, , )(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c new file mode 100644 index 0000000..b11472e --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-bfloat.c @@ -0,0 +1,21 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svuzp1_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svuzp1_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.uzp1.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svuzp1_bf16'}} + return SVE_ACLE_FUNC(svuzp1,_bf16,,)(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64-bfloat.c new file mode 100644 index 0000000..8c9e032 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp1-fp64-bfloat.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svuzp1_bf16(svbfloat16_t op1, svbfloat16_t op2) { + // CHECK-LABEL: test_svuzp1_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.uzp1q.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svuzp1q_bf16'}} + return SVE_ACLE_FUNC(svuzp1q, _bf16, , )(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c new file mode 100644 index 0000000..dd62539 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-bfloat.c @@ -0,0 +1,21 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svuzp2_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svuzp2_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.uzp2.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svuzp2_bf16'}} + return SVE_ACLE_FUNC(svuzp2,_bf16,,)(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64-bfloat.c new file mode 100644 index 0000000..477ec51 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_uzp2-fp64-bfloat.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svuzp2_bf16(svbfloat16_t op1, svbfloat16_t op2) { + // CHECK-LABEL: test_svuzp2_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.uzp2q.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svuzp2q_bf16'}} + return SVE_ACLE_FUNC(svuzp2q, _bf16, , )(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c new file mode 100644 index 0000000..1712860 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-bfloat.c @@ -0,0 +1,21 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svzip1_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svzip1_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.zip1.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svzip1_bf16'}} + return SVE_ACLE_FUNC(svzip1,_bf16,,)(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64-bfloat.c new file mode 100644 index 0000000..06969ff --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip1-fp64-bfloat.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svzip1_bf16(svbfloat16_t op1, svbfloat16_t op2) { + // CHECK-LABEL: test_svzip1_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.zip1q.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svzip1q_bf16'}} + return SVE_ACLE_FUNC(svzip1q, _bf16, , )(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c new file mode 100644 index 0000000..a25daa6 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-bfloat.c @@ -0,0 +1,21 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svzip2_bf16(svbfloat16_t op1, svbfloat16_t op2) +{ + // CHECK-LABEL: test_svzip2_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.zip2.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svzip2_bf16'}} + return SVE_ACLE_FUNC(svzip2,_bf16,,)(op1, op2); +} diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64-bfloat.c new file mode 100644 index 0000000..0856f06 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_zip2-fp64-bfloat.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE_MATMUL_FP64 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -D__ARM_FEATURE_SVE_BF16 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -D__ARM_FEATURE_BF16_SCALAR_ARITHMETIC -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -fallow-half-arguments-and-returns -fsyntax-only -verify -verify-ignore-unexpected=error -verify-ignore-unexpected=note %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4 +#endif + +svbfloat16_t test_svzip2_bf16(svbfloat16_t op1, svbfloat16_t op2) { + // CHECK-LABEL: test_svzip2_bf16 + // CHECK: %[[INTRINSIC:.*]] = call @llvm.aarch64.sve.zip2q.nxv8bf16( %op1, %op2) + // CHECK: ret %[[INTRINSIC]] + // expected-warning@+1 {{implicit declaration of function 'svzip2q_bf16'}} + return SVE_ACLE_FUNC(svzip2q, _bf16, , )(op1, op2); +} diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index 3dec466..46cca2a 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -1124,9 +1124,10 @@ multiclass sve_int_perm_reverse_z { def : SVE_1_Op_Pat(NAME # _S)>; def : SVE_1_Op_Pat(NAME # _D)>; - def : SVE_1_Op_Pat(NAME # _H)>; - def : SVE_1_Op_Pat(NAME # _S)>; - def : SVE_1_Op_Pat(NAME # _D)>; + def : SVE_1_Op_Pat(NAME # _H)>; + def : SVE_1_Op_Pat(NAME # _H)>; + def : SVE_1_Op_Pat(NAME # _S)>; + def : SVE_1_Op_Pat(NAME # _D)>; } class sve_int_perm_reverse_p sz8_64, string asm, PPRRegOp pprty> @@ -1320,10 +1321,11 @@ multiclass sve_int_sel_vvv { def : SVE_3_Op_Pat(NAME # _S)>; def : SVE_3_Op_Pat(NAME # _D)>; - def : SVE_3_Op_Pat(NAME # _H)>; - def : SVE_3_Op_Pat(NAME # _S)>; - def : SVE_3_Op_Pat(NAME # _D)>; - def : SVE_3_Op_Pat(NAME # _D)>; + def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _S)>; + def : SVE_3_Op_Pat(NAME # _D)>; + def : SVE_3_Op_Pat(NAME # _D)>; def : InstAlias<"mov $Zd, $Pg/m, $Zn", (!cast(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; @@ -2212,10 +2214,11 @@ multiclass sve_int_perm_bin_perm_zz opc, string asm, def : SVE_2_Op_Pat(NAME # _S)>; def : SVE_2_Op_Pat(NAME # _D)>; - def : SVE_2_Op_Pat(NAME # _H)>; - def : SVE_2_Op_Pat(NAME # _S)>; - def : SVE_2_Op_Pat(NAME # _S)>; - def : SVE_2_Op_Pat(NAME # _D)>; + def : SVE_2_Op_Pat(NAME # _H)>; + def : SVE_2_Op_Pat(NAME # _H)>; + def : SVE_2_Op_Pat(NAME # _S)>; + def : SVE_2_Op_Pat(NAME # _S)>; + def : SVE_2_Op_Pat(NAME # _D)>; } //===----------------------------------------------------------------------===// @@ -5806,9 +5809,10 @@ multiclass sve_int_perm_splice { def : SVE_3_Op_Pat(NAME # _S)>; def : SVE_3_Op_Pat(NAME # _D)>; - def : SVE_3_Op_Pat(NAME # _H)>; - def : SVE_3_Op_Pat(NAME # _S)>; - def : SVE_3_Op_Pat(NAME # _D)>; + def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _H)>; + def : SVE_3_Op_Pat(NAME # _S)>; + def : SVE_3_Op_Pat(NAME # _D)>; } class sve2_int_perm_splice_cons sz8_64, string asm, diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll index 22f87e7..ccd8080 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-perm-select.ll @@ -806,6 +806,14 @@ define @rev_i64( %a) { ret %res } +define @rev_bf16( %a) { +; CHECK-LABEL: rev_bf16 +; CHECK: rev z0.h, z0.h +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sve.rev.nxv8bf16( %a) + ret %res +} + define @rev_f16( %a) { ; CHECK-LABEL: rev_f16 ; CHECK: rev z0.h, z0.h @@ -874,6 +882,16 @@ define @splice_i64( %pg, ret %out } +define @splice_bf16( %pg, %a, %b) { +; CHECK-LABEL: splice_bf16: +; CHECK: splice z0.h, p0, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.splice.nxv8bf16( %pg, + %a, + %b) + ret %out +} + define @splice_f16( %pg, %a, %b) { ; CHECK-LABEL: splice_f16: ; CHECK: splice z0.h, p0, z0.h, z1.h @@ -1168,6 +1186,15 @@ define @trn1_f16_v4( %a, %out } +define @trn1_bf16( %a, %b) { +; CHECK-LABEL: trn1_bf16: +; CHECK: trn1 z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.trn1.nxv8bf16( %a, + %b) + ret %out +} + define @trn1_f16( %a, %b) { ; CHECK-LABEL: trn1_f16: ; CHECK: trn1 z0.h, z0.h, z1.h @@ -1280,6 +1307,15 @@ define @trn2_f16_v4( %a, %out } +define @trn2_bf16( %a, %b) { +; CHECK-LABEL: trn2_bf16: +; CHECK: trn2 z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.trn2.nxv8bf16( %a, + %b) + ret %out +} + define @trn2_f16( %a, %b) { ; CHECK-LABEL: trn2_f16: ; CHECK: trn2 z0.h, z0.h, z1.h @@ -1392,6 +1428,15 @@ define @uzp1_f16_v4( %a, %out } +define @uzp1_bf16( %a, %b) { +; CHECK-LABEL: uzp1_bf16: +; CHECK: uzp1 z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.uzp1.nxv8bf16( %a, + %b) + ret %out +} + define @uzp1_f16( %a, %b) { ; CHECK-LABEL: uzp1_f16: ; CHECK: uzp1 z0.h, z0.h, z1.h @@ -1504,6 +1549,15 @@ define @uzp2_f16_v4( %a, %out } +define @uzp2_bf16( %a, %b) { +; CHECK-LABEL: uzp2_bf16: +; CHECK: uzp2 z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.uzp2.nxv8bf16( %a, + %b) + ret %out +} + define @uzp2_f16( %a, %b) { ; CHECK-LABEL: uzp2_f16: ; CHECK: uzp2 z0.h, z0.h, z1.h @@ -1616,6 +1670,15 @@ define @zip1_f16_v4( %a, %out } +define @zip1_bf16( %a, %b) { +; CHECK-LABEL: zip1_bf16: +; CHECK: zip1 z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.zip1.nxv8bf16( %a, + %b) + ret %out +} + define @zip1_f16( %a, %b) { ; CHECK-LABEL: zip1_f16: ; CHECK: zip1 z0.h, z0.h, z1.h @@ -1728,6 +1791,15 @@ define @zip2_f16_v4( %a, %out } +define @zip2_bf16( %a, %b) { +; CHECK-LABEL: zip2_bf16: +; CHECK: zip2 z0.h, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.zip2.nxv8bf16( %a, + %b) + ret %out +} + define @zip2_f16( %a, %b) { ; CHECK-LABEL: zip2_f16: ; CHECK: zip2 z0.h, z0.h, z1.h @@ -1834,6 +1906,7 @@ declare @llvm.aarch64.sve.rev.nxv16i8() declare @llvm.aarch64.sve.rev.nxv8i16() declare @llvm.aarch64.sve.rev.nxv4i32() declare @llvm.aarch64.sve.rev.nxv2i64() +declare @llvm.aarch64.sve.rev.nxv8bf16() declare @llvm.aarch64.sve.rev.nxv8f16() declare @llvm.aarch64.sve.rev.nxv4f32() declare @llvm.aarch64.sve.rev.nxv2f64() @@ -1842,6 +1915,7 @@ declare @llvm.aarch64.sve.splice.nxv16i8(, declare @llvm.aarch64.sve.splice.nxv8i16(, , ) declare @llvm.aarch64.sve.splice.nxv4i32(, , ) declare @llvm.aarch64.sve.splice.nxv2i64(, , ) +declare @llvm.aarch64.sve.splice.nxv8bf16(, , ) declare @llvm.aarch64.sve.splice.nxv8f16(, , ) declare @llvm.aarch64.sve.splice.nxv4f32(, , ) declare @llvm.aarch64.sve.splice.nxv2f64(, , ) @@ -1879,6 +1953,7 @@ declare @llvm.aarch64.sve.trn1.nxv8i16(, @llvm.aarch64.sve.trn1.nxv4i32(, ) declare @llvm.aarch64.sve.trn1.nxv2i64(, ) declare @llvm.aarch64.sve.trn1.nxv4f16(, ) +declare @llvm.aarch64.sve.trn1.nxv8bf16(, ) declare @llvm.aarch64.sve.trn1.nxv8f16(, ) declare @llvm.aarch64.sve.trn1.nxv4f32(, ) declare @llvm.aarch64.sve.trn1.nxv2f64(, ) @@ -1892,6 +1967,7 @@ declare @llvm.aarch64.sve.trn2.nxv8i16(, @llvm.aarch64.sve.trn2.nxv4i32(, ) declare @llvm.aarch64.sve.trn2.nxv2i64(, ) declare @llvm.aarch64.sve.trn2.nxv4f16(, ) +declare @llvm.aarch64.sve.trn2.nxv8bf16(, ) declare @llvm.aarch64.sve.trn2.nxv8f16(, ) declare @llvm.aarch64.sve.trn2.nxv4f32(, ) declare @llvm.aarch64.sve.trn2.nxv2f64(, ) @@ -1905,6 +1981,7 @@ declare @llvm.aarch64.sve.uzp1.nxv8i16(, @llvm.aarch64.sve.uzp1.nxv4i32(, ) declare @llvm.aarch64.sve.uzp1.nxv2i64(, ) declare @llvm.aarch64.sve.uzp1.nxv4f16(, ) +declare @llvm.aarch64.sve.uzp1.nxv8bf16(, ) declare @llvm.aarch64.sve.uzp1.nxv8f16(, ) declare @llvm.aarch64.sve.uzp1.nxv4f32(, ) declare @llvm.aarch64.sve.uzp1.nxv2f64(, ) @@ -1918,6 +1995,7 @@ declare @llvm.aarch64.sve.uzp2.nxv8i16(, @llvm.aarch64.sve.uzp2.nxv4i32(, ) declare @llvm.aarch64.sve.uzp2.nxv2i64(, ) declare @llvm.aarch64.sve.uzp2.nxv4f16(, ) +declare @llvm.aarch64.sve.uzp2.nxv8bf16(, ) declare @llvm.aarch64.sve.uzp2.nxv8f16(, ) declare @llvm.aarch64.sve.uzp2.nxv4f32(, ) declare @llvm.aarch64.sve.uzp2.nxv2f64(, ) @@ -1931,6 +2009,7 @@ declare @llvm.aarch64.sve.zip1.nxv8i16(, @llvm.aarch64.sve.zip1.nxv4i32(, ) declare @llvm.aarch64.sve.zip1.nxv2i64(, ) declare @llvm.aarch64.sve.zip1.nxv4f16(, ) +declare @llvm.aarch64.sve.zip1.nxv8bf16(, ) declare @llvm.aarch64.sve.zip1.nxv8f16(, ) declare @llvm.aarch64.sve.zip1.nxv4f32(, ) declare @llvm.aarch64.sve.zip1.nxv2f64(, ) @@ -1944,6 +2023,7 @@ declare @llvm.aarch64.sve.zip2.nxv8i16(, @llvm.aarch64.sve.zip2.nxv4i32(, ) declare @llvm.aarch64.sve.zip2.nxv2i64(, ) declare @llvm.aarch64.sve.zip2.nxv4f16(, ) +declare @llvm.aarch64.sve.zip2.nxv8bf16(, ) declare @llvm.aarch64.sve.zip2.nxv8f16(, ) declare @llvm.aarch64.sve.zip2.nxv4f32(, ) declare @llvm.aarch64.sve.zip2.nxv2f64(, ) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll index 60e5216..dae9d33 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-sel.ll @@ -57,6 +57,16 @@ define @sel_i64( %pg, %a, ret %out } +define @sel_bf16( %pg, %a, %b) { +; CHECK-LABEL: sel_bf16: +; CHECK: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: ret + %out = call @llvm.aarch64.sve.sel.nxv8bf16( %pg, + %a, + %b) + ret %out +} + define @sel_f16( %pg, %a, %b) { ; CHECK-LABEL: sel_f16: ; CHECK: sel z0.h, p0, z0.h, z1.h @@ -92,6 +102,7 @@ declare @llvm.aarch64.sve.sel.nxv16i8(, @llvm.aarch64.sve.sel.nxv8i16(, , ) declare @llvm.aarch64.sve.sel.nxv4i32(, , ) declare @llvm.aarch64.sve.sel.nxv2i64(, , ) +declare @llvm.aarch64.sve.sel.nxv8bf16(, , ) declare @llvm.aarch64.sve.sel.nxv8f16(, , ) declare @llvm.aarch64.sve.sel.nxv4f32(, , ) declare @llvm.aarch64.sve.sel.nxv2f64(, , ) -- 2.7.4