[AArch64][SME2/SVE2p1] Add predicate-as-counter intrinsics for while*

These intrinsics are used to implement the while intrinsics that result
in a predicate-as-counter value, e.g.

  __attribute__((arm_streaming))
  svcount_t svwhilelt_c8(int64_t rn, int64_t rm, uint64_t vl)

As described in https://github.com/ARM-software/acle/pull/217

Reviewed By: hassnaa-arm

Differential Revision: https://reviews.llvm.org/D150265

diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 599917c..beb47aa 100644 (file)
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -2791,6 +2791,16 @@ let TargetPrefix = "aarch64" in {
       : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty],
                               [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
+  // While (predicate-as-counter) intrinsics
+  foreach cmp = ["ge", "gt", "hi", "hs", "le", "lo", "ls", "lt"] in {
+    foreach ty = ["c8", "c16", "c32", "c64"] in {
+      def int_aarch64_sve_while # cmp # _ # ty
+          : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty],
+                                  [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty],
+                                  [IntrNoMem, ImmArg<ArgIndex<2>>]>;
+    }
+  }
+
   //
   // SME2 Intrinsics
   //

diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 4fd6e56..2358161 100644 (file)
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -512,6 +512,12 @@ multiclass SVE2p1_Cntp_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
   def : Pat<(vtd (op vt1:$Op1, (i32 4))), (inst $Op1, 1)>;
 }
 
+multiclass SVE2p1_While_PN_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
+                               Instruction inst> {
+  def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 2))), (inst $Op1, $Op2, 0)>;
+  def : Pat<(vtd (op vt1:$Op1, vt1:$Op2, (i32 4))), (inst $Op1, $Op2, 1)>;
+}
+
 class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                        ValueType vt2, ValueType vt3, Operand ImmTy,
                        Instruction inst>
@@ -9590,6 +9596,15 @@ multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc> {
  def _H : sve2p1_int_while_rr_pn<mnemonic, 0b01, opc, PNR16_p8to15>;
  def _S : sve2p1_int_while_rr_pn<mnemonic, 0b10, opc, PNR32_p8to15>;
  def _D : sve2p1_int_while_rr_pn<mnemonic, 0b11, opc, PNR64_p8to15>;
+
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c8"),
+                            i64, !cast<Instruction>(NAME # _B)>;
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c16"),
+                            i64, !cast<Instruction>(NAME # _H)>;
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c32"),
+                            i64, !cast<Instruction>(NAME # _S)>;
+ defm : SVE2p1_While_PN_Pat<aarch64svcount, !cast<SDPatternOperator>("int_aarch64_sve_" # mnemonic # "_c64"),
+                            i64, !cast<Instruction>(NAME # _D)>;
 }
 
 

diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
new file mode 100644 (file)
index 0000000..aae0925
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
@@ -0,0 +1,717 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 < %s | FileCheck %s
+
+
+;
+; WHILEGE
+;
+
+define target("aarch64.svcount") @whilege_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilege_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilege_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILEGT
+;
+
+define target("aarch64.svcount") @whilegt_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilegt_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilegt_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILEHI
+;
+
+define target("aarch64.svcount") @whilehi_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehi_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehi_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILEHS
+;
+
+define target("aarch64.svcount") @whilehs_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilehs_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilehs_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELE
+;
+
+define target("aarch64.svcount") @whilele_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilele_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilele_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELO
+;
+
+define target("aarch64.svcount") @whilelo_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelo_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelo_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELS
+;
+
+define target("aarch64.svcount") @whilels_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilels_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilels_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+;
+; WHILELT
+;
+
+define target("aarch64.svcount") @whilelt_c8_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c8_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.b, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c8_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c8_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c16_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c16_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.h, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c16_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c16_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c32_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c32_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.s, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c32_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c32_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c64_vl2(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c64_vl2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.d, x0, x1, vlx2
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 %a, i64 %b, i32 2)
+  ret target("aarch64.svcount") %out
+}
+
+define target("aarch64.svcount") @whilelt_c64_vl4(i64 %a, i64 %b) nounwind {
+; CHECK-LABEL: whilelt_c64_vl4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    mov p0.b, p8.b
+; CHECK-NEXT:    ret
+  %out = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 %a, i64 %b, i32 4)
+  ret target("aarch64.svcount") %out
+}
+
+
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64, i64, i32)
+declare target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64, i64, i32)