[ARM] Generate 8.1-m CSINC, CSNEG and CSINV instructions.

Arm 8.1-M adds a number of related CSEL instructions, including CSINC, CSNEG and CSINV. These choose between two values given the content in CPSR and a condition, performing an increment, negation or inverse of the false value.

This adds some selection for them, either from constant values or patterns. It does not include CSEL directly, which is currently not always making code better. It is still useful, but we will have to check more carefully where it should and shouldn't be used.

Code by Ranjeet Singh and Simon Tatham, with some modifications from me.

Differential revision: https://reviews.llvm.org/D66483

llvm-svn: 370739

diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 680729d..54ddb94 100644 (file)
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -3040,18 +3040,22 @@ bool ARMBaseInstrInfo::optimizeCompareInstr(
         break;
       case ARM::VSELEQD:
       case ARM::VSELEQS:
+      case ARM::VSELEQH:
         CC = ARMCC::EQ;
         break;
       case ARM::VSELGTD:
       case ARM::VSELGTS:
+      case ARM::VSELGTH:
         CC = ARMCC::GT;
         break;
       case ARM::VSELGED:
       case ARM::VSELGES:
+      case ARM::VSELGEH:
         CC = ARMCC::GE;
         break;
-      case ARM::VSELVSS:
       case ARM::VSELVSD:
+      case ARM::VSELVSS:
+      case ARM::VSELVSH:
         CC = ARMCC::VS;
         break;
       }

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 8a4de79..0239d6a 100644 (file)
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -1634,6 +1634,9 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::WLS:           return "ARMISD::WLS";
   case ARMISD::LE:            return "ARMISD::LE";
   case ARMISD::LOOP_DEC:      return "ARMISD::LOOP_DEC";
+  case ARMISD::CSINV:         return "ARMISD::CSINV";
+  case ARMISD::CSNEG:         return "ARMISD::CSNEG";
+  case ARMISD::CSINC:         return "ARMISD::CSINC";
   }
   return nullptr;
 }
@@ -4815,6 +4818,49 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
   SDValue TrueVal = Op.getOperand(2);
   SDValue FalseVal = Op.getOperand(3);
+  ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
+  ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
+
+  if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
+      LHS.getValueType() == MVT::i32 && RHS.getValueType() == MVT::i32) {
+    unsigned TVal = CTVal->getZExtValue();
+    unsigned FVal = CFVal->getZExtValue();
+    unsigned Opcode = 0;
+
+    if (TVal == ~FVal) {
+      Opcode = ARMISD::CSINV;
+    } else if (TVal == ~FVal + 1) {
+      Opcode = ARMISD::CSNEG;
+    } else if (TVal + 1 == FVal) {
+      Opcode = ARMISD::CSINC;
+    } else if (TVal == FVal + 1) {
+      Opcode = ARMISD::CSINC;
+      std::swap(TrueVal, FalseVal);
+      std::swap(TVal, FVal);
+      CC = ISD::getSetCCInverse(CC, true);
+    }
+
+    if (Opcode) {
+      // Attempt to use ZR checking TVal is 0, possibly inverting the condition
+      // to get there. CSINC not is invertable like the other two (~(~a) == a,
+      // -(-a) == a, but (a+1)+1 != a).
+      if (FVal == 0 && Opcode != ARMISD::CSINC) {
+        std::swap(TrueVal, FalseVal);
+        std::swap(TVal, FVal);
+        CC = ISD::getSetCCInverse(CC, true);
+      }
+      if (TVal == 0)
+        TrueVal = DAG.getRegister(ARM::ZR, MVT::i32);
+
+      // Drops F's value because we can get it by inverting/negating TVal.
+      FalseVal = TrueVal;
+
+      SDValue ARMcc;
+      SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
+      EVT VT = TrueVal.getValueType();
+      return DAG.getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
+    }
+  }
 
   if (isUnsupportedFloatingType(LHS.getValueType())) {
     DAG.getTargetLoweringInfo().softenSetCCOperands(

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index d84a235..d7586bc 100644 (file)
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -240,6 +240,11 @@ class VectorType;
       // instructions.
       MEMCPY,
 
+      // V8.1MMainline condition select
+      CSINV, // Conditional select invert.
+      CSNEG, // Conditional select negate.
+      CSINC, // Conditional select increment.
+
       // Vector load N-element structure to all lanes:
       VLD1DUP = ISD::FIRST_TARGET_MEMORY_OPCODE,
       VLD2DUP,

diff --git a/llvm/lib/Target/ARM/ARMInstrFormats.td b/llvm/lib/Target/ARM/ARMInstrFormats.td
index 8885b73..043b54a 100644 (file)
--- a/llvm/lib/Target/ARM/ARMInstrFormats.td
+++ b/llvm/lib/Target/ARM/ARMInstrFormats.td
@@ -188,6 +188,13 @@ def s_cc_out : OptionalDefOperand<OtherVT, (ops CCR), (ops (i32 CPSR))> {
   let DecoderMethod = "DecodeCCOutOperand";
 }
 
+// Transform to generate the inverse of a condition code during ISel
+def inv_cond_XFORM : SDNodeXForm<imm, [{
+  ARMCC::CondCodes CC = static_cast<ARMCC::CondCodes>(N->getZExtValue());
+  return CurDAG->getTargetConstant(ARMCC::getOppositeCondition(CC), SDLoc(N),
+                                   MVT::i32);
+}]>;
+
 // VPT predicate
 
 def VPTPredNOperand : AsmOperandClass {

diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index ac1be46..5396ffb 100644 (file)
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -116,6 +116,16 @@ def ARMSmlaldx       : SDNode<"ARMISD::SMLALDX", SDT_LongMac>;
 def ARMSmlsld        : SDNode<"ARMISD::SMLSLD", SDT_LongMac>;
 def ARMSmlsldx       : SDNode<"ARMISD::SMLSLDX", SDT_LongMac>;
 
+def SDT_ARMCSel      : SDTypeProfile<1, 3,
+                                   [SDTCisSameAs<0, 1>,
+                                    SDTCisSameAs<0, 2>,
+                                    SDTCisInt<3>,
+                                    SDTCisVT<3, i32>]>;
+
+def ARMcsinv         : SDNode<"ARMISD::CSINV", SDT_ARMCSel, [SDNPOptInGlue]>;
+def ARMcsneg         : SDNode<"ARMISD::CSNEG", SDT_ARMCSel, [SDNPOptInGlue]>;
+def ARMcsinc         : SDNode<"ARMISD::CSINC", SDT_ARMCSel, [SDNPOptInGlue]>;
+
 def SDT_MulHSR       : SDTypeProfile<1, 3, [SDTCisVT<0,i32>,
                                             SDTCisSameAs<0, 1>,
                                             SDTCisSameAs<0, 2>,

diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index 7e74a0a..f8cea50 100644 (file)
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -5257,6 +5257,25 @@ def t2CSINC : CS<"csinc", 0b1001>;
 def t2CSINV : CS<"csinv", 0b1010>;
 def t2CSNEG : CS<"csneg", 0b1011>;
 
+let Predicates = [HasV8_1MMainline] in {
+  def : T2Pat<(ARMcsinc GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm),
+              (t2CSINC GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+  def : T2Pat<(ARMcsinv GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm),
+              (t2CSINV GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+  def : T2Pat<(ARMcsneg GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm),
+              (t2CSNEG GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+
+  multiclass ModifiedV8_1CSEL<Instruction Insn, dag modvalue> {
+    def : T2Pat<(ARMcmov modvalue, GPRwithZR:$tval, cmovpred:$imm),
+                (Insn GPRwithZR:$tval, GPRwithZR:$fval, imm0_31:$imm)>;
+    def : T2Pat<(ARMcmov GPRwithZR:$tval, modvalue, cmovpred:$imm),
+                (Insn GPRwithZR:$tval, GPRwithZR:$fval,
+                         (i32 (inv_cond_XFORM imm:$imm)))>;
+  }
+  defm : ModifiedV8_1CSEL<t2CSINC, (add rGPR:$fval, 1)>;
+  defm : ModifiedV8_1CSEL<t2CSINV, (xor rGPR:$fval, -1)>;
+  defm : ModifiedV8_1CSEL<t2CSNEG, (sub 0, rGPR:$fval)>;
+}
 
 // CS aliases.
 let Predicates = [HasV8_1MMainline] in {

diff --git a/llvm/test/CodeGen/Thumb2/csel.ll b/llvm/test/CodeGen/Thumb2/csel.ll
index 256dd18..17a1112 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/csel.ll
+++ b/llvm/test/CodeGen/Thumb2/csel.ll
@@ -6,9 +6,7 @@ define i32 @csinc_const_65(i32 %a) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    movs r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #6
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, r1, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -19,11 +17,9 @@ entry:
 define i32 @csinc_const_56(i32 %a) {
 ; CHECK-LABEL: csinc_const_56:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #6
+; CHECK-NEXT:    movs r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #5
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -34,11 +30,8 @@ entry:
 define i32 @csinc_const_zext(i32 %a) {
 ; CHECK-LABEL: csinc_const_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, zr, zr, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -49,11 +42,9 @@ entry:
 define i32 @csinv_const_56(i32 %a) {
 ; CHECK-LABEL: csinv_const_56:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #5
+; CHECK-NEXT:    mvn r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    mvngt r1, #5
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -64,11 +55,9 @@ entry:
 define i32 @csinv_const_65(i32 %a) {
 ; CHECK-LABEL: csinv_const_65:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    mvn r1, #5
+; CHECK-NEXT:    movs r1, #5
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #5
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -79,11 +68,8 @@ entry:
 define i32 @csinv_const_sext(i32 %a) {
 ; CHECK-LABEL: csinv_const_sext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, zr, zr, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -94,11 +80,9 @@ entry:
 define i32 @csneg_const(i32 %a) {
 ; CHECK-LABEL: csneg_const:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    mov.w r1, #-1
+; CHECK-NEXT:    movs r1, #1
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csneg r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -109,11 +93,9 @@ entry:
 define i32 @csneg_const_r(i32 %a) {
 ; CHECK-LABEL: csneg_const_r:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    mov.w r1, #-1
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt.w r1, #-1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csneg r0, r1, r1, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -139,9 +121,7 @@ define i32 @csinc_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinc_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it le
-; CHECK-NEXT:    addle r1, r2, #1
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinc r0, r1, r2, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -154,9 +134,7 @@ define i32 @csinc_swap_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinc_swap_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    addgt r2, r1, #1
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csinc r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -169,9 +147,7 @@ define i32 @csinv_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinv_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it le
-; CHECK-NEXT:    mvnle r1, r2
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csinv r0, r1, r2, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -184,9 +160,7 @@ define i32 @csinv_swap_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csinv_swap_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    mvngt r2, r1
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csinv r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -199,9 +173,7 @@ define i32 @csneg_var(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_var:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it le
-; CHECK-NEXT:    rsble r1, r2, #0
-; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    csneg r0, r1, r2, gt
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -214,9 +186,7 @@ define i32 @csneg_swap_var_sgt(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_sgt:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    rsbgt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %a, 45
@@ -229,9 +199,7 @@ define i32 @csneg_swap_var_sge(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_sge:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #44
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    rsbgt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sge i32 %a, 45
@@ -244,9 +212,7 @@ define i32 @csneg_swap_var_sle(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_sle:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #46
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    rsblt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ge
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sle i32 %a, 45
@@ -259,9 +225,7 @@ define i32 @csneg_swap_var_slt(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_slt:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    rsblt r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ge
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp slt i32 %a, 45
@@ -274,9 +238,7 @@ define i32 @csneg_swap_var_ugt(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ugt:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    rsbhi r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ls
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ugt i32 %a, 45
@@ -289,9 +251,7 @@ define i32 @csneg_swap_var_uge(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_uge:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #44
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    rsbhi r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ls
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp uge i32 %a, 45
@@ -304,9 +264,7 @@ define i32 @csneg_swap_var_ule(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ule:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #46
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    rsblo r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, hs
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ule i32 %a, 45
@@ -319,9 +277,7 @@ define i32 @csneg_swap_var_ult(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ult:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    rsblo r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, hs
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ult i32 %a, 45
@@ -334,9 +290,7 @@ define i32 @csneg_swap_var_ne(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_ne:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    rsbeq r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, ne
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp eq i32 %a, 45
@@ -349,9 +303,7 @@ define i32 @csneg_swap_var_eq(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: csneg_swap_var_eq:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, #45
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    rsbne r2, r1, #0
-; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    csneg r0, r2, r1, eq
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp ne i32 %a, 45
@@ -364,8 +316,7 @@ define i32 @csinc_inplace(i32 %a, i32 %b) {
 ; CHECK-LABEL: csinc_inplace:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r1, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    addgt r0, #1
+; CHECK-NEXT:    csinc r0, r0, r0, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %b, 45
@@ -378,8 +329,7 @@ define i32 @csinv_inplace(i32 %a, i32 %b) {
 ; CHECK-LABEL: csinv_inplace:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r1, #45
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    mvngt r0, r0
+; CHECK-NEXT:    csinv r0, r0, r0, le
 ; CHECK-NEXT:    bx lr
 entry:
   %cmp = icmp sgt i32 %b, 45

diff --git a/llvm/test/CodeGen/Thumb2/mve-abs.ll b/llvm/test/CodeGen/Thumb2/mve-abs.ll
index 081157b..df89d3e 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-abs.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-abs.ll
@@ -40,39 +40,36 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @abs_v2i64(<2 x i64> %s1) {
 ; CHECK-LABEL: abs_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    vmov r12, s2
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vmov r0, s3
-; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    vmov r4, s0
-; CHECK-NEXT:    rsbs.w r3, r12, #0
-; CHECK-NEXT:    sbc.w lr, r2, r0
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    vmov lr, s4
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    rsbs.w r3, lr, #0
+; CHECK-NEXT:    sbc.w r2, r12, r0
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq lr, r0
-; CHECK-NEXT:    vmov r0, s1
-; CHECK-NEXT:    rsbs r5, r4, #0
-; CHECK-NEXT:    sbc.w r6, r2, r0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r2, #1
-; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    csinc r1, zr, zr, pl
+; CHECK-NEXT:    ands r1, r1, #1
 ; CHECK-NEXT:    itt eq
-; CHECK-NEXT:    moveq r6, r0
-; CHECK-NEXT:    moveq r5, r4
-; CHECK-NEXT:    vmov.32 q0[0], r5
-; CHECK-NEXT:    vmov.32 q0[1], r6
-; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    moveq r2, r0
+; CHECK-NEXT:    moveq r3, lr
+; CHECK-NEXT:    vmov lr, s6
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    rsbs.w r2, lr, #0
+; CHECK-NEXT:    sbc.w r3, r12, r0
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinc r1, zr, zr, pl
+; CHECK-NEXT:    ands r1, r1, #1
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    moveq r2, lr
+; CHECK-NEXT:    vmov.32 q0[2], r2
 ; CHECK-NEXT:    it eq
-; CHECK-NEXT:    moveq r3, r12
-; CHECK-NEXT:    vmov.32 q0[2], r3
-; CHECK-NEXT:    vmov.32 q0[3], lr
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    moveq r3, r0
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = icmp slt <2 x i64> %s1, zeroinitializer
   %1 = sub nsw <2 x i64> zeroinitializer, %s1

diff --git a/llvm/test/CodeGen/Thumb2/mve-fmath.ll b/llvm/test/CodeGen/Thumb2/mve-fmath.ll
index dc60f71..d275b5d 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-fmath.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fmath.ll
@@ -1326,54 +1326,49 @@ define arm_aapcs_vfpcc <8 x half> @copysign_float16_t(<8 x half> %src1, <8 x hal
 ; CHECK-NEXT:    ldrb.w r0, [sp, #29]
 ; CHECK-NEXT:    vabs.f16 s4, s0
 ; CHECK-NEXT:    vneg.f16 s6, s4
+; CHECK-NEXT:    ldrb.w r1, [sp, #25]
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vabs.f16 s8, s1
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s4, s4, s6
-; CHECK-NEXT:    ldrb.w r1, [sp, #25]
+; CHECK-NEXT:    tst.w r1, #128
 ; CHECK-NEXT:    vmov r0, s4
 ; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    ands r1, r1, #128
+; CHECK-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-NEXT:    vabs.f16 s4, s4
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #1
 ; CHECK-NEXT:    vneg.f16 s6, s4
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    vmovx.f16 s0, s3
+; CHECK-NEXT:    lsls r1, r1, #31
 ; CHECK-NEXT:    vseleq.f16 s4, s4, s6
-; CHECK-NEXT:    vabs.f16 s0, s0
+; CHECK-NEXT:    vmovx.f16 s0, s3
 ; CHECK-NEXT:    vmov r1, s4
 ; CHECK-NEXT:    vmov.16 q1[0], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #21]
 ; CHECK-NEXT:    vmov.16 q1[1], r1
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vabs.f16 s0, s0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmovx.f16 s8, s1
 ; CHECK-NEXT:    vmov.16 q1[2], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #17]
 ; CHECK-NEXT:    vabs.f16 s8, s8
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vabs.f16 s8, s2
 ; CHECK-NEXT:    vmov.16 q1[3], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #13]
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmovx.f16 s8, s2
@@ -1381,29 +1376,26 @@ define arm_aapcs_vfpcc <8 x half> @copysign_float16_t(<8 x half> %src1, <8 x hal
 ; CHECK-NEXT:    ldrb.w r0, [sp, #9]
 ; CHECK-NEXT:    vabs.f16 s8, s8
 ; CHECK-NEXT:    vneg.f16 s2, s0
-; CHECK-NEXT:    ands r0, r0, #128
+; CHECK-NEXT:    tst.w r0, #128
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vabs.f16 s8, s3
 ; CHECK-NEXT:    vmov.16 q1[5], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #5]
 ; CHECK-NEXT:    vneg.f16 s10, s8
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s8, s8, s10
 ; CHECK-NEXT:    vmov r0, s8
 ; CHECK-NEXT:    vmov.16 q1[6], r0
 ; CHECK-NEXT:    ldrb.w r0, [sp, #1]
-; CHECK-NEXT:    ands r0, r0, #128
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    tst.w r0, #128
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    lsls r0, r0, #31
 ; CHECK-NEXT:    vseleq.f16 s0, s0, s2
 ; CHECK-NEXT:    vmov r0, s0
 ; CHECK-NEXT:    vmov.16 q1[7], r0

diff --git a/llvm/test/CodeGen/Thumb2/mve-minmax.ll b/llvm/test/CodeGen/Thumb2/mve-minmax.ll
index 3a79cdf..35e9f12 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-minmax.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-minmax.ll
@@ -55,15 +55,13 @@ define arm_aapcs_vfpcc <2 x i64> @smin_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -131,15 +129,13 @@ define arm_aapcs_vfpcc <2 x i64> @umin_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -208,15 +204,13 @@ define arm_aapcs_vfpcc <2 x i64> @smax_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -284,15 +278,13 @@ define arm_aapcs_vfpcc <2 x i64> @umax_v2i64(<2 x i64> %s1, <2 x i64> %s2) {
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    subs r2, r3, r2
 ; CHECK-NEXT:    sbcs.w r2, lr, r12
 ; CHECK-NEXT:    it lo
 ; CHECK-NEXT:    movlo r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov.32 q2[2], r1
@@ -388,22 +380,20 @@ define arm_aapcs_vfpcc <2 x double> @maxnm_float64_t(<2 x double> %src1, <2 x do
 ; CHECK-NEXT:    vmov r0, r1, d9
 ; CHECK-NEXT:    vmov r2, r3, d11
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
-; CHECK-NEXT:    mov r4, r0
-; CHECK-NEXT:    vmov r0, r1, d8
+; CHECK-NEXT:    vmov r12, r1, d8
+; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    vmov r2, r3, d10
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r4, #1
-; CHECK-NEXT:    cmp r4, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
+; CHECK-NEXT:    movne r0, #1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    csinv r4, zr, zr, eq
+; CHECK-NEXT:    mov r0, r12
 ; CHECK-NEXT:    bl __aeabi_dcmpgt
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q0[0], r0
 ; CHECK-NEXT:    vmov.32 q0[1], r0
 ; CHECK-NEXT:    vmov.32 q0[2], r4

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-and.ll b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
index 7a20141..a9b31c7 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
@@ -612,18 +612,16 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s8
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s10
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q1, q1, q3
@@ -650,10 +648,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
@@ -662,27 +659,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vand q2, q2, q3
@@ -706,10 +700,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqr_v2i1(<2 x i64> %a, <2 x i64> %b, i64 %c
 ; CHECK-NEXT:    eors r2, r1
 ; CHECK-NEXT:    eors r3, r0
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    clz r2, r2
-; CHECK-NEXT:    lsrs r2, r2, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    csinc r2, zr, zr, ne
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csinv r2, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r2
 ; CHECK-NEXT:    vmov.32 q2[1], r2
 ; CHECK-NEXT:    vmov r2, s7
@@ -718,27 +711,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqr_v2i1(<2 x i64> %a, <2 x i64> %b, i64 %c
 ; CHECK-NEXT:    eors r0, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vand q2, q3, q2

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
index 03aef30..af8cdac 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
@@ -155,11 +155,10 @@ define arm_aapcs_vfpcc i2 @bitcast_from_v2i1(<2 x i64> %a) {
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
+; CHECK-NEXT:    csinc r0, zr, zr, ne
 ; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    clz r1, r1
-; CHECK-NEXT:    lsrs r1, r1, #5
+; CHECK-NEXT:    csinc r1, zr, zr, ne
+; CHECK-NEXT:    ands r1, r1, #1
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    mvnne r1, #1
 ; CHECK-NEXT:    bfi r1, r0, #0, #1

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll b/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
index b8b518c..5be074d 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
@@ -6,13 +6,12 @@ define arm_aapcs_vfpcc <4 x i32> @build_var0_v4i1(i32 %s, i32 %t, <4 x i32> %a,
 ; CHECK-LABEL: build_var0_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #4
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #4
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -26,13 +25,12 @@ define arm_aapcs_vfpcc <4 x i32> @build_var3_v4i1(i32 %s, i32 %t, <4 x i32> %a,
 ; CHECK-LABEL: build_var3_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #12, #4
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #12, #4
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -46,16 +44,15 @@ define arm_aapcs_vfpcc <4 x i32> @build_varN_v4i1(i32 %s, i32 %t, <4 x i32> %a,
 ; CHECK-LABEL: build_varN_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #4
-; CHECK-NEXT:    bfi r2, r0, #4, #4
-; CHECK-NEXT:    bfi r2, r0, #8, #4
-; CHECK-NEXT:    bfi r2, r0, #12, #4
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #4
+; CHECK-NEXT:    bfi r1, r0, #4, #4
+; CHECK-NEXT:    bfi r1, r0, #8, #4
+; CHECK-NEXT:    bfi r1, r0, #12, #4
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -71,13 +68,12 @@ define arm_aapcs_vfpcc <8 x i16> @build_var0_v8i1(i32 %s, i32 %t, <8 x i16> %a,
 ; CHECK-LABEL: build_var0_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #2
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #2
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -91,13 +87,12 @@ define arm_aapcs_vfpcc <8 x i16> @build_var3_v8i1(i32 %s, i32 %t, <8 x i16> %a,
 ; CHECK-LABEL: build_var3_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #6, #2
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #6, #2
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -111,20 +106,19 @@ define arm_aapcs_vfpcc <8 x i16> @build_varN_v8i1(i32 %s, i32 %t, <8 x i16> %a,
 ; CHECK-LABEL: build_varN_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #2
-; CHECK-NEXT:    bfi r2, r0, #2, #2
-; CHECK-NEXT:    bfi r2, r0, #4, #2
-; CHECK-NEXT:    bfi r2, r0, #6, #2
-; CHECK-NEXT:    bfi r2, r0, #8, #2
-; CHECK-NEXT:    bfi r2, r0, #10, #2
-; CHECK-NEXT:    bfi r2, r0, #12, #2
-; CHECK-NEXT:    bfi r2, r0, #14, #2
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #2
+; CHECK-NEXT:    bfi r1, r0, #2, #2
+; CHECK-NEXT:    bfi r1, r0, #4, #2
+; CHECK-NEXT:    bfi r1, r0, #6, #2
+; CHECK-NEXT:    bfi r1, r0, #8, #2
+; CHECK-NEXT:    bfi r1, r0, #10, #2
+; CHECK-NEXT:    bfi r1, r0, #12, #2
+; CHECK-NEXT:    bfi r1, r0, #14, #2
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -140,13 +134,12 @@ define arm_aapcs_vfpcc <16 x i8> @build_var0_v16i1(i32 %s, i32 %t, <16 x i8> %a,
 ; CHECK-LABEL: build_var0_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #1
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #1
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -160,13 +153,12 @@ define arm_aapcs_vfpcc <16 x i8> @build_var3_v16i1(i32 %s, i32 %t, <16 x i8> %a,
 ; CHECK-LABEL: build_var3_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #3, #1
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #3, #1
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -180,28 +172,27 @@ define arm_aapcs_vfpcc <16 x i8> @build_varN_v16i1(i32 %s, i32 %t, <16 x i8> %a,
 ; CHECK-LABEL: build_varN_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r2, r0, #0, #1
-; CHECK-NEXT:    bfi r2, r0, #1, #1
-; CHECK-NEXT:    bfi r2, r0, #2, #1
-; CHECK-NEXT:    bfi r2, r0, #3, #1
-; CHECK-NEXT:    bfi r2, r0, #4, #1
-; CHECK-NEXT:    bfi r2, r0, #5, #1
-; CHECK-NEXT:    bfi r2, r0, #6, #1
-; CHECK-NEXT:    bfi r2, r0, #7, #1
-; CHECK-NEXT:    bfi r2, r0, #8, #1
-; CHECK-NEXT:    bfi r2, r0, #9, #1
-; CHECK-NEXT:    bfi r2, r0, #10, #1
-; CHECK-NEXT:    bfi r2, r0, #11, #1
-; CHECK-NEXT:    bfi r2, r0, #12, #1
-; CHECK-NEXT:    bfi r2, r0, #13, #1
-; CHECK-NEXT:    bfi r2, r0, #14, #1
-; CHECK-NEXT:    bfi r2, r0, #15, #1
-; CHECK-NEXT:    vmsr p0, r2
+; CHECK-NEXT:    bfi r1, r0, #0, #1
+; CHECK-NEXT:    bfi r1, r0, #1, #1
+; CHECK-NEXT:    bfi r1, r0, #2, #1
+; CHECK-NEXT:    bfi r1, r0, #3, #1
+; CHECK-NEXT:    bfi r1, r0, #4, #1
+; CHECK-NEXT:    bfi r1, r0, #5, #1
+; CHECK-NEXT:    bfi r1, r0, #6, #1
+; CHECK-NEXT:    bfi r1, r0, #7, #1
+; CHECK-NEXT:    bfi r1, r0, #8, #1
+; CHECK-NEXT:    bfi r1, r0, #9, #1
+; CHECK-NEXT:    bfi r1, r0, #10, #1
+; CHECK-NEXT:    bfi r1, r0, #11, #1
+; CHECK-NEXT:    bfi r1, r0, #12, #1
+; CHECK-NEXT:    bfi r1, r0, #13, #1
+; CHECK-NEXT:    bfi r1, r0, #14, #1
+; CHECK-NEXT:    bfi r1, r0, #15, #1
+; CHECK-NEXT:    vmsr p0, r1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -216,11 +207,10 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @build_var0_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: build_var0_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r2, #1
-; CHECK-NEXT:    rsbs r0, r2, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    vmov s8, r0
 ; CHECK-NEXT:    vldr s10, .LCPI9_0
 ; CHECK-NEXT:    vmov.f32 s9, s8
@@ -243,11 +233,10 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @build_var1_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: build_var1_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r2, #1
-; CHECK-NEXT:    rsbs r0, r2, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    vmov s10, r0
 ; CHECK-NEXT:    vldr s8, .LCPI10_0
 ; CHECK-NEXT:    vmov.f32 s9, s8
@@ -270,11 +259,10 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @build_varN_v2i1(i32 %s, i32 %t, <2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: build_varN_v2i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r2, #1
-; CHECK-NEXT:    rsbs r0, r2, #0
+; CHECK-NEXT:    csinc r0, zr, zr, hs
+; CHECK-NEXT:    and r0, r0, #1
+; CHECK-NEXT:    rsbs r0, r0, #0
 ; CHECK-NEXT:    vdup.32 q2, r0
 ; CHECK-NEXT:    vbic q1, q1, q2
 ; CHECK-NEXT:    vand q0, q0, q2

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
index 387c842..bf3f64b 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-ext.ll
@@ -57,17 +57,15 @@ define arm_aapcs_vfpcc <2 x i64> @sext_v2i1_v2i64(<2 x i64> %src) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    sbcs.w r1, r2, r1
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r2, #1
 ; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    vmov.32 q0[0], r2
-; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    csinv r1, zr, zr, eq
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[1], r1
 ; CHECK-NEXT:    vmov.32 q0[2], r0
 ; CHECK-NEXT:    vmov.32 q0[3], r0
 ; CHECK-NEXT:    bx lr
@@ -136,15 +134,13 @@ define arm_aapcs_vfpcc <2 x i64> @zext_v2i1_v2i64(<2 x i64> %src) {
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    sbcs.w r2, r0, r2
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r0, #1
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q0[0], r0
 ; CHECK-NEXT:    vmov.32 q0[2], r1
 ; CHECK-NEXT:    vand q0, q0, q1

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll b/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
index 036c7c0..27d8d07 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
@@ -167,11 +167,10 @@ define arm_aapcs_vfpcc void @store_v2i1(<2 x i1> *%dst, <2 x i64> %a) {
 ; CHECK-LE-NEXT:    vmov r3, s2
 ; CHECK-LE-NEXT:    orrs r1, r2
 ; CHECK-LE-NEXT:    vmov r2, s3
-; CHECK-LE-NEXT:    clz r1, r1
-; CHECK-LE-NEXT:    lsrs r1, r1, #5
+; CHECK-LE-NEXT:    csinc r1, zr, zr, ne
 ; CHECK-LE-NEXT:    orrs r2, r3
-; CHECK-LE-NEXT:    clz r2, r2
-; CHECK-LE-NEXT:    lsrs r2, r2, #5
+; CHECK-LE-NEXT:    csinc r2, zr, zr, ne
+; CHECK-LE-NEXT:    ands r2, r2, #1
 ; CHECK-LE-NEXT:    it ne
 ; CHECK-LE-NEXT:    mvnne r2, #1
 ; CHECK-LE-NEXT:    bfi r2, r1, #0, #1
@@ -187,11 +186,10 @@ define arm_aapcs_vfpcc void @store_v2i1(<2 x i1> *%dst, <2 x i64> %a) {
 ; CHECK-BE-NEXT:    vmov r3, s5
 ; CHECK-BE-NEXT:    orrs r1, r2
 ; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    clz r1, r1
-; CHECK-BE-NEXT:    lsrs r1, r1, #5
+; CHECK-BE-NEXT:    csinc r1, zr, zr, ne
 ; CHECK-BE-NEXT:    orrs r2, r3
-; CHECK-BE-NEXT:    clz r2, r2
-; CHECK-BE-NEXT:    lsrs r2, r2, #5
+; CHECK-BE-NEXT:    csinc r2, zr, zr, ne
+; CHECK-BE-NEXT:    ands r2, r2, #1
 ; CHECK-BE-NEXT:    it ne
 ; CHECK-BE-NEXT:    mvnne r2, #1
 ; CHECK-BE-NEXT:    bfi r2, r1, #0, #1

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-not.ll b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
index 0b950ad..32b3b80 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
@@ -327,18 +327,16 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vbic q0, q0, q2
@@ -359,18 +357,16 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vbic q0, q0, q2

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
index 2b4aaaf..df228a9 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
@@ -425,36 +425,32 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s4
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s6
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s7
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vorr q2, q3, q2
@@ -482,10 +478,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
@@ -494,27 +489,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vorr q2, q2, q3

diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll b/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
index 924d88d..5f93ecb 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-xor.ll
@@ -461,36 +461,32 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeqz_v2i1(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-NEXT:    vmov r1, s4
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s6
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s7
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    veor q2, q3, q2
@@ -518,10 +514,9 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    vmov r2, s6
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s7
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s11
@@ -530,27 +525,24 @@ define arm_aapcs_vfpcc <2 x i64> @cmpeq_v2i1(<2 x i64> %a, <2 x i64> %b, <2 x i6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s0
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vmov r0, s1
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[0], r0
 ; CHECK-NEXT:    vmov.32 q2[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q2[2], r0
 ; CHECK-NEXT:    vmov.32 q2[3], r0
 ; CHECK-NEXT:    veor q2, q2, q3

diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmp.ll b/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
index a704b4e..fadcb64 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmp.ll
@@ -378,10 +378,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
 ; CHECK-NEXT:    vmov r0, s7
@@ -389,10 +388,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r1, s6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[2], r0
 ; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vbic q0, q3, q4
@@ -420,10 +418,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
 ; CHECK-NEXT:    vmov r0, s7
@@ -431,10 +428,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, <2 x i64> %srcb,
 ; CHECK-NEXT:    vmov r1, s6
 ; CHECK-NEXT:    eors r1, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q4[2], r0
 ; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vbic q0, q3, q4
@@ -459,79 +455,67 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov lr, s10
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    subs r1, r0, r2
-; CHECK-NEXT:    asr.w r12, r0, #31
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    subs.w r1, lr, r2
+; CHECK-NEXT:    asr.w r12, lr, #31
 ; CHECK-NEXT:    sbcs.w r1, r12, r2, asr #31
 ; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    vmov r2, s10
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r1
 ; CHECK-NEXT:    vmov.32 q3[1], r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    subs.w r2, r1, lr
+; CHECK-NEXT:    subs r0, r1, r2
 ; CHECK-NEXT:    asr.w r12, r1, #31
-; CHECK-NEXT:    sbcs.w r2, r12, lr, asr #31
+; CHECK-NEXT:    sbcs.w r0, r12, r2, asr #31
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    vmov.32 q4[2], r1
-; CHECK-NEXT:    vmov.32 q3[2], r3
-; CHECK-NEXT:    vmov.32 q4[3], r1
-; CHECK-NEXT:    vmov.32 q3[3], r3
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[0], r0
 ; CHECK-NEXT:    vmov.32 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s6
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[2], r0
 ; CHECK-NEXT:    vmov.32 q5[3], r0
 ; CHECK-NEXT:    vand q1, q5, q4

diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
index 224c6f5..b8235d6 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
@@ -5,45 +5,41 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -61,30 +57,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
@@ -92,24 +86,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -130,45 +122,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -186,45 +174,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -242,45 +226,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -298,45 +278,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -354,30 +330,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
@@ -385,24 +359,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -422,45 +394,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_une_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s7
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -478,45 +446,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -535,45 +499,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -592,45 +552,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -649,45 +605,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -706,45 +658,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -765,45 +713,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s5
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s6
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s7
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -835,12 +779,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -848,25 +791,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -876,11 +819,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -890,11 +832,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -904,11 +846,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -918,25 +859,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -969,12 +905,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -984,28 +919,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1018,10 +952,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1031,15 +964,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1052,10 +983,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1069,9 +999,8 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
@@ -1080,11 +1009,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1118,12 +1046,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1131,25 +1058,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1159,11 +1086,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1173,11 +1099,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1187,11 +1113,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1201,25 +1126,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1250,12 +1170,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1263,25 +1182,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1291,11 +1210,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1305,11 +1223,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1319,11 +1237,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1333,25 +1250,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1382,12 +1294,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1395,25 +1306,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1423,11 +1334,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1437,11 +1347,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1451,11 +1361,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1465,25 +1374,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1514,12 +1418,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1527,25 +1430,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1555,11 +1458,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1569,11 +1471,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1583,11 +1485,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1597,25 +1498,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1648,12 +1544,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1663,28 +1558,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1697,10 +1591,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1710,15 +1603,13 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    mov.w r0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1731,10 +1622,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1748,9 +1638,8 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
@@ -1759,11 +1648,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1796,12 +1684,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1809,25 +1696,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1837,11 +1724,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1851,11 +1737,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmp.f16 s22, s20
@@ -1865,11 +1751,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1879,25 +1764,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -1928,12 +1808,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -1941,25 +1820,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1969,11 +1848,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -1983,11 +1861,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -1997,11 +1875,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2011,25 +1888,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2061,12 +1933,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2074,25 +1945,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2102,11 +1973,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2116,11 +1986,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2130,11 +2000,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2144,25 +2013,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2194,12 +2058,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2207,25 +2070,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2235,11 +2098,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2249,11 +2111,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2263,11 +2125,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2277,25 +2138,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2327,12 +2183,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2340,25 +2195,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2368,11 +2223,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2382,11 +2236,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2396,11 +2250,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2410,25 +2263,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2460,12 +2308,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2473,25 +2320,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2501,11 +2348,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2515,11 +2361,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2529,11 +2375,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2543,25 +2388,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
@@ -2595,12 +2435,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s8
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s12
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
@@ -2608,25 +2447,25 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s5
-; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s12, s8
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov.16 q4[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s22, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q4[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s13, s9
-; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s5
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2636,11 +2475,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s13
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s6
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2650,11 +2488,11 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s2
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
+; CHECK-MVE-NEXT:    vseleq.f16 s20, s14, s10
 ; CHECK-MVE-NEXT:    vmov r1, s20
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s6
 ; CHECK-MVE-NEXT:    vcmpe.f16 s22, s20
@@ -2664,11 +2502,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmovx.f16 s22, s14
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s22, s20
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s7
 ; CHECK-MVE-NEXT:    vmov r1, s20
@@ -2678,25 +2515,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s4
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s20, s15, s11
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s11
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s15
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s20
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q4[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4

diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
index 8901ba4..73dc885 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
@@ -5,45 +5,41 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -64,30 +60,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
@@ -95,24 +89,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -137,45 +129,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -196,45 +184,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -255,45 +239,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -315,45 +295,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -375,30 +351,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
@@ -406,24 +380,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -447,45 +419,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_une_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmp.f32 s3, s4
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -506,45 +474,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -567,45 +531,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -628,45 +588,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -688,45 +644,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -748,45 +700,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -811,45 +759,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, float %src2, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s4
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s4
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s4
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s4
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s15, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s14, s10
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -887,36 +831,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -926,10 +868,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -940,11 +881,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -954,10 +893,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -968,22 +906,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1023,10 +959,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1036,28 +971,26 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1070,24 +1003,22 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
@@ -1101,39 +1032,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1175,36 +1103,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1214,10 +1140,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1228,11 +1153,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1242,10 +1165,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1256,22 +1178,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1309,36 +1229,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1348,10 +1266,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1362,11 +1279,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1376,10 +1291,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1390,22 +1304,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1443,36 +1355,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1482,10 +1392,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1496,11 +1405,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1510,10 +1417,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1524,22 +1430,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1578,36 +1482,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1617,10 +1519,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1631,11 +1532,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -1645,10 +1544,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1659,22 +1557,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1715,10 +1611,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1728,28 +1623,26 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1762,24 +1655,22 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s10
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
@@ -1793,39 +1684,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
-; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r0, s18
+; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1866,36 +1754,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1905,10 +1791,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1919,11 +1804,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s18, s16
@@ -1933,10 +1816,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1947,22 +1829,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2000,36 +1880,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2039,10 +1917,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2053,11 +1930,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2067,10 +1942,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2081,22 +1955,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2136,36 +2008,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2175,10 +2045,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2189,11 +2058,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2203,10 +2070,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2217,22 +2083,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2272,36 +2136,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2311,10 +2173,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2325,11 +2186,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2339,10 +2198,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2353,22 +2211,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2407,36 +2263,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2446,10 +2300,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2460,11 +2313,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2474,10 +2325,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2488,22 +2338,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2542,36 +2390,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2581,10 +2427,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2595,11 +2440,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2609,10 +2452,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2623,22 +2465,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2680,36 +2520,34 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s16
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vmov r0, s12
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
+; CHECK-MVE-NEXT:    vmov r0, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s1, s16
-; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r2, s12
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmovx.f16 s20, s9
+; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2719,10 +2557,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2733,11 +2570,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s10, s6
-; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s18, s16
@@ -2747,10 +2582,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s16
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s20, s18
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2761,22 +2595,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s18, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov r0, s18
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3

diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
index a16492f..16ba72d 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
@@ -5,45 +5,41 @@
 define arm_aapcs_vfpcc <4 x float> @vcmp_oeq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oeq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -61,30 +57,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_one_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
@@ -92,24 +86,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -131,45 +123,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ogt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -187,45 +175,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_oge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_oge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -243,45 +227,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_olt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_olt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -299,45 +279,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ole_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ole_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -355,30 +331,28 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r3, #1
@@ -386,24 +360,22 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
 ; CHECK-MVE-NEXT:    mov.w r0, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -424,45 +396,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_une_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_une_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmp.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmp.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmp.f32 s3, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -480,45 +448,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ugt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -537,45 +501,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -594,45 +554,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -651,45 +607,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, #0
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -708,45 +660,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -768,45 +716,41 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {
 ; CHECK-MVE-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVE:       @ %bb.0: @ %entry
-; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
 ; CHECK-MVE-NEXT:    movs r1, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s0, s0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    vcmpe.f32 s1, s1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r2, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
+; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    mov.w r3, #0
-; CHECK-MVE-NEXT:    vcmpe.f32 s2, s2
+; CHECK-MVE-NEXT:    vcmpe.f32 s3, s3
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r3, #1
 ; CHECK-MVE-NEXT:    cmp r3, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r3, #1
+; CHECK-MVE-NEXT:    csinc r3, zr, zr, eq
 ; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    cmp r3, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s3, s11, s7
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    lsls r0, r3, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s2, s10, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
@@ -839,36 +783,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -878,10 +821,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -892,11 +834,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -906,10 +846,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -920,23 +859,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oeq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -969,11 +905,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -982,28 +917,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1016,26 +950,23 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1048,39 +979,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
-; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1115,36 +1043,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1154,10 +1081,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1168,11 +1094,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1182,10 +1106,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1196,23 +1119,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it gt
 ; CHECK-MVE-NEXT:    movgt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1243,36 +1163,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1282,10 +1201,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1296,11 +1214,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1310,10 +1226,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1324,23 +1239,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_oge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ge
 ; CHECK-MVE-NEXT:    movge r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1371,36 +1283,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1410,10 +1321,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1424,11 +1334,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1438,10 +1346,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1452,23 +1359,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_olt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it mi
 ; CHECK-MVE-NEXT:    movmi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1499,36 +1403,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1538,10 +1441,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1552,11 +1454,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1566,10 +1466,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1580,23 +1479,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ole_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ls
 ; CHECK-MVE-NEXT:    movls r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1629,11 +1525,10 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1642,28 +1537,27 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
+; CHECK-MVE-NEXT:    mov.w r0, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1676,26 +1570,23 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    vmov.16 q3[3], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s10
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1708,39 +1599,36 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
+; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmov r1, s16
+; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
 ; CHECK-MVE-NEXT:    vmov.16 q3[5], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r1, #1
-; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it eq
 ; CHECK-MVE-NEXT:    moveq r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1774,36 +1662,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmp.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmp.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1813,10 +1700,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1827,11 +1713,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmp.f16 s16, #0
@@ -1841,10 +1725,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmp.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1855,23 +1738,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_une_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it ne
 ; CHECK-MVE-NEXT:    movne r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -1902,36 +1782,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1941,10 +1820,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1955,11 +1833,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -1969,10 +1845,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -1983,23 +1858,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ugt_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it hi
 ; CHECK-MVE-NEXT:    movhi r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2031,36 +1903,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2070,10 +1941,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2084,11 +1954,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2098,10 +1966,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2112,23 +1979,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it pl
 ; CHECK-MVE-NEXT:    movpl r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2160,36 +2024,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2199,10 +2062,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2213,11 +2075,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2227,10 +2087,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2241,23 +2100,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it lt
 ; CHECK-MVE-NEXT:    movlt r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2289,36 +2145,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, #0
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2328,10 +2183,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2342,11 +2196,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, #0
@@ -2356,10 +2208,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2370,23 +2221,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it le
 ; CHECK-MVE-NEXT:    movle r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2418,36 +2266,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2457,10 +2304,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s2
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2471,11 +2317,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2485,10 +2329,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s3
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2499,23 +2342,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vc
 ; CHECK-MVE-NEXT:    movvc r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
@@ -2550,36 +2390,35 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s0, s0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s14, s8
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    mov.w r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s14, s12
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r2, #1
 ; CHECK-MVE-NEXT:    cmp r2, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r2, #1
-; CHECK-MVE-NEXT:    cmp r2, #0
+; CHECK-MVE-NEXT:    csinc r2, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s12
+; CHECK-MVE-NEXT:    lsls r2, r2, #31
+; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
 ; CHECK-MVE-NEXT:    vseleq.f16 s12, s8, s4
+; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    vmov r2, s12
-; CHECK-MVE-NEXT:    vcmpe.f16 s1, s1
+; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov.16 q3[0], r2
-; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov.16 q3[1], r1
 ; CHECK-MVE-NEXT:    mov.w r1, #0
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    movs r0, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s9, s5
-; CHECK-MVE-NEXT:    vmovx.f16 s18, s9
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s1
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2589,10 +2428,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s5
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s2, s2
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2603,11 +2441,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s10, s6
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
 ; CHECK-MVE-NEXT:    vmov r1, s16
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s2
 ; CHECK-MVE-NEXT:    vcmpe.f16 s16, s16
@@ -2617,10 +2453,9 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmovx.f16 s16, s6
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
 ; CHECK-MVE-NEXT:    vcmpe.f16 s3, s3
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s18, s16
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
@@ -2631,23 +2466,20 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r1, #1
 ; CHECK-MVE-NEXT:    cmp r1, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r1, #1
-; CHECK-MVE-NEXT:    cmp r1, #0
+; CHECK-MVE-NEXT:    csinc r1, zr, zr, eq
+; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
+; CHECK-MVE-NEXT:    lsls r1, r1, #31
+; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
 ; CHECK-MVE-NEXT:    vseleq.f16 s16, s11, s7
-; CHECK-MVE-NEXT:    movs r0, #0
 ; CHECK-MVE-NEXT:    vmrs APSR_nzcv, fpscr
 ; CHECK-MVE-NEXT:    it vs
 ; CHECK-MVE-NEXT:    movvs r0, #1
 ; CHECK-MVE-NEXT:    cmp r0, #0
-; CHECK-MVE-NEXT:    it ne
-; CHECK-MVE-NEXT:    movne r0, #1
-; CHECK-MVE-NEXT:    vmovx.f16 s0, s7
-; CHECK-MVE-NEXT:    vmovx.f16 s2, s11
-; CHECK-MVE-NEXT:    cmp r0, #0
+; CHECK-MVE-NEXT:    csinc r0, zr, zr, eq
 ; CHECK-MVE-NEXT:    vmov r1, s16
-; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
+; CHECK-MVE-NEXT:    lsls r0, r0, #31
 ; CHECK-MVE-NEXT:    vmov.16 q3[6], r1
+; CHECK-MVE-NEXT:    vseleq.f16 s0, s2, s0
 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3

diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
index 3adc4cb..38ffd29 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpr.ll
@@ -444,10 +444,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    eors r2, r1
 ; CHECK-NEXT:    eors r3, r0
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    clz r2, r2
-; CHECK-NEXT:    lsrs r2, r2, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    csinc r2, zr, zr, ne
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csinv r2, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r2
 ; CHECK-NEXT:    vmov.32 q3[1], r2
 ; CHECK-NEXT:    vmov r2, s3
@@ -455,10 +454,9 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eq_v2i64(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    eors r0, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
@@ -481,10 +479,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    eors r2, r1
 ; CHECK-NEXT:    eors r3, r0
 ; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    clz r2, r2
-; CHECK-NEXT:    lsrs r2, r2, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
+; CHECK-NEXT:    csinc r2, zr, zr, ne
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    csinv r2, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r2
 ; CHECK-NEXT:    vmov.32 q3[1], r2
 ; CHECK-NEXT:    vmov r2, s3
@@ -492,10 +489,9 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eq_v2i32(<2 x i64> %src, i64 %src2, <2 x
 ; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    eors r0, r2
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
@@ -521,79 +517,67 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_multi_v2i32(<2 x i64> %a, <2 x i32> %b, <
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    vmov r2, s8
-; CHECK-NEXT:    vmov lr, s10
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
-; CHECK-NEXT:    vmov r0, s0
-; CHECK-NEXT:    subs r1, r0, r2
-; CHECK-NEXT:    asr.w r12, r0, #31
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    subs.w r1, lr, r2
+; CHECK-NEXT:    asr.w r12, lr, #31
 ; CHECK-NEXT:    sbcs.w r1, r12, r2, asr #31
 ; CHECK-NEXT:    mov.w r1, #0
+; CHECK-NEXT:    vmov r2, s10
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r1, #1
 ; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    csinv r1, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r1
 ; CHECK-NEXT:    vmov.32 q3[1], r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    subs.w r2, r1, lr
+; CHECK-NEXT:    subs r0, r1, r2
 ; CHECK-NEXT:    asr.w r12, r1, #31
-; CHECK-NEXT:    sbcs.w r2, r12, lr, asr #31
+; CHECK-NEXT:    sbcs.w r0, r12, r2, asr #31
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    vmov.32 q4[0], r0
 ; CHECK-NEXT:    vmov.32 q4[1], r0
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
+; CHECK-NEXT:    vmov.32 q4[2], r0
+; CHECK-NEXT:    vmov.32 q4[3], r0
 ; CHECK-NEXT:    vmov r0, s4
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    vmov.32 q4[2], r1
-; CHECK-NEXT:    vmov.32 q3[2], r3
-; CHECK-NEXT:    vmov.32 q4[3], r1
-; CHECK-NEXT:    vmov.32 q3[3], r3
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[0], r0
 ; CHECK-NEXT:    vmov.32 q5[1], r0
 ; CHECK-NEXT:    vmov r0, s6
 ; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne r0, #1
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, eq
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q5[2], r0
 ; CHECK-NEXT:    vmov.32 q5[3], r0
 ; CHECK-NEXT:    vand q1, q5, q4

diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpz.ll
index 173c033..5e9fbf3 100644 (file)
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpz.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpz.ll
@@ -365,18 +365,16 @@ define arm_aapcs_vfpcc <2 x i64> @vcmp_eqz_v2i64(<2 x i64> %src, <2 x i64> %a, <
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3
@@ -396,18 +394,16 @@ define arm_aapcs_vfpcc <2 x i32> @vcmp_eqz_v2i32(<2 x i64> %src, <2 x i32> %a, <
 ; CHECK-NEXT:    vmov r1, s0
 ; CHECK-NEXT:    orrs r0, r1
 ; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[0], r0
 ; CHECK-NEXT:    vmov.32 q3[1], r0
 ; CHECK-NEXT:    vmov r0, s3
 ; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    clz r0, r0
-; CHECK-NEXT:    lsrs r0, r0, #5
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r0, #-1
+; CHECK-NEXT:    csinc r0, zr, zr, ne
+; CHECK-NEXT:    tst.w r0, #1
+; CHECK-NEXT:    csinv r0, zr, zr, eq
 ; CHECK-NEXT:    vmov.32 q3[2], r0
 ; CHECK-NEXT:    vmov.32 q3[3], r0
 ; CHECK-NEXT:    vbic q0, q2, q3