From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 24 Feb 2021 17:36:28 +0000 (-0800)
Subject: [LegalizeIntegerTypes] Further improve ExpandIntRes_SADDSUBO for targets where SADDO... 
X-Git-Tag: llvmorg-14-init~14134
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=fe50be12c8b845fffd44c508ad981901d25ac5f8;p=platform%2Fupstream%2Fllvm.git

[LegalizeIntegerTypes] Further improve ExpandIntRes_SADDSUBO for targets where SADDO/SSUBO aren't supported.

Rather than converting 3 signbits to bools and comparing them,
we can do bitwise logic on the whole vector and convert the
resulting sign bit to a bool at the end.

This is still a different algorithm than what we do in LegalizeDAG
through expandSADDOSSUBO. That algorithm needs to know that the
RHS of SSUBO is > 0, but that's costly when the type is split.

Reviewed By: RKSimon

Differential Revision: https://reviews.llvm.org/D97325
---

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 8316289..f91a4ce 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3623,18 +3623,16 @@ void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
 
 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
                                              SDValue &Lo, SDValue &Hi) {
+  assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
+         "Node has unexpected Opcode");
   SDValue LHS = Node->getOperand(0);
   SDValue RHS = Node->getOperand(1);
   SDLoc dl(Node);
 
   SDValue Ovf;
 
-  unsigned CarryOp;
-  switch(Node->getOpcode()) {
-  default: llvm_unreachable("Node has unexpected Opcode");
-  case ISD::SADDO: CarryOp = ISD::SADDO_CARRY; break;
-  case ISD::SSUBO: CarryOp = ISD::SSUBO_CARRY; break;
-  }
+  bool IsAdd = Node->getOpcode() == ISD::SADDO;
+  unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
 
   bool HasCarryOp = TLI.isOperationLegalOrCustom(
       CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
@@ -3646,8 +3644,7 @@ void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
     GetExpandedInteger(RHS, RHSL, RHSH);
     SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
 
-    Lo = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
-                     ISD::UADDO : ISD::USUBO, dl, VTList, { LHSL, RHSL });
+    Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
     Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
 
     Ovf = Hi.getValue(1);
@@ -3670,19 +3667,27 @@ void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
     //   Sub:
     //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
     //
+    // To get better codegen we can rewrite this by doing bitwise math on
+    // the integers and extract the final sign bit at the end. So the
+    // above becomes:
+    //
+    //   Add:
+    //   Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
+    //   Sub:
+    //   Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
+    //
+    // NOTE: This is different than the expansion we do in expandSADDSUBO
+    // because it is more costly to determine the RHS is > 0 for SSUBO with the
+    // integers split.
+    EVT VT = LHS.getValueType();
+    SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
+    if (IsAdd)
+      SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
+
+    SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
+    Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
     EVT OType = Node->getValueType(1);
-    SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
-
-    SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETLT);
-    SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETLT);
-    SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
-                                      Node->getOpcode() == ISD::SADDO ?
-                                      ISD::SETEQ : ISD::SETNE);
-
-    SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETLT);
-    SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
-
-    Ovf = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
+    Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
   }
 
   // Use the calculated overflow everywhere.
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
index ec098e9..16561fe 100644
--- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
@@ -366,42 +366,28 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp x7, #0 // =0
-; CHECK-NEXT:    cset w9, lt
-; CHECK-NEXT:    cmp x3, #0 // =0
-; CHECK-NEXT:    cset w10, lt
-; CHECK-NEXT:    cmp w10, w9
-; CHECK-NEXT:    cset w9, eq
-; CHECK-NEXT:    adds x11, x2, x6
+; CHECK-NEXT:    adds x8, x2, x6
 ; CHECK-NEXT:    adcs x12, x3, x7
+; CHECK-NEXT:    mov x9, #9223372036854775807
+; CHECK-NEXT:    eor x10, x3, x7
 ; CHECK-NEXT:    cmp x12, #0 // =0
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cset w15, lt
-; CHECK-NEXT:    cinv x14, x8, ge
-; CHECK-NEXT:    cmp w10, w15
-; CHECK-NEXT:    cset w10, ne
-; CHECK-NEXT:    asr x13, x12, #63
-; CHECK-NEXT:    tst w9, w10
-; CHECK-NEXT:    csel x2, x13, x11, ne
-; CHECK-NEXT:    csel x3, x14, x12, ne
-; CHECK-NEXT:    cmp x5, #0 // =0
-; CHECK-NEXT:    cset w9, lt
-; CHECK-NEXT:    cmp x1, #0 // =0
-; CHECK-NEXT:    cset w10, lt
-; CHECK-NEXT:    cmp w10, w9
-; CHECK-NEXT:    cset w9, eq
-; CHECK-NEXT:    adds x11, x0, x4
-; CHECK-NEXT:    adcs x12, x1, x5
-; CHECK-NEXT:    cmp x12, #0 // =0
-; CHECK-NEXT:    cset w13, lt
-; CHECK-NEXT:    cinv x8, x8, ge
-; CHECK-NEXT:    cmp w10, w13
-; CHECK-NEXT:    cset w10, ne
-; CHECK-NEXT:    tst w9, w10
-; CHECK-NEXT:    asr x9, x12, #63
-; CHECK-NEXT:    csel x9, x9, x11, ne
-; CHECK-NEXT:    csel x1, x8, x12, ne
-; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    eor x13, x3, x12
+; CHECK-NEXT:    cinv x14, x9, ge
+; CHECK-NEXT:    bics xzr, x13, x10
+; CHECK-NEXT:    asr x10, x12, #63
+; CHECK-NEXT:    csel x2, x10, x8, lt
+; CHECK-NEXT:    csel x3, x14, x12, lt
+; CHECK-NEXT:    adds x8, x0, x4
+; CHECK-NEXT:    adcs x10, x1, x5
+; CHECK-NEXT:    eor x11, x1, x5
+; CHECK-NEXT:    cmp x10, #0 // =0
+; CHECK-NEXT:    eor x12, x1, x10
+; CHECK-NEXT:    cinv x9, x9, ge
+; CHECK-NEXT:    bics xzr, x12, x11
+; CHECK-NEXT:    asr x11, x10, #63
+; CHECK-NEXT:    csel x8, x11, x8, lt
+; CHECK-NEXT:    csel x1, x9, x10, lt
+; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
index 0ce0184..bfc3858 100644
--- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
@@ -369,42 +369,28 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp x7, #0 // =0
-; CHECK-NEXT:    cset w9, lt
-; CHECK-NEXT:    cmp x3, #0 // =0
-; CHECK-NEXT:    cset w10, lt
-; CHECK-NEXT:    cmp w10, w9
-; CHECK-NEXT:    cset w9, ne
-; CHECK-NEXT:    subs x11, x2, x6
+; CHECK-NEXT:    subs x8, x2, x6
 ; CHECK-NEXT:    sbcs x12, x3, x7
+; CHECK-NEXT:    mov x9, #9223372036854775807
+; CHECK-NEXT:    eor x10, x3, x7
 ; CHECK-NEXT:    cmp x12, #0 // =0
-; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cset w15, lt
-; CHECK-NEXT:    cinv x14, x8, ge
-; CHECK-NEXT:    cmp w10, w15
-; CHECK-NEXT:    cset w10, ne
-; CHECK-NEXT:    asr x13, x12, #63
-; CHECK-NEXT:    tst w9, w10
-; CHECK-NEXT:    csel x2, x13, x11, ne
-; CHECK-NEXT:    csel x3, x14, x12, ne
-; CHECK-NEXT:    cmp x5, #0 // =0
-; CHECK-NEXT:    cset w9, lt
-; CHECK-NEXT:    cmp x1, #0 // =0
-; CHECK-NEXT:    cset w10, lt
-; CHECK-NEXT:    cmp w10, w9
-; CHECK-NEXT:    cset w9, ne
-; CHECK-NEXT:    subs x11, x0, x4
-; CHECK-NEXT:    sbcs x12, x1, x5
-; CHECK-NEXT:    cmp x12, #0 // =0
-; CHECK-NEXT:    cset w13, lt
-; CHECK-NEXT:    cinv x8, x8, ge
-; CHECK-NEXT:    cmp w10, w13
-; CHECK-NEXT:    cset w10, ne
-; CHECK-NEXT:    tst w9, w10
-; CHECK-NEXT:    asr x9, x12, #63
-; CHECK-NEXT:    csel x9, x9, x11, ne
-; CHECK-NEXT:    csel x1, x8, x12, ne
-; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    eor x13, x3, x12
+; CHECK-NEXT:    cinv x14, x9, ge
+; CHECK-NEXT:    tst x10, x13
+; CHECK-NEXT:    asr x10, x12, #63
+; CHECK-NEXT:    csel x2, x10, x8, lt
+; CHECK-NEXT:    csel x3, x14, x12, lt
+; CHECK-NEXT:    subs x8, x0, x4
+; CHECK-NEXT:    sbcs x10, x1, x5
+; CHECK-NEXT:    eor x11, x1, x5
+; CHECK-NEXT:    cmp x10, #0 // =0
+; CHECK-NEXT:    eor x12, x1, x10
+; CHECK-NEXT:    cinv x9, x9, ge
+; CHECK-NEXT:    tst x11, x12
+; CHECK-NEXT:    asr x11, x10, #63
+; CHECK-NEXT:    csel x8, x11, x8, lt
+; CHECK-NEXT:    csel x1, x9, x10, lt
+; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/ARM/sadd_sat.ll b/llvm/test/CodeGen/ARM/sadd_sat.ll
index 0b49803..6d5e3e3 100644
--- a/llvm/test/CodeGen/ARM/sadd_sat.ll
+++ b/llvm/test/CodeGen/ARM/sadd_sat.ll
@@ -94,130 +94,74 @@ define i32 @func(i32 %x, i32 %y) nounwind {
 define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T1-LABEL: func2:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    .pad #4
-; CHECK-T1-NEXT:    sub sp, #4
-; CHECK-T1-NEXT:    str r2, [sp] @ 4-byte Spill
-; CHECK-T1-NEXT:    mov r2, r0
-; CHECK-T1-NEXT:    movs r4, #1
-; CHECK-T1-NEXT:    movs r0, #0
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    mov r5, r4
-; CHECK-T1-NEXT:    bmi .LBB1_2
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    mov r4, r1
+; CHECK-T1-NEXT:    eors r4, r3
+; CHECK-T1-NEXT:    adds r0, r0, r2
+; CHECK-T1-NEXT:    adcs r3, r1
+; CHECK-T1-NEXT:    eors r1, r3
+; CHECK-T1-NEXT:    bics r1, r4
+; CHECK-T1-NEXT:    bpl .LBB1_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    mov r5, r0
+; CHECK-T1-NEXT:    asrs r0, r3, #31
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    mov r7, r4
+; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_4
 ; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    mov r7, r0
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bpl .LBB1_5
+; CHECK-T1-NEXT:    b .LBB1_6
 ; CHECK-T1-NEXT:  .LBB1_4:
-; CHECK-T1-NEXT:    subs r6, r7, r5
-; CHECK-T1-NEXT:    rsbs r5, r6, #0
-; CHECK-T1-NEXT:    adcs r5, r6
-; CHECK-T1-NEXT:    ldr r6, [sp] @ 4-byte Reload
-; CHECK-T1-NEXT:    adds r6, r2, r6
-; CHECK-T1-NEXT:    adcs r1, r3
-; CHECK-T1-NEXT:    mov r2, r4
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r1, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_6
-; CHECK-T1-NEXT:  @ %bb.5:
-; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:  .LBB1_5:
+; CHECK-T1-NEXT:    mov r2, r3
 ; CHECK-T1-NEXT:  .LBB1_6:
-; CHECK-T1-NEXT:    subs r0, r7, r2
-; CHECK-T1-NEXT:    subs r2, r0, #1
-; CHECK-T1-NEXT:    sbcs r0, r2
-; CHECK-T1-NEXT:    ands r5, r0
-; CHECK-T1-NEXT:    beq .LBB1_8
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:    asrs r6, r1, #31
-; CHECK-T1-NEXT:  .LBB1_8:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    bmi .LBB1_10
-; CHECK-T1-NEXT:  @ %bb.9:
-; CHECK-T1-NEXT:    lsls r2, r4, #31
-; CHECK-T1-NEXT:    cmp r5, #0
-; CHECK-T1-NEXT:    beq .LBB1_11
-; CHECK-T1-NEXT:    b .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_10:
-; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
-; CHECK-T1-NEXT:    cmp r5, #0
-; CHECK-T1-NEXT:    bne .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_11:
-; CHECK-T1-NEXT:    mov r2, r1
-; CHECK-T1-NEXT:  .LBB1_12:
-; CHECK-T1-NEXT:    mov r0, r6
 ; CHECK-T1-NEXT:    mov r1, r2
-; CHECK-T1-NEXT:    add sp, #4
-; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    pop {r4, pc}
 ; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  @ %bb.7:
 ; CHECK-T1-NEXT:  .LCPI1_0:
 ; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
 ; CHECK-T2-LABEL: func2:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    .save {r7, lr}
-; CHECK-T2-NEXT:    push {r7, lr}
-; CHECK-T2-NEXT:    cmp r1, #0
-; CHECK-T2-NEXT:    mov.w lr, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi.w lr, #1
 ; CHECK-T2-NEXT:    adds r0, r0, r2
-; CHECK-T2-NEXT:    adcs.w r2, r1, r3
-; CHECK-T2-NEXT:    mov.w r1, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r1, #1
-; CHECK-T2-NEXT:    mov.w r12, #0
-; CHECK-T2-NEXT:    subs.w r1, lr, r1
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    movne r1, #1
+; CHECK-T2-NEXT:    eor.w r12, r1, r3
+; CHECK-T2-NEXT:    adc.w r2, r1, r3
+; CHECK-T2-NEXT:    eors r1, r2
+; CHECK-T2-NEXT:    bic.w r3, r1, r12
+; CHECK-T2-NEXT:    mov.w r1, #-2147483648
 ; CHECK-T2-NEXT:    cmp r3, #0
 ; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi.w r12, #1
-; CHECK-T2-NEXT:    sub.w r3, lr, r12
-; CHECK-T2-NEXT:    clz r3, r3
-; CHECK-T2-NEXT:    lsrs r3, r3, #5
-; CHECK-T2-NEXT:    ands r3, r1
-; CHECK-T2-NEXT:    mov.w r1, #-2147483648
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    asrne r0, r2, #31
+; CHECK-T2-NEXT:    asrmi r0, r2, #31
 ; CHECK-T2-NEXT:    cmp r2, #0
 ; CHECK-T2-NEXT:    it mi
 ; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
 ; CHECK-T2-NEXT:    cmp r3, #0
-; CHECK-T2-NEXT:    it eq
-; CHECK-T2-NEXT:    moveq r1, r2
-; CHECK-T2-NEXT:    pop {r7, pc}
+; CHECK-T2-NEXT:    it pl
+; CHECK-T2-NEXT:    movpl r1, r2
+; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func2:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    .save {r11, lr}
-; CHECK-ARM-NEXT:    push {r11, lr}
-; CHECK-ARM-NEXT:    cmp r1, #0
-; CHECK-ARM-NEXT:    mov lr, #0
-; CHECK-ARM-NEXT:    movmi lr, #1
 ; CHECK-ARM-NEXT:    adds r0, r0, r2
-; CHECK-ARM-NEXT:    adcs r2, r1, r3
-; CHECK-ARM-NEXT:    mov r1, #0
-; CHECK-ARM-NEXT:    movmi r1, #1
-; CHECK-ARM-NEXT:    mov r12, #0
-; CHECK-ARM-NEXT:    subs r1, lr, r1
-; CHECK-ARM-NEXT:    movne r1, #1
-; CHECK-ARM-NEXT:    cmp r3, #0
-; CHECK-ARM-NEXT:    movmi r12, #1
-; CHECK-ARM-NEXT:    sub r3, lr, r12
-; CHECK-ARM-NEXT:    clz r3, r3
-; CHECK-ARM-NEXT:    lsr r3, r3, #5
-; CHECK-ARM-NEXT:    ands r3, r3, r1
+; CHECK-ARM-NEXT:    eor r12, r1, r3
+; CHECK-ARM-NEXT:    adc r2, r1, r3
+; CHECK-ARM-NEXT:    eor r1, r1, r2
+; CHECK-ARM-NEXT:    bic r3, r1, r12
 ; CHECK-ARM-NEXT:    mov r1, #-2147483648
-; CHECK-ARM-NEXT:    asrne r0, r2, #31
+; CHECK-ARM-NEXT:    cmp r3, #0
+; CHECK-ARM-NEXT:    asrmi r0, r2, #31
 ; CHECK-ARM-NEXT:    cmp r2, #0
 ; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
 ; CHECK-ARM-NEXT:    cmp r3, #0
-; CHECK-ARM-NEXT:    moveq r1, r2
-; CHECK-ARM-NEXT:    pop {r11, pc}
+; CHECK-ARM-NEXT:    movpl r1, r2
+; CHECK-ARM-NEXT:    bx lr
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y)
   ret i64 %tmp
 }
diff --git a/llvm/test/CodeGen/ARM/sadd_sat_plus.ll b/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
index 4028e57..f0a0b72 100644
--- a/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
+++ b/llvm/test/CodeGen/ARM/sadd_sat_plus.ll
@@ -78,132 +78,80 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-T1-LABEL: func64:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    .pad #4
-; CHECK-T1-NEXT:    sub sp, #4
-; CHECK-T1-NEXT:    ldr r5, [sp, #28]
-; CHECK-T1-NEXT:    movs r2, #1
-; CHECK-T1-NEXT:    movs r4, #0
-; CHECK-T1-NEXT:    cmp r5, #0
-; CHECK-T1-NEXT:    mov r3, r2
-; CHECK-T1-NEXT:    bmi .LBB1_2
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    ldr r3, [sp, #12]
+; CHECK-T1-NEXT:    mov r2, r1
+; CHECK-T1-NEXT:    eors r2, r3
+; CHECK-T1-NEXT:    ldr r4, [sp, #8]
+; CHECK-T1-NEXT:    adds r0, r0, r4
+; CHECK-T1-NEXT:    adcs r3, r1
+; CHECK-T1-NEXT:    eors r1, r3
+; CHECK-T1-NEXT:    bics r1, r2
+; CHECK-T1-NEXT:    bpl .LBB1_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    mov r3, r4
+; CHECK-T1-NEXT:    asrs r0, r3, #31
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    mov r6, r2
+; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_4
 ; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    mov r6, r4
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bpl .LBB1_5
+; CHECK-T1-NEXT:    b .LBB1_6
 ; CHECK-T1-NEXT:  .LBB1_4:
-; CHECK-T1-NEXT:    subs r7, r6, r3
-; CHECK-T1-NEXT:    rsbs r3, r7, #0
-; CHECK-T1-NEXT:    adcs r3, r7
-; CHECK-T1-NEXT:    ldr r7, [sp, #24]
-; CHECK-T1-NEXT:    adds r0, r0, r7
-; CHECK-T1-NEXT:    adcs r1, r5
-; CHECK-T1-NEXT:    mov r5, r2
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r1, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_6
-; CHECK-T1-NEXT:  @ %bb.5:
-; CHECK-T1-NEXT:    mov r5, r4
+; CHECK-T1-NEXT:  .LBB1_5:
+; CHECK-T1-NEXT:    mov r2, r3
 ; CHECK-T1-NEXT:  .LBB1_6:
-; CHECK-T1-NEXT:    subs r4, r6, r5
-; CHECK-T1-NEXT:    subs r5, r4, #1
-; CHECK-T1-NEXT:    sbcs r4, r5
-; CHECK-T1-NEXT:    ands r3, r4
-; CHECK-T1-NEXT:    beq .LBB1_8
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:    asrs r0, r1, #31
-; CHECK-T1-NEXT:  .LBB1_8:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    bmi .LBB1_10
-; CHECK-T1-NEXT:  @ %bb.9:
-; CHECK-T1-NEXT:    lsls r2, r2, #31
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    beq .LBB1_11
-; CHECK-T1-NEXT:    b .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_10:
-; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    bne .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_11:
-; CHECK-T1-NEXT:    mov r2, r1
-; CHECK-T1-NEXT:  .LBB1_12:
 ; CHECK-T1-NEXT:    mov r1, r2
-; CHECK-T1-NEXT:    add sp, #4
-; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    pop {r4, pc}
 ; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  @ %bb.7:
 ; CHECK-T1-NEXT:  .LCPI1_0:
 ; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
 ; CHECK-T2-LABEL: func64:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    .save {r7, lr}
-; CHECK-T2-NEXT:    push {r7, lr}
-; CHECK-T2-NEXT:    ldrd r2, r12, [sp, #8]
-; CHECK-T2-NEXT:    cmp r1, #0
-; CHECK-T2-NEXT:    mov.w r3, #0
-; CHECK-T2-NEXT:    mov.w lr, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r3, #1
+; CHECK-T2-NEXT:    ldr r2, [sp]
+; CHECK-T2-NEXT:    ldr.w r12, [sp, #4]
 ; CHECK-T2-NEXT:    adds r0, r0, r2
-; CHECK-T2-NEXT:    adcs.w r2, r1, r12
-; CHECK-T2-NEXT:    mov.w r1, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r1, #1
-; CHECK-T2-NEXT:    subs r1, r3, r1
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    movne r1, #1
-; CHECK-T2-NEXT:    cmp.w r12, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi.w lr, #1
-; CHECK-T2-NEXT:    sub.w r3, r3, lr
-; CHECK-T2-NEXT:    clz r3, r3
-; CHECK-T2-NEXT:    lsrs r3, r3, #5
-; CHECK-T2-NEXT:    ands r3, r1
+; CHECK-T2-NEXT:    adc.w r2, r1, r12
+; CHECK-T2-NEXT:    eor.w r3, r1, r12
+; CHECK-T2-NEXT:    eors r1, r2
+; CHECK-T2-NEXT:    bic.w r3, r1, r3
 ; CHECK-T2-NEXT:    mov.w r1, #-2147483648
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    asrne r0, r2, #31
+; CHECK-T2-NEXT:    cmp r3, #0
+; CHECK-T2-NEXT:    it mi
+; CHECK-T2-NEXT:    asrmi r0, r2, #31
 ; CHECK-T2-NEXT:    cmp r2, #0
 ; CHECK-T2-NEXT:    it mi
 ; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
 ; CHECK-T2-NEXT:    cmp r3, #0
-; CHECK-T2-NEXT:    it eq
-; CHECK-T2-NEXT:    moveq r1, r2
-; CHECK-T2-NEXT:    pop {r7, pc}
+; CHECK-T2-NEXT:    it pl
+; CHECK-T2-NEXT:    movpl r1, r2
+; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func64:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    .save {r11, lr}
-; CHECK-ARM-NEXT:    push {r11, lr}
-; CHECK-ARM-NEXT:    ldr r2, [sp, #8]
-; CHECK-ARM-NEXT:    mov r3, #0
-; CHECK-ARM-NEXT:    ldr r12, [sp, #12]
-; CHECK-ARM-NEXT:    adds r0, r0, r2
-; CHECK-ARM-NEXT:    mov r2, #0
-; CHECK-ARM-NEXT:    adc lr, r1, r12
-; CHECK-ARM-NEXT:    cmp r1, #0
-; CHECK-ARM-NEXT:    mov r1, #0
-; CHECK-ARM-NEXT:    movwmi r1, #1
-; CHECK-ARM-NEXT:    cmp lr, #0
-; CHECK-ARM-NEXT:    movwmi r2, #1
-; CHECK-ARM-NEXT:    subs r2, r1, r2
-; CHECK-ARM-NEXT:    movwne r2, #1
-; CHECK-ARM-NEXT:    cmp r12, #0
-; CHECK-ARM-NEXT:    movwmi r3, #1
-; CHECK-ARM-NEXT:    sub r1, r1, r3
-; CHECK-ARM-NEXT:    clz r1, r1
-; CHECK-ARM-NEXT:    lsr r1, r1, #5
-; CHECK-ARM-NEXT:    ands r2, r1, r2
-; CHECK-ARM-NEXT:    asrne r0, lr, #31
+; CHECK-ARM-NEXT:    ldr r12, [sp]
+; CHECK-ARM-NEXT:    ldr r2, [sp, #4]
+; CHECK-ARM-NEXT:    adds r0, r0, r12
+; CHECK-ARM-NEXT:    eor r3, r1, r2
+; CHECK-ARM-NEXT:    adc r2, r1, r2
+; CHECK-ARM-NEXT:    eor r1, r1, r2
+; CHECK-ARM-NEXT:    bic r3, r1, r3
 ; CHECK-ARM-NEXT:    mov r1, #-2147483648
-; CHECK-ARM-NEXT:    cmp lr, #0
-; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r3, #0
+; CHECK-ARM-NEXT:    asrmi r0, r2, #31
 ; CHECK-ARM-NEXT:    cmp r2, #0
-; CHECK-ARM-NEXT:    moveq r1, lr
-; CHECK-ARM-NEXT:    pop {r11, pc}
+; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r3, #0
+; CHECK-ARM-NEXT:    movpl r1, r2
+; CHECK-ARM-NEXT:    bx lr
   %a = mul i64 %y, %z
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %z)
   ret i64 %tmp
diff --git a/llvm/test/CodeGen/ARM/ssub_sat.ll b/llvm/test/CodeGen/ARM/ssub_sat.ll
index 50ab854..5edbb8e 100644
--- a/llvm/test/CodeGen/ARM/ssub_sat.ll
+++ b/llvm/test/CodeGen/ARM/ssub_sat.ll
@@ -96,129 +96,73 @@ define i32 @func(i32 %x, i32 %y) nounwind {
 define i64 @func2(i64 %x, i64 %y) nounwind {
 ; CHECK-T1-LABEL: func2:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    .pad #4
-; CHECK-T1-NEXT:    sub sp, #4
-; CHECK-T1-NEXT:    str r2, [sp] @ 4-byte Spill
-; CHECK-T1-NEXT:    mov r2, r0
-; CHECK-T1-NEXT:    movs r4, #1
-; CHECK-T1-NEXT:    movs r0, #0
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    mov r5, r4
-; CHECK-T1-NEXT:    bmi .LBB1_2
+; CHECK-T1-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-T1-NEXT:    push {r4, r5, r7, lr}
+; CHECK-T1-NEXT:    mov r5, r1
+; CHECK-T1-NEXT:    eors r5, r3
+; CHECK-T1-NEXT:    subs r0, r0, r2
+; CHECK-T1-NEXT:    mov r4, r1
+; CHECK-T1-NEXT:    sbcs r4, r3
+; CHECK-T1-NEXT:    eors r1, r4
+; CHECK-T1-NEXT:    ands r1, r5
+; CHECK-T1-NEXT:    bpl .LBB1_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    mov r5, r0
+; CHECK-T1-NEXT:    asrs r0, r4, #31
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    mov r7, r4
+; CHECK-T1-NEXT:    cmp r4, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_4
 ; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    mov r7, r0
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bpl .LBB1_5
+; CHECK-T1-NEXT:    b .LBB1_6
 ; CHECK-T1-NEXT:  .LBB1_4:
-; CHECK-T1-NEXT:    subs r5, r7, r5
-; CHECK-T1-NEXT:    subs r6, r5, #1
-; CHECK-T1-NEXT:    sbcs r5, r6
-; CHECK-T1-NEXT:    ldr r6, [sp] @ 4-byte Reload
-; CHECK-T1-NEXT:    subs r6, r2, r6
-; CHECK-T1-NEXT:    sbcs r1, r3
-; CHECK-T1-NEXT:    mov r2, r4
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r1, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_6
-; CHECK-T1-NEXT:  @ %bb.5:
-; CHECK-T1-NEXT:    mov r2, r0
+; CHECK-T1-NEXT:  .LBB1_5:
+; CHECK-T1-NEXT:    mov r2, r4
 ; CHECK-T1-NEXT:  .LBB1_6:
-; CHECK-T1-NEXT:    subs r0, r7, r2
-; CHECK-T1-NEXT:    subs r2, r0, #1
-; CHECK-T1-NEXT:    sbcs r0, r2
-; CHECK-T1-NEXT:    ands r5, r0
-; CHECK-T1-NEXT:    beq .LBB1_8
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:    asrs r6, r1, #31
-; CHECK-T1-NEXT:  .LBB1_8:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    bmi .LBB1_10
-; CHECK-T1-NEXT:  @ %bb.9:
-; CHECK-T1-NEXT:    lsls r2, r4, #31
-; CHECK-T1-NEXT:    cmp r5, #0
-; CHECK-T1-NEXT:    beq .LBB1_11
-; CHECK-T1-NEXT:    b .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_10:
-; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
-; CHECK-T1-NEXT:    cmp r5, #0
-; CHECK-T1-NEXT:    bne .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_11:
-; CHECK-T1-NEXT:    mov r2, r1
-; CHECK-T1-NEXT:  .LBB1_12:
-; CHECK-T1-NEXT:    mov r0, r6
 ; CHECK-T1-NEXT:    mov r1, r2
-; CHECK-T1-NEXT:    add sp, #4
-; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  @ %bb.7:
 ; CHECK-T1-NEXT:  .LCPI1_0:
 ; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
 ; CHECK-T2-LABEL: func2:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    .save {r4, lr}
-; CHECK-T2-NEXT:    push {r4, lr}
-; CHECK-T2-NEXT:    cmp r3, #0
-; CHECK-T2-NEXT:    mov.w lr, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi.w lr, #1
-; CHECK-T2-NEXT:    cmp r1, #0
-; CHECK-T2-NEXT:    mov.w r4, #0
-; CHECK-T2-NEXT:    mov.w r12, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r4, #1
-; CHECK-T2-NEXT:    subs.w lr, r4, lr
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    movne.w lr, #1
 ; CHECK-T2-NEXT:    subs r0, r0, r2
-; CHECK-T2-NEXT:    sbcs.w r2, r1, r3
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi.w r12, #1
-; CHECK-T2-NEXT:    subs.w r1, r4, r12
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    movne r1, #1
-; CHECK-T2-NEXT:    ands.w r3, lr, r1
+; CHECK-T2-NEXT:    eor.w r12, r1, r3
+; CHECK-T2-NEXT:    sbc.w r2, r1, r3
+; CHECK-T2-NEXT:    eors r1, r2
+; CHECK-T2-NEXT:    ands.w r3, r12, r1
 ; CHECK-T2-NEXT:    mov.w r1, #-2147483648
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    asrne r0, r2, #31
+; CHECK-T2-NEXT:    it mi
+; CHECK-T2-NEXT:    asrmi r0, r2, #31
 ; CHECK-T2-NEXT:    cmp r2, #0
 ; CHECK-T2-NEXT:    it mi
 ; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
 ; CHECK-T2-NEXT:    cmp r3, #0
-; CHECK-T2-NEXT:    it eq
-; CHECK-T2-NEXT:    moveq r1, r2
-; CHECK-T2-NEXT:    pop {r4, pc}
+; CHECK-T2-NEXT:    it pl
+; CHECK-T2-NEXT:    movpl r1, r2
+; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func2:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    .save {r4, lr}
-; CHECK-ARM-NEXT:    push {r4, lr}
-; CHECK-ARM-NEXT:    cmp r3, #0
-; CHECK-ARM-NEXT:    mov lr, #0
-; CHECK-ARM-NEXT:    movmi lr, #1
-; CHECK-ARM-NEXT:    cmp r1, #0
-; CHECK-ARM-NEXT:    mov r4, #0
-; CHECK-ARM-NEXT:    mov r12, #0
-; CHECK-ARM-NEXT:    movmi r4, #1
-; CHECK-ARM-NEXT:    subs lr, r4, lr
-; CHECK-ARM-NEXT:    movne lr, #1
 ; CHECK-ARM-NEXT:    subs r0, r0, r2
-; CHECK-ARM-NEXT:    sbcs r2, r1, r3
-; CHECK-ARM-NEXT:    movmi r12, #1
-; CHECK-ARM-NEXT:    subs r1, r4, r12
-; CHECK-ARM-NEXT:    movne r1, #1
-; CHECK-ARM-NEXT:    ands r3, lr, r1
-; CHECK-ARM-NEXT:    asrne r0, r2, #31
+; CHECK-ARM-NEXT:    eor r12, r1, r3
+; CHECK-ARM-NEXT:    sbc r2, r1, r3
+; CHECK-ARM-NEXT:    eor r1, r1, r2
+; CHECK-ARM-NEXT:    ands r3, r12, r1
 ; CHECK-ARM-NEXT:    mov r1, #-2147483648
+; CHECK-ARM-NEXT:    asrmi r0, r2, #31
 ; CHECK-ARM-NEXT:    cmp r2, #0
 ; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
 ; CHECK-ARM-NEXT:    cmp r3, #0
-; CHECK-ARM-NEXT:    moveq r1, r2
-; CHECK-ARM-NEXT:    pop {r4, pc}
+; CHECK-ARM-NEXT:    movpl r1, r2
+; CHECK-ARM-NEXT:    bx lr
   %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y)
   ret i64 %tmp
 }
diff --git a/llvm/test/CodeGen/ARM/ssub_sat_plus.ll b/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
index a4b4105..989e0aa 100644
--- a/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
+++ b/llvm/test/CodeGen/ARM/ssub_sat_plus.ll
@@ -83,131 +83,79 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; CHECK-T1-LABEL: func64:
 ; CHECK-T1:       @ %bb.0:
-; CHECK-T1-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-T1-NEXT:    .pad #4
-; CHECK-T1-NEXT:    sub sp, #4
-; CHECK-T1-NEXT:    ldr r5, [sp, #28]
-; CHECK-T1-NEXT:    movs r2, #1
-; CHECK-T1-NEXT:    movs r4, #0
-; CHECK-T1-NEXT:    cmp r5, #0
-; CHECK-T1-NEXT:    mov r3, r2
-; CHECK-T1-NEXT:    bmi .LBB1_2
+; CHECK-T1-NEXT:    .save {r4, lr}
+; CHECK-T1-NEXT:    push {r4, lr}
+; CHECK-T1-NEXT:    ldr r2, [sp, #12]
+; CHECK-T1-NEXT:    mov r4, r1
+; CHECK-T1-NEXT:    eors r4, r2
+; CHECK-T1-NEXT:    ldr r3, [sp, #8]
+; CHECK-T1-NEXT:    subs r0, r0, r3
+; CHECK-T1-NEXT:    mov r3, r1
+; CHECK-T1-NEXT:    sbcs r3, r2
+; CHECK-T1-NEXT:    eors r1, r3
+; CHECK-T1-NEXT:    ands r1, r4
+; CHECK-T1-NEXT:    bpl .LBB1_2
 ; CHECK-T1-NEXT:  @ %bb.1:
-; CHECK-T1-NEXT:    mov r3, r4
+; CHECK-T1-NEXT:    asrs r0, r3, #31
 ; CHECK-T1-NEXT:  .LBB1_2:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    mov r6, r2
+; CHECK-T1-NEXT:    cmp r3, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_4
 ; CHECK-T1-NEXT:  @ %bb.3:
-; CHECK-T1-NEXT:    mov r6, r4
+; CHECK-T1-NEXT:    movs r2, #1
+; CHECK-T1-NEXT:    lsls r2, r2, #31
+; CHECK-T1-NEXT:    cmp r1, #0
+; CHECK-T1-NEXT:    bpl .LBB1_5
+; CHECK-T1-NEXT:    b .LBB1_6
 ; CHECK-T1-NEXT:  .LBB1_4:
-; CHECK-T1-NEXT:    subs r3, r6, r3
-; CHECK-T1-NEXT:    subs r7, r3, #1
-; CHECK-T1-NEXT:    sbcs r3, r7
-; CHECK-T1-NEXT:    ldr r7, [sp, #24]
-; CHECK-T1-NEXT:    subs r0, r0, r7
-; CHECK-T1-NEXT:    sbcs r1, r5
-; CHECK-T1-NEXT:    mov r5, r2
+; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
+; CHECK-T1-NEXT:    cmp r1, #0
 ; CHECK-T1-NEXT:    bmi .LBB1_6
-; CHECK-T1-NEXT:  @ %bb.5:
-; CHECK-T1-NEXT:    mov r5, r4
+; CHECK-T1-NEXT:  .LBB1_5:
+; CHECK-T1-NEXT:    mov r2, r3
 ; CHECK-T1-NEXT:  .LBB1_6:
-; CHECK-T1-NEXT:    subs r4, r6, r5
-; CHECK-T1-NEXT:    subs r5, r4, #1
-; CHECK-T1-NEXT:    sbcs r4, r5
-; CHECK-T1-NEXT:    ands r3, r4
-; CHECK-T1-NEXT:    beq .LBB1_8
-; CHECK-T1-NEXT:  @ %bb.7:
-; CHECK-T1-NEXT:    asrs r0, r1, #31
-; CHECK-T1-NEXT:  .LBB1_8:
-; CHECK-T1-NEXT:    cmp r1, #0
-; CHECK-T1-NEXT:    bmi .LBB1_10
-; CHECK-T1-NEXT:  @ %bb.9:
-; CHECK-T1-NEXT:    lsls r2, r2, #31
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    beq .LBB1_11
-; CHECK-T1-NEXT:    b .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_10:
-; CHECK-T1-NEXT:    ldr r2, .LCPI1_0
-; CHECK-T1-NEXT:    cmp r3, #0
-; CHECK-T1-NEXT:    bne .LBB1_12
-; CHECK-T1-NEXT:  .LBB1_11:
-; CHECK-T1-NEXT:    mov r2, r1
-; CHECK-T1-NEXT:  .LBB1_12:
 ; CHECK-T1-NEXT:    mov r1, r2
-; CHECK-T1-NEXT:    add sp, #4
-; CHECK-T1-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-T1-NEXT:    pop {r4, pc}
 ; CHECK-T1-NEXT:    .p2align 2
-; CHECK-T1-NEXT:  @ %bb.13:
+; CHECK-T1-NEXT:  @ %bb.7:
 ; CHECK-T1-NEXT:  .LCPI1_0:
 ; CHECK-T1-NEXT:    .long 2147483647 @ 0x7fffffff
 ;
 ; CHECK-T2-LABEL: func64:
 ; CHECK-T2:       @ %bb.0:
-; CHECK-T2-NEXT:    .save {r4, lr}
-; CHECK-T2-NEXT:    push {r4, lr}
-; CHECK-T2-NEXT:    ldr.w r12, [sp, #12]
-; CHECK-T2-NEXT:    movs r2, #0
-; CHECK-T2-NEXT:    movs r3, #0
-; CHECK-T2-NEXT:    ldr r4, [sp, #8]
-; CHECK-T2-NEXT:    cmp.w r12, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r2, #1
-; CHECK-T2-NEXT:    cmp r1, #0
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi r3, #1
-; CHECK-T2-NEXT:    subs r2, r3, r2
-; CHECK-T2-NEXT:    mov.w lr, #0
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    movne r2, #1
-; CHECK-T2-NEXT:    subs r0, r0, r4
-; CHECK-T2-NEXT:    sbcs.w r4, r1, r12
-; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    movmi.w lr, #1
-; CHECK-T2-NEXT:    subs.w r1, r3, lr
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    movne r1, #1
-; CHECK-T2-NEXT:    ands r2, r1
+; CHECK-T2-NEXT:    ldr r2, [sp]
+; CHECK-T2-NEXT:    ldr.w r12, [sp, #4]
+; CHECK-T2-NEXT:    subs r0, r0, r2
+; CHECK-T2-NEXT:    sbc.w r2, r1, r12
+; CHECK-T2-NEXT:    eor.w r3, r1, r12
+; CHECK-T2-NEXT:    eors r1, r2
+; CHECK-T2-NEXT:    ands r3, r1
 ; CHECK-T2-NEXT:    mov.w r1, #-2147483648
-; CHECK-T2-NEXT:    it ne
-; CHECK-T2-NEXT:    asrne r0, r4, #31
-; CHECK-T2-NEXT:    cmp r4, #0
 ; CHECK-T2-NEXT:    it mi
-; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
+; CHECK-T2-NEXT:    asrmi r0, r2, #31
 ; CHECK-T2-NEXT:    cmp r2, #0
-; CHECK-T2-NEXT:    it eq
-; CHECK-T2-NEXT:    moveq r1, r4
-; CHECK-T2-NEXT:    pop {r4, pc}
+; CHECK-T2-NEXT:    it mi
+; CHECK-T2-NEXT:    mvnmi r1, #-2147483648
+; CHECK-T2-NEXT:    cmp r3, #0
+; CHECK-T2-NEXT:    it pl
+; CHECK-T2-NEXT:    movpl r1, r2
+; CHECK-T2-NEXT:    bx lr
 ;
 ; CHECK-ARM-LABEL: func64:
 ; CHECK-ARM:       @ %bb.0:
-; CHECK-ARM-NEXT:    .save {r4, lr}
-; CHECK-ARM-NEXT:    push {r4, lr}
-; CHECK-ARM-NEXT:    ldr lr, [sp, #12]
-; CHECK-ARM-NEXT:    cmp r1, #0
-; CHECK-ARM-NEXT:    mov r3, #0
-; CHECK-ARM-NEXT:    mov r4, #0
-; CHECK-ARM-NEXT:    movwmi r3, #1
-; CHECK-ARM-NEXT:    cmp lr, #0
-; CHECK-ARM-NEXT:    movwmi r4, #1
-; CHECK-ARM-NEXT:    ldr r12, [sp, #8]
-; CHECK-ARM-NEXT:    subs r4, r3, r4
-; CHECK-ARM-NEXT:    mov r2, #0
-; CHECK-ARM-NEXT:    movwne r4, #1
+; CHECK-ARM-NEXT:    ldr r12, [sp]
+; CHECK-ARM-NEXT:    ldr r2, [sp, #4]
 ; CHECK-ARM-NEXT:    subs r0, r0, r12
-; CHECK-ARM-NEXT:    sbcs r12, r1, lr
-; CHECK-ARM-NEXT:    movwmi r2, #1
-; CHECK-ARM-NEXT:    subs r1, r3, r2
-; CHECK-ARM-NEXT:    movwne r1, #1
-; CHECK-ARM-NEXT:    ands r2, r4, r1
-; CHECK-ARM-NEXT:    asrne r0, r12, #31
+; CHECK-ARM-NEXT:    eor r3, r1, r2
+; CHECK-ARM-NEXT:    sbc r2, r1, r2
+; CHECK-ARM-NEXT:    eor r1, r1, r2
+; CHECK-ARM-NEXT:    ands r3, r3, r1
 ; CHECK-ARM-NEXT:    mov r1, #-2147483648
-; CHECK-ARM-NEXT:    cmp r12, #0
-; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    asrmi r0, r2, #31
 ; CHECK-ARM-NEXT:    cmp r2, #0
-; CHECK-ARM-NEXT:    moveq r1, r12
-; CHECK-ARM-NEXT:    pop {r4, pc}
+; CHECK-ARM-NEXT:    mvnmi r1, #-2147483648
+; CHECK-ARM-NEXT:    cmp r3, #0
+; CHECK-ARM-NEXT:    movpl r1, r2
+; CHECK-ARM-NEXT:    bx lr
   %a = mul i64 %y, %z
   %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %z)
   ret i64 %tmp
diff --git a/llvm/test/CodeGen/RISCV/sadd_sat.ll b/llvm/test/CodeGen/RISCV/sadd_sat.ll
index e6f7d17..440890f 100644
--- a/llvm/test/CodeGen/RISCV/sadd_sat.ll
+++ b/llvm/test/CodeGen/RISCV/sadd_sat.ll
@@ -90,15 +90,11 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    slti a5, a2, 0
-; RV32I-NEXT:    slti a4, a4, 0
-; RV32I-NEXT:    xor a5, a4, a5
-; RV32I-NEXT:    snez a5, a5
-; RV32I-NEXT:    slti a3, a3, 0
+; RV32I-NEXT:    xor a5, a4, a2
 ; RV32I-NEXT:    xor a3, a4, a3
-; RV32I-NEXT:    seqz a3, a3
+; RV32I-NEXT:    not a3, a3
 ; RV32I-NEXT:    and a3, a3, a5
-; RV32I-NEXT:    bnez a3, .LBB1_4
+; RV32I-NEXT:    bltz a3, .LBB1_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    mv a1, a2
 ; RV32I-NEXT:    ret
@@ -138,15 +134,10 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; RV32IZbb-NEXT:  # %bb.1:
 ; RV32IZbb-NEXT:    addi a1, a1, -1
 ; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    slti a5, a2, 0
-; RV32IZbb-NEXT:    slti a4, a4, 0
-; RV32IZbb-NEXT:    xor a5, a4, a5
-; RV32IZbb-NEXT:    snez a5, a5
-; RV32IZbb-NEXT:    slti a3, a3, 0
+; RV32IZbb-NEXT:    xor a5, a4, a2
 ; RV32IZbb-NEXT:    xor a3, a4, a3
-; RV32IZbb-NEXT:    seqz a3, a3
-; RV32IZbb-NEXT:    and a3, a3, a5
-; RV32IZbb-NEXT:    bnez a3, .LBB1_4
+; RV32IZbb-NEXT:    andn a3, a5, a3
+; RV32IZbb-NEXT:    bltz a3, .LBB1_4
 ; RV32IZbb-NEXT:  # %bb.3:
 ; RV32IZbb-NEXT:    mv a1, a2
 ; RV32IZbb-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll b/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
index 156141b..d38298a 100644
--- a/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
@@ -99,15 +99,11 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    slti a4, a3, 0
-; RV32I-NEXT:    slti a2, a2, 0
-; RV32I-NEXT:    xor a4, a2, a4
-; RV32I-NEXT:    snez a4, a4
-; RV32I-NEXT:    slti a5, a5, 0
+; RV32I-NEXT:    xor a4, a2, a3
 ; RV32I-NEXT:    xor a2, a2, a5
-; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    not a2, a2
 ; RV32I-NEXT:    and a2, a2, a4
-; RV32I-NEXT:    bnez a2, .LBB1_4
+; RV32I-NEXT:    bltz a2, .LBB1_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    mv a1, a3
 ; RV32I-NEXT:    ret
@@ -147,15 +143,10 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; RV32IZbb-NEXT:  # %bb.1:
 ; RV32IZbb-NEXT:    addi a1, a1, -1
 ; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    slti a4, a3, 0
-; RV32IZbb-NEXT:    slti a2, a2, 0
-; RV32IZbb-NEXT:    xor a4, a2, a4
-; RV32IZbb-NEXT:    snez a4, a4
-; RV32IZbb-NEXT:    slti a5, a5, 0
+; RV32IZbb-NEXT:    xor a4, a2, a3
 ; RV32IZbb-NEXT:    xor a2, a2, a5
-; RV32IZbb-NEXT:    seqz a2, a2
-; RV32IZbb-NEXT:    and a2, a2, a4
-; RV32IZbb-NEXT:    bnez a2, .LBB1_4
+; RV32IZbb-NEXT:    andn a2, a4, a2
+; RV32IZbb-NEXT:    bltz a2, .LBB1_4
 ; RV32IZbb-NEXT:  # %bb.3:
 ; RV32IZbb-NEXT:    mv a1, a3
 ; RV32IZbb-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/ssub_sat.ll b/llvm/test/CodeGen/RISCV/ssub_sat.ll
index e89744f..47565d4 100644
--- a/llvm/test/CodeGen/RISCV/ssub_sat.ll
+++ b/llvm/test/CodeGen/RISCV/ssub_sat.ll
@@ -81,30 +81,25 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 define i64 @func2(i64 %x, i64 %y) nounwind {
 ; RV32I-LABEL: func2:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a6, a1
+; RV32I-NEXT:    mv a4, a1
 ; RV32I-NEXT:    sltu a1, a0, a2
-; RV32I-NEXT:    sub a5, a6, a3
-; RV32I-NEXT:    sub t0, a5, a1
+; RV32I-NEXT:    sub a5, a4, a3
+; RV32I-NEXT:    sub a5, a5, a1
 ; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    bgez t0, .LBB1_2
+; RV32I-NEXT:    bgez a5, .LBB1_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    slti a7, t0, 0
-; RV32I-NEXT:    slti a4, a6, 0
-; RV32I-NEXT:    xor a5, a4, a7
-; RV32I-NEXT:    snez a5, a5
-; RV32I-NEXT:    slti a3, a3, 0
+; RV32I-NEXT:    xor a6, a4, a5
 ; RV32I-NEXT:    xor a3, a4, a3
-; RV32I-NEXT:    snez a3, a3
-; RV32I-NEXT:    and a3, a3, a5
-; RV32I-NEXT:    bnez a3, .LBB1_4
+; RV32I-NEXT:    and a3, a3, a6
+; RV32I-NEXT:    bltz a3, .LBB1_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    sub a0, a0, a2
-; RV32I-NEXT:    mv a1, t0
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB1_4:
-; RV32I-NEXT:    srai a0, t0, 31
+; RV32I-NEXT:    srai a0, a5, 31
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: func2:
@@ -129,30 +124,25 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ;
 ; RV32IZbb-LABEL: func2:
 ; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a6, a1
+; RV32IZbb-NEXT:    mv a4, a1
 ; RV32IZbb-NEXT:    sltu a1, a0, a2
-; RV32IZbb-NEXT:    sub a5, a6, a3
-; RV32IZbb-NEXT:    sub t0, a5, a1
+; RV32IZbb-NEXT:    sub a5, a4, a3
+; RV32IZbb-NEXT:    sub a5, a5, a1
 ; RV32IZbb-NEXT:    lui a1, 524288
-; RV32IZbb-NEXT:    bgez t0, .LBB1_2
+; RV32IZbb-NEXT:    bgez a5, .LBB1_2
 ; RV32IZbb-NEXT:  # %bb.1:
 ; RV32IZbb-NEXT:    addi a1, a1, -1
 ; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    slti a7, t0, 0
-; RV32IZbb-NEXT:    slti a4, a6, 0
-; RV32IZbb-NEXT:    xor a5, a4, a7
-; RV32IZbb-NEXT:    snez a5, a5
-; RV32IZbb-NEXT:    slti a3, a3, 0
+; RV32IZbb-NEXT:    xor a6, a4, a5
 ; RV32IZbb-NEXT:    xor a3, a4, a3
-; RV32IZbb-NEXT:    snez a3, a3
-; RV32IZbb-NEXT:    and a3, a3, a5
-; RV32IZbb-NEXT:    bnez a3, .LBB1_4
+; RV32IZbb-NEXT:    and a3, a3, a6
+; RV32IZbb-NEXT:    bltz a3, .LBB1_4
 ; RV32IZbb-NEXT:  # %bb.3:
 ; RV32IZbb-NEXT:    sub a0, a0, a2
-; RV32IZbb-NEXT:    mv a1, t0
+; RV32IZbb-NEXT:    mv a1, a5
 ; RV32IZbb-NEXT:    ret
 ; RV32IZbb-NEXT:  .LBB1_4:
-; RV32IZbb-NEXT:    srai a0, t0, 31
+; RV32IZbb-NEXT:    srai a0, a5, 31
 ; RV32IZbb-NEXT:    ret
 ;
 ; RV64IZbb-LABEL: func2:
diff --git a/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll b/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
index 8c0b5ac..f94f5f6 100644
--- a/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/ssub_sat_plus.ll
@@ -88,30 +88,25 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; RV32I-LABEL: func64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a6, a1
+; RV32I-NEXT:    mv a2, a1
 ; RV32I-NEXT:    sltu a1, a0, a4
-; RV32I-NEXT:    sub a3, a6, a5
-; RV32I-NEXT:    sub t0, a3, a1
+; RV32I-NEXT:    sub a3, a2, a5
+; RV32I-NEXT:    sub a3, a3, a1
 ; RV32I-NEXT:    lui a1, 524288
-; RV32I-NEXT:    bgez t0, .LBB1_2
+; RV32I-NEXT:    bgez a3, .LBB1_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    slti a7, t0, 0
-; RV32I-NEXT:    slti a2, a6, 0
-; RV32I-NEXT:    xor a3, a2, a7
-; RV32I-NEXT:    snez a3, a3
-; RV32I-NEXT:    slti a5, a5, 0
+; RV32I-NEXT:    xor a6, a2, a3
 ; RV32I-NEXT:    xor a2, a2, a5
-; RV32I-NEXT:    snez a2, a2
-; RV32I-NEXT:    and a2, a2, a3
-; RV32I-NEXT:    bnez a2, .LBB1_4
+; RV32I-NEXT:    and a2, a2, a6
+; RV32I-NEXT:    bltz a2, .LBB1_4
 ; RV32I-NEXT:  # %bb.3:
 ; RV32I-NEXT:    sub a0, a0, a4
-; RV32I-NEXT:    mv a1, t0
+; RV32I-NEXT:    mv a1, a3
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB1_4:
-; RV32I-NEXT:    srai a0, t0, 31
+; RV32I-NEXT:    srai a0, a3, 31
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: func64:
@@ -136,30 +131,25 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ;
 ; RV32IZbb-LABEL: func64:
 ; RV32IZbb:       # %bb.0:
-; RV32IZbb-NEXT:    mv a6, a1
+; RV32IZbb-NEXT:    mv a2, a1
 ; RV32IZbb-NEXT:    sltu a1, a0, a4
-; RV32IZbb-NEXT:    sub a3, a6, a5
-; RV32IZbb-NEXT:    sub t0, a3, a1
+; RV32IZbb-NEXT:    sub a3, a2, a5
+; RV32IZbb-NEXT:    sub a3, a3, a1
 ; RV32IZbb-NEXT:    lui a1, 524288
-; RV32IZbb-NEXT:    bgez t0, .LBB1_2
+; RV32IZbb-NEXT:    bgez a3, .LBB1_2
 ; RV32IZbb-NEXT:  # %bb.1:
 ; RV32IZbb-NEXT:    addi a1, a1, -1
 ; RV32IZbb-NEXT:  .LBB1_2:
-; RV32IZbb-NEXT:    slti a7, t0, 0
-; RV32IZbb-NEXT:    slti a2, a6, 0
-; RV32IZbb-NEXT:    xor a3, a2, a7
-; RV32IZbb-NEXT:    snez a3, a3
-; RV32IZbb-NEXT:    slti a5, a5, 0
+; RV32IZbb-NEXT:    xor a6, a2, a3
 ; RV32IZbb-NEXT:    xor a2, a2, a5
-; RV32IZbb-NEXT:    snez a2, a2
-; RV32IZbb-NEXT:    and a2, a2, a3
-; RV32IZbb-NEXT:    bnez a2, .LBB1_4
+; RV32IZbb-NEXT:    and a2, a2, a6
+; RV32IZbb-NEXT:    bltz a2, .LBB1_4
 ; RV32IZbb-NEXT:  # %bb.3:
 ; RV32IZbb-NEXT:    sub a0, a0, a4
-; RV32IZbb-NEXT:    mv a1, t0
+; RV32IZbb-NEXT:    mv a1, a3
 ; RV32IZbb-NEXT:    ret
 ; RV32IZbb-NEXT:  .LBB1_4:
-; RV32IZbb-NEXT:    srai a0, t0, 31
+; RV32IZbb-NEXT:    srai a0, a3, 31
 ; RV32IZbb-NEXT:    ret
 ;
 ; RV64IZbb-LABEL: func64:
diff --git a/llvm/test/CodeGen/RISCV/xaluo.ll b/llvm/test/CodeGen/RISCV/xaluo.ll
index 63ac7b2..6d06528 100644
--- a/llvm/test/CodeGen/RISCV/xaluo.ll
+++ b/llvm/test/CodeGen/RISCV/xaluo.ll
@@ -123,14 +123,11 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV32-NEXT:    add a2, a0, a2
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a5, a5, a0
-; RV32-NEXT:    slti a0, a5, 0
-; RV32-NEXT:    slti a1, a1, 0
-; RV32-NEXT:    xor a0, a1, a0
-; RV32-NEXT:    snez a0, a0
-; RV32-NEXT:    slti a3, a3, 0
+; RV32-NEXT:    xor a0, a1, a5
 ; RV32-NEXT:    xor a1, a1, a3
-; RV32-NEXT:    seqz a1, a1
+; RV32-NEXT:    not a1, a1
 ; RV32-NEXT:    and a0, a1, a0
+; RV32-NEXT:    slti a0, a0, 0
 ; RV32-NEXT:    sw a2, 0(a4)
 ; RV32-NEXT:    sw a5, 4(a4)
 ; RV32-NEXT:    ret
@@ -154,18 +151,15 @@ entry:
 define zeroext i1 @saddo2.i64(i64 %v1, i64* %res) {
 ; RV32-LABEL: saddo2.i64:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    addi a3, zero, -1
-; RV32-NEXT:    slt a3, a3, a1
-; RV32-NEXT:    addi a4, a0, 4
-; RV32-NEXT:    sltu a0, a4, a0
-; RV32-NEXT:    add a5, a1, a0
-; RV32-NEXT:    slti a0, a5, 0
-; RV32-NEXT:    slti a1, a1, 0
-; RV32-NEXT:    xor a0, a1, a0
-; RV32-NEXT:    snez a0, a0
-; RV32-NEXT:    and a0, a3, a0
-; RV32-NEXT:    sw a4, 0(a2)
-; RV32-NEXT:    sw a5, 4(a2)
+; RV32-NEXT:    addi a3, a0, 4
+; RV32-NEXT:    sltu a0, a3, a0
+; RV32-NEXT:    add a4, a1, a0
+; RV32-NEXT:    xor a0, a1, a4
+; RV32-NEXT:    not a1, a1
+; RV32-NEXT:    and a0, a1, a0
+; RV32-NEXT:    slti a0, a0, 0
+; RV32-NEXT:    sw a3, 0(a2)
+; RV32-NEXT:    sw a4, 4(a2)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: saddo2.i64:
@@ -189,11 +183,9 @@ define zeroext i1 @saddo3.i64(i64 %v1, i64* %res) {
 ; RV32-NEXT:    sltu a0, a3, a0
 ; RV32-NEXT:    add a0, a1, a0
 ; RV32-NEXT:    addi a4, a0, -1
-; RV32-NEXT:    slti a0, a4, 0
-; RV32-NEXT:    slti a1, a1, 0
-; RV32-NEXT:    xor a0, a1, a0
-; RV32-NEXT:    snez a0, a0
+; RV32-NEXT:    xor a0, a1, a4
 ; RV32-NEXT:    and a0, a1, a0
+; RV32-NEXT:    slti a0, a0, 0
 ; RV32-NEXT:    sw a3, 0(a2)
 ; RV32-NEXT:    sw a4, 4(a2)
 ; RV32-NEXT:    ret
@@ -329,18 +321,14 @@ define zeroext i1 @ssubo.i64(i64 %v1, i64 %v2, i64* %res) {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    sltu a6, a0, a2
 ; RV32-NEXT:    sub a5, a1, a3
-; RV32-NEXT:    sub a7, a5, a6
-; RV32-NEXT:    slti a6, a7, 0
-; RV32-NEXT:    slti a1, a1, 0
-; RV32-NEXT:    xor a5, a1, a6
-; RV32-NEXT:    snez a5, a5
-; RV32-NEXT:    slti a3, a3, 0
+; RV32-NEXT:    sub a5, a5, a6
+; RV32-NEXT:    xor a6, a1, a5
 ; RV32-NEXT:    xor a1, a1, a3
-; RV32-NEXT:    snez a1, a1
-; RV32-NEXT:    and a1, a1, a5
+; RV32-NEXT:    and a1, a1, a6
+; RV32-NEXT:    slti a1, a1, 0
 ; RV32-NEXT:    sub a0, a0, a2
 ; RV32-NEXT:    sw a0, 0(a4)
-; RV32-NEXT:    sw a7, 4(a4)
+; RV32-NEXT:    sw a5, 4(a4)
 ; RV32-NEXT:    mv a0, a1
 ; RV32-NEXT:    ret
 ;
@@ -776,15 +764,11 @@ define i64 @saddo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    add a5, a0, a2
 ; RV32-NEXT:    sltu a5, a5, a0
 ; RV32-NEXT:    add a4, a4, a5
-; RV32-NEXT:    slti a4, a4, 0
-; RV32-NEXT:    slti a5, a1, 0
-; RV32-NEXT:    xor a4, a5, a4
-; RV32-NEXT:    snez a6, a4
-; RV32-NEXT:    slti a4, a3, 0
-; RV32-NEXT:    xor a4, a5, a4
-; RV32-NEXT:    seqz a4, a4
-; RV32-NEXT:    and a4, a4, a6
-; RV32-NEXT:    bnez a4, .LBB24_2
+; RV32-NEXT:    xor a4, a1, a4
+; RV32-NEXT:    xor a5, a1, a3
+; RV32-NEXT:    not a5, a5
+; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    bltz a4, .LBB24_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a2
 ; RV32-NEXT:    mv a1, a3
@@ -816,15 +800,12 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    add a2, a0, a2
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a4, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    slti a1, a1, 0
 ; RV32-NEXT:    xor a0, a1, a0
-; RV32-NEXT:    snez a0, a0
-; RV32-NEXT:    slti a2, a3, 0
-; RV32-NEXT:    xor a1, a1, a2
-; RV32-NEXT:    seqz a1, a1
+; RV32-NEXT:    xor a1, a1, a3
+; RV32-NEXT:    not a1, a1
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    addi a1, zero, -1
+; RV32-NEXT:    slt a0, a1, a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: saddo.not.i64:
@@ -1027,15 +1008,10 @@ define i64 @ssubo.select.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    sltu a4, a0, a2
 ; RV32-NEXT:    sub a5, a1, a3
 ; RV32-NEXT:    sub a4, a5, a4
-; RV32-NEXT:    slti a4, a4, 0
-; RV32-NEXT:    slti a5, a1, 0
-; RV32-NEXT:    xor a4, a5, a4
-; RV32-NEXT:    snez a6, a4
-; RV32-NEXT:    slti a4, a3, 0
-; RV32-NEXT:    xor a4, a5, a4
-; RV32-NEXT:    snez a4, a4
-; RV32-NEXT:    and a4, a4, a6
-; RV32-NEXT:    bnez a4, .LBB32_2
+; RV32-NEXT:    xor a4, a1, a4
+; RV32-NEXT:    xor a5, a1, a3
+; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    bltz a4, .LBB32_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a2
 ; RV32-NEXT:    mv a1, a3
@@ -1066,15 +1042,11 @@ define i1 @ssub.not.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    sub a2, a1, a3
 ; RV32-NEXT:    sub a0, a2, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    slti a1, a1, 0
 ; RV32-NEXT:    xor a0, a1, a0
-; RV32-NEXT:    snez a0, a0
-; RV32-NEXT:    slti a2, a3, 0
-; RV32-NEXT:    xor a1, a1, a2
-; RV32-NEXT:    snez a1, a1
+; RV32-NEXT:    xor a1, a1, a3
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    addi a1, zero, -1
+; RV32-NEXT:    slt a0, a1, a0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ssub.not.i64:
@@ -1562,15 +1534,12 @@ define zeroext i1 @saddo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    add a2, a0, a2
 ; RV32-NEXT:    sltu a0, a2, a0
 ; RV32-NEXT:    add a0, a4, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    slti a1, a1, 0
 ; RV32-NEXT:    xor a0, a1, a0
-; RV32-NEXT:    snez a0, a0
-; RV32-NEXT:    slti a2, a3, 0
-; RV32-NEXT:    xor a1, a1, a2
-; RV32-NEXT:    seqz a1, a1
+; RV32-NEXT:    xor a1, a1, a3
+; RV32-NEXT:    not a1, a1
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    beqz a0, .LBB47_2
+; RV32-NEXT:    addi a1, zero, -1
+; RV32-NEXT:    blt a1, a0, .LBB47_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
@@ -1734,15 +1703,11 @@ define zeroext i1 @ssubo.br.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    sltu a0, a0, a2
 ; RV32-NEXT:    sub a2, a1, a3
 ; RV32-NEXT:    sub a0, a2, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    slti a1, a1, 0
 ; RV32-NEXT:    xor a0, a1, a0
-; RV32-NEXT:    snez a0, a0
-; RV32-NEXT:    slti a2, a3, 0
-; RV32-NEXT:    xor a1, a1, a2
-; RV32-NEXT:    snez a1, a1
+; RV32-NEXT:    xor a1, a1, a3
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    beqz a0, .LBB51_2
+; RV32-NEXT:    addi a1, zero, -1
+; RV32-NEXT:    blt a1, a0, .LBB51_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    mv a0, zero
 ; RV32-NEXT:    ret
diff --git a/llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll b/llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll
index 65c88cc..172213ca 100644
--- a/llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-saturating-arith.ll
@@ -34,58 +34,52 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @sadd_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
 ; CHECK-LABEL: sadd_int64_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    vmov r0, s7
-; CHECK-NEXT:    vmov r2, s3
-; CHECK-NEXT:    vmov lr, s6
-; CHECK-NEXT:    vmov r4, s1
-; CHECK-NEXT:    vmov r6, s4
-; CHECK-NEXT:    vmov r7, s0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    cset r1, mi
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    cset r3, mi
-; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    cset r12, eq
-; CHECK-NEXT:    adds.w r1, r1, lr
-; CHECK-NEXT:    adcs.w lr, r2, r0
-; CHECK-NEXT:    cset r2, mi
-; CHECK-NEXT:    cmp r3, r2
-; CHECK-NEXT:    cset r3, ne
-; CHECK-NEXT:    and.w r3, r3, r12
-; CHECK-NEXT:    ands r12, r3, #1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r4, s4
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    adds r2, r2, r3
 ; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    eor.w r12, r1, r0
+; CHECK-NEXT:    adcs r0, r1
+; CHECK-NEXT:    eors r1, r0
+; CHECK-NEXT:    bic.w r1, r1, r12
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    cset r1, mi
+; CHECK-NEXT:    ands r12, r1, #1
+; CHECK-NEXT:    vmov r1, s1
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    asrne.w r1, lr, #31
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    cset r0, mi
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    cset r5, mi
-; CHECK-NEXT:    cmp r5, r0
-; CHECK-NEXT:    cset r0, eq
-; CHECK-NEXT:    adds r6, r6, r7
-; CHECK-NEXT:    adcs r3, r4
-; CHECK-NEXT:    cset r4, mi
-; CHECK-NEXT:    cmp r5, r4
-; CHECK-NEXT:    cset r5, ne
-; CHECK-NEXT:    ands r0, r5
-; CHECK-NEXT:    ands r0, r0, #1
+; CHECK-NEXT:    asrne r2, r0, #31
+; CHECK-NEXT:    adds r4, r4, r5
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    eor.w lr, r1, r3
+; CHECK-NEXT:    adcs r3, r1
+; CHECK-NEXT:    eors r1, r3
+; CHECK-NEXT:    bic.w r1, r1, lr
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    cset r1, mi
+; CHECK-NEXT:    ands r1, r1, #1
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    asrne r6, r3, #31
-; CHECK-NEXT:    vmov q0[2], q0[0], r6, r1
-; CHECK-NEXT:    mvn r1, #-2147483648
+; CHECK-NEXT:    asrne r4, r3, #31
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT:    cset r2, mi
 ; CHECK-NEXT:    tst.w r2, #1
-; CHECK-NEXT:    cinv r2, r1, eq
+; CHECK-NEXT:    cinv r2, r5, eq
 ; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r2, r2, lr, ne
-; CHECK-NEXT:    tst.w r4, #1
-; CHECK-NEXT:    cinv r1, r1, eq
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r3, ne
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r2
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    csel r0, r2, r0, ne
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    cset r2, mi
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    cinv r2, r5, eq
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csel r1, r2, r3, ne
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
   ret <2 x i64> %0
@@ -193,58 +187,50 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ssub_int64_t(<2 x i64> %src1, <2 x i64> %src2) {
 ; CHECK-LABEL: ssub_int64_t:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    vmov r3, s6
+; CHECK-NEXT:    vmov r2, s2
 ; CHECK-NEXT:    vmov r0, s7
-; CHECK-NEXT:    vmov r2, s3
-; CHECK-NEXT:    vmov lr, s6
-; CHECK-NEXT:    vmov r4, s1
-; CHECK-NEXT:    vmov r6, s4
-; CHECK-NEXT:    vmov r7, s0
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    cset r1, mi
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    cset r3, mi
-; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    vmov r1, s2
-; CHECK-NEXT:    cset r12, ne
-; CHECK-NEXT:    subs.w r1, r1, lr
-; CHECK-NEXT:    sbcs.w lr, r2, r0
-; CHECK-NEXT:    cset r2, mi
-; CHECK-NEXT:    cmp r3, r2
-; CHECK-NEXT:    cset r3, ne
-; CHECK-NEXT:    and.w r3, r3, r12
-; CHECK-NEXT:    ands r12, r3, #1
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    vmov r4, s4
+; CHECK-NEXT:    vmov r5, s0
+; CHECK-NEXT:    subs r2, r2, r3
 ; CHECK-NEXT:    vmov r3, s5
+; CHECK-NEXT:    eor.w r12, r1, r0
+; CHECK-NEXT:    sbc.w r0, r1, r0
+; CHECK-NEXT:    eors r1, r0
+; CHECK-NEXT:    ands.w r1, r1, r12
+; CHECK-NEXT:    cset r1, mi
+; CHECK-NEXT:    ands r12, r1, #1
+; CHECK-NEXT:    vmov r1, s1
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    asrne.w r1, lr, #31
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    cset r0, mi
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    cset r5, mi
-; CHECK-NEXT:    cmp r5, r0
-; CHECK-NEXT:    cset r0, ne
-; CHECK-NEXT:    subs r6, r7, r6
-; CHECK-NEXT:    sbcs.w r3, r4, r3
-; CHECK-NEXT:    cset r4, mi
-; CHECK-NEXT:    cmp r5, r4
-; CHECK-NEXT:    cset r5, ne
-; CHECK-NEXT:    ands r0, r5
-; CHECK-NEXT:    ands r0, r0, #1
+; CHECK-NEXT:    asrne r2, r0, #31
+; CHECK-NEXT:    subs r4, r5, r4
+; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    eor.w lr, r1, r3
+; CHECK-NEXT:    sbc.w r3, r1, r3
+; CHECK-NEXT:    eors r1, r3
+; CHECK-NEXT:    ands.w r1, r1, lr
+; CHECK-NEXT:    cset r1, mi
+; CHECK-NEXT:    ands r1, r1, #1
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    asrne r6, r3, #31
-; CHECK-NEXT:    vmov q0[2], q0[0], r6, r1
-; CHECK-NEXT:    mvn r1, #-2147483648
+; CHECK-NEXT:    asrne r4, r3, #31
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r4, r2
+; CHECK-NEXT:    cset r2, mi
 ; CHECK-NEXT:    tst.w r2, #1
-; CHECK-NEXT:    cinv r2, r1, eq
+; CHECK-NEXT:    cinv r2, r5, eq
 ; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    csel r2, r2, lr, ne
-; CHECK-NEXT:    tst.w r4, #1
-; CHECK-NEXT:    cinv r1, r1, eq
-; CHECK-NEXT:    cmp r0, #0
-; CHECK-NEXT:    csel r0, r1, r3, ne
-; CHECK-NEXT:    vmov q0[3], q0[1], r0, r2
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    csel r0, r2, r0, ne
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    cset r2, mi
+; CHECK-NEXT:    tst.w r2, #1
+; CHECK-NEXT:    cinv r2, r5, eq
+; CHECK-NEXT:    cmp r1, #0
+; CHECK-NEXT:    csel r1, r2, r3, ne
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %0 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %src1, <2 x i64> %src2)
   ret <2 x i64> %0