return true;
}
+/// Match special-case patterns that check for unsigned add overflow.
+static bool matchUAddWithOverflowConstantEdgeCases(CmpInst *Cmp,
+ BinaryOperator *&Add) {
+ // Add = add A, 1; Cmp = icmp eq A,-1 (overflow if A is max val)
+ // Add = add A,-1; Cmp = icmp ne A, 0 (overflow if A is non-zero)
+ Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1);
+ ICmpInst::Predicate Pred = Cmp->getPredicate();
+ if (Pred == ICmpInst::ICMP_EQ && match(B, m_AllOnes()))
+ B = ConstantInt::get(B->getType(), 1);
+ else if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt()))
+ B = ConstantInt::get(B->getType(), -1);
+ else
+ return false;
+
+ // Check the users of the variable operand of the compare looking for an add
+ // with the adjusted constant.
+ for (User *U : A->users()) {
+ if (match(U, m_Add(m_Specific(A), m_Specific(B)))) {
+ Add = cast<BinaryOperator>(U);
+ return true;
+ }
+ }
+ return false;
+}
+
/// Try to combine the compare into a call to the llvm.uadd.with.overflow
/// intrinsic. Return true if any changes were made.
static bool combineToUAddWithOverflow(CmpInst *Cmp, const TargetLowering &TLI,
Value *A, *B;
BinaryOperator *Add;
if (!match(Cmp, m_UAddWithOverflow(m_Value(A), m_Value(B), m_BinOp(Add))))
- return false;
+ if (!matchUAddWithOverflowConstantEdgeCases(Cmp, Add))
+ return false;
if (!TLI.shouldFormOverflowOp(ISD::UADDO,
TLI.getValueType(DL, Add->getType())))
if (Add->getParent() != Cmp->getParent() && !Add->hasOneUse())
return false;
-#ifndef NDEBUG
- // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption
- // for now:
- if (Add->hasOneUse())
- assert(*Add->user_begin() == Cmp && "expected!");
-#endif
-
if (!replaceMathCmpWithIntrinsic(Add, Cmp, Intrinsic::uadd_with_overflow))
return false;
define i1 @uaddo_i64_increment_alt(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_increment_alt:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmn x0, #1 // =1
-; CHECK-NEXT: add x8, x0, #1 // =1
-; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: adds x8, x0, #1 // =1
+; CHECK-NEXT: cset w0, hs
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: ret
%a = add i64 %x, 1
define i1 @uaddo_i64_increment_alt_dom(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_increment_alt_dom:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmn x0, #1 // =1
-; CHECK-NEXT: cset w8, eq
-; CHECK-NEXT: add x9, x0, #1 // =1
-; CHECK-NEXT: mov w0, w8
-; CHECK-NEXT: str x9, [x1]
+; CHECK-NEXT: adds x8, x0, #1 // =1
+; CHECK-NEXT: cset w0, hs
+; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: ret
%ov = icmp eq i64 %x, -1
%a = add i64 %x, 1
define i1 @uaddo_i64_decrement_alt(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_decrement_alt:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmp x0, #0 // =0
-; CHECK-NEXT: sub x8, x0, #1 // =1
-; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: subs x8, x0, #1 // =1
+; CHECK-NEXT: cset w0, hs
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: ret
%a = add i64 %x, -1
define i1 @uaddo_i64_decrement_alt_dom(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_decrement_alt_dom:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmp x0, #0 // =0
-; CHECK-NEXT: cset w8, ne
-; CHECK-NEXT: sub x9, x0, #1 // =1
-; CHECK-NEXT: mov w0, w8
-; CHECK-NEXT: str x9, [x1]
+; CHECK-NEXT: subs x8, x0, #1 // =1
+; CHECK-NEXT: cset w0, hs
+; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: ret
%ov = icmp ne i64 %x, 0
%a = add i64 %x, -1
define void @test_18446744073709551615(i64*, i64*) {
; CHECK-LABEL: test_18446744073709551615:
; CHECK: # %bb.0:
-; CHECK-NEXT: movq (%rdi), %rax
-; CHECK-NEXT: leaq -1(%rax), %rcx
-; CHECK-NEXT: movq %rcx, (%rdi)
-; CHECK-NEXT: testq %rax, %rax
-; CHECK-NEXT: setne %al
-; CHECK-NEXT: addb $-1, %al
+; CHECK-NEXT: addq $-1, (%rdi)
; CHECK-NEXT: adcq $0, (%rsi)
; CHECK-NEXT: retq
%3 = load i64, i64* %0, align 8
define i1 @uaddo_i64_increment_alt(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_increment_alt:
; CHECK: # %bb.0:
-; CHECK-NEXT: leaq 1(%rdi), %rax
-; CHECK-NEXT: movq %rax, (%rsi)
-; CHECK-NEXT: cmpq $-1, %rdi
+; CHECK-NEXT: incq %rdi
; CHECK-NEXT: sete %al
+; CHECK-NEXT: movq %rdi, (%rsi)
; CHECK-NEXT: retq
%a = add i64 %x, 1
store i64 %a, i64* %p
define i1 @uaddo_i64_increment_alt_dom(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_increment_alt_dom:
; CHECK: # %bb.0:
-; CHECK-NEXT: cmpq $-1, %rdi
-; CHECK-NEXT: sete %al
; CHECK-NEXT: incq %rdi
+; CHECK-NEXT: sete %al
; CHECK-NEXT: movq %rdi, (%rsi)
; CHECK-NEXT: retq
%ov = icmp eq i64 %x, -1
define i1 @uaddo_i64_decrement_alt(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_decrement_alt:
; CHECK: # %bb.0:
-; CHECK-NEXT: leaq -1(%rdi), %rax
-; CHECK-NEXT: movq %rax, (%rsi)
-; CHECK-NEXT: testq %rdi, %rdi
-; CHECK-NEXT: setne %al
+; CHECK-NEXT: addq $-1, %rdi
+; CHECK-NEXT: setb %al
+; CHECK-NEXT: movq %rdi, (%rsi)
; CHECK-NEXT: retq
%a = add i64 %x, -1
store i64 %a, i64* %p
define i1 @uaddo_i64_decrement_alt_dom(i64 %x, i64* %p) {
; CHECK-LABEL: uaddo_i64_decrement_alt_dom:
; CHECK: # %bb.0:
-; CHECK-NEXT: testq %rdi, %rdi
-; CHECK-NEXT: setne %al
-; CHECK-NEXT: decq %rdi
+; CHECK-NEXT: addq $-1, %rdi
+; CHECK-NEXT: setb %al
; CHECK-NEXT: movq %rdi, (%rsi)
; CHECK-NEXT: retq
%ov = icmp ne i64 %x, 0
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: pushq %rax
; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: movq {{.*}}(%rip), %rax
-; CHECK-NEXT: leaq -1(%rax), %rsi
+; CHECK-NEXT: movq {{.*}}(%rip), %rsi
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: addq $-1, %rsi
+; CHECK-NEXT: setb %al
; CHECK-NEXT: movq %rsi, {{.*}}(%rip)
-; CHECK-NEXT: xorl %ecx, %ecx
-; CHECK-NEXT: testq %rax, %rax
-; CHECK-NEXT: setne %cl
-; CHECK-NEXT: movl %ecx, {{.*}}(%rip)
+; CHECK-NEXT: movl %eax, {{.*}}(%rip)
; CHECK-NEXT: movl $.L.str, %edi
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: callq printf
define i1 @uaddo_i64_increment_alt(i64 %x, i64* %p) {
; CHECK-LABEL: @uaddo_i64_increment_alt(
-; CHECK-NEXT: [[A:%.*]] = add i64 [[X:%.*]], 1
-; CHECK-NEXT: store i64 [[A]], i64* [[P:%.*]]
-; CHECK-NEXT: [[OV:%.*]] = icmp eq i64 [[X]], -1
-; CHECK-NEXT: ret i1 [[OV]]
+; CHECK-NEXT: [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 1)
+; CHECK-NEXT: [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT: [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT: store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT: ret i1 [[OV1]]
;
%a = add i64 %x, 1
store i64 %a, i64* %p
define i1 @uaddo_i64_increment_alt_dom(i64 %x, i64* %p) {
; CHECK-LABEL: @uaddo_i64_increment_alt_dom(
-; CHECK-NEXT: [[OV:%.*]] = icmp eq i64 [[X:%.*]], -1
-; CHECK-NEXT: [[A:%.*]] = add i64 [[X]], 1
-; CHECK-NEXT: store i64 [[A]], i64* [[P:%.*]]
-; CHECK-NEXT: ret i1 [[OV]]
+; CHECK-NEXT: [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 1)
+; CHECK-NEXT: [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT: [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT: store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT: ret i1 [[OV1]]
;
%ov = icmp eq i64 %x, -1
%a = add i64 %x, 1
define i1 @uaddo_i64_decrement_alt(i64 %x, i64* %p) {
; CHECK-LABEL: @uaddo_i64_decrement_alt(
-; CHECK-NEXT: [[A:%.*]] = add i64 [[X:%.*]], -1
-; CHECK-NEXT: store i64 [[A]], i64* [[P:%.*]]
-; CHECK-NEXT: [[OV:%.*]] = icmp ne i64 [[X]], 0
-; CHECK-NEXT: ret i1 [[OV]]
+; CHECK-NEXT: [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 -1)
+; CHECK-NEXT: [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT: [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT: store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT: ret i1 [[OV1]]
;
%a = add i64 %x, -1
store i64 %a, i64* %p
define i1 @uaddo_i64_decrement_alt_dom(i64 %x, i64* %p) {
; CHECK-LABEL: @uaddo_i64_decrement_alt_dom(
-; CHECK-NEXT: [[OV:%.*]] = icmp ne i64 [[X:%.*]], 0
-; CHECK-NEXT: [[A:%.*]] = add i64 [[X]], -1
-; CHECK-NEXT: store i64 [[A]], i64* [[P:%.*]]
-; CHECK-NEXT: ret i1 [[OV]]
+; CHECK-NEXT: [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 -1)
+; CHECK-NEXT: [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT: [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT: store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT: ret i1 [[OV1]]
;
%ov = icmp ne i64 %x, 0
%a = add i64 %x, -1