return SDValue();
}
+static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1,
+ ISD::CondCode Cond, const SDLoc &dl,
+ SelectionDAG &DAG) {
+ // If we are testing for all-bits-clear, we might be able to do that with
+ // less shifting since bit-order does not matter.
+ if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
+ return SDValue();
+
+ auto *C1 = isConstOrConstSplat(N1, /* AllowUndefs */ true);
+ if (!C1 || !C1->isZero())
+ return SDValue();
+
+ if (!N0.hasOneUse() ||
+ (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
+ return SDValue();
+
+ unsigned BitWidth = N0.getScalarValueSizeInBits();
+ auto *ShAmtC = isConstOrConstSplat(N0.getOperand(2));
+ if (!ShAmtC || ShAmtC->getAPIntValue().uge(BitWidth))
+ return SDValue();
+
+ // Canonicalize fshr as fshl to reduce pattern-matching.
+ unsigned ShAmt = ShAmtC->getZExtValue();
+ if (N0.getOpcode() == ISD::FSHR)
+ ShAmt = BitWidth - ShAmt;
+
+ // Match an 'or' with a specific operand 'Other' in either commuted variant.
+ SDValue X, Y;
+ auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
+ if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
+ return false;
+ if (Or.getOperand(0) == Other) {
+ X = Or.getOperand(0);
+ Y = Or.getOperand(1);
+ return true;
+ }
+ if (Or.getOperand(1) == Other) {
+ X = Or.getOperand(1);
+ Y = Or.getOperand(0);
+ return true;
+ }
+ return false;
+ };
+
+ EVT OpVT = N0.getValueType();
+ EVT ShAmtVT = N0.getOperand(2).getValueType();
+ SDValue F0 = N0.getOperand(0);
+ SDValue F1 = N0.getOperand(1);
+ if (matchOr(F0, F1)) {
+ // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
+ SDValue NewShAmt = DAG.getConstant(ShAmt, dl, ShAmtVT);
+ SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt);
+ SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
+ return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
+ }
+ if (matchOr(F1, F0)) {
+ // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
+ SDValue NewShAmt = DAG.getConstant(BitWidth - ShAmt, dl, ShAmtVT);
+ SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt);
+ SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
+ return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
+ }
+
+ return SDValue();
+}
+
/// Try to simplify a setcc built with the specified operands and cc. If it is
/// unable to simplify it, return a null SDValue.
SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
return V;
+ if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
+ return V;
+
if (auto *N1C = isConstOrConstSplat(N1)) {
const APInt &C1 = N1C->getAPIntValue();
define i1 @fshl_or_eq_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_eq_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: ror w8, w0, #27
-; CHECK-NEXT: orr w8, w8, w1, lsl #5
+; CHECK-NEXT: orr w8, w0, w1, lsl #5
; CHECK-NEXT: cmp w8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
define i1 @fshl_or_commute_eq_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_commute_eq_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: ror w8, w0, #27
-; CHECK-NEXT: orr w8, w8, w1, lsl #5
+; CHECK-NEXT: orr w8, w0, w1, lsl #5
; CHECK-NEXT: cmp w8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
define <4 x i1> @fshl_or2_eq_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_eq_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr v1.16b, v0.16b, v1.16b
-; CHECK-NEXT: shl v0.4s, v0.4s, #25
; CHECK-NEXT: ushr v1.4s, v1.4s, #7
-; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT: cmeq v0.4s, v0.4s, #0
; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
define <4 x i1> @fshl_or2_commute_eq_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_commute_eq_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr v1.16b, v1.16b, v0.16b
-; CHECK-NEXT: shl v0.4s, v0.4s, #25
; CHECK-NEXT: ushr v1.4s, v1.4s, #7
-; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT: cmeq v0.4s, v0.4s, #0
; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
define i1 @fshr_or_eq_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or_eq_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl w8, w0, #16
-; CHECK-NEXT: orr w9, w0, w1
-; CHECK-NEXT: extr w8, w9, w8, #24
+; CHECK-NEXT: orr w8, w0, w1, lsl #8
; CHECK-NEXT: tst w8, #0xffff
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
define i1 @fshr_or_commute_eq_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or_commute_eq_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsl w8, w0, #16
-; CHECK-NEXT: orr w9, w1, w0
-; CHECK-NEXT: extr w8, w9, w8, #24
+; CHECK-NEXT: orr w8, w0, w1, lsl #8
; CHECK-NEXT: tst w8, #0xffff
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
define i1 @fshr_or2_eq_0(i64 %x, i64 %y) {
; CHECK-LABEL: fshr_or2_eq_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: ror x8, x0, #3
-; CHECK-NEXT: orr x8, x8, x1, lsr #3
+; CHECK-NEXT: orr x8, x0, x1, lsr #3
; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
define i1 @fshl_or_ne_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: ror w8, w0, #25
-; CHECK-NEXT: orr w8, w8, w1, lsl #7
+; CHECK-NEXT: orr w8, w0, w1, lsl #7
; CHECK-NEXT: cmp w8, #0
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
define i1 @fshl_or_commute_ne_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_commute_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: ror w8, w0, #25
-; CHECK-NEXT: orr w8, w8, w1, lsl #7
+; CHECK-NEXT: orr w8, w0, w1, lsl #7
; CHECK-NEXT: cmp w8, #0
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
define <4 x i1> @fshl_or2_ne_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr v1.16b, v0.16b, v1.16b
-; CHECK-NEXT: shl v0.4s, v0.4s, #5
; CHECK-NEXT: ushr v1.4s, v1.4s, #27
-; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT: cmtst v0.4s, v0.4s, v0.4s
; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
define <4 x i1> @fshl_or2_commute_ne_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_commute_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr v1.16b, v1.16b, v0.16b
-; CHECK-NEXT: shl v0.4s, v0.4s, #5
; CHECK-NEXT: ushr v1.4s, v1.4s, #27
-; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b
; CHECK-NEXT: cmtst v0.4s, v0.4s, v0.4s
; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: ret
define i1 @fshr_or_ne_0(i64 %x, i64 %y) {
; CHECK-LABEL: fshr_or_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr w8, w0, w1
-; CHECK-NEXT: extr x8, x8, x0, #1
+; CHECK-NEXT: orr x8, x0, x1, lsl #63
; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
define i1 @fshr_or_commute_ne_0(i64 %x, i64 %y) {
; CHECK-LABEL: fshr_or_commute_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr w8, w1, w0
-; CHECK-NEXT: extr x8, x8, x0, #1
+; CHECK-NEXT: orr x8, x0, x1, lsl #63
; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
define i1 @fshr_or2_ne_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or2_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr w8, w0, w1
-; CHECK-NEXT: lsl w8, w8, #16
-; CHECK-NEXT: extr w8, w0, w8, #18
+; CHECK-NEXT: and w8, w1, #0xfffc
+; CHECK-NEXT: orr w8, w0, w8, lsr #2
; CHECK-NEXT: tst w8, #0xffff
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
define i1 @fshr_or2_commute_ne_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or2_commute_ne_0:
; CHECK: // %bb.0:
-; CHECK-NEXT: orr w8, w1, w0
-; CHECK-NEXT: lsl w8, w8, #16
-; CHECK-NEXT: extr w8, w0, w8, #18
+; CHECK-NEXT: and w8, w1, #0xfffc
+; CHECK-NEXT: orr w8, w0, w8, lsr #2
; CHECK-NEXT: tst w8, #0xffff
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
; X64-NEXT: addq $1, %rax
; X64-NEXT: adcq $0, %rdx
; X64-NEXT: movq %rax, %rcx
+; X64-NEXT: shrq $60, %rcx
; X64-NEXT: orq %rdx, %rcx
-; X64-NEXT: shrdq $60, %rdx, %rcx
; X64-NEXT: jne .LBB0_1
; X64-NEXT: # %bb.2: # %exit
; X64-NEXT: retq
;
; X64-LABEL: opt_setcc_srl_eq_zero:
; X64: # %bb.0:
+; X64-NEXT: shrq $17, %rdi
; X64-NEXT: orq %rsi, %rdi
-; X64-NEXT: shrdq $17, %rsi, %rdi
; X64-NEXT: sete %al
; X64-NEXT: retq
%srl = lshr i128 %a, 17
;
; X64-LABEL: opt_setcc_srl_ne_zero:
; X64: # %bb.0:
+; X64-NEXT: shrq $17, %rdi
; X64-NEXT: orq %rsi, %rdi
-; X64-NEXT: shrdq $17, %rsi, %rdi
; X64-NEXT: setne %al
; X64-NEXT: retq
%srl = lshr i128 %a, 17
;
; X64-LABEL: opt_setcc_shl_eq_zero:
; X64: # %bb.0:
+; X64-NEXT: shlq $17, %rsi
; X64-NEXT: orq %rdi, %rsi
-; X64-NEXT: shldq $17, %rdi, %rsi
; X64-NEXT: sete %al
; X64-NEXT: retq
%shl = shl i128 %a, 17
;
; X64-LABEL: opt_setcc_shl_ne_zero:
; X64: # %bb.0:
+; X64-NEXT: shlq $17, %rsi
; X64-NEXT: orq %rdi, %rsi
-; X64-NEXT: shldq $17, %rdi, %rsi
; X64-NEXT: setne %al
; X64-NEXT: retq
%shl = shl i128 %a, 17
;
; X64-LABEL: opt_setcc_expanded_shl_correct_shifts:
; X64: # %bb.0:
+; X64-NEXT: shlq $17, %rdi
; X64-NEXT: orq %rsi, %rdi
-; X64-NEXT: shldq $17, %rsi, %rdi
; X64-NEXT: sete %al
; X64-NEXT: retq
%shl.a = shl i64 %a, 17
define i1 @fshl_or_eq_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_eq_0:
; CHECK: # %bb.0:
+; CHECK-NEXT: shll $5, %esi
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldl $5, %edi, %esi
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = or i32 %x, %y
define i1 @fshl_or_commute_eq_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_commute_eq_0:
; CHECK: # %bb.0:
+; CHECK-NEXT: shll $5, %esi
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldl $5, %edi, %esi
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = or i32 %y, %x
define <4 x i1> @fshl_or2_eq_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_eq_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: por %xmm0, %xmm1
+; CHECK-NEXT: pxor %xmm2, %xmm2
; CHECK-NEXT: psrld $7, %xmm1
-; CHECK-NEXT: pslld $25, %xmm0
; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pxor %xmm1, %xmm1
-; CHECK-NEXT: pcmpeqd %xmm1, %xmm0
+; CHECK-NEXT: pcmpeqd %xmm2, %xmm0
; CHECK-NEXT: retq
%or = or <4 x i32> %x, %y
%f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %or, <4 x i32> <i32 25, i32 25, i32 25, i32 25>)
define <4 x i1> @fshl_or2_commute_eq_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_commute_eq_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: por %xmm0, %xmm1
+; CHECK-NEXT: pxor %xmm2, %xmm2
; CHECK-NEXT: psrld $7, %xmm1
-; CHECK-NEXT: pslld $25, %xmm0
; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pxor %xmm1, %xmm1
-; CHECK-NEXT: pcmpeqd %xmm1, %xmm0
+; CHECK-NEXT: pcmpeqd %xmm2, %xmm0
; CHECK-NEXT: retq
%or = or <4 x i32> %y, %x
%f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %or, <4 x i32> <i32 25, i32 25, i32 25, i32 25>)
define i1 @fshr_or_eq_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or_eq_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldw $8, %di, %si
+; CHECK-NEXT: shll $8, %esi
+; CHECK-NEXT: orw %di, %si
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = or i16 %x, %y
define i1 @fshr_or_commute_eq_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or_commute_eq_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldw $8, %di, %si
+; CHECK-NEXT: shll $8, %esi
+; CHECK-NEXT: orw %di, %si
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = or i16 %y, %x
define i1 @fshr_or2_eq_0(i64 %x, i64 %y) {
; CHECK-LABEL: fshr_or2_eq_0:
; CHECK: # %bb.0:
+; CHECK-NEXT: shrq $3, %rsi
; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: shrdq $3, %rdi, %rsi
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = or i64 %x, %y
define i1 @fshr_or2_commute_eq_0(i64 %x, i64 %y) {
; CHECK-LABEL: fshr_or2_commute_eq_0:
; CHECK: # %bb.0:
+; CHECK-NEXT: shrq $3, %rsi
; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: shrdq $3, %rdi, %rsi
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = or i64 %y, %x
define i1 @fshl_or_ne_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_ne_0:
; CHECK: # %bb.0:
+; CHECK-NEXT: shll $7, %esi
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldl $7, %edi, %esi
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i32 %x, %y
define i1 @fshl_or_commute_ne_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_commute_ne_0:
; CHECK: # %bb.0:
+; CHECK-NEXT: shll $7, %esi
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldl $7, %edi, %esi
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i32 %y, %x
define <4 x i1> @fshl_or2_ne_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_ne_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: por %xmm0, %xmm1
+; CHECK-NEXT: pxor %xmm2, %xmm2
; CHECK-NEXT: psrld $27, %xmm1
-; CHECK-NEXT: pslld $5, %xmm0
; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pxor %xmm1, %xmm1
-; CHECK-NEXT: pcmpeqd %xmm1, %xmm0
+; CHECK-NEXT: pcmpeqd %xmm2, %xmm0
; CHECK-NEXT: pcmpeqd %xmm1, %xmm1
; CHECK-NEXT: pxor %xmm1, %xmm0
; CHECK-NEXT: retq
define <4 x i1> @fshl_or2_commute_ne_0(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: fshl_or2_commute_ne_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: por %xmm0, %xmm1
+; CHECK-NEXT: pxor %xmm2, %xmm2
; CHECK-NEXT: psrld $27, %xmm1
-; CHECK-NEXT: pslld $5, %xmm0
; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pxor %xmm1, %xmm1
-; CHECK-NEXT: pcmpeqd %xmm1, %xmm0
+; CHECK-NEXT: pcmpeqd %xmm2, %xmm0
; CHECK-NEXT: pcmpeqd %xmm1, %xmm1
; CHECK-NEXT: pxor %xmm1, %xmm0
; CHECK-NEXT: retq
define i1 @fshr_or_ne_0(i64 %x, i64 %y) {
; CHECK-LABEL: fshr_or_ne_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldq $63, %rdi, %rsi
+; CHECK-NEXT: shlq $63, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i64 %x, %y
define i1 @fshr_or_commute_ne_0(i64 %x, i64 %y) {
; CHECK-LABEL: fshr_or_commute_ne_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldq $63, %rdi, %rsi
+; CHECK-NEXT: shlq $63, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i64 %y, %x
define i1 @fshr_or2_ne_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or2_ne_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shrdw $2, %di, %si
+; CHECK-NEXT: movzwl %si, %eax
+; CHECK-NEXT: shrl $2, %eax
+; CHECK-NEXT: orw %di, %ax
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i16 %x, %y
define i1 @fshr_or2_commute_ne_0(i16 %x, i16 %y) {
; CHECK-LABEL: fshr_or2_commute_ne_0:
; CHECK: # %bb.0:
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shrdw $2, %di, %si
+; CHECK-NEXT: movzwl %si, %eax
+; CHECK-NEXT: shrl $2, %eax
+; CHECK-NEXT: orw %di, %ax
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i16 %y, %x