setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom);
setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom);
setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom);
- setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom);
// Even though i128 is not a legal type, we still need to custom lower
// the atomic operations in order to exploit SystemZ instructions.
setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
- setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
+
+ // We can use the CC result of compare-and-swap to implement
+ // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
+ setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Custom);
+ setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Custom);
+ setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
}
-// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two
-// into a fullword ATOMIC_CMP_SWAPW operation.
+// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
SelectionDAG &DAG) const {
auto *Node = cast<AtomicSDNode>(Op.getNode());
-
- // We have native support for 32-bit compare and swap.
- EVT NarrowVT = Node->getMemoryVT();
- EVT WideVT = MVT::i32;
- if (NarrowVT == WideVT)
- return Op;
-
- int64_t BitSize = NarrowVT.getSizeInBits();
SDValue ChainIn = Node->getOperand(0);
SDValue Addr = Node->getOperand(1);
SDValue CmpVal = Node->getOperand(2);
SDValue SwapVal = Node->getOperand(3);
MachineMemOperand *MMO = Node->getMemOperand();
SDLoc DL(Node);
+
+ // We have native support for 32-bit and 64-bit compare and swap, but we
+ // still need to expand extracting the "success" result from the CC.
+ EVT NarrowVT = Node->getMemoryVT();
+ EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
+ if (NarrowVT == WideVT) {
+ SDVTList Tys = DAG.getVTList(WideVT, MVT::Other, MVT::Glue);
+ SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
+ SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP,
+ DL, Tys, Ops, NarrowVT, MMO);
+ SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(2),
+ SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
+
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(1));
+ return SDValue();
+ }
+
+ // Convert 8-bit and 16-bit compare and swap to a loop, implemented
+ // via a fullword ATOMIC_CMP_SWAPW operation.
+ int64_t BitSize = NarrowVT.getSizeInBits();
EVT PtrVT = Addr.getValueType();
// Get the address of the containing word.
DAG.getConstant(0, DL, WideVT), BitShift);
// Construct the ATOMIC_CMP_SWAPW node.
- SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
+ SDVTList VTList = DAG.getVTList(WideVT, MVT::Other, MVT::Glue);
SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
VTList, Ops, NarrowVT, MMO);
- return AtomicOp;
+ SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(2),
+ SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
+
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
+ DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(1));
+ return SDValue();
}
SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
case ISD::ATOMIC_LOAD_UMAX:
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
- case ISD::ATOMIC_CMP_SWAP:
+ case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
return lowerATOMIC_CMP_SWAP(Op, DAG);
case ISD::STACKSAVE:
return lowerSTACKSAVE(Op, DAG);
Results.push_back(Res);
break;
}
- case ISD::ATOMIC_CMP_SWAP: {
+ case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
SDLoc DL(N);
- SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::Other);
+ SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::Other, MVT::Glue);
SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
lowerI128ToGR128(DAG, N->getOperand(2)),
lowerI128ToGR128(DAG, N->getOperand(3)) };
MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP_128,
DL, Tys, Ops, MVT::i128, MMO);
+ SDValue Success = emitSETCC(DAG, DL, Res.getValue(2),
+ SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
+ Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));
Results.push_back(lowerGR128ToI128(DAG, Res));
+ Results.push_back(Success);
Results.push_back(Res.getValue(1));
break;
}
OPCODE(ATOMIC_LOADW_UMIN);
OPCODE(ATOMIC_LOADW_UMAX);
OPCODE(ATOMIC_CMP_SWAPW);
+ OPCODE(ATOMIC_CMP_SWAP);
OPCODE(ATOMIC_LOAD_128);
OPCODE(ATOMIC_STORE_128);
OPCODE(ATOMIC_CMP_SWAP_128);
// Operand 5: the width of the field in bits (8 or 16)
ATOMIC_CMP_SWAPW,
+ // Atomic compare-and-swap returning glue (condition code).
+ // Val, OUTCHAIN, glue = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
+ ATOMIC_CMP_SWAP,
+
// 128-bit atomic load.
// Val, OUTCHAIN = ATOMIC_LOAD_128(INCHAIN, ptr)
ATOMIC_LOAD_128,
ATOMIC_STORE_128,
// 128-bit atomic compare-and-swap.
- // Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
+ // Val, OUTCHAIN, glue = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
ATOMIC_CMP_SWAP_128,
// Byte swapping load.
// Compare and swap.
let Defs = [CC] in {
- defm CS : CmpSwapRSPair<"cs", 0xBA, 0xEB14, atomic_cmp_swap_32, GR32>;
- def CSG : CmpSwapRSY<"csg", 0xEB30, atomic_cmp_swap_64, GR64>;
+ defm CS : CmpSwapRSPair<"cs", 0xBA, 0xEB14, z_atomic_cmp_swap, GR32>;
+ def CSG : CmpSwapRSY<"csg", 0xEB30, z_atomic_cmp_swap, GR64>;
}
// Compare double and swap.
SDTCisVT<4, i32>,
SDTCisVT<5, i32>,
SDTCisVT<6, i32>]>;
+def SDT_ZAtomicCmpSwap : SDTypeProfile<1, 3,
+ [SDTCisInt<0>,
+ SDTCisPtrTy<1>,
+ SDTCisSameAs<0, 2>,
+ SDTCisSameAs<0, 3>]>;
def SDT_ZAtomicLoad128 : SDTypeProfile<1, 1,
[SDTCisVT<0, untyped>,
SDTCisPtrTy<1>]>;
def z_atomic_loadw_max : AtomicWOp<"ATOMIC_LOADW_MAX">;
def z_atomic_loadw_umin : AtomicWOp<"ATOMIC_LOADW_UMIN">;
def z_atomic_loadw_umax : AtomicWOp<"ATOMIC_LOADW_UMAX">;
-def z_atomic_cmp_swapw : AtomicWOp<"ATOMIC_CMP_SWAPW", SDT_ZAtomicCmpSwapW>;
+
+def z_atomic_cmp_swap : SDNode<"SystemZISD::ATOMIC_CMP_SWAP",
+ SDT_ZAtomicCmpSwap,
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad,
+ SDNPOutGlue, SDNPMemOperand]>;
+def z_atomic_cmp_swapw : SDNode<"SystemZISD::ATOMIC_CMP_SWAPW",
+ SDT_ZAtomicCmpSwapW,
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad,
+ SDNPOutGlue, SDNPMemOperand]>;
def z_atomic_load_128 : SDNode<"SystemZISD::ATOMIC_LOAD_128",
SDT_ZAtomicLoad128,
def z_atomic_cmp_swap_128 : SDNode<"SystemZISD::ATOMIC_CMP_SWAP_128",
SDT_ZAtomicCmpSwap128,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad,
- SDNPMemOperand]>;
+ SDNPOutGlue, SDNPMemOperand]>;
def z_mvc : SDNode<"SystemZISD::MVC", SDT_ZMemMemLength,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
%res = extractvalue { i8, i1 } %pair, 0
ret i8 %res
}
+
+; Check generating the comparison result.
+define i32 @f3(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
+; CHECK-MAIN-LABEL: f3:
+; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
+; CHECK-MAIN: sll %r3, 3
+; CHECK-MAIN: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
+; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r3)
+; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
+; CHECK-MAIN: crjlh [[TMP]], %r4, [[EXIT:\.[^ ]*]]
+; CHECK-MAIN: risbg %r5, [[TMP]], 32, 55, 0
+; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
+; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
+; CHECK-MAIN: jl [[LOOP]]
+; CHECK-MAIN: [[EXIT]]:
+; CHECK-MAIN-NEXT: ipm %r2
+; CHECK-MAIN-NEXT: afi %r2, -268435456
+; CHECK-MAIN-NEXT: srl %r2, 31
+; CHECK-MAIN-NOT: %r2
+; CHECK-MAIN: br %r14
+;
+; CHECK-SHIFT-LABEL: f3:
+; CHECK-SHIFT: sll [[SHIFT:%r[1-9]+]], 3
+; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], [[SHIFT]]
+; CHECK-SHIFT: rll
+; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -8([[NEGSHIFT]])
+ %pair = cmpxchg i8 *%src, i8 %cmp, i8 %swap seq_cst seq_cst
+ %val = extractvalue { i8, i1 } %pair, 1
+ %res = zext i1 %val to i32
+ ret i32 %res
+}
+
%res = extractvalue { i16, i1 } %pair, 0
ret i16 %res
}
+
+; Check generating the comparison result.
+define i32 @f3(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
+; CHECK-MAIN-LABEL: f3:
+; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
+; CHECK-MAIN: sll %r3, 3
+; CHECK-MAIN: l [[OLD:%r[0-9]+]], 0([[RISBG]])
+; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
+; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r3)
+; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
+; CHECK-MAIN: crjlh [[TMP]], %r4, [[EXIT:\.[^ ]*]]
+; CHECK-MAIN: risbg %r5, [[TMP]], 32, 47, 0
+; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
+; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
+; CHECK-MAIN: jl [[LOOP]]
+; CHECK-MAIN: [[EXIT]]:
+; CHECK-MAIN-NEXT: ipm %r2
+; CHECK-MAIN-NEXT: afi %r2, -268435456
+; CHECK-MAIN-NEXT: srl %r2, 31
+; CHECK-MAIN-NOT: %r2
+; CHECK-MAIN: br %r14
+;
+; CHECK-SHIFT-LABEL: f3:
+; CHECK-SHIFT: sll %r3, 3
+; CHECK-SHIFT: lcr [[NEGSHIFT:%r[1-9]+]], %r3
+; CHECK-SHIFT: rll
+; CHECK-SHIFT: rll {{%r[0-9]+}}, %r5, -16([[NEGSHIFT]])
+ %pair = cmpxchg i16 *%src, i16 %cmp, i16 %swap seq_cst seq_cst
+ %val = extractvalue { i16, i1 } %pair, 1
+ %res = zext i1 %val to i32
+ ret i32 %res
+}
+
%val = extractvalue { i32, i1 } %pair, 0
ret i32 %val
}
+
+; Check generating the comparison result.
+; CHECK-LABEL: f13
+; CHECK: cs %r2, %r3, 0(%r4)
+; CHECK-NEXT: ipm %r2
+; CHECK-NEXT: afi %r2, -268435456
+; CHECK-NEXT: srl %r2, 31
+; CHECK: br %r14
+define i32 @f13(i32 %cmp, i32 %swap, i32 *%src) {
+ %pairval = cmpxchg i32 *%src, i32 %cmp, i32 %swap seq_cst seq_cst
+ %val = extractvalue { i32, i1 } %pairval, 1
+ %res = zext i1 %val to i32
+ ret i32 %res
+}
%val = extractvalue { i64, i1 } %pairval, 0
ret i64 %val
}
+
+; Check generating the comparison result.
+; CHECK-LABEL: f10
+; CHECK: csg %r2, %r3, 0(%r4)
+; CHECK-NEXT: ipm %r2
+; CHECK-NEXT: afi %r2, -268435456
+; CHECK-NEXT: srl %r2, 31
+; CHECK: br %r14
+define i32 @f10(i64 %cmp, i64 %swap, i64 *%src) {
+ %pairval = cmpxchg i64 *%src, i64 %cmp, i64 %swap seq_cst seq_cst
+ %val = extractvalue { i64, i1 } %pairval, 1
+ %res = zext i1 %val to i32
+ ret i32 %res
+}
+
ret i16 %res
}
-; Now use the comparison result.
-; CHECK-LABEL: f5
-; CHECK: llcr [[REG:%r[0-9]+]], [[RES:%r[0-9]+]]
-; CHECK: cr [[REG]], %r3
-define zeroext i8 @f5(i8* nocapture, i8 zeroext, i8 zeroext) {
- %cx = cmpxchg i8* %0, i8 %1, i8 %2 seq_cst seq_cst
- %res = extractvalue { i8, i1 } %cx, 1
- %xres = sext i1 %res to i8
- ret i8 %xres
-}
-
-; Now use the comparison result and zero-extended old value.
-; CHECK-LABEL: f6
-; CHECK: llcr [[REG:%r[0-9]+]], [[RES:%r[0-9]+]]
-; CHECK: st [[REG]], 0(%r5)
-; CHECK: cr [[REG]], %r3
-define zeroext i8 @f6(i8* nocapture, i8 zeroext, i8 zeroext, i32*) {
- %cx = cmpxchg i8* %0, i8 %1, i8 %2 seq_cst seq_cst
- %old = extractvalue { i8, i1 } %cx, 0
- %xold = zext i8 %old to i32
- store i32 %xold, i32* %3
- %res = extractvalue { i8, i1 } %cx, 1
- %xres = sext i1 %res to i8
- ret i8 %xres
-}
%val = extractvalue { i128, i1 } %pairval, 0
ret i128 %val
}
+
+; Check generating the comparison result.
+; CHECK-LABEL: f10
+; CHECK-DAG: lg %r1, 8(%r3)
+; CHECK-DAG: lg %r0, 0(%r3)
+; CHECK-DAG: lg %r13, 8(%r2)
+; CHECK-DAG: lg %r12, 0(%r2)
+; CHECK: cdsg %r12, %r0, 0(%r4)
+; CHECK-NEXT: ipm %r2
+; CHECK-NEXT: afi %r2, -268435456
+; CHECK-NEXT: srl %r2, 31
+; CHECK: br %r14
+define i32 @f10(i128 %cmp, i128 %swap, i128 *%src) {
+ %pairval = cmpxchg i128 *%src, i128 %cmp, i128 %swap seq_cst seq_cst
+ %val = extractvalue { i128, i1 } %pairval, 1
+ %res = zext i1 %val to i32
+ ret i32 %res
+}
projects / platform / upstream / llvm.git / commitdiff