VECREDUCE_UMAX,
VECREDUCE_UMIN,
+ // The `llvm.experimental.stackmap` intrinsic.
+ // Operands: input chain, glue, <id>, <numShadowBytes>, [live0[, live1...]]
+ // Outputs: output chain, glue
+ STACKMAP,
+
// Vector Predication
#define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID,
#include "llvm/IR/VPIntrinsics.def"
void Select_FREEZE(SDNode *N);
void Select_ARITH_FENCE(SDNode *N);
+ void Select_STACKMAP(SDNode *N);
private:
void DoInstructionSelection();
case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
+ case ISD::STACKMAP:
+ Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
+ break;
}
if (!Res.getNode())
return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
ST->getMemOperand());
}
+
+SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
+ assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
+ SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
+ SDValue Op = N->getOperand(OpNo);
+ NewOps[OpNo] = GetSoftPromotedHalf(Op);
+ SDValue NewNode =
+ DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
+
+ for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
+ ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
+
+ return SDValue(); // Signal that we replaced the node ourselves.
+}
#include "LegalizeTypes.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/CodeGen/StackMaps.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
break;
case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
+ case ISD::STACKMAP:
+ Res = PromoteIntOp_STACKMAP(N, OpNo);
+ break;
}
// If the result is null, the sub-method took care of registering results etc.
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
+ assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
+ SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
+ SDValue Operand = N->getOperand(OpNo);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
+ NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
+ return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+}
+
//===----------------------------------------------------------------------===//
// Integer Result Expansion
//===----------------------------------------------------------------------===//
case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
+ case ISD::STACKMAP:
+ Res = ExpandIntOp_STACKMAP(N, OpNo);
+ break;
}
// If the result is null, the sub-method took care of registering results etc.
return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
}
+
+SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
+ assert(OpNo > 1);
+
+ SDValue Op = N->getOperand(OpNo);
+ SDLoc DL = SDLoc(N);
+ SmallVector<SDValue> NewOps;
+
+ // Copy operands before the one being expanded.
+ for (unsigned I = 0; I < OpNo; I++)
+ NewOps.push_back(N->getOperand(I));
+
+ if (Op->getOpcode() == ISD::Constant) {
+ ConstantSDNode *CN = cast<ConstantSDNode>(Op);
+ EVT Ty = Op.getValueType();
+ if (CN->getConstantIntValue()->getValue().getActiveBits() < 64) {
+ NewOps.push_back(
+ DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
+ NewOps.push_back(DAG.getTargetConstant(CN->getZExtValue(), DL, Ty));
+ } else {
+ // FIXME: https://github.com/llvm/llvm-project/issues/55609
+ return SDValue();
+ }
+ } else {
+ // FIXME: Non-constant operands are not yet handled:
+ // - https://github.com/llvm/llvm-project/issues/26431
+ // - https://github.com/llvm/llvm-project/issues/55957
+ return SDValue();
+ }
+
+ // Copy remaining operands.
+ for (unsigned I = OpNo + 1; I < N->getNumOperands(); I++)
+ NewOps.push_back(N->getOperand(I));
+
+ SDValue NewNode = DAG.getNode(N->getOpcode(), DL, N->getVTList(), NewOps);
+
+ for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
+ ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
+
+ return SDValue(); // Signal that we have replaced the node already.
+}
SDValue PromoteIntOp_VECREDUCE(SDNode *N);
SDValue PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_SET_ROUNDING(SDNode *N);
+ SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo);
void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
SDValue ExpandIntOp_RETURNADDR(SDNode *N);
SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
SDValue ExpandIntOp_SPLAT_VECTOR(SDNode *N);
+ SDValue ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo);
void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
ISD::CondCode &CCCode, const SDLoc &dl);
SDValue SoftPromoteHalfOp_SETCC(SDNode *N);
SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);
+ SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo);
//===--------------------------------------------------------------------===//
// Scalarization Support: LegalizeVectorTypes.cpp
}
}
-/// Lower llvm.experimental.stackmap directly to its target opcode.
+/// Lower llvm.experimental.stackmap.
void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
- // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
+ // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
// [live variables...])
assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
InFlag = Chain.getValue(1);
- // Add the <id> and <numBytes> constants.
- SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
- Ops.push_back(DAG.getTargetConstant(
- cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
- SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
- Ops.push_back(DAG.getTargetConstant(
- cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
- MVT::i32));
-
- // Push live variables for the stack map.
- addStackMapLiveVars(CI, 2, DL, Ops, *this);
-
- // We are not pushing any register mask info here on the operands list,
- // because the stackmap doesn't clobber anything.
-
- // Push the chain and the glue flag.
+ // Add the STACKMAP operands, starting with DAG house-keeping.
Ops.push_back(Chain);
Ops.push_back(InFlag);
+ // Add the <id>, <numShadowBytes> operands.
+ //
+ // These do not require legalisation, and can be emitted directly to target
+ // constant nodes.
+ SDValue ID = getValue(CI.getArgOperand(0));
+ assert(ID.getValueType() == MVT::i64);
+ SDValue IDConst = DAG.getTargetConstant(
+ cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType());
+ Ops.push_back(IDConst);
+
+ SDValue Shad = getValue(CI.getArgOperand(1));
+ assert(Shad.getValueType() == MVT::i32);
+ SDValue ShadConst = DAG.getTargetConstant(
+ cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType());
+ Ops.push_back(ShadConst);
+
+ // Add the live variables.
+ for (unsigned I = 2; I < CI.arg_size(); I++) {
+ SDValue Op = getValue(CI.getArgOperand(I));
+
+ // Things on the stack are pointer-typed, meaning that they are already
+ // legal and can be emitted directly to target nodes.
+ if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ Ops.push_back(DAG.getTargetFrameIndex(
+ FI->getIndex(), TLI.getFrameIndexTy(DAG.getDataLayout())));
+ } else {
+ // Otherwise emit a target independent node to be legalised.
+ Ops.push_back(getValue(CI.getArgOperand(I)));
+ }
+ }
+
// Create the STACKMAP node.
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
- SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
- Chain = SDValue(SM, 0);
+ Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
InFlag = Chain.getValue(1);
Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
case ISD::VECREDUCE_UMIN: return "vecreduce_umin";
case ISD::VECREDUCE_FMAX: return "vecreduce_fmax";
case ISD::VECREDUCE_FMIN: return "vecreduce_fmin";
+ case ISD::STACKMAP:
+ return "stackmap";
// Vector Predication
#define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...) \
#include "llvm/CodeGen/SchedulerRegistry.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/StackProtector.h"
#include "llvm/CodeGen/SwiftErrorValueTracking.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
N->getOperand(0));
}
+void SelectionDAGISel::Select_STACKMAP(SDNode *N) {
+ std::vector<SDValue> Ops;
+ auto *It = N->op_begin();
+ SDLoc DL(N);
+
+ // Stash the chain and glue operands so we can move them to the end.
+ SDValue Chain = *It++;
+ SDValue InFlag = *It++;
+
+ // <id> operand.
+ SDValue ID = *It++;
+ assert(ID.getValueType() == MVT::i64);
+ Ops.push_back(ID);
+
+ // <numShadowBytes> operand.
+ SDValue Shad = *It++;
+ assert(Shad.getValueType() == MVT::i32);
+ Ops.push_back(Shad);
+
+ // Live variable operands.
+ for (; It != N->op_end(); It++) {
+ SDNode *OpNode = It->getNode();
+ SDValue O;
+
+ // FrameIndex nodes should have been directly emitted to TargetFrameIndex
+ // nodes at DAG-construction time.
+ assert(OpNode->getOpcode() != ISD::FrameIndex);
+
+ if (OpNode->getOpcode() == ISD::Constant) {
+ Ops.push_back(
+ CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
+ O = CurDAG->getTargetConstant(
+ cast<ConstantSDNode>(OpNode)->getZExtValue(), DL, It->getValueType());
+ } else {
+ O = *It;
+ }
+ Ops.push_back(O);
+ }
+
+ Ops.push_back(Chain);
+ Ops.push_back(InFlag);
+
+ SDVTList NodeTys = CurDAG->getVTList(MVT::Other, MVT::Glue);
+ CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops);
+}
+
/// GetVBR - decode a vbr encoding whose top bit is set.
LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t
GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
case ISD::ARITH_FENCE:
Select_ARITH_FENCE(NodeToMatch);
return;
+ case ISD::STACKMAP:
+ Select_STACKMAP(NodeToMatch);
+ return;
}
assert(!NodeToMatch->isMachineOpcode() && "Node already selected!");
--- /dev/null
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -enable-patchpoint-liveness=false | FileCheck %s
+
+; CHECK-LABEL: .section __LLVM_STACKMAPS,__llvm_stackmaps
+; CHECK-NEXT: __LLVM_StackMaps:
+
+; Header
+; CHECK-NEXT: .byte 3
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 0
+
+; NumFunctions
+; CHECK-NEXT: .long 1
+; NumConstants
+; CHECK-NEXT: .long 0
+; NumRecords
+; CHECK-NEXT: .long 1
+
+; StackSizeRecord[NumFunctions]
+; StackSizeRecord[0]
+; CHECK-NEXT: .quad _main
+; CHECK-NEXT: .quad 24
+; CHECK-NEXT: .quad 1
+
+; Constants[NumConstants] (empty)
+
+; StkMapRecord[NumRecords]
+; StkMapRecord[0]
+; CHECK-NEXT: .quad 0
+; CHECK-NEXT: .long {{.*}}
+; CHECK-NEXT: .short {{.*}}
+; NumLocations
+; CHECK-NEXT: .short 11
+; Location[NumLocations]
+; Location[0]
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 1
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[1]
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 8
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 22
+; Location[2]
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 1
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[3]
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 16
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[4]
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 16
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[5]
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 8
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 66
+; Location[6]
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 4
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[7]
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 8
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[8]
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 4
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[9]
+; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 8
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+; Location[10]
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .short 1
+; CHECK-NEXT: .short {{.*}}
+; CHECK-NEXT: .short 0
+; CHECK-NEXT: .long 0
+
+@p32 = external global i8 addrspace(270)*
+
+%struct1 = type {i32, i64}
+%struct2 = type {i1, i1, i1}
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
+
+define dso_local i32 @main(i32 %argc, i8** %argv) {
+entry:
+ %i1reg = icmp eq i32 %argc, 5
+ %i7reg = zext i1 %i1reg to i7
+ %halfreg = sitofp i32 %argc to half
+ %ptr32 = load i8 addrspace(270)*, i8 addrspace(270)** @p32
+ %structreg1 = insertvalue %struct1 zeroinitializer, i32 %argc, 0
+ %structreg2 = insertvalue %struct2 zeroinitializer, i1 %i1reg, 0
+ call void (i64, i32, ...) @llvm.experimental.stackmap(
+ i64 0,
+ i32 0,
+ i1 %i1reg,
+ i7 22,
+ i7 %i7reg,
+ half 1.0,
+ half %halfreg,
+ i128 66,
+ ; FIXME: fix and test vectors. At the moment even legally sized vectors
+ ; are broken:
+ ; https://github.com/llvm/llvm-project/issues/55613
+ ;
+ ; FIXME: test non-constant i128 once these are fixed:
+ ; - https://github.com/llvm/llvm-project/issues/26431
+ ; - https://github.com/llvm/llvm-project/issues/55957
+ i8 addrspace(270)* %ptr32,
+ ; FIXME: The stackmap record generated for structs is incorrect:
+ ; - https://github.com/llvm/llvm-project/issues/55649
+ ; - https://github.com/llvm/llvm-project/issues/55957
+ %struct1 zeroinitializer,
+ %struct1 %structreg1,
+ %struct2 zeroinitializer,
+ %struct2 %structreg2)
+ ret i32 0
+}