From 70697ff74d16019bac34e8aa5461751904e2ef89 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Wed, 16 Mar 2016 23:08:00 +0000 Subject: [PATCH] Extract out a SelectionDAGBuilder::LowerAsStatepoint; NFC Summary: This is a step towards implementing "direct" lowering of calls and invokes with deopt operand bundles into STATEPOINT nodes (as opposed to having them mandatorily pass through RewriteStatepointsForGC, which is the case today). This change extracts out a `SelectionDAGBuilder::LowerAsStatepoint` helper function that is able to lower a "statepoint like thing", and uses it to lower `gc.statepoint` calls. This is an NFC now, but in a later change we will use `LowerAsStatepoint` to directly lower calls and invokes with operand bundles without going through an intermediate `gc.statepoint` IR representation. FYI: I expect `SelectionDAGBuilder::StatepointInfo` will evolve as I add support for lowering non gc.statepoints, right now it is fairly tightly coupled with an IR level `gc.statepoint`. Reviewers: reames, pgavlin, JosephTremoulet Subscribers: sanjoy, mcrosier, llvm-commits Differential Revision: http://reviews.llvm.org/D18106 llvm-svn: 263671 --- .../lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h | 48 ++++ .../CodeGen/SelectionDAG/StatepointLowering.cpp | 293 +++++++++++---------- 2 files changed, 197 insertions(+), 144 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 941a0c0..485501f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -726,6 +726,54 @@ public: /// references that need to refer to the last resulting block. void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last); + /// Describes a gc.statepoint or a gc.statepoint like thing for the purposes + /// of lowering into a STATEPOINT node. Right now it only abstracts an actual + /// gc.statepoint, but that will change in the future. + struct StatepointLoweringInfo { + /// Bases[i] is the base pointer for Ptrs[i]. Together they denote the set + /// of gc pointers this STATEPOINT has to relocate. + ArrayRef Bases; + ArrayRef Ptrs; + + /// The set of gc.relocate calls associated with this gc.statepoint. + ArrayRef GCRelocates; + + /// The full list of gc arguments to the gc.statepoint being lowered. + ArrayRef GCArgs; + + /// The gc.statepoint instruction. + const Instruction *StatepointInstr = nullptr; + + /// The list of gc transition arguments present in the gc.statepoint being + /// lowered. + ArrayRef GCTransitionArgs; + + /// The ID that the resulting STATEPOINT instruction has to report. + unsigned ID = -1; + + /// Information regarding the underlying call instruction. + TargetLowering::CallLoweringInfo CLI; + + /// The deoptimization state associated with this gc.statepoint call, if + /// any. + ArrayRef DeoptState; + + /// Flags associated with the meta arguments being lowered. + uint64_t StatepointFlags = -1; + + /// The number of patchable bytes the call needs to get lowered into. + unsigned NumPatchBytes = -1; + + /// The exception handling unwind destination, in case this represents an + /// invoke of gc.statepoint. + const BasicBlock *EHPadBB = nullptr; + + explicit StatepointLoweringInfo(SelectionDAG &DAG) : CLI(DAG) {} + }; + + /// Lower \p SLI into a STATEPOINT instruction. + SDValue LowerAsStatepoint(StatepointLoweringInfo &SLI); + // This function is responsible for the whole statepoint lowering process. // It uniformly handles invoke and call statepoints. void LowerStatepoint(ImmutableStatepoint Statepoint, diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index ff01368..ae28089 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -279,46 +279,13 @@ removeDuplicatesGCPtrs(SmallVectorImpl &Bases, /// Extract call from statepoint, lower it and return pointer to the /// call node. Also update NodeMap so that getValue(statepoint) will /// reference lowered call result -static SDNode * -lowerCallFromStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB, - SelectionDAGBuilder &Builder, - SmallVectorImpl &PendingExports) { - - ImmutableCallSite CS(ISP.getCallSite()); - - SDValue ActualCallee; - - if (ISP.getNumPatchBytes() > 0) { - // If we've been asked to emit a nop sequence instead of a call instruction - // for this statepoint then don't lower the call target, but use a constant - // `null` instead. Not lowering the call target lets statepoint clients get - // away without providing a physical address for the symbolic call target at - // link time. - - const auto &TLI = Builder.DAG.getTargetLoweringInfo(); - const auto &DL = Builder.DAG.getDataLayout(); - - unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace(); - ActualCallee = Builder.DAG.getConstant(0, Builder.getCurSDLoc(), - TLI.getPointerTy(DL, AS)); - } else { - ActualCallee = Builder.getValue(ISP.getCalledValue()); - } - - assert(CS.getCallingConv() != CallingConv::AnyReg && - "anyregcc is not supported on statepoints!"); - - Type *DefTy = ISP.getActualReturnType(); - bool HasDef = !DefTy->isVoidTy(); - - TargetLowering::CallLoweringInfo CLI(Builder.DAG); - Builder.populateCallLoweringInfo( - CLI, ISP.getCallSite(), ImmutableStatepoint::CallArgsBeginPos, - ISP.getNumCallArgs(), ActualCallee, DefTy, false); +static std::pair lowerCallFromStatepointLoweringInfo( + SelectionDAGBuilder::StatepointLoweringInfo &SI, + SelectionDAGBuilder &Builder, SmallVectorImpl &PendingExports) { SDValue ReturnValue, CallEndVal; - std::tie(ReturnValue, CallEndVal) = Builder.lowerInvokable(CLI, EHPadBB); - + std::tie(ReturnValue, CallEndVal) = + Builder.lowerInvokable(SI.CLI, SI.EHPadBB); SDNode *CallEnd = CallEndVal.getNode(); // Get a call instruction from the call sequence chain. Tail calls are not @@ -337,6 +304,7 @@ lowerCallFromStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB, // to grab the return value from the return register(s), or it can be a LOAD // to load a value returned by reference via a stack slot. + bool HasDef = !SI.CLI.RetTy->isVoidTy(); if (HasDef) { if (CallEnd->getOpcode() == ISD::LOAD) CallEnd = CallEnd->getOperand(0).getNode(); @@ -346,44 +314,7 @@ lowerCallFromStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB, } assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!"); - - // Export the result value if needed - const Instruction *GCResult = ISP.getGCResult(); - if (HasDef && GCResult) { - if (GCResult->getParent() != CS.getParent()) { - // Result value will be used in a different basic block so we need to - // export it now. - // Default exporting mechanism will not work here because statepoint call - // has a different type than the actual call. It means that by default - // llvm will create export register of the wrong type (always i32 in our - // case). So instead we need to create export register with correct type - // manually. - // TODO: To eliminate this problem we can remove gc.result intrinsics - // completely and make statepoint call to return a tuple. - unsigned Reg = Builder.FuncInfo.CreateRegs(ISP.getActualReturnType()); - RegsForValue RFV( - *Builder.DAG.getContext(), Builder.DAG.getTargetLoweringInfo(), - Builder.DAG.getDataLayout(), Reg, ISP.getActualReturnType()); - SDValue Chain = Builder.DAG.getEntryNode(); - - RFV.getCopyToRegs(ReturnValue, Builder.DAG, Builder.getCurSDLoc(), Chain, - nullptr); - PendingExports.push_back(Chain); - Builder.FuncInfo.ValueMap[CS.getInstruction()] = Reg; - } else { - // Result value will be used in a same basic block. Don't export it or - // perform any explicit register copies. - // We'll replace the actuall call node shortly. gc_result will grab - // this value. - Builder.setValue(CS.getInstruction(), ReturnValue); - } - } else { - // The token value is never used from here on, just generate a poison value - Builder.setValue(CS.getInstruction(), - Builder.DAG.getIntPtrConstant(-1, Builder.getCurSDLoc())); - } - - return CallEnd->getOperand(0).getNode(); + return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode()); } /// Callect all gc pointers coming into statepoint intrinsic, clean them up, @@ -503,18 +434,13 @@ static void lowerIncomingStatepointValue(SDValue Incoming, /// completion, 'Ops' will contain ready to use operands for machine code /// statepoint. The chain nodes will have already been created and the DAG root /// will be set to the last value spilled (if any were). -static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, - ImmutableStatepoint StatepointSite, - SelectionDAGBuilder &Builder) { - - // Lower the deopt and gc arguments for this statepoint. Layout will - // be: deopt argument length, deopt arguments.., gc arguments... - - SmallVector Bases, Ptrs; - SmallVector Relocations; - getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite, - Builder); +static void +lowerStatepointMetaArgs(SmallVectorImpl &Ops, + SelectionDAGBuilder::StatepointLoweringInfo &SI, + SelectionDAGBuilder &Builder) { +// Lower the deopt and gc arguments for this statepoint. Layout will +// be: deopt argument length, deopt arguments.., gc arguments... #ifndef NDEBUG // Check that each of the gc pointer and bases we've gotten out of the // safepoint is something the strategy thinks might be a pointer (or vector @@ -522,14 +448,14 @@ static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, // errors during statepoint insertion. TODO: This should actually be in the // Verifier, but we can't get to the GCStrategy from there (yet). GCStrategy &S = Builder.GFI->getStrategy(); - for (const Value *V : Bases) { + for (const Value *V : SI.Bases) { auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); if (Opt.hasValue()) { assert(Opt.getValue() && "non gc managed base pointer found in statepoint"); } } - for (const Value *V : Ptrs) { + for (const Value *V : SI.Ptrs) { auto Opt = S.isGCManagedPointer(V->getType()->getScalarType()); if (Opt.hasValue()) { assert(Opt.getValue() && @@ -543,30 +469,23 @@ static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, // particular value. This is purely an optimization over the code below and // doesn't change semantics at all. It is important for performance that we // reserve slots for both deopt and gc values before lowering either. - for (const Value *V : StatepointSite.vm_state_args()) { + for (const Value *V : SI.DeoptState) { reservePreviousStackSlotForValue(V, Builder); } - for (unsigned i = 0; i < Bases.size(); ++i) { - reservePreviousStackSlotForValue(Bases[i], Builder); - reservePreviousStackSlotForValue(Ptrs[i], Builder); + for (unsigned i = 0; i < SI.Bases.size(); ++i) { + reservePreviousStackSlotForValue(SI.Bases[i], Builder); + reservePreviousStackSlotForValue(SI.Ptrs[i], Builder); } // First, prefix the list with the number of unique values to be // lowered. Note that this is the number of *Values* not the // number of SDValues required to lower them. - const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs(); + const int NumVMSArgs = SI.DeoptState.size(); pushStackMapConstant(Ops, Builder, NumVMSArgs); - assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(), - StatepointSite.vm_state_end())); - - // The vm state arguments are lowered in an opaque manner. We do - // not know what type of values are contained within. We skip the - // first one since that happens to be the total number we lowered - // explicitly just above. We could have left it in the loop and - // not done it explicitly, but it's far easier to understand this - // way. - for (const Value *V : StatepointSite.vm_state_args()) { + // The vm state arguments are lowered in an opaque manner. We do not know + // what type of values are contained within. + for (const Value *V : SI.DeoptState) { SDValue Incoming = Builder.getValue(V); lowerIncomingStatepointValue(Incoming, Ops, Builder); } @@ -576,11 +495,11 @@ static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, // arrays interwoven with each (lowered) base pointer immediately followed by // it's (lowered) derived pointer. i.e // (base[0], ptr[0], base[1], ptr[1], ...) - for (unsigned i = 0; i < Bases.size(); ++i) { - const Value *Base = Bases[i]; + for (unsigned i = 0; i < SI.Bases.size(); ++i) { + const Value *Base = SI.Bases[i]; lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder); - const Value *Ptr = Ptrs[i]; + const Value *Ptr = SI.Ptrs[i]; lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder); } @@ -589,7 +508,7 @@ static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, // allocas and give control over placement to the consumer. In this case, // it is the contents of the slot which may get updated, not the pointer to // the alloca - for (Value *V : StatepointSite.gc_args()) { + for (Value *V : SI.GCArgs) { SDValue Incoming = Builder.getValue(V); if (FrameIndexSDNode *FI = dyn_cast(Incoming)) { // This handles allocas as arguments to the statepoint @@ -601,11 +520,10 @@ static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, // Record computed locations for all lowered values. // This can not be embedded in lowering loops as we need to record *all* // values, while previous loops account only values with unique SDValues. - const Instruction *StatepointInstr = - StatepointSite.getCallSite().getInstruction(); + const Instruction *StatepointInstr = SI.StatepointInstr; auto &SpillMap = Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr]; - for (const GCRelocateInst *Relocate : StatepointSite.getRelocates()) { + for (const GCRelocateInst *Relocate : SI.GCRelocates) { const Value *V = Relocate->getDerivedPtr(); SDValue SDV = Builder.getValue(V); SDValue Loc = Builder.StatepointLowering.getLocation(SDV); @@ -641,8 +559,8 @@ void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) { LowerStatepoint(ImmutableStatepoint(&CI)); } -void SelectionDAGBuilder::LowerStatepoint( - ImmutableStatepoint ISP, const BasicBlock *EHPadBB /*= nullptr*/) { +SDValue SelectionDAGBuilder::LowerAsStatepoint( + SelectionDAGBuilder::StatepointLoweringInfo &SI) { // The basic scheme here is that information about both the original call and // the safepoint is encoded in the CallInst. We create a temporary call and // lower it, then reverse engineer the calling sequence. @@ -651,36 +569,25 @@ void SelectionDAGBuilder::LowerStatepoint( // Clear state StatepointLowering.startNewStatepoint(*this); - ImmutableCallSite CS(ISP.getCallSite()); - #ifndef NDEBUG - // Consistency check. Check only relocates in the same basic block as thier - // statepoint. - for (const User *U : CS->users()) { - const CallInst *Call = cast(U); - if (isa(Call) && Call->getParent() == CS.getParent()) - StatepointLowering.scheduleRelocCall(*Call); - } -#endif - -#ifndef NDEBUG - // If this is a malformed statepoint, report it early to simplify debugging. - // This should catch any IR level mistake that's made when constructing or - // transforming statepoints. - ISP.verify(); - - // Check that the associated GCStrategy expects to encounter statepoints. - assert(GFI->getStrategy().useStatepoints() && - "GCStrategy does not expect to encounter statepoints"); + for (auto *Reloc : SI.GCRelocates) + if (Reloc->getParent() == SI.StatepointInstr->getParent()) + StatepointLowering.scheduleRelocCall(*Reloc); #endif // Lower statepoint vmstate and gcstate arguments SmallVector LoweredMetaArgs; - lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this); + lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this); + + // Now that we've emitted the spills, we need to update the root so that the + // call sequence is ordered correctly. + SI.CLI.setChain(getRoot()); // Get call node, we will replace it later with statepoint - SDNode *CallNode = - lowerCallFromStatepoint(ISP, EHPadBB, *this, PendingExports); + SDValue ReturnVal; + SDNode *CallNode; + std::tie(ReturnVal, CallNode) = + lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports); // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END // nodes with all the appropriate arguments and return values. @@ -703,8 +610,8 @@ void SelectionDAGBuilder::LowerStatepoint( // followed by a SRCVALUE for the pointer that may be used during lowering // (e.g. to form MachinePointerInfo values for loads/stores). const bool IsGCTransition = - (ISP.getFlags() & (uint64_t)StatepointFlags::GCTransition) == - (uint64_t)StatepointFlags::GCTransition; + (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) == + (uint64_t)StatepointFlags::GCTransition; if (IsGCTransition) { SmallVector TSOps; @@ -712,7 +619,7 @@ void SelectionDAGBuilder::LowerStatepoint( TSOps.push_back(Chain); // Add GC transition arguments - for (const Value *V : ISP.gc_transition_args()) { + for (const Value *V : SI.GCTransitionArgs) { TSOps.push_back(getValue(V)); if (V->getType()->isPointerTy()) TSOps.push_back(DAG.getSrcValue(V)); @@ -737,9 +644,9 @@ void SelectionDAGBuilder::LowerStatepoint( SmallVector Ops; // Add the and constants. - Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64)); + Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64)); Ops.push_back( - DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32)); + DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32)); // Calculate and push starting position of vmstate arguments // Get number of arguments incoming directly into call node @@ -761,10 +668,10 @@ void SelectionDAGBuilder::LowerStatepoint( Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt); // Add a constant argument for the calling convention - pushStackMapConstant(Ops, *this, CS.getCallingConv()); + pushStackMapConstant(Ops, *this, SI.CLI.CallConv); // Add a constant argument for the flags - uint64_t Flags = ISP.getFlags(); + uint64_t Flags = SI.StatepointFlags; assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) && "Unknown flag used"); pushStackMapConstant(Ops, *this, Flags); @@ -802,7 +709,7 @@ void SelectionDAGBuilder::LowerStatepoint( TEOps.push_back(SDValue(StatepointMCNode, 0)); // Add GC transition arguments - for (const Value *V : ISP.gc_transition_args()) { + for (const Value *V : SI.GCTransitionArgs) { TEOps.push_back(getValue(V)); if (V->getType()->isPointerTy()) TEOps.push_back(DAG.getSrcValue(V)); @@ -832,6 +739,104 @@ void SelectionDAGBuilder::LowerStatepoint( // return value of each gc.relocate to the respective output of the // previously emitted STATEPOINT value. Unfortunately, this doesn't appear // to actually be possible today. + + return ReturnVal; +} + +void +SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, + const BasicBlock *EHPadBB /*= nullptr*/) { + SmallVector Bases; + SmallVector Ptrs; + SmallVector GCRelocates; + + getIncomingStatepointGCValues(Bases, Ptrs, GCRelocates, ISP, *this); + + assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg && + "anyregcc is not supported on statepoints!"); + +#ifndef NDEBUG + // If this is a malformed statepoint, report it early to simplify debugging. + // This should catch any IR level mistake that's made when constructing or + // transforming statepoints. + ISP.verify(); + + // Check that the associated GCStrategy expects to encounter statepoints. + assert(GFI->getStrategy().useStatepoints() && + "GCStrategy does not expect to encounter statepoints"); +#endif + + SDValue ActualCallee; + + if (ISP.getNumPatchBytes() > 0) { + // If we've been asked to emit a nop sequence instead of a call instruction + // for this statepoint then don't lower the call target, but use a constant + // `null` instead. Not lowering the call target lets statepoint clients get + // away without providing a physical address for the symbolic call target at + // link time. + + const auto &TLI = DAG.getTargetLoweringInfo(); + const auto &DL = DAG.getDataLayout(); + + unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace(); + ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS)); + } else { + ActualCallee = getValue(ISP.getCalledValue()); + } + + StatepointLoweringInfo SI(DAG); + populateCallLoweringInfo(SI.CLI, ISP.getCallSite(), + ImmutableStatepoint::CallArgsBeginPos, + ISP.getNumCallArgs(), ActualCallee, + ISP.getActualReturnType(), false /* IsPatchPoint */); + + SI.Bases = Bases; + SI.Ptrs = Ptrs; + SI.GCRelocates = GCRelocates; + SI.GCArgs = ArrayRef(ISP.gc_args_begin(), ISP.gc_args_end()); + SI.StatepointInstr = ISP.getInstruction(); + SI.GCTransitionArgs = + ArrayRef(ISP.gc_args_begin(), ISP.gc_args_end()); + SI.ID = ISP.getID(); + SI.DeoptState = ArrayRef(ISP.vm_state_begin(), ISP.vm_state_end()); + SI.StatepointFlags = ISP.getFlags(); + SI.NumPatchBytes = ISP.getNumPatchBytes(); + SI.EHPadBB = EHPadBB; + + SDValue ReturnValue = LowerAsStatepoint(SI); + + // Export the result value if needed + const Instruction *GCResult = ISP.getGCResult(); + Type *RetTy = ISP.getActualReturnType(); + if (!RetTy->isVoidTy() && GCResult) { + if (GCResult->getParent() != ISP.getCallSite().getParent()) { + // Result value will be used in a different basic block so we need to + // export it now. Default exporting mechanism will not work here because + // statepoint call has a different type than the actual call. It means + // that by default llvm will create export register of the wrong type + // (always i32 in our case). So instead we need to create export register + // with correct type manually. + // TODO: To eliminate this problem we can remove gc.result intrinsics + // completely and make statepoint call to return a tuple. + unsigned Reg = FuncInfo.CreateRegs(RetTy); + RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), + DAG.getDataLayout(), Reg, RetTy); + SDValue Chain = DAG.getEntryNode(); + + RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr); + PendingExports.push_back(Chain); + FuncInfo.ValueMap[ISP.getInstruction()] = Reg; + } else { + // Result value will be used in a same basic block. Don't export it or + // perform any explicit register copies. + // We'll replace the actuall call node shortly. gc_result will grab + // this value. + setValue(ISP.getInstruction(), ReturnValue); + } + } else { + // The token value is never used from here on, just generate a poison value + setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc())); + } } void SelectionDAGBuilder::visitGCResult(const CallInst &CI) { -- 2.7.4