// base relation will remain. Internally, we add a mixture of the two
// types, then update all the second type to the first type
using DefiningValueMapTy = MapVector<Value *, Value *>;
+using PointerToBaseTy = MapVector<Value *, Value *>;
using StatepointLiveSetTy = SetVector<Value *>;
using RematerializedValueMapTy =
MapVector<AssertingVH<Instruction>, AssertingVH<Value>>;
/// The set of values known to be live across this safepoint
StatepointLiveSetTy LiveSet;
- /// Mapping from live pointers to a base-defining-value
- MapVector<Value *, Value *> PointerToBase;
-
/// The *new* gc.statepoint instruction itself. This produces the token
/// that normal path gc.relocates and the gc.result are tied to.
GCStatepointInst *StatepointToken;
// post condition: PointerToBase contains one (derived, base) pair for every
// pointer in live. Note that derived can be equal to base if the original
// pointer was a base pointer.
-static void
-findBasePointers(const StatepointLiveSetTy &live,
- MapVector<Value *, Value *> &PointerToBase,
- DominatorTree *DT, DefiningValueMapTy &DVCache) {
+static void findBasePointers(const StatepointLiveSetTy &live,
+ PointerToBaseTy &PointerToBase, DominatorTree *DT,
+ DefiningValueMapTy &DVCache) {
for (Value *ptr : live) {
Value *base = findBasePointer(ptr, DVCache);
assert(base && "failed to find base pointer");
/// parse point.
static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache,
CallBase *Call,
- PartiallyConstructedSafepointRecord &result) {
- MapVector<Value *, Value *> PointerToBase;
+ PartiallyConstructedSafepointRecord &result,
+ PointerToBaseTy &PointerToBase) {
StatepointLiveSetTy PotentiallyDerivedPointers = result.LiveSet;
// We assume that all pointers passed to deopt are base pointers; as an
// optimization, we can use this to avoid seperately materializing the base
PointerToBase[V] = V;
}
findBasePointers(PotentiallyDerivedPointers, PointerToBase, &DT, DVCache);
-
- if (PrintBasePointers) {
- errs() << "Base Pairs (w/o Relocation):\n";
- for (auto &Pair : PointerToBase) {
- errs() << " derived ";
- Pair.first->printAsOperand(errs(), false);
- errs() << " base ";
- Pair.second->printAsOperand(errs(), false);
- errs() << "\n";;
- }
- }
-
- result.PointerToBase = PointerToBase;
}
/// Given an updated version of the dataflow liveness results, update the
/// liveset and base pointer maps for the call site CS.
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
CallBase *Call,
- PartiallyConstructedSafepointRecord &result);
+ PartiallyConstructedSafepointRecord &result,
+ PointerToBaseTy &PointerToBase);
static void recomputeLiveInValues(
Function &F, DominatorTree &DT, ArrayRef<CallBase *> toUpdate,
- MutableArrayRef<struct PartiallyConstructedSafepointRecord> records) {
+ MutableArrayRef<struct PartiallyConstructedSafepointRecord> records,
+ PointerToBaseTy &PointerToBase) {
// TODO-PERF: reuse the original liveness, then simply run the dataflow
// again. The old values are still live and will help it stabilize quickly.
GCPtrLivenessData RevisedLivenessData;
computeLiveInValues(DT, F, RevisedLivenessData);
for (size_t i = 0; i < records.size(); i++) {
struct PartiallyConstructedSafepointRecord &info = records[i];
- recomputeLiveInValues(RevisedLivenessData, toUpdate[i], info);
+ recomputeLiveInValues(RevisedLivenessData, toUpdate[i], info,
+ PointerToBase);
}
}
const SmallVectorImpl<Value *> &BasePtrs,
const SmallVectorImpl<Value *> &LiveVariables,
PartiallyConstructedSafepointRecord &Result,
- std::vector<DeferredReplacement> &Replacements) {
+ std::vector<DeferredReplacement> &Replacements,
+ const PointerToBaseTy &PointerToBase) {
assert(BasePtrs.size() == LiveVariables.size());
// Then go ahead and use the builder do actually do the inserts. We insert
auto &Context = Call->getContext();
auto &DL = Call->getModule()->getDataLayout();
auto GetBaseAndOffset = [&](Value *Derived) {
- assert(Result.PointerToBase.count(Derived));
+ assert(PointerToBase.count(Derived));
unsigned AddressSpace = Derived->getType()->getPointerAddressSpace();
unsigned IntPtrSize = DL.getPointerSizeInBits(AddressSpace);
- Value *Base = Result.PointerToBase.find(Derived)->second;
+ Value *Base = PointerToBase.find(Derived)->second;
Value *Base_int = Builder.CreatePtrToInt(
Base, Type::getIntNTy(Context, IntPtrSize));
Value *Derived_int = Builder.CreatePtrToInt(
static void
makeStatepointExplicit(DominatorTree &DT, CallBase *Call,
PartiallyConstructedSafepointRecord &Result,
- std::vector<DeferredReplacement> &Replacements) {
+ std::vector<DeferredReplacement> &Replacements,
+ const PointerToBaseTy &PointerToBase) {
const auto &LiveSet = Result.LiveSet;
- const auto &PointerToBase = Result.PointerToBase;
// Convert to vector for efficient cross referencing.
SmallVector<Value *, 64> BaseVec, LiveVec;
assert(LiveVec.size() == BaseVec.size());
// Do the actual rewriting and delete the old statepoint
- makeStatepointExplicitImpl(Call, BaseVec, LiveVec, Result, Replacements);
+ makeStatepointExplicitImpl(Call, BaseVec, LiveVec, Result, Replacements,
+ PointerToBase);
}
// Helper function for the relocationViaAlloca.
// relocated values we don't do any user adjustments here.
static void rematerializeLiveValues(CallBase *Call,
PartiallyConstructedSafepointRecord &Info,
+ PointerToBaseTy &PointerToBase,
TargetTransformInfo &TTI) {
const unsigned int ChainLengthThreshold = 10;
for (Value *LiveValue: Info.LiveSet) {
// For each live pointer find its defining chain
SmallVector<Instruction *, 3> ChainToBase;
- assert(Info.PointerToBase.count(LiveValue));
+ assert(PointerToBase.count(LiveValue));
Value *RootOfChain =
findRematerializableChainToBasePointer(ChainToBase,
LiveValue);
// Handle the scenario where the RootOfChain is not equal to the
// Base Value, but they are essentially the same phi values.
- if (RootOfChain != Info.PointerToBase[LiveValue]) {
+ if (RootOfChain != PointerToBase[LiveValue]) {
PHINode *OrigRootPhi = dyn_cast<PHINode>(RootOfChain);
- PHINode *AlternateRootPhi = dyn_cast<PHINode>(Info.PointerToBase[LiveValue]);
+ PHINode *AlternateRootPhi = dyn_cast<PHINode>(PointerToBase[LiveValue]);
if (!OrigRootPhi || !AlternateRootPhi)
continue;
// PHI nodes that have the same incoming values, and belonging to the same
Instruction *InsertBefore = Call->getNextNode();
assert(InsertBefore);
Instruction *RematerializedValue = rematerializeChain(
- InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
+ InsertBefore, RootOfChain, PointerToBase[LiveValue]);
Info.RematerializedValues[RematerializedValue] = LiveValue;
} else {
auto *Invoke = cast<InvokeInst>(Call);
&*Invoke->getUnwindDest()->getFirstInsertionPt();
Instruction *NormalRematerializedValue = rematerializeChain(
- NormalInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
+ NormalInsertBefore, RootOfChain, PointerToBase[LiveValue]);
Instruction *UnwindRematerializedValue = rematerializeChain(
- UnwindInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
+ UnwindInsertBefore, RootOfChain, PointerToBase[LiveValue]);
Info.RematerializedValues[NormalRematerializedValue] = LiveValue;
Info.RematerializedValues[UnwindRematerializedValue] = LiveValue;
// site.
findLiveReferences(F, DT, ToUpdate, Records);
+ /// Global mapping from live pointers to a base-defining-value.
+ PointerToBaseTy PointerToBase;
+
// B) Find the base pointers for each live pointer
for (size_t i = 0; i < Records.size(); i++) {
PartiallyConstructedSafepointRecord &info = Records[i];
- findBasePointers(DT, DVCache, ToUpdate[i], info);
+ findBasePointers(DT, DVCache, ToUpdate[i], info, PointerToBase);
+ }
+ if (PrintBasePointers) {
+ errs() << "Base Pairs (w/o Relocation):\n";
+ for (auto &Pair : PointerToBase) {
+ errs() << " derived ";
+ Pair.first->printAsOperand(errs(), false);
+ errs() << " base ";
+ Pair.second->printAsOperand(errs(), false);
+ errs() << "\n";
+ ;
+ }
}
// The base phi insertion logic (for any safepoint) may have inserted new
PartiallyConstructedSafepointRecord &Info = Records[i];
SmallVector<Value *, 128> Bases;
- for (auto Pair : Info.PointerToBase)
- Bases.push_back(Pair.second);
+ for (auto *Derived : Info.LiveSet) {
+ assert(PointerToBase.count(Derived) && "Missed base for derived pointer");
+ Bases.push_back(PointerToBase[Derived]);
+ }
insertUseHolderAfter(ToUpdate[i], Bases, Holders);
}
// By selecting base pointers, we've effectively inserted new uses. Thus, we
// need to rerun liveness. We may *also* have inserted new defs, but that's
// not the key issue.
- recomputeLiveInValues(F, DT, ToUpdate, Records);
+ recomputeLiveInValues(F, DT, ToUpdate, Records, PointerToBase);
if (PrintBasePointers) {
- for (auto &Info : Records) {
- errs() << "Base Pairs: (w/Relocation)\n";
- for (auto Pair : Info.PointerToBase) {
- errs() << " derived ";
- Pair.first->printAsOperand(errs(), false);
- errs() << " base ";
- Pair.second->printAsOperand(errs(), false);
- errs() << "\n";
- }
+ errs() << "Base Pairs: (w/Relocation)\n";
+ for (auto Pair : PointerToBase) {
+ errs() << " derived ";
+ Pair.first->printAsOperand(errs(), false);
+ errs() << " base ";
+ Pair.second->printAsOperand(errs(), false);
+ errs() << "\n";
}
}
// Note that the relocation placement code relies on this filtering for
// correctness as it expects the base to be in the liveset, which isn't true
// if the base is constant.
- for (auto &Info : Records)
- for (auto &BasePair : Info.PointerToBase)
- if (isa<Constant>(BasePair.second))
- Info.LiveSet.remove(BasePair.first);
+ for (auto &Info : Records) {
+ Info.LiveSet.remove_if([&](Value *LiveV) {
+ assert(PointerToBase.count(LiveV) && "Missed base for derived pointer");
+ return isa<Constant>(PointerToBase[LiveV]);
+ });
+ }
for (CallInst *CI : Holders)
CI->eraseFromParent();
// some values instead of relocating them. This is purely an optimization and
// does not influence correctness.
for (size_t i = 0; i < Records.size(); i++)
- rematerializeLiveValues(ToUpdate[i], Records[i], TTI);
+ rematerializeLiveValues(ToUpdate[i], Records[i], PointerToBase, TTI);
// We need this to safely RAUW and delete call or invoke return values that
// may themselves be live over a statepoint. For details, please see usage in
// previous statepoint can not be a live variable, thus we can and remove
// the old statepoint calls as we go.)
for (size_t i = 0; i < Records.size(); i++)
- makeStatepointExplicit(DT, ToUpdate[i], Records[i], Replacements);
+ makeStatepointExplicit(DT, ToUpdate[i], Records[i], Replacements,
+ PointerToBase);
ToUpdate.clear(); // prevent accident use of invalid calls.
// these live sets, and migrate to using that data structure from this point
// onward.
Info.LiveSet.clear();
- Info.PointerToBase.clear();
}
+ PointerToBase.clear();
// Do all the fixups of the original live variables to their relocated selves
SmallVector<Value *, 128> Live;
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
CallBase *Call,
- PartiallyConstructedSafepointRecord &Info) {
+ PartiallyConstructedSafepointRecord &Info,
+ PointerToBaseTy &PointerToBase) {
StatepointLiveSetTy Updated;
findLiveSetAtInst(Call, RevisedLivenessData, Updated);
// We may have base pointers which are now live that weren't before. We need
// to update the PointerToBase structure to reflect this.
for (auto V : Updated)
- Info.PointerToBase.insert({V, V});
-
-#ifndef NDEBUG
- for (auto V : Updated)
- assert(Info.PointerToBase.count(V) &&
- "Must be able to find base for live value!");
-#endif
-
- // Remove any stale base mappings - this can happen since our liveness is
- // more precise then the one inherent in the base pointer analysis.
- DenseSet<Value *> ToErase;
- for (auto KVPair : Info.PointerToBase)
- if (!Updated.count(KVPair.first))
- ToErase.insert(KVPair.first);
-
- for (auto *V : ToErase)
- Info.PointerToBase.erase(V);
-
-#ifndef NDEBUG
- for (auto KVPair : Info.PointerToBase)
- assert(Updated.count(KVPair.first) && "record for non-live value");
-#endif
+ PointerToBase.insert({ V, V });
Info.LiveSet = Updated;
}