void mergeInStatus(VectorizationSafetyStatus S);
};
+class RuntimePointerChecking;
+/// A grouping of pointers. A single memcheck is required between
+/// two groups.
+struct RuntimeCheckingPtrGroup {
+ /// Create a new pointer checking group containing a single
+ /// pointer, with index \p Index in RtCheck.
+ RuntimeCheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck);
+
+ /// Tries to add the pointer recorded in RtCheck at index
+ /// \p Index to this pointer checking group. We can only add a pointer
+ /// to a checking group if we will still be able to get
+ /// the upper and lower bounds of the check. Returns true in case
+ /// of success, false otherwise.
+ bool addPointer(unsigned Index);
+
+ /// Constitutes the context of this pointer checking group. For each
+ /// pointer that is a member of this group we will retain the index
+ /// at which it appears in RtCheck.
+ RuntimePointerChecking &RtCheck;
+ /// The SCEV expression which represents the upper bound of all the
+ /// pointers in this group.
+ const SCEV *High;
+ /// The SCEV expression which represents the lower bound of all the
+ /// pointers in this group.
+ const SCEV *Low;
+ /// Indices of all the pointers that constitute this grouping.
+ SmallVector<unsigned, 2> Members;
+};
+
+/// A memcheck which made up of a pair of grouped pointers.
+typedef std::pair<const RuntimeCheckingPtrGroup *,
+ const RuntimeCheckingPtrGroup *>
+ RuntimePointerCheck;
+
/// Holds information about the memory runtime legality checks to verify
/// that a group of pointers do not overlap.
class RuntimePointerChecking {
+ friend struct RuntimeCheckingPtrGroup;
+
public:
struct PointerInfo {
/// Holds the pointer value that we need to check.
/// No run-time memory checking is necessary.
bool empty() const { return Pointers.empty(); }
- /// A grouping of pointers. A single memcheck is required between
- /// two groups.
- struct CheckingPtrGroup {
- /// Create a new pointer checking group containing a single
- /// pointer, with index \p Index in RtCheck.
- CheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck)
- : RtCheck(RtCheck), High(RtCheck.Pointers[Index].End),
- Low(RtCheck.Pointers[Index].Start) {
- Members.push_back(Index);
- }
-
- /// Tries to add the pointer recorded in RtCheck at index
- /// \p Index to this pointer checking group. We can only add a pointer
- /// to a checking group if we will still be able to get
- /// the upper and lower bounds of the check. Returns true in case
- /// of success, false otherwise.
- bool addPointer(unsigned Index);
-
- /// Constitutes the context of this pointer checking group. For each
- /// pointer that is a member of this group we will retain the index
- /// at which it appears in RtCheck.
- RuntimePointerChecking &RtCheck;
- /// The SCEV expression which represents the upper bound of all the
- /// pointers in this group.
- const SCEV *High;
- /// The SCEV expression which represents the lower bound of all the
- /// pointers in this group.
- const SCEV *Low;
- /// Indices of all the pointers that constitute this grouping.
- SmallVector<unsigned, 2> Members;
- };
-
- /// A memcheck which made up of a pair of grouped pointers.
- ///
- /// These *have* to be const for now, since checks are generated from
- /// CheckingPtrGroups in LAI::addRuntimeChecks which is a const member
- /// function. FIXME: once check-generation is moved inside this class (after
- /// the PtrPartition hack is removed), we could drop const.
- typedef std::pair<const CheckingPtrGroup *, const CheckingPtrGroup *>
- PointerCheck;
-
/// Generate the checks and store it. This also performs the grouping
/// of pointers to reduce the number of memchecks necessary.
void generateChecks(MemoryDepChecker::DepCandidates &DepCands,
bool UseDependencies);
/// Returns the checks that generateChecks created.
- const SmallVector<PointerCheck, 4> &getChecks() const { return Checks; }
+ const SmallVector<RuntimePointerCheck, 4> &getChecks() const {
+ return Checks;
+ }
/// Decide if we need to add a check between two groups of pointers,
/// according to needsChecking.
- bool needsChecking(const CheckingPtrGroup &M,
- const CheckingPtrGroup &N) const;
+ bool needsChecking(const RuntimeCheckingPtrGroup &M,
+ const RuntimeCheckingPtrGroup &N) const;
/// Returns the number of run-time checks required according to
/// needsChecking.
void print(raw_ostream &OS, unsigned Depth = 0) const;
/// Print \p Checks.
- void printChecks(raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks,
+ void printChecks(raw_ostream &OS,
+ const SmallVectorImpl<RuntimePointerCheck> &Checks,
unsigned Depth = 0) const;
/// This flag indicates if we need to add the runtime check.
SmallVector<PointerInfo, 2> Pointers;
/// Holds a partitioning of pointers into "check groups".
- SmallVector<CheckingPtrGroup, 2> CheckingGroups;
+ SmallVector<RuntimeCheckingPtrGroup, 2> CheckingGroups;
/// Check if pointers are in the same partition
///
bool UseDependencies);
/// Generate the checks and return them.
- SmallVector<PointerCheck, 4>
- generateChecks() const;
+ SmallVector<RuntimePointerCheck, 4> generateChecks() const;
/// Holds a pointer to the ScalarEvolution analysis.
ScalarEvolution *SE;
/// Set of run-time checks required to establish independence of
/// otherwise may-aliasing pointers in the loop.
- SmallVector<PointerCheck, 4> Checks;
+ SmallVector<RuntimePointerCheck, 4> Checks;
};
/// Drive the analysis of memory accesses in the loop
/// Returns a pair of instructions where the first element is the first
/// instruction generated in possibly a sequence of instructions and the
/// second value is the final comparator value or NULL if no check is needed.
- std::pair<Instruction *, Instruction *>
- addRuntimeChecks(Instruction *Loc,
- const SmallVectorImpl<RuntimePointerChecking::PointerCheck>
- &PointerChecks) const;
+ std::pair<Instruction *, Instruction *> addRuntimeChecks(
+ Instruction *Loc,
+ const SmallVectorImpl<RuntimePointerCheck> &PointerChecks) const;
/// The diagnostics report generated for the analysis. E.g. why we
/// couldn't analyze the loop.
#ifndef LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H
#define LLVM_TRANSFORMS_UTILS_LOOPVERSIONING_H
-#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
class LoopAccessInfo;
class LoopInfo;
class ScalarEvolution;
+struct RuntimeCheckingPtrGroup;
+typedef std::pair<const RuntimeCheckingPtrGroup *,
+ const RuntimeCheckingPtrGroup *>
+ RuntimePointerCheck;
/// This class emits a version of the loop where run-time checks ensure
/// that may-alias pointers can't overlap.
Loop *getNonVersionedLoop() { return NonVersionedLoop; }
/// Sets the runtime alias checks for versioning the loop.
- void setAliasChecks(
- SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks);
+ void setAliasChecks(SmallVector<RuntimePointerCheck, 4> Checks);
/// Sets the runtime SCEV checks for versioning the loop.
void setSCEVChecks(SCEVUnionPredicate Check);
ValueToValueMapTy VMap;
/// The set of alias checks that we are versioning for.
- SmallVector<RuntimePointerChecking::PointerCheck, 4> AliasChecks;
+ SmallVector<RuntimePointerCheck, 4> AliasChecks;
/// The set of SCEV checks that we are versioning for.
SCEVUnionPredicate Preds;
/// Maps a pointer to the pointer checking group that the pointer
/// belongs to.
- DenseMap<const Value *, const RuntimePointerChecking::CheckingPtrGroup *>
- PtrToGroup;
+ DenseMap<const Value *, const RuntimeCheckingPtrGroup *> PtrToGroup;
/// The alias scope corresponding to a pointer checking group.
- DenseMap<const RuntimePointerChecking::CheckingPtrGroup *, MDNode *>
- GroupToScope;
+ DenseMap<const RuntimeCheckingPtrGroup *, MDNode *> GroupToScope;
/// The list of alias scopes that a pointer checking group can't alias.
- DenseMap<const RuntimePointerChecking::CheckingPtrGroup *, MDNode *>
+ DenseMap<const RuntimeCheckingPtrGroup *, MDNode *>
GroupToNonAliasingScopeList;
/// Analyses used.
return OrigSCEV;
}
+RuntimeCheckingPtrGroup::RuntimeCheckingPtrGroup(
+ unsigned Index, RuntimePointerChecking &RtCheck)
+ : RtCheck(RtCheck), High(RtCheck.Pointers[Index].End),
+ Low(RtCheck.Pointers[Index].Start) {
+ Members.push_back(Index);
+}
+
/// Calculate Start and End points of memory access.
/// Let's assume A is the first access and B is a memory access on N-th loop
/// iteration. Then B is calculated as:
Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, Sc);
}
-SmallVector<RuntimePointerChecking::PointerCheck, 4>
+SmallVector<RuntimePointerCheck, 4>
RuntimePointerChecking::generateChecks() const {
- SmallVector<PointerCheck, 4> Checks;
+ SmallVector<RuntimePointerCheck, 4> Checks;
for (unsigned I = 0; I < CheckingGroups.size(); ++I) {
for (unsigned J = I + 1; J < CheckingGroups.size(); ++J) {
- const RuntimePointerChecking::CheckingPtrGroup &CGI = CheckingGroups[I];
- const RuntimePointerChecking::CheckingPtrGroup &CGJ = CheckingGroups[J];
+ const RuntimeCheckingPtrGroup &CGI = CheckingGroups[I];
+ const RuntimeCheckingPtrGroup &CGJ = CheckingGroups[J];
if (needsChecking(CGI, CGJ))
Checks.push_back(std::make_pair(&CGI, &CGJ));
Checks = generateChecks();
}
-bool RuntimePointerChecking::needsChecking(const CheckingPtrGroup &M,
- const CheckingPtrGroup &N) const {
+bool RuntimePointerChecking::needsChecking(
+ const RuntimeCheckingPtrGroup &M, const RuntimeCheckingPtrGroup &N) const {
for (unsigned I = 0, EI = M.Members.size(); EI != I; ++I)
for (unsigned J = 0, EJ = N.Members.size(); EJ != J; ++J)
if (needsChecking(M.Members[I], N.Members[J]))
return I;
}
-bool RuntimePointerChecking::CheckingPtrGroup::addPointer(unsigned Index) {
+bool RuntimeCheckingPtrGroup::addPointer(unsigned Index) {
const SCEV *Start = RtCheck.Pointers[Index].Start;
const SCEV *End = RtCheck.Pointers[Index].End;
// pointers to the same underlying object.
if (!UseDependencies) {
for (unsigned I = 0; I < Pointers.size(); ++I)
- CheckingGroups.push_back(CheckingPtrGroup(I, *this));
+ CheckingGroups.push_back(RuntimeCheckingPtrGroup(I, *this));
return;
}
MemoryDepChecker::MemAccessInfo Access(Pointers[I].PointerValue,
Pointers[I].IsWritePtr);
- SmallVector<CheckingPtrGroup, 2> Groups;
+ SmallVector<RuntimeCheckingPtrGroup, 2> Groups;
auto LeaderI = DepCands.findValue(DepCands.getLeaderValue(Access));
// Because DepCands is constructed by visiting accesses in the order in
// Go through all the existing sets and see if we can find one
// which can include this pointer.
- for (CheckingPtrGroup &Group : Groups) {
+ for (RuntimeCheckingPtrGroup &Group : Groups) {
// Don't perform more than a certain amount of comparisons.
// This should limit the cost of grouping the pointers to something
// reasonable. If we do end up hitting this threshold, the algorithm
// We couldn't add this pointer to any existing set or the threshold
// for the number of comparisons has been reached. Create a new group
// to hold the current pointer.
- Groups.push_back(CheckingPtrGroup(Pointer, *this));
+ Groups.push_back(RuntimeCheckingPtrGroup(Pointer, *this));
}
// We've computed the grouped checks for this partition.
}
void RuntimePointerChecking::printChecks(
- raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks,
+ raw_ostream &OS, const SmallVectorImpl<RuntimePointerCheck> &Checks,
unsigned Depth) const {
unsigned N = 0;
for (const auto &Check : Checks) {
/// Expand code for the lower and upper bound of the pointer group \p CG
/// in \p TheLoop. \return the values for the bounds.
-static PointerBounds
-expandBounds(const RuntimePointerChecking::CheckingPtrGroup *CG, Loop *TheLoop,
- Instruction *Loc, SCEVExpander &Exp, ScalarEvolution *SE,
- const RuntimePointerChecking &PtrRtChecking) {
+static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG,
+ Loop *TheLoop, Instruction *Loc,
+ SCEVExpander &Exp, ScalarEvolution *SE,
+ const RuntimePointerChecking &PtrRtChecking) {
Value *Ptr = PtrRtChecking.Pointers[CG->Members[0]].PointerValue;
const SCEV *Sc = SE->getSCEV(Ptr);
/// Turns a collection of checks into a collection of expanded upper and
/// lower bounds for both pointers in the check.
-static SmallVector<std::pair<PointerBounds, PointerBounds>, 4> expandBounds(
- const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks,
- Loop *L, Instruction *Loc, ScalarEvolution *SE, SCEVExpander &Exp,
- const RuntimePointerChecking &PtrRtChecking) {
+static SmallVector<std::pair<PointerBounds, PointerBounds>, 4>
+expandBounds(const SmallVectorImpl<RuntimePointerCheck> &PointerChecks, Loop *L,
+ Instruction *Loc, ScalarEvolution *SE, SCEVExpander &Exp,
+ const RuntimePointerChecking &PtrRtChecking) {
SmallVector<std::pair<PointerBounds, PointerBounds>, 4> ChecksWithBounds;
// Here we're relying on the SCEV Expander's cache to only emit code for the
// same bounds once.
transform(
PointerChecks, std::back_inserter(ChecksWithBounds),
- [&](const RuntimePointerChecking::PointerCheck &Check) {
+ [&](const RuntimePointerCheck &Check) {
PointerBounds
First = expandBounds(Check.first, L, Loc, Exp, SE, PtrRtChecking),
Second = expandBounds(Check.second, L, Loc, Exp, SE, PtrRtChecking);
std::pair<Instruction *, Instruction *> LoopAccessInfo::addRuntimeChecks(
Instruction *Loc,
- const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &PointerChecks)
- const {
+ const SmallVectorImpl<RuntimePointerCheck> &PointerChecks) const {
const DataLayout &DL = TheLoop->getHeader()->getModule()->getDataLayout();
auto *SE = PSE->getSE();
SCEVExpander Exp(*SE, DL, "induction");
/// \p PtrToPartition contains the partition number for pointers. Partition
/// number -1 means that the pointer is used in multiple partitions. In this
/// case we can't safely omit the check.
- SmallVector<RuntimePointerChecking::PointerCheck, 4>
- includeOnlyCrossPartitionChecks(
- const SmallVectorImpl<RuntimePointerChecking::PointerCheck> &AllChecks,
+ SmallVector<RuntimePointerCheck, 4> includeOnlyCrossPartitionChecks(
+ const SmallVectorImpl<RuntimePointerCheck> &AllChecks,
const SmallVectorImpl<int> &PtrToPartition,
const RuntimePointerChecking *RtPtrChecking) {
- SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks;
+ SmallVector<RuntimePointerCheck, 4> Checks;
copy_if(AllChecks, std::back_inserter(Checks),
- [&](const RuntimePointerChecking::PointerCheck &Check) {
+ [&](const RuntimePointerCheck &Check) {
for (unsigned PtrIdx1 : Check.first->Members)
for (unsigned PtrIdx2 : Check.second->Members)
// Only include this check if there is a pair of pointers
/// Determine the pointer alias checks to prove that there are no
/// intervening stores.
- SmallVector<RuntimePointerChecking::PointerCheck, 4> collectMemchecks(
+ SmallVector<RuntimePointerCheck, 4> collectMemchecks(
const SmallVectorImpl<StoreToLoadForwardingCandidate> &Candidates) {
SmallPtrSet<Value *, 4> PtrsWrittenOnFwdingPath =
std::mem_fn(&StoreToLoadForwardingCandidate::getLoadPtr));
const auto &AllChecks = LAI.getRuntimePointerChecking()->getChecks();
- SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks;
+ SmallVector<RuntimePointerCheck, 4> Checks;
copy_if(AllChecks, std::back_inserter(Checks),
- [&](const RuntimePointerChecking::PointerCheck &Check) {
+ [&](const RuntimePointerCheck &Check) {
for (auto PtrIdx1 : Check.first->Members)
for (auto PtrIdx2 : Check.second->Members)
if (needsChecking(PtrIdx1, PtrIdx2, PtrsWrittenOnFwdingPath,
// Check intervening may-alias stores. These need runtime checks for alias
// disambiguation.
- SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks =
- collectMemchecks(Candidates);
+ SmallVector<RuntimePointerCheck, 4> Checks = collectMemchecks(Candidates);
// Too many checks are likely to outweigh the benefits of forwarding.
if (Checks.size() > Candidates.size() * CheckPerElim) {
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/MemorySSA.h"
}
void LoopVersioning::setAliasChecks(
- SmallVector<RuntimePointerChecking::PointerCheck, 4> Checks) {
+ SmallVector<RuntimePointerCheck, 4> Checks) {
AliasChecks = std::move(Checks);
}
// Go through the checks and for each pointer group, collect the scopes for
// each non-aliasing pointer group.
- DenseMap<const RuntimePointerChecking::CheckingPtrGroup *,
- SmallVector<Metadata *, 4>>
+ DenseMap<const RuntimeCheckingPtrGroup *, SmallVector<Metadata *, 4>>
GroupToNonAliasingScopes;
for (const auto &Check : AliasChecks)
projects / platform / upstream / llvm.git / commitdiff