//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/IPO/FunctionSpecialization.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/InlineCost.h"
cl::desc("Force function specialization for every call site with a "
"constant argument"));
+static cl::opt<unsigned> FuncSpecializationMaxIters(
+ "func-specialization-max-iters", cl::Hidden,
+ cl::desc("The maximum number of iterations function specialization is run"),
+ cl::init(1));
+
static cl::opt<unsigned> MaxClonesThreshold(
"func-specialization-max-clones", cl::Hidden,
cl::desc("The maximum number of clones allowed for a single function "
cl::desc("Enable function specialization on the address of global values"));
// Disabled by default as it can significantly increase compilation times.
+// Running nikic's compile time tracker on x86 with instruction count as the
+// metric shows 3-4% regression for SPASS while being neutral for all other
+// benchmarks of the llvm test suite.
//
// https://llvm-compile-time-tracker.com
// https://github.com/nikic/llvm-compile-time-tracker
cl::desc("Enable specialization of functions that take a literal constant "
"as an argument."));
-Constant *FunctionSpecializer::getPromotableAlloca(AllocaInst *Alloca,
- CallInst *Call) {
+namespace {
+// Bookkeeping struct to pass data from the analysis and profitability phase
+// to the actual transform helper functions.
+struct SpecializationInfo {
+ SmallVector<ArgInfo, 8> Args; // Stores the {formal,actual} argument pairs.
+ InstructionCost Gain; // Profitability: Gain = Bonus - Cost.
+};
+} // Anonymous namespace
+
+using FuncList = SmallVectorImpl<Function *>;
+using CallArgBinding = std::pair<CallBase *, Constant *>;
+using CallSpecBinding = std::pair<CallBase *, SpecializationInfo>;
+// We are using MapVector because it guarantees deterministic iteration
+// order across executions.
+using SpecializationMap = SmallMapVector<CallBase *, SpecializationInfo, 8>;
+
+static Constant *getPromotableAlloca(AllocaInst *Alloca, CallInst *Call) {
Value *StoreValue = nullptr;
for (auto *User : Alloca->users()) {
// We can't use llvm::isAllocaPromotable() as that would fail because of
// Bail if there is any other unknown usage.
return nullptr;
}
- return getCandidateConstant(StoreValue);
+ return dyn_cast_or_null<Constant>(StoreValue);
}
// A constant stack value is an AllocaInst that has a single constant
// value stored to it. Return this constant if such an alloca stack value
// is a function argument.
-Constant *FunctionSpecializer::getConstantStackValue(CallInst *Call,
- Value *Val) {
+static Constant *getConstantStackValue(CallInst *Call, Value *Val,
+ SCCPSolver &Solver) {
if (!Val)
return nullptr;
Val = Val->stripPointerCasts();
// ret void
// }
//
-void FunctionSpecializer::promoteConstantStackValues() {
+static void constantArgPropagation(FuncList &WorkList, Module &M,
+ SCCPSolver &Solver) {
// Iterate over the argument tracked functions see if there
// are any new constant values for the call instruction via
// stack variables.
- for (Function &F : M) {
- if (!Solver.isArgumentTrackedFunction(&F))
- continue;
+ for (auto *F : WorkList) {
- for (auto *User : F.users()) {
+ for (auto *User : F->users()) {
auto *Call = dyn_cast<CallInst>(User);
if (!Call)
continue;
- if (!Solver.isBlockExecutable(Call->getParent()))
- continue;
-
bool Changed = false;
for (const Use &U : Call->args()) {
unsigned Idx = Call->getArgOperandNo(&U);
if (!Call->onlyReadsMemory(Idx) || !ArgOpType->isPointerTy())
continue;
- auto *ConstVal = getConstantStackValue(Call, ArgOp);
+ auto *ConstVal = getConstantStackValue(Call, ArgOp, Solver);
if (!ConstVal)
continue;
}
// ssa_copy intrinsics are introduced by the SCCP solver. These intrinsics
-// interfere with the promoteConstantStackValues() optimization.
+// interfere with the constantArgPropagation optimization.
static void removeSSACopy(Function &F) {
for (BasicBlock &BB : F) {
for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
}
}
-/// Remove any ssa_copy intrinsics that may have been introduced.
-void FunctionSpecializer::cleanUpSSA() {
- for (Function *F : SpecializedFuncs)
- removeSSACopy(*F);
+static void removeSSACopy(Module &M) {
+ for (Function &F : M)
+ removeSSACopy(F);
}
-/// Attempt to specialize functions in the module to enable constant
-/// propagation across function boundaries.
-///
-/// \returns true if at least one function is specialized.
-bool FunctionSpecializer::run() {
- bool Changed = false;
+namespace {
+class FunctionSpecializer {
- for (Function &F : M) {
- if (!isCandidateFunction(&F))
- continue;
+ /// The IPSCCP Solver.
+ SCCPSolver &Solver;
- auto Cost = getSpecializationCost(&F);
- if (!Cost.isValid()) {
- LLVM_DEBUG(dbgs() << "FnSpecialization: Invalid specialization cost.\n");
- continue;
- }
-
- LLVM_DEBUG(dbgs() << "FnSpecialization: Specialization cost for "
- << F.getName() << " is " << Cost << "\n");
+ /// Analysis manager, needed to invalidate analyses.
+ FunctionAnalysisManager *FAM;
- SmallVector<CallSpecBinding, 8> Specializations;
- if (!findSpecializations(&F, Cost, Specializations)) {
- LLVM_DEBUG(
- dbgs() << "FnSpecialization: No possible specializations found\n");
- continue;
- }
+ /// Analyses used to help determine if a function should be specialized.
+ std::function<AssumptionCache &(Function &)> GetAC;
+ std::function<TargetTransformInfo &(Function &)> GetTTI;
+ std::function<TargetLibraryInfo &(Function &)> GetTLI;
- Changed = true;
+ SmallPtrSet<Function *, 4> SpecializedFuncs;
+ SmallPtrSet<Function *, 4> FullySpecialized;
+ SmallVector<Instruction *> ReplacedWithConstant;
+ DenseMap<Function *, CodeMetrics> FunctionMetrics;
- SmallVector<Function *, 4> Clones;
- for (CallSpecBinding &Specialization : Specializations)
- Clones.push_back(createSpecialization(&F, Specialization));
+public:
+ FunctionSpecializer(SCCPSolver &Solver, FunctionAnalysisManager *FAM,
+ std::function<AssumptionCache &(Function &)> GetAC,
+ std::function<TargetTransformInfo &(Function &)> GetTTI,
+ std::function<TargetLibraryInfo &(Function &)> GetTLI)
+ : Solver(Solver), FAM(FAM), GetAC(GetAC), GetTTI(GetTTI), GetTLI(GetTLI) {
+ }
- Solver.solveWhileResolvedUndefsIn(Clones);
- updateCallSites(&F, Specializations);
+ ~FunctionSpecializer() {
+ // Eliminate dead code.
+ removeDeadInstructions();
+ removeDeadFunctions();
}
- promoteConstantStackValues();
+ /// Attempt to specialize functions in the module to enable constant
+ /// propagation across function boundaries.
+ ///
+ /// \returns true if at least one function is specialized.
+ bool specializeFunctions(FuncList &Candidates, FuncList &WorkList) {
+ bool Changed = false;
+ for (auto *F : Candidates) {
+ if (!isCandidateFunction(F))
+ continue;
- LLVM_DEBUG(if (NbFunctionsSpecialized) dbgs()
- << "FnSpecialization: Specialized " << NbFunctionsSpecialized
- << " functions in module " << M.getName() << "\n");
+ auto Cost = getSpecializationCost(F);
+ if (!Cost.isValid()) {
+ LLVM_DEBUG(
+ dbgs() << "FnSpecialization: Invalid specialization cost.\n");
+ continue;
+ }
- NumFuncSpecialized += NbFunctionsSpecialized;
- return Changed;
-}
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Specialization cost for "
+ << F->getName() << " is " << Cost << "\n");
-void FunctionSpecializer::removeDeadFunctions() {
- for (Function *F : FullySpecialized) {
- LLVM_DEBUG(dbgs() << "FnSpecialization: Removing dead function "
- << F->getName() << "\n");
- if (FAM)
- FAM->clear(*F, F->getName());
- F->eraseFromParent();
- }
- FullySpecialized.clear();
-}
+ SmallVector<CallSpecBinding, 8> Specializations;
+ if (!findSpecializations(F, Cost, Specializations)) {
+ LLVM_DEBUG(
+ dbgs() << "FnSpecialization: No possible specializations found\n");
+ continue;
+ }
+
+ Changed = true;
+ for (auto &Entry : Specializations)
+ specializeFunction(F, Entry.second, WorkList);
+ }
-// Compute the code metrics for function \p F.
-CodeMetrics &FunctionSpecializer::analyzeFunction(Function *F) {
- auto I = FunctionMetrics.insert({F, CodeMetrics()});
- CodeMetrics &Metrics = I.first->second;
- if (I.second) {
- // The code metrics were not cached.
- SmallPtrSet<const Value *, 32> EphValues;
- CodeMetrics::collectEphemeralValues(F, &(GetAC)(*F), EphValues);
- for (BasicBlock &BB : *F)
- Metrics.analyzeBasicBlock(&BB, (GetTTI)(*F), EphValues);
-
- LLVM_DEBUG(dbgs() << "FnSpecialization: Code size of function "
- << F->getName() << " is " << Metrics.NumInsts
- << " instructions\n");
+ updateSpecializedFuncs(Candidates, WorkList);
+ NumFuncSpecialized += NbFunctionsSpecialized;
+ return Changed;
}
- return Metrics;
-}
-/// Clone the function \p F and remove the ssa_copy intrinsics added by
-/// the SCCPSolver in the cloned version.
-static Function *cloneCandidateFunction(Function *F) {
- ValueToValueMapTy Mappings;
- Function *Clone = CloneFunction(F, Mappings);
- removeSSACopy(*Clone);
- return Clone;
-}
+ void removeDeadInstructions() {
+ for (auto *I : ReplacedWithConstant) {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Removing dead instruction " << *I
+ << "\n");
+ I->eraseFromParent();
+ }
+ ReplacedWithConstant.clear();
+ }
-/// This function decides whether it's worthwhile to specialize function
-/// \p F based on the known constant values its arguments can take on. It
-/// only discovers potential specialization opportunities without actually
-/// applying them.
-///
-/// \returns true if any specializations have been found.
-bool FunctionSpecializer::findSpecializations(
- Function *F, InstructionCost Cost,
- SmallVectorImpl<CallSpecBinding> &WorkList) {
- // Get a list of interesting arguments.
- SmallVector<Argument *, 4> Args;
- for (Argument &Arg : F->args())
- if (isArgumentInteresting(&Arg))
- Args.push_back(&Arg);
-
- if (!Args.size())
- return false;
+ void removeDeadFunctions() {
+ for (auto *F : FullySpecialized) {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Removing dead function "
+ << F->getName() << "\n");
+ if (FAM)
+ FAM->clear(*F, F->getName());
+ F->eraseFromParent();
+ }
+ FullySpecialized.clear();
+ }
- // Find all the call sites for the function.
- SpecializationMap Specializations;
- for (User *U : F->users()) {
- if (!isa<CallInst>(U) && !isa<InvokeInst>(U))
- continue;
- auto &CS = *cast<CallBase>(U);
+ bool tryToReplaceWithConstant(Value *V) {
+ if (!V->getType()->isSingleValueType() || isa<CallBase>(V) ||
+ V->user_empty())
+ return false;
+
+ const ValueLatticeElement &IV = Solver.getLatticeValueFor(V);
+ if (isOverdefined(IV))
+ return false;
+ auto *Const =
+ isConstant(IV) ? Solver.getConstant(IV) : UndefValue::get(V->getType());
+
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Replacing " << *V
+ << "\nFnSpecialization: with " << *Const << "\n");
+
+ // Record uses of V to avoid visiting irrelevant uses of const later.
+ SmallVector<Instruction *> UseInsts;
+ for (auto *U : V->users())
+ if (auto *I = dyn_cast<Instruction>(U))
+ if (Solver.isBlockExecutable(I->getParent()))
+ UseInsts.push_back(I);
+
+ V->replaceAllUsesWith(Const);
+
+ for (auto *I : UseInsts)
+ Solver.visit(I);
+
+ // Remove the instruction from Block and Solver.
+ if (auto *I = dyn_cast<Instruction>(V)) {
+ if (I->isSafeToRemove()) {
+ ReplacedWithConstant.push_back(I);
+ Solver.removeLatticeValueFor(I);
+ }
+ }
+ return true;
+ }
- // Skip irrelevant users.
- if (CS.getCalledFunction() != F)
- continue;
+private:
+ // The number of functions specialised, used for collecting statistics and
+ // also in the cost model.
+ unsigned NbFunctionsSpecialized = 0;
+
+ // Compute the code metrics for function \p F.
+ CodeMetrics &analyzeFunction(Function *F) {
+ auto I = FunctionMetrics.insert({F, CodeMetrics()});
+ CodeMetrics &Metrics = I.first->second;
+ if (I.second) {
+ // The code metrics were not cached.
+ SmallPtrSet<const Value *, 32> EphValues;
+ CodeMetrics::collectEphemeralValues(F, &(GetAC)(*F), EphValues);
+ for (BasicBlock &BB : *F)
+ Metrics.analyzeBasicBlock(&BB, (GetTTI)(*F), EphValues);
+
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Code size of function "
+ << F->getName() << " is " << Metrics.NumInsts
+ << " instructions\n");
+ }
+ return Metrics;
+ }
- // If the call site has attribute minsize set, that callsite won't be
- // specialized.
- if (CS.hasFnAttr(Attribute::MinSize))
- continue;
+ /// Clone the function \p F and remove the ssa_copy intrinsics added by
+ /// the SCCPSolver in the cloned version.
+ Function *cloneCandidateFunction(Function *F, ValueToValueMapTy &Mappings) {
+ Function *Clone = CloneFunction(F, Mappings);
+ removeSSACopy(*Clone);
+ return Clone;
+ }
- // If the parent of the call site will never be executed, we don't need
- // to worry about the passed value.
- if (!Solver.isBlockExecutable(CS.getParent()))
- continue;
+ /// This function decides whether it's worthwhile to specialize function
+ /// \p F based on the known constant values its arguments can take on. It
+ /// only discovers potential specialization opportunities without actually
+ /// applying them.
+ ///
+ /// \returns true if any specializations have been found.
+ bool findSpecializations(Function *F, InstructionCost Cost,
+ SmallVectorImpl<CallSpecBinding> &WorkList) {
+ // Get a list of interesting arguments.
+ SmallVector<Argument *, 4> Args;
+ for (Argument &Arg : F->args())
+ if (isArgumentInteresting(&Arg))
+ Args.push_back(&Arg);
+
+ if (!Args.size())
+ return false;
+
+ // Find all the call sites for the function.
+ SpecializationMap Specializations;
+ for (User *U : F->users()) {
+ if (!isa<CallInst>(U) && !isa<InvokeInst>(U))
+ continue;
+ auto &CS = *cast<CallBase>(U);
+ // If the call site has attribute minsize set, that callsite won't be
+ // specialized.
+ if (CS.hasFnAttr(Attribute::MinSize))
+ continue;
- // Examine arguments and create specialization candidates from call sites
- // with constant arguments.
- bool Added = false;
- for (Argument *A : Args) {
- Constant *C = getCandidateConstant(CS.getArgOperand(A->getArgNo()));
- if (!C)
+ // If the parent of the call site will never be executed, we don't need
+ // to worry about the passed value.
+ if (!Solver.isBlockExecutable(CS.getParent()))
continue;
- if (!Added) {
- Specializations[&CS] = {{}, 0 - Cost, nullptr};
- Added = true;
+ // Examine arguments and create specialization candidates from call sites
+ // with constant arguments.
+ bool Added = false;
+ for (Argument *A : Args) {
+ Constant *C = getCandidateConstant(CS.getArgOperand(A->getArgNo()));
+ if (!C)
+ continue;
+
+ if (!Added) {
+ Specializations[&CS] = {{}, 0 - Cost};
+ Added = true;
+ }
+
+ SpecializationInfo &S = Specializations.back().second;
+ S.Gain += getSpecializationBonus(A, C, Solver.getLoopInfo(*F));
+ S.Args.push_back({A, C});
}
+ Added = false;
+ }
- SpecializationInfo &S = Specializations.back().second;
- S.Gain += getSpecializationBonus(A, C, Solver.getLoopInfo(*F));
- S.Args.push_back({A, C});
+ // Remove unprofitable specializations.
+ if (!ForceFunctionSpecialization)
+ Specializations.remove_if(
+ [](const auto &Entry) { return Entry.second.Gain <= 0; });
+
+ // Clear the MapVector and return the underlying vector.
+ WorkList = Specializations.takeVector();
+
+ // Sort the candidates in descending order.
+ llvm::stable_sort(WorkList, [](const auto &L, const auto &R) {
+ return L.second.Gain > R.second.Gain;
+ });
+
+ // Truncate the worklist to 'MaxClonesThreshold' candidates if necessary.
+ if (WorkList.size() > MaxClonesThreshold) {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Number of candidates exceed "
+ << "the maximum number of clones threshold.\n"
+ << "FnSpecialization: Truncating worklist to "
+ << MaxClonesThreshold << " candidates.\n");
+ WorkList.erase(WorkList.begin() + MaxClonesThreshold, WorkList.end());
}
- Added = false;
- }
- // Remove unprofitable specializations.
- if (!ForceFunctionSpecialization)
- Specializations.remove_if(
- [](const auto &Entry) { return Entry.second.Gain <= 0; });
-
- // Clear the MapVector and return the underlying vector.
- WorkList = Specializations.takeVector();
-
- // Sort the candidates in descending order.
- llvm::stable_sort(WorkList, [](const auto &L, const auto &R) {
- return L.second.Gain > R.second.Gain;
- });
-
- // Truncate the worklist to 'MaxClonesThreshold' candidates if necessary.
- if (WorkList.size() > MaxClonesThreshold) {
- LLVM_DEBUG(dbgs() << "FnSpecialization: Number of candidates exceed "
- << "the maximum number of clones threshold.\n"
- << "FnSpecialization: Truncating worklist to "
- << MaxClonesThreshold << " candidates.\n");
- WorkList.erase(WorkList.begin() + MaxClonesThreshold, WorkList.end());
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Specializations for function "
+ << F->getName() << "\n";
+ for (const auto &Entry
+ : WorkList) {
+ dbgs() << "FnSpecialization: Gain = " << Entry.second.Gain
+ << "\n";
+ for (const ArgInfo &Arg : Entry.second.Args)
+ dbgs() << "FnSpecialization: FormalArg = "
+ << Arg.Formal->getNameOrAsOperand()
+ << ", ActualArg = "
+ << Arg.Actual->getNameOrAsOperand() << "\n";
+ });
+
+ return !WorkList.empty();
}
- LLVM_DEBUG(dbgs() << "FnSpecialization: Specializations for function "
- << F->getName() << "\n";
- for (const auto &Entry
- : WorkList) {
- dbgs() << "FnSpecialization: Gain = " << Entry.second.Gain
- << "\n";
- for (const ArgInfo &Arg : Entry.second.Args)
- dbgs() << "FnSpecialization: FormalArg = "
- << Arg.Formal->getNameOrAsOperand()
- << ", ActualArg = " << Arg.Actual->getNameOrAsOperand()
- << "\n";
- });
+ bool isCandidateFunction(Function *F) {
+ // Do not specialize the cloned function again.
+ if (SpecializedFuncs.contains(F))
+ return false;
- return !WorkList.empty();
-}
+ // If we're optimizing the function for size, we shouldn't specialize it.
+ if (F->hasOptSize() ||
+ shouldOptimizeForSize(F, nullptr, nullptr, PGSOQueryType::IRPass))
+ return false;
-bool FunctionSpecializer::isCandidateFunction(Function *F) {
- if (F->isDeclaration())
- return false;
+ // Exit if the function is not executable. There's no point in specializing
+ // a dead function.
+ if (!Solver.isBlockExecutable(&F->getEntryBlock()))
+ return false;
- if (F->hasFnAttribute(Attribute::NoDuplicate))
- return false;
+ // It wastes time to specialize a function which would get inlined finally.
+ if (F->hasFnAttribute(Attribute::AlwaysInline))
+ return false;
- if (!Solver.isArgumentTrackedFunction(F))
- return false;
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Try function: " << F->getName()
+ << "\n");
+ return true;
+ }
- // Do not specialize the cloned function again.
- if (SpecializedFuncs.contains(F))
- return false;
+ void specializeFunction(Function *F, SpecializationInfo &S,
+ FuncList &WorkList) {
+ ValueToValueMapTy Mappings;
+ Function *Clone = cloneCandidateFunction(F, Mappings);
+
+ // Rewrite calls to the function so that they call the clone instead.
+ rewriteCallSites(Clone, S.Args, Mappings);
+
+ // Initialize the lattice state of the arguments of the function clone,
+ // marking the argument on which we specialized the function constant
+ // with the given value.
+ Solver.markArgInFuncSpecialization(Clone, S.Args);
+
+ // Mark all the specialized functions
+ WorkList.push_back(Clone);
+ NbFunctionsSpecialized++;
+
+ // If the function has been completely specialized, the original function
+ // is no longer needed. Mark it unreachable.
+ if (F->getNumUses() == 0 || all_of(F->users(), [F](User *U) {
+ if (auto *CS = dyn_cast<CallBase>(U))
+ return CS->getFunction() == F;
+ return false;
+ })) {
+ Solver.markFunctionUnreachable(F);
+ FullySpecialized.insert(F);
+ }
+ }
- // If we're optimizing the function for size, we shouldn't specialize it.
- if (F->hasOptSize() ||
- shouldOptimizeForSize(F, nullptr, nullptr, PGSOQueryType::IRPass))
- return false;
+ /// Compute and return the cost of specializing function \p F.
+ InstructionCost getSpecializationCost(Function *F) {
+ CodeMetrics &Metrics = analyzeFunction(F);
+ // If the code metrics reveal that we shouldn't duplicate the function, we
+ // shouldn't specialize it. Set the specialization cost to Invalid.
+ // Or if the lines of codes implies that this function is easy to get
+ // inlined so that we shouldn't specialize it.
+ if (Metrics.notDuplicatable || !Metrics.NumInsts.isValid() ||
+ (!ForceFunctionSpecialization &&
+ !F->hasFnAttribute(Attribute::NoInline) &&
+ Metrics.NumInsts < SmallFunctionThreshold))
+ return InstructionCost::getInvalid();
+
+ // Otherwise, set the specialization cost to be the cost of all the
+ // instructions in the function and penalty for specializing more functions.
+ unsigned Penalty = NbFunctionsSpecialized + 1;
+ return Metrics.NumInsts * InlineConstants::getInstrCost() * Penalty;
+ }
- // Exit if the function is not executable. There's no point in specializing
- // a dead function.
- if (!Solver.isBlockExecutable(&F->getEntryBlock()))
- return false;
+ InstructionCost getUserBonus(User *U, llvm::TargetTransformInfo &TTI,
+ const LoopInfo &LI) {
+ auto *I = dyn_cast_or_null<Instruction>(U);
+ // If not an instruction we do not know how to evaluate.
+ // Keep minimum possible cost for now so that it doesnt affect
+ // specialization.
+ if (!I)
+ return std::numeric_limits<unsigned>::min();
+
+ InstructionCost Cost =
+ TTI.getInstructionCost(U, TargetTransformInfo::TCK_SizeAndLatency);
+
+ // Increase the cost if it is inside the loop.
+ unsigned LoopDepth = LI.getLoopDepth(I->getParent());
+ Cost *= std::pow((double)AvgLoopIterationCount, LoopDepth);
+
+ // Traverse recursively if there are more uses.
+ // TODO: Any other instructions to be added here?
+ if (I->mayReadFromMemory() || I->isCast())
+ for (auto *User : I->users())
+ Cost += getUserBonus(User, TTI, LI);
+
+ return Cost;
+ }
- // It wastes time to specialize a function which would get inlined finally.
- if (F->hasFnAttribute(Attribute::AlwaysInline))
- return false;
+ /// Compute a bonus for replacing argument \p A with constant \p C.
+ InstructionCost getSpecializationBonus(Argument *A, Constant *C,
+ const LoopInfo &LI) {
+ Function *F = A->getParent();
+ auto &TTI = (GetTTI)(*F);
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Analysing bonus for constant: "
+ << C->getNameOrAsOperand() << "\n");
+
+ InstructionCost TotalCost = 0;
+ for (auto *U : A->users()) {
+ TotalCost += getUserBonus(U, TTI, LI);
+ LLVM_DEBUG(dbgs() << "FnSpecialization: User cost ";
+ TotalCost.print(dbgs()); dbgs() << " for: " << *U << "\n");
+ }
- LLVM_DEBUG(dbgs() << "FnSpecialization: Try function: " << F->getName()
- << "\n");
- return true;
-}
+ // The below heuristic is only concerned with exposing inlining
+ // opportunities via indirect call promotion. If the argument is not a
+ // (potentially casted) function pointer, give up.
+ Function *CalledFunction = dyn_cast<Function>(C->stripPointerCasts());
+ if (!CalledFunction)
+ return TotalCost;
+
+ // Get TTI for the called function (used for the inline cost).
+ auto &CalleeTTI = (GetTTI)(*CalledFunction);
+
+ // Look at all the call sites whose called value is the argument.
+ // Specializing the function on the argument would allow these indirect
+ // calls to be promoted to direct calls. If the indirect call promotion
+ // would likely enable the called function to be inlined, specializing is a
+ // good idea.
+ int Bonus = 0;
+ for (User *U : A->users()) {
+ if (!isa<CallInst>(U) && !isa<InvokeInst>(U))
+ continue;
+ auto *CS = cast<CallBase>(U);
+ if (CS->getCalledOperand() != A)
+ continue;
-Function *
-FunctionSpecializer::createSpecialization(Function *F,
- CallSpecBinding &Specialization) {
- Function *Clone = cloneCandidateFunction(F);
- Specialization.second.Clone = Clone;
+ // Get the cost of inlining the called function at this call site. Note
+ // that this is only an estimate. The called function may eventually
+ // change in a way that leads to it not being inlined here, even though
+ // inlining looks profitable now. For example, one of its called
+ // functions may be inlined into it, making the called function too large
+ // to be inlined into this call site.
+ //
+ // We apply a boost for performing indirect call promotion by increasing
+ // the default threshold by the threshold for indirect calls.
+ auto Params = getInlineParams();
+ Params.DefaultThreshold += InlineConstants::IndirectCallThreshold;
+ InlineCost IC =
+ getInlineCost(*CS, CalledFunction, Params, CalleeTTI, GetAC, GetTLI);
+
+ // We clamp the bonus for this call to be between zero and the default
+ // threshold.
+ if (IC.isAlways())
+ Bonus += Params.DefaultThreshold;
+ else if (IC.isVariable() && IC.getCostDelta() > 0)
+ Bonus += IC.getCostDelta();
+
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Inlining bonus " << Bonus
+ << " for user " << *U << "\n");
+ }
- // Initialize the lattice state of the arguments of the function clone,
- // marking the argument on which we specialized the function constant
- // with the given value.
- Solver.markArgInFuncSpecialization(Clone, Specialization.second.Args);
+ return TotalCost + Bonus;
+ }
- Solver.addArgumentTrackedFunction(Clone);
- Solver.markBlockExecutable(&Clone->front());
+ /// Determine if it is possible to specialise the function for constant values
+ /// of the formal parameter \p A.
+ bool isArgumentInteresting(Argument *A) {
+ // No point in specialization if the argument is unused.
+ if (A->user_empty())
+ return false;
+
+ // For now, don't attempt to specialize functions based on the values of
+ // composite types.
+ Type *ArgTy = A->getType();
+ if (!ArgTy->isSingleValueType())
+ return false;
+
+ // Specialization of integer and floating point types needs to be explicitly
+ // enabled.
+ if (!EnableSpecializationForLiteralConstant &&
+ (ArgTy->isIntegerTy() || ArgTy->isFloatingPointTy()))
+ return false;
+
+ // SCCP solver does not record an argument that will be constructed on
+ // stack.
+ if (A->hasByValAttr() && !A->getParent()->onlyReadsMemory())
+ return false;
+
+ // Check the lattice value and decide if we should attemt to specialize,
+ // based on this argument. No point in specialization, if the lattice value
+ // is already a constant.
+ const ValueLatticeElement &LV = Solver.getLatticeValueFor(A);
+ if (LV.isUnknownOrUndef() || LV.isConstant() ||
+ (LV.isConstantRange() && LV.getConstantRange().isSingleElement())) {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Nothing to do, argument "
+ << A->getNameOrAsOperand() << " is already constant\n");
+ return false;
+ }
- // Mark all the specialized functions
- SpecializedFuncs.insert(Clone);
- NbFunctionsSpecialized++;
+ return true;
+ }
- return Clone;
-}
+ /// Check if the valuy \p V (an actual argument) is a constant or can only
+ /// have a constant value. Return that constant.
+ Constant *getCandidateConstant(Value *V) {
+ if (isa<PoisonValue>(V))
+ return nullptr;
-/// Compute and return the cost of specializing function \p F.
-InstructionCost FunctionSpecializer::getSpecializationCost(Function *F) {
- CodeMetrics &Metrics = analyzeFunction(F);
- // If the code metrics reveal that we shouldn't duplicate the function, we
- // shouldn't specialize it. Set the specialization cost to Invalid.
- // Or if the lines of codes implies that this function is easy to get
- // inlined so that we shouldn't specialize it.
- if (Metrics.notDuplicatable || !Metrics.NumInsts.isValid() ||
- (!ForceFunctionSpecialization &&
- !F->hasFnAttribute(Attribute::NoInline) &&
- Metrics.NumInsts < SmallFunctionThreshold))
- return InstructionCost::getInvalid();
-
- // Otherwise, set the specialization cost to be the cost of all the
- // instructions in the function and penalty for specializing more functions.
- unsigned Penalty = NbFunctionsSpecialized + 1;
- return Metrics.NumInsts * InlineConstants::getInstrCost() * Penalty;
-}
+ // TrackValueOfGlobalVariable only tracks scalar global variables.
+ if (auto *GV = dyn_cast<GlobalVariable>(V)) {
+ // Check if we want to specialize on the address of non-constant
+ // global values.
+ if (!GV->isConstant() && !SpecializeOnAddresses)
+ return nullptr;
-static InstructionCost getUserBonus(User *U, llvm::TargetTransformInfo &TTI,
- const LoopInfo &LI) {
- auto *I = dyn_cast_or_null<Instruction>(U);
- // If not an instruction we do not know how to evaluate.
- // Keep minimum possible cost for now so that it doesnt affect
- // specialization.
- if (!I)
- return std::numeric_limits<unsigned>::min();
-
- InstructionCost Cost =
- TTI.getInstructionCost(U, TargetTransformInfo::TCK_SizeAndLatency);
-
- // Increase the cost if it is inside the loop.
- unsigned LoopDepth = LI.getLoopDepth(I->getParent());
- Cost *= std::pow((double)AvgLoopIterationCount, LoopDepth);
-
- // Traverse recursively if there are more uses.
- // TODO: Any other instructions to be added here?
- if (I->mayReadFromMemory() || I->isCast())
- for (auto *User : I->users())
- Cost += getUserBonus(User, TTI, LI);
-
- return Cost;
-}
+ if (!GV->getValueType()->isSingleValueType())
+ return nullptr;
+ }
-/// Compute a bonus for replacing argument \p A with constant \p C.
-InstructionCost
-FunctionSpecializer::getSpecializationBonus(Argument *A, Constant *C,
- const LoopInfo &LI) {
- Function *F = A->getParent();
- auto &TTI = (GetTTI)(*F);
- LLVM_DEBUG(dbgs() << "FnSpecialization: Analysing bonus for constant: "
- << C->getNameOrAsOperand() << "\n");
-
- InstructionCost TotalCost = 0;
- for (auto *U : A->users()) {
- TotalCost += getUserBonus(U, TTI, LI);
- LLVM_DEBUG(dbgs() << "FnSpecialization: User cost ";
- TotalCost.print(dbgs()); dbgs() << " for: " << *U << "\n");
- }
+ // Select for possible specialisation values that are constants or
+ // are deduced to be constants or constant ranges with a single element.
+ Constant *C = dyn_cast<Constant>(V);
+ if (!C) {
+ const ValueLatticeElement &LV = Solver.getLatticeValueFor(V);
+ if (LV.isConstant())
+ C = LV.getConstant();
+ else if (LV.isConstantRange() &&
+ LV.getConstantRange().isSingleElement()) {
+ assert(V->getType()->isIntegerTy() && "Non-integral constant range");
+ C = Constant::getIntegerValue(
+ V->getType(), *LV.getConstantRange().getSingleElement());
+ } else
+ return nullptr;
+ }
- // The below heuristic is only concerned with exposing inlining
- // opportunities via indirect call promotion. If the argument is not a
- // (potentially casted) function pointer, give up.
- Function *CalledFunction = dyn_cast<Function>(C->stripPointerCasts());
- if (!CalledFunction)
- return TotalCost;
-
- // Get TTI for the called function (used for the inline cost).
- auto &CalleeTTI = (GetTTI)(*CalledFunction);
-
- // Look at all the call sites whose called value is the argument.
- // Specializing the function on the argument would allow these indirect
- // calls to be promoted to direct calls. If the indirect call promotion
- // would likely enable the called function to be inlined, specializing is a
- // good idea.
- int Bonus = 0;
- for (User *U : A->users()) {
- if (!isa<CallInst>(U) && !isa<InvokeInst>(U))
- continue;
- auto *CS = cast<CallBase>(U);
- if (CS->getCalledOperand() != A)
- continue;
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Found interesting argument "
+ << V->getNameOrAsOperand() << "\n");
- // Get the cost of inlining the called function at this call site. Note
- // that this is only an estimate. The called function may eventually
- // change in a way that leads to it not being inlined here, even though
- // inlining looks profitable now. For example, one of its called
- // functions may be inlined into it, making the called function too large
- // to be inlined into this call site.
- //
- // We apply a boost for performing indirect call promotion by increasing
- // the default threshold by the threshold for indirect calls.
- auto Params = getInlineParams();
- Params.DefaultThreshold += InlineConstants::IndirectCallThreshold;
- InlineCost IC =
- getInlineCost(*CS, CalledFunction, Params, CalleeTTI, GetAC, GetTLI);
-
- // We clamp the bonus for this call to be between zero and the default
- // threshold.
- if (IC.isAlways())
- Bonus += Params.DefaultThreshold;
- else if (IC.isVariable() && IC.getCostDelta() > 0)
- Bonus += IC.getCostDelta();
-
- LLVM_DEBUG(dbgs() << "FnSpecialization: Inlining bonus " << Bonus
- << " for user " << *U << "\n");
+ return C;
}
- return TotalCost + Bonus;
-}
+ /// Rewrite calls to function \p F to call function \p Clone instead.
+ ///
+ /// This function modifies calls to function \p F as long as the actual
+ /// arguments match those in \p Args. Note that for recursive calls we
+ /// need to compare against the cloned formal arguments.
+ ///
+ /// Callsites that have been marked with the MinSize function attribute won't
+ /// be specialized and rewritten.
+ void rewriteCallSites(Function *Clone, const SmallVectorImpl<ArgInfo> &Args,
+ ValueToValueMapTy &Mappings) {
+ assert(!Args.empty() && "Specialization without arguments");
+ Function *F = Args[0].Formal->getParent();
+
+ SmallVector<CallBase *, 8> CallSitesToRewrite;
+ for (auto *U : F->users()) {
+ if (!isa<CallInst>(U) && !isa<InvokeInst>(U))
+ continue;
+ auto &CS = *cast<CallBase>(U);
+ if (!CS.getCalledFunction() || CS.getCalledFunction() != F)
+ continue;
+ CallSitesToRewrite.push_back(&CS);
+ }
-/// Determine if it is possible to specialise the function for constant values
-/// of the formal parameter \p A.
-bool FunctionSpecializer::isArgumentInteresting(Argument *A) {
- // No point in specialization if the argument is unused.
- if (A->user_empty())
- return false;
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Replacing call sites of "
+ << F->getName() << " with " << Clone->getName() << "\n");
+
+ for (auto *CS : CallSitesToRewrite) {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: "
+ << CS->getFunction()->getName() << " ->" << *CS
+ << "\n");
+ if (/* recursive call */
+ (CS->getFunction() == Clone &&
+ all_of(Args,
+ [CS, &Mappings](const ArgInfo &Arg) {
+ unsigned ArgNo = Arg.Formal->getArgNo();
+ return CS->getArgOperand(ArgNo) == Mappings[Arg.Formal];
+ })) ||
+ /* normal call */
+ all_of(Args, [CS](const ArgInfo &Arg) {
+ unsigned ArgNo = Arg.Formal->getArgNo();
+ return CS->getArgOperand(ArgNo) == Arg.Actual;
+ })) {
+ CS->setCalledFunction(Clone);
+ Solver.markOverdefined(CS);
+ }
+ }
+ }
- // For now, don't attempt to specialize functions based on the values of
- // composite types.
- Type *ArgTy = A->getType();
- if (!ArgTy->isSingleValueType())
- return false;
+ void updateSpecializedFuncs(FuncList &Candidates, FuncList &WorkList) {
+ for (auto *F : WorkList) {
+ SpecializedFuncs.insert(F);
- // Specialization of integer and floating point types needs to be explicitly
- // enabled.
- if (!EnableSpecializationForLiteralConstant &&
- (ArgTy->isIntegerTy() || ArgTy->isFloatingPointTy()))
- return false;
+ // Initialize the state of the newly created functions, marking them
+ // argument-tracked and executable.
+ if (F->hasExactDefinition() && !F->hasFnAttribute(Attribute::Naked))
+ Solver.addTrackedFunction(F);
- // SCCP solver does not record an argument that will be constructed on
- // stack.
- if (A->hasByValAttr() && !A->getParent()->onlyReadsMemory())
- return false;
+ Solver.addArgumentTrackedFunction(F);
+ Candidates.push_back(F);
+ Solver.markBlockExecutable(&F->front());
- // Check the lattice value and decide if we should attemt to specialize,
- // based on this argument. No point in specialization, if the lattice value
- // is already a constant.
- const ValueLatticeElement &LV = Solver.getLatticeValueFor(A);
- if (LV.isUnknownOrUndef() || LV.isConstant() ||
- (LV.isConstantRange() && LV.getConstantRange().isSingleElement())) {
- LLVM_DEBUG(dbgs() << "FnSpecialization: Nothing to do, argument "
- << A->getNameOrAsOperand() << " is already constant\n");
- return false;
+ // Replace the function arguments for the specialized functions.
+ for (Argument &Arg : F->args())
+ if (!Arg.use_empty() && tryToReplaceWithConstant(&Arg))
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Replaced constant argument: "
+ << Arg.getNameOrAsOperand() << "\n");
+ }
}
+};
+} // namespace
+
+bool llvm::runFunctionSpecialization(
+ Module &M, FunctionAnalysisManager *FAM, const DataLayout &DL,
+ std::function<TargetLibraryInfo &(Function &)> GetTLI,
+ std::function<TargetTransformInfo &(Function &)> GetTTI,
+ std::function<AssumptionCache &(Function &)> GetAC,
+ function_ref<AnalysisResultsForFn(Function &)> GetAnalysis) {
+ SCCPSolver Solver(DL, GetTLI, M.getContext());
+ FunctionSpecializer FS(Solver, FAM, GetAC, GetTTI, GetTLI);
+ bool Changed = false;
- return true;
-}
+ // Loop over all functions, marking arguments to those with their addresses
+ // taken or that are external as overdefined.
+ for (Function &F : M) {
+ if (F.isDeclaration())
+ continue;
+ if (F.hasFnAttribute(Attribute::NoDuplicate))
+ continue;
-/// Check if the valuy \p V (an actual argument) is a constant or can only
-/// have a constant value. Return that constant.
-Constant *FunctionSpecializer::getCandidateConstant(Value *V) {
- if (isa<PoisonValue>(V))
- return nullptr;
+ LLVM_DEBUG(dbgs() << "\nFnSpecialization: Analysing decl: " << F.getName()
+ << "\n");
+ Solver.addAnalysis(F, GetAnalysis(F));
- // TrackValueOfGlobalVariable only tracks scalar global variables.
- if (auto *GV = dyn_cast<GlobalVariable>(V)) {
- // Check if we want to specialize on the address of non-constant
- // global values.
- if (!GV->isConstant() && !SpecializeOnAddresses)
- return nullptr;
+ // Determine if we can track the function's arguments. If so, add the
+ // function to the solver's set of argument-tracked functions.
+ if (canTrackArgumentsInterprocedurally(&F)) {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Can track arguments\n");
+ Solver.addArgumentTrackedFunction(&F);
+ continue;
+ } else {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Can't track arguments!\n"
+ << "FnSpecialization: Doesn't have local linkage, or "
+ << "has its address taken\n");
+ }
- if (!GV->getValueType()->isSingleValueType())
- return nullptr;
- }
+ // Assume the function is called.
+ Solver.markBlockExecutable(&F.front());
- // Select for possible specialisation values that are constants or
- // are deduced to be constants or constant ranges with a single element.
- Constant *C = dyn_cast<Constant>(V);
- if (!C) {
- const ValueLatticeElement &LV = Solver.getLatticeValueFor(V);
- if (LV.isConstant())
- C = LV.getConstant();
- else if (LV.isConstantRange() && LV.getConstantRange().isSingleElement()) {
- assert(V->getType()->isIntegerTy() && "Non-integral constant range");
- C = Constant::getIntegerValue(V->getType(),
- *LV.getConstantRange().getSingleElement());
- } else
- return nullptr;
+ // Assume nothing about the incoming arguments.
+ for (Argument &AI : F.args())
+ Solver.markOverdefined(&AI);
}
- LLVM_DEBUG(dbgs() << "FnSpecialization: Found interesting argument "
- << V->getNameOrAsOperand() << "\n");
+ // Determine if we can track any of the module's global variables. If so, add
+ // the global variables we can track to the solver's set of tracked global
+ // variables.
+ for (GlobalVariable &G : M.globals()) {
+ G.removeDeadConstantUsers();
+ if (canTrackGlobalVariableInterprocedurally(&G))
+ Solver.trackValueOfGlobalVariable(&G);
+ }
- return C;
-}
+ auto &TrackedFuncs = Solver.getArgumentTrackedFunctions();
+ SmallVector<Function *, 16> FuncDecls(TrackedFuncs.begin(),
+ TrackedFuncs.end());
-/// Redirects callsites of function \p F to its specialized copies.
-void FunctionSpecializer::updateCallSites(
- Function *F, SmallVectorImpl<CallSpecBinding> &Specializations) {
- SmallVector<CallBase *, 8> ToUpdate;
- for (User *U : F->users()) {
- if (auto *CS = dyn_cast<CallBase>(U))
- if (CS->getCalledFunction() == F &&
- Solver.isBlockExecutable(CS->getParent()))
- ToUpdate.push_back(CS);
+ // No tracked functions, so nothing to do: don't run the solver and remove
+ // the ssa_copy intrinsics that may have been introduced.
+ if (TrackedFuncs.empty()) {
+ removeSSACopy(M);
+ return false;
}
- unsigned NCallsLeft = ToUpdate.size();
- for (CallBase *CS : ToUpdate) {
- // Decrement the counter if the callsite is either recursive or updated.
- bool ShouldDecrementCount = CS->getFunction() == F;
- for (CallSpecBinding &Specialization : Specializations) {
- Function *Clone = Specialization.second.Clone;
- SmallVectorImpl<ArgInfo> &Args = Specialization.second.Args;
+ // Solve for constants.
+ auto RunSCCPSolver = [&](auto &WorkList) {
+ bool ResolvedUndefs = true;
+
+ while (ResolvedUndefs) {
+ // Not running the solver unnecessary is checked in regression test
+ // nothing-to-do.ll, so if this debug message is changed, this regression
+ // test needs updating too.
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Running solver\n");
+
+ Solver.solve();
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Resolving undefs\n");
+ ResolvedUndefs = false;
+ for (Function *F : WorkList)
+ if (Solver.resolvedUndefsIn(*F))
+ ResolvedUndefs = true;
+ }
- if (any_of(Args, [CS, this](const ArgInfo &Arg) {
- unsigned ArgNo = Arg.Formal->getArgNo();
- return getCandidateConstant(CS->getArgOperand(ArgNo)) != Arg.Actual;
- }))
- continue;
+ for (auto *F : WorkList) {
+ for (BasicBlock &BB : *F) {
+ if (!Solver.isBlockExecutable(&BB))
+ continue;
+ // FIXME: The solver may make changes to the function here, so set
+ // Changed, even if later function specialization does not trigger.
+ for (auto &I : make_early_inc_range(BB))
+ Changed |= FS.tryToReplaceWithConstant(&I);
+ }
+ }
+ };
- LLVM_DEBUG(dbgs() << "FnSpecialization: Replacing call site " << *CS
- << " with " << Clone->getName() << "\n");
+#ifndef NDEBUG
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Worklist fn decls:\n");
+ for (auto *F : FuncDecls)
+ LLVM_DEBUG(dbgs() << "FnSpecialization: *) " << F->getName() << "\n");
+#endif
- CS->setCalledFunction(Clone);
- ShouldDecrementCount = true;
- break;
- }
- if (ShouldDecrementCount)
- --NCallsLeft;
- }
+ // Initially resolve the constants in all the argument tracked functions.
+ RunSCCPSolver(FuncDecls);
+
+ SmallVector<Function *, 8> WorkList;
+ unsigned I = 0;
+ while (FuncSpecializationMaxIters != I++ &&
+ FS.specializeFunctions(FuncDecls, WorkList)) {
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Finished iteration " << I << "\n");
+
+ // Run the solver for the specialized functions.
+ RunSCCPSolver(WorkList);
+
+ // Replace some unresolved constant arguments.
+ constantArgPropagation(FuncDecls, M, Solver);
- // If the function has been completely specialized, the original function
- // is no longer needed. Mark it unreachable.
- if (NCallsLeft == 0) {
- Solver.markFunctionUnreachable(F);
- FullySpecialized.insert(F);
+ WorkList.clear();
+ Changed = true;
}
+
+ LLVM_DEBUG(dbgs() << "FnSpecialization: Number of specializations = "
+ << NumFuncSpecialized << "\n");
+
+ // Remove any ssa_copy intrinsics that may have been introduced.
+ removeSSACopy(M);
+ return Changed;
}
#include "llvm/InitializePasses.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Transforms/IPO.h"
-#include "llvm/Transforms/IPO/FunctionSpecialization.h"
#include "llvm/Transforms/Scalar/SCCP.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/SCCPSolver.h"
STATISTIC(NumInstReplaced,
"Number of instructions replaced with (simpler) instruction");
-static cl::opt<bool> SpecializeFunctions("specialize-functions",
- cl::init(false), cl::Hidden, cl::desc("Enable function specialization"));
-
-static cl::opt<unsigned> FuncSpecializationMaxIters(
- "func-specialization-max-iters", cl::init(1), cl::Hidden, cl::desc(
- "The maximum number of iterations function specialization is run"));
-
static void findReturnsToZap(Function &F,
SmallVector<ReturnInst *, 8> &ReturnsToZap,
SCCPSolver &Solver) {
}
static bool runIPSCCP(
- Module &M, const DataLayout &DL, FunctionAnalysisManager *FAM,
+ Module &M, const DataLayout &DL,
std::function<const TargetLibraryInfo &(Function &)> GetTLI,
- std::function<TargetTransformInfo &(Function &)> GetTTI,
- std::function<AssumptionCache &(Function &)> GetAC,
function_ref<AnalysisResultsForFn(Function &)> getAnalysis) {
SCCPSolver Solver(DL, GetTLI, M.getContext());
- FunctionSpecializer Specializer(Solver, M, FAM, GetTLI, GetTTI, GetAC);
// Loop over all functions, marking arguments to those with their addresses
// taken or that are external as overdefined.
}
// Solve for constants.
- Solver.solveWhileResolvedUndefsIn(M);
-
- if (SpecializeFunctions) {
- unsigned Iters = 0;
- while (Iters++ < FuncSpecializationMaxIters && Specializer.run());
+ bool ResolvedUndefs = true;
+ Solver.solve();
+ while (ResolvedUndefs) {
+ LLVM_DEBUG(dbgs() << "RESOLVING UNDEFS\n");
+ ResolvedUndefs = false;
+ for (Function &F : M) {
+ if (Solver.resolvedUndefsIn(F))
+ ResolvedUndefs = true;
+ }
+ if (ResolvedUndefs)
+ Solver.solve();
}
+ bool MadeChanges = false;
+
// Iterate over all of the instructions in the module, replacing them with
// constants if we have found them to be of constant values.
- bool MadeChanges = false;
+
for (Function &F : M) {
if (F.isDeclaration())
continue;
NumInstRemoved, NumInstReplaced);
}
- DomTreeUpdater DTU = SpecializeFunctions && Specializer.isClonedFunction(&F)
- ? DomTreeUpdater(DomTreeUpdater::UpdateStrategy::Lazy)
- : Solver.getDTU(F);
-
+ DomTreeUpdater DTU = Solver.getDTU(F);
// Change dead blocks to unreachable. We do it after replacing constants
// in all executable blocks, because changeToUnreachable may remove PHI
// nodes in executable blocks we found values for. The function's entry
auto GetTLI = [&FAM](Function &F) -> const TargetLibraryInfo & {
return FAM.getResult<TargetLibraryAnalysis>(F);
};
- auto GetTTI = [&FAM](Function &F) -> TargetTransformInfo & {
- return FAM.getResult<TargetIRAnalysis>(F);
- };
- auto GetAC = [&FAM](Function &F) -> AssumptionCache & {
- return FAM.getResult<AssumptionAnalysis>(F);
- };
auto getAnalysis = [&FAM](Function &F) -> AnalysisResultsForFn {
DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
return {
std::make_unique<PredicateInfo>(F, DT, FAM.getResult<AssumptionAnalysis>(F)),
&DT, FAM.getCachedResult<PostDominatorTreeAnalysis>(F),
- SpecializeFunctions ? &FAM.getResult<LoopAnalysis>(F) : nullptr };
+ nullptr};
};
- if (!runIPSCCP(M, DL, &FAM, GetTLI, GetTTI, GetAC, getAnalysis))
+ if (!runIPSCCP(M, DL, GetTLI, getAnalysis))
return PreservedAnalyses::all();
PreservedAnalyses PA;
auto GetTLI = [this](Function &F) -> const TargetLibraryInfo & {
return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
};
- auto GetTTI = [this](Function &F) -> TargetTransformInfo & {
- return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
- };
- auto GetAC = [this](Function &F) -> AssumptionCache & {
- return this->getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
- };
auto getAnalysis = [this](Function &F) -> AnalysisResultsForFn {
DominatorTree &DT =
this->getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
nullptr};
};
- return runIPSCCP(M, DL, nullptr, GetTLI, GetTTI, GetAC, getAnalysis);
+ return runIPSCCP(M, DL, GetTLI, getAnalysis);
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<AssumptionCacheTracker>();
AU.addRequired<DominatorTreeWrapperPass>();
AU.addRequired<TargetLibraryInfoWrapperPass>();
- AU.addRequired<TargetTransformInfoWrapperPass>();
}
};
// createIPSCCPPass - This is the public interface to this file.
ModulePass *llvm::createIPSCCPPass() { return new IPSCCPLegacyPass(); }
+PreservedAnalyses FunctionSpecializationPass::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ const DataLayout &DL = M.getDataLayout();
+ auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
+ return FAM.getResult<TargetLibraryAnalysis>(F);
+ };
+ auto GetTTI = [&FAM](Function &F) -> TargetTransformInfo & {
+ return FAM.getResult<TargetIRAnalysis>(F);
+ };
+ auto GetAC = [&FAM](Function &F) -> AssumptionCache & {
+ return FAM.getResult<AssumptionAnalysis>(F);
+ };
+ auto GetAnalysis = [&FAM](Function &F) -> AnalysisResultsForFn {
+ DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
+ return {std::make_unique<PredicateInfo>(
+ F, DT, FAM.getResult<AssumptionAnalysis>(F)),
+ &DT, FAM.getCachedResult<PostDominatorTreeAnalysis>(F),
+ &FAM.getResult<LoopAnalysis>(F)};
+ };
+
+ if (!runFunctionSpecialization(M, &FAM, DL, GetTLI, GetTTI, GetAC, GetAnalysis))
+ return PreservedAnalyses::all();
+
+ PreservedAnalyses PA;
+ PA.preserve<DominatorTreeAnalysis>();
+ PA.preserve<PostDominatorTreeAnalysis>();
+ PA.preserve<FunctionAnalysisManagerModuleProxy>();
+ return PA;
+}
+
+namespace {
+struct FunctionSpecializationLegacyPass : public ModulePass {
+ static char ID; // Pass identification, replacement for typeid
+ FunctionSpecializationLegacyPass() : ModulePass(ID) {}
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<AssumptionCacheTracker>();
+ AU.addRequired<DominatorTreeWrapperPass>();
+ AU.addRequired<TargetLibraryInfoWrapperPass>();
+ AU.addRequired<TargetTransformInfoWrapperPass>();
+ }
+
+ bool runOnModule(Module &M) override {
+ if (skipModule(M))
+ return false;
+
+ const DataLayout &DL = M.getDataLayout();
+ auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
+ return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+ };
+ auto GetTTI = [this](Function &F) -> TargetTransformInfo & {
+ return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+ };
+ auto GetAC = [this](Function &F) -> AssumptionCache & {
+ return this->getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+ };
+
+ auto GetAnalysis = [this](Function &F) -> AnalysisResultsForFn {
+ DominatorTree &DT =
+ this->getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
+ return {
+ std::make_unique<PredicateInfo>(
+ F, DT,
+ this->getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
+ F)),
+ nullptr, // We cannot preserve the LI, DT, or PDT with the legacy pass
+ nullptr, // manager, so set them to nullptr.
+ nullptr};
+ };
+ return runFunctionSpecialization(M, nullptr, DL, GetTLI, GetTTI, GetAC, GetAnalysis);
+ }
+};
+} // namespace
+
+char FunctionSpecializationLegacyPass::ID = 0;
+
+INITIALIZE_PASS_BEGIN(
+ FunctionSpecializationLegacyPass, "function-specialization",
+ "Propagate constant arguments by specializing the function", false, false)
+
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_END(FunctionSpecializationLegacyPass, "function-specialization",
+ "Propagate constant arguments by specializing the function",
+ false, false)
+
+ModulePass *llvm::createFunctionSpecializationPass() {
+ return new FunctionSpecializationLegacyPass();
+}
projects / platform / upstream / llvm.git / commitdiff