class CallBase;
class Function;
-enum class InlinePriorityMode : int { NoPriority, Size, Cost, OptRatio };
+enum class InlinePriorityMode : int { Size, Cost, OptRatio };
template <typename T> class InlineOrder {
public:
getInlineOrder(InlinePriorityMode UseInlinePriority,
FunctionAnalysisManager &FAM, const InlineParams &Params);
-template <typename T, typename Container = SmallVector<T, 16>>
-class DefaultInlineOrder : public InlineOrder<T> {
- using reference = T &;
- using const_reference = const T &;
-
-public:
- size_t size() override { return Calls.size() - FirstIndex; }
-
- void push(const T &Elt) override { Calls.push_back(Elt); }
-
- T pop() override {
- assert(size() > 0);
- return Calls[FirstIndex++];
- }
-
- const_reference front() override {
- assert(size() > 0);
- return Calls[FirstIndex];
- }
-
- void erase_if(function_ref<bool(T)> Pred) override {
- Calls.erase(std::remove_if(Calls.begin() + FirstIndex, Calls.end(), Pred),
- Calls.end());
- }
-
-private:
- Container Calls;
- size_t FirstIndex = 0;
-};
-
} // namespace llvm
#endif // LLVM_ANALYSIS_INLINEORDER_H
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
-#include "llvm/Analysis/InlineOrder.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
// this model, but it is uniformly spread across all the functions in the SCC
// and eventually they all become too large to inline, rather than
// incrementally maknig a single function grow in a super linear fashion.
- DefaultInlineOrder<std::pair<CallBase *, int>> Calls;
+ SmallVector<std::pair<CallBase *, int>, 16> Calls;
// Populate the initial list of calls in this SCC.
for (auto &N : InitialC) {
if (auto *CB = dyn_cast<CallBase>(&I))
if (Function *Callee = CB->getCalledFunction()) {
if (!Callee->isDeclaration())
- Calls.push({CB, -1});
+ Calls.push_back({CB, -1});
else if (!isa<IntrinsicInst>(I)) {
using namespace ore;
setInlineRemark(*CB, "unavailable definition");
// be deleted as a batch after inlining.
SmallVector<Function *, 4> DeadFunctionsInComdats;
- // Loop forward over all of the calls.
- while (!Calls.empty()) {
+ // Loop forward over all of the calls. Note that we cannot cache the size as
+ // inlining can introduce new calls that need to be processed.
+ for (int I = 0; I < (int)Calls.size(); ++I) {
// We expect the calls to typically be batched with sequences of calls that
// have the same caller, so we first set up some shared infrastructure for
// this caller. We also do any pruning we can at this layer on the caller
// alone.
- Function &F = *Calls.front().first->getCaller();
+ Function &F = *Calls[I].first->getCaller();
LazyCallGraph::Node &N = *CG.lookup(F);
- if (CG.lookupSCC(N) != C) {
- Calls.pop();
+ if (CG.lookupSCC(N) != C)
continue;
- }
LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n"
<< " Function size: " << F.getInstructionCount()
// We bail out as soon as the caller has to change so we can update the
// call graph and prepare the context of that new caller.
bool DidInline = false;
- while (!Calls.empty() && Calls.front().first->getCaller() == &F) {
- auto P = Calls.pop();
+ for (; I < (int)Calls.size() && Calls[I].first->getCaller() == &F; ++I) {
+ auto &P = Calls[I];
CallBase *CB = P.first;
const int InlineHistoryID = P.second;
Function &Callee = *CB->getCalledFunction();
}
if (NewCallee) {
if (!NewCallee->isDeclaration()) {
- Calls.push({ICB, NewHistoryID});
+ Calls.push_back({ICB, NewHistoryID});
// Continually inlining through an SCC can result in huge compile
// times and bloated code since we arbitrarily stop at some point
// when the inliner decides it's not profitable to inline anymore.
if (Callee.isDiscardableIfUnused() && Callee.hasZeroLiveUses() &&
!CG.isLibFunction(Callee)) {
if (Callee.hasLocalLinkage() || !Callee.hasComdat()) {
- Calls.erase_if([&](const std::pair<CallBase *, int> &Call) {
- return Call.first->getCaller() == &Callee;
- });
+ Calls.erase(
+ std::remove_if(Calls.begin() + I + 1, Calls.end(),
+ [&](const std::pair<CallBase *, int> &Call) {
+ return Call.first->getCaller() == &Callee;
+ }),
+ Calls.end());
+
// Clear the body and queue the function itself for deletion when we
// finish inlining and call graph updates.
// Note that after this point, it is an error to do anything other
Advice->recordInlining();
}
+ // Back the call index up by one to put us in a good position to go around
+ // the outer loop.
+ --I;
+
if (!DidInline)
continue;
Changed = true;
static cl::opt<InlinePriorityMode> UseInlinePriority(
"inline-priority-mode", cl::init(InlinePriorityMode::Size), cl::Hidden,
cl::desc("Choose the priority mode to use in module inline"),
- cl::values(clEnumValN(InlinePriorityMode::NoPriority, "no priority",
- "Use no priority, visit callsites in bottom-up."),
- clEnumValN(InlinePriorityMode::Size, "size",
+ cl::values(clEnumValN(InlinePriorityMode::Size, "size",
"Use callee size priority."),
clEnumValN(InlinePriorityMode::Cost, "cost",
"Use inline cost priority.")));