return ILV.getOrCreateScalarValue(V, Instance);
}
-void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent) const {
+void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n"
<< Indent << "\"INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";
IG->getInsertPos()->printAsOperand(O, false);
O << ", ";
- getAddr()->printAsOperand(O);
+ getAddr()->printAsOperand(O, SlotTracker);
VPValue *Mask = getMask();
if (Mask) {
O << ", ";
- Mask->printAsOperand(O);
+ Mask->printAsOperand(O, SlotTracker);
}
O << "\\l\"";
for (unsigned i = 0; i < IG->getFactor(); ++i)
#define DEBUG_TYPE "vplan"
raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
- if (const VPInstruction *Instr = dyn_cast<VPInstruction>(&V))
- Instr->print(OS);
- else
- V.printAsOperand(OS);
+ const VPInstruction *Instr = dyn_cast<VPInstruction>(&V);
+ VPSlotTracker SlotTracker(
+ (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
+ V.print(OS, SlotTracker);
return OS;
}
+void VPValue::print(raw_ostream &OS, VPSlotTracker &SlotTracker) const {
+ if (const VPInstruction *Instr = dyn_cast<VPInstruction>(this))
+ Instr->print(OS, SlotTracker);
+ else
+ printAsOperand(OS, SlotTracker);
+}
+
// Get the top-most entry block of \p Start. This is the entry block of the
// containing VPlan. This function is templated to support both const and non-const blocks
template <typename T> static T *getPlanEntry(T *Start) {
generateInstruction(State, Part);
}
-void VPInstruction::print(raw_ostream &O, const Twine &Indent) const {
+void VPInstruction::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"EMIT ";
- print(O);
+ print(O, SlotTracker);
O << "\\l\"";
}
void VPInstruction::print(raw_ostream &O) const {
- printAsOperand(O);
+ VPSlotTracker SlotTracker(getParent()->getPlan());
+ print(O, SlotTracker);
+}
+
+void VPInstruction::print(raw_ostream &O, VPSlotTracker &SlotTracker) const {
+ printAsOperand(O, SlotTracker);
O << " = ";
switch (getOpcode()) {
for (const VPValue *Operand : operands()) {
O << " ";
- Operand->printAsOperand(O);
+ Operand->printAsOperand(O, SlotTracker);
}
}
OS << "\\n" << DOT::EscapeString(Plan.getName());
if (!Plan.Value2VPValue.empty() || Plan.BackedgeTakenCount) {
OS << ", where:";
- if (Plan.BackedgeTakenCount)
- OS << "\\n" << *Plan.BackedgeTakenCount << " := BackedgeTakenCount";
+ if (Plan.BackedgeTakenCount) {
+ OS << "\\n";
+ Plan.BackedgeTakenCount->print(OS, SlotTracker);
+ OS << " := BackedgeTakenCount";
+ }
for (auto Entry : Plan.Value2VPValue) {
- OS << "\\n" << *Entry.second;
+ OS << "\\n";
+ Entry.second->print(OS, SlotTracker);
OS << DOT::EscapeString(" := ");
Entry.first->printAsOperand(OS, false);
}
if (Pred) {
OS << " +\n" << Indent << " \"BlockPredicate: ";
if (const VPInstruction *PredI = dyn_cast<VPInstruction>(Pred)) {
- PredI->printAsOperand(OS);
+ PredI->printAsOperand(OS, SlotTracker);
OS << " (" << DOT::EscapeString(PredI->getParent()->getName())
<< ")\\l\"";
} else
- Pred->printAsOperand(OS);
+ Pred->printAsOperand(OS, SlotTracker);
}
for (const VPRecipeBase &Recipe : *BasicBlock)
- Recipe.print(OS, Indent);
+ Recipe.print(OS, Indent, SlotTracker);
// Dump the condition bit.
const VPValue *CBV = BasicBlock->getCondBit();
if (CBV) {
OS << " +\n" << Indent << " \"CondBit: ";
if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) {
- CBI->printAsOperand(OS);
+ CBI->printAsOperand(OS, SlotTracker);
OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\"";
} else {
- CBV->printAsOperand(OS);
+ CBV->printAsOperand(OS, SlotTracker);
OS << "\"";
}
}
O << DOT::EscapeString(IngredientString);
}
-void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent) const {
+void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"WIDEN\\l\"";
for (auto &Instr : make_range(Begin, End))
O << " +\n" << Indent << "\" " << VPlanIngredient(&Instr) << "\\l\"";
}
-void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O,
- const Twine &Indent) const {
+void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"WIDEN-INDUCTION";
if (Trunc) {
O << "\\l\"";
O << " " << VPlanIngredient(IV) << "\\l\"";
}
-void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent) const {
+void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"WIDEN-GEP ";
O << (IsPtrLoopInvariant ? "Inv" : "Var");
size_t IndicesNumber = IsIndexLoopInvariant.size();
O << " +\n" << Indent << "\" " << VPlanIngredient(GEP) << "\\l\"";
}
-void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
+void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"WIDEN-PHI " << VPlanIngredient(Phi) << "\\l\"";
}
-void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent) const {
+void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"BLEND ";
Phi->printAsOperand(O, false);
O << " =";
O << " ";
Phi->getIncomingValue(I)->printAsOperand(O, false);
O << "/";
- User->getOperand(I)->printAsOperand(O);
+ User->getOperand(I)->printAsOperand(O, SlotTracker);
}
}
O << "\\l\"";
}
-void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent) const {
+void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n"
<< Indent << "\"" << (IsUniform ? "CLONE " : "REPLICATE ")
<< VPlanIngredient(Ingredient);
O << "\\l\"";
}
-void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
+void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n"
<< Indent << "\"PHI-PREDICATED-INSTRUCTION " << VPlanIngredient(PredInst)
<< "\\l\"";
}
-void VPWidenMemoryInstructionRecipe::print(raw_ostream &O,
- const Twine &Indent) const {
+void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"WIDEN " << VPlanIngredient(&Instr);
O << ", ";
- getAddr()->printAsOperand(O);
+ getAddr()->printAsOperand(O, SlotTracker);
VPValue *Mask = getMask();
if (Mask) {
O << ", ";
- Mask->printAsOperand(O);
+ Mask->printAsOperand(O, SlotTracker);
}
O << "\\l\"";
}
User->setOperand(I, New);
}
+void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const {
+ unsigned Slot = Tracker.getSlot(this);
+ if (Slot == unsigned(-1))
+ OS << "<badref>";
+ else
+ OS << "%vp" << Tracker.getSlot(this);
+}
+
void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
Old2NewTy &Old2New,
InterleavedAccessInfo &IAI) {
Old2NewTy Old2New;
visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI);
}
+
+void VPSlotTracker::assignSlot(const VPValue *V) {
+ assert(Slots.find(V) == Slots.end() && "VPValue already has a slot!");
+ Slots[V] = NextSlot++;
+}
+
+void VPSlotTracker::assignSlots(const VPBlockBase *VPBB) {
+ if (auto *Region = dyn_cast<VPRegionBlock>(VPBB))
+ assignSlots(Region);
+ else
+ assignSlots(cast<VPBasicBlock>(VPBB));
+}
+
+void VPSlotTracker::assignSlots(const VPRegionBlock *Region) {
+ ReversePostOrderTraversal<const VPBlockBase *> RPOT(Region->getEntry());
+ for (const VPBlockBase *Block : RPOT)
+ assignSlots(Block);
+}
+
+void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
+ for (const VPRecipeBase &Recipe : *VPBB) {
+ if (const auto *VPI = dyn_cast<VPInstruction>(&Recipe))
+ assignSlot(VPI);
+ }
+}
+
+void VPSlotTracker::assignSlots(const VPlan &Plan) {
+
+ for (const VPValue *V : Plan.VPExternalDefs)
+ assignSlot(V);
+
+ for (auto &E : Plan.Value2VPValue)
+ if (!isa<VPInstruction>(E.second))
+ assignSlot(E.second);
+
+ for (const VPValue *V : Plan.VPCBVs)
+ assignSlot(V);
+
+ if (Plan.BackedgeTakenCount)
+ assignSlot(Plan.BackedgeTakenCount);
+
+ ReversePostOrderTraversal<const VPBlockBase *> RPOT(Plan.getEntry());
+ for (const VPBlockBase *Block : RPOT)
+ assignSlots(Block);
+}
virtual void execute(struct VPTransformState &State) = 0;
/// Each recipe prints itself.
- virtual void print(raw_ostream &O, const Twine &Indent) const = 0;
+ void print(raw_ostream &O, const Twine &Indent);
+ virtual void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const = 0;
/// Insert an unlinked recipe into a basic block immediately before
/// the specified recipe.
void execute(VPTransformState &State) override;
/// Print the Recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
/// Print the VPInstruction.
void print(raw_ostream &O) const;
+ void print(raw_ostream &O, VPSlotTracker &SlotTracker) const;
/// Return true if this instruction may modify memory.
bool mayWriteToMemory() const {
}
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// A recipe for handling GEP instructions.
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// A recipe for handling phi nodes of integer and floating-point inductions,
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// A recipe for handling all phi nodes except for integer and FP inductions.
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// VPInterleaveRecipe is a recipe for transforming an interleave group of load
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
const InterleaveGroup<Instruction> *getInterleaveGroup() { return IG; }
};
void setAlsoPack(bool Pack) { AlsoPack = Pack; }
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// A recipe for generating conditional branches on the bits of a mask.
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override {
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override {
O << " +\n" << Indent << "\"BRANCH-ON-MASK ";
if (User)
- O << *User->getOperand(0);
+ User->getOperand(0)->print(O, SlotTracker);
else
O << " All-One";
O << "\\l\"";
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// A Recipe for widening load/store operations.
void execute(VPTransformState &State) override;
/// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
};
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
}
};
+class VPSlotTracker;
/// VPlan models a candidate for vectorization, encoding various decisions take
/// to produce efficient output IR, including which branches, basic-blocks and
/// output IR instructions to generate, and their cost. VPlan holds a
/// VPBlock.
class VPlan {
friend class VPlanPrinter;
+ friend class VPSlotTracker;
private:
/// Hold the single entry to the Hierarchical CFG of the VPlan.
SmallVector<VPValue *, 4> VPCBVs;
public:
- VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {}
+ VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {
+ if (Entry)
+ Entry->setPlan(this);
+ }
~VPlan() {
if (Entry)
unsigned BID = 0;
SmallDenseMap<const VPBlockBase *, unsigned> BlockID;
- VPlanPrinter(raw_ostream &O, const VPlan &P) : OS(O), Plan(P) {}
+ VPSlotTracker SlotTracker;
+
+ VPlanPrinter(raw_ostream &O, const VPlan &P)
+ : OS(O), Plan(P), SlotTracker(&P) {}
/// Handle indentation.
void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
// Forward declarations.
class VPUser;
+class VPSlotTracker;
+
// This is the base class of the VPlan Def/Use graph, used for modeling the data
// flow into, within and out of the VPlan. VPValues can stand for live-ins
// coming from the input IR, instructions which VPlan will generate if executed
/// for any other purpose, as the values may change as LLVM evolves.
unsigned getVPValueID() const { return SubclassID; }
- void printAsOperand(raw_ostream &OS) const {
- OS << "%vp" << (unsigned short)(unsigned long long)this;
- }
+ void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const;
+ void print(raw_ostream &OS, VPSlotTracker &Tracker) const;
unsigned getNumUsers() const { return Users.size(); }
void addUser(VPUser &User) { Users.push_back(&User); }
return const_operand_range(op_begin(), op_end());
}
};
+class VPlan;
+class VPBasicBlock;
+class VPRegionBlock;
+
+/// This class can be used to assign consecutive numbers to all VPValues in a
+/// VPlan and allows querying the numbering for printing, similar to the
+/// ModuleSlotTracker for IR values.
+class VPSlotTracker {
+private:
+ DenseMap<const VPValue *, unsigned> Slots;
+ unsigned NextSlot = 0;
+
+ void assignSlots(const VPBlockBase *VPBB);
+ void assignSlots(const VPRegionBlock *Region);
+ void assignSlots(const VPBasicBlock *VPBB);
+ void assignSlot(const VPValue *V);
+
+ void assignSlots(const VPlan &Plan);
+
+public:
+ VPSlotTracker(const VPlan *Plan) {
+ if (Plan)
+ assignSlots(*Plan);
+ }
+
+ unsigned getSlot(const VPValue *V) const {
+ auto I = Slots.find(V);
+ if (I == Slots.end())
+ return -1;
+ return I->second;
+ }
+};
} // namespace llvm
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "gtest/gtest.h"
+#include <string>
namespace llvm {
namespace {
}
}
+TEST(VPBasicBlockTest, print) {
+ VPInstruction *I1 = new VPInstruction(10, {});
+ VPInstruction *I2 = new VPInstruction(1, {I1});
+ VPInstruction *I3 = new VPInstruction(2, {I1, I2});
+
+ VPBasicBlock *VPBB1 = new VPBasicBlock();
+ VPBB1->appendRecipe(I1);
+ VPBB1->appendRecipe(I2);
+ VPBB1->appendRecipe(I3);
+
+ VPInstruction *I4 = new VPInstruction(4, {I3, I2});
+ VPInstruction *I5 = new VPInstruction(5, {I1});
+ VPBasicBlock *VPBB2 = new VPBasicBlock();
+ VPBB2->appendRecipe(I4);
+ VPBB2->appendRecipe(I5);
+
+ VPBlockUtils::connectBlocks(VPBB1, VPBB2);
+
+ // Check printing an instruction without associated VPlan.
+ {
+ std::string I3Dump;
+ raw_string_ostream OS(I3Dump);
+ I3->print(OS);
+ OS.flush();
+ EXPECT_EQ("<badref> = br <badref> <badref>", I3Dump);
+ }
+
+ VPlan Plan;
+ Plan.setEntry(VPBB1);
+ std::string FullDump;
+ raw_string_ostream(FullDump) << Plan;
+
+ EXPECT_EQ(R"(digraph VPlan {
+graph [labelloc=t, fontsize=30; label="Vectorization Plan"]
+node [shape=rect, fontname=Courier, fontsize=30]
+edge [fontname=Courier, fontsize=30]
+compound=true
+ N0 [label =
+ ":\n" +
+ "EMIT %vp0 = catchswitch\l" +
+ "EMIT %vp1 = ret %vp0\l" +
+ "EMIT %vp2 = br %vp0 %vp1\l"
+ ]
+ N0 -> N1 [ label=""]
+ N1 [label =
+ ":\n" +
+ "EMIT %vp3 = indirectbr %vp2 %vp1\l" +
+ "EMIT %vp4 = invoke %vp0\l"
+ ]
+}
+)",
+ FullDump);
+
+ {
+ std::string I3Dump;
+ raw_string_ostream OS(I3Dump);
+ I3->print(OS);
+ OS.flush();
+ EXPECT_EQ("%vp2 = br %vp0 %vp1", I3Dump);
+ }
+
+ {
+ std::string I2Dump;
+ raw_string_ostream OS(I2Dump);
+ OS << *I2;
+ OS.flush();
+ EXPECT_EQ("%vp1 = ret %vp0", I2Dump);
+ }
+}
+
} // namespace
} // namespace llvm