#include "llvm/MCA/SourceMgr.h"
#include "llvm/MCA/Stages/Stage.h"
-#include <queue>
-
namespace llvm {
struct MCSchedModel;
class MCSubtargetInfo;
class RegisterFile;
class ResourceManager;
+struct StallInfo {
+ enum class StallKind { DEFAULT, REGISTER_DEPS, DISPATCH, DELAY };
+
+ InstRef IR;
+ unsigned CyclesLeft;
+ StallKind Kind;
+
+ StallInfo() : IR(), CyclesLeft(), Kind(StallKind::DEFAULT) {}
+
+ bool isValid() const { return (bool)IR; }
+
+ StallKind getStallKind() const { return Kind; }
+ unsigned getCyclesLeft() const { return CyclesLeft; }
+ const InstRef &getInstruction() const { return IR; }
+ InstRef &getInstruction() { return IR; }
+
+ void clear() {
+ IR.invalidate();
+ CyclesLeft = 0;
+ Kind = StallKind::DEFAULT;
+ }
+
+ void update(const InstRef &Inst, unsigned Cycles, StallKind SK) {
+ IR = Inst;
+ CyclesLeft = Cycles;
+ Kind = SK;
+ }
+
+ void cycleEnd() {
+ if (!isValid())
+ return;
+
+ if (!CyclesLeft)
+ return;
+
+ --CyclesLeft;
+ }
+};
+
class InOrderIssueStage final : public Stage {
const MCSchedModel &SM;
const MCSubtargetInfo &STI;
/// Number of instructions issued in the current cycle.
unsigned NumIssued;
- /// If an instruction cannot execute due to an unmet register or resource
- /// dependency, the it is stalled for StallCyclesLeft.
- InstRef StalledInst;
- unsigned StallCyclesLeft;
+ StallInfo SI;
/// Instruction that is issued in more than 1 cycle.
InstRef CarriedOver;
InOrderIssueStage(const InOrderIssueStage &Other) = delete;
InOrderIssueStage &operator=(const InOrderIssueStage &Other) = delete;
- /// If IR has an unmet register or resource dependency, canExecute returns
- /// false. StallCycles is set to the number of cycles left before the
- /// instruction can be issued.
- bool canExecute(const InstRef &IR, unsigned *StallCycles) const;
+ /// Returns true if IR can execute during this cycle.
+ /// In case of stall, it updates SI with information about the stalled
+ /// instruction and the stall reason.
+ bool canExecute(const InstRef &IR);
- /// Issue the instruction, or update StallCycles if IR is stalled.
- Error tryIssue(InstRef &IR, unsigned *StallCycles);
+ /// Issue the instruction, or update the StallInfo.
+ Error tryIssue(InstRef &IR);
/// Update status of instructions from IssuedInst.
void updateIssuedInst();
/// Continue to issue the CarriedOver instruction.
void updateCarriedOver();
+ /// Notifies a stall event to the Stage listener. Stall information is
+ /// obtained from the internal StallInfo field.
+ void notifyStallEvent();
+
+ void notifyInstructionIssued(const InstRef &IR,
+ ArrayRef<ResourceUse> UsedRes);
+ void notifyInstructionDispatched(const InstRef &IR, unsigned Ops,
+ ArrayRef<unsigned> UsedRegs);
+ void notifyInstructionExecuted(const InstRef &IR);
+ void notifyInstructionRetired(const InstRef &IR,
+ ArrayRef<unsigned> FreedRegs);
+
/// Retire instruction once it is executed.
void retireInstruction(InstRef &IR);
InOrderIssueStage(RegisterFile &PRF, const MCSchedModel &SM,
const MCSubtargetInfo &STI)
: SM(SM), STI(STI), PRF(PRF), RM(std::make_unique<ResourceManager>(SM)),
- NumIssued(0), StallCyclesLeft(0), CarryOver(0), Bandwidth(0),
- LastWriteBackCycle(0) {}
+ NumIssued(), SI(), CarryOver(), Bandwidth(), LastWriteBackCycle() {}
bool isAvailable(const InstRef &) const override;
bool hasWorkToComplete() const override;
void WriteRef::commit() {
assert(Write && Write->isExecuted() && "Cannot commit before write back!");
+ RegisterID = Write->getRegisterID();
+ WriteResID = Write->getWriteResourceID();
Write = nullptr;
}
assert(RegID && "Adding an invalid register definition?");
LLVM_DEBUG({
- dbgs() << "RegisterFile: addRegisterWrite [ " << Write.getSourceIndex()
- << ", " << MRI.getName(RegID) << "]\n";
+ dbgs() << "[PRF] addRegisterWrite [ " << Write.getSourceIndex() << ", "
+ << MRI.getName(RegID) << "]\n";
});
// If RenameAs is equal to RegID, then RegID is subject to register renaming
const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
MCPhysReg RegID = RS.getRegisterID();
assert(RegID && RegID < RegisterMappings.size());
- LLVM_DEBUG(dbgs() << "RegisterFile: collecting writes for register "
+ LLVM_DEBUG(dbgs() << "[PRF] collecting writes for register "
<< MRI.getName(RegID) << '\n');
// Check if this is an alias.
});
}
+RegisterFile::RAWHazard
+RegisterFile::checkRAWHazards(const MCSubtargetInfo &STI,
+ const ReadState &RS) const {
+ RAWHazard Hazard;
+ SmallVector<WriteRef, 4> Writes;
+ SmallVector<WriteRef, 4> CommittedWrites;
+
+ const MCSchedModel &SM = STI.getSchedModel();
+ const ReadDescriptor &RD = RS.getDescriptor();
+ const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
+
+ collectWrites(STI, RS, Writes, CommittedWrites);
+ for (const WriteRef &WR : Writes) {
+ const WriteState *WS = WR.getWriteState();
+ unsigned WriteResID = WS->getWriteResourceID();
+ int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
+
+ if (WS->getCyclesLeft() == UNKNOWN_CYCLES) {
+ if (Hazard.isValid())
+ continue;
+
+ Hazard.RegisterID = WR.getRegisterID();
+ Hazard.CyclesLeft = UNKNOWN_CYCLES;
+ continue;
+ }
+
+ int CyclesLeft = WS->getCyclesLeft() - ReadAdvance;
+ if (CyclesLeft > 0) {
+ if (Hazard.CyclesLeft < CyclesLeft) {
+ Hazard.RegisterID = WR.getRegisterID();
+ Hazard.CyclesLeft = CyclesLeft;
+ }
+ }
+ }
+ Writes.clear();
+
+ for (const WriteRef &WR : CommittedWrites) {
+ unsigned WriteResID = WR.getWriteResourceID();
+ int NegReadAdvance = -STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
+ int Elapsed = static_cast<int>(getElapsedCyclesFromWriteBack(WR));
+ int CyclesLeft = NegReadAdvance - Elapsed;
+ assert(CyclesLeft > 0 && "Write should not be in the CommottedWrites set!");
+ if (Hazard.CyclesLeft < CyclesLeft) {
+ Hazard.RegisterID = WR.getRegisterID();
+ Hazard.CyclesLeft = CyclesLeft;
+ }
+ }
+
+ return Hazard;
+}
+
void RegisterFile::addRegisterRead(ReadState &RS,
const MCSubtargetInfo &STI) const {
MCPhysReg RegID = RS.getRegisterID();
// microarchitectural registers in register file #0 was changed by the
// users via flag -reg-file-size. Alternatively, the scheduling model
// specified a too small number of registers for this register file.
- LLVM_DEBUG(dbgs() << "Not enough registers in the register file.\n");
+ LLVM_DEBUG(
+ dbgs() << "[PRF] Not enough registers in the register file.\n");
// FIXME: Normalize the instruction register count to match the
// NumPhysRegs value. This is a highly unusual case, and is not expected
//===----------------------------------------------------------------------===//
#include "llvm/MCA/Stages/InOrderIssueStage.h"
-
-#include "llvm/MC/MCSchedule.h"
-#include "llvm/MCA/HWEventListener.h"
#include "llvm/MCA/HardwareUnits/RegisterFile.h"
#include "llvm/MCA/HardwareUnits/ResourceManager.h"
#include "llvm/MCA/HardwareUnits/RetireControlUnit.h"
#include "llvm/MCA/Instruction.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Error.h"
-
-#include <algorithm>
#define DEBUG_TYPE "llvm-mca"
namespace llvm {
namespace mca {
bool InOrderIssueStage::hasWorkToComplete() const {
- return !IssuedInst.empty() || StalledInst || CarriedOver;
+ return !IssuedInst.empty() || SI.isValid() || CarriedOver;
}
bool InOrderIssueStage::isAvailable(const InstRef &IR) const {
- if (StalledInst || CarriedOver)
+ if (SI.isValid() || CarriedOver)
return false;
const Instruction &Inst = *IR.getInstruction();
/// Return a number of cycles left until register requirements of the
/// instructions are met.
static unsigned checkRegisterHazard(const RegisterFile &PRF,
- const MCSchedModel &SM,
const MCSubtargetInfo &STI,
const InstRef &IR) {
- unsigned StallCycles = 0;
- SmallVector<WriteRef, 4> Writes;
- SmallVector<WriteRef, 4> CommittedWrites;
-
for (const ReadState &RS : IR.getInstruction()->getUses()) {
- const ReadDescriptor &RD = RS.getDescriptor();
- const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
-
- PRF.collectWrites(STI, RS, Writes, CommittedWrites);
- for (const WriteRef &WR : Writes) {
- const WriteState *WS = WR.getWriteState();
- unsigned WriteResID = WS->getWriteResourceID();
- int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
- LLVM_DEBUG(dbgs() << "[E] ReadAdvance for #" << IR << ": " << ReadAdvance
- << '\n');
-
- if (WS->getCyclesLeft() == UNKNOWN_CYCLES) {
- // Try again in the next cycle until the value is known
- StallCycles = std::max(StallCycles, 1U);
- continue;
- }
-
- int CyclesLeft = WS->getCyclesLeft() - ReadAdvance;
- if (CyclesLeft > 0) {
- LLVM_DEBUG(dbgs() << "[E] Register hazard: " << WS->getRegisterID()
- << '\n');
- StallCycles = std::max(StallCycles, (unsigned)CyclesLeft);
- }
- }
- Writes.clear();
-
- for (const WriteRef &WR : CommittedWrites) {
- unsigned WriteResID = WR.getWriteResourceID();
- assert(!WR.getWriteState() && "Should be already committed!");
- assert(WR.hasKnownWriteBackCycle() && "Invalid write!");
- assert(STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID) < 0);
- unsigned ReadAdvance = static_cast<unsigned>(
- -STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID));
- unsigned Elapsed = PRF.getElapsedCyclesFromWriteBack(WR);
- assert(Elapsed < ReadAdvance && "Should not have been added to the set!");
- unsigned CyclesLeft = (ReadAdvance - Elapsed);
- StallCycles = std::max(StallCycles, CyclesLeft);
- }
+ RegisterFile::RAWHazard Hazard = PRF.checkRAWHazards(STI, RS);
+ if (Hazard.isValid())
+ return Hazard.hasUnknownCycles() ? 1U : Hazard.CyclesLeft;
}
- return StallCycles;
+ return 0;
}
-bool InOrderIssueStage::canExecute(const InstRef &IR,
- unsigned *StallCycles) const {
- *StallCycles = 0;
-
- if (unsigned RegStall = checkRegisterHazard(PRF, SM, STI, IR)) {
- *StallCycles = RegStall;
- // FIXME: add a parameter to HWStallEvent to indicate a number of cycles.
- for (unsigned I = 0; I < RegStall; ++I) {
- notifyEvent<HWStallEvent>(
- HWStallEvent(HWStallEvent::RegisterFileStall, IR));
- notifyEvent<HWPressureEvent>(
- HWPressureEvent(HWPressureEvent::REGISTER_DEPS, IR));
- }
- } else if (hasResourceHazard(*RM, IR)) {
- *StallCycles = 1;
- notifyEvent<HWStallEvent>(
- HWStallEvent(HWStallEvent::DispatchGroupStall, IR));
- notifyEvent<HWPressureEvent>(
- HWPressureEvent(HWPressureEvent::RESOURCES, IR));
- } else if (LastWriteBackCycle) {
+bool InOrderIssueStage::canExecute(const InstRef &IR) {
+ assert(!SI.getCyclesLeft() && "Should not have reached this code!");
+ assert(!SI.isValid() && "Should not have reached this code!");
+
+ if (unsigned Cycles = checkRegisterHazard(PRF, STI, IR)) {
+ SI.update(IR, Cycles, StallInfo::StallKind::REGISTER_DEPS);
+ return false;
+ }
+
+ if (hasResourceHazard(*RM, IR)) {
+ SI.update(IR, /* delay */ 1, StallInfo::StallKind::DISPATCH);
+ return false;
+ }
+
+ if (LastWriteBackCycle) {
if (!IR.getInstruction()->getDesc().RetireOOO) {
unsigned NextWriteBackCycle = findFirstWriteBackCycle(IR);
- // Delay the instruction to ensure that writes occur in program order
+ // Delay the instruction to ensure that writes happen in program order.
if (NextWriteBackCycle < LastWriteBackCycle) {
- *StallCycles = LastWriteBackCycle - NextWriteBackCycle;
+ SI.update(IR, LastWriteBackCycle - NextWriteBackCycle,
+ StallInfo::StallKind::DELAY);
+ return false;
}
}
}
- return *StallCycles == 0;
+ return true;
}
static void addRegisterReadWrite(RegisterFile &PRF, Instruction &IS,
PRF.addRegisterWrite(WriteRef(SourceIndex, &WS), UsedRegs);
}
-static void notifyInstructionIssue(
- const InstRef &IR,
- const SmallVectorImpl<std::pair<ResourceRef, ResourceCycles>> &UsedRes,
- const Stage &S) {
-
- S.notifyEvent<HWInstructionEvent>(
+void InOrderIssueStage::notifyInstructionIssued(const InstRef &IR,
+ ArrayRef<ResourceUse> UsedRes) {
+ notifyEvent<HWInstructionEvent>(
HWInstructionEvent(HWInstructionEvent::Ready, IR));
- S.notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, UsedRes));
+ notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, UsedRes));
LLVM_DEBUG(dbgs() << "[E] Issued #" << IR << "\n");
}
-static void notifyInstructionDispatch(const InstRef &IR, unsigned Ops,
- const SmallVectorImpl<unsigned> &UsedRegs,
- const Stage &S) {
-
- S.notifyEvent<HWInstructionEvent>(
+void InOrderIssueStage::notifyInstructionDispatched(
+ const InstRef &IR, unsigned Ops, ArrayRef<unsigned> UsedRegs) {
+ notifyEvent<HWInstructionEvent>(
HWInstructionDispatchedEvent(IR, UsedRegs, Ops));
LLVM_DEBUG(dbgs() << "[E] Dispatched #" << IR << "\n");
}
+void InOrderIssueStage::notifyInstructionExecuted(const InstRef &IR) {
+ notifyEvent<HWInstructionEvent>(
+ HWInstructionEvent(HWInstructionEvent::Executed, IR));
+ LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR << " is executed\n");
+}
+
+void InOrderIssueStage::notifyInstructionRetired(const InstRef &IR,
+ ArrayRef<unsigned> FreedRegs) {
+ notifyEvent<HWInstructionEvent>(HWInstructionRetiredEvent(IR, FreedRegs));
+ LLVM_DEBUG(dbgs() << "[E] Retired #" << IR << " \n");
+}
+
llvm::Error InOrderIssueStage::execute(InstRef &IR) {
- if (llvm::Error E = tryIssue(IR, &StallCyclesLeft))
+ if (llvm::Error E = tryIssue(IR))
return E;
- if (StallCyclesLeft) {
- StalledInst = IR;
- }
+ if (SI.isValid())
+ notifyStallEvent();
return llvm::ErrorSuccess();
}
-llvm::Error InOrderIssueStage::tryIssue(InstRef &IR, unsigned *StallCycles) {
+llvm::Error InOrderIssueStage::tryIssue(InstRef &IR) {
Instruction &IS = *IR.getInstruction();
unsigned SourceIndex = IR.getSourceIndex();
const InstrDesc &Desc = IS.getDesc();
- if (!canExecute(IR, StallCycles)) {
- LLVM_DEBUG(dbgs() << "[E] Stalled #" << IR << " for " << *StallCycles
- << " cycles\n");
+ if (!canExecute(IR)) {
+ LLVM_DEBUG(dbgs() << "[N] Stalled #" << SI.getInstruction() << " for "
+ << SI.getCyclesLeft() << " cycles\n");
Bandwidth = 0;
return llvm::ErrorSuccess();
}
addRegisterReadWrite(PRF, IS, SourceIndex, STI, UsedRegs);
unsigned NumMicroOps = IS.getNumMicroOps();
- notifyInstructionDispatch(IR, NumMicroOps, UsedRegs, *this);
+ notifyInstructionDispatched(IR, NumMicroOps, UsedRegs);
- SmallVector<std::pair<ResourceRef, ResourceCycles>, 4> UsedResources;
+ SmallVector<ResourceUse, 4> UsedResources;
RM->issueInstruction(Desc, UsedResources);
IS.execute(SourceIndex);
// Replace resource masks with valid resource processor IDs.
- for (std::pair<ResourceRef, ResourceCycles> &Use : UsedResources) {
+ for (ResourceUse &Use : UsedResources) {
uint64_t Mask = Use.first.first;
Use.first.first = RM->resolveResourceMask(Mask);
}
- notifyInstructionIssue(IR, UsedResources, *this);
+ notifyInstructionIssued(IR, UsedResources);
bool ShouldCarryOver = NumMicroOps > Bandwidth;
if (ShouldCarryOver) {
IS.cycleEvent();
if (!IS.isExecuted()) {
- LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR
+ LLVM_DEBUG(dbgs() << "[N] Instruction #" << IR
<< " is still executing\n");
++I;
continue;
}
PRF.onInstructionExecuted(&IS);
- notifyEvent<HWInstructionEvent>(
- HWInstructionEvent(HWInstructionEvent::Executed, IR));
- LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR << " is executed\n");
+ notifyInstructionExecuted(IR);
++NumExecuted;
retireInstruction(*I);
if (!CarriedOver)
return;
- assert(!StalledInst && "A stalled instruction cannot be carried over.");
+ assert(!SI.isValid() && "A stalled instruction cannot be carried over.");
if (CarryOver > Bandwidth) {
CarryOver -= Bandwidth;
Bandwidth = 0;
LLVM_DEBUG(dbgs() << "[N] Carry over (" << CarryOver << "uops left) #"
- << CarriedOver << " \n");
+ << CarriedOver << " \n");
return;
}
- LLVM_DEBUG(dbgs() << "[N] Carry over (complete) #" << CarriedOver
- << " \n");
+ LLVM_DEBUG(dbgs() << "[N] Carry over (complete) #" << CarriedOver << " \n");
if (CarriedOver.getInstruction()->getDesc().EndGroup)
Bandwidth = 0;
for (const WriteState &WS : IS.getDefs())
PRF.removeRegisterWrite(WS, FreedRegs);
- notifyEvent<HWInstructionEvent>(HWInstructionRetiredEvent(IR, FreedRegs));
- LLVM_DEBUG(dbgs() << "[E] Retired #" << IR << " \n");
+ notifyInstructionRetired(IR, FreedRegs);
+}
+
+void InOrderIssueStage::notifyStallEvent() {
+ assert(SI.getCyclesLeft() && "A zero cycles stall?");
+ assert(SI.isValid() && "Invalid stall information found!");
+
+ const InstRef &IR = SI.getInstruction();
+
+ switch (SI.getStallKind()) {
+ default:
+ break;
+ case StallInfo::StallKind::REGISTER_DEPS: {
+ notifyEvent<HWStallEvent>(
+ HWStallEvent(HWStallEvent::RegisterFileStall, IR));
+ notifyEvent<HWPressureEvent>(
+ HWPressureEvent(HWPressureEvent::REGISTER_DEPS, IR));
+ break;
+ }
+ case StallInfo::StallKind::DISPATCH: {
+ notifyEvent<HWStallEvent>(
+ HWStallEvent(HWStallEvent::DispatchGroupStall, IR));
+ notifyEvent<HWPressureEvent>(
+ HWPressureEvent(HWPressureEvent::RESOURCES, IR));
+ break;
+ }
+ }
}
llvm::Error InOrderIssueStage::cycleStart() {
updateCarriedOver();
// Issue instructions scheduled for this cycle
- if (!StallCyclesLeft && StalledInst) {
- if (llvm::Error E = tryIssue(StalledInst, &StallCyclesLeft))
- return E;
- }
+ if (SI.isValid()) {
+ if (!SI.getCyclesLeft()) {
+ // Make a copy of the reference, and try issue it again.
+ // Do not take the instruction reference because SI.clear() will
+ // invalidate it.
+ InstRef IR = SI.getInstruction();
+ SI.clear();
+
+ if (llvm::Error E = tryIssue(IR))
+ return E;
+ }
- if (!StallCyclesLeft) {
- StalledInst.invalidate();
- assert(NumIssued <= SM.IssueWidth && "Overflow.");
- } else {
- // The instruction is still stalled, cannot issue any new instructions in
- // this cycle.
- Bandwidth = 0;
+ if (SI.getCyclesLeft()) {
+ // The instruction is still stalled, cannot issue any new instructions in
+ // this cycle.
+ notifyStallEvent();
+ Bandwidth = 0;
+ return llvm::ErrorSuccess();
+ }
}
+ assert(NumIssued <= SM.IssueWidth && "Overflow.");
return llvm::ErrorSuccess();
}
llvm::Error InOrderIssueStage::cycleEnd() {
PRF.cycleEnd();
-
- if (StallCyclesLeft > 0)
- --StallCyclesLeft;
+ SI.cycleEnd();
if (LastWriteBackCycle > 0)
--LastWriteBackCycle;
projects / platform / upstream / llvm.git / commitdiff