//===----------------------------------------------------------------------===//
//
/// \file
-/// This pass adds instructions to enable whole quad mode for pixel
-/// shaders, and whole wavefront mode for all programs.
+/// This pass adds instructions to enable whole quad mode (strict or non-strict)
+/// for pixel shaders, and strict whole wavefront mode for all programs.
+///
+/// The "strict" prefix indicates that inactive lanes do not take part in
+/// control flow, specifically an inactive lane enabled by a strict WQM/WWM will
+/// always be enabled irrespective of control flow decisions. Conversely in
+/// non-strict WQM inactive lanes may control flow decisions.
///
/// Whole quad mode is required for derivative computations, but it interferes
/// with shader side effects (stores and atomics). It ensures that WQM is
/// ...
/// S_MOV_B64 EXEC, Tmp
///
-/// We also compute when a sequence of instructions requires Whole Wavefront
-/// Mode (StrictWWM) and insert instructions to save and restore it:
+/// We also compute when a sequence of instructions requires strict whole
+/// wavefront mode (StrictWWM) and insert instructions to save and restore it:
+///
+/// S_OR_SAVEEXEC_B64 Tmp, -1
+/// ...
+/// S_MOV_B64 EXEC, Tmp
+///
+/// When a sequence of instructions requires strict whole quad mode (StrictWQM)
+/// we use a similar save and restore mechanism and force whole quad mode for
+/// those instructions:
///
-/// S_OR_SAVEEXEC_B64 Tmp, -1
-/// ...
-/// S_MOV_B64 EXEC, Tmp
+/// S_MOV_B64 Tmp, EXEC
+/// S_WQM_B64 EXEC, EXEC
+/// ...
+/// S_MOV_B64 EXEC, Tmp
///
/// In order to avoid excessive switching during sequences of Exact
/// instructions, the pass first analyzes which instructions must be run in WQM
enum {
StateWQM = 0x1,
StateStrictWWM = 0x2,
- StateExact = 0x4,
+ StateStrictWQM = 0x4,
+ StateExact = 0x8,
+ StateStrict = StateStrictWWM | StateStrictWQM,
};
struct PrintState {
#ifndef NDEBUG
static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
- if (PS.State & StateWQM)
- OS << "WQM";
- if (PS.State & StateStrictWWM) {
- if (PS.State & StateWQM)
- OS << '|';
- OS << "StrictWWM";
- }
- if (PS.State & StateExact) {
- if (PS.State & (StateWQM | StateStrictWWM))
- OS << '|';
- OS << "Exact";
- }
+ static const std::pair<char, const char *> Mapping[] = {
+ std::make_pair(StateWQM, "WQM"),
+ std::make_pair(StateStrictWWM, "StrictWWM"),
+ std::make_pair(StateStrictWQM, "StrictWQM"),
+ std::make_pair(StateExact, "Exact")};
+ char State = PS.State;
+ for (auto M : Mapping) {
+ if (State & M.first) {
+ OS << M.second;
+ State &= ~M.first;
+
+ if (State)
+ OS << '|';
+ }
+ }
+ assert(State == 0);
return OS;
}
#endif
DenseMap<const MachineInstr *, InstrInfo> Instructions;
MapVector<MachineBasicBlock *, BlockInfo> Blocks;
- // Tracks state (WQM/StrictWWM/Exact) after a given instruction
+ // Tracks state (WQM/StrictWWM/StrictWQM/Exact) after a given instruction
DenseMap<const MachineInstr *, char> StateTransition;
SmallVector<MachineInstr *, 2> LiveMaskQueries;
Register SaveWQM);
void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
Register SavedWQM);
- void toStrictWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
- Register SaveOrig);
- void fromStrictWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
- Register SavedOrig, char NonStrictWWMState);
+ void toStrictMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
+ Register SaveOrig, char StrictStateNeeded);
+ void fromStrictMode(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator Before, Register SavedOrig,
+ char NonStrictState, char CurrentStrictState);
MachineBasicBlock *splitBlock(MachineBasicBlock *BB, MachineInstr *TermMI);
GlobalFlags |= StateStrictWWM;
LowerToMovInstrs.push_back(&MI);
continue;
+ } else if (Opcode == AMDGPU::STRICT_WQM) {
+ // STRICT_WQM is similar to STRICTWWM, but instead of enabling all
+ // threads of the wave like STRICTWWM, STRICT_WQM enables all threads in
+ // quads that have at least one active thread.
+ markInstructionUses(MI, StateStrictWQM, Worklist);
+ GlobalFlags |= StateStrictWQM;
+ LowerToMovInstrs.push_back(&MI);
+ continue;
} else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 ||
Opcode == AMDGPU::V_SET_INACTIVE_B64) {
- III.Disabled = StateStrictWWM;
+ III.Disabled = StateStrict;
MachineOperand &Inactive = MI.getOperand(2);
if (Inactive.isReg()) {
if (Inactive.isUndef()) {
Worklist.push_back(&MBB);
}
GlobalFlags |= StateExact;
- III.Disabled = StateWQM | StateStrictWWM;
+ III.Disabled = StateWQM | StateStrict;
continue;
} else {
if (Opcode == AMDGPU::SI_PS_LIVE || Opcode == AMDGPU::SI_LIVE_MASK) {
// Propagate backwards within block
if (MachineInstr *PrevMI = MI.getPrevNode()) {
- char InNeeds = (II.Needs & ~StateStrictWWM) | II.OutNeeds;
+ char InNeeds = (II.Needs & ~StateStrict) | II.OutNeeds;
if (!PrevMI->isPHI()) {
InstrInfo &PrevII = Instructions[PrevMI];
if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
if (II.Needs != 0)
markInstructionUses(MI, II.Needs, Worklist);
- // Ensure we process a block containing StrictWWM, even if it does not require
- // any WQM transitions.
+ // Ensure we process a block containing StrictWWM/StrictWQM, even if it does
+ // not require any WQM transitions.
if (II.Needs & StateStrictWWM)
BI.Needs |= StateStrictWWM;
+ if (II.Needs & StateStrictWQM)
+ BI.Needs |= StateStrictWQM;
}
void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
StateTransition[MI] = StateWQM;
}
-void SIWholeQuadMode::toStrictWWM(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator Before,
- Register SaveOrig) {
+void SIWholeQuadMode::toStrictMode(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator Before,
+ Register SaveOrig, char StrictStateNeeded) {
MachineInstr *MI;
-
assert(SaveOrig);
- MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICT_WWM),
- SaveOrig)
- .addImm(-1);
+ assert(StrictStateNeeded == StateStrictWWM ||
+ StrictStateNeeded == StateStrictWQM);
+
+ if (StrictStateNeeded == StateStrictWWM) {
+ MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICT_WWM),
+ SaveOrig)
+ .addImm(-1);
+ } else {
+ MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICT_WQM),
+ SaveOrig)
+ .addImm(-1);
+ }
LIS->InsertMachineInstrInMaps(*MI);
StateTransition[MI] = StateStrictWWM;
}
-void SIWholeQuadMode::fromStrictWWM(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator Before,
- Register SavedOrig,
- char NonStrictWWMState) {
+void SIWholeQuadMode::fromStrictMode(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator Before,
+ Register SavedOrig, char NonStrictState,
+ char CurrentStrictState) {
MachineInstr *MI;
assert(SavedOrig);
- MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICT_WWM), Exec)
- .addReg(SavedOrig);
+ assert(CurrentStrictState == StateStrictWWM ||
+ CurrentStrictState == StateStrictWQM);
+
+ if (CurrentStrictState == StateStrictWWM) {
+ MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICT_WWM),
+ Exec)
+ .addReg(SavedOrig);
+ } else {
+ MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICT_WQM),
+ Exec)
+ .addReg(SavedOrig);
+ }
LIS->InsertMachineInstrInMaps(*MI);
- StateTransition[MI] = NonStrictWWMState;
+ StateTransition[MI] = NonStrictState;
}
void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
<< ":\n");
Register SavedWQMReg;
- Register SavedNonStrictWWMReg;
+ Register SavedNonStrictReg;
bool WQMFromExec = IsEntry;
char State = (IsEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
- char NonStrictWWMState = 0;
+ char NonStrictState = 0;
const TargetRegisterClass *BoolRC = TRI->getBoolRC();
auto II = MBB.getFirstNonPHI(), IE = MBB.end();
// Exact or vice versa.
MachineBasicBlock::iterator FirstWQM = IE;
- // This stores the first instruction where it's safe to switch from StrictWWM
- // to Exact/WQM or to switch to StrictWWM. It must always be the same as, or
- // after, FirstWQM since if it's safe to switch to/from StrictWWM, it must be
- // safe to switch to/from WQM as well.
- MachineBasicBlock::iterator FirstStrictWWM = IE;
+ // This stores the first instruction where it's safe to switch from Strict
+ // mode to Exact/WQM or to switch to Strict mode. It must always be the same
+ // as, or after, FirstWQM since if it's safe to switch to/from Strict, it must
+ // be safe to switch to/from WQM as well.
+ MachineBasicBlock::iterator FirstStrict = IE;
// Record initial state is block information.
BI.InitialState = State;
for (;;) {
MachineBasicBlock::iterator Next = II;
- char Needs = StateExact | StateWQM; // StrictWWM is disabled by default
+ char Needs = StateExact | StateWQM; // Strict mode is disabled by default.
char OutNeeds = 0;
if (FirstWQM == IE)
FirstWQM = II;
- if (FirstStrictWWM == IE)
- FirstStrictWWM = II;
+ if (FirstStrict == IE)
+ FirstStrict = II;
// First, figure out the allowed states (Needs) based on the propagated
// flags.
if (III != Instructions.end()) {
if (III->second.Needs & StateStrictWWM)
Needs = StateStrictWWM;
+ else if (III->second.Needs & StateStrictWQM)
+ Needs = StateStrictWQM;
else if (III->second.Needs & StateWQM)
Needs = StateWQM;
else
}
} else {
// If the instruction doesn't actually need a correct EXEC, then we can
- // safely leave StrictWWM enabled.
- Needs = StateExact | StateWQM | StateStrictWWM;
+ // safely leave Strict mode enabled.
+ Needs = StateExact | StateWQM | StateStrict;
}
if (MI.isTerminator() && OutNeeds == StateExact)
// Now, transition if necessary.
if (!(Needs & State)) {
MachineBasicBlock::iterator First;
- if (State == StateStrictWWM || Needs == StateStrictWWM) {
- // We must switch to or from StrictWWM
- First = FirstStrictWWM;
+ if (State == StateStrictWWM || Needs == StateStrictWWM ||
+ State == StateStrictWQM || Needs == StateStrictWQM) {
+ // We must switch to or from Strict mode.
+ First = FirstStrict;
} else {
- // We only need to switch to/from WQM, so we can use FirstWQM
+ // We only need to switch to/from WQM, so we can use FirstWQM.
First = FirstWQM;
}
- // Whether we need to save SCC depends on start and end states
+ // Whether we need to save SCC depends on start and end states.
bool SaveSCC = false;
switch (State) {
case StateExact:
case StateStrictWWM:
- // Exact/WWM -> WWM: save SCC
- // Exact/WWM -> WQM: save SCC if WQM mask is generated from exec
- // Exact/WWM -> Exact: no save
- SaveSCC =
- (Needs & StateStrictWWM) || ((Needs & StateWQM) && WQMFromExec);
+ case StateStrictWQM:
+ // Exact/Strict -> Strict: save SCC
+ // Exact/Strict -> WQM: save SCC if WQM mask is generated from exec
+ // Exact/Strict -> Exact: no save
+ SaveSCC = (Needs & StateStrict) || ((Needs & StateWQM) && WQMFromExec);
break;
case StateWQM:
- // WQM -> Exact/WMM: save SCC
+ // WQM -> Exact/Strict: save SCC
SaveSCC = !(Needs & StateWQM);
break;
default:
MachineBasicBlock::iterator Before =
prepareInsertion(MBB, First, II, Needs == StateWQM, SaveSCC);
- if (State == StateStrictWWM) {
- assert(SavedNonStrictWWMReg);
- fromStrictWWM(MBB, Before, SavedNonStrictWWMReg, NonStrictWWMState);
- LIS->createAndComputeVirtRegInterval(SavedNonStrictWWMReg);
- SavedNonStrictWWMReg = 0;
- State = NonStrictWWMState;
+ if (State & StateStrict) {
+ assert(State == StateStrictWWM || State == StateStrictWQM);
+ assert(SavedNonStrictReg);
+ fromStrictMode(MBB, Before, SavedNonStrictReg, NonStrictState, State);
+
+ LIS->createAndComputeVirtRegInterval(SavedNonStrictReg);
+ SavedNonStrictReg = 0;
+ State = NonStrictState;
}
- if (Needs == StateStrictWWM) {
- NonStrictWWMState = State;
- assert(!SavedNonStrictWWMReg);
- SavedNonStrictWWMReg = MRI->createVirtualRegister(BoolRC);
- toStrictWWM(MBB, Before, SavedNonStrictWWMReg);
- State = StateStrictWWM;
+ if (Needs & StateStrict) {
+ NonStrictState = State;
+ assert(Needs == StateStrictWWM || Needs == StateStrictWQM);
+ assert(!SavedNonStrictReg);
+ SavedNonStrictReg = MRI->createVirtualRegister(BoolRC);
+
+ toStrictMode(MBB, Before, SavedNonStrictReg, Needs);
+ State = Needs;
+
} else {
if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
if (!WQMFromExec && (OutNeeds & StateWQM)) {
}
}
- if (Needs != (StateExact | StateWQM | StateStrictWWM)) {
+ if (Needs != (StateExact | StateWQM | StateStrict)) {
if (Needs != (StateExact | StateWQM))
FirstWQM = IE;
- FirstStrictWWM = IE;
+ FirstStrict = IE;
}
if (II == IE)
II = Next;
}
assert(!SavedWQMReg);
- assert(!SavedNonStrictWWMReg);
+ assert(!SavedNonStrictReg);
}
void SIWholeQuadMode::lowerLiveMaskQueries() {
}
bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
+ LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName()
+ << " ------------- \n");
+ LLVM_DEBUG(MF.dump(););
+
Instructions.clear();
Blocks.clear();
LiveMaskQueries.clear();
LiveMaskReg = Exec;
- // Shader is simple does not need WQM/StrictWWM or any complex lowering
- if (!(GlobalFlags & (StateWQM | StateStrictWWM)) &&
- LowerToCopyInstrs.empty() && LowerToMovInstrs.empty() &&
- KillInstrs.empty()) {
+ // Shader is simple does not need any state changes or any complex lowering
+ if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() &&
+ LowerToMovInstrs.empty() && KillInstrs.empty()) {
lowerLiveMaskQueries();
return !LiveMaskQueries.empty();
}
; Check that we don't leave WWM on for computations that don't require WWM,
; since that will lead clobbering things that aren't supposed to be clobbered
; in cases like this.
+; We enforce this by checking that v_add gets emitted in the same block as
+; WWM computations.
;
;CHECK-LABEL: {{^}}test_wwm3:
+;CHECK: %if
;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
;CHECK: buffer_load_dword
;CHECK: v_add_f32_e32
;CHECK: s_mov_b64 exec, [[ORIG]]
;CHECK: v_add_f32_e32
+;CHECK: %endif
define amdgpu_ps float @test_wwm3(i32 inreg %idx) {
main_body:
; use mbcnt to make sure the branch is divergent
; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM
; write could clobber disabled channels in the non-WWM one.
+; We enforce this by checking that v_mov gets emitted in the same block as
+; WWM computations.
;
;CHECK-LABEL: {{^}}test_wwm4:
+;CHECK: %if
;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
;CHECK: buffer_load_dword
;CHECK: v_add_f32_e32
;CHECK: s_mov_b64 exec, [[ORIG]]
;CHECK-NEXT: v_mov_b32_e32
+;CHECK: %endif
define amdgpu_ps float @test_wwm4(i32 inreg %idx) {
main_body:
; use mbcnt to make sure the branch is divergent
;VI-CHECK: flat_load_dword
;CHECK: v_add_f32_e32
;CHECK: s_mov_b64 exec, [[ORIG2]]
+;CHECK: %endif
define amdgpu_ps float @test_wwm6_then() {
main_body:
%src0 = load volatile float, float addrspace(1)* undef
;SI-CHECK: buffer_load_dword
;VI-CHECK: flat_load_dword
;CHECK: s_mov_b64 exec, [[ORIG2]]
+;CHECK: %endloop
define amdgpu_ps float @test_wwm6_loop() {
main_body:
%src0 = load volatile float, float addrspace(1)* undef
ret void
}
+; Check that Strict WQM is triggered by the strict_wqm intrinsic.
+;
+;CHECK-LABEL: {{^}}test_strict_wqm1:
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test_strict_wqm1(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+ %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+ %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0)
+ %out = fadd float %src0, %src1
+ %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)
+ ret float %out.0
+}
+
+; Same as above, but with an integer type.
+;
+;CHECK-LABEL: {{^}}test_strict_wqm2:
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_{{[iu]}}32_e32
+define amdgpu_ps float @test_strict_wqm2(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+ %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+ %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0)
+ %src0.0 = bitcast float %src0 to i32
+ %src1.0 = bitcast float %src1 to i32
+ %out = add i32 %src0.0, %src1.0
+ %out.0 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %out)
+ %out.1 = bitcast i32 %out.0 to float
+ ret float %out.1
+}
+
+; Check that we don't leave Strict WQM on for computations that don't require it,
+; since that will lead clobbering things that aren't supposed to be clobbered
+; in cases like this.
+; We enforce this by checking that v_add gets emitted in the same block as
+; WWM computations.
+;
+;CHECK-LABEL: {{^}}test_strict_wqm3:
+;CHECK: %if
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: v_add_f32_e32
+;CHECK: %endif
+define amdgpu_ps float @test_strict_wqm3(i32 inreg %idx) {
+main_body:
+ ; use mbcnt to make sure the branch is divergent
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %cc = icmp uge i32 %hi, 32
+ br i1 %cc, label %endif, label %if
+
+if:
+ %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0)
+ %out = fadd float %src, %src
+ %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)
+ %out.1 = fadd float %src, %out.0
+ br label %endif
+
+endif:
+ %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ]
+ ret float %out.2
+}
+
+; Check that Strict WQM writes aren't coalesced with non-strict writes, since
+; the Strict WQM write could clobber disabled channels in the non-strict one.
+; We enforce this by checking that v_mov gets emitted in the same block as
+; WWM computations.
+;
+;CHECK-LABEL: {{^}}test_strict_wqm4:
+;CHECK: %if
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK-NEXT: v_mov_b32_e32
+;CHECK: %endif
+define amdgpu_ps float @test_strict_wqm4(i32 inreg %idx) {
+main_body:
+ ; use mbcnt to make sure the branch is divergent
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %cc = icmp uge i32 %hi, 32
+ br i1 %cc, label %endif, label %if
+
+if:
+ %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0)
+ %out = fadd float %src, %src
+ %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)
+ br label %endif
+
+endif:
+ %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
+ ret float %out.1
+}
+
+; Make sure the transition from Exact to Strict WQM then WQM works properly.
+;
+;CHECK-LABEL: {{^}}test_strict_wqm5:
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: buffer_store_dword
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: s_wqm_b64 exec, exec
+define amdgpu_ps float @test_strict_wqm5(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+ %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+ %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0)
+ %temp = fadd float %src1, %src1
+ %temp.0 = call float @llvm.amdgcn.strict.wqm.f32(float %temp)
+ %out = fadd float %temp.0, %temp.0
+ %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)
+ ret float %out.0
+}
+
+; Check that Strict WQM is turned on correctly across basic block boundaries.
+; if..then..endif version
+;
+;CHECK-LABEL: {{^}}test_strict_wqm6_then:
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: %if
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+;CHECK: %endif
+define amdgpu_ps float @test_strict_wqm6_then() {
+main_body:
+ %src0 = load volatile float, float addrspace(1)* undef
+ ; use mbcnt to make sure the branch is divergent
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %cc = icmp uge i32 %hi, 32
+ br i1 %cc, label %endif, label %if
+
+if:
+ %src1 = load volatile float, float addrspace(1)* undef
+ %out = fadd float %src0, %src1
+ %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)
+ br label %endif
+
+endif:
+ %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
+ ret float %out.1
+}
+
+; Check that Strict WQM is turned on correctly across basic block boundaries.
+; loop version
+;
+;CHECK-LABEL: {{^}}test_strict_wqm6_loop:
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: %loop
+;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+;CHECK: %endloop
+define amdgpu_ps float @test_strict_wqm6_loop() {
+main_body:
+ %src0 = load volatile float, float addrspace(1)* undef
+ ; use mbcnt to make sure the branch is divergent
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ br label %loop
+
+loop:
+ %counter = phi i32 [ %lo, %main_body ], [ %counter.1, %loop ]
+ %src1 = load volatile float, float addrspace(1)* undef
+ %out = fadd float %src0, %src1
+ %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)
+ %counter.1 = sub i32 %counter, 1
+ %cc = icmp ne i32 %counter.1, 0
+ br i1 %cc, label %loop, label %endloop
+
+endloop:
+ ret float %out.0
+}
+
; Check that enabling WQM anywhere enables WQM for the set.inactive source.
;
;CHECK-LABEL: {{^}}test_set_inactive2:
; Check that we don't leave WWM on for computations that don't require WWM,
; since that will lead clobbering things that aren't supposed to be clobbered
; in cases like this.
+; We enforce this by checking that v_add gets emitted in the same block as
+; WWM computations.
;
;CHECK-LABEL: {{^}}test_strict_wwm3:
+;CHECK: %if
;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
;CHECK: buffer_load_dword
;CHECK: v_add_f32_e32
;CHECK: s_mov_b64 exec, [[ORIG]]
;CHECK: v_add_f32_e32
+;CHECK: %endif
define amdgpu_ps float @test_strict_wwm3(i32 inreg %idx) {
main_body:
; use mbcnt to make sure the branch is divergent
; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM
; write could clobber disabled channels in the non-WWM one.
+; We enforce this by checking that v_mov gets emitted in the same block as
+; WWM computations.
;
;CHECK-LABEL: {{^}}test_strict_wwm4:
+;CHECK: %if
;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
;CHECK: buffer_load_dword
;CHECK: v_add_f32_e32
;CHECK: s_mov_b64 exec, [[ORIG]]
;CHECK-NEXT: v_mov_b32_e32
+;CHECK: %endif
define amdgpu_ps float @test_strict_wwm4(i32 inreg %idx) {
main_body:
; use mbcnt to make sure the branch is divergent
;VI-CHECK: flat_load_dword
;CHECK: v_add_f32_e32
;CHECK: s_mov_b64 exec, [[ORIG2]]
+;CHECK: %endif
define amdgpu_ps float @test_strict_wwm6_then() {
main_body:
%src0 = load volatile float, float addrspace(1)* undef
;SI-CHECK: buffer_load_dword
;VI-CHECK: flat_load_dword
;CHECK: s_mov_b64 exec, [[ORIG2]]
+;CHECK: %endloop
define amdgpu_ps float @test_strict_wwm6_loop() {
main_body:
%src0 = load volatile float, float addrspace(1)* undef
ret float %r
}
+; Check a case of a block being entirely WQM except for a bit of STRICT WQM.
+;
+;CHECK-LABEL: {{^}}test_strict_wqm_within_wqm:
+;CHECK: %IF
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: ds_swizzle
+;
+define amdgpu_ps float @test_strict_wqm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {
+main_body:
+ %c.bc = bitcast i32 %c to float
+ %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0
+ %tex0 = extractelement <4 x float> %tex, i32 0
+ %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0
+ %cmp = icmp eq i32 %z, 0
+ br i1 %cmp, label %IF, label %ENDIF
+IF:
+ %dataf = extractelement <4 x float> %dtex, i32 0
+ %data1 = fptosi float %dataf to i32
+ %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data1, i32 2079)
+ %data3 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %data2)
+ %data3f = sitofp i32 %data3 to float
+ br label %ENDIF
+
+ENDIF:
+ %r = phi float [ %data3f, %IF ], [ 0.0, %main_body ]
+ ret float %r
+}
+
+;CHECK-LABEL: {{^}}test_strict_wqm_strict_wwm_wqm:
+;CHECK: buffer_store_dword
+
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+
+;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+
+;CHECK: s_mov_b64 [[ORIG3:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: v_add
+;CHECK: s_mov_b64 exec, [[ORIG3]]
+
+;TODO: StrictWQM -> WQM transition could be improved. WQM could use the exec from the previous state instead of calling s_wqm again.
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: image_sample
+
+define amdgpu_ps float @test_strict_wqm_strict_wwm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, <4 x i32> inreg %res2, float %inp, <8 x i32> inreg %res3) {
+main_body:
+ call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+ %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+ %temp = fadd float %reload, %reload
+ %temp2 = call float @llvm.amdgcn.strict.wqm.f32(float %temp)
+ %temp3 = fadd float %temp2, %temp2
+ %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res2, i32 %idx0, i32 0, i32 0, i32 0)
+ %temp4 = call float @llvm.amdgcn.strict.wwm.f32(float %reload_wwm)
+ %temp5 = fadd float %temp3, %temp4
+ %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res3, <4 x i32> %res, i1 false, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+ %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+ ret float %out
+}
+
+;CHECK-LABEL: {{^}}test_strict_wwm_strict_wqm_wqm:
+;CHECK: buffer_store_dword
+
+;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+
+;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+
+;CHECK: s_or_saveexec_b64 [[ORIG3:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: v_add
+;CHECK: s_mov_b64 exec, [[ORIG3]]
+
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: image_sample
+define amdgpu_ps float @test_strict_wwm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) {
+main_body:
+ call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+ %temp = fadd float %reload, %reload
+ %temp2 = call float @llvm.amdgcn.strict.wwm.f32(float %temp)
+ %temp3 = fadd float %temp2, %temp2
+ %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ %temp4 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm)
+ %temp5 = fadd float %temp3, %temp4
+ %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ ret float %out
+}
+
+;CHECK-LABEL: {{^}}test_wqm_strict_wqm_wqm:
+;CHECK: buffer_store_dword
+
+;CHECK: s_wqm_b64 exec, exec
+
+;TODO: WQM -> StrictWQM transition could be improved. StrictWQM could use the exec from the previous state instead of calling s_wqm again.
+;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+
+;CHECK: image_sample
+
+define amdgpu_ps float @test_wqm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) {
+main_body:
+ call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+ %temp = fadd float %reload, %reload
+ %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0)
+ %temp2 = fadd float %tex, %tex
+ %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ %temp3 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm)
+ %temp4 = fadd float %temp2, %temp3
+ %tex2 = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp4, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.buffer.store.f32(float %tex2, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+ ret float %out
+}
declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1
declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1
declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #3
declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
+declare float @llvm.amdgcn.image.sample.1d.f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
declare void @llvm.amdgcn.kill(i1) #1
declare float @llvm.amdgcn.wqm.f32(float) #3
declare i32 @llvm.amdgcn.wqm.i32(i32) #3
declare i32 @llvm.amdgcn.strict.wwm.i32(i32) #3
declare float @llvm.amdgcn.wwm.f32(float) #3
declare i32 @llvm.amdgcn.wwm.i32(i32) #3
+declare float @llvm.amdgcn.strict.wqm.f32(float) #3
+declare i32 @llvm.amdgcn.strict.wqm.i32(i32) #3
declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4
declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3
projects / platform / upstream / llvm.git / commitdiff