ST.getMaxNumSGPRs(MF) / 4);
}
+static ArrayRef<MCPhysReg> getAllSGPRs(const GCNSubtarget &ST,
+ const MachineFunction &MF) {
+ return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), ST.getMaxNumSGPRs(MF));
+}
+
// Find a scratch register that we can use at the start of the prologue to
// re-align the stack pointer. We avoid using callee-save registers since they
// may appear to be free when this is called from canUseAsPrologue (during
// Shift down registers reserved for the scratch RSRC.
Register SIFrameLowering::getEntryFunctionReservedScratchRsrcReg(
- MachineFunction &MF, Register ScratchWaveOffsetReg) const {
+ MachineFunction &MF) const {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo();
Register ScratchRsrcReg = MFI->getScratchRSrcReg();
- if (ScratchRsrcReg == AMDGPU::NoRegister ||
- !MRI.isPhysRegUsed(ScratchRsrcReg))
- return AMDGPU::NoRegister;
+ if (!ScratchRsrcReg || !MRI.isPhysRegUsed(ScratchRsrcReg))
+ return Register();
if (ST.hasSGPRInitBug() ||
ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
// Skip the last N reserved elements because they should have already been
// reserved for VCC etc.
+ Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
for (MCPhysReg Reg : AllSGPR128s) {
// Pick the first unallocated one. Make sure we don't clobber the other
- // reserved input we needed.
- //
- // FIXME: The preloaded SGPR count is not accurate for shaders as the
- // scratch wave offset may be in a fixed SGPR or
- // SITargetLowering::allocateSystemSGPRs may choose some free SGPR for the
- // scratch wave offset. We explicitly avoid the scratch wave offset to
- // account for this.
+ // reserved input we needed. Also for PAL, make sure we don't clobber
+ // the GIT pointer passed in SGPR0 or SGPR8.
if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg) &&
- !TRI->isSubRegisterEq(Reg, ScratchWaveOffsetReg)) {
+ !TRI->isSubRegisterEq(Reg, GITPtrLoReg)) {
MRI.replaceRegWith(ScratchRsrcReg, Reg);
MFI->setScratchRSrcReg(Reg);
return Reg;
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo();
+ const SIRegisterInfo *TRI = &TII->getRegisterInfo();
MachineRegisterInfo &MRI = MF.getRegInfo();
const Function &F = MF.getFunction();
assert(MFI->isEntryFunction());
- Register ScratchWaveOffsetReg = MFI->getPreloadedReg(
+ Register PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET);
// FIXME: Hack to not crash in situations which emitted an error.
- if (ScratchWaveOffsetReg == AMDGPU::NoRegister)
+ if (!PreloadedScratchWaveOffsetReg)
return;
// We need to do the replacement of the private segment buffer register even
// if there are no stack objects. There could be stores to undef or a
// constant without an associated object.
//
- // This will return `AMDGPU::NoRegister` in cases where there are no actual
+ // This will return `Register()` in cases where there are no actual
// uses of the SRSRC.
- Register ScratchRsrcReg =
- getEntryFunctionReservedScratchRsrcReg(MF, ScratchWaveOffsetReg);
+ Register ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
// Make the selected register live throughout the function.
- if (ScratchRsrcReg != AMDGPU::NoRegister) {
+ if (ScratchRsrcReg) {
for (MachineBasicBlock &OtherBB : MF) {
if (&OtherBB != &MBB) {
OtherBB.addLiveIn(ScratchRsrcReg);
// Now that we have fixed the reserved SRSRC we need to locate the
// (potentially) preloaded SRSRC.
- Register PreloadedScratchRsrcReg = AMDGPU::NoRegister;
+ Register PreloadedScratchRsrcReg;
if (ST.isAmdHsaOrMesa(F)) {
PreloadedScratchRsrcReg =
MFI->getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
- if (ScratchRsrcReg != AMDGPU::NoRegister &&
- PreloadedScratchRsrcReg != AMDGPU::NoRegister) {
+ if (ScratchRsrcReg && PreloadedScratchRsrcReg) {
// We added live-ins during argument lowering, but since they were not
// used they were deleted. We're adding the uses now, so add them back.
MRI.addLiveIn(PreloadedScratchRsrcReg);
DebugLoc DL;
MachineBasicBlock::iterator I = MBB.begin();
+ // We found the SRSRC first because it needs four registers and has an
+ // alignment requirement. If the SRSRC that we found is clobbering with
+ // the scratch wave offset, which may be in a fixed SGPR or a free SGPR
+ // chosen by SITargetLowering::allocateSystemSGPRs, COPY the scratch
+ // wave offset to a free SGPR.
+ Register ScratchWaveOffsetReg;
+ if (TRI->isSubRegisterEq(ScratchRsrcReg, PreloadedScratchWaveOffsetReg)) {
+ ArrayRef<MCPhysReg> AllSGPRs = getAllSGPRs(ST, MF);
+ unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
+ AllSGPRs = AllSGPRs.slice(
+ std::min(static_cast<unsigned>(AllSGPRs.size()), NumPreloaded));
+ Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
+ for (MCPhysReg Reg : AllSGPRs) {
+ if (!MRI.isPhysRegUsed(Reg) && MRI.isAllocatable(Reg) &&
+ !TRI->isSubRegisterEq(ScratchRsrcReg, Reg) && GITPtrLoReg != Reg) {
+ ScratchWaveOffsetReg = Reg;
+ BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg)
+ .addReg(PreloadedScratchWaveOffsetReg, RegState::Kill);
+ break;
+ }
+ }
+ } else {
+ ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
+ }
+ assert(ScratchWaveOffsetReg);
+
if (MF.getFrameInfo().hasCalls()) {
Register SPReg = MFI->getStackPtrOffsetReg();
assert(SPReg != AMDGPU::SP_REG);
BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), FPReg).addImm(0);
}
- if (MFI->hasFlatScratchInit() || ScratchRsrcReg != AMDGPU::NoRegister) {
- MRI.addLiveIn(ScratchWaveOffsetReg);
- MBB.addLiveIn(ScratchWaveOffsetReg);
+ if (MFI->hasFlatScratchInit() || ScratchRsrcReg) {
+ MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
+ MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
}
if (MFI->hasFlatScratchInit()) {
emitEntryFunctionFlatScratchInit(MF, MBB, I, DL, ScratchWaveOffsetReg);
}
- if (ScratchRsrcReg != AMDGPU::NoRegister) {
+ if (ScratchRsrcReg) {
emitEntryFunctionScratchRsrcRegSetup(MF, MBB, I, DL,
PreloadedScratchRsrcReg,
ScratchRsrcReg, ScratchWaveOffsetReg);
const MCInstrDesc &GetPC64 = TII->get(AMDGPU::S_GETPC_B64);
BuildMI(MBB, I, DL, GetPC64, Rsrc01);
}
- auto GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
- if (ST.hasMergedShaders()) {
- switch (MF.getFunction().getCallingConv()) {
- case CallingConv::AMDGPU_HS:
- case CallingConv::AMDGPU_GS:
- // Low GIT address is passed in s8 rather than s0 for an LS+HS or
- // ES+GS merged shader on gfx9+.
- GitPtrLo = AMDGPU::SGPR8;
- break;
- default:
- break;
- }
- }
+ Register GitPtrLo = MFI->getGITPtrLoReg(MF);
MF.getRegInfo().addLiveIn(GitPtrLo);
MBB.addLiveIn(GitPtrLo);
BuildMI(MBB, I, DL, SMovB32, RsrcLo)
.addImm(0) // dlc
.addReg(ScratchRsrcReg, RegState::ImplicitDefine)
.addMemOperand(MMO);
- } else if (ST.isMesaGfxShader(Fn) ||
- (PreloadedScratchRsrcReg == AMDGPU::NoRegister)) {
+ } else if (ST.isMesaGfxShader(Fn) || !PreloadedScratchRsrcReg) {
assert(!ST.isAmdHsaOrMesa(Fn));
const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
.addImm(Rsrc23 >> 32)
.addReg(ScratchRsrcReg, RegState::ImplicitDefine);
} else if (ST.isAmdHsaOrMesa(Fn)) {
- assert(PreloadedScratchRsrcReg != AMDGPU::NoRegister);
+ assert(PreloadedScratchRsrcReg);
if (ScratchRsrcReg != PreloadedScratchRsrcReg) {
BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
Reg.FI.getValue());
}
- if (ScratchExecCopy != AMDGPU::NoRegister) {
+ if (ScratchExecCopy) {
// FIXME: Split block and make terminator.
unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
LiveRegs.addReg(ScratchExecCopy);
}
-
if (FuncInfo->FramePointerSaveIndex) {
const int FI = FuncInfo->FramePointerSaveIndex.getValue();
assert(!MFI.isDeadObjectIndex(FI) &&
Register ScratchSPReg = findScratchNonCalleeSaveRegister(
MRI, LiveRegs, AMDGPU::SReg_32_XM0RegClass);
- assert(ScratchSPReg != AMDGPU::NoRegister &&
- ScratchSPReg != FuncInfo->SGPRForFPSaveRestoreCopy);
+ assert(ScratchSPReg && ScratchSPReg != FuncInfo->SGPRForFPSaveRestoreCopy);
// s_add_u32 tmp_reg, s32, NumBytes
// s_and_b32 s32, tmp_reg, 0b111...0000
continue;
const SIRegisterInfo &TRI = TII->getRegisterInfo();
- if (ScratchExecCopy == AMDGPU::NoRegister) {
+ if (!ScratchExecCopy) {
// See emitPrologue
if (LiveRegs.empty()) {
LiveRegs.init(*ST.getRegisterInfo());
FuncInfo->getStackPtrOffsetReg(), Reg.FI.getValue());
}
- if (ScratchExecCopy != AMDGPU::NoRegister) {
+ if (ScratchExecCopy) {
// FIXME: Split block and make terminator.
unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
for (auto &CS : CSI) {
if (CS.getReg() == FuncInfo->getFrameOffsetReg()) {
- if (FuncInfo->SGPRForFPSaveRestoreCopy != AMDGPU::NoRegister)
+ if (FuncInfo->SGPRForFPSaveRestoreCopy)
CS.setDstReg(FuncInfo->SGPRForFPSaveRestoreCopy);
break;
}
;
; GCN-LABEL: {{^}}ps_main:
-; GCN-DAG: s_mov_b32 s4, SCRATCH_RSRC_DWORD0
-; GCN-DAG: s_mov_b32 s5, SCRATCH_RSRC_DWORD1
-; GCN-DAG: s_mov_b32 s6, -1
-; SI-DAG: s_mov_b32 s7, 0xe8f000
-; VI-DAG: s_mov_b32 s7, 0xe80000
-; GFX9-DAG: s_mov_b32 s7, 0xe00000
-; GFX10_W32-DAG: s_mov_b32 s7, 0x31c16000
-; GFX10_W64-DAG: s_mov_b32 s7, 0x31e16000
+; GCN-DAG: s_mov_b32 s0, SCRATCH_RSRC_DWORD0
+; GCN-DAG: s_mov_b32 s1, SCRATCH_RSRC_DWORD1
+; GCN-DAG: s_mov_b32 s2, -1
+; SI-DAG: s_mov_b32 s3, 0xe8f000
+; VI-DAG: s_mov_b32 s3, 0xe80000
+; GFX9-DAG: s_mov_b32 s3, 0xe00000
+; GFX10_W32-DAG: s_mov_b32 s3, 0x31c16000
+; GFX10_W64-DAG: s_mov_b32 s3, 0x31e16000
; GCN-DAG: v_lshlrev_b32_e32 [[BYTES:v[0-9]+]], 2, v0
; GCN-DAG: v_and_b32_e32 [[CLAMP_IDX:v[0-9]+]], 0x1fc, [[BYTES]]
; GCN-NOT: s_mov_b32 s0
}
; GCN-LABEL: {{^}}vs_main:
-; GCN-DAG: s_mov_b32 s4, SCRATCH_RSRC_DWORD0
+; GCN-DAG: s_mov_b32 s0, SCRATCH_RSRC_DWORD0
; GCN-NOT: s_mov_b32 s0
; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen
; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen
}
; GCN-LABEL: {{^}}cs_main:
-; GCN-DAG: s_mov_b32 s4, SCRATCH_RSRC_DWORD0
+; GCN-DAG: s_mov_b32 s0, SCRATCH_RSRC_DWORD0
; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen
; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen
define amdgpu_cs float @cs_main(i32 %idx) {
}
; GCN-LABEL: {{^}}hs_main:
-; SIVI: s_mov_b32 s4, SCRATCH_RSRC_DWORD0
+; SIVI: s_mov_b32 s0, SCRATCH_RSRC_DWORD0
; SIVI-NOT: s_mov_b32 s0
; SIVI: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen
; SIVI: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen
}
; GCN-LABEL: {{^}}gs_main:
-; SIVI: s_mov_b32 s4, SCRATCH_RSRC_DWORD0
+; SIVI: s_mov_b32 s0, SCRATCH_RSRC_DWORD0
; SIVI: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen
; SIVI: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen