From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 16 Jul 2019 19:22:21 +0000 (+0000)
Subject: AMDGPU/GlobalISel: Select private loads
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=7161fb0be59e56becefb8646583cde912bcbfa5c;p=platform%2Fupstream%2Fllvm.git

AMDGPU/GlobalISel: Select private loads

llvm-svn: 366248
---

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 1ccb90b..13ca1ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -57,6 +57,13 @@ def gi_flat_offset_signed :
     GIComplexOperandMatcher<s64, "selectFlatOffsetSigned">,
     GIComplexPatternEquiv<FLATOffsetSigned>;
 
+def gi_mubuf_scratch_offset :
+    GIComplexOperandMatcher<s32, "selectMUBUFScratchOffset">,
+    GIComplexPatternEquiv<MUBUFScratchOffset>;
+def gi_mubuf_scratch_offen :
+    GIComplexOperandMatcher<s32, "selectMUBUFScratchOffen">,
+    GIComplexPatternEquiv<MUBUFScratchOffen>;
+
 
 class GISelSop2Pat <
   SDPatternOperator node,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 25e72bb..901a2ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -17,10 +17,11 @@
 #include "AMDGPURegisterInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
-#include "SIMachineFunctionInfo.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -34,6 +35,7 @@
 #define DEBUG_TYPE "amdgpu-isel"
 
 using namespace llvm;
+using namespace MIPatternMatch;
 
 #define GET_GLOBALISEL_IMPL
 #define AMDGPUSubtarget GCNSubtarget
@@ -1594,3 +1596,135 @@ InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectFlatOffsetSigned(MachineOperand &Root) const {
   return selectFlatOffsetImpl<true>(Root);
 }
+
+// FIXME: Implement
+static bool signBitIsZero(const MachineOperand &Op,
+                          const MachineRegisterInfo &MRI) {
+  return false;
+}
+
+static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
+  auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
+  return PSV && PSV->isStack();
+}
+
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
+  MachineInstr *MI = Root.getParent();
+  MachineBasicBlock *MBB = MI->getParent();
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
+
+  int64_t Offset = 0;
+  if (mi_match(Root.getReg(), MRI, m_ICst(Offset))) {
+    Register HighBits = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+
+    // TODO: Should this be inside the render function? The iterator seems to
+    // move.
+    BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32),
+            HighBits)
+      .addImm(Offset & ~4095);
+
+    return {{[=](MachineInstrBuilder &MIB) { // rsrc
+               MIB.addReg(Info->getScratchRSrcReg());
+             },
+             [=](MachineInstrBuilder &MIB) { // vaddr
+               MIB.addReg(HighBits);
+             },
+             [=](MachineInstrBuilder &MIB) { // soffset
+               const MachineMemOperand *MMO = *MI->memoperands_begin();
+               const MachinePointerInfo &PtrInfo = MMO->getPointerInfo();
+
+               Register SOffsetReg = isStackPtrRelative(PtrInfo)
+                                         ? Info->getStackPtrOffsetReg()
+                                         : Info->getScratchWaveOffsetReg();
+               MIB.addReg(SOffsetReg);
+             },
+             [=](MachineInstrBuilder &MIB) { // offset
+               MIB.addImm(Offset & 4095);
+             }}};
+  }
+
+  assert(Offset == 0);
+
+  // Try to fold a frame index directly into the MUBUF vaddr field, and any
+  // offsets.
+  Optional<int> FI;
+  Register VAddr = Root.getReg();
+  if (const MachineInstr *RootDef = MRI.getVRegDef(Root.getReg())) {
+    if (isBaseWithConstantOffset(Root, MRI)) {
+      const MachineOperand &LHS = RootDef->getOperand(1);
+      const MachineOperand &RHS = RootDef->getOperand(2);
+      const MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
+      const MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
+      if (LHSDef && RHSDef) {
+        int64_t PossibleOffset =
+            RHSDef->getOperand(1).getCImm()->getSExtValue();
+        if (SIInstrInfo::isLegalMUBUFImmOffset(PossibleOffset) &&
+            (!STI.privateMemoryResourceIsRangeChecked() ||
+             signBitIsZero(LHS, MRI))) {
+          if (LHSDef->getOpcode() == AMDGPU::G_FRAME_INDEX)
+            FI = LHSDef->getOperand(1).getIndex();
+          else
+            VAddr = LHS.getReg();
+          Offset = PossibleOffset;
+        }
+      }
+    } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) {
+      FI = RootDef->getOperand(1).getIndex();
+    }
+  }
+
+  // If we don't know this private access is a local stack object, it needs to
+  // be relative to the entry point's scratch wave offset register.
+  // TODO: Should split large offsets that don't fit like above.
+  // TODO: Don't use scratch wave offset just because the offset didn't fit.
+  Register SOffset = FI.hasValue() ? Info->getStackPtrOffsetReg()
+                                   : Info->getScratchWaveOffsetReg();
+
+  return {{[=](MachineInstrBuilder &MIB) { // rsrc
+             MIB.addReg(Info->getScratchRSrcReg());
+           },
+           [=](MachineInstrBuilder &MIB) { // vaddr
+             if (FI.hasValue())
+               MIB.addFrameIndex(FI.getValue());
+             else
+               MIB.addReg(VAddr);
+           },
+           [=](MachineInstrBuilder &MIB) { // soffset
+             MIB.addReg(SOffset);
+           },
+           [=](MachineInstrBuilder &MIB) { // offset
+             MIB.addImm(Offset);
+           }}};
+}
+
+InstructionSelector::ComplexRendererFns
+AMDGPUInstructionSelector::selectMUBUFScratchOffset(
+    MachineOperand &Root) const {
+  MachineInstr *MI = Root.getParent();
+  MachineBasicBlock *MBB = MI->getParent();
+  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
+
+  int64_t Offset = 0;
+  if (!mi_match(Root.getReg(), MRI, m_ICst(Offset)) ||
+      !SIInstrInfo::isLegalMUBUFImmOffset(Offset))
+    return {};
+
+  const MachineFunction *MF = MBB->getParent();
+  const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
+  const MachineMemOperand *MMO = *MI->memoperands_begin();
+  const MachinePointerInfo &PtrInfo = MMO->getPointerInfo();
+
+  Register SOffsetReg = isStackPtrRelative(PtrInfo)
+                            ? Info->getStackPtrOffsetReg()
+                            : Info->getScratchWaveOffsetReg();
+  return {{
+      [=](MachineInstrBuilder &MIB) {
+        MIB.addReg(Info->getScratchRSrcReg());
+      },                                                         // rsrc
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(SOffsetReg); }, // soffset
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }      // offset
+  }};
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index e30d745..4f489dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -128,6 +128,11 @@ private:
   InstructionSelector::ComplexRendererFns
   selectFlatOffsetSigned(MachineOperand &Root) const;
 
+  InstructionSelector::ComplexRendererFns
+  selectMUBUFScratchOffen(MachineOperand &Root) const;
+  InstructionSelector::ComplexRendererFns
+  selectMUBUFScratchOffset(MachineOperand &Root) const;
+
   const SIInstrInfo &TII;
   const SIRegisterInfo &TRI;
   const AMDGPURegisterBankInfo &RBI;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
new file mode 100644
index 0000000..e969f45
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
@@ -0,0 +1,1158 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+
+---
+
+name: load_private_s32_from_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_4
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_4
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_s32_from_2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_2
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 2, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_2
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 2, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_LOAD %0 :: (load 2, align 2, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_s32_from_1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_LOAD %0 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_v2s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ; GFX6-LABEL: name: load_private_v2s32
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 8, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]]
+    ; GFX9-LABEL: name: load_private_v2s32
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 8, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUFFER_LOAD_DWORDX2_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<2 x s32>) = G_LOAD %0 :: (load 8, align 8, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_private_v4s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_v4s32
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 16, align 4, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]]
+    ; GFX9-LABEL: name: load_private_v4s32
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 16, align 4, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<4 x  s32>) = G_LOAD %0 :: (load 16, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_private_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s64
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; GFX9-LABEL: name: load_private_s64
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s64) = G_LOAD %0 :: (load 8, align 8, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_private_v2s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_v2s64
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_128(<2 x s64>) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
+    ; GFX9-LABEL: name: load_private_v2s64
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_128(<2 x s64>) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x s64>)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<2 x s64>) = G_LOAD %0 :: (load 16, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_private_v2p1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_v2p1
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_128(<2 x p1>) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    ; GFX9-LABEL: name: load_private_v2p1
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_128(<2 x p1>) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<2 x p1>)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<2 x p1>) = G_LOAD %0 :: (load 16, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_private_s128
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s128
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_128(s128) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    ; GFX9-LABEL: name: load_private_s128
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_128(s128) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](s128)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s128) = G_LOAD %0 :: (load 16, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+
+name: load_private_p3_from_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_p3_from_4
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[LOAD]](p3)
+    ; GFX9-LABEL: name: load_private_p3_from_4
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[LOAD]](p3)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(p3) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_p5_from_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_p5_from_4
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vgpr_32(p5) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[LOAD]](p5)
+    ; GFX9-LABEL: name: load_private_p5_from_4
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vgpr_32(p5) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[LOAD]](p5)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(p5) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_p999_from_8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_p999_from_8
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    ; GFX9-LABEL: name: load_private_p999_from_8
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_64(p999) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[LOAD]](p999)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(p999) = G_LOAD %0 :: (load 8, align 8, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_private_v2p3
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_v2p3
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_64(<2 x p3>) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX9-LABEL: name: load_private_v2p3
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_64(<2 x p3>) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x p3>)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<2 x p3>) = G_LOAD %0 :: (load 8, align 8, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+---
+
+name: load_private_v2s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_v2s16
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; GFX9-LABEL: name: load_private_v2s16
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<2 x s16>) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_v4s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_v4s16
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; GFX9-LABEL: name: load_private_v4s16
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p5) :: (load 8, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<4 x s16>) = G_LOAD %0 :: (load 8, align 8, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+
+...
+
+# ---
+
+# name: load_private_v6s16
+# legalized:       true
+# regBankSelected: true
+# tracksRegLiveness: true
+# machineFunctionInfo:
+#   scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+#   scratchWaveOffsetReg: $sgpr4
+#   stackPtrOffsetReg: $sgpr32
+
+# body: |
+#   bb.0:
+#     liveins:  $vgpr0
+
+#     %0:vgpr(p5) = COPY $vgpr0
+#     %1:vgpr(<6 x  s16>) = G_LOAD %0 :: (load 12, align 4, addrspace 5)
+#     $vgpr0_vgpr1_vgpr2 = COPY %1
+
+# ...
+
+---
+
+name: load_private_v8s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_v8s16
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX6: [[LOAD:%[0-9]+]]:vreg_128(<8 x s16>) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    ; GFX9-LABEL: name: load_private_v8s16
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; GFX9: [[LOAD:%[0-9]+]]:vreg_128(<8 x s16>) = G_LOAD [[COPY]](p5) :: (load 16, align 4, addrspace 5)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[LOAD]](<8 x s16>)
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(<8 x  s16>) = G_LOAD %0 :: (load 16, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+################################################################################
+### Stress addressing modes
+################################################################################
+
+---
+
+name: load_private_s32_from_1_gep_2047
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_2047
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2047, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_2047
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2047, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 2047
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_2048
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_2048
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_2048
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2048, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 2048
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_m2047
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_m2047
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965249, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_m2047
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965249, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 -2047
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_m2048
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_m2048
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_m2048
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 -2048
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_4095
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_4095
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_4095
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 4095
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_4096
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_4096
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_4096
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 4096
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_m4095
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_m4095
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_m4095
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 -4095
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_m4096
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_m4096
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_m4096
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 -4096
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_8191
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_8191
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8191, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_8191
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8191, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 8191
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_8192
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_8192
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8192, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_8192
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8192, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 8192
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_m8191
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_m8191
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_m8191
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 -8191
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_gep_m8192
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+    liveins:  $vgpr0
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_gep_m8192
+    ; GFX6: liveins: $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_gep_m8192
+    ; GFX9: liveins: $vgpr0
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 -8192
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_4_constant_0
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+
+    ; GFX6-LABEL: name: load_private_s32_from_4_constant_0
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
+    ; GFX9-LABEL: name: load_private_s32_from_4_constant_0
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
+    %0:vgpr(p5) = G_CONSTANT i32 0
+    %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_s32_from_4_constant_sgpr_16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+
+    ; GFX6-LABEL: name: load_private_s32_from_4_constant_sgpr_16
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 16, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
+    ; GFX9-LABEL: name: load_private_s32_from_4_constant_sgpr_16
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 16, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
+    %0:sgpr(p5) = G_CONSTANT i32 16
+    %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_s32_from_1_constant_4095
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_constant_4095
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFSET]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_constant_4095
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFSET]]
+    %0:vgpr(p5) = G_CONSTANT i32 4095
+    %1:vgpr(s32) = G_LOAD %0 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_s32_from_1_constant_4096
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+
+body: |
+  bb.0:
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_constant_4096
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_constant_4096
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = G_CONSTANT i32 4096
+    %1:vgpr(s32) = G_LOAD %0 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_s32_from_fi
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+stack:
+  - { id: 0, size: 4, alignment: 4 }
+
+body: |
+  bb.0:
+
+    ; GFX6-LABEL: name: load_private_s32_from_fi
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_fi
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+    %0:vgpr(p5) = G_FRAME_INDEX %stack.0
+    %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
+    $vgpr0 = COPY %1
+
+...
+
+---
+
+name: load_private_s32_from_1_fi_offset_4095
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+stack:
+  - { id: 0, size: 4096, alignment: 4 }
+
+body: |
+  bb.0:
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_fi_offset_4095
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_fi_offset_4095
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = G_FRAME_INDEX %stack.0
+    %1:vgpr(s32) = G_CONSTANT i32 4095
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...
+
+---
+
+name: load_private_s32_from_1_fi_offset_4096
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
+  scratchWaveOffsetReg: $sgpr4
+  stackPtrOffsetReg: $sgpr32
+stack:
+  - { id: 0, size: 8192, alignment: 4 }
+
+body: |
+  bb.0:
+
+    ; GFX6-LABEL: name: load_private_s32_from_1_fi_offset_4096
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+    ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], 0, implicit $exec
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    ; GFX9-LABEL: name: load_private_s32_from_1_fi_offset_4096
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], 0, implicit $exec
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
+    %0:vgpr(p5) = G_FRAME_INDEX %stack.0
+    %1:vgpr(s32) = G_CONSTANT i32 4096
+    %2:vgpr(p5) = G_GEP %0, %1
+    %3:vgpr(s32) = G_LOAD %2 :: (load 1, align 1, addrspace 5)
+    $vgpr0 = COPY %3
+
+...