[AMDGPU] Support unaligned flat scratch in TLI

Adjust SITargetLowering::allowsMisalignedMemoryAccessesImpl for
unaligned flat scratch support. Mostly needed for global isel.

Differential Revision: https://reviews.llvm.org/D93669

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5fb1924bdd9fc8d8f53bd441665c186a5fb8d65f..81fdfa0343b335e7b812640ffd1c7cbd891c2ba2 100644 (file)
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1470,12 +1470,21 @@ bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
     }
   }
 
+  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
+    bool AlignedBy4 = Alignment >= Align(4);
+    if (IsFast)
+      *IsFast = AlignedBy4;
+
+    return AlignedBy4 ||
+           Subtarget->enableFlatScratch() ||
+           Subtarget->hasUnalignedScratchAccess();
+  }
+
   // FIXME: We have to be conservative here and assume that flat operations
   // will access scratch.  If we had access to the IR function, then we
   // could determine if any private memory was used in the function.
-  if (!Subtarget->hasUnalignedScratchAccess() &&
-      (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
-       AddrSpace == AMDGPUAS::FLAT_ADDRESS)) {
+  if (AddrSpace == AMDGPUAS::FLAT_ADDRESS &&
+      !Subtarget->hasUnalignedScratchAccess()) {
     bool AlignedBy4 = Alignment >= Align(4);
     if (IsFast)
       *IsFast = AlignedBy4;

diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index 271f6c703980a797f79e0d28c7cb9be2a8948f93..8e37b413ddf585b489f216a7c1076dea8c7e7b2c 100644 (file)
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -271,16 +271,9 @@ define amdgpu_kernel void @vload2_private(i16 addrspace(1)* nocapture readonly %
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    scratch_store_short off, v0, vcc_hi offset:8
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
-; FLATSCR-NEXT:    scratch_load_ushort v0, off, vcc_hi offset:4
+; FLATSCR-NEXT:    scratch_load_dword v0, off, vcc_hi offset:4
 ; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
-; FLATSCR-NEXT:    scratch_load_ushort v3, off, vcc_hi offset:6
-; FLATSCR-NEXT:    s_mov_b32 vcc_hi, 0
-; FLATSCR-NEXT:    s_waitcnt vmcnt(1)
-; FLATSCR-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; FLATSCR-NEXT:    v_mov_b32_e32 v1, v3
-; FLATSCR-NEXT:    scratch_load_short_d16_hi v1, off, vcc_hi offset:8
-; FLATSCR-NEXT:    v_lshl_or_b32 v0, v3, 16, v0
+; FLATSCR-NEXT:    scratch_load_dword v1, off, vcc_hi offset:6
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; FLATSCR-NEXT:    s_endpgm

diff --git a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
index 5d5cfd318edfba18e03a0ad8ba8e70dfaa957ac8..645eead8c2971e5798243898c942b96b4b7b0b38 100644 (file)
--- a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
@@ -1,6 +1,7 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=UNALIGNED %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s
+; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,UNALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,FLATSCR,ALIGNED %s
 
 ; SI-LABEL: {{^}}local_unaligned_load_store_i16:
 ; SI: ds_read_u8
@@ -602,64 +603,70 @@ define amdgpu_kernel void @local_store_align1_v16i8(<16 x i8> addrspace(3)* %out
 }
 
 ; SI-LABEL: {{^}}private_load_align1_f64:
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; FLATSCR: scratch_load_dwordx2
 define double @private_load_align1_f64(double addrspace(5)* %in) {
   %x = load double, double addrspace(5)* %in, align 1
   ret double %x
 }
 
 ; SI-LABEL: {{^}}private_store_align1_f64:
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; FLATSCR: scratch_store_dwordx2
 define void @private_store_align1_f64(double addrspace(5)* %out, double %x) #0 {
   store double %x, double addrspace(5)* %out, align 1
   ret void
 }
 
 ; SI-LABEL: {{^}}private_load_align4_f64:
-; SI: buffer_load_dword
-; SI: buffer_load_dword
+; MUBUF: buffer_load_dword
+; MUBUF: buffer_load_dword
+; FLATSCR: scratch_load_dwordx2
 define double @private_load_align4_f64(double addrspace(5)* %in) {
   %x = load double, double addrspace(5)* %in, align 4
   ret double %x
 }
 
 ; SI-LABEL: {{^}}private_store_align4_f64:
-; SI: buffer_store_dword
-; SI: buffer_store_dword
+; MUBUF: buffer_store_dword
+; MUBUF: buffer_store_dword
+; FLATSCR: scratch_store_dwordx2
 define void @private_store_align4_f64(double addrspace(5)* %out, double %x) #0 {
   store double %x, double addrspace(5)* %out, align 4
   ret void
 }
 
 ; SI-LABEL: {{^}}private_load_align2_f64:
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; FLATSCR: scratch_load_dwordx2
 define double @private_load_align2_f64(double addrspace(5)* %in) {
   %x = load double, double addrspace(5)* %in, align 2
   ret double %x
 }
 
 ; SI-LABEL: {{^}}private_store_align2_f64:
-; SI: buffer_store_short
-; SI: buffer_store_short
-; SI: buffer_store_short
-; SI: buffer_store_short
+; MUBUF: buffer_store_short
+; MUBUF: buffer_store_short
+; MUBUF: buffer_store_short
+; MUBUF: buffer_store_short
+; FLATSCR: scratch_store_dwordx2
 define void @private_store_align2_f64(double addrspace(5)* %out, double %x) #0 {
   store double %x, double addrspace(5)* %out, align 2
   ret void

diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
index a46f4d4175b7cbec6e14332b576754db8c2372ec..5bbacbf6a034cbf71183b85d2873bf1389731ac8 100644 (file)
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
@@ -9,9 +9,6 @@ target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:3
 ; ALL-LABEL: @load_unknown_offset_align1_i8(
 ; ALL: alloca [128 x i8], align 1
 ; UNALIGNED: load <2 x i8>, <2 x i8> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
-
-; ALIGNED: load i8, i8 addrspace(5)* %ptr0, align 1{{$}}
-; ALIGNED: load i8, i8 addrspace(5)* %ptr1, align 1{{$}}
 define amdgpu_kernel void @load_unknown_offset_align1_i8(i8 addrspace(1)* noalias %out, i32 %offset) #0 {
   %alloca = alloca [128 x i8], align 1, addrspace(5)
   %ptr0 = getelementptr inbounds [128 x i8], [128 x i8] addrspace(5)* %alloca, i32 0, i32 %offset
@@ -60,13 +57,11 @@ define amdgpu_kernel void @load_unknown_offset_align1_i32(i32 addrspace(1)* noal
   ret void
 }
 
-; FIXME: Should always increase alignment of the load
 ; Make sure alloca alignment isn't decreased
 ; ALL-LABEL: @load_alloca16_unknown_offset_align1_i32(
 ; ALL: alloca [128 x i32], align 16
 
-; UNALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
-; ALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}}
+; ALL: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}}
 define amdgpu_kernel void @load_alloca16_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 {
   %alloca = alloca [128 x i32], align 16, addrspace(5)
   %ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset
@@ -128,11 +123,8 @@ define amdgpu_kernel void @store_unknown_offset_align1_i32(i32 addrspace(1)* noa
 }
 
 ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32(
-; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
-; ALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
-; UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5)
+; ALL: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 4
 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
   %alloca = alloca [8 x i32], align 1, addrspace(5)
   %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
@@ -148,11 +140,8 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
 }
 
 ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8(
-; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
-; ALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
-; UNALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5)
+; ALL: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 4
 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
   %alloca = alloca [8 x i8], align 1, addrspace(5)
   %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*
@@ -168,11 +157,8 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
 }
 
 ; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i32(
-; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
-; ALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
-; UNALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5)
+; ALL: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4
 define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
   %alloca = alloca [8 x i32], align 1, addrspace(5)
   %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
@@ -188,11 +174,8 @@ define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
 }
 
 ; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i8(
-; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
-; ALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
-; UNALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5)
+; ALL: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4
 define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() {
   %alloca = alloca [8 x i8], align 1, addrspace(5)
   %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*