From 48c0df5d4602e60a5b6c38c1df3628f7171cf580 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Fri, 19 Jul 2019 20:01:24 +0000 Subject: [PATCH] AMDGPU: Don't rely on m0 being -1 for GWS offsets This only works if the high bits of m0 are also 0, so m0 would have to be set to 0xffff. llvm-svn: 366608 --- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 10 +++-- .../CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll | 51 ++++++++++++++-------- .../test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll | 30 +++++++++---- .../CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll | 7 +-- .../CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll | 7 +-- .../AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll | 7 +-- .../CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll | 7 +-- 7 files changed, 76 insertions(+), 43 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index ea73053..e69fdb3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -2158,10 +2158,12 @@ void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) { // offset field) % 64. Some versions of the programming guide omit the m0 // part, or claim it's from offset 0. if (ConstantSDNode *ConstOffset = dyn_cast(BaseOffset)) { - // If we have a constant offset, try to use the default value for m0 as a - // base to possibly avoid setting it up. - glueCopyToM0(N, CurDAG->getTargetConstant(-1, SL, MVT::i32)); - ImmOffset = ConstOffset->getZExtValue() + 1; + // If we have a constant offset, try to use the 0 in m0 as the base. + // TODO: Look into changing the default m0 initialization value. If the + // default -1 only set the low 16-bits, we could leave it as-is and add 1 to + // the immediate offset. + glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32)); + ImmOffset = ConstOffset->getZExtValue(); } else { if (CurDAG->isBaseWithConstantOffset(BaseOffset)) { ImmOffset = BaseOffset.getConstantOperandVal(1); diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll index 756e86c..7ecbcf8 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.barrier.ll @@ -12,13 +12,14 @@ ; Minimum offset ; GCN-LABEL: {{^}}gws_barrier_offset0: ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]] -; NOLOOP-DAG: s_mov_b32 m0, -1{{$}} +; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} ; NOLOOP: v_mov_b32_e32 v0, [[BAR_NUM]] -; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}} +; NOLOOP: ds_gws_barrier v0 gds{{$}} +; LOOP: s_mov_b32 m0, 0{{$}} ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]: ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 -; LOOP-NEXT: ds_gws_barrier v0 offset:1 gds +; LOOP-NEXT: ds_gws_barrier v0 gds ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 @@ -26,7 +27,7 @@ ; MIR-LABEL: name: gws_barrier_offset0{{$}} ; MIR: BUNDLE implicit{{( killed)?}} $vgpr0, implicit $m0, implicit $exec { -; MIR-NEXT: DS_GWS_BARRIER $vgpr0, 1, -1, implicit $m0, implicit $exec :: (load 4 from custom GWSResource) +; MIR-NEXT: DS_GWS_BARRIER $vgpr0, 0, -1, implicit $m0, implicit $exec :: (load 4 from custom GWSResource) ; MIR-NEXT: S_WAITCNT 0 ; MIR-NEXT: } define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 { @@ -39,9 +40,9 @@ define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 { ; Maximum offset ; GCN-LABEL: {{^}}gws_barrier_offset63: ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]] -; NOLOOP-DAG: s_mov_b32 m0, -1{{$}} +; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} ; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]] -; NOLOOP: ds_gws_barrier v0 offset:64 gds{{$}} +; NOLOOP: ds_gws_barrier v0 offset:63 gds{{$}} define amdgpu_kernel void @gws_barrier_offset63(i32 %val) #0 { call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 63) ret void @@ -104,8 +105,18 @@ define amdgpu_kernel void @gws_barrier_vgpr_offset_add(i32 %val) #0 { ; Check if m0 initialization is shared ; GCN-LABEL: {{^}}gws_barrier_save_m0_barrier_constant_offset: -; NOLOOP: s_mov_b32 m0, -1 -; NOLOOP-NOT: s_mov_b32 m0 +; NOLOOP: s_mov_b32 m0, 0 +; NOLOOP: ds_gws_barrier v0 offset:10 gds + +; LOOP: s_mov_b32 m0, -1 +; LOOP: ds_write_b32 +; LOOP: s_mov_b32 m0, 0 +; LOOP: s_setreg_imm32_b32 +; LOOP: ds_gws_barrier v0 offset:10 gds +; LOOP: s_cbranch_scc1 + +; LOOP: s_mov_b32 m0, -1 +; LOOP: ds_write_b32 define amdgpu_kernel void @gws_barrier_save_m0_barrier_constant_offset(i32 %val) #0 { store i32 1, i32 addrspace(3)* @lds call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 10) @@ -115,7 +126,8 @@ define amdgpu_kernel void @gws_barrier_save_m0_barrier_constant_offset(i32 %val) ; Make sure this increments lgkmcnt ; GCN-LABEL: {{^}}gws_barrier_lgkmcnt: -; NOLOOP: ds_gws_barrier v0 offset:1 gds{{$}} +; NOLOOP: s_mov_b32 m0, 0{{$}} +; NOLOOP: ds_gws_barrier v0 gds{{$}} ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; NOLOOP-NEXT: s_setpc_b64 define void @gws_barrier_lgkmcnt(i32 %val) { @@ -134,7 +146,8 @@ define amdgpu_kernel void @gws_barrier_wait_before(i32 %val, i32 addrspace(1)* % } ; GCN-LABEL: {{^}}gws_barrier_wait_after: -; NOLOOP: ds_gws_barrier v0 offset:8 gds +; NOLOOP: s_mov_b32 m0, 0{{$}} +; NOLOOP: ds_gws_barrier v0 offset:7 gds ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; NOLOOP-NEXT: load_dword define amdgpu_kernel void @gws_barrier_wait_after(i32 %val, i32 addrspace(1)* %ptr) #0 { @@ -145,9 +158,10 @@ define amdgpu_kernel void @gws_barrier_wait_after(i32 %val, i32 addrspace(1)* %p ; Does not imply memory fence on its own ; GCN-LABEL: {{^}}gws_barrier_fence_before: +; NOLOOP: s_mov_b32 m0, 0{{$}} ; NOLOOP: store_dword ; NOLOOP: s_waitcnt vmcnt(0) lgkmcnt(0) -; NOLOOP: ds_gws_barrier v0 offset:8 gds +; NOLOOP: ds_gws_barrier v0 offset:7 gds ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) define amdgpu_kernel void @gws_barrier_fence_before(i32 %val, i32 addrspace(1)* %ptr) #0 { store i32 0, i32 addrspace(1)* %ptr @@ -158,7 +172,8 @@ define amdgpu_kernel void @gws_barrier_fence_before(i32 %val, i32 addrspace(1)* ; FIXME: Extra waitcnt ; GCN-LABEL: {{^}}gws_barrier_fence_after: -; NOLOOP: ds_gws_barrier v0 offset:8 gds +; NOLOOP: s_mov_b32 m0, 0{{$}} +; NOLOOP: ds_gws_barrier v0 offset:7 gds ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; NOLOOP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 @@ -172,10 +187,10 @@ define amdgpu_kernel void @gws_barrier_fence_after(i32 %val, i32 addrspace(1)* % ; FIXME: Should a wait be inserted here, or is an explicit fence needed? ; GCN-LABEL: {{^}}gws_init_barrier: -; NOLOOP: s_mov_b32 m0, -1 -; NOLOOP: ds_gws_init v0 offset:8 gds +; NOLOOP: s_mov_b32 m0, 0 +; NOLOOP: ds_gws_init v0 offset:7 gds ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; NOLOOP-NEXT: ds_gws_barrier v0 offset:8 gds +; NOLOOP-NEXT: ds_gws_barrier v0 offset:7 gds ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) define amdgpu_kernel void @gws_init_barrier(i32 %val) #0 { call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7) @@ -185,12 +200,12 @@ define amdgpu_kernel void @gws_init_barrier(i32 %val) #0 { ; FIXME: Why vmcnt, not expcnt? ; GCN-LABEL: {{^}}gws_init_fence_barrier: -; NOLOOP: s_mov_b32 m0, -1 -; NOLOOP: ds_gws_init v0 offset:8 gds +; NOLOOP: s_mov_b32 m0, 0 +; NOLOOP: ds_gws_init v0 offset:7 gds ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; NOLOOP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 -; NOLOOP-NEXT: ds_gws_barrier v0 offset:8 gds +; NOLOOP-NEXT: ds_gws_barrier v0 offset:7 gds ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) define amdgpu_kernel void @gws_init_fence_barrier(i32 %val) #0 { call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll index 11f9264..15daa71 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.init.ll @@ -7,13 +7,13 @@ ; Minimum offset ; GCN-LABEL: {{^}}gws_init_offset0: ; GCN-DAG: s_load_dword [[BAR_NUM:s[0-9]+]] -; GCN-DAG: s_mov_b32 m0, -1{{$}} +; GCN-DAG: s_mov_b32 m0, 0{{$}} ; GCN: v_mov_b32_e32 v0, [[BAR_NUM]] -; NOLOOP: ds_gws_init v0 offset:1 gds{{$}} +; NOLOOP: ds_gws_init v0 gds{{$}} ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]: ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 -; LOOP-NEXT: ds_gws_init v0 offset:1 gds +; LOOP-NEXT: ds_gws_init v0 gds ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 @@ -26,14 +26,15 @@ define amdgpu_kernel void @gws_init_offset0(i32 %val) #0 { ; Maximum offset ; GCN-LABEL: {{^}}gws_init_offset63: ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]] -; NOLOOP-DAG: s_mov_b32 m0, -1{{$}} +; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} ; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]] -; NOLOOP: ds_gws_init v0 offset:64 gds{{$}} +; NOLOOP: ds_gws_init v0 offset:63 gds{{$}} +; LOOP: s_mov_b32 m0, 0{{$}} ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]: ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 -; LOOP-NEXT: ds_gws_init v0 offset:64 gds +; LOOP-NEXT: ds_gws_init v0 offset:63 gds ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 @@ -100,8 +101,18 @@ define amdgpu_kernel void @gws_init_vgpr_offset_add(i32 %val) #0 { ; Check if m0 initialization is shared. ; GCN-LABEL: {{^}}gws_init_save_m0_init_constant_offset: -; NOLOOP: s_mov_b32 m0, -1 -; NOLOOP-NOT: s_mov_b32 m0 +; NOLOOP: s_mov_b32 m0, 0 +; NOLOOP: ds_gws_init v0 offset:10 gds + +; LOOP: s_mov_b32 m0, -1 +; LOOP: ds_write_b32 +; LOOP: s_mov_b32 m0, 0 +; LOOP: s_setreg_imm32_b32 +; LOOP: ds_gws_init v0 offset:10 gds +; LOOP: s_cbranch_scc1 + +; LOOP: s_mov_b32 m0, -1 +; LOOP: ds_write_b32 define amdgpu_kernel void @gws_init_save_m0_init_constant_offset(i32 %val) #0 { store i32 1, i32 addrspace(3)* @lds call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 10) @@ -110,7 +121,8 @@ define amdgpu_kernel void @gws_init_save_m0_init_constant_offset(i32 %val) #0 { } ; GCN-LABEL: {{^}}gws_init_lgkmcnt: -; NOLOOP: ds_gws_init v0 offset:1 gds{{$}} +; NOLOOP: s_mov_b32 m0, 0{{$}} +; NOLOOP: ds_gws_init v0 gds{{$}} ; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; NOLOOP-NEXT: s_setpc_b64 define void @gws_init_lgkmcnt(i32 %val) { diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll index 7363115e..21e09c1 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.br.ll @@ -6,13 +6,14 @@ ; GCN-LABEL: {{^}}gws_sema_br_offset0: ; NOLOOP-DAG: s_load_dword [[BAR_NUM:s[0-9]+]] -; NOLOOP-DAG: s_mov_b32 m0, -1{{$}} +; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} ; NOLOOP: v_mov_b32_e32 v0, [[BAR_NUM]] -; NOLOOP: ds_gws_sema_br v0 offset:1 gds{{$}} +; NOLOOP: ds_gws_sema_br v0 gds{{$}} +; LOOP: s_mov_b32 m0, 0{{$}} ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]: ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 -; LOOP-NEXT: ds_gws_sema_br v0 offset:1 gds +; LOOP-NEXT: ds_gws_sema_br v0 gds ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll index ba231c3..dd1a801 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.p.ll @@ -5,13 +5,14 @@ ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,GFX10 %s ; GCN-LABEL: {{^}}gws_sema_p_offset0: -; NOLOOP-DAG: s_mov_b32 m0, -1{{$}} -; NOLOOP: ds_gws_sema_p offset:1 gds{{$}} +; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} +; NOLOOP: ds_gws_sema_p gds{{$}} +; LOOP: s_mov_b32 m0, 0{{$}} ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]: ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 ; GFX8-NEXT: s_nop 0 -; LOOP-NEXT: ds_gws_sema_p offset:1 gds +; LOOP-NEXT: ds_gws_sema_p gds ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll index ab5f95b..2d690ec 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll @@ -7,13 +7,14 @@ ; GFX6ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.ds.gws.sema.release.all ; GCN-LABEL: {{^}}gws_sema_release_all_offset0: -; NOLOOP-DAG: s_mov_b32 m0, -1{{$}} -; NOLOOP: ds_gws_sema_release_all offset:1 gds{{$}} +; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} +; NOLOOP: ds_gws_sema_release_all gds{{$}} +; LOOP: s_mov_b32 m0, 0{{$}} ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]: ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 ; GFX8-NEXT: s_nop 0 -; LOOP-NEXT: ds_gws_sema_release_all offset:1 gds +; LOOP-NEXT: ds_gws_sema_release_all gds ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll index ed55db2..6526461 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.v.ll @@ -5,13 +5,14 @@ ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,GFX10 %s ; GCN-LABEL: {{^}}gws_sema_v_offset0: -; NOLOOP-DAG: s_mov_b32 m0, -1{{$}} -; NOLOOP: ds_gws_sema_v offset:1 gds{{$}} +; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} +; NOLOOP: ds_gws_sema_v gds{{$}} +; LOOP: s_mov_b32 m0, 0{{$}} ; LOOP: [[LOOP:BB[0-9]+_[0-9]+]]: ; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 ; GFX8-NEXT: s_nop 0 -; LOOP-NEXT: ds_gws_sema_v offset:1 gds +; LOOP-NEXT: ds_gws_sema_v gds ; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) ; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 -- 2.7.4