From e381833ba5794fde6d36e3fbcc9b22ffbe438d17 Mon Sep 17 00:00:00 2001 From: Joe Nash Date: Tue, 24 Aug 2021 14:40:04 -0400 Subject: [PATCH] [AMDGPU] Support global_atomic_fmin/max on gfx10 Makes patterns added for gfx90a usable with the gfx10 versions of the insts. Reviewed By: rampitec Differential Revision: https://reviews.llvm.org/D108654 Change-Id: I86167bf6b4823f975f74ccb619bd6190331ba16b --- llvm/lib/Target/AMDGPU/FLATInstructions.td | 15 +- .../AMDGPU/fp-min-max-global-atomics-gfx10.ll | 201 +++++++++++++++++++++ 2 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/fp-min-max-global-atomics-gfx10.ll diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index 90f26e5..07559aa 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -788,15 +788,15 @@ let SubtargetPredicate = isGFX10Plus, is_flat_global = 1 in { defm GLOBAL_ATOMIC_FCMPSWAP : FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32>; defm GLOBAL_ATOMIC_FMIN : - FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32>; + FLAT_Global_Atomic_Pseudo<"global_atomic_fmin", VGPR_32, f32, int_amdgcn_global_atomic_fmin>; defm GLOBAL_ATOMIC_FMAX : - FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32>; + FLAT_Global_Atomic_Pseudo<"global_atomic_fmax", VGPR_32, f32, int_amdgcn_global_atomic_fmax>; defm GLOBAL_ATOMIC_FCMPSWAP_X2 : FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap_x2", VReg_64, f64>; defm GLOBAL_ATOMIC_FMIN_X2 : - FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64>; + FLAT_Global_Atomic_Pseudo<"global_atomic_fmin_x2", VReg_64, f64, int_amdgcn_global_atomic_fmin>; defm GLOBAL_ATOMIC_FMAX_X2 : - FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64>; + FLAT_Global_Atomic_Pseudo<"global_atomic_fmax_x2", VReg_64, f64, int_amdgcn_global_atomic_fmax>; } // End SubtargetPredicate = isGFX10Plus, is_flat_global = 1 let is_flat_global = 1 in { @@ -1237,6 +1237,13 @@ defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SWAP_X2", atomic_swap_global_64, i64 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP_X2", AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR_X2", atomic_load_xor_global_64, i64>; +let OtherPredicates = [isGFX10Plus] in { +defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN", atomic_load_fmin_global_32, f32>; +defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX", atomic_load_fmax_global_32, f32>; +defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMIN_X2", atomic_load_fmin_global_64, f64>; +defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_FMAX_X2", atomic_load_fmax_global_64, f64>; +} + let OtherPredicates = [HasAtomicFaddInsts] in { defm : GlobalFLATNoRtnAtomicPats ; defm : GlobalFLATNoRtnAtomicPats ; diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-global-atomics-gfx10.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-global-atomics-gfx10.ll new file mode 100644 index 0000000..823d441 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-global-atomics-gfx10.ll @@ -0,0 +1,201 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefix=GFX10 + +; RUN: llc < %s -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX10 + +declare double @llvm.amdgcn.global.atomic.fmin.f64.p1f64.f64(double addrspace(1)* %ptr, double %data) +declare double @llvm.amdgcn.global.atomic.fmax.f64.p1f64.f64(double addrspace(1)* %ptr, double %data) +declare float @llvm.amdgcn.global.atomic.fmin.f32.p1f32.f32(float addrspace(1)* %ptr, float %data) +declare float @llvm.amdgcn.global.atomic.fmax.f32.p1f32.f32(float addrspace(1)* %ptr, float %data) + +define amdgpu_kernel void @global_atomic_fmin_f32_noret(float addrspace(1)* %ptr, float %data) { +; GFX10-LABEL: global_atomic_fmin_f32_noret: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dword s4, s[0:1], 0x2c +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: global_atomic_fmin v0, v1, s[2:3] +; GFX10-NEXT: s_endpgm +; +; G_GFX10-LABEL: global_atomic_fmin_f32_noret: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_clause 0x1 +; G_GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; G_GFX10-NEXT: s_load_dword s4, s[0:1], 0x2c +; G_GFX10-NEXT: s_waitcnt lgkmcnt(0) +; G_GFX10-NEXT: v_mov_b32_e32 v0, s2 +; G_GFX10-NEXT: v_mov_b32_e32 v1, s3 +; G_GFX10-NEXT: v_mov_b32_e32 v2, s4 +; G_GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off glc +; G_GFX10-NEXT: s_endpgm +main_body: + %ret = call float @llvm.amdgcn.global.atomic.fmin.f32.p1f32.f32(float addrspace(1)* %ptr, float %data) + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_f32_noret(float addrspace(1)* %ptr, float %data) { +; GFX10-LABEL: global_atomic_fmax_f32_noret: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dword s4, s[0:1], 0x2c +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX10-NEXT: v_mov_b32_e32 v0, 0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v1, s4 +; GFX10-NEXT: global_atomic_fmax v0, v1, s[2:3] +; GFX10-NEXT: s_endpgm +; +; G_GFX10-LABEL: global_atomic_fmax_f32_noret: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_clause 0x1 +; G_GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; G_GFX10-NEXT: s_load_dword s4, s[0:1], 0x2c +; G_GFX10-NEXT: s_waitcnt lgkmcnt(0) +; G_GFX10-NEXT: v_mov_b32_e32 v0, s2 +; G_GFX10-NEXT: v_mov_b32_e32 v1, s3 +; G_GFX10-NEXT: v_mov_b32_e32 v2, s4 +; G_GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc +; G_GFX10-NEXT: s_endpgm +main_body: + %ret = call float @llvm.amdgcn.global.atomic.fmax.f32.p1f32.f32(float addrspace(1)* %ptr, float %data) + ret void +} + +define float @global_atomic_fmax_f32_rtn(float addrspace(1)* %ptr, float %data) { +; GFX10-LABEL: global_atomic_fmax_f32_rtn: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; G_GFX10-LABEL: global_atomic_fmax_f32_rtn: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; G_GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; G_GFX10-NEXT: global_atomic_fmax v0, v[0:1], v2, off glc +; G_GFX10-NEXT: s_waitcnt vmcnt(0) +; G_GFX10-NEXT: s_setpc_b64 s[30:31] +main_body: + %ret = call float @llvm.amdgcn.global.atomic.fmax.f32.p1f32.f32(float addrspace(1)* %ptr, float %data) + ret float %ret +} + +define float @global_atomic_fmin_f32_rtn(float addrspace(1)* %ptr, float %data) { +; GFX10-LABEL: global_atomic_fmin_f32_rtn: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off glc +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; G_GFX10-LABEL: global_atomic_fmin_f32_rtn: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; G_GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; G_GFX10-NEXT: global_atomic_fmin v0, v[0:1], v2, off glc +; G_GFX10-NEXT: s_waitcnt vmcnt(0) +; G_GFX10-NEXT: s_setpc_b64 s[30:31] +main_body: + %ret = call float @llvm.amdgcn.global.atomic.fmin.f32.p1f32.f32(float addrspace(1)* %ptr, float %data) + ret float %ret +} + +define amdgpu_kernel void @global_atomic_fmin_f64_noret(double addrspace(1)* %ptr, double %data) { +; GFX10-LABEL: global_atomic_fmin_f64_noret: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GFX10-NEXT: v_mov_b32_e32 v2, 0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v0, s2 +; GFX10-NEXT: v_mov_b32_e32 v1, s3 +; GFX10-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1] +; GFX10-NEXT: s_endpgm +; +; G_GFX10-LABEL: global_atomic_fmin_f64_noret: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; G_GFX10-NEXT: s_waitcnt lgkmcnt(0) +; G_GFX10-NEXT: v_mov_b32_e32 v0, s0 +; G_GFX10-NEXT: v_mov_b32_e32 v2, s2 +; G_GFX10-NEXT: v_mov_b32_e32 v1, s1 +; G_GFX10-NEXT: v_mov_b32_e32 v3, s3 +; G_GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off glc +; G_GFX10-NEXT: s_endpgm +main_body: + %ret = call double @llvm.amdgcn.global.atomic.fmin.f64.p1f64.f64(double addrspace(1)* %ptr, double %data) + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_f64_noret(double addrspace(1)* %ptr, double %data) { +; GFX10-LABEL: global_atomic_fmax_f64_noret: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; GFX10-NEXT: v_mov_b32_e32 v2, 0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v0, s2 +; GFX10-NEXT: v_mov_b32_e32 v1, s3 +; GFX10-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1] +; GFX10-NEXT: s_endpgm +; +; G_GFX10-LABEL: global_atomic_fmax_f64_noret: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; G_GFX10-NEXT: s_waitcnt lgkmcnt(0) +; G_GFX10-NEXT: v_mov_b32_e32 v0, s0 +; G_GFX10-NEXT: v_mov_b32_e32 v2, s2 +; G_GFX10-NEXT: v_mov_b32_e32 v1, s1 +; G_GFX10-NEXT: v_mov_b32_e32 v3, s3 +; G_GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc +; G_GFX10-NEXT: s_endpgm +main_body: + %ret = call double @llvm.amdgcn.global.atomic.fmax.f64.p1f64.f64(double addrspace(1)* %ptr, double %data) + ret void +} + +define double @global_atomic_fmax_f64_rtn(double addrspace(1)* %ptr, double %data) { +; GFX10-LABEL: global_atomic_fmax_f64_rtn: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; G_GFX10-LABEL: global_atomic_fmax_f64_rtn: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; G_GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; G_GFX10-NEXT: global_atomic_fmax_x2 v[0:1], v[0:1], v[2:3], off glc +; G_GFX10-NEXT: s_waitcnt vmcnt(0) +; G_GFX10-NEXT: s_setpc_b64 s[30:31] +main_body: + %ret = call double @llvm.amdgcn.global.atomic.fmax.f64.p1f64.f64(double addrspace(1)* %ptr, double %data) + ret double %ret +} + +define double @global_atomic_fmin_f64_rtn(double addrspace(1)* %ptr, double %data) { +; GFX10-LABEL: global_atomic_fmin_f64_rtn: +; GFX10: ; %bb.0: ; %main_body +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off glc +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; G_GFX10-LABEL: global_atomic_fmin_f64_rtn: +; G_GFX10: ; %bb.0: ; %main_body +; G_GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; G_GFX10-NEXT: s_waitcnt_vscnt null, 0x0 +; G_GFX10-NEXT: global_atomic_fmin_x2 v[0:1], v[0:1], v[2:3], off glc +; G_GFX10-NEXT: s_waitcnt vmcnt(0) +; G_GFX10-NEXT: s_setpc_b64 s[30:31] +main_body: + %ret = call double @llvm.amdgcn.global.atomic.fmin.f64.p1f64.f64(double addrspace(1)* %ptr, double %data) + ret double %ret +} -- 2.7.4