From 8708089896767fe8bcaf95b24ff7b472d928a65f Mon Sep 17 00:00:00 2001 From: Michal Paszkowski Date: Wed, 21 Dec 2022 22:51:59 +0100 Subject: [PATCH] [SPIR-V] Add atomic_flag builtin implementation This change provides implementation details for atomic_flag builtins and adds an extended atomic_flag.ll test from the LLVM SPIR-V Translator. Differential Revision: https://reviews.llvm.org/D136310 --- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 103 +++++++++++++++++---- llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 4 + llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll | 40 ++++++++ 3 files changed, 130 insertions(+), 17 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 3ebb60b..89c25e7 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -416,6 +416,36 @@ static Register buildConstantIntReg(uint64_t Val, MachineIRBuilder &MIRBuilder, return GR->buildConstantInt(Val, MIRBuilder, IntType); } +static Register buildScopeReg(Register CLScopeRegister, + MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR, + const MachineRegisterInfo *MRI) { + auto CLScope = + static_cast(getIConstVal(CLScopeRegister, MRI)); + SPIRV::Scope::Scope Scope = getSPIRVScope(CLScope); + + if (CLScope == static_cast(Scope)) + return CLScopeRegister; + + return buildConstantIntReg(Scope, MIRBuilder, GR); +} + +static Register buildMemSemanticsReg(Register SemanticsRegister, + Register PtrRegister, + const MachineRegisterInfo *MRI, + SPIRVGlobalRegistry *GR) { + std::memory_order Order = + static_cast(getIConstVal(SemanticsRegister, MRI)); + unsigned Semantics = + getSPIRVMemSemantics(Order) | + getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister)); + + if (Order == Semantics) + return SemanticsRegister; + + return Register(); +} + /// Helper function for translating atomic init to OpStore. static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder) { @@ -585,31 +615,26 @@ static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { const MachineRegisterInfo *MRI = MIRBuilder.getMRI(); - Register ScopeRegister; SPIRV::Scope::Scope Scope = SPIRV::Scope::Workgroup; + Register ScopeRegister; + if (Call->Arguments.size() >= 4) { - assert(Call->Arguments.size() == 4 && "Extra args for explicit atomic RMW"); - auto CLScope = static_cast( - getIConstVal(Call->Arguments[3], MRI)); - Scope = getSPIRVScope(CLScope); - if (CLScope == static_cast(Scope)) - ScopeRegister = Call->Arguments[3]; + assert(Call->Arguments.size() == 4 && + "Too many args for explicit atomic RMW"); + ScopeRegister = buildScopeReg(Call->Arguments[3], MIRBuilder, GR, MRI); } + if (!ScopeRegister.isValid()) ScopeRegister = buildConstantIntReg(Scope, MIRBuilder, GR); Register PtrRegister = Call->Arguments[0]; - Register MemSemanticsReg; unsigned Semantics = SPIRV::MemorySemantics::None; - if (Call->Arguments.size() >= 3) { - std::memory_order Order = - static_cast(getIConstVal(Call->Arguments[2], MRI)); - Semantics = - getSPIRVMemSemantics(Order) | - getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister)); - if (Order == Semantics) - MemSemanticsReg = Call->Arguments[2]; - } + Register MemSemanticsReg; + + if (Call->Arguments.size() >= 3) + MemSemanticsReg = + buildMemSemanticsReg(Call->Arguments[2], PtrRegister, MRI, GR); + if (!MemSemanticsReg.isValid()) MemSemanticsReg = buildConstantIntReg(Semantics, MIRBuilder, GR); @@ -623,6 +648,47 @@ static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, return true; } +/// Helper function for building atomic flag instructions (e.g. +/// OpAtomicFlagTestAndSet). +static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, + unsigned Opcode, MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + const MachineRegisterInfo *MRI = MIRBuilder.getMRI(); + + Register PtrRegister = Call->Arguments[0]; + unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent; + Register MemSemanticsReg; + + if (Call->Arguments.size() >= 2) + MemSemanticsReg = + buildMemSemanticsReg(Call->Arguments[1], PtrRegister, MRI, GR); + + if (!MemSemanticsReg.isValid()) + MemSemanticsReg = buildConstantIntReg(Semantics, MIRBuilder, GR); + + assert((Opcode != SPIRV::OpAtomicFlagClear || + (Semantics != SPIRV::MemorySemantics::Acquire && + Semantics != SPIRV::MemorySemantics::AcquireRelease)) && + "Invalid memory order argument!"); + + SPIRV::Scope::Scope Scope = SPIRV::Scope::Device; + Register ScopeRegister; + + if (Call->Arguments.size() >= 3) + ScopeRegister = buildScopeReg(Call->Arguments[2], MIRBuilder, GR, MRI); + + if (!ScopeRegister.isValid()) + ScopeRegister = buildConstantIntReg(Scope, MIRBuilder, GR); + + auto MIB = MIRBuilder.buildInstr(Opcode); + if (Opcode == SPIRV::OpAtomicFlagTestAndSet) + MIB.addDef(Call->ReturnRegister) + .addUse(GR->getSPIRVTypeID(Call->ReturnType)); + + MIB.addUse(PtrRegister).addUse(ScopeRegister).addUse(MemSemanticsReg); + return true; +} + /// Helper function for building barriers, i.e., memory/control ordering /// operations. static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode, @@ -992,6 +1058,9 @@ static bool generateAtomicInst(const SPIRV::IncomingCall *Call, return buildAtomicRMWInst(Call, Opcode, MIRBuilder, GR); case SPIRV::OpMemoryBarrier: return buildBarrierInst(Call, SPIRV::OpMemoryBarrier, MIRBuilder, GR); + case SPIRV::OpAtomicFlagTestAndSet: + case SPIRV::OpAtomicFlagClear: + return buildAtomicFlagInst(Call, Opcode, MIRBuilder, GR); default: return false; } diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td index c82354b..3c43070 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -527,6 +527,10 @@ defm : DemangledNativeBuiltin<"atomic_fetch_sub_explicit", OpenCL_std, Atomic, 4 defm : DemangledNativeBuiltin<"atomic_fetch_or_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicOr>; defm : DemangledNativeBuiltin<"atomic_fetch_xor_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicXor>; defm : DemangledNativeBuiltin<"atomic_fetch_and_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicAnd>; +defm : DemangledNativeBuiltin<"atomic_flag_test_and_set", OpenCL_std, Atomic, 1, 1, OpAtomicFlagTestAndSet>; +defm : DemangledNativeBuiltin<"atomic_flag_test_and_set_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagTestAndSet>; +defm : DemangledNativeBuiltin<"atomic_flag_clear", OpenCL_std, Atomic, 1, 1, OpAtomicFlagClear>; +defm : DemangledNativeBuiltin<"atomic_flag_clear_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagClear>; // Barrier builtin records: defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>; diff --git a/llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll b/llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll new file mode 100644 index 0000000..3c563d3 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll @@ -0,0 +1,40 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s + +;; Types: +; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 +; CHECK-DAG: %[[#BOOL:]] = OpTypeBool +;; Constants: +; CHECK-DAG: %[[#SEQ_CONSI_SEMAN:]] = OpConstant %[[#INT]] 16 +; CHECK-DAG: %[[#DEVICE_SCOPE:]] = OpConstant %[[#INT]] 1 +; CHECK-DAG: %[[#WORKGROUP_SCOPE:]] = OpConstant %[[#INT]] 2 +;; Instructions: +; CHECK-DAG: %[[#PARAM:]] = OpFunctionParameter %[[#]] +; CHECK: %[[#]] = OpAtomicFlagTestAndSet %[[#BOOL]] %[[#PARAM]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: %[[#]] = OpAtomicFlagTestAndSet %[[#BOOL]] %[[#PARAM]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: %[[#]] = OpAtomicFlagTestAndSet %[[#BOOL]] %[[#PARAM]] %[[#WORKGROUP_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: OpAtomicFlagClear %[[#PARAM]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: OpAtomicFlagClear %[[#PARAM]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: OpAtomicFlagClear %[[#PARAM]] %[[#WORKGROUP_SCOPE]] %[[#SEQ_CONSI_SEMAN]] + +define spir_kernel void @testAtomicFlag(ptr %object) { +entry: + %call1 = call spir_func zeroext i1 @_Z24atomic_flag_test_and_setPU3AS4VU7_Atomici(ptr %object) + %call2 = call spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order(ptr %object, i32 5) + %call5 = call spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(ptr %object, i32 5, i32 1) + call spir_func void @_Z17atomic_flag_clearPU3AS4VU7_Atomici(ptr %object) + call spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order(ptr %object, i32 5) + call spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(ptr %object, i32 5, i32 1) + ret void +} + +declare spir_func zeroext i1 @_Z24atomic_flag_test_and_setPU3AS4VU7_Atomici(ptr) + +declare spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order(ptr, i32) + +declare spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(ptr, i32, i32) + +declare spir_func void @_Z17atomic_flag_clearPU3AS4VU7_Atomici(ptr) + +declare spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order(ptr, i32) + +declare spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(ptr, i32, i32) -- 2.7.4