From: Tobias Gysi Date: Wed, 1 Mar 2023 08:15:37 +0000 (+0100) Subject: [mlir][llvm] Add AliasAnalysis and AccessGroup interfaces. X-Git-Tag: upstream/17.0.6~16200 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=edfefd7fc50b995c4dfd81d5f9db53c1a9e11580;p=platform%2Fupstream%2Fllvm.git [mlir][llvm] Add AliasAnalysis and AccessGroup interfaces. The revision introduces two interfaces that provide access to the alias analysis and access group metadata attributes. The AliasAnalysis interface combines all alias analysis related attributes (alias, noalias, and tbaa) similar to LLVM's getAAMetadata method, while the AccessGroup interface is dedicated to the access group metadata. Previously, only the load and store operations supported alias analysis and access group metadata. This revision extends this support to the atomic operations. A follow up revision will also add support for the memcopy, memset, and memove intrinsics. The interfaces then provide convenient access to the metadata attributes and eliminate the need of TypeSwitch or string based attribute access. The revision still relies on string based attribute access for the translation to LLVM IR (except for tbaa metadata). Only once the the memory access intrinsics also implement the new interfaces, the translation to LLVM IR can be fully switched to use interface based attribute accesses. Depends on D144875 Reviewed By: ftynse Differential Revision: https://reviews.llvm.org/D144851 --- diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h index 9387ad74f531..9822c092ea47 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h @@ -15,6 +15,7 @@ #define MLIR_DIALECT_LLVMIR_LLVMDIALECT_H_ #include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/Dialect.h" @@ -34,8 +35,6 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" -#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h.inc" - namespace llvm { class Type; class LLVMContext; diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h new file mode 100644 index 000000000000..961e0be9d267 --- /dev/null +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.h @@ -0,0 +1,36 @@ +//===- LLVMInterfaces.h - LLVM Interfaces -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines op interfaces for the LLVM dialect in MLIR. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_LLVMIR_LLVMINTERFACES_H_ +#define MLIR_DIALECT_LLVMIR_LLVMINTERFACES_H_ + +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" + +namespace mlir { +namespace LLVM { +namespace detail { + +/// Verifies the access groups attribute of memory operations that implement the +/// access group interface. +LogicalResult verifyAccessGroupOpInterface(Operation *op); + +/// Verifies the alias analysis attributes of memory operations that implement +/// the alias analysis interface. +LogicalResult verifyAliasAnalysisOpInterface(Operation *op); + +} // namespace detail +} // namespace LLVM +} // namespace mlir + +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h.inc" + +#endif // MLIR_DIALECT_LLVMIR_LLVMINTERFACES_H_ diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td index 37e6b75ad451..dc5d86d8bac8 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td @@ -48,6 +48,125 @@ def FastmathFlagsInterface : OpInterface<"FastmathFlagsInterface"> { ]; } +def AccessGroupOpInterface : OpInterface<"AccessGroupOpInterface"> { + let description = [{ + An interface for memory operations that can carry access groups metadata. + It provides setters and getters for the operation's access groups attribute. + The default implementations of the interface methods expect the operation + to have an attribute of type ArrayAttr named access_groups. + }]; + + let cppNamespace = "::mlir::LLVM"; + let verify = [{ return detail::verifyAccessGroupOpInterface($_op); }]; + + let methods = [ + InterfaceMethod< + /*desc=*/ "Returns the access groups attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getAccessGroupsOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getAccessGroupsAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the access groups attribute", + /*returnType=*/ "void", + /*methodName=*/ "setAccessGroups", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setAccessGroupsAttr(attr); + }] + > + ]; +} + +def AliasAnalysisOpInterface : OpInterface<"AliasAnalysisOpInterface"> { + let description = [{ + An interface for memory operations that can carry alias analysis metadata. + It provides setters and getters for the operation's alias analysis + attributes. The default implementations of the interface methods expect + the operation to have attributes of type ArrayAttr named alias_scopes, + noalias_scopes, and tbaa. + }]; + + let cppNamespace = "::mlir::LLVM"; + let verify = [{ return detail::verifyAliasAnalysisOpInterface($_op); }]; + + let methods = [ + InterfaceMethod< + /*desc=*/ "Returns the alias scopes attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getAliasScopesOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getAliasScopesAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the alias scopes attribute", + /*returnType=*/ "void", + /*methodName=*/ "setAliasScopes", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setAliasScopesAttr(attr); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns the noalias scopes attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getNoAliasScopesOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getNoaliasScopesAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the noalias scopes attribute", + /*returnType=*/ "void", + /*methodName=*/ "setNoAliasScopes", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setNoaliasScopesAttr(attr); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns the tbaa attribute or nullptr", + /*returnType=*/ "ArrayAttr", + /*methodName=*/ "getTBAATagsOrNull", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + return op.getTbaaAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Sets the tbaa attribute", + /*returnType=*/ "void", + /*methodName=*/ "setTBAATags", + /*args=*/ (ins "const ArrayAttr":$attr), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + ConcreteOp op = cast(this->getOperation()); + op.setTbaaAttr(attr); + }] + > + ]; +} + //===----------------------------------------------------------------------===// // LLVM dialect type interfaces. //===----------------------------------------------------------------------===// @@ -76,5 +195,4 @@ def LLVM_PointerElementTypeInterface ]; } - #endif // LLVMIR_INTERFACES diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index 2ee34b239a52..a7678176d9f8 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -269,16 +269,9 @@ class LLVM_OpBase traits = []> : } //===----------------------------------------------------------------------===// -// Base classes for LLVM dialect operations. +// Base classes for LLVM enum attributes. //===----------------------------------------------------------------------===// -// Base class for LLVM operations. All operations get an "llvm." prefix in -// their name automatically. LLVM operations have either zero or one result, -// this class is specialized below for both cases and should not be used -// directly. -class LLVM_Op traits = []> : - LLVM_OpBase; - // Case of the LLVM enum attribute backed by I64Attr with customized string // representation that corresponds to what is visible in the textual IR form. // The parameters are as follows: @@ -290,7 +283,6 @@ class LLVM_Op traits = []> : // is printed/parsed as `weak` in MLIR custom textual format. class LLVM_EnumAttrCase : I64EnumAttrCase { - // The name of the equivalent enumerant in LLVM. string llvmEnumerant = llvmSym; } @@ -314,8 +306,6 @@ class LLVM_EnumAttr cases, list unsupportedCases = []> : I64EnumAttr { - - // List of unsupported cases that have no conversion to an MLIR value. list unsupported = unsupportedCases; @@ -331,11 +321,50 @@ class LLVM_CEnumAttr values> { - list lst = !foreach(x, values, - !subst("$0", !cast(x), pattern)); +//===----------------------------------------------------------------------===// +// Patterns for LLVM dialect operations. +//===----------------------------------------------------------------------===// + +// Patterns with code to set flags and metadata of memory operations after their +// translation to LLVM IR instructions. Operations may use the patterns to +// implement their "llvmBuilder". The patterns assume the `op` and `inst` +// variables exist and refer to the original MLIR operation and the translated +// LLVM IR instruction, respectively. +class LLVM_MemOpPatterns { + code setAlignmentCode = [{ + if ($alignment.has_value()) { + auto align = *$alignment; + if (align != 0) + inst->setAlignment(llvm::Align(align)); + } + }]; + code setVolatileCode = [{ + inst->setVolatile($volatile_); + }]; + code setSyncScopeCode = [{ + if ($syncscope.has_value()) { + llvm::LLVMContext &llvmContext = builder.getContext(); + inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope)); + } + }]; + code setOrderingCode = [{ + inst->setAtomic(convertAtomicOrderingToLLVM($ordering)); + }]; + code setNonTemporalMetadataCode = [{ + if ($nontemporal) { + llvm::MDNode *metadata = llvm::MDNode::get( + inst->getContext(), llvm::ConstantAsMetadata::get( + builder.getInt32(1))); + inst->setMetadata(llvm::LLVMContext::MD_nontemporal, metadata); + } + }]; + code setAccessGroupsMetadataCode = [{ + moduleTranslation.setAccessGroupsMetadata(op, inst); + }]; + code setAliasAnalysisMetadataCode = [{ + moduleTranslation.setAliasScopeMetadata(op, inst); + moduleTranslation.setTBAAMetadata(op, inst); + }]; } // Patterns with code obtaining the LLVM IR type of the given operand or result @@ -352,6 +381,36 @@ def LLVM_IntrPatterns { .getBody()[$0])}]; } +// For every value in the list, substitutes the value in the place of "$0" in +// "pattern" and stores the list of strings as "lst". +class ListIntSubst values> { + list lst = !foreach(x, values, + !subst("$0", !cast(x), pattern)); +} + +//===----------------------------------------------------------------------===// +// Base classes for LLVM dialect operations. +//===----------------------------------------------------------------------===// + +// Base class for LLVM operations. All operations get an "llvm." prefix in +// their name automatically and should either have zero or one result. +class LLVM_Op traits = []> : + LLVM_OpBase; + +// Base class for LLVM memory access operations that implement the access group +// and alias analysis interfaces. The "aliasAttrs" list contains the arguments +// required by the access group and alias analysis interfaces. Derived +// operations should append the "aliasAttrs" to their argument list. +class LLVM_MemAccessOpBase traits = []> : + LLVM_Op, + DeclareOpInterfaceMethods], traits)>, + LLVM_MemOpPatterns { + dag aliasAttrs = (ins OptionalAttr:$access_groups, + OptionalAttr:$alias_scopes, + OptionalAttr:$noalias_scopes, + OptionalAttr:$tbaa); +} // Base class for LLVM intrinsics operation. It is similar to LLVM_Op, but // provides the "llvmBuilder" field for constructing the intrinsic. diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 80818ff4f628..8660d6392942 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -169,48 +169,8 @@ def LLVM_FRemOp : LLVM_FloatArithmeticOp<"frem", "FRem">; def LLVM_FNegOp : LLVM_UnaryFloatArithmeticOp< LLVM_ScalarOrVectorOf, "fneg", "FNeg">; -// Common code definitions used to set memory operation attributes and flags. -class MemoryOpBase { - code setAlignmentCode = [{ - if ($alignment.has_value()) { - auto align = *$alignment; - if (align != 0) - inst->setAlignment(llvm::Align(align)); - } - }]; - code setVolatileCode = [{ - inst->setVolatile($volatile_); - }]; - code setSyncScopeCode = [{ - if ($syncscope.has_value()) { - llvm::LLVMContext &llvmContext = builder.getContext(); - inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope)); - } - }]; - code setOrderingCode = [{ - inst->setAtomic(convertAtomicOrderingToLLVM($ordering)); - }]; - code setNonTemporalMetadataCode = [{ - if ($nontemporal) { - llvm::MDNode *metadata = llvm::MDNode::get( - inst->getContext(), llvm::ConstantAsMetadata::get( - builder.getInt32(1))); - inst->setMetadata(llvm::LLVMContext::MD_nontemporal, metadata); - } - }]; - code setAccessGroupsMetadataCode = [{ - moduleTranslation.setAccessGroupsMetadata(op, inst); - }]; - code setAliasScopeMetadataCode = [{ - moduleTranslation.setAliasScopeMetadata(op, inst); - }]; - code setTBAAMetadataCode = [{ - moduleTranslation.setTBAAMetadata(op, inst); - }]; -} - // Memory-related operations. -def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpBase { +def LLVM_AllocaOp : LLVM_Op<"alloca">, LLVM_MemOpPatterns { let arguments = (ins AnyInteger:$arraySize, OptionalAttr:$alignment, OptionalAttr:$elem_type); @@ -347,18 +307,16 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure]> { let hasVerifier = 1; } -def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { - let arguments = (ins Arg, "", [MemRead]>:$addr, - OptionalAttr:$access_groups, - OptionalAttr:$alias_scopes, - OptionalAttr:$noalias_scopes, - OptionalAttr:$tbaa, - OptionalAttr:$alignment, - UnitAttr:$volatile_, - UnitAttr:$nontemporal, - DefaultValuedAttr:$ordering, - OptionalAttr:$syncscope); +def LLVM_LoadOp : LLVM_MemAccessOpBase<"load"> { + dag args = (ins Arg, "", [MemRead]>:$addr, + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr< + AtomicOrdering, "AtomicOrdering::not_atomic">:$ordering, + OptionalAttr:$syncscope); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); let results = (outs LLVM_LoadableType:$res); string llvmInstName = "Load"; let description = [{ @@ -392,16 +350,13 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { }]; string llvmBuilder = [{ auto *inst = builder.CreateLoad($_resultType, $addr, $volatile_); + $res = inst; }] # setOrderingCode # setSyncScopeCode # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode - # setAliasScopeMetadataCode - # setTBAAMetadataCode - # [{ - $res = inst; - }]; + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *loadInst = cast(inst); unsigned alignment = loadInst->getAlign().value(); @@ -423,19 +378,17 @@ def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase { let hasVerifier = 1; } -def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { - let arguments = (ins LLVM_LoadableType:$value, - Arg,"",[MemWrite]>:$addr, - OptionalAttr:$access_groups, - OptionalAttr:$alias_scopes, - OptionalAttr:$noalias_scopes, - OptionalAttr:$tbaa, - OptionalAttr:$alignment, - UnitAttr:$volatile_, - UnitAttr:$nontemporal, - DefaultValuedAttr:$ordering, - OptionalAttr:$syncscope); +def LLVM_StoreOp : LLVM_MemAccessOpBase<"store"> { + dag args = (ins LLVM_LoadableType:$value, + Arg,"",[MemWrite]>:$addr, + OptionalAttr:$alignment, + UnitAttr:$volatile_, + UnitAttr:$nontemporal, + DefaultValuedAttr< + AtomicOrdering, "AtomicOrdering::not_atomic">:$ordering, + OptionalAttr:$syncscope); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); string llvmInstName = "Store"; let description = [{ The `store` operation is used to write to memory. A store may be marked as @@ -473,8 +426,7 @@ def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase { # setAlignmentCode # setNonTemporalMetadataCode # setAccessGroupsMetadataCode - # setAliasScopeMetadataCode - # setTBAAMetadataCode; + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *storeInst = cast(inst); unsigned alignment = storeInst->getAlign().value(); @@ -1754,15 +1706,17 @@ def LLVM_ConstantOp def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, AnyInteger]>; -def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ +def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [ TypesMatchWith<"result #0 and operand #1 have the same type", - "val", "res", "$_self">]>, MemoryOpBase { - let arguments = (ins AtomicBinOp:$bin_op, - LLVM_PointerTo:$ptr, - LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering, - OptionalAttr:$syncscope, - OptionalAttr:$alignment, - UnitAttr:$volatile_); + "val", "res", "$_self">]> { + dag args = (ins AtomicBinOp:$bin_op, + LLVM_PointerTo:$ptr, + LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$volatile_); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); let results = (outs LLVM_AtomicRMWType:$res); let assemblyFormat = [{ (`volatile` $volatile_^)? $bin_op $ptr `,` $val @@ -1777,7 +1731,9 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ $res = inst; }] # setVolatileCode # setSyncScopeCode - # setAlignmentCode; + # setAlignmentCode + # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *atomicInst = cast(inst); unsigned alignment = atomicInst->getAlign().value(); @@ -1786,7 +1742,7 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ convertAtomicOrderingFromLLVM(atomicInst->getOrdering()), getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile()); }]; - list llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1]; + list llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1]; let builders = [ OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val, "LLVM::AtomicOrdering":$ordering, @@ -1799,20 +1755,22 @@ def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [ def LLVM_AtomicCmpXchgType : AnyTypeOf<[AnyInteger, LLVM_AnyPointer]>; -def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ +def LLVM_AtomicCmpXchgOp : LLVM_MemAccessOpBase<"cmpxchg", [ TypesMatchWith<"operand #1 and operand #2 have the same type", "val", "cmp", "$_self">, TypesMatchWith<"result #0 has an LLVM struct type consisting of " "the type of operand #2 and a bool", "val", "res", - "getValAndBoolStructType($_self)">]>, MemoryOpBase { - let arguments = (ins LLVM_PointerTo:$ptr, - LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, - AtomicOrdering:$success_ordering, - AtomicOrdering:$failure_ordering, - OptionalAttr:$syncscope, - OptionalAttr:$alignment, - UnitAttr:$weak, - UnitAttr:$volatile_); + "getValAndBoolStructType($_self)">]> { + dag args = (ins LLVM_PointerTo:$ptr, + LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val, + AtomicOrdering:$success_ordering, + AtomicOrdering:$failure_ordering, + OptionalAttr:$syncscope, + OptionalAttr:$alignment, + UnitAttr:$weak, + UnitAttr:$volatile_); + // Append the aliasing related attributes define in LLVM_MemAccessOpBase. + let arguments = !con(args, aliasAttrs); let results = (outs LLVM_AnyStruct:$res); let assemblyFormat = [{ (`weak` $weak^)? (`volatile` $volatile_^)? $ptr `,` $cmp `,` $val @@ -1828,7 +1786,9 @@ def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ inst->setWeak($weak); }] # setVolatileCode # setSyncScopeCode - # setAlignmentCode; + # setAlignmentCode + # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *cmpXchgInst = cast(inst); unsigned alignment = cmpXchgInst->getAlign().value(); @@ -1851,7 +1811,7 @@ def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [ let hasVerifier = 1; } -def LLVM_FenceOp : LLVM_Op<"fence">, MemoryOpBase { +def LLVM_FenceOp : LLVM_Op<"fence">, LLVM_MemOpPatterns { let arguments = (ins AtomicOrdering:$ordering, OptionalAttr:$syncscope); let assemblyFormat = "(`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict"; diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h index faca8fc5e4fb..90168a1e3011 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h @@ -14,6 +14,7 @@ #ifndef MLIR_TARGET_LLVMIR_MODULETRANSLATION_H #define MLIR_TARGET_LLVMIR_MODULETRANSLATION_H +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/IR/Operation.h" #include "mlir/IR/SymbolTable.h" #include "mlir/IR/Value.h" @@ -130,9 +131,8 @@ public: // Sets LLVM metadata for memory operations that have alias scope information. void setAliasScopeMetadata(Operation *op, llvm::Instruction *inst); - /// Sets LLVM TBAA metadata for memory operations that have - /// TBAA attributes. - void setTBAAMetadata(Operation *op, llvm::Instruction *inst); + /// Sets LLVM TBAA metadata for memory operations that have TBAA attributes. + void setTBAAMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst); /// Sets LLVM loop metadata for branch operations that have a loop annotation /// attribute. diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt index 55c172e2565b..ebfe0258e079 100644 --- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt +++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt @@ -4,6 +4,7 @@ add_mlir_dialect_library(MLIRLLVMDialect IR/FunctionCallUtils.cpp IR/LLVMAttrs.cpp IR/LLVMDialect.cpp + IR/LLVMInterfaces.cpp IR/LLVMTypes.cpp IR/LLVMTypeSyntax.cpp diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 0e04b240474b..c776fdd7ed5c 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "TypeDetail.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinOps.h" @@ -46,8 +47,6 @@ using mlir::LLVM::linkage::getMaxEnumValForLinkage; static constexpr const char kElemTypeAttrName[] = "elem_type"; -#include "mlir/Dialect/LLVMIR/LLVMInterfaces.cpp.inc" - static auto processFMFAttr(ArrayRef attrs) { SmallVector filteredAttrs( llvm::make_filter_range(attrs, [&](NamedAttribute attr) { @@ -715,95 +714,6 @@ Type LLVM::GEPOp::getSourceElementType() { // LoadOp //===----------------------------------------------------------------------===// -/// Verifies the given array attribute contains symbol references and checks the -/// referenced symbol types using the provided verification function. -LogicalResult verifyMemOpSymbolRefs( - Operation *op, StringRef name, ArrayAttr symbolRefs, - llvm::function_ref - verifySymbolType) { - assert(symbolRefs && "expected a non-null attribute"); - - // Verify that the attribute is a symbol ref array attribute, - // because this constraint is not verified for all attribute - // names processed here (e.g. 'tbaa'). This verification - // is redundant in some cases. - if (!llvm::all_of(symbolRefs, [](Attribute attr) { - return attr && attr.isa(); - })) - return op->emitOpError("attribute '") - << name - << "' failed to satisfy constraint: symbol ref array attribute"; - - for (SymbolRefAttr symbolRef : symbolRefs.getAsRange()) { - StringAttr metadataName = symbolRef.getRootReference(); - StringAttr symbolName = symbolRef.getLeafReference(); - // We want @metadata::@symbol, not just @symbol - if (metadataName == symbolName) { - return op->emitOpError() << "expected '" << symbolRef - << "' to specify a fully qualified reference"; - } - auto metadataOp = SymbolTable::lookupNearestSymbolFrom( - op->getParentOp(), metadataName); - if (!metadataOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to reference a metadata op"; - Operation *symbolOp = - SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); - if (!symbolOp) - return op->emitOpError() - << "expected '" << symbolRef << "' to be a valid reference"; - if (failed(verifySymbolType(symbolOp, symbolRef))) { - return failure(); - } - } - - return success(); -} - -/// Verifies the given array attribute contains symbol references that point to -/// metadata operations of the given type. -template -static LogicalResult -verifyMemOpSymbolRefsPointTo(Operation *op, StringRef name, - std::optional symbolRefs) { - if (!symbolRefs) - return success(); - - auto verifySymbolType = [op](Operation *symbolOp, - SymbolRefAttr symbolRef) -> LogicalResult { - if (!isa(symbolOp)) { - return op->emitOpError() - << "expected '" << symbolRef << "' to resolve to a " - << OpTy::getOperationName(); - } - return success(); - }; - return verifyMemOpSymbolRefs(op, name, *symbolRefs, verifySymbolType); -} - -/// Verifies the types of the metadata operations referenced by aliasing and -/// access group metadata. -template -LogicalResult verifyMemOpMetadata(OpTy memOp) { - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getAccessGroupsAttrName(), memOp.getAccessGroups()))) - return failure(); - - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getAliasScopesAttrName(), memOp.getAliasScopes()))) - return failure(); - - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getNoaliasScopesAttrName(), memOp.getNoaliasScopes()))) - return failure(); - - if (failed(verifyMemOpSymbolRefsPointTo( - memOp, memOp.getTbaaAttrName(), memOp.getTbaa()))) - return failure(); - - return success(); -} - /// Returns true if the given type is supported by atomic operations. All /// integer and float types with limited bit width are supported. Additionally, /// depending on the operation pointers may be supported as well. @@ -850,12 +760,8 @@ LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType, LogicalResult LoadOp::verify() { Type valueType = getResult().getType(); - if (failed(verifyAtomicMemOp( - *this, valueType, - {AtomicOrdering::release, AtomicOrdering::acq_rel}))) - return failure(); - - return verifyMemOpMetadata(*this); + return verifyAtomicMemOp(*this, valueType, + {AtomicOrdering::release, AtomicOrdering::acq_rel}); } void LoadOp::build(OpBuilder &builder, OperationState &state, Value addr, @@ -871,12 +777,12 @@ void LoadOp::build(OpBuilder &builder, OperationState &state, Type type, bool isNonTemporal, AtomicOrdering ordering, StringRef syncscope) { build(builder, state, type, addr, - /*access_groups=*/nullptr, - /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, - /*tbaa=*/nullptr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, isNonTemporal, ordering, - syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope), + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, + /*tbaa=*/nullptr); } // Extract the pointee type from the LLVM pointer type wrapped in MLIR. Return @@ -930,23 +836,20 @@ static void printLoadType(OpAsmPrinter &printer, Operation *op, Type type, LogicalResult StoreOp::verify() { Type valueType = getValue().getType(); - if (failed(verifyAtomicMemOp( - *this, valueType, - {AtomicOrdering::acquire, AtomicOrdering::acq_rel}))) - return failure(); - - return verifyMemOpMetadata(*this); + return verifyAtomicMemOp(*this, valueType, + {AtomicOrdering::acquire, AtomicOrdering::acq_rel}); } void StoreOp::build(OpBuilder &builder, OperationState &state, Value value, Value addr, unsigned alignment, bool isVolatile, bool isNonTemporal, AtomicOrdering ordering, StringRef syncscope) { - build(builder, state, value, addr, /*access_groups=*/nullptr, - /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, + build(builder, state, value, addr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, isNonTemporal, ordering, - syncscope.empty() ? nullptr : builder.getStringAttr(syncscope)); + syncscope.empty() ? nullptr : builder.getStringAttr(syncscope), + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); } /// Parses the StoreOp type either using the typed or opaque pointer format. @@ -2326,7 +2229,9 @@ void AtomicRMWOp::build(OpBuilder &builder, OperationState &state, unsigned alignment, bool isVolatile) { build(builder, state, val.getType(), binOp, ptr, val, ordering, !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, - alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile); + alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, + /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); } LogicalResult AtomicRMWOp::verify() { @@ -2378,7 +2283,8 @@ void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state, successOrdering, failureOrdering, !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak, - isVolatile); + isVolatile, /*access_groups=*/nullptr, + /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); } LogicalResult AtomicCmpXchgOp::verify() { @@ -2964,22 +2870,17 @@ struct LLVMInlinerInterface : public DialectInlinerInterface { bool isLegalToInline(Operation *op, Region *, bool, IRMapping &) const final { if (isPure(op)) return true; - return llvm::TypeSwitch(op) - .Case([&](auto memOp) { - // Some attributes on load and store operations require handling - // during inlining. Since this is not yet implemented, refuse to - // inline memory operations that have any of these attributes. - if (memOp.getAccessGroups()) - return false; - if (memOp.getAliasScopes()) - return false; - if (memOp.getNoaliasScopes()) - return false; - return true; - }) - .Case([](auto) { return true; }) - .Default([](auto) { return false; }); + // Some attributes on memory operations require handling during + // inlining. Since this is not yet implemented, refuse to inline memory + // operations that have any of these attributes. + if (auto iface = dyn_cast(op)) + if (iface.getAliasScopesOrNull() || iface.getNoAliasScopesOrNull()) + return false; + if (auto iface = dyn_cast(op)) + if (iface.getAccessGroupsOrNull()) + return false; + return isa(op); } /// Handle the given inlined return by replacing it with a branch. This diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp new file mode 100644 index 000000000000..95e0ea955367 --- /dev/null +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp @@ -0,0 +1,115 @@ +//===- LLVMInterfaces.cpp - LLVM Interfaces ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines op interfaces for the LLVM dialect in MLIR. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +using namespace mlir; +using namespace mlir::LLVM; + +/// Verifies the given array attribute contains symbol references and checks the +/// referenced symbol types using the provided verification function. +static LogicalResult +verifySymbolRefs(Operation *op, StringRef name, ArrayAttr symbolRefs, + llvm::function_ref + verifySymbolType) { + assert(symbolRefs && "expected a non-null attribute"); + + // Verify that the attribute is a symbol ref array attribute, + // because this constraint is not verified for all attribute + // names processed here (e.g. 'tbaa'). This verification + // is redundant in some cases. + if (!llvm::all_of(symbolRefs, [](Attribute attr) { + return attr && attr.isa(); + })) + return op->emitOpError() << name + << " attribute failed to satisfy constraint: " + "symbol ref array attribute"; + + for (SymbolRefAttr symbolRef : symbolRefs.getAsRange()) { + StringAttr metadataName = symbolRef.getRootReference(); + StringAttr symbolName = symbolRef.getLeafReference(); + // We want @metadata::@symbol, not just @symbol + if (metadataName == symbolName) { + return op->emitOpError() << "expected '" << symbolRef + << "' to specify a fully qualified reference"; + } + auto metadataOp = SymbolTable::lookupNearestSymbolFrom( + op->getParentOp(), metadataName); + if (!metadataOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to reference a metadata op"; + Operation *symbolOp = + SymbolTable::lookupNearestSymbolFrom(metadataOp, symbolName); + if (!symbolOp) + return op->emitOpError() + << "expected '" << symbolRef << "' to be a valid reference"; + if (failed(verifySymbolType(symbolOp, symbolRef))) { + return failure(); + } + } + + return success(); +} + +/// Verifies the given array attribute contains symbol references that point to +/// metadata operations of the given type. +template +LogicalResult verifySymbolRefsPointTo(Operation *op, StringRef name, + ArrayAttr symbolRefs) { + if (!symbolRefs) + return success(); + + auto verifySymbolType = [op](Operation *symbolOp, + SymbolRefAttr symbolRef) -> LogicalResult { + if (!isa(symbolOp)) { + return op->emitOpError() + << "expected '" << symbolRef << "' to resolve to a " + << OpTy::getOperationName(); + } + return success(); + }; + return verifySymbolRefs(op, name, symbolRefs, verifySymbolType); +} + +//===----------------------------------------------------------------------===// +// AccessGroupOpInterface +//===----------------------------------------------------------------------===// + +LogicalResult mlir::LLVM::detail::verifyAccessGroupOpInterface(Operation *op) { + auto iface = cast(op); + if (failed(verifySymbolRefsPointTo( + iface, "access groups", iface.getAccessGroupsOrNull()))) + return failure(); + return success(); +} + +//===----------------------------------------------------------------------===// +// AliasAnalysisOpInterface +//===----------------------------------------------------------------------===// + +LogicalResult +mlir::LLVM::detail::verifyAliasAnalysisOpInterface(Operation *op) { + auto iface = cast(op); + if (failed(verifySymbolRefsPointTo( + iface, "alias scopes", iface.getAliasScopesOrNull()))) + return failure(); + if (failed(verifySymbolRefsPointTo( + iface, "noalias scopes", iface.getNoAliasScopesOrNull()))) + return failure(); + if (failed(verifySymbolRefsPointTo( + iface, "tbaa tags", iface.getTBAATagsOrNull()))) + return failure(); + return success(); +} + +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.cpp.inc" diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp index ed9722bd8d9e..2a6093c12e75 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp @@ -12,6 +12,7 @@ #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Support/LLVM.h" #include "mlir/Target/LLVMIR/ModuleImport.h" @@ -77,30 +78,6 @@ static ArrayRef getSupportedMetadataImpl() { return convertibleMetadata; } -namespace { -/// Helper class to attach metadata attributes to specific operation types. It -/// specializes TypeSwitch to take an Operation and return a LogicalResult. -template -struct AttributeSetter { - AttributeSetter(Operation *op) : op(op) {} - - /// Calls `attachFn` on the provided Operation if it has one of - /// the given operation types. Returns failure otherwise. - template - LogicalResult apply(CallableT &&attachFn) { - return llvm::TypeSwitch(op) - .Case([&attachFn](auto concreteOp) { - attachFn(concreteOp); - return success(); - }) - .Default([&](auto) { return failure(); }); - } - -private: - Operation *op; -}; -} // namespace - /// Converts the given profiling metadata `node` to an MLIR profiling attribute /// and attaches it to the imported operation if the translation succeeds. /// Returns failure otherwise. @@ -148,11 +125,13 @@ static LogicalResult setProfilingAttr(OpBuilder &builder, llvm::MDNode *node, branchWeights.push_back(branchWeight->getZExtValue()); } - return AttributeSetter(op).apply( - [&](auto branchWeightOp) { + return TypeSwitch(op) + .Case([&](auto branchWeightOp) { branchWeightOp.setBranchWeightsAttr( builder.getI32VectorAttr(branchWeights)); - }); + return success(); + }) + .Default([](auto) { return failure(); }); } /// Searches the symbol reference pointing to the metadata operation that @@ -164,9 +143,12 @@ static LogicalResult setTBAAAttr(const llvm::MDNode *node, Operation *op, if (!tbaaTagSym) return failure(); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setTbaaAttr(ArrayAttr::get(memOp.getContext(), tbaaTagSym)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setTBAATags(ArrayAttr::get(iface.getContext(), tbaaTagSym)); + return success(); } /// Looks up all the symbol references pointing to the access group operations @@ -181,12 +163,14 @@ static LogicalResult setAccessGroupsAttr(const llvm::MDNode *node, if (failed(accessGroups)) return failure(); - SmallVector accessGroupAttrs(accessGroups->begin(), - accessGroups->end()); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setAccessGroupsAttr( - ArrayAttr::get(memOp.getContext(), accessGroupAttrs)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setAccessGroups(ArrayAttr::get( + iface.getContext(), + SmallVector{accessGroups->begin(), accessGroups->end()})); + return success(); } /// Converts the given loop metadata node to an MLIR loop annotation attribute @@ -218,12 +202,14 @@ static LogicalResult setAliasScopesAttr(const llvm::MDNode *node, Operation *op, if (failed(aliasScopes)) return failure(); - SmallVector aliasScopeAttrs(aliasScopes->begin(), - aliasScopes->end()); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setAliasScopesAttr( - ArrayAttr::get(memOp.getContext(), aliasScopeAttrs)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setAliasScopes(ArrayAttr::get( + iface.getContext(), + SmallVector{aliasScopes->begin(), aliasScopes->end()})); + return success(); } /// Looks up all the symbol references pointing to the alias scope operations @@ -233,17 +219,19 @@ static LogicalResult setAliasScopesAttr(const llvm::MDNode *node, Operation *op, static LogicalResult setNoaliasScopesAttr(const llvm::MDNode *node, Operation *op, LLVM::ModuleImport &moduleImport) { - FailureOr> noaliasScopes = + FailureOr> noAliasScopes = moduleImport.lookupAliasScopeAttrs(node); - if (failed(noaliasScopes)) + if (failed(noAliasScopes)) return failure(); - SmallVector noaliasScopeAttrs(noaliasScopes->begin(), - noaliasScopes->end()); - return AttributeSetter(op).apply([&](auto memOp) { - memOp.setNoaliasScopesAttr( - ArrayAttr::get(memOp.getContext(), noaliasScopeAttrs)); - }); + auto iface = dyn_cast(op); + if (!iface) + return failure(); + + iface.setNoAliasScopes(ArrayAttr::get( + iface.getContext(), + SmallVector{noAliasScopes->begin(), noAliasScopes->end()})); + return success(); } namespace { diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index 04eddde310cf..164cd1f3e2bb 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -18,6 +18,7 @@ #include "LoopAnnotationTranslation.h" #include "mlir/Dialect/DLTI/DLTI.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h" #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h" #include "mlir/Dialect/OpenMP/OpenMPDialect.h" #include "mlir/IR/Attributes.h" @@ -1091,31 +1092,25 @@ llvm::MDNode *ModuleTranslation::getTBAANode(Operation *op, return tbaaMetadataMapping.lookup(tagOp); } -void ModuleTranslation::setTBAAMetadata(Operation *op, +void ModuleTranslation::setTBAAMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst) { - auto populateTBAAMetadata = [&](std::optional tagRefs) { - if (!tagRefs || tagRefs->empty()) - return; - - // LLVM IR currently does not support attaching more than one - // TBAA access tag to a memory accessing instruction. - // It may be useful to support this in future, but for the time being - // just ignore the metadata if MLIR operation has multiple access tags. - if (tagRefs->size() > 1) { - op->emitWarning() << "TBAA access tags were not translated, because LLVM " - "IR only supports a single tag per instruction"; - return; - } + ArrayAttr tagRefs = op.getTBAATagsOrNull(); + if (!tagRefs || tagRefs.empty()) + return; - SymbolRefAttr tagRef = (*tagRefs)[0].cast(); - llvm::MDNode *node = getTBAANode(op, tagRef); - inst->setMetadata(llvm::LLVMContext::MD_tbaa, node); - }; + // LLVM IR currently does not support attaching more than one TBAA access tag + // to a memory accessing instruction. It may be useful to support this in + // future, but for the time being just ignore the metadata if MLIR operation + // has multiple access tags. + if (tagRefs.size() > 1) { + op.emitWarning() << "TBAA access tags were not translated, because LLVM " + "IR only supports a single tag per instruction"; + return; + } - llvm::TypeSwitch(op) - .Case( - [&](auto memOp) { populateTBAAMetadata(memOp.getTbaa()); }) - .Default([](auto) { llvm_unreachable("expected LoadOp or StoreOp"); }); + SymbolRefAttr tagRef = tagRefs[0].cast(); + llvm::MDNode *node = getTBAANode(op, tagRef); + inst->setMetadata(llvm::LLVMContext::MD_tbaa, node); } LogicalResult ModuleTranslation::createTBAAMetadata() { diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir index d063e9b7a5c2..3e019144a199 100644 --- a/mlir/test/Dialect/LLVMIR/invalid.mlir +++ b/mlir/test/Dialect/LLVMIR/invalid.mlir @@ -938,9 +938,9 @@ module { // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : !llvm.ptr) { // expected-error@below {{expected '@func1' to specify a fully qualified reference}} - %0 = llvm.load %arg0 { "access_groups" = [@func1] } : !llvm.ptr + %0 = llvm.load %arg0 { "access_groups" = [@func1] } : !llvm.ptr -> i32 llvm.return } llvm.func @func1() { @@ -951,9 +951,9 @@ module { // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : i32, %arg1 : !llvm.ptr) { // expected-error@below {{expected '@accessGroups::@group1' to reference a metadata op}} - %0 = llvm.load %arg0 { "access_groups" = [@accessGroups::@group1] } : !llvm.ptr + llvm.store %arg0, %arg1 { "access_groups" = [@accessGroups::@group1] } : i32, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -963,9 +963,9 @@ module { // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : !llvm.ptr, %arg1 : f32) { // expected-error@below {{expected '@metadata::@group1' to be a valid reference}} - %0 = llvm.load %arg0 { "access_groups" = [@metadata::@group1] } : !llvm.ptr + %0 = llvm.atomicrmw fadd %arg0, %arg1 monotonic { "access_groups" = [@metadata::@group1] } : !llvm.ptr, f32 llvm.return } llvm.metadata @metadata { @@ -975,9 +975,9 @@ module { // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @accessGroups(%arg0 : !llvm.ptr, %arg1 : i32, %arg2 : i32) { // expected-error@below {{expected '@metadata::@scope' to resolve to a llvm.access_group}} - %0 = llvm.load %arg0 { "access_groups" = [@metadata::@scope] } : !llvm.ptr + %0 = llvm.cmpxchg %arg0, %arg1, %arg2 acq_rel monotonic { "access_groups" = [@metadata::@scope] } : !llvm.ptr, i32 llvm.return } llvm.metadata @metadata { @@ -989,9 +989,9 @@ module { // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @aliasScope(%arg0 : !llvm.ptr, %arg1 : i32, %arg2 : i32) { // expected-error@below {{attribute 'alias_scopes' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "alias_scopes" = "test" } : !llvm.ptr + %0 = llvm.cmpxchg %arg0, %arg1, %arg2 acq_rel monotonic { "alias_scopes" = "test" } : !llvm.ptr, i32 llvm.return } } @@ -999,9 +999,9 @@ module { // ----- module { - llvm.func @accessGroups(%arg0 : !llvm.ptr) { + llvm.func @noAliasScopes(%arg0 : !llvm.ptr) { // expected-error@below {{attribute 'noalias_scopes' failed to satisfy constraint: symbol ref array attribute}} - %0 = llvm.load %arg0 { "noalias_scopes" = "test" } : !llvm.ptr + %0 = llvm.load %arg0 { "noalias_scopes" = "test" } : !llvm.ptr -> i32 llvm.return } } @@ -1009,9 +1009,9 @@ module { // ----- module { - llvm.func @aliasScope(%arg0 : !llvm.ptr) { + llvm.func @aliasScope(%arg0 : i32, %arg1 : !llvm.ptr) { // expected-error@below {{expected '@metadata::@group' to resolve to a llvm.alias_scope}} - %0 = llvm.load %arg0 { "alias_scopes" = [@metadata::@group] } : !llvm.ptr + llvm.store %arg0, %arg1 { "alias_scopes" = [@metadata::@group] } : i32, !llvm.ptr llvm.return } llvm.metadata @metadata { @@ -1022,9 +1022,9 @@ module { // ----- module { - llvm.func @aliasScope(%arg0 : !llvm.ptr) { + llvm.func @aliasScope(%arg0 : !llvm.ptr, %arg1 : f32) { // expected-error@below {{expected '@metadata::@group' to resolve to a llvm.alias_scope}} - %0 = llvm.load %arg0 { "noalias_scopes" = [@metadata::@group] } : !llvm.ptr + %0 = llvm.atomicrmw fadd %arg0, %arg1 monotonic { "noalias_scopes" = [@metadata::@group] } : !llvm.ptr, f32 llvm.return } llvm.metadata @metadata { diff --git a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll index 9db045dde27a..8eec16884e3f 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-alias-scopes.ll @@ -59,3 +59,23 @@ define void @two_domains(ptr %arg1) { !3 = !{!3, !1} !4 = !{!2} !5 = !{!3} + +; // ----- + +; CHECK-LABEL: llvm.func @supported_ops +define void @supported_ops(ptr %arg1, float %arg2, i32 %arg3, i32 %arg4) { + ; CHECK: llvm.load {{.*}}alias_scopes = + %1 = load i32, ptr %arg1, !alias.scope !3 + ; CHECK: llvm.store {{.*}}alias_scopes = + store i32 %1, ptr %arg1, !alias.scope !3 + ; CHECK: llvm.atomicrmw {{.*}}alias_scopes = + %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !alias.scope !3 + ; CHECK: llvm.cmpxchg {{.*}}alias_scopes = + %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !alias.scope !3 + ret void +} + +!0 = distinct !{!0, !"The domain"} +!1 = distinct !{!1} +!2 = !{!2, !0} +!3 = !{!2} diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll index ff2d9c84fc13..eee7f6f55de7 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -27,6 +27,25 @@ define void @access_group(ptr %arg1) { ; // ----- +; CHECK-LABEL: llvm.func @supported_ops +define void @supported_ops(ptr %arg1, float %arg2, i32 %arg3, i32 %arg4) { + ; CHECK: llvm.load {{.*}}access_groups = + %1 = load i32, ptr %arg1, !llvm.access.group !0 + ; CHECK: llvm.store {{.*}}access_groups = + store i32 %1, ptr %arg1, !llvm.access.group !0 + ; CHECK: llvm.atomicrmw {{.*}}access_groups = + %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !llvm.access.group !0 + ; CHECK: llvm.cmpxchg {{.*}}access_groups = + %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !llvm.access.group !0 + ret void +} + +!0 = !{!1, !2} +!1 = distinct !{} +!2 = distinct !{} + +; // ----- + ; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation ; CHECK-LABEL: @simple diff --git a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll index 9e3b7c2ca826..87865195ac45 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-tbaa.ll @@ -29,7 +29,7 @@ define dso_local void @tbaa1(ptr %0, ptr %1) { !4 = !{!"other scalar type", !5, i64 0} !5 = !{!"Other language TBAA"} -// ----- +; // ----- ; CHECK-LABEL: llvm.metadata @__llvm_global_metadata { ; CHECK-NEXT: llvm.tbaa_root @[[R0:tbaa_root_[0-9]+]] {id = "Simple C/C++ TBAA"} @@ -68,3 +68,22 @@ define dso_local void @tbaa2(ptr %0, ptr %1) { !11 = !{!12, !13, i64 0} !12 = !{!"agg1_t", !13, i64 0, !13, i64 4} !13 = !{!"int", !9, i64 0} + +; // ----- + +; CHECK-LABEL: llvm.func @supported_ops +define void @supported_ops(ptr %arg1, float %arg2, i32 %arg3, i32 %arg4) { + ; CHECK: llvm.load {{.*}}tbaa = + %1 = load i32, ptr %arg1, !tbaa !0 + ; CHECK: llvm.store {{.*}}tbaa = + store i32 %1, ptr %arg1, !tbaa !0 + ; CHECK: llvm.atomicrmw {{.*}}tbaa = + %2 = atomicrmw fmax ptr %arg1, float %arg2 acquire, !tbaa !0 + ; CHECK: llvm.cmpxchg {{.*}}tbaa = + %3 = cmpxchg ptr %arg1, i32 %arg3, i32 %arg4 monotonic seq_cst, !tbaa !0 + ret void +} + +!0 = !{!1, !1, i64 0} +!1 = !{!"scalar type", !2, i64 0} +!2 = !{!"Simple C/C++ TBAA"} diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index f897429e563e..147d210fcfe9 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1975,27 +1975,29 @@ llvm.func @switch_weights(%arg0: i32) -> i32 { // ----- module { - llvm.func @aliasScope(%arg1 : !llvm.ptr, %arg2 : !llvm.ptr, %arg3 : !llvm.ptr) { + llvm.func @aliasScope(%arg1 : !llvm.ptr) { %0 = llvm.mlir.constant(0 : i32) : i32 - llvm.store %0, %arg1 { alias_scopes = [@metadata::@scope1], noalias_scopes = [@metadata::@scope2, @metadata::@scope3] } : !llvm.ptr - llvm.store %0, %arg2 { alias_scopes = [@metadata::@scope2], noalias_scopes = [@metadata::@scope1, @metadata::@scope3] } : !llvm.ptr - %1 = llvm.load %arg3 { alias_scopes = [@metadata::@scope3], noalias_scopes = [@metadata::@scope1, @metadata::@scope2] } : !llvm.ptr + llvm.store %0, %arg1 {alias_scopes = [@metadata::@scope1], noalias_scopes = [@metadata::@scope2, @metadata::@scope3]} : i32, !llvm.ptr + %1 = llvm.load %arg1 {alias_scopes = [@metadata::@scope2], noalias_scopes = [@metadata::@scope1, @metadata::@scope3]} : !llvm.ptr -> i32 + %2 = llvm.atomicrmw add %arg1, %0 monotonic {alias_scopes = [@metadata::@scope3], noalias_scopes = [@metadata::@scope1, @metadata::@scope2]} : !llvm.ptr, i32 + %3 = llvm.cmpxchg %arg1, %1, %2 acq_rel monotonic {alias_scopes = [@metadata::@scope3]} : !llvm.ptr, i32 llvm.return } llvm.metadata @metadata { - llvm.alias_scope_domain @domain { description = "The domain"} - llvm.alias_scope @scope1 { domain = @domain, description = "The first scope" } - llvm.alias_scope @scope2 { domain = @domain } - llvm.alias_scope @scope3 { domain = @domain } + llvm.alias_scope_domain @domain {description = "The domain"} + llvm.alias_scope @scope1 {domain = @domain, description = "The first scope"} + llvm.alias_scope @scope2 {domain = @domain} + llvm.alias_scope @scope3 {domain = @domain} } } // Function // CHECK-LABEL: aliasScope // CHECK: store {{.*}}, !alias.scope ![[SCOPES1:[0-9]+]], !noalias ![[SCOPES23:[0-9]+]] -// CHECK: store {{.*}}, !alias.scope ![[SCOPES2:[0-9]+]], !noalias ![[SCOPES13:[0-9]+]] -// CHECK: load {{.*}}, !alias.scope ![[SCOPES3:[0-9]+]], !noalias ![[SCOPES12:[0-9]+]] +// CHECK: load {{.*}}, !alias.scope ![[SCOPES2:[0-9]+]], !noalias ![[SCOPES13:[0-9]+]] +// CHECK: atomicrmw {{.*}}, !alias.scope ![[SCOPES3:[0-9]+]], !noalias ![[SCOPES12:[0-9]+]] +// CHECK: cmpxchg {{.*}}, !alias.scope ![[SCOPES3]] // Metadata // CHECK-DAG: ![[DOMAIN:[0-9]+]] = distinct !{![[DOMAIN]], !"The domain"} diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir index c0ea4b7812c4..178633977dbf 100644 --- a/mlir/test/Target/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -45,7 +45,7 @@ llvm.func @isvectorized() { llvm.func @vectorizeOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation >} ^bb1: @@ -111,7 +111,7 @@ llvm.func @unrollOptions2() { llvm.return } -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}} // CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unroll.enable"} // CHECK-DAG: ![[VEC_NODE2:[0-9]+]] = !{!"llvm.loop.unroll.full"} @@ -236,7 +236,7 @@ llvm.func @unswitchOptions() { // CHECK-LABEL: @loopOptions llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { %0 = llvm.mlir.constant(0 : i32) : i32 - %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) + %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr) llvm.br ^bb3(%0 : i32) ^bb3(%1: i32): %2 = llvm.icmp "slt" %1, %arg1 : i32 @@ -249,7 +249,13 @@ llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) { ^bb4: %3 = llvm.add %1, %arg2 : i32 // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]] - %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr + %5 = llvm.load %4 {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr -> i32 + // CHECK: store i32 %{{.*}}, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + llvm.store %5, %4 {access_groups = [@metadata::@group1, @metadata::@group2]} : i32, !llvm.ptr + // CHECK: = atomicrmw add ptr %{{.*}}, i32 %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + %6 = llvm.atomicrmw add %4, %5 monotonic {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr, i32 + // CHECK: = cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE]] + %7 = llvm.cmpxchg %4, %5, %6 acq_rel monotonic {access_groups = [@metadata::@group1, @metadata::@group2]} : !llvm.ptr, i32 // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]] llvm.br ^bb3(%3 : i32) {loop_annotation = #llvm.loop_annotation< licm = , @@ -266,7 +272,7 @@ llvm.metadata @metadata { llvm.access_group @group2 } -// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} // CHECK-DAG: ![[PA_NODE:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]} // CHECK-DAG: ![[GROUP_NODE1:[0-9]+]] = distinct !{} // CHECK-DAG: ![[GROUP_NODE2:[0-9]+]] = distinct !{} diff --git a/mlir/test/Target/LLVMIR/tbaa.mlir b/mlir/test/Target/LLVMIR/tbaa.mlir index 84f27be92326..4004066090b6 100644 --- a/mlir/test/Target/LLVMIR/tbaa.mlir +++ b/mlir/test/Target/LLVMIR/tbaa.mlir @@ -65,6 +65,10 @@ module { %5 = llvm.getelementptr inbounds %arg0[%0, 0] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.struct<"struct.agg1_t", (i32, i32)> // CHECK: store i32 %{{.*}}, ptr %{{.*}},{{.*}}!tbaa ![[STAG:[0-9]*]] llvm.store %4, %5 {tbaa = [@__tbaa::@tbaa_tag_7]} : i32, !llvm.ptr + // CHECK: atomicrmw add ptr %{{.*}}, i32 %{{.*}} !tbaa ![[STAG]] + %6 = llvm.atomicrmw add %5, %4 monotonic {tbaa = [@__tbaa::@tbaa_tag_7]} : !llvm.ptr, i32 + // CHECK: cmpxchg ptr %{{.*}}, i32 %{{.*}}, i32 %{{.*}} !tbaa ![[STAG]] + %7 = llvm.cmpxchg %5, %6, %4 acq_rel monotonic {tbaa = [@__tbaa::@tbaa_tag_7]} : !llvm.ptr, i32 llvm.return } }