def LLVM_FNegOp : LLVM_UnaryFloatArithmeticOp<
LLVM_ScalarOrVectorOf<LLVM_AnyFloat>, "fneg", "FNeg">;
-// Common code definition that is used to verify and set the alignment attribute
-// of LLVM ops that accept such an attribute.
-class MemoryOpWithAlignmentBase {
+// Common code definitions used to set memory operation attributes and flags.
+class MemoryOpBase {
code setAlignmentCode = [{
if ($alignment.has_value()) {
auto align = *$alignment;
inst->setAlignment(llvm::Align(align));
}
}];
-}
-
-// Code definition that is used for nontemporal metadata creation.
-class MemoryOpWithAlignmentAndAttributes : MemoryOpWithAlignmentBase {
+ code setVolatileCode = [{
+ inst->setVolatile($volatile_);
+ }];
+ code setSyncScopeCode = [{
+ if ($syncscope.has_value()) {
+ llvm::LLVMContext &llvmContext = builder.getContext();
+ inst->setSyncScopeID(llvmContext.getOrInsertSyncScopeID(*$syncscope));
+ }
+ }];
code setNonTemporalMetadataCode = [{
if ($nontemporal) {
llvm::Module *module = builder.GetInsertBlock()->getModule();
inst->setMetadata(module->getMDKindID("nontemporal"), metadata);
}
}];
-
code setAccessGroupsMetadataCode = [{
moduleTranslation.setAccessGroupsMetadata(op, inst);
}];
-
code setAliasScopeMetadataCode = [{
moduleTranslation.setAliasScopeMetadata(op, inst);
}];
-
code setTBAAMetadataCode = [{
moduleTranslation.setTBAAMetadata(op, inst);
}];
}
// Memory-related operations.
-def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpWithAlignmentBase {
+def LLVM_AllocaOp : LLVM_Op<"alloca">, MemoryOpBase {
let arguments = (ins AnyInteger:$arraySize,
OptionalAttr<I64Attr>:$alignment,
OptionalAttr<TypeAttr>:$elem_type);
let hasVerifier = 1;
}
-def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpWithAlignmentAndAttributes {
+def LLVM_LoadOp : LLVM_Op<"load">, MemoryOpBase {
let arguments = (ins Arg<LLVM_PointerTo<LLVM_LoadableType>, "", [MemRead]>:$addr,
OptionalAttr<SymbolRefArrayAttr>:$access_groups,
OptionalAttr<SymbolRefArrayAttr>:$alias_scopes,
let hasVerifier = 1;
}
-def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpWithAlignmentAndAttributes {
+def LLVM_StoreOp : LLVM_Op<"store">, MemoryOpBase {
let arguments = (ins LLVM_LoadableType:$value,
Arg<LLVM_PointerTo<LLVM_LoadableType>,"",[MemWrite]>:$addr,
OptionalAttr<SymbolRefArrayAttr>:$access_groups,
def LLVM_AtomicRMWType : AnyTypeOf<[LLVM_AnyFloat, AnyInteger]>;
-// FIXME: Need to add alignment and syncscope attribute to MLIR atomicrmw
-// operation.
def LLVM_AtomicRMWOp : LLVM_Op<"atomicrmw", [
TypesMatchWith<"result #0 and operand #1 have the same type",
- "val", "res", "$_self">]> {
+ "val", "res", "$_self">]>, MemoryOpBase {
let arguments = (ins AtomicBinOp:$bin_op,
LLVM_PointerTo<LLVM_AtomicRMWType>:$ptr,
- LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering);
+ LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering,
+ OptionalAttr<StrAttr>:$syncscope,
+ OptionalAttr<I64Attr>:$alignment,
+ UnitAttr:$volatile_);
let results = (outs LLVM_AtomicRMWType:$res);
- let hasVerifier = 1;
let assemblyFormat = [{
- $bin_op $ptr `,` $val $ordering
- attr-dict `:` qualified(type($ptr)) `,` type($val)
+ (`volatile` $volatile_^)? $bin_op $ptr `,` $val
+ (`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict `:`
+ qualified(type($ptr)) `,` type($val)
}];
string llvmInstName = "AtomicRMW";
string llvmBuilder = [{
- $res = builder.CreateAtomicRMW(
+ auto *inst = builder.CreateAtomicRMW(
convertAtomicBinOpToLLVM($bin_op), $ptr, $val, llvm::MaybeAlign(),
convertAtomicOrderingToLLVM($ordering));
- }];
+ $res = inst;
+ }] # setVolatileCode
+ # setSyncScopeCode
+ # setAlignmentCode;
string mlirBuilder = [{
auto *atomicInst = cast<llvm::AtomicRMWInst>(inst);
- $res = $_builder.create<LLVM::AtomicRMWOp>($_location, $_resultType,
+ unsigned alignment = atomicInst->getAlign().value();
+ $res = $_builder.create<LLVM::AtomicRMWOp>($_location,
convertAtomicBinOpFromLLVM(atomicInst->getOperation()), $ptr, $val,
- convertAtomicOrderingFromLLVM(atomicInst->getOrdering()));
+ convertAtomicOrderingFromLLVM(atomicInst->getOrdering()),
+ getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile());
}];
- // Only $ptr and $val are llvm instruction operands.
- list<int> llvmArgIndices = [-1, 0, 1, -1];
+ list<int> llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1];
+ let builders = [
+ OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val,
+ "LLVM::AtomicOrdering":$ordering,
+ CArg<"StringRef", "StringRef()">:$syncscope,
+ CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile
+ )>
+ ];
+ let hasVerifier = 1;
}
def LLVM_AtomicCmpXchgType : AnyTypeOf<[AnyInteger, LLVM_AnyPointer]>;
-// FIXME: Need to add alignment attribute to MLIR cmpxchg operation.
def LLVM_AtomicCmpXchgOp : LLVM_Op<"cmpxchg", [
TypesMatchWith<"operand #1 and operand #2 have the same type",
"val", "cmp", "$_self">,
TypesMatchWith<"result #0 has an LLVM struct type consisting of "
- "the type of operand #2 and a bool",
- "val", "res", "getValAndBoolStructType($_self)">]> {
+ "the type of operand #2 and a bool", "val", "res",
+ "getValAndBoolStructType($_self)">]>, MemoryOpBase {
let arguments = (ins LLVM_PointerTo<LLVM_AtomicCmpXchgType>:$ptr,
LLVM_AtomicCmpXchgType:$cmp, LLVM_AtomicCmpXchgType:$val,
AtomicOrdering:$success_ordering,
- AtomicOrdering:$failure_ordering);
+ AtomicOrdering:$failure_ordering,
+ OptionalAttr<StrAttr>:$syncscope,
+ OptionalAttr<I64Attr>:$alignment,
+ UnitAttr:$weak,
+ UnitAttr:$volatile_);
let results = (outs LLVM_AnyStruct:$res);
- let hasVerifier = 1;
let assemblyFormat = [{
- $ptr `,` $cmp `,` $val $success_ordering $failure_ordering
+ (`weak` $weak^)? (`volatile` $volatile_^)? $ptr `,` $cmp `,` $val
+ (`syncscope` `(` $syncscope^ `)`)? $success_ordering $failure_ordering
attr-dict `:` qualified(type($ptr)) `,` type($val)
}];
string llvmInstName = "AtomicCmpXchg";
string llvmBuilder = [{
- $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(),
- convertAtomicOrderingToLLVM($success_ordering),
- convertAtomicOrderingToLLVM($failure_ordering));
- }];
+ auto *inst = builder.CreateAtomicCmpXchg($ptr, $cmp, $val,
+ llvm::MaybeAlign(), convertAtomicOrderingToLLVM($success_ordering),
+ convertAtomicOrderingToLLVM($failure_ordering));
+ $res = inst;
+ inst->setWeak($weak);
+ }] # setVolatileCode
+ # setSyncScopeCode
+ # setAlignmentCode;
string mlirBuilder = [{
auto *cmpXchgInst = cast<llvm::AtomicCmpXchgInst>(inst);
+ unsigned alignment = cmpXchgInst->getAlign().value();
$res = $_builder.create<LLVM::AtomicCmpXchgOp>(
- $_location, $_resultType, $ptr, $cmp, $val,
+ $_location, $ptr, $cmp, $val,
convertAtomicOrderingFromLLVM(cmpXchgInst->getSuccessOrdering()),
- convertAtomicOrderingFromLLVM(cmpXchgInst->getFailureOrdering()));
+ convertAtomicOrderingFromLLVM(cmpXchgInst->getFailureOrdering()),
+ getLLVMSyncScope(cmpXchgInst), alignment, cmpXchgInst->isWeak(),
+ cmpXchgInst->isVolatile());
}];
+ let builders = [
+ OpBuilder<(ins "Value":$ptr, "Value":$cmp, "Value":$val,
+ "LLVM::AtomicOrdering":$successOrdering,
+ "LLVM::AtomicOrdering":$failureOrdering,
+ CArg<"StringRef", "StringRef()">:$syncscope,
+ CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isWeak,
+ CArg<"bool", "false">:$isVolatile
+ )>
+ ];
+ let hasVerifier = 1;
}
-def LLVM_FenceOp : LLVM_Op<"fence"> {
- let arguments = (ins AtomicOrdering:$ordering, StrAttr:$syncscope);
- let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+def LLVM_FenceOp : LLVM_Op<"fence">, MemoryOpBase {
+ let arguments = (ins AtomicOrdering:$ordering,
+ OptionalAttr<StrAttr>:$syncscope);
+ let assemblyFormat = "(`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict";
string llvmInstName = "Fence";
let llvmBuilder = [{
- llvm::LLVMContext &llvmContext = builder.getContext();
- builder.CreateFence(convertAtomicOrderingToLLVM($ordering),
- llvmContext.getOrInsertSyncScopeID($syncscope));
- }];
+ auto *inst = builder.CreateFence(convertAtomicOrderingToLLVM($ordering));
+ }] # setSyncScopeCode;
string mlirBuilder = [{
llvm::FenceInst *fenceInst = cast<llvm::FenceInst>(inst);
$_op = $_builder.create<LLVM::FenceOp>(
convertAtomicOrderingFromLLVM(fenceInst->getOrdering()),
getLLVMSyncScope(fenceInst));
}];
- let hasCustomAssemblyFormat = 1;
+ let builders = [
+ LLVM_VoidResultTypeOpBuilder,
+ LLVM_ZeroResultOpBuilder,
+ OpBuilder<(ins "LLVM::AtomicOrdering":$ordering,
+ CArg<"StringRef", "StringRef()">:$syncscope)>
+ ];
let hasVerifier = 1;
}
// Append the cmpxchg op to the end of the loop block.
auto successOrdering = LLVM::AtomicOrdering::acq_rel;
auto failureOrdering = LLVM::AtomicOrdering::monotonic;
- auto boolType = IntegerType::get(rewriter.getContext(), 1);
- auto pairType = LLVM::LLVMStructType::getLiteral(rewriter.getContext(),
- {valueType, boolType});
auto cmpxchg = rewriter.create<LLVM::AtomicCmpXchgOp>(
- loc, pairType, dataPtr, loopArgument, result, successOrdering,
- failureOrdering);
+ loc, dataPtr, loopArgument, result, successOrdering, failureOrdering);
// Extract the %new_loaded and %ok values from the pair.
Value newLoaded = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 0);
Value ok = rewriter.create<LLVM::ExtractValueOp>(loc, cmpxchg, 1);
auto maybeKind = matchSimpleAtomicOp(atomicOp);
if (!maybeKind)
return failure();
- auto resultType = adaptor.getValue().getType();
auto memRefType = atomicOp.getMemRefType();
auto dataPtr =
getStridedElementPtr(atomicOp.getLoc(), memRefType, adaptor.getMemref(),
adaptor.getIndices(), rewriter);
rewriter.replaceOpWithNewOp<LLVM::AtomicRMWOp>(
- atomicOp, resultType, *maybeKind, dataPtr, adaptor.getValue(),
+ atomicOp, *maybeKind, dataPtr, adaptor.getValue(),
LLVM::AtomicOrdering::acq_rel);
return success();
}
builder.setInsertionPointToEnd(atomicBlock);
Value loaded = builder.create<LLVM::LoadOp>(reduce.getLoc(),
atomicBlock->getArgument(1));
- builder.create<LLVM::AtomicRMWOp>(reduce.getLoc(), type, atomicKind,
+ builder.create<LLVM::AtomicRMWOp>(reduce.getLoc(), atomicKind,
atomicBlock->getArgument(0), loaded,
LLVM::AtomicOrdering::monotonic);
builder.create<omp::YieldOp>(reduce.getLoc(), ArrayRef<Value>());
OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
//===----------------------------------------------------------------------===//
-// Utility functions for parsing atomic ops
-//===----------------------------------------------------------------------===//
-
-// Helper function to parse a keyword into the specified attribute named by
-// `attrName`. The keyword must match one of the string values defined by the
-// AtomicOrdering enum. The resulting I64 attribute is added to the `result`
-// state.
-static ParseResult parseAtomicOrdering(OpAsmParser &parser,
- OperationState &result,
- StringRef attrName) {
- SMLoc loc;
- StringRef ordering;
- if (parser.getCurrentLocation(&loc) || parser.parseKeyword(&ordering))
- return failure();
-
- // Replace the keyword `ordering` with an integer attribute.
- auto kind = symbolizeAtomicOrdering(ordering);
- if (!kind) {
- return parser.emitError(loc)
- << "'" << ordering << "' is an incorrect value of the '" << attrName
- << "' attribute";
- }
-
- auto value = static_cast<int64_t>(*kind);
- auto attr = parser.getBuilder().getI64IntegerAttr(value);
- result.addAttribute(attrName, attr);
+// AtomicRMWOp
+//===----------------------------------------------------------------------===//
- return success();
+void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
+ AtomicBinOp binOp, Value ptr, Value val,
+ AtomicOrdering ordering, StringRef syncscope,
+ unsigned alignment, bool isVolatile) {
+ build(builder, state, val.getType(), binOp, ptr, val, ordering,
+ !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
+ alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile);
}
-//===----------------------------------------------------------------------===//
-// Verifier for LLVM::AtomicRMWOp.
-//===----------------------------------------------------------------------===//
-
LogicalResult AtomicRMWOp::verify() {
auto ptrType = getPtr().getType().cast<LLVM::LLVMPointerType>();
auto valType = getVal().getType();
}
//===----------------------------------------------------------------------===//
-// Verifier for LLVM::AtomicCmpXchgOp.
+// AtomicCmpXchgOp
//===----------------------------------------------------------------------===//
/// Returns an LLVM struct type that contains a value type and a boolean type.
return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType});
}
+void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state,
+ Value ptr, Value cmp, Value val,
+ AtomicOrdering successOrdering,
+ AtomicOrdering failureOrdering, StringRef syncscope,
+ unsigned alignment, bool isWeak, bool isVolatile) {
+ build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val,
+ successOrdering, failureOrdering,
+ !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
+ alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak,
+ isVolatile);
+}
+
LogicalResult AtomicCmpXchgOp::verify() {
auto ptrType = getPtr().getType().cast<LLVM::LLVMPointerType>();
if (!ptrType)
}
//===----------------------------------------------------------------------===//
-// Printer, parser and verifier for LLVM::FenceOp.
+// FenceOp
//===----------------------------------------------------------------------===//
-// <operation> ::= `llvm.fence` (`syncscope(`strAttr`)`)? keyword
-// attribute-dict?
-ParseResult FenceOp::parse(OpAsmParser &parser, OperationState &result) {
- StringAttr sScope;
- StringRef syncscopeKeyword = "syncscope";
- if (!failed(parser.parseOptionalKeyword(syncscopeKeyword))) {
- if (parser.parseLParen() ||
- parser.parseAttribute(sScope, syncscopeKeyword, result.attributes) ||
- parser.parseRParen())
- return failure();
- } else {
- result.addAttribute(syncscopeKeyword,
- parser.getBuilder().getStringAttr(""));
- }
- if (parseAtomicOrdering(parser, result, "ordering") ||
- parser.parseOptionalAttrDict(result.attributes))
- return failure();
- return success();
-}
-
-void FenceOp::print(OpAsmPrinter &p) {
- StringRef syncscopeKeyword = "syncscope";
- p << ' ';
- if (!(*this)->getAttr(syncscopeKeyword).cast<StringAttr>().getValue().empty())
- p << "syncscope(" << (*this)->getAttr(syncscopeKeyword) << ") ";
- p << stringifyAtomicOrdering(getOrdering());
+void FenceOp::build(OpBuilder &builder, OperationState &state,
+ AtomicOrdering ordering, StringRef syncscope) {
+ build(builder, state, ordering,
+ syncscope.empty() ? nullptr : builder.getStringAttr(syncscope));
}
LogicalResult FenceOp::verify() {
}
}
-/// Converts the sync scope identifier of `fenceInst` to the string
-/// representation necessary to build the LLVM dialect fence operation.
-static StringRef getLLVMSyncScope(llvm::FenceInst *fenceInst) {
- llvm::LLVMContext &llvmContext = fenceInst->getContext();
- SmallVector<StringRef> syncScopeNames;
- llvmContext.getSyncScopeNames(syncScopeNames);
- for (StringRef name : syncScopeNames)
- if (fenceInst->getSyncScopeID() == llvmContext.getOrInsertSyncScopeID(name))
- return name;
+/// Converts the sync scope identifier of `inst` to the string representation
+/// necessary to build an atomic LLVM dialect operation. Returns the empty
+/// string if the operation has either no sync scope or the default system-level
+/// sync scope attached. The atomic operations only set their sync scope
+/// attribute if they have a non-default sync scope attached.
+static StringRef getLLVMSyncScope(llvm::Instruction *inst) {
+ std::optional<llvm::SyncScope::ID> syncScopeID =
+ llvm::getAtomicSyncScopeID(inst);
+ if (!syncScopeID)
+ return "";
+
+ // Search the sync scope name for the given identifier. The default
+ // system-level sync scope thereby maps to the empty string.
+ SmallVector<StringRef> syncScopeName;
+ llvm::LLVMContext &llvmContext = inst->getContext();
+ llvmContext.getSyncScopeNames(syncScopeName);
+ auto *it = llvm::find_if(syncScopeName, [&](StringRef name) {
+ return *syncScopeID == llvmContext.getOrInsertSyncScopeID(name);
+ });
+ if (it != syncScopeName.end())
+ return *it;
llvm_unreachable("incorrect sync scope identifier");
}
func.func @atomicrmw(%ptr : !llvm.ptr, %val : f32) {
// CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} monotonic : !llvm.ptr, f32
%0 = llvm.atomicrmw fadd %ptr, %val monotonic : !llvm.ptr, f32
+ // CHECK: llvm.atomicrmw volatile fsub %{{.*}}, %{{.*}} syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32
+ %1 = llvm.atomicrmw volatile fsub %ptr, %val syncscope("singlethread") monotonic {alignment = 16 : i64} : !llvm.ptr, f32
llvm.return
}
func.func @cmpxchg(%ptr : !llvm.ptr, %cmp : i32, %new : i32) {
// CHECK: llvm.cmpxchg %{{.*}}, %{{.*}}, %{{.*}} acq_rel monotonic : !llvm.ptr, i32
%0 = llvm.cmpxchg %ptr, %cmp, %new acq_rel monotonic : !llvm.ptr, i32
+ // CHECK: llvm.cmpxchg weak volatile %{{.*}}, %{{.*}}, %{{.*}} syncscope("singlethread") acq_rel monotonic {alignment = 16 : i64} : !llvm.ptr, i32
+ %1 = llvm.cmpxchg weak volatile %ptr, %cmp, %new syncscope("singlethread") acq_rel monotonic {alignment = 16 : i64} : !llvm.ptr, i32
llvm.return
}
%16 = atomicrmw uinc_wrap ptr %ptr1, i32 %val1 acquire
; CHECK: llvm.atomicrmw udec_wrap %[[PTR1]], %[[VAL1]] acquire
%17 = atomicrmw udec_wrap ptr %ptr1, i32 %val1 acquire
+
+ ; CHECK: llvm.atomicrmw volatile
+ ; CHECK-SAME: syncscope("singlethread")
+ ; CHECK-SAME: {alignment = 8 : i64}
+ %18 = atomicrmw volatile udec_wrap ptr %ptr1, i32 %val1 syncscope("singlethread") acquire, align 8
ret void
}
%1 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 seq_cst seq_cst
; CHECK: llvm.cmpxchg %[[PTR1]], %[[VAL1]], %[[VAL2]] monotonic seq_cst
%2 = cmpxchg ptr %ptr1, i32 %val1, i32 %val2 monotonic seq_cst
+
+ ; CHECK: llvm.cmpxchg weak volatile
+ ; CHECK-SAME: syncscope("singlethread")
+ ; CHECK-SAME: {alignment = 8 : i64}
+ %3 = cmpxchg weak volatile ptr %ptr1, i32 %val1, i32 %val2 syncscope("singlethread") monotonic seq_cst, align 8
ret void
}
%15 = llvm.atomicrmw uinc_wrap %i32_ptr, %i32 monotonic : !llvm.ptr<i32>, i32
// CHECK: atomicrmw udec_wrap ptr %{{.*}}, i32 %{{.*}} monotonic
%16 = llvm.atomicrmw udec_wrap %i32_ptr, %i32 monotonic : !llvm.ptr<i32>, i32
+
+ // CHECK: atomicrmw volatile
+ // CHECK-SAME: syncscope("singlethread")
+ // CHECK-SAME: align 8
+ %17 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr<i32>, i32
llvm.return
}
%1 = llvm.extractvalue %0[0] : !llvm.struct<(i32, i1)>
// CHECK: %{{[0-9]+}} = extractvalue { i32, i1 } %{{[0-9]+}}, 1
%2 = llvm.extractvalue %0[1] : !llvm.struct<(i32, i1)>
+
+ // CHECK: cmpxchg weak volatile
+ // CHECK-SAME: syncscope("singlethread")
+ // CHECK-SAME: align 8
+ %3 = llvm.cmpxchg weak volatile %ptr, %cmp, %val syncscope("singlethread") acq_rel monotonic {alignment = 8 : i64} : !llvm.ptr<i32>, i32
llvm.return
}
projects / platform / upstream / llvm.git / commitdiff