static StringRef getDataLayoutAttrName() { return "llvm.data_layout"; }
static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; }
static StringRef getAliasScopesAttrName() { return "alias_scopes"; }
- static StringRef getLoopAttrName() { return "llvm.loop"; }
static StringRef getAccessGroupsAttrName() { return "access_groups"; }
/// Names of llvm parameter attributes.
// Terminators.
def LLVM_BrOp : LLVM_TerminatorOp<"br",
[DeclareOpInterfaceMethods<BranchOpInterface>, Pure]> {
- let arguments = (ins Variadic<LLVM_Type>:$destOperands);
+ let arguments = (ins
+ Variadic<LLVM_Type>:$destOperands,
+ OptionalAttr<LoopAnnotationAttr>:$loop_annotation
+ );
let successors = (successor AnySuccessor:$dest);
let assemblyFormat = [{
$dest (`(` $destOperands^ `:` type($destOperands) `)`)? attr-dict
OpBuilder<(ins "Block *":$dest), [{
build($_builder, $_state, ValueRange(), dest);
}]>,
+ OpBuilder<(ins "ValueRange":$operands, "Block *":$dest), [{
+ build($_builder, $_state, operands, /*loop_annotation=*/{}, dest);
+ }]>,
LLVM_TerminatorPassthroughOpBuilder
];
+ let hasVerifier = 1;
}
def LLVM_CondBrOp : LLVM_TerminatorOp<"cond_br",
[AttrSizedOperandSegments, DeclareOpInterfaceMethods<BranchOpInterface>,
let arguments = (ins I1:$condition,
Variadic<LLVM_Type>:$trueDestOperands,
Variadic<LLVM_Type>:$falseDestOperands,
- OptionalAttr<ElementsAttr>:$branch_weights);
+ OptionalAttr<ElementsAttr>:$branch_weights,
+ OptionalAttr<LoopAnnotationAttr>:$loop_annotation);
let successors = (successor AnySuccessor:$trueDest, AnySuccessor:$falseDest);
let assemblyFormat = [{
$condition ( `weights` `(` $branch_weights^ `)` )? `,`
OpBuilder<(ins "Value":$condition, "Block *":$trueDest,
"ValueRange":$trueOperands, "Block *":$falseDest,
"ValueRange":$falseOperands,
- CArg<"std::optional<std::pair<uint32_t, uint32_t>>", "{}">:$weights),
- [{
- ElementsAttr weightsAttr;
- if (weights) {
- weightsAttr =
- $_builder.getI32VectorAttr({static_cast<int32_t>(weights->first),
- static_cast<int32_t>(weights->second)});
- }
- build($_builder, $_state, condition, trueOperands, falseOperands, weightsAttr,
- trueDest, falseDest);
- }]>,
+ CArg<"std::optional<std::pair<uint32_t, uint32_t>>", "{}">:$weights)>,
OpBuilder<(ins "Value":$condition, "Block *":$trueDest,
"Block *":$falseDest, CArg<"ValueRange", "{}">:$falseOperands),
[{
build($_builder, $_state, condition, trueDest, ValueRange(), falseDest,
falseOperands);
+ }]>,
+ OpBuilder<(ins "Value":$condition, "ValueRange":$trueOperands, "ValueRange":$falseOperands,
+ "ElementsAttr":$branchWeights, "Block *":$trueDest, "Block *":$falseDest),
+ [{
+ build($_builder, $_state, condition, trueOperands, falseOperands, branchWeights,
+ {}, trueDest, falseDest);
}]>, LLVM_TerminatorPassthroughOpBuilder];
+ let hasVerifier = 1;
}
//===----------------------------------------------------------------------===//
// LLVM::BrOp
//===----------------------------------------------------------------------===//
+/// Check if the `loopAttr` references correct symbols.
+static LogicalResult verifyLoopAnnotationAttr(LoopAnnotationAttr loopAttr,
+ Operation *op) {
+ if (!loopAttr)
+ return success();
+ // If the `llvm.loop` attribute is present, enforce the following structure,
+ // which the module translation can assume.
+ ArrayRef<SymbolRefAttr> parallelAccesses = loopAttr.getParallelAccesses();
+ if (parallelAccesses.empty())
+ return success();
+ for (SymbolRefAttr accessGroupRef : parallelAccesses) {
+ StringAttr metadataName = accessGroupRef.getRootReference();
+ auto metadataOp = SymbolTable::lookupNearestSymbolFrom<LLVM::MetadataOp>(
+ op->getParentOp(), metadataName);
+ if (!metadataOp)
+ return op->emitOpError() << "expected '" << accessGroupRef
+ << "' to reference a metadata op";
+ StringAttr accessGroupName = accessGroupRef.getLeafReference();
+ Operation *accessGroupOp =
+ SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName);
+ if (!accessGroupOp)
+ return op->emitOpError() << "expected '" << accessGroupRef
+ << "' to reference an access_group op";
+ }
+ return success();
+}
+
SuccessorOperands BrOp::getSuccessorOperands(unsigned index) {
assert(index == 0 && "invalid successor index");
return SuccessorOperands(getDestOperandsMutable());
}
+LogicalResult BrOp::verify() {
+ return verifyLoopAnnotationAttr(getLoopAnnotationAttr(), *this);
+}
+
//===----------------------------------------------------------------------===//
// LLVM::CondBrOp
//===----------------------------------------------------------------------===//
: getFalseDestOperandsMutable());
}
+LogicalResult CondBrOp::verify() {
+ return verifyLoopAnnotationAttr(getLoopAnnotationAttr(), *this);
+}
+
+void CondBrOp::build(OpBuilder &builder, OperationState &result,
+ Value condition, Block *trueDest, ValueRange trueOperands,
+ Block *falseDest, ValueRange falseOperands,
+ std::optional<std::pair<uint32_t, uint32_t>> weights) {
+ ElementsAttr weightsAttr;
+ if (weights)
+ weightsAttr =
+ builder.getI32VectorAttr({static_cast<int32_t>(weights->first),
+ static_cast<int32_t>(weights->second)});
+
+ build(builder, result, condition, trueOperands, falseOperands, weightsAttr,
+ /*loop_annotation=*/{}, trueDest, falseDest);
+}
+
//===----------------------------------------------------------------------===//
// LLVM::SwitchOp
//===----------------------------------------------------------------------===//
/// Verify LLVM dialect attributes.
LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op,
NamedAttribute attr) {
- // If the `llvm.loop` attribute is present, enforce the following structure,
- // which the module translation can assume.
- if (attr.getName() == LLVMDialect::getLoopAttrName()) {
- auto loopAttr = attr.getValue().dyn_cast<LoopAnnotationAttr>();
- if (!loopAttr)
- return op->emitOpError() << "expected '" << LLVMDialect::getLoopAttrName()
- << "' to be a loop annotation attribute";
- ArrayRef<SymbolRefAttr> parallelAccesses = loopAttr.getParallelAccesses();
- if (parallelAccesses.empty())
- return success();
- for (SymbolRefAttr accessGroupRef : parallelAccesses) {
- StringAttr metadataName = accessGroupRef.getRootReference();
- auto metadataOp = SymbolTable::lookupNearestSymbolFrom<LLVM::MetadataOp>(
- op->getParentOp(), metadataName);
- if (!metadataOp)
- return op->emitOpError() << "expected '" << accessGroupRef
- << "' to reference a metadata op";
- StringAttr accessGroupName = accessGroupRef.getLeafReference();
- Operation *accessGroupOp =
- SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName);
- if (!accessGroupOp)
- return op->emitOpError() << "expected '" << accessGroupRef
- << "' to reference an access_group op";
- }
- }
-
// If the data layout attribute is present, it must use the LLVM data layout
// syntax. Try parsing it and report errors in case of failure. Users of this
// attribute may assume it is well-formed and can pass it to the (asserting)
if (!attr)
return failure();
- op->setAttr(LLVMDialect::getLoopAttrName(), attr);
- return success();
+ return TypeSwitch<Operation *, LogicalResult>(op)
+ .Case<LLVM::BrOp, LLVM::CondBrOp>([&](auto branchOp) {
+ branchOp.setLoopAnnotationAttr(attr);
+ return success();
+ })
+ .Default([](auto) { return failure(); });
}
namespace {
void ModuleTranslation::setLoopMetadata(Operation *op,
llvm::Instruction *inst) {
- auto attr =
- op->getAttrOfType<LoopAnnotationAttr>(LLVMDialect::getLoopAttrName());
+ LoopAnnotationAttr attr =
+ TypeSwitch<Operation *, LoopAnnotationAttr>(op)
+ .Case<LLVM::BrOp, LLVM::CondBrOp>(
+ [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); });
if (!attr)
return;
llvm::MDNode *loopMD = loopAnnotationTranslation->translate(attr, op);
module {
llvm.func @loopOptions() {
- // expected-error@below {{expected 'llvm.loop' to be a loop annotation attribute}}
- llvm.br ^bb4 {llvm.loop = "test"}
- ^bb4:
- llvm.return
- }
-}
-
-// -----
-
-module {
- llvm.func @loopOptions() {
// expected-error@below {{expected '@func1' to reference a metadata op}}
- llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation<parallelAccesses = @func1>}
+ llvm.br ^bb4 {loop_annotation = #llvm.loop_annotation<parallelAccesses = @func1>}
^bb4:
llvm.return
}
module {
llvm.func @loopOptions() {
// expected-error@below {{expected '@metadata' to reference an access_group op}}
- llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation<parallelAccesses = @metadata>}
+ llvm.br ^bb4 {loop_annotation = #llvm.loop_annotation<parallelAccesses = @metadata>}
^bb4:
llvm.return
}
; CHECK-LABEL: @simple
define void @simple(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @vectorize
define void @vectorize(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @interleave
define void @interleave(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @unroll
define void @unroll(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @unroll_disable
define void @unroll_disable(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @unroll_and_jam
define void @unroll_and_jam(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @licm
define void @licm(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @distribute
define void @distribute(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @pipeline
define void @pipeline(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @peeled
define void @peeled(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
; CHECK-LABEL: @unswitched
define void @unswitched(i64 %n, ptr %A) {
entry:
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
define void @parallel_accesses(ptr %arg) {
entry:
%0 = load i32, ptr %arg, !llvm.access.group !0
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
entry:
%0 = load i32, ptr %arg, !llvm.access.group !0
%1 = load i32, ptr %arg, !llvm.access.group !3
-; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+; CHECK: llvm.br ^{{.*}} {loop_annotation = #[[$ANNOT_ATTR]]}
br label %end, !llvm.loop !1
end:
ret void
// CHECK-LABEL: @disableNonForced
llvm.func @disableNonForced() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<disableNonforced = true>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<disableNonforced = true>}
^bb1:
llvm.return
}
// CHECK-LABEL: @mustprogress
llvm.func @mustprogress() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<mustProgress = true>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<mustProgress = true>}
^bb1:
llvm.return
}
// CHECK-LABEL: @isvectorized
llvm.func @isvectorized() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<isVectorized = true>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<isVectorized = true>}
^bb1:
llvm.return
}
// CHECK-LABEL: @vectorizeOptions
llvm.func @vectorizeOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<vectorize = <
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<vectorize = <
disable = false, predicateEnable = true, scalableEnable = false, width = 16 : i32,
followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup>
>}
// CHECK-LABEL: @interleaveOptions
llvm.func @interleaveOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<interleave = <count = 32 : i32>>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<interleave = <count = 32 : i32>>}
^bb1:
llvm.return
}
// CHECK-LABEL: @unrollOptions
llvm.func @unrollOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unroll = <
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<unroll = <
disable = true, count = 64 : i32, runtimeDisable = false, full = false,
followupUnrolled = #followup, followupRemainder = #followup, followupAll = #followup>
>}
// CHECK-LABEL: @unrollOptions2
llvm.func @unrollOptions2() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unroll = <disable = false, full = true>>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<unroll = <disable = false, full = true>>}
^bb1:
llvm.return
}
// CHECK-LABEL: @unrollAndJamOptions
llvm.func @unrollAndJamOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unrollAndJam = <
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<unrollAndJam = <
disable = false, count = 8 : i32, followupOuter = #followup, followupInner = #followup,
followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup>
>}
// CHECK-LABEL: @licmOptions
llvm.func @licmOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<licm = <disable = false, versioningDisable = true>>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<licm = <disable = false, versioningDisable = true>>}
^bb1:
llvm.return
}
// CHECK-LABEL: @licmOptions2
llvm.func @licmOptions2() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<licm = <disable = true, versioningDisable = false>>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<licm = <disable = true, versioningDisable = false>>}
^bb1:
llvm.return
}
// CHECK-LABEL: @distributeOptions
llvm.func @distributeOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<distribute = <
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<distribute = <
disable = true, followupCoincident = #followup, followupSequential = #followup,
followupFallback = #followup, followupAll = #followup>
>}
// CHECK-LABEL: @pipelineOptions
llvm.func @pipelineOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<pipeline = <disable = false, initiationinterval = 1 : i32>>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<pipeline = <disable = false, initiationinterval = 1 : i32>>}
^bb1:
llvm.return
}
// CHECK-LABEL: @peeledOptions
llvm.func @peeledOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<peeled = <count = 3 : i32>>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<peeled = <count = 3 : i32>>}
^bb1:
llvm.return
}
// CHECK-LABEL: @unswitchOptions
llvm.func @unswitchOptions() {
// CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unswitch = <partialDisable = true>>}
+ llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation<unswitch = <partialDisable = true>>}
^bb1:
llvm.return
}
^bb3(%1: i32):
%2 = llvm.icmp "slt" %1, %arg1 : i32
// CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = #llvm.loop_annotation<
+ llvm.cond_br %2, ^bb4, ^bb5 {loop_annotation = #llvm.loop_annotation<
licm = <disable = true>,
interleave = <count = 1>,
unroll = <disable = true>, pipeline = <disable = true, initiationinterval = 2>,
// CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]]
%5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr<i32>
// CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]]
- llvm.br ^bb3(%3 : i32) {llvm.loop = #llvm.loop_annotation<
+ llvm.br ^bb3(%3 : i32) {loop_annotation = #llvm.loop_annotation<
licm = <disable = true>,
interleave = <count = 1>,
unroll = <disable = true>, pipeline = <disable = true, initiationinterval = 2>,