}
//===----------------------------------------------------------------------===//
-// LoopOptionsAttr
+// Loop Attributes
//===----------------------------------------------------------------------===//
-def LoopOptionsAttr : LLVM_Attr<"LoopOptions", "loopopts"> {
+def LoopVectorizeAttr : LLVM_Attr<"LoopVectorize", "loop_vectorize"> {
let description = [{
- This attributes encapsulates "loop options". It is means to decorate
- branches that are "latches" (loop backedges) and maps to the `!llvm.loop`
- metadatas: https://llvm.org/docs/LangRef.html#llvm-loop
- It store the options as a pair <enum,int64_t> in a sorted array and expose
- APIs to retrieve the value for each option with a stronger type (bool for
- example).
+ This attribute defines vectorization specific loop annotations that map to
+ the "!llvm.loop.vectorize" metadata.
}];
let parameters = (ins
- ArrayRefParameter<"std::pair<LoopOptionCase, int64_t>", "">:$options
+ OptionalParameter<"BoolAttr">:$disable,
+ OptionalParameter<"BoolAttr">:$predicateEnable,
+ OptionalParameter<"BoolAttr">:$scalableEnable,
+ OptionalParameter<"IntegerAttr">:$width,
+ OptionalParameter<"LoopAnnotationAttr">:$followupVectorized,
+ OptionalParameter<"LoopAnnotationAttr">:$followupEpilogue,
+ OptionalParameter<"LoopAnnotationAttr">:$followupAll
);
- let extraClassDeclaration = [{
- using OptionValuePair = std::pair<LoopOptionCase, int64_t>;
- using OptionsArray = ArrayRef<std::pair<LoopOptionCase, int64_t>>;
- std::optional<bool> disableUnroll();
- std::optional<bool> disableLICM();
- std::optional<int64_t> interleaveCount();
+ let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def LoopInterleaveAttr : LLVM_Attr<"LoopInterleave", "loop_interleave"> {
+ let description = [{
+ This attribute defines interleaving specific loop annotations that map to
+ the "!llvm.loop.interleave" metadata.
}];
- let builders = [
- /// Build the LoopOptions Attribute from a sorted array of individual options.
- AttrBuilder<(ins "ArrayRef<std::pair<LoopOptionCase, int64_t>>":$sortedOptions)>,
- AttrBuilder<(ins "LoopOptionsAttrBuilder &":$optionBuilders)>
- ];
- let hasCustomAssemblyFormat = 1;
- let skipDefaultBuilders = 1;
+ let parameters = (ins
+ "IntegerAttr":$count
+ );
+
+ let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def LoopUnrollAttr : LLVM_Attr<"LoopUnroll", "loop_unroll"> {
+ let description = [{
+ This attribute defines unrolling specific loop annotations that map to
+ the "!llvm.loop.unroll" metadata.
+ }];
+
+ let parameters = (ins
+ OptionalParameter<"BoolAttr">:$disable,
+ OptionalParameter<"IntegerAttr">:$count,
+ OptionalParameter<"BoolAttr">:$runtimeDisable,
+ OptionalParameter<"BoolAttr">:$full,
+ OptionalParameter<"LoopAnnotationAttr">:$followup,
+ OptionalParameter<"LoopAnnotationAttr">:$followupRemainder
+ );
+
+ let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def LoopUnrollAndJamAttr : LLVM_Attr<"LoopUnrollAndJam", "loop_unroll_and_jam"> {
+ let description = [{
+ This attribute defines "unroll and jam" specific loop annotations that map to
+ the "!llvm.loop.unroll_and_jam" metadata.
+ }];
+
+ let parameters = (ins
+ OptionalParameter<"BoolAttr">:$disable,
+ OptionalParameter<"IntegerAttr">:$count,
+ OptionalParameter<"LoopAnnotationAttr">:$followupOuter,
+ OptionalParameter<"LoopAnnotationAttr">:$followupInner,
+ OptionalParameter<"LoopAnnotationAttr">:$followupRemainderOuter,
+ OptionalParameter<"LoopAnnotationAttr">:$followupRemainderInner,
+ OptionalParameter<"LoopAnnotationAttr">:$followupAll
+ );
+
+ let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def LoopLICMAttr : LLVM_Attr<"LoopLICM", "loop_licm"> {
+ let description = [{
+ This attribute encapsulates loop invariant code motion (licm) specific loop
+ annotations. The fields correspond to the "!llvm.licm.disable" and the
+ "!llvm.loop.licm_versioning.disable" metadata.
+ }];
+
+ let parameters = (ins
+ OptionalParameter<"BoolAttr">:$disable,
+ OptionalParameter<"BoolAttr">:$versioningDisable
+ );
+
+ let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def LoopDistributeAttr : LLVM_Attr<"LoopDistribute", "loop_distribute"> {
+ let description = [{
+ This attribute defines distribution specific loop annotations that map to
+ the "!llvm.loop.distribute" metadata.
+ }];
+
+ let parameters = (ins
+ OptionalParameter<"BoolAttr">:$disable,
+ OptionalParameter<"LoopAnnotationAttr">:$followupCoincident,
+ OptionalParameter<"LoopAnnotationAttr">:$followupSequential,
+ OptionalParameter<"LoopAnnotationAttr">:$followupFallback,
+ OptionalParameter<"LoopAnnotationAttr">:$followupAll
+ );
+
+ let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def LoopPipelineAttr : LLVM_Attr<"LoopPipeline", "loop_pipeline"> {
+ let description = [{
+ This attribute defines pipelining specific loop annotations that map to
+ the "!llvm.loop.pipeline" metadata.
+ }];
+
+ let parameters = (ins
+ OptionalParameter<"BoolAttr">:$disable,
+ OptionalParameter<"IntegerAttr">:$initiationinterval
+ );
+
+ let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def LoopAnnotationAttr : LLVM_Attr<"LoopAnnotation", "loop_annotation"> {
+ let description = [{
+ This attributes encapsulates "loop metadata". It is meant to decorate
+ branches that are "latches" (loop backedges) and maps to the `!llvm.loop`
+ metadatas: https://llvm.org/docs/LangRef.html#llvm-loop
+ It stores annotations in attribute parameters and groups related options in
+ nested attributes to provide structured access.
+ }];
+
+ let parameters = (ins
+ OptionalParameter<"BoolAttr">:$disableNonforced,
+ OptionalParameter<"LoopVectorizeAttr">:$vectorize,
+ OptionalParameter<"LoopInterleaveAttr">:$interleave,
+ OptionalParameter<"LoopUnrollAttr">:$unroll,
+ OptionalParameter<"LoopUnrollAndJamAttr">:$unrollAndJam,
+ OptionalParameter<"LoopLICMAttr">:$licm,
+ OptionalParameter<"LoopDistributeAttr">:$distribute,
+ OptionalParameter<"LoopPipelineAttr">:$pipeline,
+ OptionalParameter<"BoolAttr">:$mustProgress,
+ OptionalArrayRefParameter<"SymbolRefAttr">:$parallelAccesses
+ );
+
+ let assemblyFormat = "`<` struct(params) `>`";
}
//===----------------------------------------------------------------------===//
namespace mlir {
namespace LLVM {
-class LoopOptionsAttrBuilder;
/// This class represents the base attribute for all debug info attributes.
class DINodeAttr : public Attribute {
#define GET_ATTRDEF_CLASSES
#include "mlir/Dialect/LLVMIR/LLVMOpsAttrDefs.h.inc"
-namespace mlir {
-namespace LLVM {
-
-/// Builder class for LoopOptionsAttr. This helper class allows to progressively
-/// build a LoopOptionsAttr one option at a time, and pay the price of attribute
-/// creation once all the options are in place.
-class LoopOptionsAttrBuilder {
-public:
- /// Construct a empty builder.
- LoopOptionsAttrBuilder() = default;
-
- /// Construct a builder with an initial list of options from an existing
- /// LoopOptionsAttr.
- LoopOptionsAttrBuilder(LoopOptionsAttr attr);
-
- /// Set the `disable_licm` option to the provided value. If no value
- /// is provided the option is deleted.
- LoopOptionsAttrBuilder &setDisableLICM(std::optional<bool> value);
-
- /// Set the `interleave_count` option to the provided value. If no value
- /// is provided the option is deleted.
- LoopOptionsAttrBuilder &setInterleaveCount(std::optional<uint64_t> count);
-
- /// Set the `disable_unroll` option to the provided value. If no value
- /// is provided the option is deleted.
- LoopOptionsAttrBuilder &setDisableUnroll(std::optional<bool> value);
-
- /// Set the `disable_pipeline` option to the provided value. If no value
- /// is provided the option is deleted.
- LoopOptionsAttrBuilder &setDisablePipeline(std::optional<bool> value);
-
- /// Set the `pipeline_initiation_interval` option to the provided value.
- /// If no value is provided the option is deleted.
- LoopOptionsAttrBuilder &
- setPipelineInitiationInterval(std::optional<uint64_t> count);
-
- /// Returns true if any option has been set.
- bool empty() { return options.empty(); }
-
-private:
- template <typename T>
- LoopOptionsAttrBuilder &setOption(LoopOptionCase tag, std::optional<T> value);
-
- friend class LoopOptionsAttr;
- SmallVector<LoopOptionsAttr::OptionValuePair> options;
-};
-
-} // namespace LLVM
-} // namespace mlir
-
#endif // MLIR_DIALECT_LLVMIR_LLVMATTRS_H_
}
//===----------------------------------------------------------------------===//
-// LoopOptions
-//===----------------------------------------------------------------------===//
-
-def LOptDisableUnroll : I32EnumAttrCase<"disable_unroll", 1>;
-def LOptDisableLICM : I32EnumAttrCase<"disable_licm", 2>;
-def LOptInterleaveCount : I32EnumAttrCase<"interleave_count", 3>;
-def LOptDisablePipeline : I32EnumAttrCase<"disable_pipeline", 4>;
-def LOptPipelineInitiationInterval : I32EnumAttrCase<"pipeline_initiation_interval", 5>;
-
-def LoopOptionCase : I32EnumAttr<
- "LoopOptionCase",
- "LLVM loop option",
- [LOptDisableUnroll, LOptDisableLICM, LOptInterleaveCount,
- LOptDisablePipeline, LOptPipelineInitiationInterval
- ]> {
- let cppNamespace = "::mlir::LLVM";
-}
-
-//===----------------------------------------------------------------------===//
// UnnamedAddr
//===----------------------------------------------------------------------===//
static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; }
static StringRef getAliasScopesAttrName() { return "alias_scopes"; }
static StringRef getLoopAttrName() { return "llvm.loop"; }
- static StringRef getParallelAccessAttrName() { return "parallel_access"; }
- static StringRef getLoopOptionsAttrName() { return "options"; }
static StringRef getAccessGroupsAttrName() { return "access_groups"; }
static StringRef getStructAttrsAttrName() { return "llvm.struct_attrs"; }
static StringRef getTBAAAttrName() { return "llvm.tbaa"; }
namespace detail {
class DebugTranslation;
+class LoopAnnotationTranslation;
} // namespace detail
class DINodeAttr;
llvm::MDNode *getAliasScope(Operation &opInst,
SymbolRefAttr aliasScopeRef) const;
- /// Returns the LLVM metadata corresponding to a llvm loop's codegen
- /// options attribute.
- llvm::MDNode *lookupLoopOptionsMetadata(Attribute options) const {
- return loopOptionsMetadataMapping.lookup(options);
- }
-
- void mapLoopOptionsMetadata(Attribute options, llvm::MDNode *metadata) {
- auto result = loopOptionsMetadataMapping.try_emplace(options, metadata);
- (void)result;
- assert(result.second &&
- "attempting to map loop options that was already mapped");
- }
-
// Sets LLVM metadata for memory operations that are in a parallel loop.
void setAccessGroupsMetadata(Operation *op, llvm::Instruction *inst);
/// TBAA attributes.
void setTBAAMetadata(Operation *op, llvm::Instruction *inst);
+ /// Sets LLVM loop metadata for branch operations that have a loop annotation
+ /// attribute.
+ void setLoopMetadata(Operation *op, llvm::Instruction *inst);
+
/// Converts the type from MLIR LLVM dialect to LLVM.
llvm::Type *convertType(Type type);
/// A converter for translating debug information.
std::unique_ptr<detail::DebugTranslation> debugTranslation;
+ /// A converter for translating loop annotations.
+ std::unique_ptr<detail::LoopAnnotationTranslation> loopAnnotationTranslation;
+
/// Builder for LLVM IR generation of OpenMP constructs.
std::unique_ptr<llvm::OpenMPIRBuilder> ompBuilder;
/// identified via their branches) and contained memory accesses.
DenseMap<Operation *, llvm::MDNode *> accessGroupMetadataMapping;
- /// Mapping from an attribute describing loop codegen options to its LLVM
- /// metadata. The metadata is attached to Latch block branches with this
- /// attribute.
- DenseMap<Attribute, llvm::MDNode *> loopOptionsMetadataMapping;
-
/// Mapping from an alias scope metadata operation to its LLVM metadata.
/// This map is populated on module entry.
DenseMap<Operation *, llvm::MDNode *> aliasScopeMetadataMapping;
}
//===----------------------------------------------------------------------===//
-// LoopOptionsAttrBuilder
-//===----------------------------------------------------------------------===//
-
-LoopOptionsAttrBuilder::LoopOptionsAttrBuilder(LoopOptionsAttr attr)
- : options(attr.getOptions().begin(), attr.getOptions().end()) {}
-
-template <typename T>
-LoopOptionsAttrBuilder &
-LoopOptionsAttrBuilder::setOption(LoopOptionCase tag, std::optional<T> value) {
- auto option = llvm::find_if(
- options, [tag](auto option) { return option.first == tag; });
- if (option != options.end()) {
- if (value)
- option->second = *value;
- else
- options.erase(option);
- } else {
- options.push_back(LoopOptionsAttr::OptionValuePair(tag, *value));
- }
- return *this;
-}
-
-LoopOptionsAttrBuilder &
-LoopOptionsAttrBuilder::setDisableLICM(std::optional<bool> value) {
- return setOption(LoopOptionCase::disable_licm, value);
-}
-
-/// Set the `interleave_count` option to the provided value. If no value
-/// is provided the option is deleted.
-LoopOptionsAttrBuilder &
-LoopOptionsAttrBuilder::setInterleaveCount(std::optional<uint64_t> count) {
- return setOption(LoopOptionCase::interleave_count, count);
-}
-
-/// Set the `disable_unroll` option to the provided value. If no value
-/// is provided the option is deleted.
-LoopOptionsAttrBuilder &
-LoopOptionsAttrBuilder::setDisableUnroll(std::optional<bool> value) {
- return setOption(LoopOptionCase::disable_unroll, value);
-}
-
-/// Set the `disable_pipeline` option to the provided value. If no value
-/// is provided the option is deleted.
-LoopOptionsAttrBuilder &
-LoopOptionsAttrBuilder::setDisablePipeline(std::optional<bool> value) {
- return setOption(LoopOptionCase::disable_pipeline, value);
-}
-
-/// Set the `pipeline_initiation_interval` option to the provided value.
-/// If no value is provided the option is deleted.
-LoopOptionsAttrBuilder &LoopOptionsAttrBuilder::setPipelineInitiationInterval(
- std::optional<uint64_t> count) {
- return setOption(LoopOptionCase::pipeline_initiation_interval, count);
-}
-
-//===----------------------------------------------------------------------===//
-// LoopOptionsAttr
-//===----------------------------------------------------------------------===//
-
-template <typename T>
-static std::optional<T>
-getOption(ArrayRef<std::pair<LoopOptionCase, int64_t>> options,
- LoopOptionCase option) {
- auto it =
- lower_bound(options, option, [](auto optionPair, LoopOptionCase option) {
- return optionPair.first < option;
- });
- if (it == options.end())
- return {};
- return static_cast<T>(it->second);
-}
-
-std::optional<bool> LoopOptionsAttr::disableUnroll() {
- return getOption<bool>(getOptions(), LoopOptionCase::disable_unroll);
-}
-
-std::optional<bool> LoopOptionsAttr::disableLICM() {
- return getOption<bool>(getOptions(), LoopOptionCase::disable_licm);
-}
-
-std::optional<int64_t> LoopOptionsAttr::interleaveCount() {
- return getOption<int64_t>(getOptions(), LoopOptionCase::interleave_count);
-}
-
-/// Build the LoopOptions Attribute from a sorted array of individual options.
-LoopOptionsAttr LoopOptionsAttr::get(
- MLIRContext *context,
- ArrayRef<std::pair<LoopOptionCase, int64_t>> sortedOptions) {
- assert(llvm::is_sorted(sortedOptions, llvm::less_first()) &&
- "LoopOptionsAttr ctor expects a sorted options array");
- return Base::get(context, sortedOptions);
-}
-
-/// Build the LoopOptions Attribute from a sorted array of individual options.
-LoopOptionsAttr LoopOptionsAttr::get(MLIRContext *context,
- LoopOptionsAttrBuilder &optionBuilders) {
- llvm::sort(optionBuilders.options, llvm::less_first());
- return Base::get(context, optionBuilders.options);
-}
-
-void LoopOptionsAttr::print(AsmPrinter &printer) const {
- printer << "<";
- llvm::interleaveComma(getOptions(), printer, [&](auto option) {
- printer << stringifyEnum(option.first) << " = ";
- switch (option.first) {
- case LoopOptionCase::disable_licm:
- case LoopOptionCase::disable_unroll:
- case LoopOptionCase::disable_pipeline:
- printer << (option.second ? "true" : "false");
- break;
- case LoopOptionCase::interleave_count:
- case LoopOptionCase::pipeline_initiation_interval:
- printer << option.second;
- break;
- }
- });
- printer << ">";
-}
-
-Attribute LoopOptionsAttr::parse(AsmParser &parser, Type type) {
- if (failed(parser.parseLess()))
- return {};
-
- SmallVector<std::pair<LoopOptionCase, int64_t>> options;
- llvm::SmallDenseSet<LoopOptionCase> seenOptions;
- auto parseLoopOptions = [&]() -> ParseResult {
- StringRef optionName;
- if (parser.parseKeyword(&optionName))
- return failure();
-
- auto option = symbolizeLoopOptionCase(optionName);
- if (!option)
- return parser.emitError(parser.getNameLoc(), "unknown loop option: ")
- << optionName;
- if (!seenOptions.insert(*option).second)
- return parser.emitError(parser.getNameLoc(), "loop option present twice");
- if (failed(parser.parseEqual()))
- return failure();
-
- int64_t value;
- switch (*option) {
- case LoopOptionCase::disable_licm:
- case LoopOptionCase::disable_unroll:
- case LoopOptionCase::disable_pipeline:
- if (succeeded(parser.parseOptionalKeyword("true")))
- value = 1;
- else if (succeeded(parser.parseOptionalKeyword("false")))
- value = 0;
- else {
- return parser.emitError(parser.getNameLoc(),
- "expected boolean value 'true' or 'false'");
- }
- break;
- case LoopOptionCase::interleave_count:
- case LoopOptionCase::pipeline_initiation_interval:
- if (failed(parser.parseInteger(value)))
- return parser.emitError(parser.getNameLoc(), "expected integer value");
- break;
- }
- options.push_back(std::make_pair(*option, value));
- return success();
- };
- if (parser.parseCommaSeparatedList(parseLoopOptions) || parser.parseGreater())
- return {};
-
- llvm::sort(options, llvm::less_first());
- return get(parser.getContext(), options);
-}
-
-//===----------------------------------------------------------------------===//
// MemoryEffectsAttr
//===----------------------------------------------------------------------===//
.Case<DIVoidResultTypeAttr, DIBasicTypeAttr, DICompileUnitAttr,
DICompositeTypeAttr, DIDerivedTypeAttr, DIFileAttr,
DILexicalBlockAttr, DILexicalBlockFileAttr, DILocalVariableAttr,
- DISubprogramAttr, DISubroutineTypeAttr>([&](auto attr) {
+ DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr,
+ LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr,
+ LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr,
+ LoopPipelineAttr>([&](auto attr) {
os << decltype(attr)::getMnemonic();
return AliasResult::OverridableAlias;
})
// If the `llvm.loop` attribute is present, enforce the following structure,
// which the module translation can assume.
if (attr.getName() == LLVMDialect::getLoopAttrName()) {
- auto loopAttr = attr.getValue().dyn_cast<DictionaryAttr>();
+ auto loopAttr = attr.getValue().dyn_cast<LoopAnnotationAttr>();
if (!loopAttr)
return op->emitOpError() << "expected '" << LLVMDialect::getLoopAttrName()
- << "' to be a dictionary attribute";
- std::optional<NamedAttribute> parallelAccessGroup =
- loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName());
- if (parallelAccessGroup) {
- auto accessGroups = parallelAccessGroup->getValue().dyn_cast<ArrayAttr>();
- if (!accessGroups)
- return op->emitOpError()
- << "expected '" << LLVMDialect::getParallelAccessAttrName()
- << "' to be an array attribute";
- for (Attribute attr : accessGroups) {
- auto accessGroupRef = attr.dyn_cast<SymbolRefAttr>();
- if (!accessGroupRef)
- return op->emitOpError()
- << "expected '" << attr << "' to be a symbol reference";
- StringAttr metadataName = accessGroupRef.getRootReference();
- auto metadataOp =
- SymbolTable::lookupNearestSymbolFrom<LLVM::MetadataOp>(
- op->getParentOp(), metadataName);
- if (!metadataOp)
- return op->emitOpError()
- << "expected '" << attr << "' to reference a metadata op";
- StringAttr accessGroupName = accessGroupRef.getLeafReference();
- Operation *accessGroupOp =
- SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName);
- if (!accessGroupOp)
- return op->emitOpError()
- << "expected '" << attr << "' to reference an access_group op";
- }
+ << "' to be a loop annotation attribute";
+ ArrayRef<SymbolRefAttr> parallelAccesses = loopAttr.getParallelAccesses();
+ if (parallelAccesses.empty())
+ return success();
+ for (SymbolRefAttr accessGroupRef : parallelAccesses) {
+ StringAttr metadataName = accessGroupRef.getRootReference();
+ auto metadataOp = SymbolTable::lookupNearestSymbolFrom<LLVM::MetadataOp>(
+ op->getParentOp(), metadataName);
+ if (!metadataOp)
+ return op->emitOpError() << "expected '" << accessGroupRef
+ << "' to reference a metadata op";
+ StringAttr accessGroupName = accessGroupRef.getLeafReference();
+ Operation *accessGroupOp =
+ SymbolTable::lookupNearestSymbolFrom(metadataOp, accessGroupName);
+ if (!accessGroupOp)
+ return op->emitOpError() << "expected '" << accessGroupRef
+ << "' to reference an access_group op";
}
-
- std::optional<NamedAttribute> loopOptions =
- loopAttr.getNamed(LLVMDialect::getLoopOptionsAttrName());
- if (loopOptions && !loopOptions->getValue().isa<LoopOptionsAttr>())
- return op->emitOpError()
- << "expected '" << LLVMDialect::getLoopOptionsAttrName()
- << "' to be a `loopopts` attribute";
}
if (attr.getName() == LLVMDialect::getStructAttrsAttrName()) {
ConvertToLLVMIR.cpp
DebugTranslation.cpp
DebugImporter.cpp
+ LoopAnnotationTranslation.cpp
ModuleTranslation.cpp
ModuleImport.cpp
TypeToLLVM.cpp
add_mlir_translation_library(MLIRTargetLLVMIRExport
DebugTranslation.cpp
+ LoopAnnotationTranslation.cpp
ModuleTranslation.cpp
TypeToLLVM.cpp
return ret;
}
-/// Returns an LLVM metadata node corresponding to a loop option. This metadata
-/// is attached to an llvm.loop node.
-static llvm::MDNode *getLoopOptionMetadata(llvm::LLVMContext &ctx,
- LoopOptionCase option,
- int64_t value) {
- StringRef name;
- llvm::Constant *cstValue = nullptr;
- switch (option) {
- case LoopOptionCase::disable_licm:
- name = "llvm.licm.disable";
- cstValue = llvm::ConstantInt::getBool(ctx, value);
- break;
- case LoopOptionCase::disable_unroll:
- name = "llvm.loop.unroll.disable";
- cstValue = llvm::ConstantInt::getBool(ctx, value);
- break;
- case LoopOptionCase::interleave_count:
- name = "llvm.loop.interleave.count";
- cstValue = llvm::ConstantInt::get(
- llvm::IntegerType::get(ctx, /*NumBits=*/32), value);
- break;
- case LoopOptionCase::disable_pipeline:
- name = "llvm.loop.pipeline.disable";
- cstValue = llvm::ConstantInt::getBool(ctx, value);
- break;
- case LoopOptionCase::pipeline_initiation_interval:
- name = "llvm.loop.pipeline.initiationinterval";
- cstValue = llvm::ConstantInt::get(
- llvm::IntegerType::get(ctx, /*NumBits=*/32), value);
- break;
- }
- return llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
- llvm::ConstantAsMetadata::get(cstValue)});
-}
-
-static void setLoopMetadata(Operation &opInst, llvm::Instruction &llvmInst,
- llvm::IRBuilderBase &builder,
- LLVM::ModuleTranslation &moduleTranslation) {
- if (Attribute attr = opInst.getAttr(LLVMDialect::getLoopAttrName())) {
- llvm::Module *module = builder.GetInsertBlock()->getModule();
- llvm::MDNode *loopMD = moduleTranslation.lookupLoopOptionsMetadata(attr);
- if (!loopMD) {
- llvm::LLVMContext &ctx = module->getContext();
-
- SmallVector<llvm::Metadata *> loopOptions;
- // Reserve operand 0 for loop id self reference.
- auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt);
- loopOptions.push_back(dummy.get());
-
- auto loopAttr = attr.cast<DictionaryAttr>();
- auto parallelAccessGroup =
- loopAttr.getNamed(LLVMDialect::getParallelAccessAttrName());
- if (parallelAccessGroup) {
- SmallVector<llvm::Metadata *> parallelAccess;
- parallelAccess.push_back(
- llvm::MDString::get(ctx, "llvm.loop.parallel_accesses"));
- for (SymbolRefAttr accessGroupRef : parallelAccessGroup->getValue()
- .cast<ArrayAttr>()
- .getAsRange<SymbolRefAttr>())
- parallelAccess.push_back(
- moduleTranslation.getAccessGroup(opInst, accessGroupRef));
- loopOptions.push_back(llvm::MDNode::get(ctx, parallelAccess));
- }
-
- if (auto loopOptionsAttr = loopAttr.getAs<LoopOptionsAttr>(
- LLVMDialect::getLoopOptionsAttrName())) {
- for (auto option : loopOptionsAttr.getOptions())
- loopOptions.push_back(
- getLoopOptionMetadata(ctx, option.first, option.second));
- }
-
- // Create loop options and set the first operand to itself.
- loopMD = llvm::MDNode::get(ctx, loopOptions);
- loopMD->replaceOperandWith(0, loopMD);
-
- // Store a map from this Attribute to the LLVM metadata in case we
- // encounter it again.
- moduleTranslation.mapLoopOptionsMetadata(attr, loopMD);
- }
-
- llvmInst.setMetadata(module->getMDKindID("llvm.loop"), loopMD);
- }
-}
-
/// Convert the value of a DenseI64ArrayAttr to a vector of unsigned indices.
static SmallVector<unsigned> extractPosition(ArrayRef<int64_t> indices) {
SmallVector<unsigned> position;
llvm::BranchInst *branch =
builder.CreateBr(moduleTranslation.lookupBlock(brOp.getSuccessor()));
moduleTranslation.mapBranch(&opInst, branch);
- setLoopMetadata(opInst, *branch, builder, moduleTranslation);
+ moduleTranslation.setLoopMetadata(&opInst, branch);
return success();
}
if (auto condbrOp = dyn_cast<LLVM::CondBrOp>(opInst)) {
moduleTranslation.lookupBlock(condbrOp.getSuccessor(0)),
moduleTranslation.lookupBlock(condbrOp.getSuccessor(1)), branchWeights);
moduleTranslation.mapBranch(&opInst, branch);
- setLoopMetadata(opInst, *branch, builder, moduleTranslation);
+ moduleTranslation.setLoopMetadata(&opInst, branch);
return success();
}
if (auto switchOp = dyn_cast<LLVM::SwitchOp>(opInst)) {
--- /dev/null
+#include "LoopAnnotationTranslation.h"
+
+using namespace mlir;
+using namespace mlir::LLVM;
+using namespace mlir::LLVM::detail;
+
+namespace {
+/// Helper class that keeps the state of one attribute to metadata conversion.
+struct LoopAnnotationConversion {
+ LoopAnnotationConversion(LoopAnnotationAttr attr,
+ ModuleTranslation &moduleTranslation, Operation *op,
+ LoopAnnotationTranslation &loopAnnotationTranslation)
+ : attr(attr), moduleTranslation(moduleTranslation), op(op),
+ loopAnnotationTranslation(loopAnnotationTranslation),
+ ctx(moduleTranslation.getLLVMContext()) {}
+
+ /// Converts this struct's loop annotation into a corresponding LLVMIR
+ /// metadata representation.
+ llvm::MDNode *convert();
+
+ /// Conversion functions for different payload attribute kinds.
+ void addUnitNode(StringRef name);
+ void addUnitNode(StringRef name, BoolAttr attr);
+ void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false);
+ void convertI32Node(StringRef name, IntegerAttr attr);
+ void convertFollowupNode(StringRef name, LoopAnnotationAttr attr);
+
+ /// Conversion functions for each for each loop annotation sub-attribute.
+ void convertLoopOptions(LoopVectorizeAttr options);
+ void convertLoopOptions(LoopInterleaveAttr options);
+ void convertLoopOptions(LoopUnrollAttr options);
+ void convertLoopOptions(LoopUnrollAndJamAttr options);
+ void convertLoopOptions(LoopLICMAttr options);
+ void convertLoopOptions(LoopDistributeAttr options);
+ void convertLoopOptions(LoopPipelineAttr options);
+
+ LoopAnnotationAttr attr;
+ ModuleTranslation &moduleTranslation;
+ Operation *op;
+ LoopAnnotationTranslation &loopAnnotationTranslation;
+ llvm::LLVMContext &ctx;
+ llvm::SmallVector<llvm::Metadata *> metadataNodes;
+};
+} // namespace
+
+void LoopAnnotationConversion::addUnitNode(StringRef name) {
+ metadataNodes.push_back(
+ llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name)}));
+}
+
+void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) {
+ if (attr && attr.getValue())
+ addUnitNode(name);
+}
+
+void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr,
+ bool negated) {
+ if (!attr)
+ return;
+ bool val = negated ^ attr.getValue();
+ llvm::Constant *cstValue = llvm::ConstantInt::getBool(ctx, val);
+ metadataNodes.push_back(
+ llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
+ llvm::ConstantAsMetadata::get(cstValue)}));
+}
+
+void LoopAnnotationConversion::convertI32Node(StringRef name,
+ IntegerAttr attr) {
+ if (!attr)
+ return;
+ uint32_t val = attr.getInt();
+ llvm::Constant *cstValue = llvm::ConstantInt::get(
+ llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false);
+ metadataNodes.push_back(
+ llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
+ llvm::ConstantAsMetadata::get(cstValue)}));
+}
+
+void LoopAnnotationConversion::convertFollowupNode(StringRef name,
+ LoopAnnotationAttr attr) {
+ if (!attr)
+ return;
+
+ llvm::MDNode *node = loopAnnotationTranslation.translate(attr, op);
+
+ metadataNodes.push_back(
+ llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name), node}));
+}
+
+void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) {
+ convertBoolNode("llvm.loop.vectorize.enable", options.getDisable(), true);
+ convertBoolNode("llvm.loop.vectorize.predicate.enable",
+ options.getPredicateEnable());
+ convertBoolNode("llvm.loop.vectorize.scalable.enable",
+ options.getScalableEnable());
+ convertI32Node("llvm.loop.vectorize.width", options.getWidth());
+ convertFollowupNode("llvm.loop.vectorize.followup_vectorized",
+ options.getFollowupVectorized());
+ convertFollowupNode("llvm.loop.vectorize.followup_epilogue",
+ options.getFollowupEpilogue());
+ convertFollowupNode("llvm.loop.vectorize.followup_all",
+ options.getFollowupAll());
+}
+
+void LoopAnnotationConversion::convertLoopOptions(LoopInterleaveAttr options) {
+ convertI32Node("llvm.loop.interleave.count", options.getCount());
+}
+
+void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) {
+ if (auto disable = options.getDisable())
+ addUnitNode(disable.getValue() ? "llvm.loop.unroll.disable"
+ : "llvm.loop.unroll.enable");
+ convertI32Node("llvm.loop.unroll.count", options.getCount());
+ convertBoolNode("llvm.loop.unroll.runtime.disable",
+ options.getRuntimeDisable());
+ addUnitNode("llvm.loop.unroll.full", options.getFull());
+ convertFollowupNode("llvm.loop.unroll.followup", options.getFollowup());
+ convertFollowupNode("llvm.loop.unroll.followup_remainder",
+ options.getFollowupRemainder());
+}
+
+void LoopAnnotationConversion::convertLoopOptions(
+ LoopUnrollAndJamAttr options) {
+ if (auto disable = options.getDisable())
+ addUnitNode(disable.getValue() ? "llvm.loop.unroll_and_jam.disable"
+ : "llvm.loop.unroll_and_jam.enable");
+ convertI32Node("llvm.loop.unroll_and_jam.count", options.getCount());
+ convertFollowupNode("llvm.loop.unroll_and_jam.followup_outer",
+ options.getFollowupOuter());
+ convertFollowupNode("llvm.loop.unroll_and_jam.followup_inner",
+ options.getFollowupInner());
+ convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer",
+ options.getFollowupRemainderOuter());
+ convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner",
+ options.getFollowupRemainderInner());
+ convertFollowupNode("llvm.loop.unroll_and_jam.followup_all",
+ options.getFollowupAll());
+}
+
+void LoopAnnotationConversion::convertLoopOptions(LoopLICMAttr options) {
+ addUnitNode("llvm.licm.disable", options.getDisable());
+ addUnitNode("llvm.loop.licm_versioning.disable",
+ options.getVersioningDisable());
+}
+
+void LoopAnnotationConversion::convertLoopOptions(LoopDistributeAttr options) {
+ convertBoolNode("llvm.loop.distribute.enable", options.getDisable(), true);
+ convertFollowupNode("llvm.loop.distribute.followup_coincident",
+ options.getFollowupCoincident());
+ convertFollowupNode("llvm.loop.distribute.followup_sequential",
+ options.getFollowupSequential());
+ convertFollowupNode("llvm.loop.distribute.followup_fallback",
+ options.getFollowupFallback());
+ convertFollowupNode("llvm.loop.distribute.followup_all",
+ options.getFollowupAll());
+}
+
+void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) {
+ convertBoolNode("llvm.loop.pipeline.disable", options.getDisable());
+ convertI32Node("llvm.loop.pipeline.initiationinterval",
+ options.getInitiationinterval());
+}
+
+llvm::MDNode *LoopAnnotationConversion::convert() {
+
+ // Reserve operand 0 for loop id self reference.
+ auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt);
+ metadataNodes.push_back(dummy.get());
+
+ addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced());
+ addUnitNode("llvm.loop.mustprogress", attr.getMustProgress());
+
+ if (auto options = attr.getVectorize())
+ convertLoopOptions(options);
+ if (auto options = attr.getInterleave())
+ convertLoopOptions(options);
+ if (auto options = attr.getUnroll())
+ convertLoopOptions(options);
+ if (auto options = attr.getUnrollAndJam())
+ convertLoopOptions(options);
+ if (auto options = attr.getLicm())
+ convertLoopOptions(options);
+ if (auto options = attr.getDistribute())
+ convertLoopOptions(options);
+ if (auto options = attr.getPipeline())
+ convertLoopOptions(options);
+
+ ArrayRef<SymbolRefAttr> parallelAccessGroups = attr.getParallelAccesses();
+ if (!parallelAccessGroups.empty()) {
+ SmallVector<llvm::Metadata *> parallelAccess;
+ parallelAccess.push_back(
+ llvm::MDString::get(ctx, "llvm.loop.parallel_accesses"));
+ for (SymbolRefAttr accessGroupRef : parallelAccessGroups)
+ parallelAccess.push_back(
+ moduleTranslation.getAccessGroup(*op, accessGroupRef));
+ metadataNodes.push_back(llvm::MDNode::get(ctx, parallelAccess));
+ }
+
+ // Create loop options and set the first operand to itself.
+ llvm::MDNode *loopMD = llvm::MDNode::get(ctx, metadataNodes);
+ loopMD->replaceOperandWith(0, loopMD);
+
+ return loopMD;
+}
+
+llvm::MDNode *LoopAnnotationTranslation::translate(LoopAnnotationAttr attr,
+ Operation *op) {
+ if (!attr)
+ return nullptr;
+
+ llvm::MDNode *loopMD = lookupLoopMetadata(attr);
+ if (loopMD)
+ return loopMD;
+
+ loopMD =
+ LoopAnnotationConversion(attr, moduleTranslation, op, *this).convert();
+ // Store a map from this Attribute to the LLVM metadata in case we
+ // encounter it again.
+ mapLoopMetadata(attr, loopMD);
+ return loopMD;
+}
--- /dev/null
+//===- LoopAnnotationTranslation.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the translation between an MLIR loop annotations and
+// the corresponding LLVMIR metadata representation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_
+#define MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_
+
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Target/LLVMIR/ModuleTranslation.h"
+
+namespace mlir {
+namespace LLVM {
+namespace detail {
+
+/// A helper class that converts a LoopAnnotationAttr into a corresponding
+/// llvm::MDNode.
+class LoopAnnotationTranslation {
+public:
+ LoopAnnotationTranslation(LLVM::ModuleTranslation &moduleTranslation)
+ : moduleTranslation(moduleTranslation) {}
+
+ llvm::MDNode *translate(LoopAnnotationAttr attr, Operation *op);
+
+private:
+ /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute.
+ llvm::MDNode *lookupLoopMetadata(Attribute options) const {
+ return loopMetadataMapping.lookup(options);
+ }
+
+ void mapLoopMetadata(Attribute options, llvm::MDNode *metadata) {
+ auto result = loopMetadataMapping.try_emplace(options, metadata);
+ (void)result;
+ assert(result.second &&
+ "attempting to map loop options that was already mapped");
+ }
+
+ /// Mapping from an attribute describing loop metadata to its LLVM metadata.
+ /// The metadata is attached to Latch block branches with this attribute.
+ DenseMap<Attribute, llvm::MDNode *> loopMetadataMapping;
+
+ LLVM::ModuleTranslation &moduleTranslation;
+};
+
+} // namespace detail
+} // namespace LLVM
+} // namespace mlir
+
+#endif // MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONTRANSLATION_H_
#include "AttrKindDetail.h"
#include "DebugTranslation.h"
+#include "LoopAnnotationTranslation.h"
#include "mlir/Dialect/DLTI/DLTI.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
: mlirModule(module), llvmModule(std::move(llvmModule)),
debugTranslation(
std::make_unique<DebugTranslation>(module, *this->llvmModule)),
+ loopAnnotationTranslation(
+ std::make_unique<LoopAnnotationTranslation>(*this)),
typeTranslator(this->llvmModule->getContext()),
iface(module->getContext()) {
assert(satisfiesLLVMModule(mlirModule) &&
return success();
}
+void ModuleTranslation::setLoopMetadata(Operation *op,
+ llvm::Instruction *inst) {
+ auto attr =
+ op->getAttrOfType<LoopAnnotationAttr>(LLVMDialect::getLoopAttrName());
+ if (!attr)
+ return;
+ llvm::MDNode *loopMD = loopAnnotationTranslation->translate(attr, op);
+ inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD);
+}
+
llvm::Type *ModuleTranslation::convertType(Type type) {
return typeTranslator.translateType(type);
}
module {
llvm.func @loopOptions() {
- // expected-error@below {{expected 'llvm.loop' to be a dictionary attribute}}
+ // expected-error@below {{expected 'llvm.loop' to be a loop annotation attribute}}
llvm.br ^bb4 {llvm.loop = "test"}
^bb4:
llvm.return
module {
llvm.func @loopOptions() {
- // expected-error@below {{expected 'parallel_access' to be an array attribute}}
- llvm.br ^bb4 {llvm.loop = {parallel_access = "loop"}}
- ^bb4:
- llvm.return
- }
-}
-
-// -----
-
-module {
- llvm.func @loopOptions() {
- // expected-error@below {{expected '"loop"' to be a symbol reference}}
- llvm.br ^bb4 {llvm.loop = {parallel_access = ["loop"]}}
- ^bb4:
- llvm.return
- }
-}
-
-// -----
-
-module {
- llvm.func @loopOptions() {
// expected-error@below {{expected '@func1' to reference a metadata op}}
- llvm.br ^bb4 {llvm.loop = {parallel_access = [@func1]}}
+ llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation<parallelAccesses = @func1>}
^bb4:
llvm.return
}
module {
llvm.func @loopOptions() {
// expected-error@below {{expected '@metadata' to reference an access_group op}}
- llvm.br ^bb4 {llvm.loop = {parallel_access = [@metadata]}}
+ llvm.br ^bb4 {llvm.loop = #llvm.loop_annotation<parallelAccesses = @metadata>}
^bb4:
llvm.return
}
// -----
module {
- llvm.func @loopOptions() {
- // expected-error@below {{expected 'options' to be a `loopopts` attribute}}
- llvm.br ^bb4 {llvm.loop = {options = "name"}}
- ^bb4:
- llvm.return
- }
-}
-
-// -----
-
-module {
- llvm.func @loopOptions() {
- // expected-error@below {{unknown loop option: name}}
- llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts<name>}}
- ^bb4:
- llvm.return
- }
-}
-
-// -----
-
-module {
- llvm.func @loopOptions() {
- // expected-error@below {{loop option present twice}}
- llvm.br ^bb4 {llvm.loop = {options = #llvm.loopopts<disable_licm = true, disable_licm = true>}}
- ^bb4:
- llvm.return
- }
-}
-
-// -----
-
-module {
- llvm.func @accessGroups(%arg0 : !llvm.ptr<i32>) {
- // expected-error@below {{attribute 'access_groups' failed to satisfy constraint: symbol ref array attribute}}
- %0 = llvm.load %arg0 { "access_groups" = "test" } : !llvm.ptr<i32>
- llvm.return
- }
-}
-
-// -----
-
-module {
llvm.func @accessGroups(%arg0 : !llvm.ptr<i32>) {
// expected-error@below {{expected '@func1' to specify a fully qualified reference}}
%0 = llvm.load %arg0 { "access_groups" = [@func1] } : !llvm.ptr<i32>
--- /dev/null
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+
+// CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
+#followup = #llvm.loop_annotation<disableNonforced = true>
+
+// CHECK-DAG: #[[VECTORIZE:.*]] = #llvm.loop_vectorize<disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32, followupVectorized = #[[FOLLOWUP]], followupEpilogue = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
+#vectorize = #llvm.loop_vectorize<
+ disable = false, predicateEnable = false, scalableEnable = true, width = 16 : i32,
+ followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup
+>
+
+// CHECK-DAG: #[[INTERLEAVE:.*]] = #llvm.loop_interleave<count = 32 : i32>
+#interleave = #llvm.loop_interleave<count = 32 : i32>
+
+// CHECK-DAG: #[[UNROLL:.*]] = #llvm.loop_unroll<disable = true, count = 32 : i32, runtimeDisable = true, full = false, followup = #[[FOLLOWUP]], followupRemainder = #[[FOLLOWUP]]>
+#unroll = #llvm.loop_unroll<
+ disable = true, count = 32 : i32, runtimeDisable = true, full = false,
+ followup = #followup, followupRemainder = #followup
+>
+
+// CHECK-DAG: #[[UNROLL_AND_JAM:.*]] = #llvm.loop_unroll_and_jam<disable = false, count = 16 : i32, followupOuter = #[[FOLLOWUP]], followupInner = #[[FOLLOWUP]], followupRemainderOuter = #[[FOLLOWUP]], followupRemainderInner = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
+#unrollAndJam = #llvm.loop_unroll_and_jam<
+ disable = false, count = 16 : i32, followupOuter = #followup, followupInner = #followup,
+ followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup
+>
+
+// CHECK-DAG: #[[LICM:.*]] = #llvm.loop_licm<disable = false, versioningDisable = true>
+#licm = #llvm.loop_licm<disable = false, versioningDisable = true>
+
+// CHECK-DAG: #[[DISTRIBUTE:.*]] = #llvm.loop_distribute<disable = true, followupCoincident = #[[FOLLOWUP]], followupSequential = #[[FOLLOWUP]], followupFallback = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
+#distribute = #llvm.loop_distribute<
+ disable = true, followupCoincident = #followup, followupSequential = #followup,
+ followupFallback = #followup, followupAll = #followup
+>
+
+// CHECK-DAG: #[[PIPELINE:.*]] = #llvm.loop_pipeline<disable = true, initiationinterval = 1 : i32>
+#pipeline = #llvm.loop_pipeline<disable = true, initiationinterval = 1 : i32>
+
+// CHECK: #[[LOOP_ANNOT:.*]] = #llvm.loop_annotation<
+// CHECK-DAG: disableNonforced = false
+// CHECK-DAG: mustProgress = true
+// CHECK-DAG: unroll = #[[UNROLL]]
+// CHECK-DAG: unrollAndJam = #[[UNROLL_AND_JAM]]
+// CHECK-DAG: licm = #[[LICM]]
+// CHECK-DAG: distribute = #[[DISTRIBUTE]]
+// CHECK-DAG: pipeline = #[[PIPELINE]]
+// CHECK-DAG: parallelAccesses = @metadata::@group1, @metadata::@group2>
+#loopMD = #llvm.loop_annotation<disableNonforced = false,
+ mustProgress = true,
+ vectorize = #vectorize,
+ interleave = #interleave,
+ unroll = #unroll,
+ unrollAndJam = #unrollAndJam,
+ licm = #licm,
+ distribute = #distribute,
+ pipeline = #pipeline,
+ parallelAccesses = @metadata::@group1, @metadata::@group2>
+
+// CHECK: llvm.func @loop_annotation
+llvm.func @loop_annotation() {
+ // CHECK: llvm.br ^bb1 {llvm.loop = #[[LOOP_ANNOT]]
+ llvm.br ^bb1 {llvm.loop = #loopMD}
+^bb1:
+ llvm.return
+}
+
+llvm.metadata @metadata {
+ llvm.access_group @group1
+ llvm.access_group @group2
+}
return
}
-module {
- // CHECK-LABEL: @loopOptions
- llvm.func @loopOptions() {
- // CHECK: llvm.br
- // CHECK-SAME: llvm.loop = {options = #llvm.loopopts<disable_unroll = true, disable_licm = true, interleave_count = 1, disable_pipeline = true, pipeline_initiation_interval = 1>}, parallel_access = [@metadata::@group1]}
- llvm.br ^bb1 {llvm.loop = {options = #llvm.loopopts<disable_unroll = true, disable_licm = true, interleave_count = 1, disable_pipeline = true, pipeline_initiation_interval = 1>}, parallel_access = [@metadata::@group1]}
- ^bb1:
- llvm.return
- }
- // CHECK: llvm.metadata @metadata attributes {test_attribute} {
- llvm.metadata @metadata attributes {test_attribute} {
- // CHECK: llvm.access_group @group1
- llvm.access_group @group1
- }
-}
-
// CHECK-LABEL: llvm.func @vararg_func
llvm.func @vararg_func(%arg0: i32, ...) {
// CHECK: %{{.*}} = llvm.mlir.constant(1 : i32) : i32
// -----
module {
- llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) {
- %0 = llvm.mlir.constant(0 : i32) : i32
- %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr<i32>)
- llvm.br ^bb3(%0 : i32)
- ^bb3(%1: i32):
- %2 = llvm.icmp "slt" %1, %arg1 : i32
- // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
- llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts<disable_licm = true, disable_unroll = true, interleave_count = 1, disable_pipeline = true, pipeline_initiation_interval = 2>}}
- ^bb4:
- %3 = llvm.add %1, %arg2 : i32
- // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]]
- %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr<i32>
- // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]]
- llvm.br ^bb3(%3 : i32) {llvm.loop = {parallel_access = [@metadata::@group1, @metadata::@group2], options = #llvm.loopopts<disable_unroll = true, disable_licm = true, interleave_count = 1, disable_pipeline = true, pipeline_initiation_interval = 2>}}
- ^bb5:
- llvm.return
- }
-
- llvm.metadata @metadata {
- llvm.access_group @group1
- llvm.access_group @group2
- }
-}
-
-// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[PA_NODE:[0-9]+]], ![[UNROLL_DISABLE_NODE:[0-9]+]], ![[LICM_DISABLE_NODE:[0-9]+]], ![[INTERLEAVE_NODE:[0-9]+]], ![[PIPELINE_DISABLE_NODE:[0-9]+]], ![[II_NODE:[0-9]+]]}
-// CHECK: ![[PA_NODE]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]}
-// CHECK: ![[GROUP_NODE1]] = distinct !{}
-// CHECK: ![[GROUP_NODE2]] = distinct !{}
-// CHECK: ![[UNROLL_DISABLE_NODE]] = !{!"llvm.loop.unroll.disable", i1 true}
-// CHECK: ![[LICM_DISABLE_NODE]] = !{!"llvm.licm.disable", i1 true}
-// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 1}
-// CHECK: ![[PIPELINE_DISABLE_NODE]] = !{!"llvm.loop.pipeline.disable", i1 true}
-// CHECK: ![[II_NODE]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2}
-// CHECK: ![[ACCESS_GROUPS_NODE]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]}
-
-// -----
-
-module {
llvm.func @aliasScope(%arg1 : !llvm.ptr<i32>, %arg2 : !llvm.ptr<i32>, %arg3 : !llvm.ptr<i32>) {
%0 = llvm.mlir.constant(0 : i32) : i32
llvm.store %0, %arg1 { alias_scopes = [@metadata::@scope1], noalias_scopes = [@metadata::@scope2, @metadata::@scope3] } : !llvm.ptr<i32>
--- /dev/null
+// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: @disableNonForced
+llvm.func @disableNonForced() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<disableNonforced = true>}
+^bb1:
+ llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.disable_nonforced"}
+
+// -----
+
+// CHECK-LABEL: @mustprogress
+llvm.func @mustprogress() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<mustProgress = true>}
+^bb1:
+ llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.mustprogress"}
+
+// -----
+
+
+#followup = #llvm.loop_annotation<disableNonforced = true>
+
+// CHECK-LABEL: @vectorizeOptions
+llvm.func @vectorizeOptions() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<vectorize = <
+ disable = false, predicateEnable = true, scalableEnable = false, width = 16 : i32,
+ followupVectorized = #followup, followupEpilogue = #followup, followupAll = #followup>
+ >}
+^bb1:
+ llvm.return
+}
+
+// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"}
+// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]}
+// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.width", i32 16}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_vectorized", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_epilogue", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_all", ![[FOLLOWUP]]}
+
+// -----
+
+// CHECK-LABEL: @interleaveOptions
+llvm.func @interleaveOptions() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<interleave = <count = 32 : i32>>}
+^bb1:
+ llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], ![[INTERLEAVE_NODE:[0-9]+]]}
+// CHECK: ![[INTERLEAVE_NODE]] = !{!"llvm.loop.interleave.count", i32 32}
+
+// -----
+
+#followup = #llvm.loop_annotation<disableNonforced = true>
+
+// CHECK-LABEL: @unrollOptions
+llvm.func @unrollOptions() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unroll = <
+ disable = true, count = 64 : i32, runtimeDisable = false, full = false,
+ followup = #followup, followupRemainder = #followup>
+ >}
+^bb1:
+ llvm.return
+}
+
+// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"}
+// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]}
+// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.disable"}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.count", i32 64}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.runtime.disable", i1 false}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll.followup_remainder", ![[FOLLOWUP]]}
+
+// -----
+
+// CHECK-LABEL: @unrollOptions2
+llvm.func @unrollOptions2() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unroll = <disable = false, full = true>>}
+^bb1:
+ llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.unroll.enable"}
+// CHECK-DAG: ![[VEC_NODE2:[0-9]+]] = !{!"llvm.loop.unroll.full"}
+
+// -----
+
+#followup = #llvm.loop_annotation<disableNonforced = true>
+
+// CHECK-LABEL: @unrollAndJamOptions
+llvm.func @unrollAndJamOptions() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<unrollAndJam = <
+ disable = false, count = 8 : i32, followupOuter = #followup, followupInner = #followup,
+ followupRemainderOuter = #followup, followupRemainderInner = #followup, followupAll = #followup>
+ >}
+^bb1:
+ llvm.return
+}
+
+// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"}
+// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]}
+// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.enable"}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.count", i32 8}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_inner", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.unroll_and_jam.followup_all", ![[FOLLOWUP]]}
+
+// -----
+
+// CHECK-LABEL: @licmOptions
+llvm.func @licmOptions() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<licm = <disable = false, versioningDisable = true>>}
+^bb1:
+ llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.licm_versioning.disable"}
+
+// -----
+
+// CHECK-LABEL: @licmOptions2
+llvm.func @licmOptions2() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<licm = <disable = true, versioningDisable = false>>}
+^bb1:
+ llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.licm.disable"}
+
+// -----
+
+#followup = #llvm.loop_annotation<disableNonforced = true>
+
+// CHECK-LABEL: @distributeOptions
+llvm.func @distributeOptions() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<distribute = <
+ disable = true, followupCoincident = #followup, followupSequential = #followup,
+ followupFallback = #followup, followupAll = #followup>
+ >}
+^bb1:
+ llvm.return
+}
+
+// CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"}
+// CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]}
+// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.enable", i1 false}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_coincident", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_sequential", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_fallback", ![[FOLLOWUP]]}
+// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.distribute.followup_all", ![[FOLLOWUP]]}
+
+// -----
+
+// CHECK-LABEL: @pipelineOptions
+llvm.func @pipelineOptions() {
+ // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.br ^bb1 {llvm.loop = #llvm.loop_annotation<pipeline = <disable = false, initiationinterval = 1 : i32>>}
+^bb1:
+ llvm.return
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 false}
+// CHECK-DAG: ![[VEC_NODE0:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 1}
+
+// -----
+
+// CHECK-LABEL: @loopOptions
+llvm.func @loopOptions(%arg1 : i32, %arg2 : i32) {
+ %0 = llvm.mlir.constant(0 : i32) : i32
+ %4 = llvm.alloca %arg1 x i32 : (i32) -> (!llvm.ptr<i32>)
+ llvm.br ^bb3(%0 : i32)
+ ^bb3(%1: i32):
+ %2 = llvm.icmp "slt" %1, %arg1 : i32
+ // CHECK: br i1 {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]]
+ llvm.cond_br %2, ^bb4, ^bb5 {llvm.loop = #llvm.loop_annotation<
+ licm = <disable = true>,
+ interleave = <count = 1>,
+ unroll = <disable = true>, pipeline = <disable = true, initiationinterval = 2>,
+ parallelAccesses = @metadata::@group1, @metadata::@group2>}
+ ^bb4:
+ %3 = llvm.add %1, %arg2 : i32
+ // CHECK: = load i32, ptr %{{.*}} !llvm.access.group ![[ACCESS_GROUPS_NODE:[0-9]+]]
+ %5 = llvm.load %4 { access_groups = [@metadata::@group1, @metadata::@group2] } : !llvm.ptr<i32>
+ // CHECK: br label {{.*}} !llvm.loop ![[LOOP_NODE]]
+ llvm.br ^bb3(%3 : i32) {llvm.loop = #llvm.loop_annotation<
+ licm = <disable = true>,
+ interleave = <count = 1>,
+ unroll = <disable = true>, pipeline = <disable = true, initiationinterval = 2>,
+ parallelAccesses = @metadata::@group1, @metadata::@group2>}
+
+ ^bb5:
+ llvm.return
+}
+
+llvm.metadata @metadata {
+ llvm.access_group @group1
+ llvm.access_group @group2
+}
+
+// CHECK: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}}
+// CHECK-DAG: ![[PA_NODE:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[GROUP_NODE1:[0-9]+]], ![[GROUP_NODE2:[0-9]+]]}
+// CHECK-DAG: ![[GROUP_NODE1:[0-9]+]] = distinct !{}
+// CHECK-DAG: ![[GROUP_NODE2:[0-9]+]] = distinct !{}
+// CHECK-DAG: ![[UNROLL_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.unroll.disable"}
+// CHECK-DAG: ![[LICM_DISABLE_NODE:[0-9]+]] = !{!"llvm.licm.disable"}
+// CHECK-DAG: ![[INTERLEAVE_NODE:[0-9]+]] = !{!"llvm.loop.interleave.count", i32 1}
+// CHECK-DAG: ![[PIPELINE_DISABLE_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.disable", i1 true}
+// CHECK-DAG: ![[II_NODE:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 2}
+// CHECK-DAG: ![[ACCESS_GROUPS_NODE:[0-9]+]] = !{![[GROUP_NODE1]], ![[GROUP_NODE2]]}
projects / platform / upstream / llvm.git / commitdiff