namespace detail {
class DebugImporter;
+class LoopAnnotationImporter;
} // namespace detail
/// Module import implementation class that provides methods to import globals
return tbaaMapping.lookup(node);
}
- /// Returns the MLIR symbol reference mapped to the given LLVM access
- /// group metadata `node`.
- SymbolRefAttr lookupAccessGroupAttr(const llvm::MDNode *node) const {
- return accessGroupMapping.lookup(node);
- }
+ /// Returns the symbol references pointing to the access group operations that
+ /// map to the access group nodes starting from the access group metadata
+ /// `node`. Returns failure, if any of the symbol references cannot be found.
+ FailureOr<SmallVector<SymbolRefAttr>>
+ lookupAccessGroupAttrs(const llvm::MDNode *node) const;
+
+ /// Returns the loop annotation attribute that corresponds to the given LLVM
+ /// loop metadata `node`.
+ LoopAnnotationAttr translateLoopAnnotationAttr(const llvm::MDNode *node,
+ Location loc) const;
private:
/// Clears the block and value mapping before processing a new region.
LLVM::TypeFromLLVMIRTranslator typeTranslator;
/// Stateful debug information importer.
std::unique_ptr<detail::DebugImporter> debugImporter;
+ /// Loop annotation importer.
+ std::unique_ptr<detail::LoopAnnotationImporter> loopAnnotationImporter;
};
} // namespace LLVM
ConvertToLLVMIR.cpp
DebugTranslation.cpp
DebugImporter.cpp
+ LoopAnnotationImporter.cpp
LoopAnnotationTranslation.cpp
ModuleTranslation.cpp
ModuleImport.cpp
add_mlir_translation_library(MLIRTargetLLVMIRImport
DebugImporter.cpp
+ LoopAnnotationImporter.cpp
ModuleImport.cpp
TypeFromLLVM.cpp
static ArrayRef<unsigned> getSupportedMetadataImpl() {
static const SmallVector<unsigned> convertibleMetadata = {
llvm::LLVMContext::MD_prof, llvm::LLVMContext::MD_tbaa,
- llvm::LLVMContext::MD_access_group};
+ llvm::LLVMContext::MD_access_group, llvm::LLVMContext::MD_loop};
return convertibleMetadata;
}
return success();
}
-/// Searches the symbol references pointing to the access group operations that
-/// map to the access group nodes starting from the access group metadata
+/// Looks up all the symbol references pointing to the access group operations
+/// that map to the access group nodes starting from the access group metadata
/// `node`, and attaches all of them to the imported operation if the lookups
/// succeed. Returns failure otherwise.
static LogicalResult setAccessGroupAttr(const llvm::MDNode *node, Operation *op,
LLVM::ModuleImport &moduleImport) {
- // An access group node is either access group or an access group list.
- SmallVector<Attribute> accessGroups;
- if (!node->getNumOperands())
- accessGroups.push_back(moduleImport.lookupAccessGroupAttr(node));
- for (const llvm::MDOperand &operand : node->operands()) {
- auto *node = cast<llvm::MDNode>(operand.get());
- accessGroups.push_back(moduleImport.lookupAccessGroupAttr(node));
- }
- // Exit if one of the access group node lookups failed.
- if (llvm::is_contained(accessGroups, nullptr))
+ FailureOr<SmallVector<SymbolRefAttr>> accessGroups =
+ moduleImport.lookupAccessGroupAttrs(node);
+ if (failed(accessGroups))
return failure();
+ SmallVector<Attribute> accessGroupAttrs(accessGroups->begin(),
+ accessGroups->end());
op->setAttr(LLVMDialect::getAccessGroupsAttrName(),
- ArrayAttr::get(op->getContext(), accessGroups));
+ ArrayAttr::get(op->getContext(), accessGroupAttrs));
return success();
}
+
+/// Converts the given loop metadata node to an MLIR loop annotation attribute
+/// and attaches it to the imported operation if the translation succeeds.
+/// Returns failure otherwise.
+static LogicalResult setLoopAttr(const llvm::MDNode *node, Operation *op,
+ LLVM::ModuleImport &moduleImport) {
+ LoopAnnotationAttr attr =
+ moduleImport.translateLoopAnnotationAttr(node, op->getLoc());
+ if (!attr)
+ return failure();
+
+ op->setAttr(LLVMDialect::getLoopAttrName(), attr);
+ return success();
+}
+
namespace {
/// Implementation of the dialect interface that converts operations belonging
return setTBAAAttr(node, op, moduleImport);
if (kind == llvm::LLVMContext::MD_access_group)
return setAccessGroupAttr(node, op, moduleImport);
+ if (kind == llvm::LLVMContext::MD_loop)
+ return setLoopAttr(node, op, moduleImport);
// A handler for a supported metadata kind is missing.
llvm_unreachable("unknown metadata type");
--- /dev/null
+//===- LoopAnnotationImporter.cpp - Loop annotation import ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LoopAnnotationImporter.h"
+#include "llvm/IR/Constants.h"
+
+using namespace mlir;
+using namespace mlir::LLVM;
+using namespace mlir::LLVM::detail;
+
+namespace {
+/// Helper class that keeps the state of one metadata to attribute conversion.
+struct LoopMetadataConversion {
+ LoopMetadataConversion(const llvm::MDNode *node, ModuleImport &moduleImport,
+ Location loc,
+ LoopAnnotationImporter &loopAnnotationImporter)
+ : node(node), moduleImport(moduleImport), loc(loc),
+ loopAnnotationImporter(loopAnnotationImporter),
+ ctx(loc->getContext()){};
+ /// Converts this structs loop metadata node into a LoopAnnotationAttr.
+ LoopAnnotationAttr convert();
+
+ LogicalResult initPropertyMap();
+
+ /// Helper function to get and erase a property.
+ const llvm::MDNode *lookupAndEraseProperty(StringRef name);
+
+ /// Helper functions to lookup and convert MDNodes into a specifc attribute
+ /// kind. These functions return null-attributes if there is no node with the
+ /// specified name, or failure, if the node is ill-formatted.
+ FailureOr<BoolAttr> lookupUnitNode(StringRef name);
+ FailureOr<BoolAttr> lookupBoolNode(StringRef name, bool negated = false);
+ FailureOr<IntegerAttr> lookupIntNode(StringRef name);
+ FailureOr<llvm::MDNode *> lookupMDNode(StringRef name);
+ FailureOr<SmallVector<llvm::MDNode *>> lookupMDNodes(StringRef name);
+ FailureOr<LoopAnnotationAttr> lookupFollowupNode(StringRef name);
+ FailureOr<BoolAttr> lookupBooleanUnitNode(StringRef enableName,
+ StringRef disableName,
+ bool negated = false);
+
+ /// Conversion functions for sub-attributes.
+ FailureOr<LoopVectorizeAttr> convertVectorizeAttr();
+ FailureOr<LoopInterleaveAttr> convertInterleaveAttr();
+ FailureOr<LoopUnrollAttr> convertUnrollAttr();
+ FailureOr<LoopUnrollAndJamAttr> convertUnrollAndJamAttr();
+ FailureOr<LoopLICMAttr> convertLICMAttr();
+ FailureOr<LoopDistributeAttr> convertDistributeAttr();
+ FailureOr<LoopPipelineAttr> convertPipelineAttr();
+ FailureOr<SmallVector<SymbolRefAttr>> convertParallelAccesses();
+
+ llvm::StringMap<const llvm::MDNode *> propertyMap;
+ const llvm::MDNode *node;
+ ModuleImport &moduleImport;
+ Location loc;
+ LoopAnnotationImporter &loopAnnotationImporter;
+ MLIRContext *ctx;
+};
+} // namespace
+
+LogicalResult LoopMetadataConversion::initPropertyMap() {
+ // Check if it's a valid node.
+ if (node->getNumOperands() == 0 ||
+ dyn_cast<llvm::MDNode>(node->getOperand(0)) != node)
+ return emitWarning(loc) << "invalid loop node";
+
+ for (const llvm::MDOperand &operand : llvm::drop_begin(node->operands())) {
+ // Skip over DILocations.
+ if (isa<llvm::DILocation>(operand))
+ continue;
+
+ auto *property = dyn_cast<llvm::MDNode>(operand);
+ if (!property)
+ return emitWarning(loc) << "expected all loop properties to be either "
+ "debug locations or metadata nodes";
+
+ if (property->getNumOperands() == 0)
+ return emitWarning(loc) << "cannot import empty loop property";
+
+ auto *nameNode = dyn_cast<llvm::MDString>(property->getOperand(0));
+ if (!nameNode)
+ return emitWarning(loc) << "cannot import loop property without a name";
+ StringRef name = nameNode->getString();
+
+ bool succ = propertyMap.try_emplace(name, property).second;
+ if (!succ)
+ return emitWarning(loc)
+ << "cannot import loop properties with duplicated names " << name;
+ }
+
+ return success();
+}
+
+const llvm::MDNode *
+LoopMetadataConversion::lookupAndEraseProperty(StringRef name) {
+ auto it = propertyMap.find(name);
+ if (it == propertyMap.end())
+ return nullptr;
+ const llvm::MDNode *property = it->getValue();
+ propertyMap.erase(it);
+ return property;
+}
+
+FailureOr<BoolAttr> LoopMetadataConversion::lookupUnitNode(StringRef name) {
+ const llvm::MDNode *property = lookupAndEraseProperty(name);
+ if (!property)
+ return BoolAttr(nullptr);
+
+ if (property->getNumOperands() != 1)
+ return emitWarning(loc)
+ << "expected metadata node " << name << " to hold no value";
+
+ return BoolAttr::get(ctx, true);
+}
+
+FailureOr<BoolAttr> LoopMetadataConversion::lookupBooleanUnitNode(
+ StringRef enableName, StringRef disableName, bool negated) {
+ auto enable = lookupUnitNode(enableName);
+ auto disable = lookupUnitNode(disableName);
+ if (failed(enable) || failed(disable))
+ return failure();
+
+ if (*enable && *disable)
+ return emitWarning(loc)
+ << "expected metadata nodes " << enableName << " and " << disableName
+ << " to be mutually exclusive.";
+
+ if (*enable)
+ return BoolAttr::get(ctx, !negated);
+
+ if (*disable)
+ return BoolAttr::get(ctx, negated);
+ return BoolAttr(nullptr);
+}
+
+FailureOr<BoolAttr> LoopMetadataConversion::lookupBoolNode(StringRef name,
+ bool negated) {
+ const llvm::MDNode *property = lookupAndEraseProperty(name);
+ if (!property)
+ return BoolAttr(nullptr);
+
+ auto emitNodeWarning = [&]() {
+ return emitWarning(loc)
+ << "expected metadata node " << name << " to hold a boolean value";
+ };
+
+ if (property->getNumOperands() != 2)
+ return emitNodeWarning();
+ llvm::ConstantInt *val =
+ llvm::mdconst::dyn_extract<llvm::ConstantInt>(property->getOperand(1));
+ if (!val || val->getBitWidth() != 1)
+ return emitNodeWarning();
+
+ return BoolAttr::get(ctx, val->getValue().getLimitedValue(1) ^ negated);
+}
+
+FailureOr<IntegerAttr> LoopMetadataConversion::lookupIntNode(StringRef name) {
+ const llvm::MDNode *property = lookupAndEraseProperty(name);
+ if (!property)
+ return IntegerAttr(nullptr);
+
+ auto emitNodeWarning = [&]() {
+ return emitWarning(loc)
+ << "expected metadata node " << name << " to hold an i32 value";
+ };
+
+ if (property->getNumOperands() != 2)
+ return emitNodeWarning();
+
+ llvm::ConstantInt *val =
+ llvm::mdconst::dyn_extract<llvm::ConstantInt>(property->getOperand(1));
+ if (!val || val->getBitWidth() != 32)
+ return emitNodeWarning();
+
+ return IntegerAttr::get(IntegerType::get(ctx, 32),
+ val->getValue().getLimitedValue());
+}
+
+FailureOr<llvm::MDNode *> LoopMetadataConversion::lookupMDNode(StringRef name) {
+ const llvm::MDNode *property = lookupAndEraseProperty(name);
+ if (!property)
+ return nullptr;
+
+ auto emitNodeWarning = [&]() {
+ return emitWarning(loc)
+ << "expected metadata node " << name << " to hold an MDNode";
+ };
+
+ if (property->getNumOperands() != 2)
+ return emitNodeWarning();
+
+ auto *node = dyn_cast<llvm::MDNode>(property->getOperand(1));
+ if (!node)
+ return emitNodeWarning();
+
+ return node;
+}
+
+FailureOr<SmallVector<llvm::MDNode *>>
+LoopMetadataConversion::lookupMDNodes(StringRef name) {
+ const llvm::MDNode *property = lookupAndEraseProperty(name);
+ SmallVector<llvm::MDNode *> res;
+ if (!property)
+ return res;
+
+ auto emitNodeWarning = [&]() {
+ return emitWarning(loc) << "expected metadata node " << name
+ << " to hold one or multiple MDNodes";
+ };
+
+ if (property->getNumOperands() < 2)
+ return emitNodeWarning();
+
+ for (unsigned i = 1, e = property->getNumOperands(); i < e; ++i) {
+ auto *node = dyn_cast<llvm::MDNode>(property->getOperand(i));
+ if (!node)
+ return emitNodeWarning();
+ res.push_back(node);
+ }
+
+ return res;
+}
+
+FailureOr<LoopAnnotationAttr>
+LoopMetadataConversion::lookupFollowupNode(StringRef name) {
+ auto node = lookupMDNode(name);
+ if (failed(node))
+ return failure();
+ if (*node == nullptr)
+ return LoopAnnotationAttr(nullptr);
+
+ return loopAnnotationImporter.translate(*node, loc);
+}
+
+static bool isEmptyOrNull(const Attribute attr) { return !attr; }
+
+template <typename T>
+static bool isEmptyOrNull(const SmallVectorImpl<T> &vec) {
+ return vec.empty();
+}
+
+/// Helper function that only creates and attribute of type T if all argument
+/// conversion were successfull and at least one of them holds a non-null value.
+template <typename T, typename... P>
+static T createIfNonNull(MLIRContext *ctx, const P &...args) {
+ bool anyFailed = (failed(args) || ...);
+ if (anyFailed)
+ return {};
+
+ bool allEmpty = (isEmptyOrNull(*args) && ...);
+ if (allEmpty)
+ return {};
+
+ return T::get(ctx, *args...);
+}
+
+FailureOr<LoopVectorizeAttr> LoopMetadataConversion::convertVectorizeAttr() {
+ FailureOr<BoolAttr> enable =
+ lookupBoolNode("llvm.loop.vectorize.enable", true);
+ FailureOr<BoolAttr> predicateEnable =
+ lookupBoolNode("llvm.loop.vectorize.predicate.enable");
+ FailureOr<BoolAttr> scalableEnable =
+ lookupBoolNode("llvm.loop.vectorize.scalable.enable");
+ FailureOr<IntegerAttr> width = lookupIntNode("llvm.loop.vectorize.width");
+ FailureOr<LoopAnnotationAttr> followupVec =
+ lookupFollowupNode("llvm.loop.vectorize.followup_vectorized");
+ FailureOr<LoopAnnotationAttr> followupEpi =
+ lookupFollowupNode("llvm.loop.vectorize.followup_epilogue");
+ FailureOr<LoopAnnotationAttr> followupAll =
+ lookupFollowupNode("llvm.loop.vectorize.followup_all");
+
+ return createIfNonNull<LoopVectorizeAttr>(ctx, enable, predicateEnable,
+ scalableEnable, width, followupVec,
+ followupEpi, followupAll);
+}
+
+FailureOr<LoopInterleaveAttr> LoopMetadataConversion::convertInterleaveAttr() {
+ FailureOr<IntegerAttr> count = lookupIntNode("llvm.loop.interleave.count");
+ return createIfNonNull<LoopInterleaveAttr>(ctx, count);
+}
+
+FailureOr<LoopUnrollAttr> LoopMetadataConversion::convertUnrollAttr() {
+ FailureOr<BoolAttr> disable = lookupBooleanUnitNode(
+ "llvm.loop.unroll.enable", "llvm.loop.unroll.disable", /*negated=*/true);
+ FailureOr<IntegerAttr> count = lookupIntNode("llvm.loop.unroll.count");
+ FailureOr<BoolAttr> runtimeDisable =
+ lookupUnitNode("llvm.loop.unroll.runtime.disable");
+ FailureOr<BoolAttr> full = lookupUnitNode("llvm.loop.unroll.full");
+ FailureOr<LoopAnnotationAttr> followup =
+ lookupFollowupNode("llvm.loop.unroll.followup");
+ FailureOr<LoopAnnotationAttr> followupRemainder =
+ lookupFollowupNode("llvm.loop.unroll.followup_remainder");
+
+ return createIfNonNull<LoopUnrollAttr>(ctx, disable, count, runtimeDisable,
+ full, followup, followupRemainder);
+}
+
+FailureOr<LoopUnrollAndJamAttr>
+LoopMetadataConversion::convertUnrollAndJamAttr() {
+ FailureOr<BoolAttr> disable = lookupBooleanUnitNode(
+ "llvm.loop.unroll_and_jam.enable", "llvm.loop.unroll_and_jam.disable",
+ /*negated=*/true);
+ FailureOr<IntegerAttr> count =
+ lookupIntNode("llvm.loop.unroll_and_jam.count");
+ FailureOr<LoopAnnotationAttr> followupOuter =
+ lookupFollowupNode("llvm.loop.unroll_and_jam.followup_outer");
+ FailureOr<LoopAnnotationAttr> followupInner =
+ lookupFollowupNode("llvm.loop.unroll_and_jam.followup_inner");
+ FailureOr<LoopAnnotationAttr> followupRemainderOuter =
+ lookupFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer");
+ FailureOr<LoopAnnotationAttr> followupRemainderInner =
+ lookupFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner");
+ FailureOr<LoopAnnotationAttr> followupAll =
+ lookupFollowupNode("llvm.loop.unroll_and_jam.followup_all");
+ return createIfNonNull<LoopUnrollAndJamAttr>(
+ ctx, disable, count, followupOuter, followupInner, followupRemainderOuter,
+ followupRemainderInner, followupAll);
+}
+
+FailureOr<LoopLICMAttr> LoopMetadataConversion::convertLICMAttr() {
+ FailureOr<BoolAttr> disable = lookupUnitNode("llvm.licm.disable");
+ FailureOr<BoolAttr> versioningDisable =
+ lookupUnitNode("llvm.loop.licm_versioning.disable");
+ return createIfNonNull<LoopLICMAttr>(ctx, disable, versioningDisable);
+}
+
+FailureOr<LoopDistributeAttr> LoopMetadataConversion::convertDistributeAttr() {
+ FailureOr<BoolAttr> disable =
+ lookupBoolNode("llvm.loop.distribute.enable", true);
+ FailureOr<LoopAnnotationAttr> followupCoincident =
+ lookupFollowupNode("llvm.loop.distribute.followup_coincident");
+ FailureOr<LoopAnnotationAttr> followupSequential =
+ lookupFollowupNode("llvm.loop.distribute.followup_sequential");
+ FailureOr<LoopAnnotationAttr> followupFallback =
+ lookupFollowupNode("llvm.loop.distribute.followup_fallback");
+ FailureOr<LoopAnnotationAttr> followupAll =
+ lookupFollowupNode("llvm.loop.distribute.followup_all");
+ return createIfNonNull<LoopDistributeAttr>(ctx, disable, followupCoincident,
+ followupSequential,
+ followupFallback, followupAll);
+}
+
+FailureOr<LoopPipelineAttr> LoopMetadataConversion::convertPipelineAttr() {
+ FailureOr<BoolAttr> disable = lookupBoolNode("llvm.loop.pipeline.disable");
+ FailureOr<IntegerAttr> initiationinterval =
+ lookupIntNode("llvm.loop.pipeline.initiationinterval");
+ return createIfNonNull<LoopPipelineAttr>(ctx, disable, initiationinterval);
+}
+
+FailureOr<SmallVector<SymbolRefAttr>>
+LoopMetadataConversion::convertParallelAccesses() {
+ FailureOr<SmallVector<llvm::MDNode *>> nodes =
+ lookupMDNodes("llvm.loop.parallel_accesses");
+ if (failed(nodes))
+ return failure();
+ SmallVector<SymbolRefAttr> refs;
+ for (llvm::MDNode *node : *nodes) {
+ FailureOr<SmallVector<SymbolRefAttr>> accessGroups =
+ moduleImport.lookupAccessGroupAttrs(node);
+ if (failed(accessGroups))
+ return emitWarning(loc) << "could not lookup access group";
+ llvm::append_range(refs, *accessGroups);
+ }
+ return refs;
+}
+
+LoopAnnotationAttr LoopMetadataConversion::convert() {
+ if (failed(initPropertyMap()))
+ return {};
+
+ FailureOr<BoolAttr> disableNonForced =
+ lookupUnitNode("llvm.loop.disable_nonforced");
+ FailureOr<LoopVectorizeAttr> vecAttr = convertVectorizeAttr();
+ FailureOr<LoopInterleaveAttr> interleaveAttr = convertInterleaveAttr();
+ FailureOr<LoopUnrollAttr> unrollAttr = convertUnrollAttr();
+ FailureOr<LoopUnrollAndJamAttr> unrollAndJamAttr = convertUnrollAndJamAttr();
+ FailureOr<LoopLICMAttr> licmAttr = convertLICMAttr();
+ FailureOr<LoopDistributeAttr> distributeAttr = convertDistributeAttr();
+ FailureOr<LoopPipelineAttr> pipelineAttr = convertPipelineAttr();
+ FailureOr<BoolAttr> mustProgress = lookupUnitNode("llvm.loop.mustprogress");
+ FailureOr<SmallVector<SymbolRefAttr>> parallelAccesses =
+ convertParallelAccesses();
+
+ // Drop the metadata if there are parts that cannot be imported.
+ if (!propertyMap.empty()) {
+ for (auto name : propertyMap.keys())
+ emitWarning(loc) << "unknown loop annotation " << name;
+ return {};
+ }
+
+ return createIfNonNull<LoopAnnotationAttr>(
+ ctx, disableNonForced, vecAttr, interleaveAttr, unrollAttr,
+ unrollAndJamAttr, licmAttr, distributeAttr, pipelineAttr, mustProgress,
+ parallelAccesses);
+}
+
+LoopAnnotationAttr LoopAnnotationImporter::translate(const llvm::MDNode *node,
+ Location loc) {
+ if (!node)
+ return {};
+
+ // Note: This check is necessary to distinguish between failed translations
+ // and not yet attempted translations.
+ auto it = loopMetadataMapping.find(node);
+ if (it != loopMetadataMapping.end())
+ return it->getSecond();
+
+ LoopAnnotationAttr attr =
+ LoopMetadataConversion(node, moduleImport, loc, *this).convert();
+
+ mapLoopMetadata(node, attr);
+ return attr;
+}
--- /dev/null
+//===- LoopAnnotationImporter.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the translation between LLVMIR loop metadata and the
+// corresponding MLIR representation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_
+#define MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_
+
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Target/LLVMIR/ModuleImport.h"
+
+namespace mlir {
+namespace LLVM {
+namespace detail {
+
+/// A helper class that converts a `llvm.loop` metadata node into a
+/// corresponding LoopAnnotationAttr.
+class LoopAnnotationImporter {
+public:
+ explicit LoopAnnotationImporter(ModuleImport &moduleImport)
+ : moduleImport(moduleImport) {}
+ LoopAnnotationAttr translate(const llvm::MDNode *node, Location loc);
+
+private:
+ /// Returns the LLVM metadata corresponding to a llvm loop metadata attribute.
+ LoopAnnotationAttr lookupLoopMetadata(const llvm::MDNode *node) const {
+ return loopMetadataMapping.lookup(node);
+ }
+
+ void mapLoopMetadata(const llvm::MDNode *metadata, LoopAnnotationAttr attr) {
+ auto result = loopMetadataMapping.try_emplace(metadata, attr);
+ (void)result;
+ assert(result.second &&
+ "attempting to map loop options that was already mapped");
+ }
+
+ ModuleImport &moduleImport;
+ DenseMap<const llvm::MDNode *, LoopAnnotationAttr> loopMetadataMapping;
+};
+
+} // namespace detail
+} // namespace LLVM
+} // namespace mlir
+
+#endif // MLIR_LIB_TARGET_LLVMIR_LOOPANNOTATIONIMPORTER_H_
#include "AttrKindDetail.h"
#include "DebugImporter.h"
+#include "LoopAnnotationImporter.h"
#include "mlir/Dialect/DLTI/DLTI.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
mlirModule(mlirModule), llvmModule(std::move(llvmModule)),
iface(mlirModule->getContext()),
typeTranslator(*mlirModule->getContext()),
- debugImporter(std::make_unique<DebugImporter>(mlirModule)) {
+ debugImporter(std::make_unique<DebugImporter>(mlirModule)),
+ loopAnnotationImporter(std::make_unique<LoopAnnotationImporter>(*this)) {
builder.setInsertionPointToStart(mlirModule.getBody());
}
return success();
}
+FailureOr<SmallVector<SymbolRefAttr>>
+ModuleImport::lookupAccessGroupAttrs(const llvm::MDNode *node) const {
+ // An access group node is either a single access group or an access group
+ // list.
+ SmallVector<SymbolRefAttr> accessGroups;
+ if (!node->getNumOperands())
+ accessGroups.push_back(accessGroupMapping.lookup(node));
+ for (const llvm::MDOperand &operand : node->operands()) {
+ auto *node = cast<llvm::MDNode>(operand.get());
+ accessGroups.push_back(accessGroupMapping.lookup(node));
+ }
+ // Exit if one of the access group node lookups failed.
+ if (llvm::is_contained(accessGroups, nullptr))
+ return failure();
+ return accessGroups;
+}
+
+LoopAnnotationAttr
+ModuleImport::translateLoopAnnotationAttr(const llvm::MDNode *node,
+ Location loc) const {
+ return loopAnnotationImporter->translate(node, loc);
+}
+
OwningOpRef<ModuleOp>
mlir::translateLLVMIRToModule(std::unique_ptr<llvm::Module> llvmModule,
MLIRContext *context) {
!0 = !{!1}
!1 = distinct !{!"unsupported access group"}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: expected all loop properties to be either debug locations or metadata nodes
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, i32 42}
+define void @invalid_loop_node(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, i32 42}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: cannot import empty loop property
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1}
+define void @invalid_loop_node(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = distinct !{}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: cannot import loop property without a name
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1}
+define void @invalid_loop_node(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = distinct !{i1 0}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: cannot import loop properties with duplicated names llvm.loop.disable_nonforced
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !1}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1, !1}
+!1 = !{!"llvm.loop.disable_nonforced"}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: expected metadata node llvm.loop.disable_nonforced to hold no value
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.disable_nonforced", i1 0}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: expected metadata nodes llvm.loop.unroll.enable and llvm.loop.unroll.disable to be mutually exclusive
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !2}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.unroll.enable"}
+!2 = !{!"llvm.loop.unroll.disable"}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.enable to hold a boolean value
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable"}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.width to hold an i32 value
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", !0}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: expected metadata node llvm.loop.vectorize.followup_all to hold an MDNode
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.followup_all", i32 42}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: expected metadata node llvm.loop.parallel_accesses to hold one or multiple MDNodes
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.parallel_accesses", i32 42}
+
+; // -----
+
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unknown loop annotation llvm.loop.typo
+; CHECK: import-failure.ll
+; CHECK-SAME: warning: unhandled metadata: !0 = distinct !{!0, !1, !2}
+define void @unsupported_loop_annotation(i64 %n, ptr %A) {
+entry:
+ br label %end, !llvm.loop !0
+end:
+ ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.disable_nonforced"}
+!2 = !{!"llvm.loop.typo"}
!3 = distinct !{}
!4 = distinct !{}
!5 = distinct !{}
+
+; // -----
+
+; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<disableNonforced = true, mustProgress = true>
+
+; CHECK-LABEL: @simple
+define void @simple(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.disable_nonforced"}
+!3 = !{!"llvm.loop.mustprogress"}
+
+; // -----
+
+; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
+; CHECK-DAG: #[[VECTORIZE_ATTR:.*]] = #llvm.loop_vectorize<disable = false, predicateEnable = true, scalableEnable = false, width = 16 : i32, followupVectorized = #[[FOLLOWUP]], followupEpilogue = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<vectorize = #[[VECTORIZE_ATTR]]>
+
+; CHECK-LABEL: @vectorize
+define void @vectorize(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8}
+!2 = !{!"llvm.loop.vectorize.enable", i1 1}
+!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 1}
+!4 = !{!"llvm.loop.vectorize.scalable.enable", i1 0}
+!5 = !{!"llvm.loop.vectorize.width", i32 16}
+!6 = !{!"llvm.loop.vectorize.followup_vectorized", !9}
+!7 = !{!"llvm.loop.vectorize.followup_epilogue", !9}
+!8 = !{!"llvm.loop.vectorize.followup_all", !9}
+
+!9 = distinct !{!9, !10}
+!10 = !{!"llvm.loop.disable_nonforced"}
+
+; // -----
+
+; CHECK-DAG: #[[INTERLEAVE_ATTR:.*]] = #llvm.loop_interleave<count = 8 : i32>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<interleave = #[[INTERLEAVE_ATTR]]>
+
+; CHECK-LABEL: @interleave
+define void @interleave(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.interleave.count", i32 8}
+
+; // -----
+
+; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
+; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll<disable = false, count = 16 : i32, runtimeDisable = true, full = true, followup = #[[FOLLOWUP]], followupRemainder = #[[FOLLOWUP]]>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<unroll = #[[UNROLL_ATTR]]>
+
+; CHECK-LABEL: @unroll
+define void @unroll(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2, !3, !4, !5, !6, !7}
+!2 = !{!"llvm.loop.unroll.enable"}
+!3 = !{!"llvm.loop.unroll.count", i32 16}
+!4 = !{!"llvm.loop.unroll.runtime.disable"}
+!5 = !{!"llvm.loop.unroll.full"}
+!6 = !{!"llvm.loop.unroll.followup", !8}
+!7 = !{!"llvm.loop.unroll.followup_remainder", !8}
+
+!8 = distinct !{!8, !9}
+!9 = !{!"llvm.loop.disable_nonforced"}
+
+; // -----
+
+; CHECK-DAG: #[[UNROLL_ATTR:.*]] = #llvm.loop_unroll<disable = true>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<unroll = #[[UNROLL_ATTR]]>
+
+; CHECK-LABEL: @unroll_disable
+define void @unroll_disable(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.unroll.disable"}
+
+; // -----
+
+; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
+; CHECK-DAG: #[[UNROLL_AND_JAM_ATTR:.*]] = #llvm.loop_unroll_and_jam<disable = false, count = 32 : i32, followupOuter = #[[FOLLOWUP]], followupInner = #[[FOLLOWUP]], followupRemainderOuter = #[[FOLLOWUP]], followupRemainderInner = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<unrollAndJam = #[[UNROLL_AND_JAM_ATTR]]>
+
+; CHECK-LABEL: @unroll_and_jam
+define void @unroll_and_jam(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8}
+!2 = !{!"llvm.loop.unroll_and_jam.enable"}
+!3 = !{!"llvm.loop.unroll_and_jam.count", i32 32}
+!4 = !{!"llvm.loop.unroll_and_jam.followup_outer", !9}
+!5 = !{!"llvm.loop.unroll_and_jam.followup_inner", !9}
+!6 = !{!"llvm.loop.unroll_and_jam.followup_remainder_outer", !9}
+!7 = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", !9}
+!8 = !{!"llvm.loop.unroll_and_jam.followup_all", !9}
+
+!9 = distinct !{!9, !10}
+!10 = !{!"llvm.loop.disable_nonforced"}
+
+; // -----
+
+; CHECK-DAG: #[[LICM_ATTR:.*]] = #llvm.loop_licm<disable = true, versioningDisable = true>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<licm = #[[LICM_ATTR]]>
+
+; CHECK-LABEL: @licm
+define void @licm(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.licm.disable"}
+!3 = !{!"llvm.loop.licm_versioning.disable"}
+
+; // -----
+
+; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation<disableNonforced = true>
+; CHECK-DAG: #[[DISTRIBUTE_ATTR:.*]] = #llvm.loop_distribute<disable = true, followupCoincident = #[[FOLLOWUP]], followupSequential = #[[FOLLOWUP]], followupFallback = #[[FOLLOWUP]], followupAll = #[[FOLLOWUP]]>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<distribute = #[[DISTRIBUTE_ATTR]]>
+
+; CHECK-LABEL: @distribute
+define void @distribute(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2, !3, !4, !5, !6}
+!2 = !{!"llvm.loop.distribute.enable", i1 0}
+!3 = !{!"llvm.loop.distribute.followup_coincident", !9}
+!4 = !{!"llvm.loop.distribute.followup_sequential", !9}
+!5 = !{!"llvm.loop.distribute.followup_fallback", !9}
+!6 = !{!"llvm.loop.distribute.followup_all", !9}
+
+!9 = distinct !{!9, !10}
+!10 = !{!"llvm.loop.disable_nonforced"}
+
+; // -----
+
+; CHECK-DAG: #[[PIPELINE_ATTR:.*]] = #llvm.loop_pipeline<disable = false, initiationinterval = 2 : i32>
+; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<pipeline = #[[PIPELINE_ATTR]]>
+
+; CHECK-LABEL: @pipeline
+define void @pipeline(i64 %n, ptr %A) {
+entry:
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.pipeline.disable", i1 0}
+!3 = !{!"llvm.loop.pipeline.initiationinterval", i32 2}
+
+; // -----
+
+; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<parallelAccesses = @__llvm_global_metadata::@[[GROUP0:.*]]>
+
+; CHECK: llvm.metadata @__llvm_global_metadata {
+; CHECK: llvm.access_group @[[GROUP0]]
+
+; CHECK-LABEL: @parallel_accesses
+define void @parallel_accesses(ptr %arg) {
+entry:
+ %0 = load i32, ptr %arg, !llvm.access.group !0
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!0 = distinct !{}
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.parallel_accesses", !0}
+
+; // -----
+
+; CHECK: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation<parallelAccesses = @__llvm_global_metadata::@[[GROUP0:.*]], @__llvm_global_metadata::@[[GROUP1:.*]]>
+
+; CHECK: llvm.metadata @__llvm_global_metadata {
+; CHECK: llvm.access_group @[[GROUP0]]
+; CHECK: llvm.access_group @[[GROUP1]]
+
+; CHECK-LABEL: @multiple_parallel_accesses
+define void @multiple_parallel_accesses(ptr %arg) {
+entry:
+ %0 = load i32, ptr %arg, !llvm.access.group !0
+ %1 = load i32, ptr %arg, !llvm.access.group !3
+; CHECK: llvm.br ^{{.*}} {llvm.loop = #[[$ANNOT_ATTR]]}
+ br label %end, !llvm.loop !1
+end:
+ ret void
+}
+
+!0 = distinct !{}
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.parallel_accesses", !0, !3}
+!3 = distinct !{}
projects / platform / upstream / llvm.git / commitdiff