namespace fir::support {
#define FLANG_NONCODEGEN_DIALECT_LIST \
- mlir::AffineDialect, FIROpsDialect, hlfir::hlfirDialect, \
+ mlir::affine::AffineDialect, FIROpsDialect, hlfir::hlfirDialect, \
mlir::acc::OpenACCDialect, mlir::omp::OpenMPDialect, \
mlir::scf::SCFDialect, mlir::arith::ArithDialect, \
mlir::cf::ControlFlowDialect, mlir::func::FuncDialect, \
inline void registerMLIRPassesForFortranTools() {
mlir::registerCanonicalizerPass();
mlir::registerCSEPass();
- mlir::registerAffineLoopFusionPass();
+ mlir::affine::registerAffineLoopFusionPass();
mlir::registerLoopInvariantCodeMotionPass();
- mlir::registerLoopCoalescingPass();
+ mlir::affine::registerLoopCoalescingPass();
mlir::registerStripDebugInfoPass();
mlir::registerPrintOpStatsPass();
mlir::registerInlinerPass();
mlir::registerSCCPPass();
- mlir::registerAffineScalarReplacementPass();
+ mlir::affine::registerAffineScalarReplacementPass();
mlir::registerSymbolDCEPass();
mlir::registerLocationSnapshotPass();
- mlir::registerAffinePipelineDataTransferPass();
+ mlir::affine::registerAffinePipelineDataTransferPass();
- mlir::registerAffineVectorizePass();
- mlir::registerAffineLoopUnrollPass();
- mlir::registerAffineLoopUnrollAndJamPass();
- mlir::registerSimplifyAffineStructuresPass();
- mlir::registerAffineLoopInvariantCodeMotionPass();
- mlir::registerAffineLoopTilingPass();
- mlir::registerAffineDataCopyGenerationPass();
+ mlir::affine::registerAffineVectorizePass();
+ mlir::affine::registerAffineLoopUnrollPass();
+ mlir::affine::registerAffineLoopUnrollAndJamPass();
+ mlir::affine::registerSimplifyAffineStructuresPass();
+ mlir::affine::registerAffineLoopInvariantCodeMotionPass();
+ mlir::affine::registerAffineLoopTilingPass();
+ mlir::affine::registerAffineDataCopyGenerationPass();
mlir::registerConvertAffineToStandardPass();
}
}];
let constructor = "::fir::createPromoteToAffinePass()";
let dependentDialects = [
- "fir::FIROpsDialect", "mlir::func::FuncDialect", "mlir::AffineDialect"
+ "fir::FIROpsDialect", "mlir::func::FuncDialect",
+ "mlir::affine::AffineDialect"
];
}
}];
let constructor = "::fir::createAffineDemotionPass()";
let dependentDialects = [
- "fir::FIROpsDialect", "mlir::func::FuncDialect", "mlir::AffineDialect"
+ "fir::FIROpsDialect", "mlir::func::FuncDialect",
+ "mlir::affine::AffineDialect"
];
}
namespace {
-class AffineLoadConversion : public OpConversionPattern<mlir::AffineLoadOp> {
+class AffineLoadConversion
+ : public OpConversionPattern<mlir::affine::AffineLoadOp> {
public:
- using OpConversionPattern<mlir::AffineLoadOp>::OpConversionPattern;
+ using OpConversionPattern<mlir::affine::AffineLoadOp>::OpConversionPattern;
LogicalResult
- matchAndRewrite(mlir::AffineLoadOp op, OpAdaptor adaptor,
+ matchAndRewrite(mlir::affine::AffineLoadOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
SmallVector<Value> indices(adaptor.getIndices());
- auto maybeExpandedMap =
- expandAffineMap(rewriter, op.getLoc(), op.getAffineMap(), indices);
+ auto maybeExpandedMap = affine::expandAffineMap(rewriter, op.getLoc(),
+ op.getAffineMap(), indices);
if (!maybeExpandedMap)
return failure();
}
};
-class AffineStoreConversion : public OpConversionPattern<mlir::AffineStoreOp> {
+class AffineStoreConversion
+ : public OpConversionPattern<mlir::affine::AffineStoreOp> {
public:
- using OpConversionPattern<mlir::AffineStoreOp>::OpConversionPattern;
+ using OpConversionPattern<mlir::affine::AffineStoreOp>::OpConversionPattern;
LogicalResult
- matchAndRewrite(mlir::AffineStoreOp op, OpAdaptor adaptor,
+ matchAndRewrite(mlir::affine::AffineStoreOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
SmallVector<Value> indices(op.getIndices());
- auto maybeExpandedMap =
- expandAffineMap(rewriter, op.getLoc(), op.getAffineMap(), indices);
+ auto maybeExpandedMap = affine::expandAffineMap(rewriter, op.getLoc(),
+ op.getAffineMap(), indices);
if (!maybeExpandedMap)
return failure();
if (auto blockArg = value.dyn_cast<mlir::BlockArgument>()) {
affineArgs.push_back(value);
if (isa<fir::DoLoopOp>(blockArg.getOwner()->getParentOp()) ||
- isa<mlir::AffineForOp>(blockArg.getOwner()->getParentOp()))
+ isa<mlir::affine::AffineForOp>(blockArg.getOwner()->getParentOp()))
return {mlir::getAffineDimExpr(dimCount++, value.getContext())};
return {mlir::getAffineSymbolExpr(symCount++, value.getContext())};
}
}
/// Returns affine.apply and fir.convert from array_coor and gendims
-static std::pair<mlir::AffineApplyOp, fir::ConvertOp>
+static std::pair<affine::AffineApplyOp, fir::ConvertOp>
createAffineOps(mlir::Value arrayRef, mlir::PatternRewriter &rewriter) {
auto acoOp = arrayRef.getDefiningOp<ArrayCoorOp>();
auto affineMap =
populateIndexArgs(acoOp, indexArgs, rewriter);
- auto affineApply = rewriter.create<mlir::AffineApplyOp>(acoOp.getLoc(),
- affineMap, indexArgs);
+ auto affineApply = rewriter.create<affine::AffineApplyOp>(
+ acoOp.getLoc(), affineMap, indexArgs);
auto arrayElementType = coordinateArrayElement(acoOp);
auto newType =
mlir::MemRefType::get({mlir::ShapedType::kDynamic}, arrayElementType);
static void rewriteLoad(fir::LoadOp loadOp, mlir::PatternRewriter &rewriter) {
rewriter.setInsertionPoint(loadOp);
auto affineOps = createAffineOps(loadOp.getMemref(), rewriter);
- rewriter.replaceOpWithNewOp<mlir::AffineLoadOp>(
+ rewriter.replaceOpWithNewOp<affine::AffineLoadOp>(
loadOp, affineOps.second.getResult(), affineOps.first.getResult());
}
mlir::PatternRewriter &rewriter) {
rewriter.setInsertionPoint(storeOp);
auto affineOps = createAffineOps(storeOp.getMemref(), rewriter);
- rewriter.replaceOpWithNewOp<mlir::AffineStoreOp>(storeOp, storeOp.getValue(),
- affineOps.second.getResult(),
- affineOps.first.getResult());
+ rewriter.replaceOpWithNewOp<affine::AffineStoreOp>(
+ storeOp, storeOp.getValue(), affineOps.second.getResult(),
+ affineOps.first.getResult());
}
static void rewriteMemoryOps(Block *block, mlir::PatternRewriter &rewriter) {
}
private:
- std::pair<mlir::AffineForOp, mlir::Value>
+ std::pair<affine::AffineForOp, mlir::Value>
createAffineFor(fir::DoLoopOp op, mlir::PatternRewriter &rewriter) const {
if (auto constantStep = constantIntegerLike(op.getStep()))
if (*constantStep > 0)
}
// when step for the loop is positive compile time constant
- std::pair<mlir::AffineForOp, mlir::Value>
+ std::pair<affine::AffineForOp, mlir::Value>
positiveConstantStep(fir::DoLoopOp op, int64_t step,
mlir::PatternRewriter &rewriter) const {
- auto affineFor = rewriter.create<mlir::AffineForOp>(
+ auto affineFor = rewriter.create<affine::AffineForOp>(
op.getLoc(), ValueRange(op.getLowerBound()),
mlir::AffineMap::get(0, 1,
mlir::getAffineSymbolExpr(0, op.getContext())),
return std::make_pair(affineFor, affineFor.getInductionVar());
}
- std::pair<mlir::AffineForOp, mlir::Value>
+ std::pair<affine::AffineForOp, mlir::Value>
genericBounds(fir::DoLoopOp op, mlir::PatternRewriter &rewriter) const {
auto lowerBound = mlir::getAffineSymbolExpr(0, op.getContext());
auto upperBound = mlir::getAffineSymbolExpr(1, op.getContext());
auto step = mlir::getAffineSymbolExpr(2, op.getContext());
mlir::AffineMap upperBoundMap = mlir::AffineMap::get(
0, 3, (upperBound - lowerBound + step).floorDiv(step));
- auto genericUpperBound = rewriter.create<mlir::AffineApplyOp>(
+ auto genericUpperBound = rewriter.create<affine::AffineApplyOp>(
op.getLoc(), upperBoundMap,
ValueRange({op.getLowerBound(), op.getUpperBound(), op.getStep()}));
auto actualIndexMap = mlir::AffineMap::get(
(lowerBound + mlir::getAffineDimExpr(0, op.getContext())) *
mlir::getAffineSymbolExpr(1, op.getContext()));
- auto affineFor = rewriter.create<mlir::AffineForOp>(
+ auto affineFor = rewriter.create<affine::AffineForOp>(
op.getLoc(), ValueRange(),
AffineMap::getConstantMap(0, op.getContext()),
genericUpperBound.getResult(),
1 + mlir::getAffineSymbolExpr(0, op.getContext())),
1);
rewriter.setInsertionPointToStart(affineFor.getBody());
- auto actualIndex = rewriter.create<mlir::AffineApplyOp>(
+ auto actualIndex = rewriter.create<affine::AffineApplyOp>(
op.getLoc(), actualIndexMap,
ValueRange(
{affineFor.getInductionVar(), op.getLowerBound(), op.getStep()}));
<< "AffineIfConversion: couldn't calculate affine condition\n";);
return failure();
}
- auto affineIf = rewriter.create<mlir::AffineIfOp>(
+ auto affineIf = rewriter.create<affine::AffineIfOp>(
op.getLoc(), affineCondition.getIntegerSet(),
affineCondition.getAffineArgs(), !op.getElseRegion().empty());
rewriter.startRootUpdate(affineIf);
patterns.insert<AffineIfConversion>(context, functionAnalysis);
patterns.insert<AffineLoopConversion>(context, functionAnalysis);
mlir::ConversionTarget target = *context;
- target.addLegalDialect<mlir::AffineDialect, FIROpsDialect,
+ target.addLegalDialect<mlir::affine::AffineDialect, FIROpsDialect,
mlir::scf::SCFDialect, mlir::arith::ArithDialect,
mlir::func::FuncDialect>();
target.addDynamicallyLegalOp<IfOp>([&functionAnalysis](fir::IfOp op) {
mlir::RewritePatternSet patterns(context);
patterns.insert<CharacterConvertConversion>(context);
mlir::ConversionTarget target(*context);
- target.addLegalDialect<mlir::AffineDialect, fir::FIROpsDialect,
+ target.addLegalDialect<mlir::affine::AffineDialect, fir::FIROpsDialect,
mlir::arith::ArithDialect,
mlir::func::FuncDialect>();
patterns.insert<CfgLoopConv, CfgIfConv, CfgIterWhileConv>(
context, forceLoopToExecuteOnce);
mlir::ConversionTarget target(*context);
- target.addLegalDialect<mlir::AffineDialect, mlir::cf::ControlFlowDialect,
- FIROpsDialect, mlir::func::FuncDialect>();
+ target.addLegalDialect<mlir::affine::AffineDialect,
+ mlir::cf::ControlFlowDialect, FIROpsDialect,
+ mlir::func::FuncDialect>();
// apply the patterns
target.addIllegalOp<ResultOp, DoLoopOp, IfOp, IterWhileOp>();
patterns.insert<SelectTypeConv>(context, moduleMutex);
patterns.insert<DispatchOpConv>(context, bindingTables);
mlir::ConversionTarget target(*context);
- target.addLegalDialect<mlir::AffineDialect, mlir::cf::ControlFlowDialect,
- FIROpsDialect, mlir::func::FuncDialect>();
+ target.addLegalDialect<mlir::affine::AffineDialect,
+ mlir::cf::ControlFlowDialect, FIROpsDialect,
+ mlir::func::FuncDialect>();
// apply the patterns
target.addIllegalOp<SelectTypeOp>();
/// Mark all operations within Affine dialect have dynamic legality
/// constraints.
- addDynamicallyLegalDialect<AffineDialect>([](Operation *op) { ... });
+ addDynamicallyLegalDialect<affine::AffineDialect>(
+ [](Operation *op) { ... });
/// Mark `func.return` as dynamically legal, but provide a specific legality
/// callback.
// We define the specific operations, or dialects, that are legal targets for
// this lowering. In our case, we are lowering to a combination of the
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
- target.addLegalDialect<AffineDialect, arith::ArithDialect,
+ target.addLegalDialect<affine::AffineDialect, arith::ArithDialect,
func::FuncDialect, memref::MemRefDialect>();
// We also define the Toy dialect as Illegal so that the conversion will fail
// loop induction variables.
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
- buildAffineLoopNest(
+ affine::buildAffineLoopNest(
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
// Call the processing function with the rewriter, the memref operands,
// and the loop induction variables. This function will return the value
// to store at the current index.
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
- nestedBuilder.create<AffineStoreOp>(loc, valueToStore, alloc, ivs);
+ nestedBuilder.create<affine::AffineStoreOp>(loc, valueToStore, alloc,
+ ivs);
});
// Replace this operation with the generated alloc.
// Generate loads for the element of 'lhs' and 'rhs' at the
// inner loop.
- auto loadedLhs = builder.create<AffineLoadOp>(
+ auto loadedLhs = builder.create<affine::AffineLoadOp>(
loc, binaryAdaptor.getLhs(), loopIvs);
- auto loadedRhs = builder.create<AffineLoadOp>(
+ auto loadedRhs = builder.create<affine::AffineLoadOp>(
loc, binaryAdaptor.getRhs(), loopIvs);
// Create the binary operation performed on the loaded
// The last dimension is the base case of the recursion, at this point
// we store the element at the given index.
if (dimension == valueShape.size()) {
- rewriter.create<AffineStoreOp>(
+ rewriter.create<affine::AffineStoreOp>(
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
llvm::ArrayRef(indices));
return;
// Transpose the elements by generating a load from the
// reverse indices.
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
- return builder.create<AffineLoadOp>(loc, input,
- reverseIvs);
+ return builder.create<affine::AffineLoadOp>(loc, input,
+ reverseIvs);
});
return success();
}
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, func::FuncDialect, memref::MemRefDialect>();
+ registry.insert<affine::AffineDialect, func::FuncDialect,
+ memref::MemRefDialect>();
}
void runOnOperation() final;
};
// We define the specific operations, or dialects, that are legal targets for
// this lowering. In our case, we are lowering to a combination of the
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
- target.addLegalDialect<AffineDialect, BuiltinDialect, arith::ArithDialect,
- func::FuncDialect, memref::MemRefDialect>();
+ target.addLegalDialect<affine::AffineDialect, BuiltinDialect,
+ arith::ArithDialect, func::FuncDialect,
+ memref::MemRefDialect>();
// We also define the Toy dialect as Illegal so that the conversion will fail
// if any of these operations are *not* converted. Given that we actually want
// Add optimizations if enabled.
if (enableOpt) {
- optPM.addPass(mlir::createLoopFusionPass());
- optPM.addPass(mlir::createAffineScalarReplacementPass());
+ optPM.addPass(mlir::affine::createLoopFusionPass());
+ optPM.addPass(mlir::affine::createAffineScalarReplacementPass());
}
}
// loop induction variables.
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
- buildAffineLoopNest(
+ affine::buildAffineLoopNest(
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
// Call the processing function with the rewriter, the memref operands,
// and the loop induction variables. This function will return the value
// to store at the current index.
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
- nestedBuilder.create<AffineStoreOp>(loc, valueToStore, alloc, ivs);
+ nestedBuilder.create<affine::AffineStoreOp>(loc, valueToStore, alloc,
+ ivs);
});
// Replace this operation with the generated alloc.
// Generate loads for the element of 'lhs' and 'rhs' at the
// inner loop.
- auto loadedLhs = builder.create<AffineLoadOp>(
+ auto loadedLhs = builder.create<affine::AffineLoadOp>(
loc, binaryAdaptor.getLhs(), loopIvs);
- auto loadedRhs = builder.create<AffineLoadOp>(
+ auto loadedRhs = builder.create<affine::AffineLoadOp>(
loc, binaryAdaptor.getRhs(), loopIvs);
// Create the binary operation performed on the loaded
// The last dimension is the base case of the recursion, at this point
// we store the element at the given index.
if (dimension == valueShape.size()) {
- rewriter.create<AffineStoreOp>(
+ rewriter.create<affine::AffineStoreOp>(
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
llvm::ArrayRef(indices));
return;
// Transpose the elements by generating a load from the
// reverse indices.
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
- return builder.create<AffineLoadOp>(loc, input,
- reverseIvs);
+ return builder.create<affine::AffineLoadOp>(loc, input,
+ reverseIvs);
});
return success();
}
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, func::FuncDialect, memref::MemRefDialect>();
+ registry.insert<affine::AffineDialect, func::FuncDialect,
+ memref::MemRefDialect>();
}
void runOnOperation() final;
};
// We define the specific operations, or dialects, that are legal targets for
// this lowering. In our case, we are lowering to a combination of the
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
- target.addLegalDialect<AffineDialect, BuiltinDialect, arith::ArithDialect,
- func::FuncDialect, memref::MemRefDialect>();
+ target.addLegalDialect<affine::AffineDialect, BuiltinDialect,
+ arith::ArithDialect, func::FuncDialect,
+ memref::MemRefDialect>();
// We also define the Toy dialect as Illegal so that the conversion will fail
// if any of these operations are *not* converted. Given that we actually want
// Add optimizations if enabled.
if (enableOpt) {
- optPM.addPass(mlir::createLoopFusionPass());
- optPM.addPass(mlir::createAffineScalarReplacementPass());
+ optPM.addPass(mlir::affine::createLoopFusionPass());
+ optPM.addPass(mlir::affine::createAffineScalarReplacementPass());
}
}
// loop induction variables.
SmallVector<int64_t, 4> lowerBounds(tensorType.getRank(), /*Value=*/0);
SmallVector<int64_t, 4> steps(tensorType.getRank(), /*Value=*/1);
- buildAffineLoopNest(
+ affine::buildAffineLoopNest(
rewriter, loc, lowerBounds, tensorType.getShape(), steps,
[&](OpBuilder &nestedBuilder, Location loc, ValueRange ivs) {
// Call the processing function with the rewriter, the memref operands,
// and the loop induction variables. This function will return the value
// to store at the current index.
Value valueToStore = processIteration(nestedBuilder, operands, ivs);
- nestedBuilder.create<AffineStoreOp>(loc, valueToStore, alloc, ivs);
+ nestedBuilder.create<affine::AffineStoreOp>(loc, valueToStore, alloc,
+ ivs);
});
// Replace this operation with the generated alloc.
// Generate loads for the element of 'lhs' and 'rhs' at the
// inner loop.
- auto loadedLhs = builder.create<AffineLoadOp>(
+ auto loadedLhs = builder.create<affine::AffineLoadOp>(
loc, binaryAdaptor.getLhs(), loopIvs);
- auto loadedRhs = builder.create<AffineLoadOp>(
+ auto loadedRhs = builder.create<affine::AffineLoadOp>(
loc, binaryAdaptor.getRhs(), loopIvs);
// Create the binary operation performed on the loaded
// The last dimension is the base case of the recursion, at this point
// we store the element at the given index.
if (dimension == valueShape.size()) {
- rewriter.create<AffineStoreOp>(
+ rewriter.create<affine::AffineStoreOp>(
loc, rewriter.create<arith::ConstantOp>(loc, *valueIt++), alloc,
llvm::ArrayRef(indices));
return;
// Transpose the elements by generating a load from the
// reverse indices.
SmallVector<Value, 2> reverseIvs(llvm::reverse(loopIvs));
- return builder.create<AffineLoadOp>(loc, input,
- reverseIvs);
+ return builder.create<affine::AffineLoadOp>(loc, input,
+ reverseIvs);
});
return success();
}
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ToyToAffineLoweringPass)
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, func::FuncDialect, memref::MemRefDialect>();
+ registry.insert<affine::AffineDialect, func::FuncDialect,
+ memref::MemRefDialect>();
}
void runOnOperation() final;
};
// We define the specific operations, or dialects, that are legal targets for
// this lowering. In our case, we are lowering to a combination of the
// `Affine`, `Arith`, `Func`, and `MemRef` dialects.
- target.addLegalDialect<AffineDialect, BuiltinDialect, arith::ArithDialect,
- func::FuncDialect, memref::MemRefDialect>();
+ target.addLegalDialect<affine::AffineDialect, BuiltinDialect,
+ arith::ArithDialect, func::FuncDialect,
+ memref::MemRefDialect>();
// We also define the Toy dialect as Illegal so that the conversion will fail
// if any of these operations are *not* converted. Given that we actually want
// Add optimizations if enabled.
if (enableOpt) {
- optPM.addPass(mlir::createLoopFusionPass());
- optPM.addPass(mlir::createAffineScalarReplacementPass());
+ optPM.addPass(mlir::affine::createLoopFusionPass());
+ optPM.addPass(mlir::affine::createAffineScalarReplacementPass());
}
}
#include "mlir/Support/LLVM.h"
namespace mlir {
-class AffineForOp;
class Location;
struct LogicalResult;
class OpBuilder;
class Pass;
class RewritePattern;
+class RewritePatternSet;
class Value;
class ValueRange;
-class RewritePatternSet;
+namespace affine {
+class AffineForOp;
+} // namespace affine
#define GEN_PASS_DECL_CONVERTAFFINETOSTANDARD
#include "mlir/Conversion/Passes.h.inc"
/// Emit code that computes the lower bound of the given affine loop using
/// standard arithmetic operations.
-Value lowerAffineLowerBound(AffineForOp op, OpBuilder &builder);
+Value lowerAffineLowerBound(affine::AffineForOp op, OpBuilder &builder);
/// Emit code that computes the upper bound of the given affine loop using
/// standard arithmetic operations.
-Value lowerAffineUpperBound(AffineForOp op, OpBuilder &builder);
+Value lowerAffineUpperBound(affine::AffineForOp op, OpBuilder &builder);
/// Lowers affine control flow operations (ForStmt, IfStmt and AffineApplyOp)
/// to equivalent lower-level constructs (flow of basic blocks and arithmetic
def ConvertParallelLoopToGpu : Pass<"convert-parallel-loops-to-gpu"> {
let summary = "Convert mapped scf.parallel ops to gpu launch operations";
let constructor = "mlir::createParallelLoopToGpuPass()";
- let dependentDialects = ["AffineDialect", "gpu::GPUDialect"];
+ let dependentDialects = ["affine::AffineDialect", "gpu::GPUDialect"];
}
//===----------------------------------------------------------------------===//
"dialect";
let constructor = "mlir::createConvertVectorToGPUPass()";
let dependentDialects = [
- "memref::MemRefDialect", "gpu::GPUDialect", "AffineDialect",
+ "memref::MemRefDialect", "gpu::GPUDialect", "affine::AffineDialect",
"vector::VectorDialect", "nvgpu::NVGPUDialect"
];
"dialect";
let constructor = "mlir::createConvertVectorToSCFPass()";
let dependentDialects = [
- "AffineDialect",
+ "affine::AffineDialect",
"memref::MemRefDialect",
"scf::SCFDialect",
"tensor::TensorDialect"
#include "mlir/Support/LLVM.h"
namespace mlir {
-class AffineForOp;
class ConversionTarget;
struct LogicalResult;
class MLIRContext;
class Operation;
class RewritePatternSet;
+namespace affine {
+class AffineForOp;
+} // namespace affine
+
namespace scf {
class ForOp;
} // namespace scf
// TODO: Consider removing this in favor of affine.for -> affine.parallel
// detection followed by an affine.parallel -> scf.parallel -> gpu.launch
// conversion
-LogicalResult convertAffineLoopNestToGPULaunch(AffineForOp forOp,
+LogicalResult convertAffineLoopNestToGPULaunch(affine::AffineForOp forOp,
unsigned numBlockDims,
unsigned numThreadDims);
#include <optional>
namespace mlir {
+class Operation;
+namespace affine {
class AffineApplyOp;
class AffineForOp;
class AffineValueMap;
class FlatAffineRelation;
class FlatAffineValueConstraints;
-class Operation;
/// A description of a (parallelizable) reduction in an affine loop.
struct LoopReduction {
AffineForOp forOp, unsigned maxLoopDepth,
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec);
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_AFFINEANALYSIS_H
#include <optional>
namespace mlir {
-
-class AffineCondition;
-class AffineForOp;
-class AffineIfOp;
-class AffineParallelOp;
class AffineMap;
-class AffineValueMap;
class IntegerSet;
-class MLIRContext;
-class Value;
class MemRefType;
+class MLIRContext;
struct MutableAffineMap;
+class Value;
namespace presburger {
class MultiAffineFunction;
} // namespace presburger
+namespace affine {
+class AffineCondition;
+class AffineForOp;
+class AffineIfOp;
+class AffineParallelOp;
+class AffineValueMap;
+
/// FlatAffineValueConstraints is an extension of FlatLinearValueConstraints
/// with helper functions for Affine dialect ops.
class FlatAffineValueConstraints : public FlatLinearValueConstraints {
LogicalResult getRelationFromMap(const AffineValueMap &map,
FlatAffineRelation &rel);
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_AFFINESTRUCTURES_H
#include <optional>
namespace mlir {
-
class AffineExpr;
-class AffineForOp;
class AffineMap;
class BlockArgument;
class MemRefType;
-class NestedPattern;
class Operation;
class Value;
+namespace affine {
+class AffineForOp;
+class NestedPattern;
+
/// Returns the trip count of the loop as an affine map with its corresponding
/// operands if the latter is expressible as an affine expression, and nullptr
/// otherwise. This method always succeeds as long as the lower bound is not a
// the support.
bool isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts);
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_LOOPANALYSIS_H
#include "llvm/Support/Allocator.h"
namespace mlir {
+class Operation;
+namespace affine {
class NestedPattern;
-class Operation;
/// An NestedPattern captures nested patterns in the IR.
/// It is used in conjunction with a scoped NestedPatternContext which is an
bool isLoadOrStore(Operation &op);
} // namespace matcher
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_NESTEDMATCHER_H
#include <optional>
namespace mlir {
-
-class AffineForOp;
-class AffineValueMap;
class Block;
class Location;
-struct MemRefAccess;
class Operation;
class Value;
+namespace affine {
+class AffineForOp;
+class AffineValueMap;
+struct MemRefAccess;
+
// LoopNestStateCollector walks loop nests and collects load and store
// operations, and whether or not a region holding op other than ForOp and IfOp
// was encountered in the loop nest.
simplifyConstrainedMinMaxOp(Operation *op,
FlatAffineValueConstraints constraints);
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_ANALYSIS_UTILS_H
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/OpDefinition.h"
-namespace mlir {
#include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h.inc"
-} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_IR_AFFINEMEMORYOPINTERFACES_H
Interface to query characteristics of read-like ops with affine
restrictions.
}];
+ let cppNamespace = "::mlir::affine";
let methods = [
InterfaceMethod<
/*desc=*/"Returns the memref operand to read from.",
- /*retTy=*/"Value",
+ /*retTy=*/"::mlir::Value",
/*methodName=*/"getMemRef",
/*args=*/(ins),
/*methodBody*/[{}],
/*defaultImplementation=*/ [{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return op.getOperand(op.getMemRefOperandIndex());
+ return $_op.getOperand($_op.getMemRefOperandIndex());
}]
>,
InterfaceMethod<
/*desc=*/"Returns the type of the memref operand.",
- /*retTy=*/"MemRefType",
+ /*retTy=*/"::mlir::MemRefType",
/*methodName=*/"getMemRefType",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return op.getMemRef().getType().template cast<MemRefType>();
+ return $_op.getMemRef().getType().template cast<::mlir::MemRefType>();
}]
>,
InterfaceMethod<
/*desc=*/"Returns affine map operands.",
- /*retTy=*/"Operation::operand_range",
+ /*retTy=*/"::mlir::Operation::operand_range",
/*methodName=*/"getMapOperands",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return llvm::drop_begin(op.getOperands(), 1);
+ return llvm::drop_begin($_op.getOperands(), 1);
}]
>,
InterfaceMethod<
/*desc=*/[{
Returns the affine map used to index the memref for this operation.
}],
- /*retTy=*/"AffineMap",
+ /*retTy=*/"::mlir::AffineMap",
/*methodName=*/"getAffineMap",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return op.getAffineMapAttr().getValue();
+ return $_op.getAffineMapAttr().getValue();
}]
>,
InterfaceMethod<
/*desc=*/"Returns the value read by this operation.",
- /*retTy=*/"Value",
+ /*retTy=*/"::mlir::Value",
/*methodName=*/"getValue",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- return cast<ConcreteOp>(this->getOperation());
+ return $_op;
}]
>,
];
Interface to query characteristics of write-like ops with affine
restrictions.
}];
+ let cppNamespace = "::mlir::affine";
let methods = [
InterfaceMethod<
/*desc=*/"Returns the memref operand to write to.",
- /*retTy=*/"Value",
+ /*retTy=*/"::mlir::Value",
/*methodName=*/"getMemRef",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return op.getOperand(op.getMemRefOperandIndex());
+ return $_op.getOperand($_op.getMemRefOperandIndex());
}]
>,
InterfaceMethod<
/*desc=*/"Returns the type of the memref operand.",
- /*retTy=*/"MemRefType",
+ /*retTy=*/"::mlir::MemRefType",
/*methodName=*/"getMemRefType",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return op.getMemRef().getType().template cast<MemRefType>();
+ return $_op.getMemRef().getType().template cast<::mlir::MemRefType>();
}]
>,
InterfaceMethod<
/*desc=*/"Returns affine map operands.",
- /*retTy=*/"Operation::operand_range",
+ /*retTy=*/"::mlir::Operation::operand_range",
/*methodName=*/"getMapOperands",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return llvm::drop_begin(op.getOperands(), 2);
+ return llvm::drop_begin($_op.getOperands(), 2);
}]
>,
InterfaceMethod<
/*desc=*/[{
Returns the affine map used to index the memref for this operation.
}],
- /*retTy=*/"AffineMap",
+ /*retTy=*/"::mlir::AffineMap",
/*methodName=*/"getAffineMap",
/*args=*/(ins),
/*methodName=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return op.getAffineMapAttr().getValue();
+ return $_op.getAffineMapAttr().getValue();
}]
>,
InterfaceMethod<
/*desc=*/"Returns the value to store.",
- /*retTy=*/"Value",
+ /*retTy=*/"::mlir::Value",
/*methodName=*/"getValueToStore",
/*args=*/(ins),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- return op.getOperand(op.getStoredValOperandIndex());
+ return $_op.getOperand($_op.getStoredValOperandIndex());
}]
>,
];
memref operand. The memref argument given to this interface much match
one of those memref operands.
}];
+ let cppNamespace = "::mlir::affine";
let methods = [
InterfaceMethod<
/*desc=*/"Returns the AffineMapAttr associated with 'memref'.",
- /*retTy=*/"NamedAttribute",
+ /*retTy=*/"::mlir::NamedAttribute",
/*methodName=*/"getAffineMapAttrForMemRef",
- /*args=*/(ins "Value":$memref),
+ /*args=*/(ins "::mlir::Value":$memref),
/*methodBody=*/[{}],
/*defaultImplementation=*/[{
- ConcreteOp op = cast<ConcreteOp>(this->getOperation());
- assert(memref == op.getMemRef() &&
+ assert(memref == $_op.getMemRef() &&
"Expected memref argument to match memref operand");
- return {StringAttr::get(op.getContext(), op.getMapAttrStrName()),
- op.getAffineMapAttr()};
+ return {::mlir::StringAttr::get(
+ $_op.getContext(), $_op.getMapAttrStrName()),
+ $_op.getAffineMapAttr()};
}]
>,
];
#include "mlir/Interfaces/LoopLikeInterface.h"
namespace mlir {
+namespace affine {
+
class AffineApplyOp;
class AffineBound;
class AffineValueMap;
-/// TODO: These should be renamed if they are on the mlir namespace.
-/// Ideally, they should go in a mlir::affine:: namespace.
-
/// A utility function to check if a value is defined at the top level of an
/// op with trait `AffineScope` or is a region argument for such an op. A value
/// of index type defined at the top level is always a valid symbol for all its
/// argument.
void fullyComposeAffineMapAndOperands(AffineMap *map,
SmallVectorImpl<Value> *operands);
+
+} // namespace affine
} // namespace mlir
+
#include "mlir/Dialect/Affine/IR/AffineOpsDialect.h.inc"
#define GET_OP_CLASSES
#include "mlir/Dialect/Affine/IR/AffineOps.h.inc"
namespace mlir {
+namespace affine {
+
/// Returns true if the provided value is the induction variable of an
/// AffineForOp.
bool isAffineForInductionVar(Value val);
friend class AffineForOp;
};
+} // namespace affine
} // namespace mlir
#endif
def Affine_Dialect : Dialect {
let name = "affine";
- let cppNamespace = "mlir";
+ let cppNamespace = "::mlir::affine";
let hasConstantMaterializer = 1;
let dependentDialects = ["arith::ArithDialect"];
}
#include "mlir/IR/Value.h"
namespace mlir {
+namespace affine {
/// An AffineValueMap is an affine map plus its ML value operands and
/// results for analysis purposes. The structure is still a tree form that is
SmallVector<Value, 4> results;
};
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_IR_AFFINEVALUEMAP_H
#include "llvm/ADT/SmallVector.h"
namespace mlir {
+class Operation;
+
+namespace affine {
class AffineForOp;
struct ComputationSliceState;
-class Operation;
// TODO: Extend this module to include utility functions for querying fusion
// cost/storage reduction, and for performing the loop fusion transformation.
void gatherProducerConsumerMemrefs(ArrayRef<Operation *> srcOps,
ArrayRef<Operation *> dstOps,
DenseSet<Value> &producerConsumerMemrefs);
+
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_LOOPFUSIONUTILS_H
#include <optional>
namespace mlir {
-class AffineForOp;
class AffineMap;
class LoopLikeOpInterface;
-struct MemRefRegion;
class OpBuilder;
class Value;
class ValueRange;
class ParallelOp;
} // namespace scf
+namespace affine {
+class AffineForOp;
+struct MemRefRegion;
+
/// Unrolls this for operation completely if the trip count is known to be
/// constant. Returns failure otherwise.
LogicalResult loopUnrollFull(AffineForOp forOp);
return result;
}
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_LOOPUTILS_H
class FuncOp;
} // namespace func
+namespace affine {
class AffineForOp;
/// Fusion mode to attempt. The default mode `Greedy` does both
#define GEN_PASS_REGISTRATION
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_PASSES_H
def AffineDataCopyGeneration : Pass<"affine-data-copy-generate", "func::FuncOp"> {
let summary = "Generate explicit copying for affine memory operations";
- let constructor = "mlir::createAffineDataCopyGenerationPass()";
+ let constructor = "mlir::affine::createAffineDataCopyGenerationPass()";
let dependentDialects = ["memref::MemRefDialect"];
let options = [
Option<"fastMemoryCapacity", "fast-mem-capacity", "uint64_t",
}
```
}];
- let constructor = "mlir::createLoopFusionPass()";
+ let constructor = "mlir::affine::createLoopFusionPass()";
let options = [
Option<"computeToleranceThreshold", "fusion-compute-tolerance", "double",
/*default=*/"0.30f", "Fractional increase in additional computation "
Option<"maximalFusion", "fusion-maximal", "bool", /*default=*/"false",
"Enables maximal loop fusion">,
Option<"affineFusionMode", "mode", "enum FusionMode",
- "mlir::FusionMode::Greedy", "fusion mode to attempt",
- "llvm::cl::values(clEnumValN(mlir::FusionMode::Greedy,"
+ "mlir::affine::FusionMode::Greedy", "fusion mode to attempt",
+ "llvm::cl::values(clEnumValN(mlir::affine::FusionMode::Greedy,"
" \"greedy\", \"Perform greedy (both producer-consumer and sibling) fusion\"), "
- "clEnumValN( mlir::FusionMode::ProducerConsumer, "
+ "clEnumValN( mlir::affine::FusionMode::ProducerConsumer, "
"\"producer\", \"Perform only producer-consumer fusion\"), "
- "clEnumValN( mlir::FusionMode::Sibling, "
+ "clEnumValN( mlir::affine::FusionMode::Sibling, "
"\"sibling\", \"Perform only sibling fusion\"))">,
];
let dependentDialects = ["memref::MemRefDialect"];
def AffineLoopInvariantCodeMotion
: Pass<"affine-loop-invariant-code-motion", "func::FuncOp"> {
let summary = "Hoist loop invariant instructions outside of affine loops";
- let constructor = "mlir::createAffineLoopInvariantCodeMotionPass()";
+ let constructor = "mlir::affine::createAffineLoopInvariantCodeMotionPass()";
}
def AffineLoopTiling : Pass<"affine-loop-tile", "func::FuncOp"> {
let summary = "Tile affine loop nests";
- let constructor = "mlir::createLoopTilingPass()";
+ let constructor = "mlir::affine::createLoopTilingPass()";
let options = [
Option<"cacheSizeInKiB", "cache-size", "uint64_t", /*default=*/"512",
"Set size of cache to tile for in KiB (default: 512)">,
def AffineLoopUnroll : Pass<"affine-loop-unroll", "func::FuncOp"> {
let summary = "Unroll affine loops";
- let constructor = "mlir::createLoopUnrollPass()";
+ let constructor = "mlir::affine::createLoopUnrollPass()";
let options = [
Option<"unrollFactor", "unroll-factor", "unsigned", /*default=*/"4",
"Use this unroll factor for all loops being unrolled">,
def AffineLoopUnrollAndJam : Pass<"affine-loop-unroll-jam", "func::FuncOp"> {
let summary = "Unroll and jam affine loops";
- let constructor = "mlir::createLoopUnrollAndJamPass()";
+ let constructor = "mlir::affine::createLoopUnrollAndJamPass()";
let options = [
Option<"unrollJamFactor", "unroll-jam-factor", "unsigned",
/*default=*/"4",
}
```
}];
- let constructor = "mlir::createPipelineDataTransferPass()";
+ let constructor = "mlir::affine::createPipelineDataTransferPass()";
}
def AffineScalarReplacement : Pass<"affine-scalrep", "func::FuncOp"> {
}
```
}];
- let constructor = "mlir::createAffineScalarReplacementPass()";
+ let constructor = "mlir::affine::createAffineScalarReplacementPass()";
}
def AffineVectorize : Pass<"affine-super-vectorize", "func::FuncOp"> {
def AffineParallelize : Pass<"affine-parallelize", "func::FuncOp"> {
let summary = "Convert affine.for ops into 1-D affine.parallel";
- let constructor = "mlir::createAffineParallelizePass()";
+ let constructor = "mlir::affine::createAffineParallelizePass()";
let options = [
Option<"maxNested", "max-nested", "unsigned", /*default=*/"-1u",
"Maximum number of nested parallel loops to produce. "
def AffineLoopNormalize : Pass<"affine-loop-normalize", "func::FuncOp"> {
let summary = "Apply normalization transformations to affine loop-like ops";
- let constructor = "mlir::createAffineLoopNormalizePass()";
+ let constructor = "mlir::affine::createAffineLoopNormalizePass()";
let options = [
Option<"promoteSingleIter", "promote-single-iter", "bool",
/*default=*/"true", "Promote single iteration loops">,
def LoopCoalescing : Pass<"affine-loop-coalescing", "func::FuncOp"> {
let summary = "Coalesce nested loops with independent bounds into a single "
"loop";
- let constructor = "mlir::createLoopCoalescingPass()";
+ let constructor = "mlir::affine::createLoopCoalescingPass()";
let dependentDialects = ["arith::ArithDialect"];
}
def SimplifyAffineStructures : Pass<"affine-simplify-structures", "func::FuncOp"> {
let summary = "Simplify affine expressions in maps/sets and normalize "
"memrefs";
- let constructor = "mlir::createSimplifyAffineStructuresPass()";
+ let constructor = "mlir::affine::createSimplifyAffineStructuresPass()";
}
def AffineExpandIndexOps : Pass<"affine-expand-index-ops"> {
let summary = "Lower affine operations operating on indices into more fundamental operations";
- let constructor = "mlir::createAffineExpandIndexOpsPass()";
+ let constructor = "mlir::affine::createAffineExpandIndexOpsPass()";
}
#endif // MLIR_DIALECT_AFFINE_PASSES
#include "mlir/IR/OpImplementation.h"
namespace mlir {
-class AffineForOp;
namespace func {
class FuncOp;
} // namespace func
namespace affine {
-class ForOp;
+class AffineForOp;
} // namespace affine
} // namespace mlir
#include "mlir/Support/LogicalResult.h"
namespace mlir {
-class AffineApplyOp;
class Location;
class OpBuilder;
class OpFoldResult;
enum class BoundType;
} // namespace presburger
+namespace affine {
+class AffineApplyOp;
+
/// Populate patterns that expand affine index operations into more fundamental
/// operations (not necessarily restricted to Affine dialect).
void populateAffineExpandIndexOpsPatterns(RewritePatternSet &patterns);
ValueBoundsConstraintSet::StopConditionFn stopCondition = nullptr,
bool closedUB = false);
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_TRANSFORMS_TRANSFORMS_H
#include <optional>
namespace mlir {
-
-class AffineForOp;
-class AffineIfOp;
-class AffineParallelOp;
class DominanceInfo;
class Operation;
class PostDominanceInfo;
struct LogicalResult;
+namespace affine {
+class AffineForOp;
+class AffineIfOp;
+class AffineParallelOp;
+
using ReductionLoopMap = DenseMap<Operation *, SmallVector<LoopReduction, 2>>;
/// Replaces a parallel affine.for op with a 1-d affine.parallel op. `forOp`'s
Location loc;
};
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_UTILS_H
namespace mlir {
class RewriterBase;
+namespace affine {
+
/// Fills the `combinedOffsets`, `combinedSizes` and `combinedStrides` to use
/// when combining a producer slice **into** a consumer slice.
///
const llvm::SmallBitVector &rankReducedSourceDims,
SmallVectorImpl<OpFoldResult> &resolvedSizes);
+} // namespace affine
} // namespace mlir
#endif // MLIR_DIALECT_AFFINE_VIEWLIKEINTERFACEUTILS_H
deallocation](/docs/BufferDeallocationInternals/).
}];
let dependentDialects = [
- "AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
+ "affine::AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
];
let extraClassDeclaration = [{
let cppNamespace = "::mlir::linalg";
let dependentDialects = [
"arith::ArithDialect",
- "AffineDialect",
+ "affine::AffineDialect",
"math::MathDialect",
"memref::MemRefDialect",
"tensor::TensorDialect",
"Generate rank-reducing slices instead of reassociative reshapes">
];
let dependentDialects = [
- "linalg::LinalgDialect", "AffineDialect", "memref::MemRefDialect"
+ "linalg::LinalgDialect", "affine::AffineDialect", "memref::MemRefDialect"
];
}
let summary = "Fuse elementwise operations on tensors";
let constructor = "mlir::createLinalgElementwiseOpFusionPass()";
let dependentDialects = [
- "AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"
+ "affine::AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"
];
}
"loops";
let constructor = "mlir::createConvertLinalgToAffineLoopsPass()";
let dependentDialects = [
- "AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"];
+ "affine::AffineDialect", "linalg::LinalgDialect", "memref::MemRefDialect"];
}
def LinalgLowerToLoops : Pass<"convert-linalg-to-loops", "func::FuncOp"> {
let dependentDialects = [
"linalg::LinalgDialect",
"scf::SCFDialect",
- "AffineDialect"
+ "affine::AffineDialect"
];
}
"loops";
let constructor = "mlir::createConvertLinalgToParallelLoopsPass()";
let dependentDialects = [
- "AffineDialect",
+ "affine::AffineDialect",
"linalg::LinalgDialect",
"memref::MemRefDialect",
"scf::SCFDialect"
let summary = "Bufferize the linalg dialect";
let constructor = "mlir::createLinalgBufferizePass()";
let dependentDialects = [
- "AffineDialect",
+ "affine::AffineDialect",
"bufferization::BufferizationDialect",
"linalg::LinalgDialect",
"memref::MemRefDialect",
namespace mlir {
class AffineExpr;
-class AffineForOp;
class AffineMap;
class PatternRewriter;
+namespace affine {
+class AffineForOp;
+} // namespace affine
+
namespace tensor {
class ExtractSliceOp;
} // namespace tensor
}];
let constructor = "mlir::memref::createFoldMemRefAliasOpsPass()";
let dependentDialects = [
- "AffineDialect", "memref::MemRefDialect", "vector::VectorDialect"
+ "affine::AffineDialect", "memref::MemRefDialect", "vector::VectorDialect"
];
}
```
}];
let constructor = "mlir::memref::createNormalizeMemRefsPass()";
- let dependentDialects = ["AffineDialect"];
+ let dependentDialects = ["affine::AffineDialect"];
}
def ResolveRankedShapeTypeResultDims :
}];
let constructor = "mlir::memref::createResolveShapedTypeResultDimsPass()";
let dependentDialects = [
- "AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
+ "affine::AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
];
}
}];
let constructor = "mlir::memref::createExpandStridedMetadataPass()";
let dependentDialects = [
- "AffineDialect", "memref::MemRefDialect"
+ "affine::AffineDialect", "memref::MemRefDialect"
];
}
#endif // MLIR_DIALECT_MEMREF_TRANSFORMS_PASSES
: Pass<"scf-for-loop-canonicalization"> {
let summary = "Canonicalize operations within scf.for loop bodies";
let constructor = "mlir::createSCFForLoopCanonicalizationPass()";
- let dependentDialects = ["AffineDialect", "tensor::TensorDialect",
+ let dependentDialects = ["affine::AffineDialect", "tensor::TensorDialect",
"memref::MemRefDialect"];
}
"Do not peel loops inside of the last, partial iteration of another "
"already peeled loop.">
];
- let dependentDialects = ["AffineDialect"];
+ let dependentDialects = ["affine::AffineDialect"];
}
def SCFForLoopSpecialization : Pass<"scf-for-loop-specialization"> {
"Perform tiling with fixed upper bound with inbound check "
"inside the internal loops">
];
- let dependentDialects = ["AffineDialect"];
+ let dependentDialects = ["affine::AffineDialect"];
}
def SCFForLoopRangeFolding : Pass<"scf-for-loop-range-folding"> {
#include "mlir/Support/LogicalResult.h"
namespace mlir {
-class AffineApplyOp;
class AffineMap;
-class FlatAffineValueConstraints;
struct LogicalResult;
class Operation;
class OpFoldResult;
class Value;
class ValueRange;
+namespace affine {
+class FlatAffineValueConstraints;
+} // namespace affine
+
namespace scf {
class IfOp;
/// Populate the given constraint set with induction variable constraints of a
/// "for" loop with the given range and step.
-LogicalResult addLoopRangeConstraints(FlatAffineValueConstraints &cstr,
+LogicalResult addLoopRangeConstraints(affine::FlatAffineValueConstraints &cstr,
Value iv, OpFoldResult lb,
OpFoldResult ub, OpFoldResult step);
}];
let constructor = "mlir::createSparsificationPass()";
let dependentDialects = [
- "AffineDialect",
+ "affine::AffineDialect",
"arith::ArithDialect",
"bufferization::BufferizationDialect",
"LLVM::LLVMDialect",
let hasCanonicalizer = 1;
let hasConstantMaterializer = 1;
let dependentDialects = [
- "AffineDialect",
+ "affine::AffineDialect",
"arith::ArithDialect",
"complex::ComplexDialect",
];
}];
let constructor = "mlir::tensor::createFoldTensorSubsetOpsPass()";
let dependentDialects = [
- "AffineDialect", "tensor::TensorDialect", "vector::VectorDialect"
+ "affine::AffineDialect", "tensor::TensorDialect", "vector::VectorDialect"
];
}
namespace mlir {
// Forward declarations.
-class AffineApplyOp;
-class AffineForOp;
class AffineMap;
class Block;
class Location;
class VectorType;
class VectorTransferOpInterface;
+namespace affine {
+class AffineApplyOp;
+class AffineForOp;
+} // namespace affine
+
namespace vector {
/// Helper function that creates a memref::DimOp or tensor::DimOp depending on
/// the type of `source`.
inline void registerAllDialects(DialectRegistry ®istry) {
// clang-format off
registry.insert<acc::OpenACCDialect,
- AffineDialect,
+ affine::AffineDialect,
arith::ArithDialect,
amdgpu::AMDGPUDialect,
amx::AMXDialect,
registerConversionPasses();
// Dialect passes
- registerAffinePasses();
+ affine::registerAffinePasses();
registerAsyncPasses();
arith::registerArithPasses();
bufferization::registerBufferizationPasses();
} // namespace mlir
using namespace mlir;
+using namespace mlir::affine;
using namespace mlir::vector;
/// Given a range of values, emit the code that reduces them with "min" or "max"
void ConvertLinalgToStandardPass::runOnOperation() {
auto module = getOperation();
ConversionTarget target(getContext());
- target.addLegalDialect<AffineDialect, arith::ArithDialect, func::FuncDialect,
- memref::MemRefDialect, scf::SCFDialect>();
+ target.addLegalDialect<affine::AffineDialect, arith::ArithDialect,
+ func::FuncDialect, memref::MemRefDialect,
+ scf::SCFDialect>();
target.addLegalOp<ModuleOp, func::FuncOp, func::ReturnOp>();
RewritePatternSet patterns(&getContext());
populateLinalgToStandardConversionPatterns(patterns);
#define DEBUG_TYPE "loops-to-gpu"
using namespace mlir;
+using namespace mlir::affine;
using namespace mlir::scf;
// Name of internal attribute to mark visited operations during conversion.
void runOnOperation() override {
for (Operation &op : llvm::make_early_inc_range(
getOperation().getFunctionBody().getOps())) {
- if (auto forOp = dyn_cast<AffineForOp>(&op)) {
+ if (auto forOp = dyn_cast<affine::AffineForOp>(&op)) {
if (failed(convertAffineLoopNestToGPULaunch(forOp, numBlockDims,
numThreadDims)))
signalPassFailure();
SmallVector<Value, 3> dims(dimValues.begin(), dimValues.end());
dims.push_back(prevIdx);
AffineExpr d0 = rewriter.getAffineDimExpr(offsetMap.getNumDims());
- indices[dim.getPosition()] = makeComposedAffineApply(
+ indices[dim.getPosition()] = affine::makeComposedAffineApply(
rewriter, loc, d0 + offsetMap.getResult(offsetsIdx++), dims);
continue;
}
AffineExpr d0, d1;
bindDims(xferOp.getContext(), d0, d1);
Value offset = adaptor.getIndices()[*dim];
- indices[*dim] = makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
+ indices[*dim] =
+ affine::makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
}
}
AffineExpr d0, d1;
bindDims(xferOp.getContext(), d0, d1);
Value base = xferOp.getIndices()[*dim];
- Value memrefIdx = makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
+ Value memrefIdx =
+ affine::makeComposedAffineApply(b, loc, d0 + d1, {base, iv});
cond = lb.create<arith::CmpIOp>(arith::CmpIPredicate::sgt, memrefDim,
memrefIdx);
}
AffineExpr d0, d1;
bindDims(xferOp.getContext(), d0, d1);
Value offset = memrefIndices[dim];
- memrefIndices[dim] = makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
+ memrefIndices[dim] =
+ affine::makeComposedAffineApply(b, loc, d0 + d1, {offset, iv});
return dim;
}
#define DEBUG_TYPE "affine-analysis"
using namespace mlir;
+using namespace affine;
using namespace presburger;
/// Get the value that is being reduced by `pos`-th reduction in the loop if
}
/// Populate `supportedReductions` with descriptors of the supported reductions.
-void mlir::getSupportedReductions(
+void mlir::affine::getSupportedReductions(
AffineForOp forOp, SmallVectorImpl<LoopReduction> &supportedReductions) {
unsigned numIterArgs = forOp.getNumIterOperands();
if (numIterArgs == 0)
/// Returns true if `forOp' is a parallel loop. If `parallelReductions` is
/// provided, populates it with descriptors of the parallelizable reductions and
/// treats them as not preventing parallelization.
-bool mlir::isLoopParallel(AffineForOp forOp,
- SmallVectorImpl<LoopReduction> *parallelReductions) {
+bool mlir::affine::isLoopParallel(
+ AffineForOp forOp, SmallVectorImpl<LoopReduction> *parallelReductions) {
unsigned numIterArgs = forOp.getNumIterOperands();
// Loop is not parallel if it has SSA loop-carried dependences and reduction
return viewOp && isLocallyDefined(viewOp.getViewSource(), enclosingOp);
}
-bool mlir::isLoopMemoryParallel(AffineForOp forOp) {
+bool mlir::affine::isLoopMemoryParallel(AffineForOp forOp) {
// Any memref-typed iteration arguments are treated as serializing.
if (llvm::any_of(forOp.getResultTypes(),
[](Type type) { return type.isa<BaseMemRefType>(); }))
/// and ending at operands which are not defined by AffineApplyOps.
// TODO: Add a method to AffineApplyOp which forward substitutes the
// AffineApplyOp into any user AffineApplyOps.
-void mlir::getReachableAffineApplyOps(
+void mlir::affine::getReachableAffineApplyOps(
ArrayRef<Value> operands, SmallVectorImpl<Operation *> &affineApplyOps) {
struct State {
// The ssa value for this node in the DFS traversal.
// FlatAffineValueConstraints. (For eg., by using iv - lb % step = 0 and/or by
// introducing a method in FlatAffineValueConstraints
// setExprStride(ArrayRef<int64_t> expr, int64_t stride)
-LogicalResult mlir::getIndexSet(MutableArrayRef<Operation *> ops,
- FlatAffineValueConstraints *domain) {
+LogicalResult mlir::affine::getIndexSet(MutableArrayRef<Operation *> ops,
+ FlatAffineValueConstraints *domain) {
SmallVector<Value, 4> indices;
SmallVector<Operation *, 8> loopOps;
size_t numDims = 0;
//
//
// TODO: Support AffineExprs mod/floordiv/ceildiv.
-DependenceResult mlir::checkMemrefAccessDependence(
+DependenceResult mlir::affine::checkMemrefAccessDependence(
const MemRefAccess &srcAccess, const MemRefAccess &dstAccess,
unsigned loopDepth, FlatAffineValueConstraints *dependenceConstraints,
SmallVector<DependenceComponent, 2> *dependenceComponents, bool allowRAR) {
/// Gathers dependence components for dependences between all ops in loop nest
/// rooted at 'forOp' at loop depths in range [1, maxLoopDepth].
-void mlir::getDependenceComponents(
+void mlir::affine::getDependenceComponents(
AffineForOp forOp, unsigned maxLoopDepth,
std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec) {
// Collect all load and store ops in loop nest rooted at 'forOp'.
#define DEBUG_TYPE "affine-structures"
using namespace mlir;
+using namespace affine;
using namespace presburger;
numRangeDims -= intersectRangeLHS - intersectRangeRHS;
}
-LogicalResult mlir::getRelationFromMap(AffineMap &map,
- FlatAffineRelation &rel) {
+LogicalResult mlir::affine::getRelationFromMap(AffineMap &map,
+ FlatAffineRelation &rel) {
// Get flattened affine expressions.
std::vector<SmallVector<int64_t, 8>> flatExprs;
FlatAffineValueConstraints localVarCst;
return success();
}
-LogicalResult mlir::getRelationFromMap(const AffineValueMap &map,
- FlatAffineRelation &rel) {
+LogicalResult mlir::affine::getRelationFromMap(const AffineValueMap &map,
+ FlatAffineRelation &rel) {
AffineMap affineMap = map.getAffineMap();
if (failed(getRelationFromMap(affineMap, rel)))
#include <type_traits>
using namespace mlir;
+using namespace mlir::affine;
/// Returns the trip count of the loop as an affine expression if the latter is
/// expressible as an affine expression, and nullptr otherwise. The trip count
/// expression is simplified before returning. This method only utilizes map
/// composition to construct lower and upper bounds before computing the trip
/// count expressions.
-void mlir::getTripCountMapAndOperands(
+void mlir::affine::getTripCountMapAndOperands(
AffineForOp forOp, AffineMap *tripCountMap,
SmallVectorImpl<Value> *tripCountOperands) {
MLIRContext *context = forOp.getContext();
/// otherwise. This method uses affine expression analysis (in turn using
/// getTripCount) and is able to determine constant trip count in non-trivial
/// cases.
-std::optional<uint64_t> mlir::getConstantTripCount(AffineForOp forOp) {
+std::optional<uint64_t> mlir::affine::getConstantTripCount(AffineForOp forOp) {
SmallVector<Value, 4> operands;
AffineMap map;
getTripCountMapAndOperands(forOp, &map, &operands);
/// Returns the greatest known integral divisor of the trip count. Affine
/// expression analysis is used (indirectly through getTripCount), and
/// this method is thus able to determine non-trivial divisors.
-uint64_t mlir::getLargestDivisorOfTripCount(AffineForOp forOp) {
+uint64_t mlir::affine::getLargestDivisorOfTripCount(AffineForOp forOp) {
SmallVector<Value, 4> operands;
AffineMap map;
getTripCountMapAndOperands(forOp, &map, &operands);
return !composeOp.getAffineValueMap().isFunctionOf(0, iv);
}
-DenseSet<Value> mlir::getInvariantAccesses(Value iv, ArrayRef<Value> indices) {
+DenseSet<Value> mlir::affine::getInvariantAccesses(Value iv,
+ ArrayRef<Value> indices) {
DenseSet<Value> res;
for (auto val : indices) {
if (isAccessIndexInvariant(iv, val)) {
return true;
}
-bool mlir::isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
- NestedPattern &vectorTransferMatcher) {
+bool mlir::affine::isVectorizableLoopBody(
+ AffineForOp loop, int *memRefDim, NestedPattern &vectorTransferMatcher) {
*memRefDim = -1;
VectorizableOpFun fun([memRefDim](AffineForOp loop, Operation &op) {
auto load = dyn_cast<AffineLoadOp>(op);
return isVectorizableLoopBodyWithOpCond(loop, fun, vectorTransferMatcher);
}
-bool mlir::isVectorizableLoopBody(AffineForOp loop,
- NestedPattern &vectorTransferMatcher) {
+bool mlir::affine::isVectorizableLoopBody(
+ AffineForOp loop, NestedPattern &vectorTransferMatcher) {
return isVectorizableLoopBodyWithOpCond(loop, nullptr, vectorTransferMatcher);
}
/// 'def' and all its uses have the same shift factor.
// TODO: extend this to check for memory-based dependence violation when we have
// the support.
-bool mlir::isOpwiseShiftValid(AffineForOp forOp, ArrayRef<uint64_t> shifts) {
+bool mlir::affine::isOpwiseShiftValid(AffineForOp forOp,
+ ArrayRef<uint64_t> shifts) {
auto *forBody = forOp.getBody();
assert(shifts.size() == forBody->getOperations().size());
#include "llvm/Support/raw_ostream.h"
using namespace mlir;
+using namespace mlir::affine;
llvm::BumpPtrAllocator *&NestedMatch::allocator() {
thread_local llvm::BumpPtrAllocator *allocator = nullptr;
static bool isAffineIfOp(Operation &op) { return isa<AffineIfOp>(op); }
namespace mlir {
+namespace affine {
namespace matcher {
NestedPattern Op(FilterFunctionType filter) {
}
} // namespace matcher
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "analysis-utils"
using namespace mlir;
+using namespace affine;
using namespace presburger;
using llvm::SmallDenseMap;
}
}
-void mlir::getAffineForIVs(Operation &op, SmallVectorImpl<AffineForOp> *loops) {
+void mlir::affine::getAffineForIVs(Operation &op,
+ SmallVectorImpl<AffineForOp> *loops) {
auto *currOp = op.getParentOp();
AffineForOp currAffineForOp;
// Traverse up the hierarchy collecting all 'affine.for' operation while
std::reverse(loops->begin(), loops->end());
}
-void mlir::getEnclosingAffineOps(Operation &op,
- SmallVectorImpl<Operation *> *ops) {
+void mlir::affine::getEnclosingAffineOps(Operation &op,
+ SmallVectorImpl<Operation *> *ops) {
ops->clear();
Operation *currOp = op.getParentOp();
}
std::optional<int64_t>
-mlir::getMemRefIntOrFloatEltSizeInBytes(MemRefType memRefType) {
+mlir::affine::getMemRefIntOrFloatEltSizeInBytes(MemRefType memRefType) {
auto elementType = memRefType.getElementType();
unsigned sizeInBits;
/// into account size of the vector as well.
// TODO: improve/complete this when we have target data.
std::optional<uint64_t>
-mlir::getIntOrFloatMemRefSizeInBytes(MemRefType memRefType) {
+mlir::affine::getIntOrFloatMemRefSizeInBytes(MemRefType memRefType) {
if (!memRefType.hasStaticShape())
return std::nullopt;
auto elementType = memRefType.getElementType();
}
template <typename LoadOrStoreOp>
-LogicalResult mlir::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
- bool emitError) {
+LogicalResult mlir::affine::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
+ bool emitError) {
static_assert(llvm::is_one_of<LoadOrStoreOp, AffineReadOpInterface,
AffineWriteOpInterface>::value,
"argument should be either a AffineReadOpInterface or a "
// Explicitly instantiate the template so that the compiler knows we need them!
template LogicalResult
-mlir::boundCheckLoadOrStoreOp(AffineReadOpInterface loadOp, bool emitError);
+mlir::affine::boundCheckLoadOrStoreOp(AffineReadOpInterface loadOp,
+ bool emitError);
template LogicalResult
-mlir::boundCheckLoadOrStoreOp(AffineWriteOpInterface storeOp, bool emitError);
+mlir::affine::boundCheckLoadOrStoreOp(AffineWriteOpInterface storeOp,
+ bool emitError);
// Returns in 'positions' the Block positions of 'op' in each ancestor
// Block from the Block containing operation, stopping at 'limitBlock'.
/// Returns the innermost common loop depth for the set of operations in 'ops'.
// TODO: Move this to LoopUtils.
-unsigned mlir::getInnermostCommonLoopDepth(
+unsigned mlir::affine::getInnermostCommonLoopDepth(
ArrayRef<Operation *> ops, SmallVectorImpl<AffineForOp> *surroundingLoops) {
unsigned numOps = ops.size();
assert(numOps > 0 && "Expected at least one operation");
/// then verifies if it is valid. Returns 'SliceComputationResult::Success' if
/// union was computed correctly, an appropriate failure otherwise.
SliceComputationResult
-mlir::computeSliceUnion(ArrayRef<Operation *> opsA, ArrayRef<Operation *> opsB,
- unsigned loopDepth, unsigned numCommonLoops,
- bool isBackwardSlice,
- ComputationSliceState *sliceUnion) {
+mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
+ ArrayRef<Operation *> opsB, unsigned loopDepth,
+ unsigned numCommonLoops, bool isBackwardSlice,
+ ComputationSliceState *sliceUnion) {
// Compute the union of slice bounds between all pairs in 'opsA' and
// 'opsB' in 'sliceUnionCst'.
FlatAffineValueConstraints sliceUnionCst;
// Compute slice bounds for 'srcAccess' and 'dstAccess'.
ComputationSliceState tmpSliceState;
- mlir::getComputationSliceState(i, j, &dependenceConstraints, loopDepth,
- isBackwardSlice, &tmpSliceState);
+ mlir::affine::getComputationSliceState(i, j, &dependenceConstraints,
+ loopDepth, isBackwardSlice,
+ &tmpSliceState);
if (sliceUnionCst.getNumDimAndSymbolVars() == 0) {
// Initialize 'sliceUnionCst' with the bounds computed in previous step.
// nest surrounding represented by slice loop bounds in 'slice'. Returns true
// on success, false otherwise (if a non-constant trip count was encountered).
// TODO: Make this work with non-unit step loops.
-bool mlir::buildSliceTripCountMap(
+bool mlir::affine::buildSliceTripCountMap(
const ComputationSliceState &slice,
llvm::SmallDenseMap<Operation *, uint64_t, 8> *tripCountMap) {
unsigned numSrcLoopIVs = slice.ivs.size();
}
// Return the number of iterations in the given slice.
-uint64_t mlir::getSliceIterationCount(
+uint64_t mlir::affine::getSliceIterationCount(
const llvm::SmallDenseMap<Operation *, uint64_t, 8> &sliceTripCountMap) {
uint64_t iterCount = 1;
for (const auto &count : sliceTripCountMap) {
// 'dependenceConstraints' at depth greater than 'loopDepth', and computes slice
// bounds in 'sliceState' which represent the one loop nest's IVs in terms of
// the other loop nest's IVs, symbols and constants (using 'isBackwardsSlice').
-void mlir::getComputationSliceState(
+void mlir::affine::getComputationSliceState(
Operation *depSourceOp, Operation *depSinkOp,
FlatAffineValueConstraints *dependenceConstraints, unsigned loopDepth,
bool isBackwardSlice, ComputationSliceState *sliceState) {
// entire destination index set. Subtract out the dependent destination
// iterations from destination index set and check for emptiness --- this is one
// solution.
-AffineForOp
-mlir::insertBackwardComputationSlice(Operation *srcOpInst, Operation *dstOpInst,
- unsigned dstLoopDepth,
- ComputationSliceState *sliceState) {
+AffineForOp mlir::affine::insertBackwardComputationSlice(
+ Operation *srcOpInst, Operation *dstOpInst, unsigned dstLoopDepth,
+ ComputationSliceState *sliceState) {
// Get loop nest surrounding src operation.
SmallVector<AffineForOp, 4> srcLoopIVs;
getAffineForIVs(*srcOpInst, &srcLoopIVs);
/// Returns the nesting depth of this statement, i.e., the number of loops
/// surrounding this statement.
-unsigned mlir::getNestingDepth(Operation *op) {
+unsigned mlir::affine::getNestingDepth(Operation *op) {
Operation *currOp = op;
unsigned depth = 0;
while ((currOp = currOp->getParentOp())) {
[](AffineExpr e) { return e == 0; });
}
-void mlir::getAffineIVs(Operation &op, SmallVectorImpl<Value> &ivs) {
+void mlir::affine::getAffineIVs(Operation &op, SmallVectorImpl<Value> &ivs) {
auto *currOp = op.getParentOp();
AffineForOp currAffineForOp;
// Traverse up the hierarchy collecting all 'affine.for' and affine.parallel
/// Returns the number of surrounding loops common to 'loopsA' and 'loopsB',
/// where each lists loops from outer-most to inner-most in loop nest.
-unsigned mlir::getNumCommonSurroundingLoops(Operation &a, Operation &b) {
+unsigned mlir::affine::getNumCommonSurroundingLoops(Operation &a,
+ Operation &b) {
SmallVector<Value, 4> loopsA, loopsB;
getAffineIVs(a, loopsA);
getAffineIVs(b, loopsB);
return totalSizeInBytes;
}
-std::optional<int64_t> mlir::getMemoryFootprintBytes(AffineForOp forOp,
- int memorySpace) {
+std::optional<int64_t> mlir::affine::getMemoryFootprintBytes(AffineForOp forOp,
+ int memorySpace) {
auto *forInst = forOp.getOperation();
return ::getMemoryFootprintBytes(
*forInst->getBlock(), Block::iterator(forInst),
}
/// Returns whether a loop is parallel and contains a reduction loop.
-bool mlir::isLoopParallelAndContainsReduction(AffineForOp forOp) {
+bool mlir::affine::isLoopParallelAndContainsReduction(AffineForOp forOp) {
SmallVector<LoopReduction> reductions;
if (!isLoopParallel(forOp, &reductions))
return false;
/// Returns in 'sequentialLoops' all sequential loops in loop nest rooted
/// at 'forOp'.
-void mlir::getSequentialLoops(AffineForOp forOp,
- llvm::SmallDenseSet<Value, 8> *sequentialLoops) {
+void mlir::affine::getSequentialLoops(
+ AffineForOp forOp, llvm::SmallDenseSet<Value, 8> *sequentialLoops) {
forOp->walk([&](Operation *op) {
if (auto innerFor = dyn_cast<AffineForOp>(op))
if (!isLoopParallel(innerFor))
});
}
-IntegerSet mlir::simplifyIntegerSet(IntegerSet set) {
+IntegerSet mlir::affine::simplifyIntegerSet(IntegerSet set) {
FlatAffineValueConstraints fac(set);
if (fac.isEmpty())
return IntegerSet::getEmptySet(set.getNumDims(), set.getNumSymbols(),
// ... | 0 | 0 | -1 | ... | ... | = 0
// 0 | 0 | 1 | -1 | 0 | -1 | >= 0
//
-FailureOr<AffineValueMap>
-mlir::simplifyConstrainedMinMaxOp(Operation *op,
- FlatAffineValueConstraints constraints) {
+FailureOr<AffineValueMap> mlir::affine::simplifyConstrainedMinMaxOp(
+ Operation *op, FlatAffineValueConstraints constraints) {
bool isMin = isa<AffineMinOp>(op);
assert((isMin || isa<AffineMaxOp>(op)) && "expect AffineMin/MaxOp");
MLIRContext *ctx = op->getContext();
newMap.getNumDims(), newMap.getNumSymbols());
}
}
- mlir::canonicalizeMapAndOperands(&newMap, &newOperands);
+ affine::canonicalizeMapAndOperands(&newMap, &newOperands);
return AffineValueMap(newMap, newOperands);
}
#include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h"
using namespace mlir;
+using namespace mlir::affine;
//===----------------------------------------------------------------------===//
// Affine Memory Op Interfaces
#include <optional>
using namespace mlir;
+using namespace mlir::affine;
#define DEBUG_TYPE "affine-ops"
/// `region` or is an argument of `region`. A value of index type defined at the
/// top level of a `AffineScope` region is always a valid symbol for all
/// uses in that region.
-bool mlir::isTopLevelValue(Value value, Region *region) {
+bool mlir::affine::isTopLevelValue(Value value, Region *region) {
if (auto arg = value.dyn_cast<BlockArgument>())
return arg.getParentRegion() == region;
return value.getDefiningOp()->getParentRegion() == region;
/// op with trait `AffineScope`. If the value is defined in an unlinked region,
/// conservatively assume it is not top-level. A value of index type defined at
/// the top level is always a valid symbol.
-bool mlir::isTopLevelValue(Value value) {
+bool mlir::affine::isTopLevelValue(Value value) {
if (auto arg = value.dyn_cast<BlockArgument>()) {
// The block owning the argument may be unlinked, e.g. when the surrounding
// region has not yet been attached to an Op, at which point the parent Op
/// Returns the closest region enclosing `op` that is held by an operation with
/// trait `AffineScope`; `nullptr` if there is no such region.
-Region *mlir::getAffineScope(Operation *op) {
+Region *mlir::affine::getAffineScope(Operation *op) {
auto *curOp = op;
while (auto *parentOp = curOp->getParentOp()) {
if (parentOp->hasTrait<OpTrait::AffineScope>())
// *) It is valid as a symbol.
// *) It is an induction variable.
// *) It is the result of affine apply operation with dimension id arguments.
-bool mlir::isValidDim(Value value) {
+bool mlir::affine::isValidDim(Value value) {
// The value must be an index type.
if (!value.getType().isIndex())
return false;
// *) It is valid as a symbol.
// *) It is an induction variable.
// *) It is the result of an affine apply operation with dimension id operands.
-bool mlir::isValidDim(Value value, Region *region) {
+bool mlir::affine::isValidDim(Value value, Region *region) {
// The value must be an index type.
if (!value.getType().isIndex())
return false;
// *) It is the result of an affine.apply operation with symbol operands.
// *) It is a result of the dim op on a memref whose corresponding size is a
// valid symbol.
-bool mlir::isValidSymbol(Value value) {
+bool mlir::affine::isValidSymbol(Value value) {
if (!value)
return false;
/// If `region` is null, conservatively assume the symbol definition scope does
/// not exist and only accept the values that would be symbols regardless of
/// the surrounding region structure, i.e. the first three cases above.
-bool mlir::isValidSymbol(Value value, Region *region) {
+bool mlir::affine::isValidSymbol(Value value, Region *region) {
// The value must be an index type.
if (!value.getType().isIndex())
return false;
}
/// Parses dimension and symbol list and returns true if parsing failed.
-ParseResult mlir::parseDimAndSymbolList(OpAsmParser &parser,
- SmallVectorImpl<Value> &operands,
- unsigned &numDims) {
+ParseResult mlir::affine::parseDimAndSymbolList(
+ OpAsmParser &parser, SmallVectorImpl<Value> &operands, unsigned &numDims) {
SmallVector<OpAsmParser::UnresolvedOperand, 8> opInfos;
if (parser.parseOperandList(opInfos, OpAsmParser::Delimiter::Paren))
return failure();
// its operands are valid dimension ids.
bool AffineApplyOp::isValidDim() {
return llvm::all_of(getOperands(),
- [](Value op) { return mlir::isValidDim(op); });
+ [](Value op) { return affine::isValidDim(op); });
}
// The result of the affine apply operation can be used as a dimension id if all
// operands are symbols.
bool AffineApplyOp::isValidSymbol() {
return llvm::all_of(getOperands(),
- [](Value op) { return mlir::isValidSymbol(op); });
+ [](Value op) { return affine::isValidSymbol(op); });
}
// The result of the affine apply operation can be used as a symbol in `region`
// if all its operands are symbols in `region`.
bool AffineApplyOp::isValidSymbol(Region *region) {
return llvm::all_of(getOperands(), [&](Value operand) {
- return mlir::isValidSymbol(operand, region);
+ return affine::isValidSymbol(operand, region);
});
}
*map = simplifyAffineMap(*map);
}
-void mlir::fullyComposeAffineMapAndOperands(AffineMap *map,
- SmallVectorImpl<Value> *operands) {
+void mlir::affine::fullyComposeAffineMapAndOperands(
+ AffineMap *map, SmallVectorImpl<Value> *operands) {
while (llvm::any_of(*operands, [](Value v) {
return isa_and_nonnull<AffineApplyOp>(v.getDefiningOp());
})) {
return op->getResult(0);
}
-AffineApplyOp mlir::makeComposedAffineApply(OpBuilder &b, Location loc,
- AffineMap map,
- ValueRange operands) {
+AffineApplyOp mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc,
+ AffineMap map,
+ ValueRange operands) {
AffineMap normalizedMap = map;
SmallVector<Value, 8> normalizedOperands(operands.begin(), operands.end());
composeAffineMapAndOperands(&normalizedMap, &normalizedOperands);
return b.create<AffineApplyOp>(loc, normalizedMap, normalizedOperands);
}
-AffineApplyOp mlir::makeComposedAffineApply(OpBuilder &b, Location loc,
- AffineExpr e, ValueRange values) {
+AffineApplyOp mlir::affine::makeComposedAffineApply(OpBuilder &b, Location loc,
+ AffineExpr e,
+ ValueRange values) {
return makeComposedAffineApply(
b, loc, AffineMap::inferFromExprList(ArrayRef<AffineExpr>{e}).front(),
values);
}
OpFoldResult
-mlir::makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineMap map,
- ArrayRef<OpFoldResult> operands) {
+mlir::affine::makeComposedFoldedAffineApply(OpBuilder &b, Location loc,
+ AffineMap map,
+ ArrayRef<OpFoldResult> operands) {
assert(map.getNumResults() == 1 && "building affine.apply with !=1 result");
SmallVector<Operation *> constants;
}
OpFoldResult
-mlir::makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineExpr expr,
- ArrayRef<OpFoldResult> operands) {
+mlir::affine::makeComposedFoldedAffineApply(OpBuilder &b, Location loc,
+ AffineExpr expr,
+ ArrayRef<OpFoldResult> operands) {
return makeComposedFoldedAffineApply(
b, loc, AffineMap::inferFromExprList(ArrayRef<AffineExpr>{expr}).front(),
operands);
}
-SmallVector<OpFoldResult> mlir::makeComposedFoldedMultiResultAffineApply(
+SmallVector<OpFoldResult>
+mlir::affine::makeComposedFoldedMultiResultAffineApply(
OpBuilder &b, Location loc, AffineMap map,
ArrayRef<OpFoldResult> operands) {
return llvm::to_vector(llvm::map_range(
}));
}
-Value mlir::makeComposedAffineMin(OpBuilder &b, Location loc, AffineMap map,
- ValueRange operands) {
+Value mlir::affine::makeComposedAffineMin(OpBuilder &b, Location loc,
+ AffineMap map, ValueRange operands) {
SmallVector<Value> allOperands = llvm::to_vector(operands);
composeMultiResultAffineMap(map, allOperands);
return b.createOrFold<AffineMinOp>(loc, b.getIndexType(), map, allOperands);
}
OpFoldResult
-mlir::makeComposedFoldedAffineMin(OpBuilder &b, Location loc, AffineMap map,
- ArrayRef<OpFoldResult> operands) {
+mlir::affine::makeComposedFoldedAffineMin(OpBuilder &b, Location loc,
+ AffineMap map,
+ ArrayRef<OpFoldResult> operands) {
return makeComposedFoldedMinMax<AffineMinOp>(b, loc, map, operands);
}
OpFoldResult
-mlir::makeComposedFoldedAffineMax(OpBuilder &b, Location loc, AffineMap map,
- ArrayRef<OpFoldResult> operands) {
+mlir::affine::makeComposedFoldedAffineMax(OpBuilder &b, Location loc,
+ AffineMap map,
+ ArrayRef<OpFoldResult> operands) {
return makeComposedFoldedMinMax<AffineMaxOp>(b, loc, map, operands);
}
return b.createOrFold<AffineApplyOp>(loc, map, operands);
}
-SmallVector<Value, 4> mlir::applyMapToValues(OpBuilder &b, Location loc,
- AffineMap map, ValueRange values) {
+SmallVector<Value, 4> mlir::affine::applyMapToValues(OpBuilder &b, Location loc,
+ AffineMap map,
+ ValueRange values) {
SmallVector<Value, 4> res;
res.reserve(map.getNumResults());
unsigned numDims = map.getNumDims(), numSym = map.getNumSymbols();
*operands = resultOperands;
}
-void mlir::canonicalizeMapAndOperands(AffineMap *map,
- SmallVectorImpl<Value> *operands) {
+void mlir::affine::canonicalizeMapAndOperands(
+ AffineMap *map, SmallVectorImpl<Value> *operands) {
canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
}
-void mlir::canonicalizeSetAndOperands(IntegerSet *set,
- SmallVectorImpl<Value> *operands) {
+void mlir::affine::canonicalizeSetAndOperands(
+ IntegerSet *set, SmallVectorImpl<Value> *operands) {
canonicalizeMapOrSetAndOperands<IntegerSet>(set, operands);
}
/// Returns true if the provided value is the induction variable of a
/// AffineForOp.
-bool mlir::isAffineForInductionVar(Value val) {
+bool mlir::affine::isAffineForInductionVar(Value val) {
return getForInductionVarOwner(val) != AffineForOp();
}
-bool mlir::isAffineParallelInductionVar(Value val) {
+bool mlir::affine::isAffineParallelInductionVar(Value val) {
return getAffineParallelInductionVarOwner(val) != nullptr;
}
-bool mlir::isAffineInductionVar(Value val) {
+bool mlir::affine::isAffineInductionVar(Value val) {
return isAffineForInductionVar(val) || isAffineParallelInductionVar(val);
}
-AffineForOp mlir::getForInductionVarOwner(Value val) {
+AffineForOp mlir::affine::getForInductionVarOwner(Value val) {
auto ivArg = val.dyn_cast<BlockArgument>();
if (!ivArg || !ivArg.getOwner())
return AffineForOp();
return AffineForOp();
}
-AffineParallelOp mlir::getAffineParallelInductionVarOwner(Value val) {
+AffineParallelOp mlir::affine::getAffineParallelInductionVarOwner(Value val) {
auto ivArg = val.dyn_cast<BlockArgument>();
if (!ivArg || !ivArg.getOwner())
return nullptr;
/// Extracts the induction variables from a list of AffineForOps and returns
/// them.
-void mlir::extractForInductionVars(ArrayRef<AffineForOp> forInsts,
- SmallVectorImpl<Value> *ivs) {
+void mlir::
affine::extractForInductionVars(ArrayRef<AffineForOp> forInsts,
+ SmallVectorImpl<Value> *ivs) {
ivs->reserve(forInsts.size());
for (auto forInst : forInsts)
ivs->push_back(forInst.getInductionVar());
}
-void mlir::extractInductionVars(ArrayRef<mlir::Operation *> affineOps,
- SmallVectorImpl<mlir::Value> &ivs) {
+void mlir::affine::extractInductionVars(ArrayRef<mlir::Operation *> affineOps,
+ SmallVectorImpl<mlir::Value> &ivs) {
ivs.reserve(affineOps.size());
for (Operation *op : affineOps) {
// Add constraints from forOp's bounds.
/*iterArgs=*/std::nullopt, bodyBuilderFn);
}
-void mlir::buildAffineLoopNest(
+void mlir::affine::buildAffineLoopNest(
OpBuilder &builder, Location loc, ArrayRef<int64_t> lbs,
ArrayRef<int64_t> ubs, ArrayRef<int64_t> steps,
function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
buildAffineLoopFromConstants);
}
-void mlir::buildAffineLoopNest(
+void mlir::affine::buildAffineLoopNest(
OpBuilder &builder, Location loc, ValueRange lbs, ValueRange ubs,
ArrayRef<int64_t> steps,
function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilderFn) {
buildAffineLoopFromValues);
}
-AffineForOp mlir::replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop,
- ValueRange newIterOperands,
- ValueRange newYieldedValues,
- ValueRange newIterArgs,
- bool replaceLoopResults) {
+AffineForOp mlir::affine::replaceForOpWithNewYields(OpBuilder &b,
+ AffineForOp loop,
+ ValueRange newIterOperands,
+ ValueRange newYieldedValues,
+ ValueRange newIterArgs,
+ bool replaceLoopResults) {
assert(newIterOperands.size() == newYieldedValues.size() &&
"newIterOperands must be of the same size as newYieldedValues");
// Create a new loop before the existing one, with the extra operands.
#include "mlir/Dialect/Affine/IR/AffineOps.h"
using namespace mlir;
+using namespace mlir::affine;
AffineValueMap::AffineValueMap(AffineMap map, ValueRange operands,
ValueRange results)
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
using namespace mlir;
+using namespace mlir::affine;
namespace mlir {
namespace {
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
using namespace mlir;
+using namespace mlir::affine;
using namespace mlir::transform;
//===----------------------------------------------------------------------===//
#include <optional>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINEDATACOPYGENERATION
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "affine-data-copy-generate"
using namespace mlir;
+using namespace mlir::affine;
namespace {
// TODO: We currently can't generate copies correctly when stores
// are strided. Check for strided stores.
struct AffineDataCopyGeneration
- : public impl::AffineDataCopyGenerationBase<AffineDataCopyGeneration> {
+ : public affine::impl::AffineDataCopyGenerationBase<
+ AffineDataCopyGeneration> {
AffineDataCopyGeneration() = default;
explicit AffineDataCopyGeneration(unsigned slowMemorySpace,
unsigned fastMemorySpace,
/// by the latter. Only load op's handled for now.
/// TODO: extend this to store op's.
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createAffineDataCopyGenerationPass(unsigned slowMemorySpace,
- unsigned fastMemorySpace,
- unsigned tagMemorySpace,
- int minDmaTransferSize,
- uint64_t fastMemCapacityBytes) {
+mlir::affine::createAffineDataCopyGenerationPass(
+ unsigned slowMemorySpace, unsigned fastMemorySpace, unsigned tagMemorySpace,
+ int minDmaTransferSize, uint64_t fastMemCapacityBytes) {
return std::make_unique<AffineDataCopyGeneration>(
slowMemorySpace, fastMemorySpace, tagMemorySpace, minDmaTransferSize,
fastMemCapacityBytes);
}
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createAffineDataCopyGenerationPass() {
+mlir::affine::createAffineDataCopyGenerationPass() {
return std::make_unique<AffineDataCopyGeneration>();
}
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINEEXPANDINDEXOPS
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// Lowers `affine.delinearize_index` into a sequence of division and remainder
};
class ExpandAffineIndexOpsPass
- : public impl::AffineExpandIndexOpsBase<ExpandAffineIndexOpsPass> {
+ : public affine::impl::AffineExpandIndexOpsBase<ExpandAffineIndexOpsPass> {
public:
ExpandAffineIndexOpsPass() = default;
} // namespace
-void mlir::populateAffineExpandIndexOpsPatterns(RewritePatternSet &patterns) {
+void mlir::affine::populateAffineExpandIndexOpsPatterns(
+ RewritePatternSet &patterns) {
patterns.insert<LowerDelinearizeIndexOps>(patterns.getContext());
}
-std::unique_ptr<Pass> mlir::createAffineExpandIndexOpsPass() {
+std::unique_ptr<Pass> mlir::affine::createAffineExpandIndexOpsPass() {
return std::make_unique<ExpandAffineIndexOpsPass>();
}
#include "llvm/Support/raw_ostream.h"
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINELOOPINVARIANTCODEMOTION
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "licm"
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// TODO: This code should be removed once the new LICM pass can handle its
/// uses.
struct LoopInvariantCodeMotion
- : public impl::AffineLoopInvariantCodeMotionBase<LoopInvariantCodeMotion> {
+ : public affine::impl::AffineLoopInvariantCodeMotionBase<
+ LoopInvariantCodeMotion> {
void runOnOperation() override;
void runOnAffineForOp(AffineForOp forOp);
};
SmallPtrSetImpl<Operation *> &opsToHoist);
// Returns true if the individual op is loop invariant.
-bool isOpLoopInvariant(Operation &op, Value indVar, ValueRange iterArgs,
- SmallPtrSetImpl<Operation *> &opsWithUsers,
- SmallPtrSetImpl<Operation *> &opsToHoist) {
+static bool isOpLoopInvariant(Operation &op, Value indVar, ValueRange iterArgs,
+ SmallPtrSetImpl<Operation *> &opsWithUsers,
+ SmallPtrSetImpl<Operation *> &opsToHoist) {
LLVM_DEBUG(llvm::dbgs() << "iterating on op: " << op;);
if (auto ifOp = dyn_cast<AffineIfOp>(op)) {
}
// Checks if all ops in a region (i.e. list of blocks) are loop invariant.
-bool areAllOpsInTheBlockListInvariant(
- Region &blockList, Value indVar, ValueRange iterArgs,
- SmallPtrSetImpl<Operation *> &opsWithUsers,
- SmallPtrSetImpl<Operation *> &opsToHoist) {
+static bool
+areAllOpsInTheBlockListInvariant(Region &blockList, Value indVar,
+ ValueRange iterArgs,
+ SmallPtrSetImpl<Operation *> &opsWithUsers,
+ SmallPtrSetImpl<Operation *> &opsToHoist) {
for (auto &b : blockList) {
for (auto &op : b) {
}
// Returns true if the affine.if op can be hoisted.
-bool checkInvarianceOfNestedIfOps(AffineIfOp ifOp, Value indVar,
- ValueRange iterArgs,
- SmallPtrSetImpl<Operation *> &opsWithUsers,
- SmallPtrSetImpl<Operation *> &opsToHoist) {
+static bool
+checkInvarianceOfNestedIfOps(AffineIfOp ifOp, Value indVar, ValueRange iterArgs,
+ SmallPtrSetImpl<Operation *> &opsWithUsers,
+ SmallPtrSetImpl<Operation *> &opsToHoist) {
if (!areAllOpsInTheBlockListInvariant(ifOp.getThenRegion(), indVar, iterArgs,
opsWithUsers, opsToHoist))
return false;
}
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createAffineLoopInvariantCodeMotionPass() {
+mlir::affine::createAffineLoopInvariantCodeMotionPass() {
return std::make_unique<LoopInvariantCodeMotion>();
}
#include "mlir/Dialect/Func/IR/FuncOps.h"
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINELOOPNORMALIZE
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// As currently implemented, this pass cannot fail, but it might skip over ops
/// that are already in a normalized form.
struct AffineLoopNormalizePass
- : public impl::AffineLoopNormalizeBase<AffineLoopNormalizePass> {
+ : public
affine::impl::AffineLoopNormalizeBase<AffineLoopNormalizePass> {
explicit AffineLoopNormalizePass(bool promoteSingleIter) {
this->promoteSingleIter = promoteSingleIter;
}
} // namespace
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createAffineLoopNormalizePass(bool promoteSingleIter) {
+mlir::affine::createAffineLoopNormalizePass(bool promoteSingleIter) {
return std::make_unique<AffineLoopNormalizePass>(promoteSingleIter);
}
#include <deque>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINEPARALLELIZE
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "affine-parallel"
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// Convert all parallel affine.for op into 1-D affine.parallel op.
struct AffineParallelize
- : public impl::AffineParallelizeBase<AffineParallelize> {
+ : public
affine::impl::AffineParallelizeBase<AffineParallelize> {
void runOnOperation() override;
};
}
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createAffineParallelizePass() {
+mlir::affine::createAffineParallelizePass() {
return std::make_unique<AffineParallelize>();
}
#include <algorithm>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINESCALARREPLACEMENT
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "affine-scalrep"
using namespace mlir;
+using namespace mlir::affine;
namespace {
struct AffineScalarReplacement
- : public impl::AffineScalarReplacementBase<AffineScalarReplacement> {
+ : public affine::impl::AffineScalarReplacementBase<
+ AffineScalarReplacement> {
void runOnOperation() override;
};
} // namespace
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createAffineScalarReplacementPass() {
+mlir::affine::createAffineScalarReplacementPass() {
return std::make_unique<AffineScalarReplacement>();
}
#include "llvm/Support/Debug.h"
using namespace mlir;
+using namespace mlir::affine;
#define DEBUG_TYPE "decompose-affine-ops"
#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
return count;
}
-void mlir::reorderOperandsByHoistability(RewriterBase &rewriter,
- AffineApplyOp op) {
+void mlir::affine::reorderOperandsByHoistability(RewriterBase &rewriter,
+ AffineApplyOp op) {
SmallVector<int64_t> numInvariant = llvm::to_vector(
llvm::map_range(op->getOpOperands(), [&](OpOperand &operand) {
return numEnclosingInvariantLoops(operand);
rhsOperands);
}
-FailureOr<AffineApplyOp> mlir::decompose(RewriterBase &rewriter,
- AffineApplyOp op) {
+FailureOr<AffineApplyOp> mlir::affine::decompose(RewriterBase &rewriter,
+ AffineApplyOp op) {
// 1. Preconditions: only handle dimensionless AffineApplyOp maps with a
// top-level binary expression that we can reassociate (i.e. add or mul).
AffineMap m = op.getAffineMap();
#include "llvm/Support/Debug.h"
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_LOOPCOALESCING
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define PASS_NAME "loop-coalescing"
#define DEBUG_TYPE PASS_NAME
using namespace mlir;
+using namespace mlir::affine;
namespace {
struct LoopCoalescingPass
- : public impl::LoopCoalescingBase<LoopCoalescingPass> {
+ : public affine::impl::LoopCoalescingBase<LoopCoalescingPass> {
void runOnOperation() override {
func::FuncOp func = getOperation();
} // namespace
-std::unique_ptr<OperationPass<func::FuncOp>> mlir::createLoopCoalescingPass() {
+std::unique_ptr<OperationPass<func::FuncOp>>
+mlir::affine::createLoopCoalescingPass() {
return std::make_unique<LoopCoalescingPass>();
}
#include <sstream>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINELOOPFUSION
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "affine-loop-fusion"
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// Loop fusion pass. This pass currently supports a greedy fusion policy,
// TODO: Extend this pass to check for fusion preventing dependences,
// and add support for more general loop fusion algorithms.
-struct LoopFusion : public impl::AffineLoopFusionBase<LoopFusion> {
+struct LoopFusion : public affine::impl::AffineLoopFusionBase<LoopFusion> {
LoopFusion() = default;
LoopFusion(unsigned fastMemorySpace, uint64_t localBufSizeThresholdBytes,
bool maximalFusion, enum FusionMode affineFusionMode) {
depthSliceUnions.resize(dstLoopDepthTest);
FusionStrategy strategy(FusionStrategy::ProducerConsumer);
for (unsigned i = 1; i <= dstLoopDepthTest; ++i) {
- FusionResult result = mlir::canFuseLoops(
+ FusionResult result = affine::canFuseLoops(
srcAffineForOp, dstAffineForOp,
/*dstLoopDepth=*/i, &depthSliceUnions[i - 1], strategy);
unsigned maxLegalFusionDepth = 0;
FusionStrategy strategy(memref);
for (unsigned i = 1; i <= dstLoopDepthTest; ++i) {
- FusionResult result = mlir::canFuseLoops(
+ FusionResult result = affine::canFuseLoops(
sibAffineForOp, dstAffineForOp,
/*dstLoopDepth=*/i, &depthSliceUnions[i - 1], strategy);
// further inside `fuseLoops`.
bool isInnermostInsertion = (bestDstLoopDepth == dstLoopDepthTest);
// Fuse computation slice of 'sibLoopNest' into 'dstLoopNest'.
- mlir::fuseLoops(sibAffineForOp, dstAffineForOp,
- depthSliceUnions[bestDstLoopDepth - 1],
- isInnermostInsertion);
+ affine::fuseLoops(sibAffineForOp, dstAffineForOp,
+ depthSliceUnions[bestDstLoopDepth - 1],
+ isInnermostInsertion);
auto dstForInst = cast<AffineForOp>(dstNode->op);
// Update operation position of fused loop nest (if needed).
runOnBlock(&block);
}
-std::unique_ptr<Pass>
-mlir::createLoopFusionPass(unsigned fastMemorySpace,
- uint64_t localBufSizeThreshold, bool maximalFusion,
- enum FusionMode affineFusionMode) {
+std::unique_ptr<Pass> mlir::affine::createLoopFusionPass(
+ unsigned fastMemorySpace, uint64_t localBufSizeThreshold,
+ bool maximalFusion, enum FusionMode affineFusionMode) {
return std::make_unique<LoopFusion>(fastMemorySpace, localBufSizeThreshold,
maximalFusion, affineFusionMode);
}
#include <optional>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINELOOPTILING
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
using namespace mlir;
+using namespace mlir::affine;
#define DEBUG_TYPE "affine-loop-tile"
namespace {
/// A pass to perform loop tiling on all suitable loop nests of a Function.
-struct LoopTiling : public impl::AffineLoopTilingBase<LoopTiling> {
+struct LoopTiling : public affine::impl::AffineLoopTilingBase<LoopTiling> {
LoopTiling() = default;
explicit LoopTiling(uint64_t cacheSizeBytes, bool avoidMaxMinBounds = true)
: avoidMaxMinBounds(avoidMaxMinBounds) {
/// Creates a pass to perform loop tiling on all suitable loop nests of a
/// Function.
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createLoopTilingPass(uint64_t cacheSizeBytes) {
+mlir::affine::createLoopTilingPass(uint64_t cacheSizeBytes) {
return std::make_unique<LoopTiling>(cacheSizeBytes);
}
-std::unique_ptr<OperationPass<func::FuncOp>> mlir::createLoopTilingPass() {
+std::unique_ptr<OperationPass<func::FuncOp>>
+mlir::affine::createLoopTilingPass() {
return std::make_unique<LoopTiling>();
}
/// hyper-rectangles, which are scheduled in the lexicographically increasing
/// order on the vector of loop indices. This function will return failure when
/// any dependence component is negative along any of `origLoops`.
-static bool checkTilingLegality(MutableArrayRef<mlir::AffineForOp> origLoops) {
+static bool checkTilingLegality(MutableArrayRef<AffineForOp> origLoops) {
assert(!origLoops.empty() && "no original loops provided");
// We first find out all dependences we intend to check.
#include <optional>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINELOOPUNROLL
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "affine-loop-unroll"
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// full unroll threshold was specified, in which case, fully unrolls all loops
/// with trip count less than the specified threshold. The latter is for testing
/// purposes, especially for testing outer loop unrolling.
-struct LoopUnroll : public impl::AffineLoopUnrollBase<LoopUnroll> {
+struct LoopUnroll : public affine::impl::AffineLoopUnrollBase<LoopUnroll> {
// Callback to obtain unroll factors; if this has a callable target, takes
// precedence over command-line argument or passed argument.
const std::function<unsigned(AffineForOp)> getUnrollFactor;
cleanUpUnroll);
}
-std::unique_ptr<OperationPass<func::FuncOp>> mlir::createLoopUnrollPass(
+std::unique_ptr<OperationPass<func::FuncOp>> mlir::affine::createLoopUnrollPass(
int unrollFactor, bool unrollUpToFactor, bool unrollFull,
const std::function<unsigned(AffineForOp)> &getUnrollFactor) {
return std::make_unique<LoopUnroll>(
#include <optional>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINELOOPUNROLLANDJAM
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "affine-loop-unroll-jam"
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// Loop unroll jam pass. Currently, this just unroll jams the first
/// outer loop in a Function.
struct LoopUnrollAndJam
- : public impl::AffineLoopUnrollAndJamBase<LoopUnrollAndJam> {
+ : public affine::impl::AffineLoopUnrollAndJamBase<LoopUnrollAndJam> {
explicit LoopUnrollAndJam(
std::optional<unsigned> unrollJamFactor = std::nullopt) {
if (unrollJamFactor)
} // namespace
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createLoopUnrollAndJamPass(int unrollJamFactor) {
+mlir::affine::createLoopUnrollAndJamPass(int unrollJamFactor) {
return std::make_unique<LoopUnrollAndJam>(
unrollJamFactor == -1 ? std::nullopt
: std::optional<unsigned>(unrollJamFactor));
#include "llvm/Support/Debug.h"
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINEPIPELINEDATATRANSFER
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "affine-pipeline-data-transfer"
using namespace mlir;
+using namespace mlir::affine;
namespace {
struct PipelineDataTransfer
- : public impl::AffinePipelineDataTransferBase<PipelineDataTransfer> {
+ : public affine::impl::AffinePipelineDataTransferBase<
+ PipelineDataTransfer> {
void runOnOperation() override;
void runOnAffineForOp(AffineForOp forOp);
/// Creates a pass to pipeline explicit movement of data across levels of the
/// memory hierarchy.
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createPipelineDataTransferPass() {
+mlir::affine::createPipelineDataTransferPass() {
return std::make_unique<PipelineDataTransfer>();
}
instShiftMap[dmaStartOp] = 0;
// Set shifts for DMA start op's affine operand computation slices to 0.
SmallVector<AffineApplyOp, 4> sliceOps;
- mlir::createAffineComputationSlice(dmaStartOp, &sliceOps);
+ affine::createAffineComputationSlice(dmaStartOp, &sliceOps);
if (!sliceOps.empty()) {
for (auto sliceOp : sliceOps) {
instShiftMap[sliceOp.getOperation()] = 0;
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
using namespace mlir;
+using namespace mlir::affine;
static FailureOr<OpFoldResult>
reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
}
// Simplify and return bound.
- mlir::canonicalizeMapAndOperands(&boundMap, &operands);
+ affine::canonicalizeMapAndOperands(&boundMap, &operands);
// Check for special cases where no affine.apply op is needed.
if (boundMap.isSingleConstant()) {
// Bound is a constant: return an IntegerAttr.
operands[expr.getPosition() + boundMap.getNumDims()]);
// General case: build affine.apply op.
return static_cast<OpFoldResult>(
- b.create<AffineApplyOp>(loc, boundMap, operands).getResult());
+ b.create<affine::AffineApplyOp>(loc, boundMap, operands).getResult());
}
-FailureOr<OpFoldResult> mlir::reifyShapedValueDimBound(
+FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound(
OpBuilder &b, Location loc, presburger::BoundType type, Value value,
int64_t dim, ValueBoundsConstraintSet::StopConditionFn stopCondition,
bool closedUB) {
closedUB);
}
-FailureOr<OpFoldResult> mlir::reifyIndexValueBound(
+FailureOr<OpFoldResult> mlir::affine::reifyIndexValueBound(
OpBuilder &b, Location loc, presburger::BoundType type, Value value,
ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) {
auto reifyToOperands = [&](Value v, std::optional<int64_t> d) {
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_SIMPLIFYAFFINESTRUCTURES
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
#define DEBUG_TYPE "simplify-affine-structure"
using namespace mlir;
+using namespace mlir::affine;
namespace {
/// all memrefs with non-trivial layout maps are converted to ones with trivial
/// identity layout ones.
struct SimplifyAffineStructures
- : public impl::SimplifyAffineStructuresBase<SimplifyAffineStructures> {
+ : public affine::impl::SimplifyAffineStructuresBase<
+ SimplifyAffineStructures> {
void runOnOperation() override;
/// Utility to simplify an affine attribute and update its entry in the parent
} // namespace
std::unique_ptr<OperationPass<func::FuncOp>>
-mlir::createSimplifyAffineStructuresPass() {
+mlir::affine::createSimplifyAffineStructuresPass() {
return std::make_unique<SimplifyAffineStructures>();
}
#include <optional>
namespace mlir {
+namespace affine {
#define GEN_PASS_DEF_AFFINEVECTORIZE
#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
} // namespace mlir
using namespace mlir;
+using namespace affine;
using namespace vector;
///
static std::optional<NestedPattern>
makePattern(const DenseSet<Operation *> ¶llelLoops, int vectorRank,
ArrayRef<int64_t> fastestVaryingPattern) {
- using matcher::For;
+ using affine::matcher::For;
int64_t d0 = fastestVaryingPattern.empty() ? -1 : fastestVaryingPattern[0];
int64_t d1 = fastestVaryingPattern.size() < 2 ? -1 : fastestVaryingPattern[1];
int64_t d2 = fastestVaryingPattern.size() < 3 ? -1 : fastestVaryingPattern[2];
}
static NestedPattern &vectorTransferPattern() {
- static auto pattern = matcher::Op([](Operation &op) {
+ static auto pattern = affine::matcher::Op([](Operation &op) {
return isa<vector::TransferReadOp, vector::TransferWriteOp>(op);
});
return pattern;
/// Base state for the vectorize pass.
/// Command line arguments are preempted by non-empty pass arguments.
-struct Vectorize : public impl::AffineVectorizeBase<Vectorize> {
+struct Vectorize : public affine::impl::AffineVectorizeBase<Vectorize> {
using Base::Base;
void runOnOperation() override;
return success();
}
-namespace mlir {
/// External utility to vectorize affine loops in 'loops' using the n-D
/// vectorization factors in 'vectorSizes'. By default, each vectorization
/// If `reductionLoops` is not empty, the given reduction loops may be
/// vectorized along the reduction dimension.
/// TODO: Vectorizing reductions is supported only for 1-D vectorization.
-void vectorizeAffineLoops(Operation *parentOp, DenseSet<Operation *> &loops,
- ArrayRef<int64_t> vectorSizes,
- ArrayRef<int64_t> fastestVaryingPattern,
- const ReductionLoopMap &reductionLoops) {
+void mlir::affine::vectorizeAffineLoops(
+ Operation *parentOp, DenseSet<Operation *> &loops,
+ ArrayRef<int64_t> vectorSizes, ArrayRef<int64_t> fastestVaryingPattern,
+ const ReductionLoopMap &reductionLoops) {
// Thread-safe RAII local context, BumpPtrAllocator freed on exit.
NestedPatternContext mlContext;
vectorizeLoops(parentOp, loops, vectorSizes, fastestVaryingPattern,
/// loops = {{%i2}}, to vectorize only the first innermost loop;
/// loops = {{%i3}}, to vectorize only the second innermost loop;
/// loops = {{%i1}}, to vectorize only the middle loop.
-LogicalResult
-
vectorizeAffineLoopNest(std::vector<SmallVector<AffineForOp, 2>> &loops,
- const VectorizationStrategy &strategy) {
+LogicalResult mlir::affine::vectorizeAffineLoopNest(
+
std::vector<SmallVector<AffineForOp, 2>> &loops,
+ const VectorizationStrategy &strategy) {
// Thread-safe RAII local context, BumpPtrAllocator freed on exit.
NestedPatternContext mlContext;
if (failed(verifyLoopNesting(loops)))
return failure();
return vectorizeLoopNest(loops, strategy);
}
-
-} // namespace mlir
#define DEBUG_TYPE "loop-fusion-utils"
using namespace mlir;
+using namespace mlir::affine;
// Gathers all load and store memref accesses in 'opA' into 'values', where
// 'values[memref] == true' for each store operation.
// TODO: This pass performs some computation that is the same for all the depths
// (e.g., getMaxLoopDepth). Implement a version of this utility that processes
// all the depths at once or only the legal maximal depth for maximal fusion.
-FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
- unsigned dstLoopDepth,
- ComputationSliceState *srcSlice,
- FusionStrategy fusionStrategy) {
+FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
+ AffineForOp dstForOp,
+ unsigned dstLoopDepth,
+ ComputationSliceState *srcSlice,
+ FusionStrategy fusionStrategy) {
// Return 'failure' if 'dstLoopDepth == 0'.
if (dstLoopDepth == 0) {
LLVM_DEBUG(llvm::dbgs() << "Cannot fuse loop nests at depth 0\n");
// Calculate the number of common loops surrounding 'srcForOp' and 'dstForOp'.
unsigned numCommonLoops =
- mlir::getNumCommonSurroundingLoops(*srcForOp, *dstForOp);
+ affine::getNumCommonSurroundingLoops(*srcForOp, *dstForOp);
// Filter out ops in 'opsA' to compute the slice union based on the
// assumptions made by the fusion strategy.
// Compute union of computation slices computed between all pairs of ops
// from 'forOpA' and 'forOpB'.
- SliceComputationResult sliceComputationResult =
- mlir::computeSliceUnion(strategyOpsA, opsB, dstLoopDepth, numCommonLoops,
- isSrcForOpBeforeDstForOp, srcSlice);
+ SliceComputationResult sliceComputationResult = affine::computeSliceUnion(
+ strategyOpsA, opsB, dstLoopDepth, numCommonLoops,
+ isSrcForOpBeforeDstForOp, srcSlice);
if (sliceComputationResult.value == SliceComputationResult::GenericFailure) {
LLVM_DEBUG(llvm::dbgs() << "computeSliceUnion failed\n");
return FusionResult::FailPrecondition;
/// Patch the loop body of a forOp that is a single iteration reduction loop
/// into its containing block.
-LogicalResult promoteSingleIterReductionLoop(AffineForOp forOp,
- bool siblingFusionUser) {
+static LogicalResult promoteSingleIterReductionLoop(AffineForOp forOp,
+ bool siblingFusionUser) {
// Check if the reduction loop is a single iteration loop.
std::optional<uint64_t> tripCount = getConstantTripCount(forOp);
if (!tripCount || *tripCount != 1)
/// Fuses 'srcForOp' into 'dstForOp' with destination loop block insertion point
/// and source slice loop bounds specified in 'srcSlice'.
-void mlir::fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
- const ComputationSliceState &srcSlice,
- bool isInnermostSiblingInsertion) {
+void mlir::affine::fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
+ const ComputationSliceState &srcSlice,
+ bool isInnermostSiblingInsertion) {
// Clone 'srcForOp' into 'dstForOp' at 'srcSlice->insertPoint'.
OpBuilder b(srcSlice.insertPoint->getBlock(), srcSlice.insertPoint);
IRMapping mapper;
/// Collect loop nest statistics (eg. loop trip count and operation count)
/// in 'stats' for loop nest rooted at 'forOp'. Returns true on success,
/// returns false otherwise.
-bool mlir::getLoopNestStats(AffineForOp forOpRoot, LoopNestStats *stats) {
+bool mlir::affine::getLoopNestStats(AffineForOp forOpRoot,
+ LoopNestStats *stats) {
auto walkResult = forOpRoot.walk([&](AffineForOp forOp) {
auto *childForOp = forOp.getOperation();
auto *parentForOp = forOp->getParentOp();
/// Currently, the total cost is computed by counting the total operation
/// instance count (i.e. total number of operations in the loop body * loop
/// trip count) for the entire loop nest.
-int64_t mlir::getComputeCost(AffineForOp forOp, LoopNestStats &stats) {
+int64_t mlir::affine::getComputeCost(AffineForOp forOp, LoopNestStats &stats) {
return getComputeCostHelper(forOp, stats,
/*tripCountOverrideMap=*/nullptr,
/*computeCostMap=*/nullptr);
/// the total cost is computed by counting the total operation instance count
/// (i.e. total number of operations in the loop body * loop trip count) for
/// the entire loop nest.
-bool mlir::getFusionComputeCost(AffineForOp srcForOp, LoopNestStats &srcStats,
- AffineForOp dstForOp, LoopNestStats &dstStats,
- const ComputationSliceState &slice,
- int64_t *computeCost) {
+bool mlir::affine::getFusionComputeCost(AffineForOp srcForOp,
+ LoopNestStats &srcStats,
+ AffineForOp dstForOp,
+ LoopNestStats &dstStats,
+ const ComputationSliceState &slice,
+ int64_t *computeCost) {
llvm::SmallDenseMap<Operation *, uint64_t, 8> sliceTripCountMap;
DenseMap<Operation *, int64_t> computeCostMap;
/// Returns in 'producerConsumerMemrefs' the memrefs involved in a
/// producer-consumer dependence between write ops in 'srcOps' and read ops in
/// 'dstOps'.
-void mlir::gatherProducerConsumerMemrefs(
+void mlir::affine::gatherProducerConsumerMemrefs(
ArrayRef<Operation *> srcOps, ArrayRef<Operation *> dstOps,
DenseSet<Value> &producerConsumerMemrefs) {
// Gather memrefs from stores in 'srcOps'.
#define DEBUG_TYPE "loop-utils"
using namespace mlir;
+using namespace affine;
using namespace presburger;
using llvm::SmallMapVector;
/// Promotes the loop body of a forOp to its containing block if the forOp
/// was known to have a single iteration.
// TODO: extend this for arbitrary affine bounds.
-LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) {
+LogicalResult mlir::affine::promoteIfSingleIteration(AffineForOp forOp) {
std::optional<uint64_t> tripCount = getConstantTripCount(forOp);
if (!tripCount || *tripCount != 1)
return failure();
// asserts preservation of SSA dominance. A check for that as well as that for
// memory-based dependence preservation check rests with the users of this
// method.
-LogicalResult mlir::affineForOpBodySkew(AffineForOp forOp,
- ArrayRef<uint64_t> shifts,
- bool unrollPrologueEpilogue) {
+LogicalResult mlir::affine::affineForOpBodySkew(AffineForOp forOp,
+ ArrayRef<uint64_t> shifts,
+ bool unrollPrologueEpilogue) {
assert(forOp.getBody()->getOperations().size() == shifts.size() &&
"too few/many shifts");
if (forOp.getBody()->begin() == std::prev(forOp.getBody()->end()))
}
/// Checks whether a loop nest is hyper-rectangular or not.
-LogicalResult checkIfHyperRectangular(MutableArrayRef<AffineForOp> input) {
+static LogicalResult
+checkIfHyperRectangular(MutableArrayRef<AffineForOp> input) {
FlatAffineValueConstraints cst;
SmallVector<Operation *, 8> ops(input.begin(), input.end());
// 0-d or 1-d is trivially hyper-rectangular.
/// Check if the input nest is supported for tiling and whether tiling would be
/// legal or not.
template <typename t>
-LogicalResult performPreTilingChecks(MutableArrayRef<AffineForOp> input,
- ArrayRef<t> tileSizes) {
+
static LogicalResult performPreTilingChecks(MutableArrayRef<AffineForOp> input,
+ ArrayRef<t> tileSizes) {
assert(input.size() == tileSizes.size() && "Too few/many tile sizes");
if (llvm::any_of(input,
/// Move the loop body of AffineForOp 'src' from 'src' to the start of dest
/// body.
-void moveLoopBody(AffineForOp src, AffineForOp dest) {
+static void moveLoopBody(AffineForOp src, AffineForOp dest) {
moveLoopBodyImpl(src, dest, dest.getBody()->begin());
}
/// Constructs tiled loop nest, without setting the loop bounds and move the
/// body of the original loop nest to the tiled loop nest.
-void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
- AffineForOp rootAffineForOp, unsigned width,
- MutableArrayRef<AffineForOp> tiledLoops) {
+
static void constructTiledLoopNest(MutableArrayRef<AffineForOp> origLoops,
+ AffineForOp rootAffineForOp, unsigned width,
+
MutableArrayRef<AffineForOp> tiledLoops) {
Location loc = rootAffineForOp.getLoc();
// The outermost among the loops as we add more..
}
LogicalResult
-mlir::tilePerfectlyNested(MutableArrayRef<AffineForOp> input,
- ArrayRef<unsigned> tileSizes,
- SmallVectorImpl<AffineForOp> *tiledNest) {
+mlir::
affine::tilePerfectlyNested(MutableArrayRef<AffineForOp> input,
+ ArrayRef<unsigned> tileSizes,
+
SmallVectorImpl<AffineForOp> *tiledNest) {
if (input.empty())
return success();
/// loops and intra-tile loops, using SSA values as tiling parameters. A band
/// is a contiguous set of loops.
// TODO: handle non hyper-rectangular spaces.
-LogicalResult
-mlir::tilePerfectlyNestedParametric(MutableArrayRef<AffineForOp> input,
- ArrayRef<Value> tileSizes,
- SmallVectorImpl<AffineForOp> *tiledNest) {
+LogicalResult mlir::affine::tilePerfectlyNestedParametric(
+ MutableArrayRef<AffineForOp> input, ArrayRef<Value> tileSizes,
+ SmallVectorImpl<AffineForOp> *tiledNest) {
if (input.empty())
return success();
/// (the first op being another AffineFor, and the second op - a terminator).
/// A loop is perfectly nested iff: the first op in the loop's body is another
/// AffineForOp, and the second op is a terminator).
-void mlir::getPerfectlyNestedLoops(SmallVectorImpl<AffineForOp> &nestedLoops,
- AffineForOp root) {
+void mlir::affine::getPerfectlyNestedLoops(
+ SmallVectorImpl<AffineForOp> &nestedLoops, AffineForOp root) {
for (unsigned i = 0; i < std::numeric_limits<unsigned>::max(); ++i) {
nestedLoops.push_back(root);
Block &body = root.getRegion().front();
/// Identify valid and profitable bands of loops to tile. This is currently just
/// a temporary placeholder to test the mechanics of tiled code generation.
/// Returns all maximal outermost perfect loop nests to tile.
-void mlir::getTileableBands(func::FuncOp f,
-
std::vector<SmallVector<AffineForOp, 6>> *bands) {
+void mlir::affine::getTileableBands(
+
func::FuncOp f, std::vector<SmallVector<AffineForOp, 6>> *bands) {
// Get maximal perfect nest of 'affine.for' insts starting from root
// (inclusive).
for (AffineForOp forOp : f.getOps<AffineForOp>()) {
}
/// Unrolls this loop completely.
-LogicalResult mlir::loopUnrollFull(AffineForOp forOp) {
+LogicalResult mlir::affine::loopUnrollFull(AffineForOp forOp) {
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
if (mayBeConstantTripCount.has_value()) {
uint64_t tripCount = *mayBeConstantTripCount;
/// Unrolls this loop by the specified factor or by the trip count (if constant)
/// whichever is lower.
-LogicalResult mlir::loopUnrollUpToFactor(AffineForOp forOp,
- uint64_t unrollFactor) {
+LogicalResult mlir::affine::loopUnrollUpToFactor(AffineForOp forOp,
+ uint64_t unrollFactor) {
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
if (mayBeConstantTripCount.has_value() &&
*mayBeConstantTripCount < unrollFactor)
/// Unrolls this loop by the specified factor. Returns success if the loop
/// is successfully unrolled.
-LogicalResult mlir::loopUnrollByFactor(
+LogicalResult mlir::affine::loopUnrollByFactor(
AffineForOp forOp, uint64_t unrollFactor,
function_ref<void(unsigned, Operation *, OpBuilder)> annotateFn,
bool cleanUpUnroll) {
return success();
}
-LogicalResult mlir::loopUnrollJamUpToFactor(AffineForOp forOp,
- uint64_t unrollJamFactor) {
+LogicalResult mlir::affine::loopUnrollJamUpToFactor(AffineForOp forOp,
+ uint64_t unrollJamFactor) {
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
if (mayBeConstantTripCount.has_value() &&
*mayBeConstantTripCount < unrollJamFactor)
};
/// Unrolls and jams this loop by the specified factor.
-LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
- uint64_t unrollJamFactor) {
+LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
+ uint64_t unrollJamFactor) {
assert(unrollJamFactor > 0 && "unroll jam factor should be positive");
std::optional<uint64_t> mayBeConstantTripCount = getConstantTripCount(forOp);
}
// Create a new loop with additional iterOperands, iter_args and yield
// operands. This new loop will take the loop body of the original loop.
- AffineForOp newForOp = mlir::replaceForOpWithNewYields(
+ AffineForOp newForOp = affine::replaceForOpWithNewYields(
builder, oldForOp, dupIterOperands, dupYieldOperands, dupIterArgs);
newLoopsWithIterArgs.push_back(newForOp);
// `forOp` has been replaced with a new loop.
/// Performs loop interchange on 'forOpA' and 'forOpB', where 'forOpB' is
/// nested within 'forOpA' as the only non-terminator operation in its block.
-void mlir::interchangeLoops(AffineForOp forOpA, AffineForOp forOpB) {
+void mlir::affine::interchangeLoops(AffineForOp forOpA, AffineForOp forOpB) {
assert(&*forOpA.getBody()->begin() == forOpB.getOperation());
auto &forOpABody = forOpA.getBody()->getOperations();
auto &forOpBBody = forOpB.getBody()->getOperations();
/// Checks if the loop interchange permutation 'loopPermMap' of the perfectly
/// nested sequence of loops in 'loops' would violate dependences.
-bool mlir::isValidLoopInterchangePermutation(ArrayRef<AffineForOp> loops,
- ArrayRef<unsigned> loopPermMap) {
+bool mlir::affine::isValidLoopInterchangePermutation(
+ ArrayRef<AffineForOp> loops, ArrayRef<unsigned> loopPermMap) {
// Gather dependence components for dependences between all ops in loop nest
// rooted at 'loops[0]', at loop depths in range [1, maxLoopDepth].
assert(loopPermMap.size() == loops.size());
/// Returns true if `loops` is a perfectly nested loop nest, where loops appear
/// in it from outermost to innermost.
bool LLVM_ATTRIBUTE_UNUSED
-mlir::isPerfectlyNested(ArrayRef<AffineForOp> loops) {
+mlir::
affine::isPerfectlyNested(ArrayRef<AffineForOp> loops) {
assert(!loops.empty() && "no loops provided");
// We already know that the block can't be empty.
// input[i] should move from position i -> permMap[i]. Returns the position in
// `input` that becomes the new outermost loop.
-unsigned mlir::permuteLoops(MutableArrayRef<AffineForOp> input,
- ArrayRef<unsigned> permMap) {
+unsigned mlir::
affine::permuteLoops(MutableArrayRef<AffineForOp> input,
+ ArrayRef<unsigned> permMap) {
assert(input.size() == permMap.size() && "invalid permutation map size");
// Check whether the permutation spec is valid. This is a small vector - we'll
// just sort and check if it's iota.
// Sinks all sequential loops to the innermost levels (while preserving
// relative order among them) and moves all parallel loops to the
// outermost (while again preserving relative order among them).
-AffineForOp mlir::sinkSequentialLoops(AffineForOp forOp) {
+AffineForOp mlir::affine::sinkSequentialLoops(AffineForOp forOp) {
SmallVector<AffineForOp, 4> loops;
getPerfectlyNestedLoops(loops, forOp);
if (loops.size() < 2)
}
SmallVector<SmallVector<AffineForOp, 8>, 8>
-mlir::tile(ArrayRef<AffineForOp> forOps, ArrayRef<uint64_t> sizes,
- ArrayRef<AffineForOp> targets) {
+mlir::
affine::tile(ArrayRef<AffineForOp> forOps, ArrayRef<uint64_t> sizes,
+
ArrayRef<AffineForOp> targets) {
SmallVector<SmallVector<AffineForOp, 8>, 8> res;
SmallVector<AffineForOp, 8> currentTargets(targets.begin(), targets.end());
for (auto it : llvm::zip(forOps, sizes)) {
return res;
}
-SmallVector<AffineForOp, 8> mlir::tile(ArrayRef<AffineForOp> forOps,
- ArrayRef<uint64_t> sizes,
- AffineForOp target) {
+SmallVector<AffineForOp, 8> mlir::
affine::tile(ArrayRef<AffineForOp> forOps,
+ ArrayRef<uint64_t> sizes,
+ AffineForOp target) {
SmallVector<AffineForOp, 8> res;
for (auto loops : tile(forOps, sizes, ArrayRef<AffineForOp>(target))) {
assert(loops.size() == 1);
return res;
}
-LogicalResult mlir::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
+LogicalResult mlir::
affine::coalesceLoops(MutableArrayRef<AffineForOp> loops) {
if (loops.size() < 2)
return success();
return success();
}
-void mlir::mapLoopToProcessorIds(scf::ForOp forOp, ArrayRef<Value> processorId,
- ArrayRef<Value> numProcessors) {
+void mlir::affine::mapLoopToProcessorIds(scf::ForOp forOp,
+ ArrayRef<Value> processorId,
+ ArrayRef<Value> numProcessors) {
assert(processorId.size() == numProcessors.size());
if (processorId.empty())
return;
return true;
}
-LogicalResult mlir::affineDataCopyGenerate(Block::iterator begin,
- Block::iterator end,
- const AffineCopyOptions ©Options,
- std::optional<Value> filterMemRef,
- DenseSet<Operation *> ©Nests) {
+LogicalResult
+mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
+ const AffineCopyOptions ©Options,
+ std::optional<Value> filterMemRef,
+ DenseSet<Operation *> ©Nests) {
if (begin == end)
return success();
// A convenience version of affineDataCopyGenerate for all ops in the body of
// an AffineForOp.
-LogicalResult mlir::affineDataCopyGenerate(AffineForOp forOp,
- const AffineCopyOptions ©Options,
- std::optional<Value> filterMemRef,
- DenseSet<Operation *> ©Nests) {
+LogicalResult mlir::affine::affineDataCopyGenerate(
+ AffineForOp forOp, const AffineCopyOptions ©Options,
+ std::optional<Value> filterMemRef, DenseSet<Operation *> ©Nests) {
return affineDataCopyGenerate(forOp.getBody()->begin(),
std::prev(forOp.getBody()->end()), copyOptions,
filterMemRef, copyNests);
}
-LogicalResult mlir::generateCopyForMemRegion(
+LogicalResult mlir::affine::generateCopyForMemRegion(
const MemRefRegion &memrefRegion, Operation *analyzedOp,
const AffineCopyOptions ©Options, CopyGenerateResult &result) {
Block *block = analyzedOp->getBlock();
}
/// Gathers all AffineForOps in 'func.func' grouped by loop depth.
-void mlir::gatherLoops(func::FuncOp func,
-
std::vector<SmallVector<AffineForOp, 2>> &depthToLoops) {
+void mlir::affine::gatherLoops(
+
func::FuncOp func, std::vector<SmallVector<AffineForOp, 2>> &depthToLoops) {
for (auto &block : func)
gatherLoopsInBlock(&block, /*currLoopDepth=*/0, depthToLoops);
// affine.applys, fold to constant if all result dimensions of the map are
// constant (canonicalizeMapAndOperands below already does this for single
// result bound maps), and use simplifyMap to perform algebraic simplification.
-AffineForOp mlir::createCanonicalizedAffineForOp(
+AffineForOp mlir::affine::createCanonicalizedAffineForOp(
OpBuilder b, Location loc, ValueRange lbOperands, AffineMap lbMap,
ValueRange ubOperands, AffineMap ubMap, int64_t step) {
SmallVector<Value, 4> lowerOperands(lbOperands);
}
LogicalResult
-mlir::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
- SmallVectorImpl<AffineForOp> *fullTileNest) {
+mlir::
affine::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
+
SmallVectorImpl<AffineForOp> *fullTileNest) {
if (inputNest.empty())
return success();
#define DEBUG_TYPE "affine-utils"
using namespace mlir;
+using namespace affine;
using namespace presburger;
namespace {
/// Create a sequence of operations that implement the `expr` applied to the
/// given dimension and symbol values.
-mlir::Value mlir::expandAffineExpr(OpBuilder &builder, Location loc,
- AffineExpr expr, ValueRange dimValues,
- ValueRange symbolValues) {
+mlir::Value mlir::affine::expandAffineExpr(OpBuilder &builder, Location loc,
+ AffineExpr expr,
+ ValueRange dimValues,
+ ValueRange symbolValues) {
return AffineApplyExpander(builder, dimValues, symbolValues, loc).visit(expr);
}
/// Create a sequence of operations that implement the `affineMap` applied to
/// the given `operands` (as it it were an AffineApplyOp).
std::optional<SmallVector<Value, 8>>
-mlir::expandAffineMap(OpBuilder &builder, Location loc, AffineMap affineMap,
- ValueRange operands) {
+mlir::affine::expandAffineMap(OpBuilder &builder, Location loc,
+ AffineMap affineMap, ValueRange operands) {
auto numDims = affineMap.getNumDims();
auto expanded = llvm::to_vector<8>(
llvm::map_range(affineMap.getResults(),
}
LogicalResult
-mlir::affineParallelize(AffineForOp forOp,
- ArrayRef<LoopReduction> parallelReductions) {
+mlir::affine::affineParallelize(AffineForOp forOp,
+ ArrayRef<LoopReduction> parallelReductions) {
// Fail early if there are iter arguments that are not reductions.
unsigned numReductions = parallelReductions.size();
if (numReductions != forOp.getNumIterOperands())
}
// Returns success if any hoisting happened.
-LogicalResult mlir::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
+LogicalResult mlir::affine::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
// Bail out early if the ifOp returns a result. TODO: Consider how to
// properly support this case.
if (ifOp.getNumResults() != 0)
}
// Return the min expr after replacing the given dim.
-AffineExpr mlir::substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
- AffineExpr max, bool positivePath) {
+AffineExpr mlir::affine::substWithMin(AffineExpr e, AffineExpr dim,
+ AffineExpr min, AffineExpr max,
+ bool positivePath) {
if (e == dim)
return positivePath ? min : max;
if (auto bin = e.dyn_cast<AffineBinaryOpExpr>()) {
return e;
}
-void mlir::normalizeAffineParallel(AffineParallelOp op) {
+void mlir::affine::normalizeAffineParallel(AffineParallelOp op) {
// Loops with min/max in bounds are not normalized at the moment.
if (op.hasMinMaxBounds())
return;
ubExprs, op.getContext());
op.setUpperBounds(ranges.getOperands(), newUpperMap);
}
-LogicalResult mlir::normalizeAffineFor(AffineForOp op, bool promoteSingleIter) {
+LogicalResult mlir::affine::normalizeAffineFor(AffineForOp op,
+ bool promoteSingleIter) {
if (promoteSingleIter && succeeded(promoteIfSingleIteration(op)))
return success();
}
template <typename EffectType, typename T>
-bool mlir::hasNoInterveningEffect(Operation *start, T memOp) {
+bool mlir::affine::hasNoInterveningEffect(Operation *start, T memOp) {
auto isLocallyAllocated = [](Value memref) {
auto *defOp = memref.getDefiningOp();
return defOp && hasSingleEffect<MemoryEffects::Allocate>(defOp, memref);
// 4. Ensure there is no intermediate operation which could replace the
// value in memory.
- if (!mlir::hasNoInterveningEffect<MemoryEffects::Write>(storeOp, loadOp))
+ if (!affine::hasNoInterveningEffect<MemoryEffects::Write>(storeOp, loadOp))
continue;
// We now have a candidate for forwarding.
return success();
}
-template bool mlir::hasNoInterveningEffect<mlir::MemoryEffects::Read,
- mlir::AffineReadOpInterface>(
- mlir::Operation *, mlir::AffineReadOpInterface);
+template bool
+mlir::affine::hasNoInterveningEffect<mlir::MemoryEffects::Read,
+ affine::AffineReadOpInterface>(
+ mlir::Operation *, affine::AffineReadOpInterface);
// This attempts to find stores which have no impact on the final result.
// A writing op writeA will be eliminated if there exists an op writeB if
// There cannot be an operation which reads from memory between
// the two writes.
- if (!mlir::hasNoInterveningEffect<MemoryEffects::Read>(writeA, writeB))
+ if (!affine::hasNoInterveningEffect<MemoryEffects::Read>(writeA, writeB))
continue;
opsToErase.push_back(writeA);
continue;
// 3. There is no write between loadA and loadB.
- if (!mlir::hasNoInterveningEffect<MemoryEffects::Write>(
+ if (!affine::hasNoInterveningEffect<MemoryEffects::Write>(
loadB.getOperation(), loadA))
continue;
// currently only eliminates the stores only if no other loads/uses (other
// than dealloc) remain.
//
-void mlir::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
- PostDominanceInfo &postDomInfo) {
+void mlir::affine::affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo,
+ PostDominanceInfo &postDomInfo) {
// Load op's whose results were replaced by those forwarded from stores.
SmallVector<Operation *, 8> opsToErase;
}
// Perform the replacement in `op`.
-LogicalResult mlir::replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef,
- Operation *op,
- ArrayRef<Value> extraIndices,
- AffineMap indexRemap,
- ArrayRef<Value> extraOperands,
- ArrayRef<Value> symbolOperands,
- bool allowNonDereferencingOps) {
+LogicalResult mlir::affine::replaceAllMemRefUsesWith(
+ Value oldMemRef, Value newMemRef, Operation *op,
+ ArrayRef<Value> extraIndices, AffineMap indexRemap,
+ ArrayRef<Value> extraOperands, ArrayRef<Value> symbolOperands,
+ bool allowNonDereferencingOps) {
unsigned newMemRefRank = newMemRef.getType().cast<MemRefType>().getRank();
(void)newMemRefRank; // unused in opt mode
unsigned oldMemRefRank = oldMemRef.getType().cast<MemRefType>().getRank();
return success();
}
-LogicalResult mlir::replaceAllMemRefUsesWith(
+LogicalResult mlir::affine::replaceAllMemRefUsesWith(
Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices,
AffineMap indexRemap, ArrayRef<Value> extraOperands,
ArrayRef<Value> symbolOperands, Operation *domOpFilter,
/// all the affine.apply op's supplying operands to this opInst did not have any
/// uses besides this opInst; otherwise returns the list of affine.apply
/// operations created in output argument `sliceOps`.
-void mlir::createAffineComputationSlice(
+void mlir::affine::createAffineComputationSlice(
Operation *opInst, SmallVectorImpl<AffineApplyOp> *sliceOps) {
// Collect all operands that are results of affine apply ops.
SmallVector<Value, 4> subOperands;
}
// TODO: Currently works for static memrefs with a single layout map.
-LogicalResult mlir::normalizeMemRef(memref::AllocOp *allocOp) {
+LogicalResult mlir::affine::normalizeMemRef(memref::AllocOp *allocOp) {
MemRefType memrefType = allocOp->getType();
OpBuilder b(*allocOp);
return success();
}
-MemRefType mlir::normalizeMemRefType(MemRefType memrefType,
- unsigned numSymbolicOperands) {
+MemRefType mlir::affine::normalizeMemRefType(MemRefType memrefType,
+ unsigned numSymbolicOperands) {
unsigned rank = memrefType.getRank();
if (rank == 0)
return memrefType;
return newMemRefType;
}
-DivModValue mlir::getDivMod(OpBuilder &b, Location loc, Value lhs, Value rhs) {
+DivModValue mlir::affine::getDivMod(OpBuilder &b, Location loc, Value lhs,
+ Value rhs) {
DivModValue result;
AffineExpr d0, d1;
bindDims(b.getContext(), d0, d1);
result.quotient =
- makeComposedAffineApply(b, loc, d0.floorDiv(d1), {lhs, rhs});
- result.remainder = makeComposedAffineApply(b, loc, d0 % d1, {lhs, rhs});
+ affine::makeComposedAffineApply(b, loc, d0.floorDiv(d1), {lhs, rhs});
+ result.remainder =
+ affine::makeComposedAffineApply(b, loc, d0 % d1, {lhs, rhs});
return result;
}
return result;
}
-FailureOr<SmallVector<Value>> mlir::delinearizeIndex(OpBuilder &b, Location loc,
- Value linearIndex,
- ArrayRef<Value> basis) {
+FailureOr<SmallVector<Value>>
+mlir::affine::delinearizeIndex(OpBuilder &b, Location loc, Value linearIndex,
+ ArrayRef<Value> basis) {
unsigned numDims = basis.size();
SmallVector<Value> divisors;
#include "mlir/IR/PatternMatch.h"
using namespace mlir;
+using namespace affine;
-LogicalResult mlir::mergeOffsetsSizesAndStrides(
+LogicalResult mlir::affine::mergeOffsetsSizesAndStrides(
OpBuilder &builder, Location loc, ArrayRef<OpFoldResult> producerOffsets,
ArrayRef<OpFoldResult> producerSizes,
ArrayRef<OpFoldResult> producerStrides,
return success();
}
-LogicalResult mlir::mergeOffsetsSizesAndStrides(
+LogicalResult mlir::affine::mergeOffsetsSizesAndStrides(
OpBuilder &builder, Location loc, OffsetSizeAndStrideOpInterface producer,
OffsetSizeAndStrideOpInterface consumer,
const llvm::SmallBitVector &droppedProducerDims,
combinedOffsets, combinedSizes, combinedStrides);
}
-void mlir::resolveIndicesIntoOpWithOffsetsAndStrides(
+void mlir::affine::resolveIndicesIntoOpWithOffsetsAndStrides(
RewriterBase &rewriter, Location loc,
ArrayRef<OpFoldResult> mixedSourceOffsets,
ArrayRef<OpFoldResult> mixedSourceStrides,
}
}
-void mlir::resolveSizesIntoOpWithSizes(
+void mlir::affine::resolveSizesIntoOpWithSizes(
ArrayRef<OpFoldResult> sourceSizes, ArrayRef<OpFoldResult> destSizes,
const llvm::SmallBitVector &rankReducedSourceDims,
SmallVectorImpl<OpFoldResult> &resolvedSizes) {
rewriter.create<ThreadIdOp>(loc, indexType, Dimension::z).getResult()};
threadsAndWorkGroups.push_back(blockDimsOfr[0]);
threadsAndWorkGroups.push_back(blockDimsOfr[1]);
- OpFoldResult ofr = makeComposedFoldedAffineApply(
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
rewriter, loc, tx + ty * BDX + tz * BDX * BDY, threadsAndWorkGroups);
return getValueOrCreateConstantIndexOp(rewriter, loc, ofr);
}
// `forallMappingSizes`.
Value linearId = buildLinearThreadId(rewriter, loc, this->blockDimsOfr);
AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
- OpFoldResult warpIdOfr = makeComposedFoldedAffineApply(
+ OpFoldResult warpIdOfr = affine::makeComposedFoldedAffineApply(
rewriter, loc, d0.floorDiv(kWarpSize), {linearId});
Value warpId = getValueOrCreateConstantIndexOp(rewriter, loc, warpIdOfr);
// Sizes in [x, y, z] -> [z, y x] order to properly compute strides in
SmallVector<Value> ids;
// Reverse back to be in [x, y, z] order.
for (AffineExpr e : llvm::reverse(delinearizingExprs))
- ids.push_back(makeComposedAffineApply(rewriter, loc, e, warpId));
+ ids.push_back(
+ affine::makeComposedAffineApply(rewriter, loc, e, warpId));
// clang-format off
LDBG("----linearId: " << linearId);
SmallVector<Value> ids;
// Reverse back to be in [x, y, z] order.
for (AffineExpr e : llvm::reverse(delinearizingExprs))
- ids.push_back(makeComposedAffineApply(rewriter, loc, e, linearId));
+ ids.push_back(
+ affine::makeComposedAffineApply(rewriter, loc, e, linearId));
// clang-format off
LLVM_DEBUG(llvm::interleaveComma(reverseBasisSizes,
GPUDialect::getNumWorkgroupDimensions())))) {
Value v = en.value();
auto loop = cast<scf::ForOp>(v.getParentRegion()->getParentOp());
- mapLoopToProcessorIds(loop, {threadIds[en.index()]},
- {blockDims[en.index()]});
+ affine::mapLoopToProcessorIds(loop, {threadIds[en.index()]},
+ {blockDims[en.index()]});
}
}
Location loc = getOperation()->getLoc();
IRRewriter rewriter(b);
SmallVector<OpFoldResult> allResultDimValues =
- makeComposedFoldedMultiResultAffineApply(
+ affine::makeComposedFoldedMultiResultAffineApply(
rewriter, loc, resultShapesFromInputShapesMap,
createFlatListOfOperandDims(b, loc));
int64_t pos = 0;
// plus low padding sizes.
SmallVector<OpFoldResult, 4> newOffsets;
for (const auto &p : llvm::zip(lowPads, oldOffsets)) {
- newOffsets.push_back(makeComposedFoldedAffineApply(
+ newOffsets.push_back(affine::makeComposedFoldedAffineApply(
rewriter, loc, addMap, {std::get<0>(p), std::get<1>(p)}));
}
AffineExpr s0 = builder.getAffineSymbolExpr(0);
AffineExpr s1 = builder.getAffineSymbolExpr(1);
Operation *splitPoint =
- makeComposedAffineApply(builder, target.getLoc(), s0 * s1,
- {spec->lowTileSize, spec->lowTripCount});
+ affine::makeComposedAffineApply(builder, target.getLoc(), s0 * s1,
+ {spec->lowTileSize, spec->lowTripCount});
Operation *lowTileSize = spec->lowTileSize.getDefiningOp();
Operation *highTileSize = spec->highTileSize.getDefiningOp();
assert(lowTileSize && highTileSize && splitPoint &&
AffineExpr d0, s0;
bindDims(rewriter.getContext(), d0);
bindSymbols(rewriter.getContext(), s0);
- adjustedPackedSizes.push_back(makeComposedFoldedAffineApply(
+ adjustedPackedSizes.push_back(affine::makeComposedFoldedAffineApply(
rewriter, genericOp->getLoc(), d0.ceilDiv(s0) * s0,
{loopRanges[adjustedPackedSizes.size()].size,
rewriter.getIndexAttr(paddedSizesNextMultipleOf[i])}));
TrackingListener listener(state, *this);
IRRewriter rewriter(getContext(), &listener);
SmallVector<OpFoldResult> shapeSizes =
- makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
- allShapeSizes);
+ affine::makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
+ allShapeSizes);
// If the shape size is dynamic, tile by 1.
// Otherwise, do not tile (i.e. tile size 0).
for (OpFoldResult shapeSize : shapeSizes) {
void init() {
declareDependentDialect<pdl::PDLDialect>();
declareDependentDialect<LinalgDialect>();
- declareGeneratedDialect<AffineDialect>();
+ declareGeneratedDialect<affine::AffineDialect>();
declareGeneratedDialect<arith::ArithDialect>();
declareGeneratedDialect<scf::SCFDialect>();
declareGeneratedDialect<vector::VectorDialect>();
linalgOp, "failed to get loops map from shape sizes");
}
SmallVector<OpFoldResult> sizeBounds =
- makeComposedFoldedMultiResultAffineApply(
+ affine::makeComposedFoldedMultiResultAffineApply(
rewriter, linalgLoc, shapeSizesToLoopsMap, allShapeSizes);
// The offsets and sizes from the slice operation only give you the tile
for (int64_t f : factors)
basis.push_back(b.create<arith::ConstantOp>(loc, b.getIndexAttr(f)));
FailureOr<SmallVector<Value>> multiIndex =
- delinearizeIndex(b, loc, index, basis);
+ affine::delinearizeIndex(b, loc, index, basis);
assert(!failed(multiIndex) && "Failed to linearize img2col index");
return *multiIndex;
}
AffineExpr oExpr, fExpr;
bindSymbols(b.getContext(), oExpr, fExpr);
AffineMap convMap = AffineMap::get(0, 2, stride * oExpr + fExpr);
- return makeComposedAffineApply(b, loc, convMap, ValueRange{oIndex, fIndex});
+ return affine::makeComposedAffineApply(b, loc, convMap,
+ ValueRange{oIndex, fIndex});
}
FailureOr<std::pair<Operation *, Operation *>>
cast<LinalgOp>(op.getOperation()).createFlatListOfOperandDims(b, loc);
AffineMap map = op.getShapesToLoopsMap();
IRRewriter rewriter(b);
- return makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
- allShapesSizes);
+ return affine::makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
+ allShapesSizes);
}
/// Helper method to permute the list of `values` based on the `map`.
});
for (IndexOp indexOp :
llvm::make_early_inc_range(producerBlock.getOps<IndexOp>())) {
- Value newIndex = rewriter.create<mlir::AffineApplyOp>(
+ Value newIndex = rewriter.create<affine::AffineApplyOp>(
producer.getLoc(),
consumerToProducerLoopsMap.getSubMap(indexOp.getDim()), fusedIndices);
mapper.map(indexOp.getResult(), newIndex);
assert(!ShapedType::isDynamic(std::get<0>(it)));
AffineExpr idx, acc;
bindDims(rewriter.getContext(), idx, acc);
- newIndex = rewriter.create<AffineApplyOp>(
+ newIndex = rewriter.create<affine::AffineApplyOp>(
indexOp.getLoc(), idx + acc * std::get<0>(it),
ValueRange{std::get<1>(it), newIndex});
}
populateFoldReshapeOpsByExpansionPatterns(patterns, defaultControlFn);
// General canonicalization patterns.
- AffineApplyOp::getCanonicalizationPatterns(patterns, context);
+ affine::AffineApplyOp::getCanonicalizationPatterns(patterns, context);
GenericOp::getCanonicalizationPatterns(patterns, context);
tensor::ExpandShapeOp::getCanonicalizationPatterns(patterns, context);
tensor::CollapseShapeOp::getCanonicalizationPatterns(patterns, context);
// of the enclosing loops.
for (auto forOp : packingLoops) {
// Compute an upper bound `ubVal` for the upper bound of `forOp`.
- FailureOr<OpFoldResult> loopUb = reifyIndexValueBound(
+ FailureOr<OpFoldResult> loopUb = affine::reifyIndexValueBound(
rewriter, loc, presburger::BoundType::UB, forOp.getUpperBound(),
/*stopCondition=*/
[&](Value v, std::optional<int64_t> d) {
Operation *op = v.getDefiningOp();
if (!op)
return true;
- return !isa<AffineMinOp, AffineMaxOp, AffineApplyOp>(op);
+ return !isa<affine::AffineMinOp, affine::AffineMaxOp,
+ affine::AffineApplyOp>(op);
},
/*closedUB=*/true);
assert(succeeded(loopUb) && "could not get upper bound");
AffineExpr lb, ub, step;
bindDims(rewriter.getContext(), lb, ub);
bindSymbols(rewriter.getContext(), step);
- Value res = rewriter.createOrFold<AffineApplyOp>(
+ Value res = rewriter.createOrFold<affine::AffineApplyOp>(
loc, (ub - lb).ceilDiv(step),
ValueRange{forOp.getLowerBound(), ubVal,
cast<scf::ForOp>(forOp).getStep()});
Value ivVal = forOp.getInductionVar(), lbVal = forOp.getLowerBound(),
stepVal = forOp.getStep();
auto loc = forOp->getLoc();
- return rewriter.createOrFold<AffineApplyOp>(
+ return rewriter.createOrFold<affine::AffineApplyOp>(
loc, (iv - lb).ceilDiv(step), ValueRange{ivVal, lbVal, stepVal});
}
auto loop = dyn_cast<LoopLikeOpInterface>(transferRead->getParentOp());
LLVM_DEBUG(DBGS() << "Parent op: " << *transferRead->getParentOp()
<< "\n");
- if (!isa_and_nonnull<scf::ForOp, AffineForOp>(loop))
+ if (!isa_and_nonnull<scf::ForOp, affine::AffineForOp>(loop))
return WalkResult::advance();
LLVM_DEBUG(DBGS() << "Candidate read: " << *transferRead.getOperation()
// the walk.
return WalkResult::interrupt();
})
- .Case<AffineForOp>([&](AffineForOp affineForOp) {
+ .Case<affine::AffineForOp>([&](affine::AffineForOp affineForOp) {
auto newForOp = replaceForOpWithNewYields(
b, affineForOp, transferRead.getVector(),
SmallVector<Value>{transferWrite.getVector()},
std::back_inserter(allIndices), [&](uint64_t dim) {
return rewriter.create<IndexOp>(indexOp->getLoc(), dim);
});
- rewriter.replaceOpWithNewOp<AffineApplyOp>(
+ rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
indexOp, permutationMap.getSubMap(indexOp.getDim()), allIndices);
}
}
for (auto e : map.getResults()) {
auto exprMap = AffineMap::get(dims, map.getNumSymbols(), e);
SmallVector<Value> operands(vals.begin(), vals.end());
- canonicalizeMapAndOperands(&exprMap, &operands);
- res.push_back(b.create<AffineApplyOp>(loc, exprMap, operands));
+ affine::canonicalizeMapAndOperands(&exprMap, &operands);
+ res.push_back(b.create<affine::AffineApplyOp>(loc, exprMap, operands));
}
return res;
}
.Case([&](scf::ForOp forOp) {
allIvs.push_back(forOp.getInductionVar());
})
- .Case([&](AffineForOp affineForOp) {
+ .Case([&](affine::AffineForOp affineForOp) {
allIvs.push_back(affineForOp.getInductionVar());
})
.Default([&](Operation *op) { assert(false && "unexpected op"); });
template <typename LoopTy>
static FailureOr<LinalgLoops> linalgOpToLoopsImpl(RewriterBase &rewriter,
LinalgOp linalgOp) {
- using LoadOpTy = std::conditional_t<std::is_same<LoopTy, AffineForOp>::value,
- AffineLoadOp, memref::LoadOp>;
- using StoreOpTy = std::conditional_t<std::is_same<LoopTy, AffineForOp>::value,
- AffineStoreOp, memref::StoreOp>;
+ using LoadOpTy =
+ std::conditional_t<std::is_same<LoopTy, affine::AffineForOp>::value,
+ affine::AffineLoadOp, memref::LoadOp>;
+ using StoreOpTy =
+ std::conditional_t<std::is_same<LoopTy, affine::AffineForOp>::value,
+ affine::AffineStoreOp, memref::StoreOp>;
// The flattened loopToOperandRangesMaps is expected to be an invertible
// permutation map (which is asserted in the inverse calculation).
/// other cases, it is replaced by its unique operand.
struct FoldAffineOp : public RewritePattern {
FoldAffineOp(MLIRContext *context)
- : RewritePattern(AffineApplyOp::getOperationName(), 0, context) {}
+ : RewritePattern(affine::AffineApplyOp::getOperationName(), 0, context) {}
LogicalResult matchAndRewrite(Operation *op,
PatternRewriter &rewriter) const override {
- AffineApplyOp affineApplyOp = cast<AffineApplyOp>(op);
+ auto affineApplyOp = cast<affine::AffineApplyOp>(op);
auto map = affineApplyOp.getAffineMap();
if (map.getNumResults() != 1 || map.getNumInputs() > 1)
return failure();
patterns.add<LinalgRewritePattern<LoopType>>(context);
memref::DimOp::getCanonicalizationPatterns(patterns, context);
tensor::DimOp::getCanonicalizationPatterns(patterns, context);
- AffineApplyOp::getCanonicalizationPatterns(patterns, context);
+ affine::AffineApplyOp::getCanonicalizationPatterns(patterns, context);
patterns.add<FoldAffineOp>(context);
// Just apply the patterns greedily.
(void)applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
registry.insert<memref::MemRefDialect>();
}
void runOnOperation() override {
- lowerLinalgToLoopsImpl<AffineForOp>(getOperation());
+ lowerLinalgToLoopsImpl<affine::AffineForOp>(getOperation());
}
};
/// Emits a loop nest of `affine.for` with the proper body for `linalgOp`.
FailureOr<LinalgLoops>
mlir::linalg::linalgOpToAffineLoops(RewriterBase &rewriter, LinalgOp linalgOp) {
- return linalgOpToLoopsImpl<AffineForOp>(rewriter, linalgOp);
+ return linalgOpToLoopsImpl<affine::AffineForOp>(rewriter, linalgOp);
}
/// Emits a loop nest of `scf.for` with the proper body for `linalgOp`.
// Adjust the split point so that it doesn't overflow the size.
AffineExpr d0, d1, d2;
bindDims(rewriter.getContext(), d0, d1, d2);
- OpFoldResult minSplitPoint = makeComposedFoldedAffineMin(
+ OpFoldResult minSplitPoint = affine::makeComposedFoldedAffineMin(
rewriter, op.getLoc(),
AffineMap::inferFromExprList(ArrayRef<AffineExpr>{d0, d1 + d2}).front(),
{splitPoint, offsets[dimension], sizes[dimension]});
// Compute the size of the second part. Return early if the second part would
// have an empty iteration space.
- OpFoldResult remainingSize = makeComposedFoldedAffineApply(
+ OpFoldResult remainingSize = affine::makeComposedFoldedAffineApply(
rewriter, op.getLoc(), d0 + d1 - d2,
{iterationSpace[dimension].offset, iterationSpace[dimension].size,
minSplitPoint});
});
// Create the second part.
- OpFoldResult totalOffset = makeComposedFoldedAffineApply(
+ OpFoldResult totalOffset = affine::makeComposedFoldedAffineApply(
rewriter, op.getLoc(), d0 + d1, {offsets[dimension], minSplitPoint});
SmallVector<Value> secondResults;
TilingInterface secondPart =
} // namespace mlir
using namespace mlir;
+using namespace mlir::affine;
using namespace mlir::linalg;
using namespace mlir::scf;
AffineExpr s1 = b.getAffineSymbolExpr(1);
AffineExpr s2 = b.getAffineSymbolExpr(2);
auto apply = [&](AffineExpr expr, ValueRange values) -> Value {
- return makeComposedAffineApply(b, b.getLoc(), expr, values);
+ return affine::makeComposedAffineApply(b, b.getLoc(), expr, values);
};
Value a = apply(s0.floorDiv(s1), {tripCount, divisorValue});
Value t = apply((s0 + s1 - 1).floorDiv(s1), {targetSizeValue, divisorValue});
/// Build an `affine_max` of all the `vals`.
static OpFoldResult buildMax(OpBuilder &b, Location loc,
ArrayRef<OpFoldResult> vals) {
- return makeComposedFoldedAffineMax(
+ return affine::makeComposedFoldedAffineMax(
b, loc, AffineMap::getMultiDimIdentityMap(vals.size(), loc.getContext()),
vals);
}
/// Build an `affine_min` of all the `vals`.
static OpFoldResult buildMin(OpBuilder &b, Location loc,
ArrayRef<OpFoldResult> vals) {
- return makeComposedFoldedAffineMin(
+ return affine::makeComposedFoldedAffineMin(
b, loc, AffineMap::getMultiDimIdentityMap(vals.size(), loc.getContext()),
vals);
}
void mlir::linalg::populateLinalgTilingCanonicalizationPatterns(
RewritePatternSet &patterns) {
auto *ctx = patterns.getContext();
- AffineApplyOp::getCanonicalizationPatterns(patterns, ctx);
- AffineForOp::getCanonicalizationPatterns(patterns, ctx);
- AffineMinOp::getCanonicalizationPatterns(patterns, ctx);
- AffineMaxOp::getCanonicalizationPatterns(patterns, ctx);
+ affine::AffineApplyOp::getCanonicalizationPatterns(patterns, ctx);
+ affine::AffineForOp::getCanonicalizationPatterns(patterns, ctx);
+ affine::AffineMinOp::getCanonicalizationPatterns(patterns, ctx);
+ affine::AffineMaxOp::getCanonicalizationPatterns(patterns, ctx);
arith::ConstantIndexOp::getCanonicalizationPatterns(patterns, ctx);
memref::SubViewOp::getCanonicalizationPatterns(patterns, ctx);
for (auto result : indexingMap.getResults()) {
AffineMap m = AffineMap::get(indexingMap.getNumDims(),
indexingMap.getNumSymbols(), result);
- Value v = b.create<AffineApplyOp>(loc, m, ivs);
+ Value v = b.create<affine::AffineApplyOp>(loc, m, ivs);
indices.push_back(v);
}
return indices;
return llvm::to_vector(
llvm::map_range(map.getResults(), [&](AffineExpr loopExpr) {
- OpFoldResult ofr =
- makeComposedFoldedAffineApply(b, loc, loopExpr, allShapesSizes);
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
+ b, loc, loopExpr, allShapesSizes);
return Range{b.getIndexAttr(0), ofr, b.getIndexAttr(1)};
}));
}
bindDims(b.getContext(), d0);
SmallVector<OpFoldResult> subShapeSizes =
llvm::to_vector(llvm::map_range(sizes, [&](OpFoldResult ofr) {
- return makeComposedFoldedAffineApply(b, loc, d0 - 1, ofr);
+ return affine::makeComposedFoldedAffineApply(b, loc, d0 - 1, ofr);
}));
OpOperand *outOperand = linalgOp.getDpsInitOperand(resultNumber);
/// Converts affine.apply Ops to arithmetic operations.
static void convertAffineApply(RewriterBase &rewriter, LinalgOp linalgOp) {
OpBuilder::InsertionGuard g(rewriter);
- auto toReplace = linalgOp.getBlock()->getOps<AffineApplyOp>();
+ auto toReplace = linalgOp.getBlock()->getOps<affine::AffineApplyOp>();
for (auto op : make_early_inc_range(toReplace)) {
rewriter.setInsertionPoint(op);
- auto expanded = expandAffineExpr(
+ auto expanded = affine::expandAffineExpr(
rewriter, op->getLoc(), op.getAffineMap().getResult(0),
op.getOperands().take_front(op.getAffineMap().getNumDims()),
op.getOperands().take_back(op.getAffineMap().getNumSymbols()));
// Case 2: Both values are identical AffineMinOps. (Should not happen if
// CSE is run.)
- auto minOp1 = v1.getDefiningOp<AffineMinOp>();
- auto minOp2 = v2.getDefiningOp<AffineMinOp>();
+ auto minOp1 = v1.getDefiningOp<affine::AffineMinOp>();
+ auto minOp2 = v2.getDefiningOp<affine::AffineMinOp>();
if (minOp1 && minOp2 && minOp1.getAffineMap() == minOp2.getAffineMap() &&
minOp1.getOperands() == minOp2.getOperands())
continue;
using namespace mlir;
using namespace presburger;
+using namespace mlir::affine;
using namespace mlir::linalg;
using namespace mlir::scf;
constantSteps.push_back(op.value());
}
- mlir::buildAffineLoopNest(b, loc, lbs, ubs, constantSteps,
- [&](OpBuilder &b, Location loc, ValueRange ivs) {
- bodyBuilderFn(b, loc, ivs,
- linalgOp->getOperands());
- });
+ affine::buildAffineLoopNest(b, loc, lbs, ubs, constantSteps,
+ [&](OpBuilder &b, Location loc, ValueRange ivs) {
+ bodyBuilderFn(b, loc, ivs,
+ linalgOp->getOperands());
+ });
}
/// Update the `lb`, `ub` and `step` to get per processor `lb`, `ub` and `step`.
AffineExpr d0, d1;
bindDims(b.getContext(), d0, d1);
AffineExpr s0 = getAffineSymbolExpr(0, b.getContext());
- lb = makeComposedAffineApply(b, loc, d0 + d1 * s0, {lb, procId, step});
- step = makeComposedAffineApply(b, loc, d0 * s0, {nprocs, step});
+ lb =
+ affine::makeComposedAffineApply(b, loc, d0 + d1 * s0, {lb, procId, step});
+ step = affine::makeComposedAffineApply(b, loc, d0 * s0, {nprocs, step});
}
/// Generates a loop nest consisting of scf.parallel and scf.for, depending
void init() {
declareDependentDialect<pdl::PDLDialect>();
- declareGeneratedDialect<AffineDialect>();
+ declareGeneratedDialect<affine::AffineDialect>();
declareGeneratedDialect<arith::ArithDialect>();
declareGeneratedDialect<memref::MemRefDialect>();
declareGeneratedDialect<nvgpu::NVGPUDialect>();
}
AffineMap map = AffineMap::get(0, affineApplyOperands.size(), expr);
- Value result = rewriter.create<AffineApplyOp>(op.getLoc(), map,
- affineApplyOperands);
+ Value result = rewriter.create<affine::AffineApplyOp>(
+ op.getLoc(), map, affineApplyOperands);
offsets.push_back(result);
}
}
#include "mlir/Dialect/MemRef/Transforms/Passes.h.inc"
} // namespace memref
} // namespace mlir
+
using namespace mlir;
+using namespace mlir::affine;
namespace {
AffineExpr s1 = rewriter.getAffineSymbolExpr(1);
for (auto [srcSize, indice] : llvm::zip(srcSizes, indices)) {
- finalSizes.push_back(makeComposedFoldedAffineApply(rewriter, loc, s0 - s1,
- {srcSize, indice}));
+ finalSizes.push_back(affine::makeComposedFoldedAffineApply(
+ rewriter, loc, s0 - s1, {srcSize, indice}));
}
return finalSizes;
}
// Creating maximally folded and composd affine.apply composes better with
// other transformations without interleaving canonicalization passes.
- OpFoldResult ofr = makeComposedFoldedAffineApply(
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
rewriter, loc,
AffineMap::get(/*numDims=*/groupSize,
/*numSymbols=*/0, srcIndexExpr),
// Construct the AffineApplyOp for each delinearizingExpr.
for (int64_t i = 0; i < groupSize; i++) {
- OpFoldResult ofr = makeComposedFoldedAffineApply(
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
rewriter, loc,
AffineMap::get(/*numDims=*/1, /*numSymbols=*/0,
delinearizingExprs[i]),
int64_t srcRank =
collapseShapeOp.getViewSource().getType().cast<MemRefType>().getRank();
for (int64_t i = 0; i < srcRank; i++) {
- OpFoldResult ofr = makeComposedFoldedAffineApply(
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
rewriter, loc, zeroAffineMap, dynamicIndices);
sourceIndices.push_back(
getValueOrCreateConstantIndexOp(rewriter, loc, ofr));
// Resolve sizes according to dropped dims.
SmallVector<OpFoldResult> resolvedSizes;
llvm::SmallBitVector srcDroppedDims = srcSubView.getDroppedDims();
- resolveSizesIntoOpWithSizes(srcSubView.getMixedSizes(),
- subView.getMixedSizes(), srcDroppedDims,
- resolvedSizes);
+ affine::resolveSizesIntoOpWithSizes(srcSubView.getMixedSizes(),
+ subView.getMixedSizes(), srcDroppedDims,
+ resolvedSizes);
// Resolve offsets according to source offsets and strides.
SmallVector<Value> resolvedOffsets;
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, subView.getLoc(), srcSubView.getMixedOffsets(),
srcSubView.getMixedStrides(), srcDroppedDims, subView.getMixedOffsets(),
resolvedOffsets);
llvm::map_range(indices, [](Value v) -> OpFoldResult { return v; })));
SmallVector<Value> expandedIndices;
for (unsigned i = 0, e = affineMap.getNumResults(); i < e; i++) {
- OpFoldResult ofr = makeComposedFoldedAffineApply(
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
rewriter, loc, affineMap.getSubMap({i}), indicesOfr);
expandedIndices.push_back(
getValueOrCreateConstantIndexOp(rewriter, loc, ofr));
loadOp.getIndices().end());
// For affine ops, we need to apply the map to get the operands to get the
// "actual" indices.
- if (auto affineLoadOp = dyn_cast<AffineLoadOp>(loadOp.getOperation())) {
+ if (auto affineLoadOp =
+ dyn_cast<affine::AffineLoadOp>(loadOp.getOperation())) {
AffineMap affineMap = affineLoadOp.getAffineMap();
auto expandedIndices = calculateExpandedAccessIndices(
affineMap, indices, loadOp.getLoc(), rewriter);
indices.assign(expandedIndices.begin(), expandedIndices.end());
}
SmallVector<Value> sourceIndices;
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, loadOp.getLoc(), subViewOp.getMixedOffsets(),
subViewOp.getMixedStrides(), subViewOp.getDroppedDims(), indices,
sourceIndices);
llvm::TypeSwitch<Operation *, void>(loadOp)
- .Case([&](AffineLoadOp op) {
- rewriter.replaceOpWithNewOp<AffineLoadOp>(loadOp, subViewOp.getSource(),
- sourceIndices);
+ .Case([&](affine::AffineLoadOp op) {
+ rewriter.replaceOpWithNewOp<affine::AffineLoadOp>(
+ loadOp, subViewOp.getSource(), sourceIndices);
})
.Case([&](memref::LoadOp op) {
rewriter.replaceOpWithNewOp<memref::LoadOp>(
loadOp.getIndices().end());
// For affine ops, we need to apply the map to get the operands to get the
// "actual" indices.
- if (auto affineLoadOp = dyn_cast<AffineLoadOp>(loadOp.getOperation())) {
+ if (auto affineLoadOp =
+ dyn_cast<affine::AffineLoadOp>(loadOp.getOperation())) {
AffineMap affineMap = affineLoadOp.getAffineMap();
auto expandedIndices = calculateExpandedAccessIndices(
affineMap, indices, loadOp.getLoc(), rewriter);
loadOp.getLoc(), rewriter, expandShapeOp, indices, sourceIndices)))
return failure();
llvm::TypeSwitch<Operation *, void>(loadOp)
- .Case<AffineLoadOp, memref::LoadOp>([&](auto op) {
+ .Case<affine::AffineLoadOp, memref::LoadOp>([&](auto op) {
rewriter.replaceOpWithNewOp<decltype(op)>(
loadOp, expandShapeOp.getViewSource(), sourceIndices);
})
loadOp.getIndices().end());
// For affine ops, we need to apply the map to get the operands to get the
// "actual" indices.
- if (auto affineLoadOp = dyn_cast<AffineLoadOp>(loadOp.getOperation())) {
+ if (auto affineLoadOp =
+ dyn_cast<affine::AffineLoadOp>(loadOp.getOperation())) {
AffineMap affineMap = affineLoadOp.getAffineMap();
auto expandedIndices = calculateExpandedAccessIndices(
affineMap, indices, loadOp.getLoc(), rewriter);
loadOp.getLoc(), rewriter, collapseShapeOp, indices, sourceIndices)))
return failure();
llvm::TypeSwitch<Operation *, void>(loadOp)
- .Case<AffineLoadOp, memref::LoadOp>([&](auto op) {
+ .Case<affine::AffineLoadOp, memref::LoadOp>([&](auto op) {
rewriter.replaceOpWithNewOp<decltype(op)>(
loadOp, collapseShapeOp.getViewSource(), sourceIndices);
})
storeOp.getIndices().end());
// For affine ops, we need to apply the map to get the operands to get the
// "actual" indices.
- if (auto affineStoreOp = dyn_cast<AffineStoreOp>(storeOp.getOperation())) {
+ if (auto affineStoreOp =
+ dyn_cast<affine::AffineStoreOp>(storeOp.getOperation())) {
AffineMap affineMap = affineStoreOp.getAffineMap();
auto expandedIndices = calculateExpandedAccessIndices(
affineMap, indices, storeOp.getLoc(), rewriter);
indices.assign(expandedIndices.begin(), expandedIndices.end());
}
SmallVector<Value> sourceIndices;
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, storeOp.getLoc(), subViewOp.getMixedOffsets(),
subViewOp.getMixedStrides(), subViewOp.getDroppedDims(), indices,
sourceIndices);
llvm::TypeSwitch<Operation *, void>(storeOp)
- .Case([&](AffineStoreOp op) {
- rewriter.replaceOpWithNewOp<AffineStoreOp>(
+ .Case([&](affine::AffineStoreOp op) {
+ rewriter.replaceOpWithNewOp<affine::AffineStoreOp>(
op, op.getValue(), subViewOp.getSource(), sourceIndices);
})
.Case([&](memref::StoreOp op) {
storeOp.getIndices().end());
// For affine ops, we need to apply the map to get the operands to get the
// "actual" indices.
- if (auto affineStoreOp = dyn_cast<AffineStoreOp>(storeOp.getOperation())) {
+ if (auto affineStoreOp =
+ dyn_cast<affine::AffineStoreOp>(storeOp.getOperation())) {
AffineMap affineMap = affineStoreOp.getAffineMap();
auto expandedIndices = calculateExpandedAccessIndices(
affineMap, indices, storeOp.getLoc(), rewriter);
storeOp.getLoc(), rewriter, expandShapeOp, indices, sourceIndices)))
return failure();
llvm::TypeSwitch<Operation *, void>(storeOp)
- .Case<AffineStoreOp, memref::StoreOp>([&](auto op) {
+ .Case<affine::AffineStoreOp, memref::StoreOp>([&](auto op) {
rewriter.replaceOpWithNewOp<decltype(op)>(storeOp, storeOp.getValue(),
expandShapeOp.getViewSource(),
sourceIndices);
storeOp.getIndices().end());
// For affine ops, we need to apply the map to get the operands to get the
// "actual" indices.
- if (auto affineStoreOp = dyn_cast<AffineStoreOp>(storeOp.getOperation())) {
+ if (auto affineStoreOp =
+ dyn_cast<affine::AffineStoreOp>(storeOp.getOperation())) {
AffineMap affineMap = affineStoreOp.getAffineMap();
auto expandedIndices = calculateExpandedAccessIndices(
affineMap, indices, storeOp.getLoc(), rewriter);
storeOp.getLoc(), rewriter, collapseShapeOp, indices, sourceIndices)))
return failure();
llvm::TypeSwitch<Operation *, void>(storeOp)
- .Case<AffineStoreOp, memref::StoreOp>([&](auto op) {
+ .Case<affine::AffineStoreOp, memref::StoreOp>([&](auto op) {
rewriter.replaceOpWithNewOp<decltype(op)>(
storeOp, storeOp.getValue(), collapseShapeOp.getViewSource(),
sourceIndices);
if (srcSubViewOp) {
LLVM_DEBUG(DBGS() << "srcSubViewOp : " << srcSubViewOp << "\n");
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, copyOp.getLoc(), srcSubViewOp.getMixedOffsets(),
srcSubViewOp.getMixedStrides(), srcSubViewOp.getDroppedDims(),
srcindices, foldedSrcIndices);
if (dstSubViewOp) {
LLVM_DEBUG(DBGS() << "dstSubViewOp : " << dstSubViewOp << "\n");
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, copyOp.getLoc(), dstSubViewOp.getMixedOffsets(),
dstSubViewOp.getMixedStrides(), dstSubViewOp.getDroppedDims(),
dstindices, foldedDstIndices);
}
void memref::populateFoldMemRefAliasOpPatterns(RewritePatternSet &patterns) {
- patterns.add<LoadOpOfSubViewOpFolder<AffineLoadOp>,
+ patterns.add<LoadOpOfSubViewOpFolder<affine::AffineLoadOp>,
LoadOpOfSubViewOpFolder<memref::LoadOp>,
LoadOpOfSubViewOpFolder<vector::TransferReadOp>,
LoadOpOfSubViewOpFolder<gpu::SubgroupMmaLoadMatrixOp>,
- StoreOpOfSubViewOpFolder<AffineStoreOp>,
+ StoreOpOfSubViewOpFolder<affine::AffineStoreOp>,
StoreOpOfSubViewOpFolder<memref::StoreOp>,
StoreOpOfSubViewOpFolder<vector::TransferWriteOp>,
StoreOpOfSubViewOpFolder<gpu::SubgroupMmaStoreMatrixOp>,
- LoadOpOfExpandShapeOpFolder<AffineLoadOp>,
+ LoadOpOfExpandShapeOpFolder<affine::AffineLoadOp>,
LoadOpOfExpandShapeOpFolder<memref::LoadOp>,
- StoreOpOfExpandShapeOpFolder<AffineStoreOp>,
+ StoreOpOfExpandShapeOpFolder<affine::AffineStoreOp>,
StoreOpOfExpandShapeOpFolder<memref::StoreOp>,
- LoadOpOfCollapseShapeOpFolder<AffineLoadOp>,
+ LoadOpOfCollapseShapeOpFolder<affine::AffineLoadOp>,
LoadOpOfCollapseShapeOpFolder<memref::LoadOp>,
- StoreOpOfCollapseShapeOpFolder<AffineStoreOp>,
+ StoreOpOfCollapseShapeOpFolder<affine::AffineStoreOp>,
StoreOpOfCollapseShapeOpFolder<memref::StoreOp>,
SubViewOfSubViewFolder, NvgpuAsyncCopyOpSubViewOpFolder>(
patterns.getContext());
Value stepVal = getValueOrCreateConstantIndexOp(rewriter, loc, *singleStep);
AffineExpr iv, lb, step;
bindDims(rewriter.getContext(), iv, lb, step);
- Value bufferIndex = makeComposedAffineApply(
+ Value bufferIndex = affine::makeComposedAffineApply(
rewriter, loc, ((iv - lb).floorDiv(step)) % multiBufferingFactor,
{ivVal, lbVal, stepVal});
LLVM_DEBUG(DBGS() << "--multi-buffered indexing: " << bufferIndex << "\n");
#define DEBUG_TYPE "normalize-memrefs"
using namespace mlir;
+using namespace mlir::affine;
namespace {
#include "mlir/Dialect/Vector/IR/VectorOps.h"
using namespace mlir;
+using namespace mlir::affine;
//===----------------------------------------------------------------------===//
// GetParentForOp
using Base::Base;
void init() {
- declareGeneratedDialect<AffineDialect>();
+ declareGeneratedDialect<affine::AffineDialect>();
declareGeneratedDialect<func::FuncDialect>();
registerTransformOps<
RewritePatternSet &patterns) {
MLIRContext *ctx = patterns.getContext();
patterns
- .add<AffineOpSCFCanonicalizationPattern<AffineMinOp>,
- AffineOpSCFCanonicalizationPattern<AffineMaxOp>,
+ .add<AffineOpSCFCanonicalizationPattern<affine::AffineMinOp>,
+ AffineOpSCFCanonicalizationPattern<affine::AffineMaxOp>,
DimOfIterArgFolder<tensor::DimOp>, DimOfIterArgFolder<memref::DimOp>,
DimOfLoopResultFolder<tensor::DimOp>,
DimOfLoopResultFolder<memref::DimOp>>(ctx);
} // namespace mlir
using namespace mlir;
+using namespace mlir::affine;
using scf::ForOp;
using scf::ParallelOp;
// Otherwise, we dynamically compute the bound for
// each iteration of the outer loop.
newBounds.push_back(
- b.create<AffineMinOp>(op.getLoc(), b.getIndexType(), minMap,
- ValueRange{newStep, upperBound, iv}));
+ b.create<affine::AffineMinOp>(op.getLoc(), b.getIndexType(), minMap,
+ ValueRange{newStep, upperBound, iv}));
}
auto innerLoop = b.create<ParallelOp>(
op.getLoc(), SmallVector<Value, 2>(newBounds.size(), zero), newBounds,
bindSymbols(b.getContext(), s0, s1);
AffineMap minMap = AffineMap::get(1, 2, {s0, s1 - d0}, b.getContext());
Value size = getValueOrCreateConstantIndexOp(b, loc, loopRange.size);
- return makeComposedFoldedAffineMin(
+ return affine::makeComposedFoldedAffineMin(
b, loc, minMap, SmallVector<OpFoldResult>{iv, tileSize, size});
}
#define DEBUG_TYPE "mlir-scf-affine-utils"
using namespace mlir;
+using namespace affine;
using namespace presburger;
LogicalResult scf::matchForLikeLoop(Value iv, OpFoldResult &lb,
RewriterBase::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(op);
FailureOr<AffineValueMap> simplified =
- mlir::simplifyConstrainedMinMaxOp(op, std::move(constraints));
+ affine::simplifyConstrainedMinMaxOp(op, std::move(constraints));
if (failed(simplified))
return failure();
return rewriter.replaceOpWithNewOp<AffineApplyOp>(
{rewriter.getAffineSymbolExpr(0),
rewriter.getAffineDimExpr(0) - rewriter.getAffineDimExpr(1)},
rewriter.getContext());
- Value end =
- rewriter.createOrFold<AffineMinOp>(loc, min, ValueRange{hi, iv, step});
+ Value end = rewriter.createOrFold<affine::AffineMinOp>(
+ loc, min, ValueRange{hi, iv, step});
return rewriter.create<vector::CreateMaskOp>(loc, mtp, end);
}
AffineExpr currExpr = builder.getAffineSymbolExpr(dim - startPos);
expr = (expr ? expr * currExpr : currExpr);
}
- return applyMapToValues(builder, loc,
- AffineMap::get(0, endPos - startPos + 1, expr),
- dynamicDims)[0];
+ return affine::applyMapToValues(
+ builder, loc, AffineMap::get(0, endPos - startPos + 1, expr),
+ dynamicDims)[0];
}
/// Given the `src` of a collapsing reshape op and its reassociation maps,
linearizedStaticDim *= d.value();
}
Value sourceDim = builder.create<tensor::DimOp>(loc, src, sourceDimPos);
- return applyMapToValues(
+ return affine::applyMapToValues(
builder, loc,
AffineMap::get(
0, 1, builder.getAffineSymbolExpr(0).floorDiv(linearizedStaticDim)),
};
addOpFoldResult(lowPad[dim]);
addOpFoldResult(highPad[dim]);
- shapes.push_back(applyMapToValues(
+ shapes.push_back(affine::applyMapToValues(
b, loc, AffineMap::get(1, numSymbols, expr), mapOperands)[0]);
}
reifiedReturnShapes.emplace_back(std::move(shapes));
rewriter.create<DimOp>(dimOp.getLoc(), expandShapeOp.getSrc(), srcDim);
AffineExpr expr;
bindSymbols(dimOp.getContext(), expr);
- rewriter.replaceOpWithNewOp<AffineApplyOp>(dimOp, expr.floorDiv(product),
- srcDimSz);
+ rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(
+ dimOp, expr.floorDiv(product), srcDimSz);
return success();
}
};
syms.push_back(rewriter.getAffineSymbolExpr(it.index()));
product = product ? product * syms.back() : syms.back();
}
- rewriter.replaceOpWithNewOp<AffineApplyOp>(dimOp, product, srcDimSizes);
+ rewriter.replaceOpWithNewOp<affine::AffineApplyOp>(dimOp, product,
+ srcDimSizes);
return success();
}
};
bindSymbols(builder.getContext(), s0, s1);
AffineExpr ceilDivExpr = s0.ceilDiv(s1);
for (auto tiledDim : llvm::enumerate(innerDimsPos)) {
- resultDims[tiledDim.value()] = makeComposedFoldedAffineApply(
+ resultDims[tiledDim.value()] = affine::makeComposedFoldedAffineApply(
builder, loc, ceilDivExpr,
{resultDims[tiledDim.value()], innerTileSizes[tiledDim.index()]});
}
AffineExpr dim0, dim1;
bindDims(b.getContext(), dim0, dim1);
auto ceilDiv = [&](OpFoldResult v1, OpFoldResult v2) -> OpFoldResult {
- return makeComposedFoldedAffineApply(b, loc, dim0.ceilDiv(dim1), {v1, v2});
+ return affine::makeComposedFoldedAffineApply(b, loc, dim0.ceilDiv(dim1),
+ {v1, v2});
};
SmallVector<OpFoldResult> mixedSizes;
AffineExpr sym0, sym1;
bindSymbols(b.getContext(), sym0, sym1);
auto dimMul = [&](OpFoldResult v1, OpFoldResult v2) -> OpFoldResult {
- return makeComposedFoldedAffineApply(b, loc, sym0 * sym1, {v1, v2});
+ return affine::makeComposedFoldedAffineApply(b, loc, sym0 * sym1, {v1, v2});
};
SmallVector<OpFoldResult> mixedSizes;
tensor::createDimValues(b, loc, packOp.getSource());
SmallVector<OpFoldResult> inputIndices, inputSizes;
for (auto dim : llvm::seq<int64_t>(0, inputRank)) {
- using AV = AffineValueExpr;
- AffineBuilder ab(b, loc);
+ using AV = affine::AffineValueExpr;
+ affine::AffineBuilder ab(b, loc);
AffineExpr dim0, dim1, sym;
bindDims(b.getContext(), dim0, dim1);
bindSymbols(b.getContext(), sym);
}
Location loc = unpackOp.getLoc();
- using AV = AffineValueExpr;
- AffineBuilder ab(b, loc);
+ using AV = affine::AffineValueExpr;
+ affine::AffineBuilder ab(b, loc);
AffineExpr dim0, dim1, sym0;
bindDims(b.getContext(), dim0, dim1);
bindSymbols(b.getContext(), sym0);
return info;
}
- DivModValue firstCoord =
- getDivMod(b, loc, getValueOrCreateConstantIndexOp(b, loc, tileOffset),
- getValueOrCreateConstantIndexOp(b, loc, innerTileSize));
+ affine::DivModValue firstCoord = affine::getDivMod(
+ b, loc, getValueOrCreateConstantIndexOp(b, loc, tileOffset),
+ getValueOrCreateConstantIndexOp(b, loc, innerTileSize));
OpFoldResult tileExclusiveBound =
ab.add(AV(dim0).bind(tileOffset), AV(dim1).bind(tileSize));
- DivModValue lastCoord = getDivMod(
+ affine::DivModValue lastCoord = affine::getDivMod(
b, loc,
getValueOrCreateConstantIndexOp(
b, loc,
// Add two integers.
auto addMap = AffineMap::get(2, 0, {dim0 + dim1});
auto add = [&](OpFoldResult v1, OpFoldResult v2) {
- return makeComposedFoldedAffineApply(b, loc, addMap, {v1, v2});
+ return affine::makeComposedFoldedAffineApply(b, loc, addMap, {v1, v2});
};
// Subtract two integers.
auto subMap = AffineMap::get(2, 0, {dim0 - dim1});
auto sub = [&](OpFoldResult v1, OpFoldResult v2) {
- return makeComposedFoldedAffineApply(b, loc, subMap, {v1, v2});
+ return affine::makeComposedFoldedAffineApply(b, loc, subMap, {v1, v2});
};
// Take the minimum of two integers.
auto idMap = AffineMap::getMultiDimIdentityMap(2, b.getContext());
auto min = [&](OpFoldResult v1, OpFoldResult v2) {
- return makeComposedFoldedAffineMin(b, loc, idMap, {v1, v2});
+ return affine::makeComposedFoldedAffineMin(b, loc, idMap, {v1, v2});
};
// Take the maximum of two integers.
auto max = [&](OpFoldResult v1, OpFoldResult v2) {
- return makeComposedFoldedAffineMax(b, loc, idMap, {v1, v2});
+ return affine::makeComposedFoldedAffineMax(b, loc, idMap, {v1, v2});
};
// Zero index-typed integer.
OpFoldResult zero = b.getIndexAttr(0);
AffineExpr s0, s1, s2;
bindSymbols(op->getContext(), s0, s1, s2);
AffineExpr sumExpr = s0 + s1 + s2;
- Value sum = rewriter.create<AffineApplyOp>(
+ Value sum = rewriter.create<affine::AffineApplyOp>(
loc, sumExpr, ValueRange{srcDim, lowPad, highPad});
dynamicSizes.push_back(sum);
}
#include "llvm/ADT/STLExtras.h"
using namespace mlir;
+using namespace mlir::affine;
using namespace mlir::tensor;
/// Get the dimension size of a value of RankedTensor type at the
assert(dim < sliceParams.size() && "slice should be non rank-reducing");
return std::make_pair(
dim,
- makeComposedAffineApply(
+ affine::makeComposedAffineApply(
b, loc, s0 + d0 * s1,
{indexValue,
getValueOrCreateConstantIndexOp(b, loc, sliceParams[dim].offset),
SmallVector<Value> indices(readOp.getIndices().begin(),
readOp.getIndices().end());
SmallVector<Value> sourceIndices;
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, readOp.getLoc(), extractSliceOp.getMixedOffsets(),
extractSliceOp.getMixedStrides(), extractSliceOp.getDroppedDims(),
indices, sourceIndices);
SmallVector<Value> indices(writeOp.getIndices().begin(),
writeOp.getIndices().end());
SmallVector<Value> sourceIndices;
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, writeOp.getLoc(), insertSliceOp.getMixedOffsets(),
insertSliceOp.getMixedStrides(), insertSliceOp.getDroppedDims(), indices,
sourceIndices);
// Note: the "insertSlice" case is symmetrical to the extract/subview case:
// `insertSliceOp` is passed as the "source" and `sourceInsertSliceOp` is
// passed as the destination to the helper function.
- resolveSizesIntoOpWithSizes(insertSliceOp.getMixedSizes(),
- sourceInsertSliceOp.getMixedSizes(),
- droppedDims, resolvedSizes);
+ affine::resolveSizesIntoOpWithSizes(insertSliceOp.getMixedSizes(),
+ sourceInsertSliceOp.getMixedSizes(),
+ droppedDims, resolvedSizes);
// If we are inside an InParallel region, temporarily set the insertion
// point outside: only tensor.parallel_insert_slice ops are allowed in
// Note: the "insertSlice" case is symmetrical to the extract/subview case:
// `insertSliceOp` is passed as the "source" and `sourceInsertSliceOp` is
// passed as the destination to the helper function.
- resolveIndicesIntoOpWithOffsetsAndStrides(
+ affine::resolveIndicesIntoOpWithOffsetsAndStrides(
rewriter, insertSliceOp.getLoc(), insertSliceOp.getMixedOffsets(),
insertSliceOp.getMixedStrides(), droppedDims,
sourceInsertSliceOp.getMixedOffsets(), resolvedOffsets);
return failure();
SmallVector<OpFoldResult> newOffsets, newSizes, newStrides;
- if (failed(mergeOffsetsSizesAndStrides(rewriter, nextOp.getLoc(), prevOp,
- nextOp, prevOp.getDroppedDims(),
- newOffsets, newSizes, newStrides)))
+ if (failed(affine::mergeOffsetsSizesAndStrides(
+ rewriter, nextOp.getLoc(), prevOp, nextOp, prevOp.getDroppedDims(),
+ newOffsets, newSizes, newStrides)))
return failure();
rewriter.replaceOpWithNewOp<ExtractSliceOp>(nextOp, nextOp.getType(),
bindDims(b.getContext(), d0);
auto dimOp = b.createOrFold<tensor::DimOp>(loc, source, en.index());
high[en.index()] =
- makeComposedAffineApply(b, loc, en.value() - d0, {dimOp}).getResult();
+ affine::makeComposedAffineApply(b, loc, en.value() - d0, {dimOp})
+ .getResult();
}
return b.create<PadOp>(loc, type, source, low, high, pad, nofold);
}
Value buildDistributedOffset(RewriterBase &b, Location loc, int64_t index) {
int64_t distributedSize = distributedVectorType.getDimSize(index);
AffineExpr tid = getAffineSymbolExpr(0, b.getContext());
- return b.createOrFold<AffineApplyOp>(loc, tid * distributedSize,
- ArrayRef<Value>{laneId});
+ return b.createOrFold<affine::AffineApplyOp>(loc, tid * distributedSize,
+ ArrayRef<Value>{laneId});
}
/// Create a store during the process of distributing the
unsigned vectorPos = std::get<1>(it).cast<AffineDimExpr>().getPosition();
auto scale =
rewriter.getAffineConstantExpr(targetType.getDimSize(vectorPos));
- indices[indexPos] =
- makeComposedAffineApply(rewriter, loc, d0 + scale * d1,
- {indices[indexPos], newWarpOp.getLaneid()});
+ indices[indexPos] = affine::makeComposedAffineApply(
+ rewriter, loc, d0 + scale * d1,
+ {indices[indexPos], newWarpOp.getLaneid()});
}
newWriteOp.getIndicesMutable().assign(indices);
unsigned vectorPos = std::get<1>(it).cast<AffineDimExpr>().getPosition();
int64_t scale =
distributedVal.getType().cast<VectorType>().getDimSize(vectorPos);
- indices[indexPos] =
- makeComposedAffineApply(rewriter, read.getLoc(), d0 + scale * d1,
- {indices[indexPos], warpOp.getLaneid()});
+ indices[indexPos] = affine::makeComposedAffineApply(
+ rewriter, read.getLoc(), d0 + scale * d1,
+ {indices[indexPos], warpOp.getLaneid()});
}
Value newRead = rewriter.create<vector::TransferReadOp>(
read.getLoc(), distributedVal.getType(), read.getSource(), indices,
int64_t elementsPerLane = distributedVecType.getShape()[0];
AffineExpr sym0 = getAffineSymbolExpr(0, rewriter.getContext());
// tid of extracting thread: pos / elementsPerLane
- Value broadcastFromTid = rewriter.create<AffineApplyOp>(
+ Value broadcastFromTid = rewriter.create<affine::AffineApplyOp>(
loc, sym0.ceilDiv(elementsPerLane), extractOp.getPosition());
// Extract at position: pos % elementsPerLane
Value pos =
elementsPerLane == 1
? rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult()
: rewriter
- .create<AffineApplyOp>(loc, sym0 % elementsPerLane,
- extractOp.getPosition())
+ .create<affine::AffineApplyOp>(loc, sym0 % elementsPerLane,
+ extractOp.getPosition())
.getResult();
Value extracted =
rewriter.create<vector::ExtractElementOp>(loc, distributedVec, pos);
int64_t elementsPerLane = distrType.getShape()[0];
AffineExpr sym0 = getAffineSymbolExpr(0, rewriter.getContext());
// tid of extracting thread: pos / elementsPerLane
- Value insertingLane = rewriter.create<AffineApplyOp>(
+ Value insertingLane = rewriter.create<affine::AffineApplyOp>(
loc, sym0.ceilDiv(elementsPerLane), newPos);
// Insert position: pos % elementsPerLane
Value pos =
elementsPerLane == 1
? rewriter.create<arith::ConstantIndexOp>(loc, 0).getResult()
: rewriter
- .create<AffineApplyOp>(loc, sym0 % elementsPerLane, newPos)
+ .create<affine::AffineApplyOp>(loc, sym0 % elementsPerLane,
+ newPos)
.getResult();
Value isInsertingLane = rewriter.create<arith::CmpIOp>(
loc, arith::CmpIPredicate::eq, newWarpOp.getLaneid(), insertingLane);
if (extractOp.getPosition()) {
AffineExpr sym0, sym1;
bindSymbols(extractOp.getContext(), sym0, sym1);
- OpFoldResult ofr = makeComposedFoldedAffineApply(
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
rewriter, extractOp.getLoc(), sym0 + sym1,
{newIndices[newIndices.size() - 1], extractOp.getPosition()});
if (ofr.is<Value>()) {
int64_t offset = it.value().cast<IntegerAttr>().getInt();
int64_t idx =
newIndices.size() - extractOp.getPosition().size() + it.index();
- OpFoldResult ofr = makeComposedFoldedAffineApply(
+ OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
rewriter, extractOp.getLoc(),
rewriter.getAffineSymbolExpr(0) + offset, {newIndices[idx]});
if (ofr.is<Value>()) {
static std::optional<int64_t> extractConstantIndex(Value v) {
if (auto cstOp = v.getDefiningOp<arith::ConstantIndexOp>())
return cstOp.value();
- if (auto affineApplyOp = v.getDefiningOp<AffineApplyOp>())
+ if (auto affineApplyOp = v.getDefiningOp<affine::AffineApplyOp>())
if (affineApplyOp.getAffineMap().isSingleConstant())
return affineApplyOp.getAffineMap().getSingleConstantResult();
return std::nullopt;
int64_t vectorSize = xferOp.getVectorType().getDimSize(resultIdx);
auto d0 = getAffineDimExpr(0, xferOp.getContext());
auto vs = getAffineConstantExpr(vectorSize, xferOp.getContext());
- Value sum =
- makeComposedAffineApply(b, loc, d0 + vs, xferOp.indices()[indicesIdx]);
+ Value sum = affine::makeComposedAffineApply(b, loc, d0 + vs,
+ xferOp.indices()[indicesIdx]);
Value cond = createFoldedSLE(
b, sum, vector::createOrFoldDimOp(b, loc, xferOp.source(), indicesIdx));
if (!cond)
SmallVector<AffineMap, 4> maps =
AffineMap::inferFromExprList(MapList{{i - j, k}});
// affine_min(%dimMemRef - %index, %dimAlloc)
- Value affineMin = b.create<AffineMinOp>(
+ Value affineMin = b.create<affine::AffineMinOp>(
loc, index.getType(), maps[0], ValueRange{dimMemRef, index, dimAlloc});
sizes.push_back(affineMin);
});
parent = parent->getParentOp()) {
if (parent->hasTrait<OpTrait::AutomaticAllocationScope>())
scope = parent;
- if (!isa<scf::ForOp, AffineForOp>(parent))
+ if (!isa<scf::ForOp, affine::AffineForOp>(parent))
break;
}
assert(scope && "Expected op to be inside automatic allocation scope");
auto expr = getAffineDimExpr(0, builder.getContext()) +
getAffineConstantExpr(elementOffsets[dim.index()], ctx);
auto map = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, expr);
- slicedIndices[pos] = builder.create<AffineApplyOp>(loc, map, indices[pos]);
+ slicedIndices[pos] =
+ builder.create<affine::AffineApplyOp>(loc, map, indices[pos]);
}
return slicedIndices;
}
for (auto kvp : enclosingLoopToVectorDim) {
assert(kvp.second < perm.size());
- auto invariants = getInvariantAccesses(
- cast<AffineForOp>(kvp.first).getInductionVar(), indices);
+ auto invariants = affine::getInvariantAccesses(
+ cast<affine::AffineForOp>(kvp.first).getInductionVar(), indices);
unsigned numIndices = indices.size();
unsigned countInvariantIndices = 0;
for (unsigned dim = 0; dim < numIndices; ++dim) {
/// Returns the enclosing AffineForOp, from closest to farthest.
static SetVector<Operation *> getEnclosingforOps(Block *block) {
- return getParentsOfType<AffineForOp>(block);
+ return getParentsOfType<affine::AffineForOp>(block);
}
AffineMap mlir::makePermutationMap(
#define DEBUG_TYPE "memref-bound-check"
using namespace mlir;
+using namespace mlir::affine;
namespace {
#define DEBUG_TYPE "test-memref-dependence-check"
using namespace mlir;
+using namespace mlir::affine;
namespace {
#define PASS_NAME "test-affine-data-copy"
using namespace mlir;
+using namespace mlir::affine;
namespace {
#include "mlir/Dialect/Func/IR/FuncOps.h"
using namespace mlir;
+using namespace mlir::affine;
#define DEBUG_TYPE "test-affine-parametric-tile"
#define PASS_NAME "test-affine-loop-unswitch"
using namespace mlir;
+using namespace mlir::affine;
namespace {
#define PASS_NAME "test-decompose-affine-ops"
using namespace mlir;
+using namespace mlir::affine;
namespace {
#define DEBUG_TYPE "test-loop-fusion"
using namespace mlir;
+using namespace mlir::affine;
static llvm::cl::OptionCategory clOptionsCategory(DEBUG_TYPE " options");
static bool testDependenceCheck(AffineForOp srcForOp, AffineForOp dstForOp,
unsigned i, unsigned j, unsigned loopDepth,
unsigned maxLoopDepth) {
- mlir::ComputationSliceState sliceUnion;
+ affine::ComputationSliceState sliceUnion;
for (unsigned d = loopDepth + 1; d <= maxLoopDepth; ++d) {
FusionResult result =
- mlir::canFuseLoops(srcForOp, dstForOp, d, &sliceUnion);
+ affine::canFuseLoops(srcForOp, dstForOp, d, &sliceUnion);
if (result.value == FusionResult::FailBlockDependence) {
srcForOp->emitRemark("block-level dependence preventing"
" fusion of loop nest ")
}
// Returns a string representation of 'sliceUnion'.
-static std::string getSliceStr(const mlir::ComputationSliceState &sliceUnion) {
+static std::string
+getSliceStr(const affine::ComputationSliceState &sliceUnion) {
std::string result;
llvm::raw_string_ostream os(result);
// Slice insertion point format [loop-depth, operation-block-index]
unsigned i, unsigned j, unsigned loopDepth,
unsigned maxLoopDepth) {
for (unsigned d = loopDepth + 1; d <= maxLoopDepth; ++d) {
- mlir::ComputationSliceState sliceUnion;
- FusionResult result = mlir::canFuseLoops(forOpA, forOpB, d, &sliceUnion);
+ affine::ComputationSliceState sliceUnion;
+ FusionResult result = affine::canFuseLoops(forOpA, forOpB, d, &sliceUnion);
if (result.value == FusionResult::Success) {
forOpB->emitRemark("slice (")
<< " src loop: " << i << ", dst loop: " << j << ", depth: " << d
unsigned loopDepth,
unsigned maxLoopDepth) {
for (unsigned d = loopDepth + 1; d <= maxLoopDepth; ++d) {
- mlir::ComputationSliceState sliceUnion;
- FusionResult result = mlir::canFuseLoops(forOpA, forOpB, d, &sliceUnion);
+ affine::ComputationSliceState sliceUnion;
+ FusionResult result = affine::canFuseLoops(forOpA, forOpB, d, &sliceUnion);
if (result.value == FusionResult::Success) {
- mlir::fuseLoops(forOpA, forOpB, sliceUnion);
+ affine::fuseLoops(forOpA, forOpB, sliceUnion);
// Note: 'forOpA' is removed to simplify test output. A proper loop
// fusion pass should check the data dependence graph and run memref
// region analysis to ensure removing 'forOpA' is safe.
#include "mlir/Pass/Pass.h"
using namespace mlir;
+using namespace mlir::affine;
namespace {
struct TestLoopMappingPass
explicit TestLoopMappingPass() = default;
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, scf::SCFDialect>();
+ registry.insert<affine::AffineDialect, scf::SCFDialect>();
}
void runOnOperation() override {
#define PASS_NAME "test-loop-permutation"
using namespace mlir;
+using namespace mlir::affine;
namespace {
#define PASS_NAME "test-affine-reify-value-bounds"
using namespace mlir;
+using namespace mlir::affine;
using mlir::presburger::BoundType;
namespace {
TestReifyValueBounds(const TestReifyValueBounds &pass) : PassWrapper(pass){};
void getDependentDialects(DialectRegistry ®istry) const override {
- registry
- .insert<AffineDialect, tensor::TensorDialect, memref::MemRefDialect>();
+ registry.insert<affine::AffineDialect, tensor::TensorDialect,
+ memref::MemRefDialect>();
}
void runOnOperation() override;
#define DEBUG_TYPE "affine-super-vectorizer-test"
using namespace mlir;
+using namespace mlir::affine;
static llvm::cl::OptionCategory clOptionsCategory(DEBUG_TYPE " options");
void VectorizerTestPass::testVectorShapeRatio(llvm::raw_ostream &outs) {
auto f = getOperation();
- using matcher::Op;
+ using affine::matcher::Op;
SmallVector<int64_t, 8> shape(clTestVectorShapeRatio.begin(),
clTestVectorShapeRatio.end());
auto subVectorType =
auto filter = [&](Operation &op) {
assert(subVectorType.getElementType().isF32() &&
"Only f32 supported for now");
- if (!matcher::operatesOnSuperVectorsOf(op, subVectorType)) {
+ if (!mlir::matcher::operatesOnSuperVectorsOf(op, subVectorType)) {
return false;
}
if (op.getNumResults() != 1) {
}
static NestedPattern patternTestSlicingOps() {
- using matcher::Op;
+ using affine::matcher::Op;
// Match all operations with the kTestSlicingOpName name.
auto filter = [](Operation &op) {
// Just use a custom op name for this test, it makes life easier.
void VectorizerTestPass::testComposeMaps(llvm::raw_ostream &outs) {
auto f = getOperation();
- using matcher::Op;
+ using affine::matcher::Op;
auto pattern = Op(customOpWithAffineMapAttribute);
SmallVector<NestedMatch, 8> matches;
pattern.match(f, &matches);
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestGpuMemoryPromotionPass)
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, memref::MemRefDialect, scf::SCFDialect>();
+ registry.insert<affine::AffineDialect, memref::MemRefDialect,
+ scf::SCFDialect>();
}
StringRef getArgument() const final { return "test-gpu-memory-promotion"; }
StringRef getDescription() const final {
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestDataLayoutPropagationPass)
void getDependentDialects(DialectRegistry ®istry) const override {
- registry
- .insert<AffineDialect, linalg::LinalgDialect, tensor::TensorDialect>();
+ registry.insert<affine::AffineDialect, linalg::LinalgDialect,
+ tensor::TensorDialect>();
}
StringRef getArgument() const final {
TestLinalgDecomposeOps(const TestLinalgDecomposeOps &pass)
: PassWrapper(pass){};
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, linalg::LinalgDialect>();
+ registry.insert<affine::AffineDialect, linalg::LinalgDialect>();
}
StringRef getArgument() const final { return "test-linalg-decompose-ops"; }
StringRef getDescription() const final {
TestLinalgElementwiseFusion(const TestLinalgElementwiseFusion &pass)
: PassWrapper(pass) {}
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
- tensor::TensorDialect>();
+ registry.insert<affine::AffineDialect, linalg::LinalgDialect,
+ memref::MemRefDialect, tensor::TensorDialect>();
}
StringRef getArgument() const final {
return "test-linalg-elementwise-fusion-patterns";
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLinalgGreedyFusion)
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
- scf::SCFDialect>();
+ registry.insert<affine::AffineDialect, linalg::LinalgDialect,
+ memref::MemRefDialect, scf::SCFDialect>();
}
StringRef getArgument() const final { return "test-linalg-greedy-fusion"; }
StringRef getDescription() const final {
void getDependentDialects(DialectRegistry ®istry) const override {
// clang-format off
- registry.insert<AffineDialect,
+ registry.insert<affine::AffineDialect,
bufferization::BufferizationDialect,
memref::MemRefDialect,
scf::SCFDialect,
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestPadFusionPass)
void getDependentDialects(DialectRegistry ®istry) const override {
- registry
- .insert<AffineDialect, linalg::LinalgDialect, tensor::TensorDialect>();
+ registry.insert<affine::AffineDialect, linalg::LinalgDialect,
+ tensor::TensorDialect>();
}
StringRef getArgument() const final { return "test-linalg-pad-fusion"; }
void TestComposeSubViewPass::getDependentDialects(
DialectRegistry ®istry) const {
- registry.insert<AffineDialect>();
+ registry.insert<affine::AffineDialect>();
}
void TestComposeSubViewPass::runOnOperation() {
TestMultiBufferingPass(const TestMultiBufferingPass &pass)
: PassWrapper(pass) {}
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect>();
+ registry.insert<affine::AffineDialect>();
}
StringRef getArgument() const final { return "test-multi-buffering"; }
StringRef getDescription() const final {
}
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect>();
+ registry.insert<affine::AffineDialect>();
}
Option<bool> unroll{*this, "unroll", llvm::cl::desc("Include unrolling"),
TestVectorContractionPrepareForMMTLowering() = default;
void getDependentDialects(DialectRegistry ®istry) const override {
- registry
- .insert<AffineDialect, arith::ArithDialect, vector::VectorDialect>();
+ registry.insert<affine::AffineDialect, arith::ArithDialect,
+ vector::VectorDialect>();
}
void runOnOperation() override {
: PassWrapper(pass) {}
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect>();
+ registry.insert<affine::AffineDialect>();
}
StringRef getArgument() const final {
return "test-vector-transfer-unrolling-patterns";
TestScalarVectorTransferLoweringPatterns() = default;
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, memref::MemRefDialect, tensor::TensorDialect,
- vector::VectorDialect>();
+ registry.insert<affine::AffineDialect, memref::MemRefDialect,
+ tensor::TensorDialect, vector::VectorDialect>();
}
void runOnOperation() override {
const TestVectorTransferCollapseInnerMostContiguousDims &pass) = default;
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<memref::MemRefDialect, AffineDialect>();
+ registry.insert<memref::MemRefDialect, affine::AffineDialect>();
}
StringRef getArgument() const final {
void getDependentDialects(DialectRegistry ®istry) const override {
registry.insert<scf::SCFDialect, memref::MemRefDialect, gpu::GPUDialect,
- AffineDialect>();
+ affine::AffineDialect>();
}
StringRef getArgument() const final { return "test-vector-warp-distribute"; }
TestTilingInterfacePass(const TestTilingInterfacePass &pass)
: PassWrapper(pass) {}
void getDependentDialects(DialectRegistry ®istry) const override {
- registry.insert<AffineDialect, linalg::LinalgDialect, memref::MemRefDialect,
- scf::SCFDialect, tensor::TensorDialect>();
+ registry.insert<affine::AffineDialect, linalg::LinalgDialect,
+ memref::MemRefDialect, scf::SCFDialect,
+ tensor::TensorDialect>();
linalg::registerTilingInterfaceExternalModels(registry);
tensor::registerTilingInterfaceExternalModels(registry);
}
/// represents a valid IntegerSet.
inline IntegerPolyhedron parseIntegerPolyhedron(StringRef str) {
MLIRContext context(MLIRContext::Threading::DISABLED);
- return FlatAffineValueConstraints(parseIntegerSet(str, &context));
+ return affine::FlatAffineValueConstraints(parseIntegerSet(str, &context));
}
/// Parse a list of StringRefs to IntegerRelation and combine them into a
projects / platform / upstream / llvm.git / commitdiff