class FuncBuilder;
class MLIRContext;
class Operation;
-using Instruction = Operation;
class Type;
class Value;
}
/// Base class for the dialect op conversion patterns. Specific conversions
-/// must derive this class and implement `PatternMatch match(Instruction *)`
+/// must derive this class and implement `PatternMatch match(Operation *)`
/// defined in `Pattern` and at least one of `rewrite` and `rewriteTerminator`.
//
// TODO(zinenko): this should eventually converge with RewritePattern. So far,
/// DialectOpConversion ever needs to replace an operation that does not have
/// successors. This function should not fail. If some specific cases of the
/// operation are not supported, these cases should not be matched.
- virtual SmallVector<Value *, 4> rewrite(Instruction *op,
+ virtual SmallVector<Value *, 4> rewrite(Operation *op,
ArrayRef<Value *> operands,
FuncBuilder &rewriter) const {
llvm_unreachable("unimplemented rewrite, did you mean rewriteTerminator?");
/// successors. This function should not fail the pass. If some specific
/// cases of the operation are not supported, these cases should not be
/// matched.
- virtual void rewriteTerminator(Instruction *op,
+ virtual void rewriteTerminator(Operation *op,
ArrayRef<Value *> properOperands,
ArrayRef<Block *> destinations,
ArrayRef<ArrayRef<Value *>> operands,
class FuncBuilder;
class Value;
-/// Unrolls this for instruction completely if the trip count is known to be
+/// Unrolls this for operation completely if the trip count is known to be
/// constant. Returns failure otherwise.
LogicalResult loopUnrollFull(AffineForOp forOp);
-/// Unrolls this for instruction by the specified unroll factor. Returns failure
+/// Unrolls this for operation by the specified unroll factor. Returns failure
/// if the loop cannot be unrolled either due to restrictions or due to invalid
/// unroll factors.
LogicalResult loopUnrollByFactor(AffineForOp forOp, uint64_t unrollFactor);
SmallVectorImpl<Value *> *operands,
FuncBuilder *builder);
-/// Skew the instructions in the body of a 'affine.for' instruction with the
-/// specified instruction-wise shifts. The shifts are with respect to the
+/// Skew the operations in the body of a 'affine.for' operation with the
+/// specified operation-wise shifts. The shifts are with respect to the
/// original execution order, and are multiplied by the loop 'step' before being
/// applied.
LLVM_NODISCARD
FuncBuilder *getBuilder() { return builder; }
- Instruction *createOperation(const OperationState &state) override {
+ Operation *createOperation(const OperationState &state) override {
auto *result = builder->createOperation(state);
return result;
}
/// must override). It will be passed the function-wise state, common to all
/// matches, and the state returned by the `match` call, if any. The subclass
/// must use `rewriter` to modify the function.
- virtual void rewriteOpInst(Instruction *op,
+ virtual void rewriteOpInst(Operation *op,
MLFuncGlobalLoweringState *funcWiseState,
std::unique_ptr<PatternState> opState,
MLFuncLoweringRewriter *rewriter) const = 0;
FuncBuilder builder(f);
MLFuncLoweringRewriter rewriter(&builder);
- llvm::SmallVector<Instruction *, 16> ops;
- f->walk([&ops](Instruction *inst) { ops.push_back(inst); });
+ llvm::SmallVector<Operation *, 16> ops;
+ f->walk([&ops](Operation *op) { ops.push_back(op); });
- for (Instruction *inst : ops) {
+ for (Operation *op : ops) {
for (const auto &pattern : patterns) {
- builder.setInsertionPoint(inst);
- if (auto matchResult = pattern->match(inst)) {
- pattern->rewriteOpInst(inst, funcWiseState, std::move(*matchResult),
+ builder.setInsertionPoint(op);
+ if (auto matchResult = pattern->match(op)) {
+ pattern->rewriteOpInst(op, funcWiseState, std::move(*matchResult),
&rewriter);
break;
}
/// memory hierarchy.
FunctionPassBase *createPipelineDataTransferPass();
-/// Lowers affine control flow instructions (ForStmt, IfStmt and AffineApplyOp)
+/// Lowers affine control flow operations (ForStmt, IfStmt and AffineApplyOp)
/// to equivalent lower-level constructs (flow of basic blocks and arithmetic
/// primitives).
FunctionPassBase *createLowerAffinePass();
ArrayRef<Value *> extraIndices = {},
AffineMap indexRemap = AffineMap(),
ArrayRef<Value *> extraOperands = {},
- Instruction *domInstFilter = nullptr,
- Instruction *postDomInstFilter = nullptr);
+ Operation *domInstFilter = nullptr,
+ Operation *postDomInstFilter = nullptr);
/// Creates and inserts into 'builder' a new AffineApplyOp, with the number of
/// its results equal to the number of operands, as a composition
/// these will also be collected into a single (multi-result) affine apply op.
/// The final results of the composed AffineApplyOp are returned in output
/// parameter 'results'. Returns the affine apply op created.
-Instruction *createComposedAffineApplyOp(FuncBuilder *builder, Location loc,
- ArrayRef<Value *> operands,
- ArrayRef<Instruction *> affineApplyOps,
- SmallVectorImpl<Value *> *results);
+Operation *createComposedAffineApplyOp(FuncBuilder *builder, Location loc,
+ ArrayRef<Value *> operands,
+ ArrayRef<Operation *> affineApplyOps,
+ SmallVectorImpl<Value *> *results);
-/// Given an instruction, inserts one or more single result affine apply
-/// operations, results of which are exclusively used by this instruction.
+/// Given an operation, inserts one or more single result affine apply
+/// operations, results of which are exclusively used by this operation.
/// The operands of these newly created affine apply ops are
/// guaranteed to be loop iterators or terminal symbols of a function.
///
/// (i.e., there was no affine computation slice to create).
/// 2. If all the affine.apply op's supplying operands to this opInst did not
/// have any uses other than those in this opInst.
-void createAffineComputationSlice(Instruction *opInst,
+void createAffineComputationSlice(Operation *opInst,
SmallVectorImpl<AffineApplyOp> *sliceOps);
/// Replaces (potentially nested) function attributes in the operation "op"
/// with those specified in "remappingTable".
void remapFunctionAttrs(
-
Instruction &op, const DenseMap<Attribute, FunctionAttr> &remappingTable);
+
Operation &op, const DenseMap<Attribute, FunctionAttr> &remappingTable);
/// Replaces (potentially nested) function attributes all operations of the
/// Function "fn" with those specified in "remappingTable".
namespace {
// TODO(riverriddle) Handle commutative operations.
-struct SimpleOperationInfo : public llvm::DenseMapInfo<Instruction *> {
- static unsigned getHashValue(const Instruction *opC) {
- auto *op = const_cast<Instruction *>(opC);
+struct SimpleOperationInfo : public llvm::DenseMapInfo<Operation *> {
+ static unsigned getHashValue(const Operation *opC) {
+ auto *op = const_cast<Operation *>(opC);
// Hash the operations based upon their:
- // - Instruction Name
+ // - Operation Name
// - Attributes
// - Result Types
// - Operands
hash_combine_range(op->result_type_begin(), op->result_type_end()),
hash_combine_range(op->operand_begin(), op->operand_end()));
}
- static bool isEqual(const Instruction *lhsC, const Instruction *rhsC) {
- auto *lhs = const_cast<Instruction *>(lhsC);
- auto *rhs = const_cast<Instruction *>(rhsC);
+ static bool isEqual(const Operation *lhsC, const Operation *rhsC) {
+ auto *lhs = const_cast<Operation *>(lhsC);
+ auto *rhs = const_cast<Operation *>(rhsC);
if (lhs == rhs)
return true;
if (lhs == getTombstoneKey() || lhs == getEmptyKey() ||
/// Shared implementation of operation elimination and scoped map definitions.
using AllocatorTy = llvm::RecyclingAllocator<
llvm::BumpPtrAllocator,
- llvm::ScopedHashTableVal<Instruction *, Instruction *>>;
- using ScopedMapTy = llvm::ScopedHashTable<Instruction *, Instruction *,
+ llvm::ScopedHashTableVal<Operation *, Operation *>>;
+ using ScopedMapTy = llvm::ScopedHashTable<Operation *, Operation *,
SimpleOperationInfo, AllocatorTy>;
/// Represents a single entry in the depth first traversal of a CFG.
/// Attempt to eliminate a redundant operation. Returns true if the operation
/// was marked for removal, false otherwise.
- bool simplifyOperation(Instruction *op);
+ bool simplifyOperation(Operation *op);
void simplifyBlock(DominanceInfo &domInfo, Block *bb);
void simplifyRegion(DominanceInfo &domInfo, Region ®ion);
ScopedMapTy knownValues;
/// Operations marked as dead and to be erased.
- std::vector<Instruction *> opsToErase;
+ std::vector<Operation *> opsToErase;
};
} // end anonymous namespace
/// Attempt to eliminate a redundant operation.
-bool CSE::simplifyOperation(Instruction *op) {
+bool CSE::simplifyOperation(Operation *op) {
// Don't simplify operations with nested blocks. We don't currently model
// equality comparisons correctly among other things. It is also unclear
// whether we would want to CSE such operations.
// All constants in the function post folding.
SmallVector<Value *, 8> existingConstants;
// Operations that were folded and that need to be erased.
- std::vector<Instruction *> opInstsToErase;
+ std::vector<Operation *> opInstsToErase;
- void foldInstruction(Instruction *op);
+ void foldOperation(Operation *op);
void runOnFunction() override;
};
} // end anonymous namespace
/// Attempt to fold the specified operation, updating the IR to match. If
/// constants are found, we keep track of them in the existingConstants list.
///
-void ConstantFold::foldInstruction(Instruction *op) {
+void ConstantFold::foldOperation(Operation *op) {
// If this operation is already a constant, just remember it for cleanup
// later, and don't try to fold it.
if (auto constant = op->dyn_cast<ConstantOp>()) {
existingConstants.clear();
opInstsToErase.clear();
- getFunction().walk([&](Instruction *inst) { foldInstruction(inst); });
+ getFunction().walk([&](Operation *op) { foldOperation(op); });
// At this point, these operations are dead, remove them.
// TODO: This is assuming that all constant foldable operations have no
// side effects. When we have side effect modeling, we should verify that
// the operation is effect-free before we remove it. Until then this is
// close enough.
- for (auto *inst : opInstsToErase) {
- inst->erase();
+ for (auto *op : opInstsToErase) {
+ op->erase();
}
// By the time we are done, we may have simplified a bunch of code, leaving
// Utility that looks up a list of value in the value remapping table. Returns
// an empty vector if one of the values is not mapped yet.
SmallVector<Value *, 4> lookupValues(
- const llvm::iterator_range<Instruction::operand_iterator> &operands);
+ const llvm::iterator_range<Operation::operand_iterator> &operands);
// Converts the given function to the dialect using hooks defined in
// `dialectConversion`. Returns the converted function or `nullptr` on error.
// passes them to `converter->rewriteTerminator` function defined in the
// pattern, together with `builder`.
LogicalResult convertOpWithSuccessors(DialectOpConversion *converter,
- Instruction *op, FuncBuilder &builder);
+ Operation *op, FuncBuilder &builder);
// Converts an operation without successors. Extracts the converted operands
// from `valueRemapping` and passes them to the `converter->rewrite` function
// defined in the pattern, together with `builder`.
- LogicalResult convertOp(DialectOpConversion *converter, Instruction *op,
+ LogicalResult convertOp(DialectOpConversion *converter, Operation *op,
FuncBuilder &builder);
- // Converts a block by traversing its instructions sequentially, looking for
- // the first pattern match and dispatching the instruction conversion to
+ // Converts a block by traversing its operations sequentially, looking for
+ // the first pattern match and dispatching the operation conversion to
// either `convertOp` or `convertOpWithSuccessors` depending on the presence
// of successors. If there is no match, clones the operation.
//
} // end namespace mlir
SmallVector<Value *, 4> impl::FunctionConversion::lookupValues(
- const llvm::iterator_range<Instruction::operand_iterator> &operands) {
+ const llvm::iterator_range<Operation::operand_iterator> &operands) {
SmallVector<Value *, 4> remapped;
remapped.reserve(llvm::size(operands));
for (Value *operand : operands) {
}
LogicalResult impl::FunctionConversion::convertOpWithSuccessors(
- DialectOpConversion *converter, Instruction *op, FuncBuilder &builder) {
+ DialectOpConversion *converter, Operation *op, FuncBuilder &builder) {
SmallVector<Block *, 2> destinations;
destinations.reserve(op->getNumSuccessors());
SmallVector<Value *, 4> operands = lookupValues(op->getOperands());
LogicalResult
impl::FunctionConversion::convertOp(DialectOpConversion *converter,
- Instruction *op, FuncBuilder &builder) {
+ Operation *op, FuncBuilder &builder) {
auto operands = lookupValues(op->getOperands());
assert((!operands.empty() || op->getNumOperands() == 0) &&
"converting op before ops defining its operands");
builder.setInsertionPointToStart(mapping.lookupOrNull(block));
// Iterate over ops and convert them.
- for (Instruction &inst : *block) {
- if (inst.getNumRegions() != 0) {
- inst.emitError("unsupported region instruction");
+ for (Operation &op : *block) {
+ if (op.getNumRegions() != 0) {
+ op.emitError("unsupported region operation");
return failure();
}
// Find the first matching conversion and apply it.
bool converted = false;
for (auto *conversion : conversions) {
- if (!conversion->match(&inst))
+ if (!conversion->match(&op))
continue;
- if (inst.getNumSuccessors() != 0) {
- if (failed(convertOpWithSuccessors(conversion, &inst, builder)))
+ if (op.getNumSuccessors() != 0) {
+ if (failed(convertOpWithSuccessors(conversion, &op, builder)))
return failure();
- } else if (failed(convertOp(conversion, &inst, builder))) {
+ } else if (failed(convertOp(conversion, &op, builder))) {
return failure();
}
converted = true;
}
// If there is no conversion provided for the op, clone the op as is.
if (!converted)
- builder.clone(inst, mapping);
+ builder.clone(op, mapping);
}
// Recurse to children unless they have been already visited.
/// dynamic shaped memref's for now. `numParamLoopIVs` is the number of
/// enclosing loop IVs of opInst (starting from the outermost) that the region
/// is parametric on.
-static bool getFullMemRefAsRegion(Instruction *opInst, unsigned numParamLoopIVs,
+static bool getFullMemRefAsRegion(Operation *opInst, unsigned numParamLoopIVs,
MemRefRegion *region) {
unsigned rank;
if (auto loadOp = opInst->dyn_cast<LoadOp>()) {
}
static void emitNoteForBlock(Block &block, const Twine &message) {
- auto *inst = block.getContainingOp();
- if (!inst) {
+ auto *op = block.getContainingOp();
+ if (!op) {
block.getFunction()->emitNote(message);
} else {
- inst->emitNote(message);
+ op->emitNote(message);
}
}
fastBufferShape, memRefType.getElementType(), {}, fastMemorySpace);
// Create the fast memory space buffer just before the 'affine.for'
- // instruction.
+ // operation.
fastMemRef = prologue.create<AllocOp>(loc, fastMemRefType).getResult();
// Record it.
fastBufferMap[memref] = fastMemRef;
top.create<ConstantIndexOp>(loc, strideInfos[0].numEltPerStride);
}
- // Record the last instruction just before the point where we insert the
+ // Record the last operation just before the point where we insert the
// outgoing DMAs. We later do the memref replacement later only in [begin,
// postDomFilter] so that the original memref's in the DMA ops themselves
// don't get replaced.
}
/// Generate DMAs for this block. The block is partitioned into separate
-/// `regions`; each region is either a sequence of one or more instructions
+/// `regions`; each region is either a sequence of one or more operations
/// starting and ending with a load or store op, or just a loop (which could
/// have other loops nested within). Returns false on an error, true otherwise.
bool DmaGeneration::runOnBlock(Block *block) {
return true;
// Every loop in the block starts and ends a region. A contiguous sequence of
- // operation instructions starting and ending with a load/store op is also
+ // operations starting and ending with a load/store op is also
// identified as a region. Straightline code (contiguous chunks of operation
- // instructions) are always assumed to not exhaust memory. As a result, this
+ // operations) are always assumed to not exhaust memory. As a result, this
// approach is conservative in some cases at the moment, we do a check later
// and report an error with location info.
- // TODO(bondhugula): An 'affine.if' instruction is being treated similar to an
- // operation instruction. 'affine.if''s could have 'affine.for's in them;
+ // TODO(bondhugula): An 'affine.if' operation is being treated similar to an
+ // operation. 'affine.if''s could have 'affine.for's in them;
// treat them separately.
// Get to the first load, store, or for op.
auto curBegin =
- std::find_if(block->begin(), block->end(), [&](Instruction &inst) {
- return inst.isa<LoadOp>() || inst.isa<StoreOp>() ||
- inst.isa<AffineForOp>();
+ std::find_if(block->begin(), block->end(), [&](Operation &op) {
+ return op.isa<LoadOp>() || op.isa<StoreOp>() || op.isa<AffineForOp>();
});
for (auto it = curBegin; it != block->end(); ++it) {
runOnBlock(/*begin=*/curBegin, /*end=*/it);
// Recurse onto the body of this loop.
runOnBlock(forOp.getBody());
- // The next region starts right after the 'affine.for' instruction.
+ // The next region starts right after the 'affine.for' operation.
curBegin = std::next(it);
} else {
// We have enough capacity, i.e., DMAs will be computed for the portion
}
}
-/// Generates DMAs for a contiguous sequence of instructions in `block` in the
+/// Generates DMAs for a contiguous sequence of operations in `block` in the
/// iterator range [begin, end). Returns the total size of the DMA buffers used.
// Since we generate alloc's and dealloc's for all DMA buffers (before and
-// after the range of instructions resp), all of the fast memory capacity is
+// after the range of operations resp), all of the fast memory capacity is
// assumed to be available.
uint64_t DmaGeneration::runOnBlock(Block::iterator begin, Block::iterator end) {
if (begin == end)
// To check for errors when walking the block.
bool error = false;
- // Walk this range of instructions to gather all memory regions.
- block->walk(begin, end, [&](Instruction *opInst) {
+ // Walk this range of operations to gather all memory regions.
+ block->walk(begin, end, [&](Operation *opInst) {
// Gather regions to allocate to buffers in faster memory space.
if (auto loadOp = opInst->dyn_cast<LoadOp>()) {
if (loadOp.getMemRefType().getMemorySpace() != slowMemorySpace)
return totalDmaBuffersSizeInBytes;
}
- // For a range of operation instructions, a note will be emitted at the
- // caller.
+ // For a range of operations, a note will be emitted at the caller.
AffineForOp forOp;
uint64_t sizeInKib = llvm::divideCeil(totalDmaBuffersSizeInBytes, 1024);
if (llvm::DebugFlag && (forOp = begin->dyn_cast<AffineForOp>())) {
if (totalDmaBuffersSizeInBytes > fastMemCapacityBytes) {
StringRef str = "Total size of all DMA buffers' for this block "
"exceeds fast memory capacity\n";
- if (auto *inst = block->getContainingOp())
- inst->emitError(str);
+ if (auto *op = block->getContainingOp())
+ op->emitError(str);
else
block->getFunction()->emitError(str);
}
// operations, and whether or not an IfInst was encountered in the loop nest.
struct LoopNestStateCollector {
SmallVector<AffineForOp, 4> forOps;
- SmallVector<Instruction *, 4> loadOpInsts;
- SmallVector<Instruction *, 4> storeOpInsts;
+ SmallVector<Operation *, 4> loadOpInsts;
+ SmallVector<Operation *, 4> storeOpInsts;
bool hasNonForRegion = false;
- void collect(Instruction *instToWalk) {
- instToWalk->walk([&](Instruction *opInst) {
- if (op
Inst->isa<AffineForOp>())
- forOps.push_back(op
Inst->cast<AffineForOp>());
- else if (opInst->getNumRegions() != 0)
+ void collect(Operation *opToWalk) {
+ opToWalk->walk([&](Operation *op) {
+ if (op->isa<AffineForOp>())
+ forOps.push_back(op->cast<AffineForOp>());
+ else if (op->getNumRegions() != 0)
hasNonForRegion = true;
- else if (opInst->isa<LoadOp>())
- loadOpInsts.push_back(opInst);
- else if (opInst->isa<StoreOp>())
- storeOpInsts.push_back(opInst);
+ else if (op->isa<LoadOp>())
+ loadOpInsts.push_back(op);
+ else if (op->isa<StoreOp>())
+ storeOpInsts.push_back(op);
});
}
};
// TODO(b/117228571) Replace when this is modeled through side-effects/op traits
-static bool isMemRefDereferencingOp(Instruction &op) {
+static bool isMemRefDereferencingOp(Operation &op) {
if (op.isa<LoadOp>() || op.isa<StoreOp>() || op.isa<DmaStartOp>() ||
op.isa<DmaWaitOp>())
return true;
}
// MemRefDependenceGraph is a graph data structure where graph nodes are
-// top-level instructions in a Function which contain load/store ops, and edges
+// top-level operations in a Function which contain load/store ops, and edges
// are memref dependences between the nodes.
// TODO(andydavis) Add a more flexible dependece graph representation.
// TODO(andydavis) Add a depth parameter to dependence graph construction.
// The unique identifier of this node in the graph.
unsigned id;
// The top-level statment which is (or contains) loads/stores.
- Instruction *inst;
+ Operation *op;
// List of load operations.
- SmallVector<Instruction *, 4> loads;
+ SmallVector<Operation *, 4> loads;
// List of store op insts.
- SmallVector<Instruction *, 4> stores;
- Node(unsigned id, Instruction *inst) : id(id), inst(inst) {}
+ SmallVector<Operation *, 4> stores;
+ Node(unsigned id, Operation *op) : id(id), op(op) {}
// Returns the load op count for 'memref'.
unsigned getLoadOpCount(Value *memref) {
// Returns all store ops in 'storeOps' which access 'memref'.
void getStoreOpsForMemref(Value *memref,
- SmallVectorImpl<Instruction *> *storeOps) {
+ SmallVectorImpl<Operation *> *storeOps) {
for (auto *storeOpInst : stores) {
if (memref == storeOpInst->cast<StoreOp>().getMemRef())
storeOps->push_back(storeOpInst);
// Returns all load ops in 'loadOps' which access 'memref'.
void getLoadOpsForMemref(Value *memref,
- SmallVectorImpl<Instruction *> *loadOps) {
+ SmallVectorImpl<Operation *> *loadOps) {
for (auto *loadOpInst : loads) {
if (memref == loadOpInst->cast<LoadOp>().getMemRef())
loadOps->push_back(loadOpInst);
// which contain accesses to the same memref 'value'. If the value is a
// non-memref value, then the dependence is between a graph node which
// defines an SSA value and another graph node which uses the SSA value
- // (e.g. a constant instruction defining a value which is used inside a loop
+ // (e.g. a constant operation defining a value which is used inside a loop
// nest).
Value *value;
};
return &it->second;
}
- // Adds a node with 'inst' to the graph and returns its unique identifier.
- unsigned addNode(Instruction *inst) {
- Node node(nextNodeId++, inst);
+ // Adds a node with 'op' to the graph and returns its unique identifier.
+ unsigned addNode(Operation *op) {
+ Node node(nextNodeId++, op);
nodes.insert({node.id, node});
return node.id;
}
Node *node = getNode(id);
for (auto *storeOpInst : node->stores) {
auto *memref = storeOpInst->cast<StoreOp>().getMemRef();
- auto *inst = memref->getDefiningOp();
+ auto *op = memref->getDefiningOp();
// Return true if 'memref' is a block argument.
- if (!inst)
+ if (!op)
return true;
// Return true if any use of 'memref' escapes the function.
for (auto &use : memref->getUses())
return outEdgeCount;
}
- // Computes and returns an insertion point instruction, before which the
+ // Computes and returns an insertion point operation, before which the
// the fused <srcId, dstId> loop nest can be inserted while preserving
// dependences. Returns nullptr if no such insertion point is found.
- Instruction *getFusedLoopNestInsertionPoint(unsigned srcId, unsigned dstId) {
+ Operation *getFusedLoopNestInsertionPoint(unsigned srcId, unsigned dstId) {
if (outEdges.count(srcId) == 0)
- return getNode(dstId)->inst;
+ return getNode(dstId)->op;
// Build set of insts in range (srcId, dstId) which depend on 'srcId'.
- SmallPtrSet<Instruction *, 2> srcDepInsts;
+ SmallPtrSet<Operation *, 2> srcDepInsts;
for (auto &outEdge : outEdges[srcId])
if (outEdge.id != dstId)
- srcDepInsts.insert(getNode(outEdge.id)->inst);
+ srcDepInsts.insert(getNode(outEdge.id)->op);
// Build set of insts in range (srcId, dstId) on which 'dstId' depends.
- SmallPtrSet<Instruction *, 2> dstDepInsts;
+ SmallPtrSet<Operation *, 2> dstDepInsts;
for (auto &inEdge : inEdges[dstId])
if (inEdge.id != srcId)
- dstDepInsts.insert(getNode(inEdge.id)->inst);
+ dstDepInsts.insert(getNode(inEdge.id)->op);
- Instruction *srcNodeInst = getNode(srcId)->inst;
- Instruction *dstNodeInst = getNode(dstId)->inst;
+ Operation *srcNodeInst = getNode(srcId)->op;
+ Operation *dstNodeInst = getNode(dstId)->op;
// Computing insertion point:
- // *) Walk all instruction positions in Block instruction list in the
- // range (src, dst). For each instruction 'inst' visited in this search:
- // *) Store in 'firstSrcDepPos' the first position where 'inst' has a
+ // *) Walk all operation positions in Block operation list in the
+ // range (src, dst). For each operation 'op' visited in this search:
+ // *) Store in 'firstSrcDepPos' the first position where 'op' has a
// dependence edge from 'srcNode'.
- // *) Store in 'lastDstDepPost' the last position where 'inst' has a
+ // *) Store in 'lastDstDepPost' the last position where 'op' has a
// dependence edge to 'dstNode'.
// *) Compare 'firstSrcDepPos' and 'lastDstDepPost' to determine the
- // instruction insertion point (or return null pointer if no such
+ // operation insertion point (or return null pointer if no such
// insertion point exists: 'firstSrcDepPos' <= 'lastDstDepPos').
- SmallVector<Instruction *, 2> depInsts;
+ SmallVector<Operation *, 2> depInsts;
Optional<unsigned> firstSrcDepPos;
Optional<unsigned> lastDstDepPos;
unsigned pos = 0;
for (Block::iterator it = std::next(Block::iterator(srcNodeInst));
it != Block::iterator(dstNodeInst); ++it) {
- Instruction *inst = &(*it);
- if (srcDepInsts.count(inst) > 0 && firstSrcDepPos == None)
+ Operation *op = &(*it);
+ if (srcDepInsts.count(op) > 0 && firstSrcDepPos == None)
firstSrcDepPos = pos;
- if (dstDepInsts.count(inst) > 0)
+ if (dstDepInsts.count(op) > 0)
lastDstDepPos = pos;
- depInsts.push_back(inst);
+ depInsts.push_back(op);
++pos;
}
}
// Adds ops in 'loads' and 'stores' to node at 'id'.
- void addToNode(unsigned id, const SmallVectorImpl<Instruction *> &loads,
- const SmallVectorImpl<Instruction *> &stores) {
+ void addToNode(unsigned id, const SmallVectorImpl<Operation *> &loads,
+ const SmallVectorImpl<Operation *> &stores) {
Node *node = getNode(id);
for (auto *loadOpInst : loads)
node->loads.push_back(loadOpInst);
continue;
assert(nodes.count(edge.id) > 0);
// Skip if 'edge.id' is not a loop nest.
- if (!getNode(edge.id)->
inst->isa<AffineForOp>())
+ if (!getNode(edge.id)->
op->isa<AffineForOp>())
continue;
// Visit current input edge 'edge'.
callback(edge);
void dump() const { print(llvm::errs()); }
};
-// Intializes the data dependence graph by walking instructions in 'f'.
+// Intializes the data dependence graph by walking operations in 'f'.
// Assigns each node in the graph a node id based on program order in 'f'.
// TODO(andydavis) Add support for taking a Block arg to construct the
// dependence graph at a different depth.
if (f.getBlocks().size() != 1)
return false;
- DenseMap<Instruction *, unsigned> forToNodeMap;
- for (auto &inst : f.front()) {
- if (auto forOp =
inst.dyn_cast<AffineForOp>()) {
+ DenseMap<Operation *, unsigned> forToNodeMap;
+ for (auto &op : f.front()) {
+ if (auto forOp =
op.dyn_cast<AffineForOp>()) {
// Create graph node 'id' to represent top-level 'forOp' and record
// all loads and store accesses it contains.
LoopNestStateCollector collector;
- collector.collect(&inst);
+ collector.collect(&op);
// Return false if a non 'affine.for' region was found (not currently
// supported).
if (collector.hasNonForRegion)
return false;
- Node node(nextNodeId++, &inst);
+ Node node(nextNodeId++, &op);
for (auto *opInst : collector.loadOpInsts) {
node.loads.push_back(opInst);
auto *memref = opInst->cast<LoadOp>().getMemRef();
auto *memref = opInst->cast<StoreOp>().getMemRef();
memrefAccesses[memref].insert(node.id);
}
- forToNodeMap[&inst] = node.id;
+ forToNodeMap[&op] = node.id;
nodes.insert({node.id, node});
- } else if (auto loadOp = inst.dyn_cast<LoadOp>()) {
+ } else if (auto loadOp = op.dyn_cast<LoadOp>()) {
// Create graph node for top-level load op.
- Node node(nextNodeId++, &inst);
- node.loads.push_back(&inst);
- auto *memref = inst.cast<LoadOp>().getMemRef();
+ Node node(nextNodeId++, &op);
+ node.loads.push_back(&op);
+ auto *memref = op.cast<LoadOp>().getMemRef();
memrefAccesses[memref].insert(node.id);
nodes.insert({node.id, node});
- } else if (auto storeOp = inst.dyn_cast<StoreOp>()) {
+ } else if (auto storeOp = op.dyn_cast<StoreOp>()) {
// Create graph node for top-level store op.
- Node node(nextNodeId++, &inst);
- node.stores.push_back(&inst);
- auto *memref = inst.cast<StoreOp>().getMemRef();
+ Node node(nextNodeId++, &op);
+ node.stores.push_back(&op);
+ auto *memref = op.cast<StoreOp>().getMemRef();
memrefAccesses[memref].insert(node.id);
nodes.insert({node.id, node});
- } else if (inst.getNumRegions() != 0) {
+ } else if (op.getNumRegions() != 0) {
// Return false if another region is found (not currently supported).
return false;
- } else if (inst.getNumResults() > 0 && !inst.use_empty()) {
+ } else if (op.getNumResults() > 0 && !op.use_empty()) {
// Create graph node for top-level producer of SSA values, which
// could be used by loop nest nodes.
- Node node(nextNodeId++, &inst);
+ Node node(nextNodeId++, &op);
nodes.insert({node.id, node});
}
}
const Node &node = idAndNode.second;
if (!node.loads.empty() || !node.stores.empty())
continue;
- auto *opInst = node.inst;
+ auto *opInst = node.op;
for (auto *value : opInst->getResults()) {
for (auto &use : value->getUses()) {
SmallVector<AffineForOp, 4> loops;
// and operation count) for a loop nest up until the innermost loop body.
struct LoopNestStats {
// Map from AffineForOp to immediate child AffineForOps in its loop body.
- DenseMap<
Instruction *, SmallVector<AffineForOp, 2>> loopMap;
+ DenseMap<
Operation *, SmallVector<AffineForOp, 2>> loopMap;
// Map from AffineForOp to count of operations in its loop body.
- DenseMap<Instruction *, uint64_t> opCountMap;
+ DenseMap<Operation *, uint64_t> opCountMap;
// Map from AffineForOp to its constant trip count.
- DenseMap<Instruction *, uint64_t> tripCountMap;
+ DenseMap<Operation *, uint64_t> tripCountMap;
};
// LoopNestStatsCollector walks a single loop nest and gathers per-loop
LoopNestStatsCollector(LoopNestStats *stats) : stats(stats) {}
- void collect(Instruction *inst) {
-
inst->walk<AffineForOp>([&](AffineForOp forOp) {
+ void collect(Operation *op) {
+
op->walk<AffineForOp>([&](AffineForOp forOp) {
auto *forInst = forOp.getOperation();
auto *parentInst = forOp.getOperation()->getParentOp();
if (parentInst != nullptr) {
stats->loopMap[parentInst].push_back(forOp);
}
- // Record the number of op instructions in the body of 'forOp'.
+ // Record the number of op operations in the body of 'forOp'.
unsigned count = 0;
stats->opCountMap[forInst] = 0;
- for (auto &inst : *forOp.getBody()) {
- if (!
inst.isa<AffineForOp>() && !inst.isa<AffineIfOp>())
+ for (auto &op : *forOp.getBody()) {
+ if (!
op.isa<AffineForOp>() && !op.isa<AffineIfOp>())
++count;
}
stats->opCountMap[forInst] = count;
// NOTE: this is used to compute the cost of fusing a slice of some loop nest
// within another loop.
static int64_t getComputeCost(
- Instruction *forInst, LoopNestStats *stats,
- llvm::SmallDenseMap<Instruction *, uint64_t, 8> *tripCountOverrideMap,
- DenseMap<Instruction *, int64_t> *computeCostMap) {
+ Operation *forInst, LoopNestStats *stats,
+ llvm::SmallDenseMap<Operation *, uint64_t, 8> *tripCountOverrideMap,
+ DenseMap<Operation *, int64_t> *computeCostMap) {
// 'opCount' is the total number operations in one iteration of 'forOp' body
int64_t opCount = stats->opCountMap[forInst];
if (stats->loopMap.count(forInst) > 0) {
// was encountered).
// TODO(andydavis) Make this work with non-unit step loops.
static bool buildSliceTripCountMap(
- Instruction *srcOpInst, ComputationSliceState *sliceState,
- llvm::SmallDenseMap<Instruction *, uint64_t, 8> *tripCountMap) {
+ Operation *srcOpInst, ComputationSliceState *sliceState,
+ llvm::SmallDenseMap<Operation *, uint64_t, 8> *tripCountMap) {
SmallVector<AffineForOp, 4> srcLoopIVs;
getLoopIVs(*srcOpInst, &srcLoopIVs);
unsigned numSrcLoopIVs = srcLoopIVs.size();
// Removes load operations from 'srcLoads' which operate on 'memref', and
// adds them to 'dstLoads'.
-static void
-moveLoadsAccessingMemrefTo(Value *memref,
- SmallVectorImpl<Instruction *> *srcLoads,
- SmallVectorImpl<Instruction *> *dstLoads) {
+static void moveLoadsAccessingMemrefTo(Value *memref,
+ SmallVectorImpl<Operation *> *srcLoads,
+ SmallVectorImpl<Operation *> *dstLoads) {
dstLoads->clear();
- SmallVector<Instruction *, 4> srcLoadsToKeep;
+ SmallVector<Operation *, 4> srcLoadsToKeep;
for (auto *load : *srcLoads) {
if (load->cast<LoadOp>().getMemRef() == memref)
dstLoads->push_back(load);
}
// Returns the innermost common loop depth for the set of operations in 'ops'.
-static unsigned getInnermostCommonLoopDepth(ArrayRef<Instruction *> ops) {
+static unsigned getInnermostCommonLoopDepth(ArrayRef<Operation *> ops) {
unsigned numOps = ops.size();
assert(numOps > 0);
// Returns the maximum loop depth at which no dependences between 'loadOpInsts'
// and 'storeOpInsts' are satisfied.
-static unsigned getMaxLoopDepth(ArrayRef<Instruction *> loadOpInsts,
- ArrayRef<Instruction *> storeOpInsts) {
+static unsigned getMaxLoopDepth(ArrayRef<Operation *> loadOpInsts,
+ ArrayRef<Operation *> storeOpInsts) {
// Merge loads and stores into the same array.
- SmallVector<Instruction *, 2> ops(loadOpInsts.begin(), loadOpInsts.end());
+ SmallVector<Operation *, 2> ops(loadOpInsts.begin(), loadOpInsts.end());
ops.append(storeOpInsts.begin(), storeOpInsts.end());
// Compute the innermost common loop depth for loads and stores.
// dependence componenent lexicographically negative.
// TODO(andydavis) Move this function to LoopUtils.
static bool
-computeLoopInterchangePermutation(ArrayRef<Instruction *> ops,
+computeLoopInterchangePermutation(ArrayRef<Operation *> ops,
unsigned maxLoopDepth,
SmallVectorImpl<unsigned> *loopPermMap) {
// Gather dependence components for dependences between all ops in 'ops'
// This can increase the loop depth at which we can fuse a slice, since we are
// pushing loop carried dependence to a greater depth in the loop nest.
static void sinkSequentialLoops(MemRefDependenceGraph::Node *node) {
- assert(node->
inst->isa<AffineForOp>());
+ assert(node->
op->isa<AffineForOp>());
// Get perfectly nested sequence of loops starting at root of loop nest
// (the first op being another AffineFor, and the second op - a terminator).
// TODO(andydavis,bondhugula) Share this with similar code in loop tiling.
SmallVector<AffineForOp, 4> loops;
- AffineForOp curr = node->
inst->cast<AffineForOp>();
+ AffineForOp curr = node->
op->cast<AffineForOp>();
loops.push_back(curr);
auto *currBody = curr.getBody();
while (currBody->begin() == std::prev(currBody->end(), 2) &&
return;
// Merge loads and stores into the same array.
- SmallVector<Instruction *, 2> memOps(node->loads.begin(), node->loads.end());
+ SmallVector<Operation *, 2> memOps(node->loads.begin(), node->loads.end());
memOps.append(node->stores.begin(), node->stores.end());
// Compute loop permutation in 'loopPermMap'.
}
}
assert(loopNestRootIndex != -1 && "invalid root index");
- node->inst = loops[loopNestRootIndex].getOperation();
+ node->op = loops[loopNestRootIndex].getOperation();
}
// TODO(mlir-team): improve/complete this when we have target data.
// MemRefRegion written to by 'srcStoreOpInst' at depth 'dstLoopDepth'.
// TODO(bondhugula): consider refactoring the common code from generateDma and
// this one.
-static Value *createPrivateMemRef(AffineForOp forOp,
- Instruction *srcStoreOpInst,
+static Value *createPrivateMemRef(AffineForOp forOp, Operation *srcStoreOpInst,
unsigned dstLoopDepth,
Optional<unsigned> fastMemorySpace,
uint64_t localBufSizeThreshold) {
// Does the slice have a single iteration?
static uint64_t getSliceIterationCount(
- const llvm::SmallDenseMap<Instruction *, uint64_t, 8> &sliceTripCountMap) {
+ const llvm::SmallDenseMap<Operation *, uint64_t, 8> &sliceTripCountMap) {
uint64_t iterCount = 1;
for (const auto &count : sliceTripCountMap) {
iterCount *= count.second;
return false;
// Compute MemRefRegion 'dstWriteRegion' for 'dstStoreOpInst' on 'memref'.
- SmallVector<Instruction *, 2> dstStoreOps;
+ SmallVector<Operation *, 2> dstStoreOps;
dstNode->getStoreOpsForMemref(memref, &dstStoreOps);
assert(dstStoreOps.size() == 1);
auto *dstStoreOpInst = dstStoreOps[0];
// and each load op in 'dstLoadOpInsts' at 'dstLoopDepth', and returns
// the union in 'sliceState'. Returns true on success, false otherwise.
// TODO(andydavis) Move this to a loop fusion utility function.
-static bool getSliceUnion(Instruction *srcOpInst,
- ArrayRef<Instruction *> dstLoadOpInsts,
+static bool getSliceUnion(Operation *srcOpInst,
+ ArrayRef<Operation *> dstLoadOpInsts,
unsigned numSrcLoopIVs, unsigned dstLoopDepth,
ComputationSliceState *sliceState) {
MemRefAccess srcAccess(srcOpInst);
// *) Compares the total cost of the unfused loop nests to the min cost fused
// loop nest computed in the previous step, and returns true if the latter
// is lower.
-static bool isFusionProfitable(Instruction *srcOpInst,
- Instruction *srcStoreOpInst,
- ArrayRef<Instruction *> dstLoadOpInsts,
- ArrayRef<Instruction *> dstStoreOpInsts,
+static bool isFusionProfitable(Operation *srcOpInst, Operation *srcStoreOpInst,
+ ArrayRef<Operation *> dstLoadOpInsts,
+ ArrayRef<Operation *> dstStoreOpInsts,
ComputationSliceState *sliceState,
unsigned *dstLoopDepth, bool maximalFusion) {
LLVM_DEBUG({
MemRefRegion srcWriteRegion(srcStoreOpInst->getLoc());
if (failed(srcWriteRegion.compute(srcStoreOpInst, /*loopDepth=*/0))) {
LLVM_DEBUG(llvm::dbgs()
- << "Unable to compute MemRefRegion for source instruction\n.");
+ << "Unable to compute MemRefRegion for source operation\n.");
return false;
}
// Evaluate all depth choices for materializing the slice in the destination
// loop nest.
- llvm::SmallDenseMap<Instruction *, uint64_t, 8> sliceTripCountMap;
- DenseMap<Instruction *, int64_t> computeCostMap;
+ llvm::SmallDenseMap<Operation *, uint64_t, 8> sliceTripCountMap;
+ DenseMap<Operation *, int64_t> computeCostMap;
for (unsigned i = maxDstLoopDepth; i >= 1; --i) {
// Compute the union of slice bounds of all ops in 'dstLoadOpInsts'.
if (!getSliceUnion(srcOpInst, dstLoadOpInsts, numSrcLoopIVs, i,
// bounds to be functions of 'dstLoopNest' IVs and symbols.
// *) Fuse the 'srcLoopNest' computation slice into the 'dstLoopNest',
// at a loop depth determined by the cost model in 'isFusionProfitable'.
-// *) Add the newly fused load/store operation instructions to the state,
+// *) Add the newly fused load/store operations to the state,
// and also add newly fuse load ops to 'dstLoopOps' to be considered
// as fusion dst load ops in another iteration.
// *) Remove old src loop nest and its associated state.
// is preserved in the fused loop nest.
// *) Update graph state to reflect the fusion of 'sibNode' into 'dstNode'.
//
-// Given a graph where top-level instructions are vertices in the set 'V' and
+// Given a graph where top-level operations are vertices in the set 'V' and
// edges in the set 'E' are dependences between vertices, this algorithm
// takes O(V) time for initialization, and has runtime O(V + E).
//
// Get 'dstNode' into which to attempt fusion.
auto *dstNode = mdg->getNode(dstId);
// Skip if 'dstNode' is not a loop nest.
- if (!dstNode->
inst->isa<AffineForOp>())
+ if (!dstNode->
op->isa<AffineForOp>())
continue;
// Sink sequential loops in 'dstNode' (and thus raise parallel loops)
// while preserving relative order. This can increase the maximum loop
// consumer loop nest.
sinkSequentialLoops(dstNode);
- SmallVector<Instruction *, 4> loads = dstNode->loads;
- SmallVector<Instruction *, 4> dstLoadOpInsts;
+ SmallVector<Operation *, 4> loads = dstNode->loads;
+ SmallVector<Operation *, 4> dstLoadOpInsts;
DenseSet<Value *> visitedMemrefs;
while (!loads.empty()) {
// Get memref of load on top of the stack.
// Get 'srcNode' from which to attempt fusion into 'dstNode'.
auto *srcNode = mdg->getNode(srcId);
// Skip if 'srcNode' is not a loop nest.
- if (!srcNode->
inst->isa<AffineForOp>())
+ if (!srcNode->
op->isa<AffineForOp>())
continue;
// Skip if 'srcNode' has more than one store to any memref.
// TODO(andydavis) Support fusing multi-output src loop nests.
if (mdg->getOutEdgeCount(srcNode->id, memref) > maxSrcUserCount)
continue;
- // Compute an instruction list insertion point for the fused loop
+ // Compute an operation list insertion point for the fused loop
// nest which preserves dependences.
- Instruction *insertPointInst =
+ Operation *insertPointInst =
mdg->getFusedLoopNestInsertionPoint(srcNode->id, dstNode->id);
if (insertPointInst == nullptr)
continue;
// Get unique 'srcNode' store op.
auto *srcStoreOpInst = srcNode->stores.front();
// Gather 'dstNode' store ops to 'memref'.
- SmallVector<Instruction *, 2> dstStoreOpInsts;
+ SmallVector<Operation *, 2> dstStoreOpInsts;
for (auto *storeOpInst : dstNode->stores)
if (storeOpInst->cast<StoreOp>().getMemRef() == memref)
dstStoreOpInsts.push_back(storeOpInst);
LLVM_DEBUG(llvm::dbgs() << "\tslice loop nest:\n"
<< *sliceLoopNest.getOperation() << "\n");
// Move 'dstAffineForOp' before 'insertPointInst' if needed.
- auto dstAffineForOp = dstNode->
inst->cast<AffineForOp>();
+ auto dstAffineForOp = dstNode->
op->cast<AffineForOp>();
if (insertPointInst != dstAffineForOp.getOperation()) {
dstAffineForOp.getOperation()->moveBefore(insertPointInst);
}
}
if (!writesToLiveInOrOut) {
// Create private memref for 'memref' in 'dstAffineForOp'.
- SmallVector<Instruction *, 4> storesForMemref;
+ SmallVector<Operation *, 4> storesForMemref;
for (auto *storeOpInst : sliceCollector.storeOpInsts) {
if (storeOpInst->cast<StoreOp>().getMemRef() == memref)
storesForMemref.push_back(storeOpInst);
// so it is safe to remove.
if (writesToLiveInOrOut || mdg->canRemoveNode(srcNode->id)) {
mdg->removeNode(srcNode->id);
- srcNode->inst->erase();
+ srcNode->op->erase();
} else {
// Add remaining users of 'oldMemRef' back on the worklist (if not
// already there), as its replacement with a local/private memref
// Get 'dstNode' into which to attempt fusion.
auto *dstNode = mdg->getNode(dstId);
// Skip if 'dstNode' is not a loop nest.
- if (!dstNode->
inst->isa<AffineForOp>())
+ if (!dstNode->
op->isa<AffineForOp>())
continue;
// Attempt to fuse 'dstNode' with its sibling nodes in the graph.
fuseWithSiblingNodes(dstNode);
// TODO(andydavis) Check that 'sibStoreOpInst' post-dominates all other
// stores to the same memref in 'sibNode' loop nest.
auto *sibNode = mdg->getNode(sibId);
- // Compute an instruction list insertion point for the fused loop
+ // Compute an operation list insertion point for the fused loop
// nest which preserves dependences.
- assert(sibNode->inst->getBlock() == dstNode->inst->getBlock());
- Instruction *insertPointInst =
- sibNode->inst->isBeforeInBlock(dstNode->inst)
+ assert(sibNode->op->getBlock() == dstNode->op->getBlock());
+ Operation *insertPointInst =
+ sibNode->op->isBeforeInBlock(dstNode->op)
? mdg->getFusedLoopNestInsertionPoint(sibNode->id, dstNode->id)
: mdg->getFusedLoopNestInsertionPoint(dstNode->id, sibNode->id);
if (insertPointInst == nullptr)
// Check if fusion would be profitable and at what depth.
// Get unique 'sibNode' load op to 'memref'.
- SmallVector<Instruction *, 2> sibLoadOpInsts;
+ SmallVector<Operation *, 2> sibLoadOpInsts;
sibNode->getLoadOpsForMemref(memref, &sibLoadOpInsts);
// Currently findSiblingNodeToFuse searches for siblings with one load.
assert(sibLoadOpInsts.size() == 1);
- Instruction *sibLoadOpInst = sibLoadOpInsts[0];
+ Operation *sibLoadOpInst = sibLoadOpInsts[0];
assert(!sibNode->stores.empty());
// TODO(andydavis) Choose the store which postdominates all other stores.
auto *sibStoreOpInst = sibNode->stores.back();
// Gather 'dstNode' load ops to 'memref'.
- SmallVector<Instruction *, 2> dstLoadOpInsts;
+ SmallVector<Operation *, 2> dstLoadOpInsts;
dstNode->getLoadOpsForMemref(memref, &dstLoadOpInsts);
// Gather 'dstNode' store ops to 'memref'.
- SmallVector<Instruction *, 2> dstStoreOpInsts;
+ SmallVector<Operation *, 2> dstStoreOpInsts;
dstNode->getStoreOpsForMemref(memref, &dstStoreOpInsts);
unsigned bestDstLoopDepth;
auto sliceLoopNest = mlir::insertBackwardComputationSlice(
sibLoadOpInst, dstLoadOpInsts[0], bestDstLoopDepth, &sliceState);
if (sliceLoopNest != nullptr) {
- auto dstForInst = dstNode->
inst->cast<AffineForOp>();
- // Update instruction position of fused loop nest (if needed).
+ auto dstForInst = dstNode->
op->cast<AffineForOp>();
+ // Update operation position of fused loop nest (if needed).
if (insertPointInst != dstForInst.getOperation()) {
dstForInst.getOperation()->moveBefore(insertPointInst);
}
if (outEdge.id == dstNode->id || outEdge.value != inEdge.value)
return;
auto *sibNode = mdg->getNode(sibNodeId);
- if (!sibNode->
inst->isa<AffineForOp>())
+ if (!sibNode->
op->isa<AffineForOp>())
return;
// Skip if 'outEdge' is not a read-after-write dependence.
// TODO(andydavis) Remove restrict to single load op restriction.
}
// Collect dst loop stats after memref privatizaton transformation.
- auto dstForInst = dstNode->
inst->cast<AffineForOp>();
+ auto dstForInst = dstNode->
op->cast<AffineForOp>();
LoopNestStateCollector dstLoopCollector;
dstLoopCollector.collect(dstForInst.getOperation());
// Clear and add back loads and stores
// function.
if (mdg->getOutEdgeCount(sibNode->id) == 0) {
mdg->removeNode(sibNode->id);
- sibNode->
inst->cast<AffineForOp>().erase();
+ sibNode->
op->cast<AffineForOp>().erase();
}
}
if (pair.second > 0)
continue;
auto *memref = pair.first;
- // Skip if there exist other uses (return instruction or function calls).
+ // Skip if there exist other uses (return operation or function calls).
if (!memref->use_empty())
continue;
// Use list expected to match the dep graph info.
- auto *inst = memref->getDefiningOp();
- if (
inst && inst->isa<AllocOp>())
- inst->erase();
+ auto *op = memref->getDefiningOp();
+ if (
op && op->isa<AllocOp>())
+ op->erase();
}
}
};
assert(!band.empty());
assert(band.size() == tileSizes.size() && "Incorrect number of tile sizes");
- // Check if the supplied for inst's are all successively nested.
+ // Check if the supplied for op's are all successively nested.
for (unsigned i = 1, e = band.size(); i < e; i++) {
assert(band[i].getOperation()->getParentOp() == band[i - 1].getOperation());
}
};
for (auto &block : f)
- for (auto &inst : block)
- if (auto forOp =
inst.dyn_cast<AffineForOp>())
+ for (auto &op : block)
+ if (auto forOp =
op.dyn_cast<AffineForOp>())
getMaximalPerfectLoopNest(forOp);
}
void runOnFunction() override;
- /// Unroll this for inst. Returns failure if nothing was done.
+ /// Unroll this for op. Returns failure if nothing was done.
LogicalResult runOnAffineForOp(AffineForOp forOp);
static const unsigned kDefaultUnrollFactor = 4;
hasInnerLoops |= walkPostOrder(&(*Start++));
return hasInnerLoops;
}
- bool walkPostOrder(Instruction *opInst) {
+ bool walkPostOrder(Operation *opInst) {
bool hasInnerLoops = false;
for (auto ®ion : opInst->getRegions())
for (auto &block : region)
}
}
-/// Unrolls a 'affine.for' inst. Returns success if the loop was unrolled,
+/// Unrolls a 'affine.for' op. Returns success if the loop was unrolled,
/// failure otherwise. The default unroll factor is 4.
LogicalResult LoopUnroll::runOnAffineForOp(AffineForOp forOp) {
// Use the function callback if one was provided.
//
// This file implements loop unroll and jam. Unroll and jam is a transformation
// that improves locality, in particular, register reuse, while also improving
-// instruction level parallelism. The example below shows what it does in nearly
+// operation level parallelism. The example below shows what it does in nearly
// the general case. Loop unroll and jam currently works if the bounds of the
// loops inner to the loop being unroll-jammed do not depend on the latter.
//
// S6(i+1);
//
// Note: 'if/else' blocks are not jammed. So, if there are loops inside if
-// inst's, bodies of those loops will not be jammed.
+// op's, bodies of those loops will not be jammed.
//===----------------------------------------------------------------------===//
#include "mlir/Transforms/Passes.h"
runOnAffineForOp(forOp);
}
-/// Unroll and jam a 'affine.for' inst. Default unroll jam factor is
+/// Unroll and jam a 'affine.for' op. Default unroll jam factor is
/// kDefaultUnrollJamFactor. Return failure if nothing was done.
LogicalResult LoopUnrollAndJam::runOnAffineForOp(AffineForOp forOp) {
// Unroll and jam by the factor that was passed if any.
/// Unrolls and jams this loop by the specified factor.
LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
uint64_t unrollJamFactor) {
- // Gathers all maximal sub-blocks of instructions that do not themselves
- // include a for inst (a instruction could have a descendant for inst though
+ // Gathers all maximal sub-blocks of operations that do not themselves
+ // include a for op (a operation could have a descendant for op though
// in its tree). Ignore the block terminators.
struct JamBlockGatherer {
- // Store iterators to the first and last inst of each sub-block found.
+ // Store iterators to the first and last op of each sub-block found.
std::vector<std::pair<Block::iterator, Block::iterator>> subBlocks;
// This is a linear time walk.
- void walk(Instruction *inst) {
- for (auto ®ion : inst->getRegions())
+ void walk(Operation *op) {
+ for (auto ®ion : op->getRegions())
for (auto &block : region)
walk(block);
}
using namespace mlir;
namespace {
-// Visit affine expressions recursively and build the sequence of instructions
+// Visit affine expressions recursively and build the sequence of operations
// that correspond to it. Visitation functions return an Value of the
// expression subtree they visited or `nullptr` on error.
class AffineApplyExpander
// Floor division operation (rounds towards negative infinity).
//
// For positive divisors, it can be implemented without branching and with a
- // single division instruction as
+ // single division operation as
//
// a floordiv b =
// let negative = a < 0 in
// Ceiling division operation (rounds towards positive infinity).
//
// For positive divisors, it can be implemented without branching and with a
- // single division instruction as
+ // single division operation as
//
// a ceildiv b =
// let negative = a <= 0 in
};
} // namespace
-// Create a sequence of instructions that implement the `expr` applied to the
+// Create a sequence of operations that implement the `expr` applied to the
// given dimension and symbol values.
static mlir::Value *expandAffineExpr(FuncBuilder *builder, Location loc,
AffineExpr expr,
return AffineApplyExpander(builder, dimValues, symbolValues, loc).visit(expr);
}
-// Create a sequence of instructions that implement the `affineMap` applied to
+// Create a sequence of operations that implement the `affineMap` applied to
// the given `operands` (as it it were an AffineApplyOp).
Optional<SmallVector<Value *, 8>> static expandAffineMap(
FuncBuilder *builder, Location loc, AffineMap affineMap,
return false;
}
-// Convert an "if" instruction into a flow of basic blocks.
+// Convert an "if" operation into a flow of basic blocks.
//
-// Create an SESE region for the if instruction (including its "then" and
-// optional "else" instruction blocks) and append it to the end of the current
+// Create an SESE region for the if operation (including its "then" and
+// optional "else" operation blocks) and append it to the end of the current
// region. The conditional region consists of a sequence of condition-checking
// blocks that implement the short-circuit scheme, followed by a "then" SESE
// region and an "else" SESE region, and the continuation block that
-// post-dominates all blocks of the "if" instruction. The flow of blocks that
+// post-dominates all blocks of the "if" operation. The flow of blocks that
// correspond to the "then" and "else" clauses are constructed recursively,
-// enabling easy nesting of "if" instructions and if-then-else-if chains.
+// enabling easy nesting of "if" operations and if-then-else-if chains.
//
// +--------------------------------+
// | <code before the AffineIfOp> |
auto *continueBlock = condBlock->splitBlock(ifInst);
// Create a block for the 'then' code, inserting it between the cond and
- // continue blocks. Move the instructions over from the AffineIfOp and add a
+ // continue blocks. Move the operations over from the AffineIfOp and add a
// branch to the continuation point.
Block *thenBlock = new Block();
thenBlock->insertBefore(continueBlock);
- // If the 'then' block is not empty, then splice the instructions except for
+ // If the 'then' block is not empty, then splice the operations except for
// the terminator.
auto &oldThenBlocks = ifOp.getThenBlocks();
if (!oldThenBlocks.empty()) {
}
// Convert an "affine.apply" operation into a sequence of arithmetic
-// instructions using the StandardOps dialect. Return true on error.
+// operations using the StandardOps dialect. Return true on error.
bool LowerAffinePass::lowerAffineApply(AffineApplyOp op) {
FuncBuilder builder(op.getOperation());
auto maybeExpandedMap =
// Entry point of the function convertor.
//
-// Conversion is performed by recursively visiting instructions of a Function.
+// Conversion is performed by recursively visiting operations of a Function.
// It reasons in terms of single-entry single-exit (SESE) regions that are not
// materialized in the code. Instead, the pointer to the last block of the
// region is maintained throughout the conversion as the insertion point of the
// IR builder since we never change the first block after its creation. "Block"
-// instructions such as loops and branches create new SESE regions for their
+// operations such as loops and branches create new SESE regions for their
// bodies, and surround them with additional basic blocks for the control flow.
// Individual operations are simply appended to the end of the last basic block
// of the current region. The SESE invariant allows us to easily handle nested
// corresponding Value that has been defined previously. The value flow
// starts with function arguments converted to basic block arguments.
void LowerAffinePass::runOnFunction() {
- SmallVector<Instruction *, 8> instsToRewrite;
+ SmallVector<Operation *, 8> instsToRewrite;
- // Collect all the For instructions as well as AffineIfOps and AffineApplyOps.
+ // Collect all the For operations as well as AffineIfOps and AffineApplyOps.
// We do this as a prepass to avoid invalidating the walker with our rewrite.
- getFunction().walk([&](Instruction *inst) {
- if (
inst->isa<AffineApplyOp>() || inst->isa<AffineForOp>() ||
-
inst->isa<AffineIfOp>())
- instsToRewrite.push_back(inst);
+ getFunction().walk([&](Operation *op) {
+ if (
op->isa<AffineApplyOp>() || op->isa<AffineForOp>() ||
+ instsToRewrite.push_back(op);
});
- // Rewrite all of the ifs and fors. We walked the instructions in preorder,
+ // Rewrite all of the ifs and fors. We walked the operations in preorder,
// so we know that we will rewrite them in the same order.
- for (auto *inst : instsToRewrite) {
- if (auto ifOp =
inst->dyn_cast<AffineIfOp>()) {
+ for (auto *op : instsToRewrite) {
+ if (auto ifOp =
op->dyn_cast<AffineIfOp>()) {
if (lowerAffineIf(ifOp))
return signalPassFailure();
- } else if (auto forOp =
inst->dyn_cast<AffineForOp>()) {
+ } else if (auto forOp =
op->dyn_cast<AffineForOp>()) {
if (lowerAffineFor(forOp))
return signalPassFailure();
- } else if (lowerAffineApply(
inst->cast<AffineApplyOp>())) {
+ } else if (lowerAffineApply(
op->cast<AffineApplyOp>())) {
return signalPassFailure();
}
}
}
-/// Lowers If and For instructions within a function into their lower level CFG
+/// Lowers If and For operations within a function into their lower level CFG
/// equivalent blocks.
FunctionPassBase *mlir::createLowerAffinePass() {
return new LowerAffinePass();
static PassRegistration<LowerAffinePass>
pass("lower-affine",
- "Lower If, For, AffineApply instructions to primitive equivalents");
+ "Lower If, For, AffineApply operations to primitive equivalents");
explicit VectorTransferExpander(MLIRContext *context)
: MLLoweringPattern(VectorTransferOpTy::getOperationName(), 1, context) {}
- PatternMatchResult match(Instruction *op) const override {
+ PatternMatchResult match(Operation *op) const override {
if (m_Op<VectorTransferOpTy>().match(op))
return matchSuccess();
return matchFailure();
}
- void rewriteOpInst(Instruction *op, MLFuncGlobalLoweringState *funcWiseState,
+ void rewriteOpInst(Operation *op, MLFuncGlobalLoweringState *funcWiseState,
std::unique_ptr<PatternState> opState,
MLFuncLoweringRewriter *rewriter) const override {
VectorTransferRewriter<VectorTransferOpTy>(
/// to the pass. This pass is thus a partial lowering that opens the "greybox"
/// that is the super-vector abstraction. In particular, this pass can turn the
/// vector_transfer_read and vector_transfer_write ops in either:
-/// 1. a loop nest with either scalar and vector load/store instructions; or
+/// 1. a loop nest with either scalar and vector load/store operations; or
/// 2. a loop-nest with DmaStartOp / DmaWaitOp; or
/// 3. a pre-existing blackbox library call that can be written manually or
/// synthesized using search and superoptimization.
return res;
}
-static Instruction *instantiate(FuncBuilder *b, Instruction *opInst,
- VectorType hwVectorType,
- DenseMap<Value *, Value *> *substitutionsMap);
+static Operation *instantiate(FuncBuilder *b, Operation *opInst,
+ VectorType hwVectorType,
+ DenseMap<Value *, Value *> *substitutionsMap);
/// Not all Values belong to a program slice scoped within the immediately
/// enclosing loop.
auto *opInst = v->getDefiningOp();
if (opInst->isa<ConstantOp>()) {
FuncBuilder b(opInst);
- auto *inst = instantiate(&b, opInst, hwVectorType, substitutionsMap);
- auto res =
- substitutionsMap->insert(std::make_pair(v, inst->getResult(0)));
+ auto *op = instantiate(&b, opInst, hwVectorType, substitutionsMap);
+ auto res = substitutionsMap->insert(std::make_pair(v, op->getResult(0)));
assert(res.second && "Insertion failed");
return res.first->second;
}
/// - constant splat is replaced by constant splat of `hwVectorType`.
/// TODO(ntv): add more substitutions on a per-need basis.
static SmallVector<NamedAttribute, 1>
-materializeAttributes(Instruction *opInst, VectorType hwVectorType) {
+materializeAttributes(Operation *opInst, VectorType hwVectorType) {
SmallVector<NamedAttribute, 1> res;
for (auto a : opInst->getAttrs()) {
if (auto splat = a.second.dyn_cast<SplatElementsAttr>()) {
/// substitutionsMap.
///
/// If the underlying substitution fails, this fails too and returns nullptr.
-static Instruction *instantiate(FuncBuilder *b, Instruction *opInst,
- VectorType hwVectorType,
- DenseMap<Value *, Value *> *substitutionsMap) {
+static Operation *instantiate(FuncBuilder *b, Operation *opInst,
+ VectorType hwVectorType,
+ DenseMap<Value *, Value *> *substitutionsMap) {
assert(!opInst->isa<VectorTransferReadOp>() &&
"Should call the function specialized for VectorTransferReadOp");
assert(!opInst->isa<VectorTransferWriteOp>() &&
/// `hwVectorType` int the covering of the super-vector type. For a more
/// detailed description of the problem, see the description of
/// reindexAffineIndices.
-static Instruction *instantiate(FuncBuilder *b, VectorTransferReadOp read,
- VectorType hwVectorType,
- ArrayRef<unsigned> hwVectorInstance,
- DenseMap<Value *, Value *> *substitutionsMap) {
+static Operation *instantiate(FuncBuilder *b, VectorTransferReadOp read,
+ VectorType hwVectorType,
+ ArrayRef<unsigned> hwVectorInstance,
+ DenseMap<Value *, Value *> *substitutionsMap) {
SmallVector<Value *, 8> indices =
map(makePtrDynCaster<Value>(), read.getIndices());
auto affineIndices =
/// `hwVectorType` int the covering of th3e super-vector type. For a more
/// detailed description of the problem, see the description of
/// reindexAffineIndices.
-static Instruction *instantiate(FuncBuilder *b, VectorTransferWriteOp write,
- VectorType hwVectorType,
- ArrayRef<unsigned> hwVectorInstance,
- DenseMap<Value *, Value *> *substitutionsMap) {
+static Operation *instantiate(FuncBuilder *b, VectorTransferWriteOp write,
+ VectorType hwVectorType,
+ ArrayRef<unsigned> hwVectorInstance,
+ DenseMap<Value *, Value *> *substitutionsMap) {
SmallVector<Value *, 8> indices =
map(makePtrDynCaster<Value>(), write.getIndices());
auto affineIndices =
return cloned.getOperation();
}
-/// Returns `true` if inst instance is properly cloned and inserted, false
+/// Returns `true` if op instance is properly cloned and inserted, false
/// otherwise.
/// The multi-dimensional `hwVectorInstance` belongs to the shapeRatio of
/// super-vector type to hw vector type.
-/// A cloned instance of `inst` is formed as follows:
+/// A cloned instance of `op` is formed as follows:
/// 1. vector_transfer_read: the return `superVectorType` is replaced by
/// `hwVectorType`. Additionally, affine indices are reindexed with
/// `reindexAffineIndices` using `hwVectorInstance` and vector type
/// possible.
///
/// Returns true on failure.
-static bool instantiateMaterialization(Instruction *inst,
+static bool instantiateMaterialization(Operation *op,
MaterializationState *state) {
- LLVM_DEBUG(dbgs() << "\ninstantiate: " << *inst);
+ LLVM_DEBUG(dbgs() << "\ninstantiate: " << *op);
// Create a builder here for unroll-and-jam effects.
- FuncBuilder b(inst);
+ FuncBuilder b(op);
// AffineApplyOp are ignored: instantiating the proper vector op will take
// care of AffineApplyOps by composing them properly.
- if (
inst->isa<AffineApplyOp>()) {
+ if (
op->isa<AffineApplyOp>()) {
return false;
}
- if (inst->getNumRegions() != 0)
- return inst->emitError("NYI path Op with region");
+ if (op->getNumRegions() != 0)
+ return op->emitError("NYI path Op with region");
- if (auto write = inst->dyn_cast<VectorTransferWriteOp>()) {
+ if (auto write = op->dyn_cast<VectorTransferWriteOp>()) {
auto *clone = instantiate(&b, write, state->hwVectorType,
state->hwVectorInstance, state->substitutionsMap);
return clone == nullptr;
}
- if (auto read = inst->dyn_cast<VectorTransferReadOp>()) {
+ if (auto read = op->dyn_cast<VectorTransferReadOp>()) {
auto *clone = instantiate(&b, read, state->hwVectorType,
state->hwVectorInstance, state->substitutionsMap);
if (!clone) {
// The only op with 0 results reaching this point must, by construction, be
// VectorTransferWriteOps and have been caught above. Ops with >= 2 results
// are not yet supported. So just support 1 result.
- if (inst->getNumResults() != 1) {
- return inst->emitError("NYI: ops with != 1 results");
+ if (op->getNumResults() != 1) {
+ return op->emitError("NYI: ops with != 1 results");
}
- if (inst->getResult(0)->getType() != state->superVectorType) {
- return inst->emitError("Op does not return a supervector.");
+ if (op->getResult(0)->getType() != state->superVectorType) {
+ return op->emitError("Op does not return a supervector.");
}
auto *clone =
- instantiate(&b, inst, state->hwVectorType, state->substitutionsMap);
+ instantiate(&b, op, state->hwVectorType, state->substitutionsMap);
if (!clone) {
return true;
}
state->substitutionsMap->insert(
- std::make_pair(inst->getResult(0), clone->getResult(0)));
+ std::make_pair(op->getResult(0), clone->getResult(0)));
return false;
}
/// TODO(ntv): full loops + materialized allocs.
/// TODO(ntv): partial unrolling + materialized allocs.
static bool emitSlice(MaterializationState *state,
- SetVector<Instruction *> *slice) {
+ SetVector<Operation *> *slice) {
auto ratio = shapeRatio(state->superVectorType, state->hwVectorType);
assert(ratio.hasValue() &&
"ratio of super-vector to HW-vector shape is not integral");
DenseMap<Value *, Value *> substitutionMap;
scopedState.substitutionsMap = &substitutionMap;
// slice are topologically sorted, we can just clone them in order.
- for (auto *inst : *slice) {
- auto fail = instantiateMaterialization(inst, &scopedState);
+ for (auto *op : *slice) {
+ auto fail = instantiateMaterialization(op, &scopedState);
if (fail) {
- inst->emitError("Unhandled super-vector materialization failure");
+ op->emitError("Unhandled super-vector materialization failure");
return true;
}
}
/// Materializes super-vector types into concrete hw vector types as follows:
/// 1. start from super-vector terminators (current vector_transfer_write
/// ops);
-/// 2. collect all the instructions that can be reached by transitive use-defs
+/// 2. collect all the operations that can be reached by transitive use-defs
/// chains;
/// 3. get the superVectorType for this particular terminator and the
/// corresponding hardware vector type (for now limited to F32)
/// Notes
/// =====
/// The `slice` is sorted in topological order by construction.
-/// Additionally, this set is limited to instructions in the same lexical scope
+/// Additionally, this set is limited to operations in the same lexical scope
/// because we currently disallow vectorization of defs that come from another
/// scope.
/// TODO(ntv): please document return value.
-static bool materialize(Function *f,
- const SetVector<Instruction *> &terminators,
+static bool materialize(Function *f, const SetVector<Operation *> &terminators,
MaterializationState *state) {
- DenseSet<Instruction *> seen;
+ DenseSet<Operation *> seen;
DominanceInfo domInfo(f);
for (auto *term : terminators) {
// Short-circuit test, a given terminator may have been reached by some
// Note for the justification of this restriction.
// TODO(ntv): relax scoping constraints.
auto *enclosingScope = term->getParentOp();
- auto keepIfInSameScope = [enclosingScope, &domInfo](Instruction *inst) {
- assert(inst && "NULL inst");
+ auto keepIfInSameScope = [enclosingScope, &domInfo](Operation *op) {
+ assert(op && "NULL op");
if (!enclosingScope) {
// by construction, everyone is always under the top scope (null scope).
return true;
}
- return domInfo.properlyDominates(enclosingScope, inst);
+ return domInfo.properlyDominates(enclosingScope, op);
};
- SetVector<Instruction *> slice =
+ SetVector<Operation *> slice =
getSlice(term, keepIfInSameScope, keepIfInSameScope);
assert(!slice.empty());
// Capture terminators; i.e. vector_transfer_write ops involving a strict
// super-vector of subVectorType.
- auto filter = [subVectorType](Instruction &inst) {
- if (!inst.isa<VectorTransferWriteOp>()) {
+ auto filter = [subVectorType](Operation &op) {
+ if (!op.isa<VectorTransferWriteOp>()) {
return false;
}
- return matcher::operatesOnSuperVectors(inst, subVectorType);
+ return matcher::operatesOnSuperVectors(op, subVectorType);
};
auto pat = Op(filter);
SmallVector<NestedMatch, 8> matches;
pat.match(f, &matches);
- SetVector<Instruction *> terminators;
+ SetVector<Operation *> terminators;
for (auto m : matches) {
terminators.insert(m.getMatchedOperation());
}
// iteration of the innermost loop enclosing both the store op and the load op.
//
// (* A dependence being satisfied at a block: a dependence that is satisfied by
-// virtue of the destination instruction appearing textually / lexically after
-// the source instruction within the body of a 'affine.for' instruction; thus, a
+// virtue of the destination operation appearing textually / lexically after
+// the source operation within the body of a 'affine.for' operation; thus, a
// dependence is always either satisfied by a loop or by a block).
//
// The above conditions are simple to check, sufficient, and powerful for most
// A list of memref's that are potentially dead / could be eliminated.
SmallPtrSet<Value *, 4> memrefsToErase;
// Load op's whose results were replaced by those forwarded from stores.
- std::vector<Instruction *> loadOpsToErase;
+ std::vector<Operation *> loadOpsToErase;
DominanceInfo *domInfo = nullptr;
PostDominanceInfo *postDomInfo = nullptr;
// This is a straightforward implementation not optimized for speed. Optimize
// this in the future if needed.
void MemRefDataFlowOpt::forwardStoreToLoad(LoadOp loadOp) {
- Instruction *lastWriteStoreOp = nullptr;
- Instruction *loadOpInst = loadOp.getOperation();
+ Operation *lastWriteStoreOp = nullptr;
+ Operation *loadOpInst = loadOp.getOperation();
// First pass over the use list to get minimum number of surrounding
// loops common between the load op and the store op, with min taken across
// all store ops.
- SmallVector<Instruction *, 8> storeOps;
+ SmallVector<Operation *, 8> storeOps;
unsigned minSurroundingLoops = getNestingDepth(*loadOpInst);
for (InstOperand &use : loadOp.getMemRef()->getUses()) {
auto storeOp = use.getOwner()->dyn_cast<StoreOp>();
// and loadOp.
// The list of store op candidates for forwarding - need to satisfy the
// conditions listed at the top.
- SmallVector<Instruction *, 8> fwdingCandidates;
+ SmallVector<Operation *, 8> fwdingCandidates;
// Store ops that have a dependence into the load (even if they aren't
// forwarding candidates). Each forwarding candidate will be checked for a
// post-dominance on these. 'fwdingCandidates' are a subset of depSrcStores.
- SmallVector<Instruction *, 8> depSrcStores;
+ SmallVector<Operation *, 8> depSrcStores;
for (auto *storeOpInst : storeOps) {
MemRefAccess srcAccess(storeOpInst);
MemRefAccess destAccess(loadOpInst);
// that postdominates all 'depSrcStores' (if such a store exists) is the
// unique store providing the value to the load, i.e., provably the last
// writer to that memref loc.
- if (llvm::all_of(depSrcStores, [&](Instruction *depStore) {
+ if (llvm::all_of(depSrcStores, [&](Operation *depStore) {
return postDomInfo->postDominates(storeOpInst, depStore);
})) {
lastWriteStoreOp = storeOpInst;
// to do this as well, but we'll do it here since we collected these anyway.
for (auto *memref : memrefsToErase) {
// If the memref hasn't been alloc'ed in this function, skip.
- Instruction *defInst = memref->getDefiningOp();
+ Operation *defInst = memref->getDefiningOp();
if (!defInst || !defInst->isa<AllocOp>())
- // TODO(mlir-team): if the memref was returned by a 'call' instruction, we
+ // TODO(mlir-team): if the memref was returned by a 'call' operation, we
// could still erase it if the call had no side-effects.
continue;
if (std::any_of(memref->use_begin(), memref->use_end(),
return new PipelineDataTransfer();
}
-// Returns the position of the tag memref operand given a DMA instruction.
+// Returns the position of the tag memref operand given a DMA operation.
// Temporary utility: will be replaced when DmaStart/DmaFinish abstract op's are
// added. TODO(b/117228571)
-static unsigned getTagMemRefPos(Instruction &dmaInst) {
+static unsigned getTagMemRefPos(Operation &dmaInst) {
assert(dmaInst.isa<DmaStartOp>() || dmaInst.isa<DmaWaitOp>());
if (dmaInst.isa<DmaStartOp>()) {
// Second to last operand.
return dmaInst.getNumOperands() - 2;
}
- // First operand for a dma finish instruction.
+ // First operand for a dma finish operation.
return 0;
}
/// Doubles the buffer of the supplied memref on the specified 'affine.for'
-/// instruction by adding a leading dimension of size two to the memref.
+/// operation by adding a leading dimension of size two to the memref.
/// Replaces all uses of the old memref by the new one while indexing the newly
-/// added dimension by the loop IV of the specified 'affine.for' instruction
+/// added dimension by the loop IV of the specified 'affine.for' operation
/// modulo 2. Returns false if such a replacement cannot be performed.
static bool doubleBuffer(Value *oldMemRef, AffineForOp forOp) {
auto *forBody = forOp.getBody();
dynamicDimCount++));
}
- // Create and place the alloc right before the 'affine.for' instruction.
+ // Create and place the alloc right before the 'affine.for' operation.
Value *newMemRef =
bOuter.create<AllocOp>(forInst->getLoc(), newMemRefType, allocOperands);
/// Returns success if the IR is in a valid state.
void PipelineDataTransfer::runOnFunction() {
// Do a post order walk so that inner loop DMAs are processed first. This is
- // necessary since 'affine.for' instructions nested within would otherwise
+ // necessary since 'affine.for' operations nested within would otherwise
// become invalid (erased) when the outer loop is pipelined (the pipelined one
// gets deleted and replaced by a prologue, a new steady-state loop and an
// epilogue).
return true;
}
-// Identify matching DMA start/finish instructions to overlap computation with.
+// Identify matching DMA start/finish operations to overlap computation with.
static void findMatchingStartFinishInsts(
AffineForOp forOp,
- SmallVectorImpl<std::pair<Instruction *, Instruction *>> &startWaitPairs) {
+ SmallVectorImpl<std::pair<Operation *, Operation *>> &startWaitPairs) {
- // Collect outgoing DMA instructions - needed to check for dependences below.
+ // Collect outgoing DMA operations - needed to check for dependences below.
SmallVector<DmaStartOp, 4> outgoingDmaOps;
- for (auto &inst : *forOp.getBody()) {
- auto dmaStartOp = inst.dyn_cast<DmaStartOp>();
+ for (auto &op : *forOp.getBody()) {
+ auto dmaStartOp = op.dyn_cast<DmaStartOp>();
if (dmaStartOp && dmaStartOp.isSrcMemorySpaceFaster())
outgoingDmaOps.push_back(dmaStartOp);
}
- SmallVector<Instruction *, 4> dmaStartInsts, dmaFinishInsts;
- for (auto &inst : *forOp.getBody()) {
- // Collect DMA finish instructions.
- if (inst.isa<DmaWaitOp>()) {
- dmaFinishInsts.push_back(&inst);
+ SmallVector<Operation *, 4> dmaStartInsts, dmaFinishInsts;
+ for (auto &op : *forOp.getBody()) {
+ // Collect DMA finish operations.
+ if (op.isa<DmaWaitOp>()) {
+ dmaFinishInsts.push_back(&op);
continue;
}
- auto dmaStartOp = inst.dyn_cast<DmaStartOp>();
+ auto dmaStartOp = op.dyn_cast<DmaStartOp>();
if (!dmaStartOp)
continue;
}
}
if (!escapingUses)
- dmaStartInsts.push_back(&inst);
+ dmaStartInsts.push_back(&op);
}
- // For each start instruction, we look for a matching finish instruction.
+ // For each start operation, we look for a matching finish operation.
for (auto *dmaStartInst : dmaStartInsts) {
for (auto *dmaFinishInst : dmaFinishInsts) {
if (checkTagMatch(dmaStartInst->cast<DmaStartOp>(),
return;
}
- SmallVector<std::pair<Instruction *, Instruction *>, 4> startWaitPairs;
+ SmallVector<std::pair<Operation *, Operation *>, 4> startWaitPairs;
findMatchingStartFinishInsts(forOp, startWaitPairs);
if (startWaitPairs.empty()) {
// Double the buffers for the higher memory space memref's.
// Identify memref's to replace by scanning through all DMA start
- // instructions. A DMA start instruction has two memref's - the one from the
+ // operations. A DMA start operation has two memref's - the one from the
// higher level of memory hierarchy is the one to double buffer.
// TODO(bondhugula): check whether double-buffering is even necessary.
// TODO(bondhugula): make this work with different layouts: assuming here that
startWaitPairs.clear();
findMatchingStartFinishInsts(forOp, startWaitPairs);
- // Store shift for instruction for later lookup for AffineApplyOp's.
- DenseMap<Instruction *, unsigned> instShiftMap;
+ // Store shift for operation for later lookup for AffineApplyOp's.
+ DenseMap<Operation *, unsigned> instShiftMap;
for (auto &pair : startWaitPairs) {
auto *dmaStartInst = pair.first;
assert(dmaStartInst->isa<DmaStartOp>());
instShiftMap[dmaStartInst] = 0;
- // Set shifts for DMA start inst's affine operand computation slices to 0.
+ // Set shifts for DMA start op's affine operand computation slices to 0.
SmallVector<AffineApplyOp, 4> sliceOps;
mlir::createAffineComputationSlice(dmaStartInst, &sliceOps);
if (!sliceOps.empty()) {
} else {
// If a slice wasn't created, the reachable affine.apply op's from its
// operands are the ones that go with it.
- SmallVector<Instruction *, 4> affineApplyInsts;
+ SmallVector<Operation *, 4> affineApplyInsts;
SmallVector<Value *, 4> operands(dmaStartInst->getOperands());
getReachableAffineApplyOps(operands, affineApplyInsts);
- for (auto *inst : affineApplyInsts) {
- instShiftMap[inst] = 0;
+ for (auto *op : affineApplyInsts) {
+ instShiftMap[op] = 0;
}
}
}
// Everything else (including compute ops and dma finish) are shifted by one.
- for (auto &inst : *forOp.getBody()) {
- if (instShiftMap.find(&inst) == instShiftMap.end()) {
- instShiftMap[&inst] = 1;
+ for (auto &op : *forOp.getBody()) {
+ if (instShiftMap.find(&op) == instShiftMap.end()) {
+ instShiftMap[&op] = 1;
}
}
// Get shifts stored in map.
std::vector<uint64_t> shifts(forOp.getBody()->getOperations().size());
unsigned s = 0;
- for (auto &inst : *forOp.getBody()) {
- assert(instShiftMap.find(&inst) != instShiftMap.end());
- shifts[s++] = instShiftMap[&inst];
+ for (auto &op : *forOp.getBody()) {
+ assert(instShiftMap.find(&op) != instShiftMap.end());
+ shifts[s++] = instShiftMap[&op];
- // Tagging instructions with shifts for debugging purposes.
+ // Tagging operations with shifts for debugging purposes.
LLVM_DEBUG({
- FuncBuilder b(&inst);
- inst.setAttr("shift", b.getI64IntegerAttr(shifts[s - 1]));
+ FuncBuilder b(&op);
+ op.setAttr("shift", b.getI64IntegerAttr(shifts[s - 1]));
});
}
}
if (failed(instBodySkew(forOp, shifts))) {
- LLVM_DEBUG(llvm::dbgs() << "inst body skewing failed - unexpected\n";);
+ LLVM_DEBUG(llvm::dbgs() << "op body skewing failed - unexpected\n";);
return;
}
}
namespace {
-/// Simplifies all affine expressions appearing in the operation instructions of
+/// Simplifies all affine expressions appearing in the operations of
/// the Function. This is mainly to test the simplifyAffineExpr method.
/// TODO(someone): This should just be defined as a canonicalization pattern
/// on AffineMap and driven from the existing canonicalization pass.
void runOnFunction() override;
/// Utility to simplify an affine attribute and update its entry in the parent
- /// instruction if necessary.
+ /// operation if necessary.
template <typename AttributeT>
- void simplifyAndUpdateAttribute(Instruction *inst, Identifier name,
+ void simplifyAndUpdateAttribute(Operation *op, Identifier name,
AttributeT attr) {
auto &simplified = simplifiedAttributes[attr];
if (simplified == attr)
}
// Simplification was successful, so update the attribute.
- inst->setAttr(name, simplified);
+ op->setAttr(name, simplified);
}
/// Performs basic integer set simplifications. Checks if it's empty, and
void SimplifyAffineStructures::runOnFunction() {
simplifiedAttributes.clear();
- getFunction().walk([&](Instruction *opInst) {
+ getFunction().walk([&](Operation *opInst) {
for (auto attr : opInst->getAttrs()) {
if (auto mapAttr = attr.second.dyn_cast<AffineMapAttr>())
simplifyAndUpdateAttribute(opInst, attr.first, mapAttr);
Function &func = getFunction();
UnknownLoc unknownLoc = UnknownLoc::get(&getContext());
- // Strip the debug info from the function and its instructions.
+ // Strip the debug info from the function and its operations.
func.setLoc(unknownLoc);
- func.walk([&](Instruction *inst) { inst->setLoc(unknownLoc); });
+ func.walk([&](Operation *op) { op->setLoc(unknownLoc); });
}
/// Creates a pass to strip debug information from a function.
}
static PassRegistration<StripDebugInfo>
- pass("strip-debuginfo", "Strip debug info from functions and instructions");
+ pass("strip-debuginfo", "Strip debug info from functions and operations");
worklist.reserve(64);
// Add all operations to the worklist.
- fn.walk([&](Instruction *inst) { addToWorklist(inst); });
+ fn.walk([&](Operation *op) { addToWorklist(op); });
}
/// Perform the rewrites.
void simplifyFunction();
- void addToWorklist(Instruction *op) {
+ void addToWorklist(Operation *op) {
// Check to see if the worklist already contains this op.
if (worklistMap.count(op))
return;
worklist.push_back(op);
}
- Instruction *popFromWorklist() {
+ Operation *popFromWorklist() {
auto *op = worklist.back();
worklist.pop_back();
/// If the specified operation is in the worklist, remove it. If not, this is
/// a no-op.
- void removeFromWorklist(Instruction *op) {
+ void removeFromWorklist(Operation *op) {
auto it = worklistMap.find(op);
if (it != worklistMap.end()) {
assert(worklist[it->second] == op && "malformed worklist data structure");
protected:
// Implement the hook for creating operations, and make sure that newly
// created ops are added to the worklist for processing.
- Instruction *createOperation(const OperationState &state) override {
+ Operation *createOperation(const OperationState &state) override {
auto *result = builder.createOperation(state);
addToWorklist(result);
return result;
// If an operation is about to be removed, make sure it is not in our
// worklist anymore because we'd get dangling references to it.
- void notifyOperationRemoved(Instruction *op) override {
+ void notifyOperationRemoved(Operation *op) override {
addToWorklist(op->getOperands());
removeFromWorklist(op);
}
// When the root of a pattern is about to be replaced, it can trigger
// simplifications to its users - make sure to add them to the worklist
// before the root is changed.
- void notifyRootReplaced(Instruction *op) override {
+ void notifyRootReplaced(Operation *op) override {
for (auto *result : op->getResults())
// TODO: Add a result->getUsers() iterator.
for (auto &user : result->getUses())
}
private:
- // Look over the provided operands for any defining instructions that should
+ // Look over the provided operands for any defining operations that should
// be re-added to the worklist. This function should be called when an
// operation is modified or removed, as it may trigger further
// simplifications.
template <typename Operands> void addToWorklist(Operands &&operands) {
for (Value *operand : operands) {
// If the use count of this operand is now < 2, we re-add the defining
- // instruction to the worklist.
- // TODO(riverriddle) This is based on the fact that zero use instructions
+ // operation to the worklist.
+ // TODO(riverriddle) This is based on the fact that zero use operations
// may be deleted, and that single use values often have more
// canonicalization opportunities.
if (!operand->use_empty() &&
/// need to be revisited, plus their index in the worklist. This allows us to
/// efficiently remove operations from the worklist when they are erased from
/// the function, even if they aren't the root of a pattern.
- std::vector<Instruction *> worklist;
- DenseMap<Instruction *, unsigned> worklistMap;
+ std::vector<Operation *> worklist;
+ DenseMap<Operation *, unsigned> worklistMap;
/// As part of canonicalization, we move constants to the top of the entry
/// block of the current function and de-duplicate them. This keeps track of
/// constants we have done this for.
- DenseMap<std::pair<Attribute, Type>, Instruction *> uniquedConstants;
+ DenseMap<std::pair<Attribute, Type>, Operation *> uniquedConstants;
};
}; // end anonymous namespace
continue;
}
- // Check to see if any operands to the instruction is constant and whether
+ // Check to see if any operands to the operation is constant and whether
// the operation knows how to constant fold itself.
operandConstants.assign(op->getNumOperands(), Attribute());
for (unsigned i = 0, e = op->getNumOperands(); i != e; ++i)
// Replaces all IV uses to its single iteration value.
auto *iv = forOp.getInductionVar();
- Instruction *forInst = forOp.getOperation();
+ Operation *op = forOp.getOperation();
if (!iv->use_empty()) {
if (forOp.hasConstantLowerBound()) {
- auto *mlFunc = forInst->getFunction();
+ auto *mlFunc = op->getFunction();
FuncBuilder topBuilder(mlFunc);
auto constOp = topBuilder.create<ConstantIndexOp>(
forOp.getLoc(), forOp.getConstantLowerBound());
} else {
AffineBound lb = forOp.getLowerBound();
SmallVector<Value *, 4> lbOperands(lb.operand_begin(), lb.operand_end());
- FuncBuilder builder(forInst->getBlock(), Block::iterator(forInst));
+ FuncBuilder builder(op->getBlock(), Block::iterator(op));
if (lb.getMap() == builder.getDimIdentityMap()) {
// No need of generating an affine.apply.
iv->replaceAllUsesWith(lbOperands[0]);
} else {
auto affineApplyOp = builder.create<AffineApplyOp>(
- forInst->getLoc(), lb.getMap(), lbOperands);
+ op->getLoc(), lb.getMap(), lbOperands);
iv->replaceAllUsesWith(affineApplyOp);
}
}
}
- // Move the loop body instructions, except for terminator, to the loop's
+ // Move the loop body operations, except for terminator, to the loop's
// containing block.
- auto *block = forInst->getBlock();
+ auto *block = op->getBlock();
forOp.getBody()->getOperations().back().erase();
- block->getOperations().splice(Block::iterator(forInst),
+ block->getOperations().splice(Block::iterator(op),
forOp.getBody()->getOperations());
forOp.erase();
return success();
}
-/// Promotes all single iteration for inst's in the Function, i.e., moves
+/// Promotes all single iteration for op's in the Function, i.e., moves
/// their body into the containing Block.
void mlir::promoteSingleIterationLoops(Function *f) {
// Gathers all innermost loops through a post order pruned walk.
[](AffineForOp forOp) { promoteIfSingleIteration(forOp); });
}
-/// Generates a 'affine.for' inst with the specified lower and upper bounds
-/// while generating the right IV remappings for the shifted instructions. The
-/// instruction blocks that go into the loop are specified in instGroupQueue
+/// Generates a 'affine.for' op with the specified lower and upper bounds
+/// while generating the right IV remappings for the shifted operations. The
+/// operation blocks that go into the loop are specified in instGroupQueue
/// starting from the specified offset, and in that order; the first element of
-/// the pair specifies the shift applied to that group of instructions; note
+/// the pair specifies the shift applied to that group of operations; note
/// that the shift is multiplied by the loop step before being applied. Returns
/// nullptr if the generated loop simplifies to a single iteration one.
static AffineForOp
generateLoop(AffineMap lbMap, AffineMap ubMap,
- const std::vector<std::pair<uint64_t, ArrayRef<Instruction *>>>
+ const std::vector<std::pair<uint64_t, ArrayRef<Operation *>>>
&instGroupQueue,
unsigned offset, AffineForOp srcForInst, FuncBuilder *b) {
SmallVector<Value *, 4> lbOperands(srcForInst.getLowerBoundOperands());
it != e; ++it) {
uint64_t shift = it->first;
auto insts = it->second;
- // All 'same shift' instructions get added with their operands being
- // remapped to results of cloned instructions, and their IV used remapped.
+ // All 'same shift' operations get added with their operands being
+ // remapped to results of cloned operations, and their IV used remapped.
// Generate the remapping if the shift is not zero: remappedIV = newIV -
// shift.
if (!srcIV->use_empty() && shift != 0) {
} else {
operandMap.map(srcIV, loopChunkIV);
}
- for (auto *inst : insts) {
- if (!
inst->isa<AffineTerminatorOp>())
- bodyBuilder.clone(*inst, operandMap);
+ for (auto *op : insts) {
+ if (!
op->isa<AffineTerminatorOp>())
+ bodyBuilder.clone(*op, operandMap);
}
};
if (succeeded(promoteIfSingleIteration(loopChunk)))
return loopChunk;
}
-/// Skew the instructions in the body of a 'affine.for' instruction with the
-/// specified instruction-wise shifts. The shifts are with respect to the
+/// Skew the operations in the body of a 'affine.for' operation with the
+/// specified operation-wise shifts. The shifts are with respect to the
/// original execution order, and are multiplied by the loop 'step' before being
-/// applied. A shift of zero for each instruction will lead to no change.
-// The skewing of instructions with respect to one another can be used for
+/// applied. A shift of zero for each operation will lead to no change.
+// The skewing of operations with respect to one another can be used for
// example to allow overlap of asynchronous operations (such as DMA
-// communication) with computation, or just relative shifting of instructions
+// communication) with computation, or just relative shifting of operations
// for better register reuse, locality or parallelism. As such, the shifts are
-// typically expected to be at most of the order of the number of instructions.
+// typically expected to be at most of the order of the number of operations.
// This method should not be used as a substitute for loop distribution/fission.
-// This method uses an algorithm// in time linear in the number of instructions
+// This method uses an algorithm// in time linear in the number of operations
// in the body of the for loop - (using the 'sweep line' paradigm). This method
// asserts preservation of SSA dominance. A check for that as well as that for
// memory-based depedence preservation check rests with the users of this
return success();
}
- // An array of instruction groups sorted by shift amount; each group has all
- // instructions with the same shift in the order in which they appear in the
- // body of the 'affine.for' inst.
- std::vector<std::vector<Instruction *>> sortedInstGroups(maxShift + 1);
+ // An array of operation groups sorted by shift amount; each group has all
+ // operations with the same shift in the order in which they appear in the
+ // body of the 'affine.for' op.
+ std::vector<std::vector<Operation *>> sortedInstGroups(maxShift + 1);
unsigned pos = 0;
- for (auto &inst : *forOp.getBody()) {
+ for (auto &op : *forOp.getBody()) {
auto shift = shifts[pos++];
- sortedInstGroups[shift].push_back(&inst);
+ sortedInstGroups[shift].push_back(&op);
}
// Unless the shifts have a specific pattern (which actually would be the
// Do a sweep over the sorted shifts while storing open groups in a
// vector, and generating loop portions as necessary during the sweep. A block
- // of instructions is paired with its shift.
- std::vector<std::pair<uint64_t, ArrayRef<Instruction *>>> instGroupQueue;
+ // of operations is paired with its shift.
+ std::vector<std::pair<uint64_t, ArrayRef<Operation *>>> instGroupQueue;
auto origLbMap = forOp.getLowerBoundMap();
uint64_t lbShift = 0;
"Queue expected to be empty when the first block is found");
// The interval for which the loop needs to be generated here is:
// [lbShift, min(lbShift + tripCount, d)) and the body of the
- // loop needs to have all instructions in instQueue in that order.
+ // loop needs to have all operations in instQueue in that order.
AffineForOp res;
if (lbShift + tripCount * step < d * step) {
res = generateLoop(
b.getShiftedAffineMap(origLbMap, lbShift),
b.getShiftedAffineMap(origLbMap, lbShift + tripCount * step),
instGroupQueue, 0, forOp, &b);
- // Entire loop for the queued inst groups generated, empty it.
+ // Entire loop for the queued op groups generated, empty it.
instGroupQueue.clear();
lbShift += tripCount * step;
} else {
// Start of first interval.
lbShift = d * step;
}
- // Augment the list of instructions that get into the current open interval.
+ // Augment the list of operations that get into the current open interval.
instGroupQueue.push_back({d, sortedInstGroups[d]});
}
- // Those instructions groups left in the queue now need to be processed (FIFO)
+ // Those operations groups left in the queue now need to be processed (FIFO)
// and their loops completed.
for (unsigned i = 0, e = instGroupQueue.size(); i < e; ++i) {
uint64_t ubShift = (instGroupQueue[i].first + tripCount) * step;
prologue = epilogue;
}
- // Erase the original for inst.
+ // Erase the original for op.
forOp.erase();
if (unrollPrologueEpilogue && prologue)
return failure();
// Generate the cleanup loop if trip count isn't a multiple of unrollFactor.
- Instruction *forInst = forOp.getOperation();
+ Operation *op = forOp.getOperation();
if (getLargestDivisorOfTripCount(forOp) % unrollFactor != 0) {
- FuncBuilder builder(forInst->getBlock(), ++Block::iterator(forInst));
- auto cleanupForInst = builder.clone(*
forInst)->cast<AffineForOp>();
+ FuncBuilder builder(op->getBlock(), ++Block::iterator(op));
+ auto cleanupForInst = builder.clone(*
op)->cast<AffineForOp>();
AffineMap cleanupMap;
SmallVector<Value *, 4> cleanupOperands;
getCleanupLoopLowerBound(forOp, unrollFactor, &cleanupMap, &cleanupOperands,
// 'forOp'.
FuncBuilder builder = forOp.getBodyBuilder();
- // Keep a pointer to the last non-terminator instruction in the original block
+ // Keep a pointer to the last non-terminator operation in the original block
// so that we know what to clone (since we are doing this in-place).
Block::iterator srcBlockEnd = std::prev(forOp.getBody()->end(), 2);
BlockAndValueMapping map;
map.map(oldIv, newForOp.getInductionVar());
FuncBuilder b = newForOp.getBodyBuilder();
- for (auto &inst : *forOp.getBody()) {
+ for (auto &op : *forOp.getBody()) {
// Step over newForOp in case it is nested under forOp.
- if (&inst == newForOp.getOperation()) {
+ if (&op == newForOp.getOperation()) {
continue;
}
- if (
inst.isa<AffineTerminatorOp>()) {
+ if (
op.isa<AffineTerminatorOp>()) {
continue;
}
- auto *instClone = b.clone(inst, map);
+ auto *instClone = b.clone(op, map);
unsigned idx = 0;
- for (auto r : inst.getResults()) {
+ for (auto r : op.getResults()) {
// Since we do a forward pass over the body, we iteratively augment
// the `map` with everything we clone.
map.map(r, instClone->getResult(idx++));
auto scaledStep = originalStep * factor;
forOp.setStep(scaledStep);
- auto *forInst = forOp.getOperation();
- FuncBuilder b(forInst->getBlock(), ++Block::iterator(forInst));
+ auto *op = forOp.getOperation();
+ FuncBuilder b(op->getBlock(), ++Block::iterator(op));
// Lower-bound map creation.
auto lbMap = forOp.getLowerBoundMap();
auto newForOp = b.create<AffineForOp>(t.getLoc(), lbOperands, lbMap,
ubOperands, ubMap, originalStep);
cloneLoopBodyInto(t, forOp.getInductionVar(), newForOp);
- // Remove all instructions from `t` except `newForOp`.
+ // Remove all operations from `t` except `newForOp`.
auto rit = ++newForOp.getOperation()->getReverseIterator();
auto re = t.getBody()->rend();
- for (auto &inst : llvm::make_early_inc_range(llvm::make_range(rit, re))) {
- inst.erase();
+ for (auto &op : llvm::make_early_inc_range(llvm::make_range(rit, re))) {
+ op.erase();
}
innerLoops.push_back(newForOp);
}
/// Return true if this operation dereferences one or more memref's.
// Temporary utility: will be replaced when this is modeled through
// side-effects/op traits. TODO(b/117228571)
-static bool isMemRefDereferencingOp(Instruction &op) {
+static bool isMemRefDereferencingOp(Operation &op) {
if (op.isa<LoadOp>() || op.isa<StoreOp>() || op.isa<DmaStartOp>() ||
op.isa<DmaWaitOp>())
return true;
ArrayRef<Value *> extraIndices,
AffineMap indexRemap,
ArrayRef<Value *> extraOperands,
- Instruction *domInstFilter,
- Instruction *postDomInstFilter) {
+ Operation *domInstFilter,
+ Operation *postDomInstFilter) {
unsigned newMemRefRank = newMemRef->getType().cast<MemRefType>().getRank();
(void)newMemRefRank; // unused in opt mode
unsigned oldMemRefRank = oldMemRef->getType().cast<MemRefType>().getRank();
llvm::make_unique<PostDominanceInfo>(postDomInstFilter->getFunction());
// The ops where memref replacement succeeds are replaced with new ones.
- SmallVector<Instruction *, 8> opsToErase;
+ SmallVector<Operation *, 8> opsToErase;
// Walk all uses of old memref. Operation using the memref gets replaced.
for (auto &use : llvm::make_early_inc_range(oldMemRef->getUses())) {
};
unsigned memRefOperandPos = getMemRefOperandPos();
- // Construct the new operation instruction using this memref.
+ // Construct the new operation using this memref.
OperationState state(opInst->getContext(), opInst->getLoc(),
opInst->getName());
state.setOperandListToResizable(opInst->hasResizableOperandsList());
return true;
}
-/// Given an operation instruction, inserts one or more single result affine
+/// Given an operation, inserts one or more single result affine
/// apply operations, results of which are exclusively used by this operation
-/// instruction. The operands of these newly created affine apply ops are
+/// operation. The operands of these newly created affine apply ops are
/// guaranteed to be loop iterators or terminal symbols of a function.
///
/// Before
/// uses besides this opInst; otherwise returns the list of affine.apply
/// operations created in output argument `sliceOps`.
void mlir::createAffineComputationSlice(
-
Instruction *opInst, SmallVectorImpl<AffineApplyOp> *sliceOps) {
+
Operation *opInst, SmallVectorImpl<AffineApplyOp> *sliceOps) {
// Collect all operands that are results of affine apply ops.
SmallVector<Value *, 4> subOperands;
subOperands.reserve(opInst->getNumOperands());
}
// Gather sequence of AffineApplyOps reachable from 'subOperands'.
- SmallVector<Instruction *, 4> affineApplyOps;
+ SmallVector<Operation *, 4> affineApplyOps;
getReachableAffineApplyOps(subOperands, affineApplyOps);
// Skip transforming if there are no affine maps to compose.
if (affineApplyOps.empty())
return;
- // Check if all uses of the affine apply op's lie only in this op inst, in
+ // Check if all uses of the affine apply op's lie only in this op op, in
// which case there would be nothing to do.
bool localized = true;
for (auto *op : affineApplyOps) {
}
void mlir::remapFunctionAttrs(
-
Instruction &op, const DenseMap<Attribute, FunctionAttr> &remappingTable) {
+
Operation &op, const DenseMap<Attribute, FunctionAttr> &remappingTable) {
for (auto attr : op.getAttrs()) {
// Do the remapping, if we got the same thing back, then it must contain
// functions that aren't getting remapped.
void mlir::remapFunctionAttrs(
Function &fn, const DenseMap<Attribute, FunctionAttr> &remappingTable) {
- // Look at all instructions in a Function.
- fn.walk(
- [&](Instruction *inst) { remapFunctionAttrs(*inst, remappingTable); });
+ // Look at all operations in a Function.
+ fn.walk([&](Operation *op) { remapFunctionAttrs(*op, remappingTable); });
}
void mlir::remapFunctionAttrs(
llvm::cl::desc(
"Enable testing the normalization of AffineAffineApplyOp "
"where each AffineAffineApplyOp in the composition is a single output "
- "instruction."),
+ "operation."),
llvm::cl::cat(clOptionsCategory));
namespace {
clTestVectorShapeRatio.end());
auto subVectorType =
VectorType::get(shape, FloatType::getF32(f->getContext()));
- // Only filter instructions that operate on a strict super-vector and have one
+ // Only filter operations that operate on a strict super-vector and have one
// return. This makes testing easier.
- auto filter = [subVectorType](Instruction &inst) {
+ auto filter = [subVectorType](Operation &op) {
assert(subVectorType.getElementType() ==
FloatType::getF32(subVectorType.getContext()) &&
"Only f32 supported for now");
- if (!matcher::operatesOnSuperVectors(inst, subVectorType)) {
+ if (!matcher::operatesOnSuperVectors(op, subVectorType)) {
return false;
}
- if (inst.getNumResults() != 1) {
+ if (op.getNumResults() != 1) {
return false;
}
return true;
}
}
-static std::string toString(Instruction *inst) {
+static std::string toString(Operation *op) {
std::string res;
llvm::raw_string_ostream os(res);
- inst->print(os);
+ op->print(os);
return res;
}
constexpr auto kTestSlicingOpName = "slicing-test-op";
using functional::map;
using matcher::Op;
- // Match all OpInstructions with the kTestSlicingOpName name.
- auto filter = [](Instruction &inst) {
- return inst.getName().getStringRef() == kTestSlicingOpName;
+ // Match all operations with the kTestSlicingOpName name.
+ auto filter = [](Operation &op) {
+ return op.getName().getStringRef() == kTestSlicingOpName;
};
return Op(filter);
}
SmallVector<NestedMatch, 8> matches;
patternTestSlicingOps().match(f, &matches);
for (auto m : matches) {
- SetVector<Instruction *> backwardSlice;
+ SetVector<Operation *> backwardSlice;
getBackwardSlice(m.getMatchedOperation(), &backwardSlice);
auto strs = map(toString, backwardSlice);
outs() << "\nmatched: " << *m.getMatchedOperation()
SmallVector<NestedMatch, 8> matches;
patternTestSlicingOps().match(f, &matches);
for (auto m : matches) {
- SetVector<Instruction *> forwardSlice;
+ SetVector<Operation *> forwardSlice;
getForwardSlice(m.getMatchedOperation(), &forwardSlice);
auto strs = map(toString, forwardSlice);
outs() << "\nmatched: " << *m.getMatchedOperation()
SmallVector<NestedMatch, 8> matches;
patternTestSlicingOps().match(f, &matches);
for (auto m : matches) {
- SetVector<Instruction *> staticSlice = getSlice(m.getMatchedOperation());
+ SetVector<Operation *> staticSlice = getSlice(m.getMatchedOperation());
auto strs = map(toString, staticSlice);
outs() << "\nmatched: " << *m.getMatchedOperation() << " static slice: ";
for (const auto &s : strs) {
}
}
-static bool customOpWithAffineMapAttribute(Instruction &inst) {
- return inst.getName().getStringRef() ==
+static bool customOpWithAffineMapAttribute(Operation &op) {
+ return op.getName().getStringRef() ==
VectorizerTestPass::kTestAffineMapOpName;
}
simplifyAffineMap(res).print(outs() << "\nComposed map: ");
}
-static bool affineApplyOp(Instruction &inst) {
- return inst.isa<AffineApplyOp>();
-}
+static bool affineApplyOp(Operation &op) { return op.isa<AffineApplyOp>(); }
-static bool singleResultAffineApplyOpWithoutUses(Instruction &inst) {
- auto app =
inst.dyn_cast<AffineApplyOp>();
+static bool singleResultAffineApplyOpWithoutUses(Operation &op) {
+ auto app =
op.dyn_cast<AffineApplyOp>();
return app && app.use_empty();
}
/// references along fastest varying dimensions and loops with recursive nested
/// patterns capturing imperfectly-nested loop nests; the SLP vectorizer, on
/// the other hand, performs flat pattern matching inside a single unrolled loop
-/// body and stitches together pieces of load and store instructions into full
+/// body and stitches together pieces of load and store operations into full
/// 1-D vectors. We envision that the SLP vectorizer is a good way to capture
/// innermost loop, control-flow dependent patterns that super-vectorization may
/// not be able to capture easily. In other words, super-vectorization does not
struct VectorizationStrategy {
SmallVector<int64_t, 8> vectorSizes;
- DenseMap<Instruction *, unsigned> loopToVectorDim;
+ DenseMap<Operation *, unsigned> loopToVectorDim;
};
} // end anonymous namespace
-static void vectorizeLoopIfProfitable(Instruction *loop,
- unsigned depthInPattern,
+static void vectorizeLoopIfProfitable(Operation *loop, unsigned depthInPattern,
unsigned patternDepth,
VectorizationStrategy *strategy) {
assert(patternDepth > depthInPattern &&
namespace {
struct VectorizationState {
- /// Adds an entry of pre/post vectorization instructions in the state.
- void registerReplacement(Instruction *key, Instruction *value);
+ /// Adds an entry of pre/post vectorization operations in the state.
+ void registerReplacement(Operation *key, Operation *value);
/// When the current vectorization pattern is successful, this erases the
- /// instructions that were marked for erasure in the proper order and resets
+ /// operations that were marked for erasure in the proper order and resets
/// the internal state for the next pattern.
void finishVectorizationPattern();
- // In-order tracking of original Instruction that have been vectorized.
+ // In-order tracking of original Operation that have been vectorized.
// Erase in reverse order.
- SmallVector<Instruction *, 16> toErase;
- // Set of Instruction that have been vectorized (the values in the
+ SmallVector<Operation *, 16> toErase;
+ // Set of Operation that have been vectorized (the values in the
// vectorizationMap for hashed access). The vectorizedSet is used in
- // particular to filter the instructions that have already been vectorized by
+ // particular to filter the operations that have already been vectorized by
// this pattern, when iterating over nested loops in this pattern.
- DenseSet<Instruction *> vectorizedSet;
- // Map of old scalar Instruction to new vectorized Instruction.
- DenseMap<Instruction *, Instruction *> vectorizationMap;
+ DenseSet<Operation *> vectorizedSet;
+ // Map of old scalar Operation to new vectorized Operation.
+ DenseMap<Operation *, Operation *> vectorizationMap;
// Map of old scalar Value to new vectorized Value.
DenseMap<Value *, Value *> replacementMap;
// The strategy drives which loop to vectorize by which amount.
// operations that have been vectorized. They can be retrieved from
// `vectorizationMap` but it is convenient to keep track of them in a separate
// data structure.
- DenseSet<Instruction *> roots;
- // Terminal instructions for the worklist in the vectorizeNonTerminals
+ DenseSet<Operation *> roots;
+ // Terminal operations for the worklist in the vectorizeNonTerminals
// function. They consist of the subset of store operations that have been
// vectorized. They can be retrieved from `vectorizationMap` but it is
// convenient to keep track of them in a separate data structure. Since they
// do not necessarily belong to use-def chains starting from loads (e.g
// storing a constant), we need to handle them in a post-pass.
- DenseSet<Instruction *> terminals;
- // Checks that the type of `inst` is StoreOp and adds it to the terminals
- // set.
- void registerTerminal(Instruction *inst);
+ DenseSet<Operation *> terminals;
+ // Checks that the type of `op` is StoreOp and adds it to the terminals set.
+ void registerTerminal(Operation *op);
private:
void registerReplacement(Value *key, Value *value);
} // end namespace
-void VectorizationState::registerReplacement(Instruction *key,
- Instruction *value) {
+void VectorizationState::registerReplacement(Operation *key, Operation *value) {
LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ commit vectorized op: ");
LLVM_DEBUG(key->print(dbgs()));
LLVM_DEBUG(dbgs() << " into ");
}
}
-void VectorizationState::registerTerminal(Instruction *inst) {
- assert(inst->isa<StoreOp>() && "terminal must be a StoreOp");
- assert(terminals.count(inst) == 0 &&
+void VectorizationState::registerTerminal(Operation *op) {
+ assert(op->isa<StoreOp>() && "terminal must be a StoreOp");
+ assert(terminals.count(op) == 0 &&
"terminal was already inserted previously");
- terminals.insert(inst);
+ terminals.insert(op);
}
void VectorizationState::finishVectorizationPattern() {
while (!toErase.empty()) {
- auto *inst = toErase.pop_back_val();
+ auto *op = toErase.pop_back_val();
LLVM_DEBUG(dbgs() << "\n[early-vect] finishVectorizationPattern erase: ");
- LLVM_DEBUG(inst->print(dbgs()));
- inst->erase();
+ LLVM_DEBUG(op->print(dbgs()));
+ op->erase();
}
}
using namespace functional;
loop.setStep(step);
- FilterFunctionType notVectorizedThisPattern = [state](Instruction &inst) {
- if (!matcher::isLoadOrStore(inst)) {
+ FilterFunctionType notVectorizedThisPattern = [state](Operation &op) {
+ if (!matcher::isLoadOrStore(op)) {
return false;
}
- return state->vectorizationMap.count(&inst) == 0 &&
- state->vectorizedSet.count(&inst) == 0 &&
- state->roots.count(&inst) == 0 && state->terminals.count(&inst) == 0;
+ return state->vectorizationMap.count(&op) == 0 &&
+ state->vectorizedSet.count(&op) == 0 &&
+ state->roots.count(&op) == 0 && state->terminals.count(&op) == 0;
};
auto loadAndStores = matcher::Op(notVectorizedThisPattern);
SmallVector<NestedMatch, 8> loadAndStoresMatches;
/// we can build a cost model and a search procedure.
static FilterFunctionType
isVectorizableLoopPtrFactory(unsigned fastestVaryingMemRefDimension) {
- return [fastestVaryingMemRefDimension](Instruction &forInst) {
- auto loop = for
Inst.cast<AffineForOp>();
+ return [fastestVaryingMemRefDimension](Operation &forOp) {
+ auto loop = for
Op.cast<AffineForOp>();
return isVectorizableLoopAlongFastestVaryingMemRefDim(
loop, fastestVaryingMemRefDimension);
};
/// element type.
/// If `type` is not a valid vector type or if the scalar constant is not a
/// valid vector element type, returns nullptr.
-static Value *vectorizeConstant(Instruction *inst, ConstantOp constant,
- Type type) {
+static Value *vectorizeConstant(Operation *op, ConstantOp constant, Type type) {
if (!type || !type.isa<VectorType>() ||
!VectorType::isValidElementType(constant.getType())) {
return nullptr;
}
- FuncBuilder b(inst);
- Location loc = inst->getLoc();
+ FuncBuilder b(op);
+ Location loc = op->getLoc();
auto vectorType = type.cast<VectorType>();
auto attr = SplatElementsAttr::get(vectorType, constant.getValue());
auto *constantOpInst = constant.getOperation();
return b.createOperation(state)->getResult(0);
}
-/// Tries to vectorize a given operand `op` of Instruction `inst` during
+/// Tries to vectorize a given operand `op` of Operation `op` during
/// def-chain propagation or during terminal vectorization, by applying the
/// following logic:
-/// 1. if the defining instruction is part of the vectorizedSet (i.e. vectorized
+/// 1. if the defining operation is part of the vectorizedSet (i.e. vectorized
/// useby -def propagation), `op` is already in the proper vector form;
/// 2. otherwise, the `op` may be in some other vector form that fails to
/// vectorize atm (i.e. broadcasting required), returns nullptr to indicate
/// vectorization is possible with the above logic. Returns nullptr otherwise.
///
/// TODO(ntv): handle more complex cases.
-static Value *vectorizeOperand(Value *operand, Instruction *inst,
+static Value *vectorizeOperand(Value *operand, Operation *op,
VectorizationState *state) {
LLVM_DEBUG(dbgs() << "\n[early-vect]vectorize operand: ");
LLVM_DEBUG(operand->print(dbgs()));
// 3. vectorize constant.
if (auto constant = operand->getDefiningOp()->dyn_cast<ConstantOp>()) {
return vectorizeConstant(
- inst, constant,
+ op, constant,
VectorType::get(state->strategy->vectorSizes, operand->getType()));
}
// 4. currently non-vectorizable.
return nullptr;
};
-/// Encodes Instruction-specific behavior for vectorization. In general we
+/// Encodes Operation-specific behavior for vectorization. In general we
/// assume that all operands of an op must be vectorized but this is not always
/// true. In the future, it would be nice to have a trait that describes how a
/// particular operation vectorizes. For now we implement the case distinction
/// TODO(ntv): consider adding a trait to Op to describe how it gets vectorized.
/// Maybe some Ops are not vectorizable or require some tricky logic, we cannot
/// do one-off logic here; ideally it would be TableGen'd.
-static Instruction *vectorizeOneInstruction(Instruction *opInst,
- VectorizationState *state) {
+static Operation *vectorizeOneOperation(Operation *opInst,
+ VectorizationState *state) {
// Sanity checks.
assert(!opInst->isa<LoadOp>() &&
"all loads must have already been fully vectorized independently");
// Create a clone of the op with the proper operands and return types.
// TODO(ntv): The following assumes there is always an op with a fixed
// name that works both in scalar mode and vector mode.
- // TODO(ntv): Is it worth considering an Instruction.clone operation
- // which changes the type so we can promote an Instruction with less
- // boilerplate?
+ // TODO(ntv): Is it worth considering an Operation.clone operation which
+ // changes the type so we can promote an Operation with less boilerplate?
FuncBuilder b(opInst);
OperationState newOp(b.getContext(), opInst->getLoc(),
opInst->getName().getStringRef(), vectorOperands,
/// replacementMap. If any such replacement is missing, vectorization fails.
static LogicalResult vectorizeNonTerminals(VectorizationState *state) {
// 1. create initial worklist with the uses of the roots.
- SetVector<Instruction *> worklist;
+ SetVector<Operation *> worklist;
// Note: state->roots have already been vectorized and must not be vectorized
- // again. This fits `getForwardSlice` which does not insert `inst` in the
+ // again. This fits `getForwardSlice` which does not insert `op` in the
// result.
// Note: we have to exclude terminals because some of their defs may not be
// nested under the vectorization pattern (e.g. constants defined in an
// encompassing scope).
// TODO(ntv): Use a backward slice for terminals, avoid special casing and
// merge implementations.
- for (auto *inst : state->roots) {
- getForwardSlice(inst, &worklist, [state](Instruction *inst) {
- return state->terminals.count(inst) == 0; // propagate if not terminal
+ for (auto *op : state->roots) {
+ getForwardSlice(op, &worklist, [state](Operation *op) {
+ return state->terminals.count(op) == 0; // propagate if not terminal
});
}
// We merged multiple slices, topological order may not hold anymore.
worklist = topologicalSort(worklist);
for (unsigned i = 0; i < worklist.size(); ++i) {
- auto *inst = worklist[i];
+ auto *op = worklist[i];
LLVM_DEBUG(dbgs() << "\n[early-vect] vectorize use: ");
- LLVM_DEBUG(inst->print(dbgs()));
+ LLVM_DEBUG(op->print(dbgs()));
- // Create vector form of the instruction.
- // Insert it just before inst, on success register inst as replaced.
- auto *vectorizedInst = vectorizeOneInstruction(inst, state);
+ // Create vector form of the operation.
+ // Insert it just before op, on success register op as replaced.
+ auto *vectorizedInst = vectorizeOneOperation(op, state);
if (!vectorizedInst) {
return failure();
}
// Note that we cannot just call replaceAllUsesWith because it may
// result in ops with mixed types, for ops whose operands have not all
// yet been vectorized. This would be invalid IR.
- state->registerReplacement(inst, vectorizedInst);
+ state->registerReplacement(op, vectorizedInst);
}
return success();
}
return guard.failure();
}
- // 2. Vectorize operations reached by use-def chains from root
- // except the terminals (store instructions) that need to be
- // post-processed separately.
+ // 2. Vectorize operations reached by use-def chains from root except the
+ // terminals (store operations) that need to be post-processed separately.
// TODO(ntv): add more as we expand.
if (failed(vectorizeNonTerminals(&state))) {
LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ failed vectorizeNonTerminals");
// encompassing scope).
// TODO(ntv): Use a backward slice for terminals, avoid special casing and
// merge implementations.
- for (auto *inst : state.terminals) {
- if (!vectorizeOneInstruction(inst, &state)) { // nullptr == failure
+ for (auto *op : state.terminals) {
+ if (!vectorizeOneOperation(op, &state)) { // nullptr == failure
LLVM_DEBUG(dbgs() << "\n[early-vect]+++++ failed to vectorize terminals");
return guard.failure();
}
projects / platform / upstream / llvm.git / commitdiff