RegionRange(Arg &&arg)
: RegionRange(ArrayRef<std::unique_ptr<Region>>(std::forward<Arg>(arg))) {
}
- template <typename Arg>
- RegionRange(Arg &&arg,
- typename std::enable_if_t<
- std::is_constructible<ArrayRef<Region *>, Arg>::value &&
- !std::is_convertible<Arg, Region *>::value> * = nullptr)
- : RegionRange(ArrayRef<Region *>(std::forward<Arg>(arg))) {}
RegionRange(ArrayRef<std::unique_ptr<Region>> regions);
RegionRange(ArrayRef<Region *> regions);
#include "mlir/IR/OpDefinition.h"
+//===----------------------------------------------------------------------===//
+// LoopLike Interfaces
+//===----------------------------------------------------------------------===//
+
/// Include the generated interface declarations.
#include "mlir/Interfaces/LoopLikeInterface.h.inc"
+//===----------------------------------------------------------------------===//
+// LoopLike Utilities
+//===----------------------------------------------------------------------===//
+
+namespace mlir {
+/// Move loop invariant code out of a `looplike` operation.
+void moveLoopInvariantCode(LoopLikeOpInterface looplike);
+} // namespace mlir
+
#endif // MLIR_INTERFACES_LOOPLIKEINTERFACE_H_
class Operation;
class Region;
class RegionBranchOpInterface;
-class RegionRange;
/// Given a list of regions, perform control flow sinking on them. For each
/// region, control-flow sinking moves operations that dominate the region but
///
/// Returns the number of operations sunk.
size_t
-controlFlowSink(RegionRange regions, DominanceInfo &domInfo,
+controlFlowSink(ArrayRef<Region *> regions, DominanceInfo &domInfo,
function_ref<bool(Operation *, Region *)> shouldMoveIntoRegion,
function_ref<void(Operation *, Region *)> moveIntoRegion);
+++ /dev/null
-//===- LoopInvariantCodeMotionUtils.h - LICM Utils --------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
-#define MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
-
-#include "mlir/Support/LLVM.h"
-
-namespace mlir {
-
-class LoopLikeOpInterface;
-class Operation;
-class Region;
-class RegionRange;
-class Value;
-
-/// Given a list of regions, perform loop-invariant code motion. An operation is
-/// loop-invariant if it depends only of values defined outside of the loop.
-/// LICM moves these operations out of the loop body so that they are not
-/// computed more than once.
-///
-/// Example:
-///
-/// ```mlir
-/// affine.for %arg0 = 0 to 10 {
-/// affine.for %arg1 = 0 to 10 {
-/// %v0 = arith.addi %arg0, %arg0 : i32
-/// %v1 = arith.addi %v0, %arg1 : i32
-/// }
-/// }
-/// ```
-///
-/// After LICM:
-///
-/// ```mlir
-/// affine.for %arg0 = 0 to 10 {
-/// %v0 = arith.addi %arg0, %arg0 : i32
-/// affine.for %arg1 = 0 to 10 {
-/// %v1 = arith.addi %v0, %arg1 : i32
-/// }
-/// }
-/// ```
-///
-/// Users must supply three callbacks.
-///
-/// - `isDefinedOutsideRegion` returns true if the given value is invariant with
-/// respect to the given region. A common implementation might be:
-/// `value.getParentRegion()->isProperAncestor(region)`.
-/// - `shouldMoveOutOfRegion` returns true if the provided operation can be
-/// moved of the given region, e.g. if it is side-effect free.
-/// - `moveOutOfRegion` moves the operation out of the given region. A common
-/// implementation might be: `op->moveBefore(region->getParentOp())`.
-///
-/// An operation is moved if all of its operands satisfy
-/// `isDefinedOutsideRegion` and it satisfies `shouldMoveOutOfRegion`.
-///
-/// Returns the number of operations moved.
-size_t moveLoopInvariantCode(
- RegionRange regions,
- function_ref<bool(Value, Region *)> isDefinedOutsideRegion,
- function_ref<bool(Operation *, Region *)> shouldMoveOutOfRegion,
- function_ref<void(Operation *, Region *)> moveOutOfRegion);
-
-/// Move side-effect free loop invariant code out of a loop-like op using
-/// methods provided by the interface.
-size_t moveLoopInvariantCode(LoopLikeOpInterface loopLike);
-
-} // end namespace mlir
-
-#endif // MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
+++ /dev/null
-//===- SideEffectUtils.h - Side Effect Utils --------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_TRANSFORMS_SIDEFFECTUTILS_H
-#define MLIR_TRANSFORMS_SIDEFFECTUTILS_H
-
-namespace mlir {
-
-class Operation;
-
-/// Returns true if the given operation is side-effect free.
-///
-/// An operation is side-effect free if its implementation of
-/// `MemoryEffectOpInterface` indicates that it has no memory effects. For
-/// example, it may implement `NoSideEffect` in ODS. Alternatively, if the
-/// operation `HasRecursiveSideEffects`, then it is side-effect free if all of
-/// its nested operations are side-effect free.
-///
-/// If the operation has both, then it is side-effect free if both conditions
-/// are satisfied.
-bool isSideEffectFree(Operation *op);
-
-} // end namespace mlir
-
-#endif // MLIR_TRANSFORMS_SIDEFFECTUTILS_H
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Debug.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
#include "mlir/Transforms/Passes.h"
using namespace mlir;
add_mlir_interface_library(TilingInterface)
add_mlir_interface_library(VectorInterfaces)
add_mlir_interface_library(ViewLikeInterface)
-add_mlir_interface_library(LoopLikeInterface)
+
+add_mlir_library(MLIRLoopLikeInterface
+ LoopLikeInterface.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${MLIR_MAIN_INCLUDE_DIR}/mlir/Interfaces
+
+ DEPENDS
+ MLIRLoopLikeInterfaceIncGen
+
+ LINK_LIBS PUBLIC
+ MLIRIR
+ MLIRSideEffectInterfaces
+ )
//===----------------------------------------------------------------------===//
#include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/Debug.h"
+#include <queue>
using namespace mlir;
+#define DEBUG_TYPE "loop-like"
+
+//===----------------------------------------------------------------------===//
+// LoopLike Interfaces
+//===----------------------------------------------------------------------===//
+
/// Include the definitions of the loop-like interfaces.
#include "mlir/Interfaces/LoopLikeInterface.cpp.inc"
+
+//===----------------------------------------------------------------------===//
+// LoopLike Utilities
+//===----------------------------------------------------------------------===//
+
+/// Returns true if the given operation is side-effect free as are all of its
+/// nested operations.
+///
+/// TODO: There is a duplicate function in ControlFlowSink. Move
+/// `moveLoopInvariantCode` to TransformUtils and then factor out this function.
+static bool isSideEffectFree(Operation *op) {
+ if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
+ // If the op has side-effects, it cannot be moved.
+ if (!memInterface.hasNoEffect())
+ return false;
+ // If the op does not have recursive side effects, then it can be moved.
+ if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
+ return true;
+ } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
+ // Otherwise, if the op does not implement the memory effect interface and
+ // it does not have recursive side effects, then it cannot be known that the
+ // op is moveable.
+ return false;
+ }
+
+ // Recurse into the regions and ensure that all nested ops can also be moved.
+ for (Region ®ion : op->getRegions())
+ for (Operation &op : region.getOps())
+ if (!isSideEffectFree(&op))
+ return false;
+ return true;
+}
+
+/// Checks whether the given op can be hoisted by checking that
+/// - the op and none of its contained operations depend on values inside of the
+/// loop (by means of calling definedOutside).
+/// - the op has no side-effects.
+static bool canBeHoisted(Operation *op,
+ function_ref<bool(Value)> definedOutside) {
+ if (!isSideEffectFree(op))
+ return false;
+
+ // Do not move terminators.
+ if (op->hasTrait<OpTrait::IsTerminator>())
+ return false;
+
+ // Walk the nested operations and check that all used values are either
+ // defined outside of the loop or in a nested region, but not at the level of
+ // the loop body.
+ auto walkFn = [&](Operation *child) {
+ for (Value operand : child->getOperands()) {
+ // Ignore values defined in a nested region.
+ if (op->isAncestor(operand.getParentRegion()->getParentOp()))
+ continue;
+ if (!definedOutside(operand))
+ return WalkResult::interrupt();
+ }
+ return WalkResult::advance();
+ };
+ return !op->walk(walkFn).wasInterrupted();
+}
+
+void mlir::moveLoopInvariantCode(LoopLikeOpInterface looplike) {
+ Region *loopBody = &looplike.getLoopBody();
+
+ std::queue<Operation *> worklist;
+ // Add top-level operations in the loop body to the worklist.
+ for (Operation &op : loopBody->getOps())
+ worklist.push(&op);
+
+ auto definedOutside = [&](Value value) {
+ return looplike.isDefinedOutsideOfLoop(value);
+ };
+
+ while (!worklist.empty()) {
+ Operation *op = worklist.front();
+ worklist.pop();
+ // Skip ops that have already been moved. Check if the op can be hoisted.
+ if (op->getParentRegion() != loopBody || !canBeHoisted(op, definedOutside))
+ continue;
+
+ looplike.moveOutOfLoop(op);
+
+ // Since the op has been moved, we need to check its users within the
+ // top-level of the loop body.
+ for (Operation *user : op->getUsers())
+ if (user->getParentRegion() == loopBody)
+ worklist.push(user);
+ }
+}
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Transforms/ControlFlowSinkUtils.h"
#include "mlir/Transforms/Passes.h"
-#include "mlir/Transforms/SideEffectUtils.h"
using namespace mlir;
};
} // end anonymous namespace
+/// Returns true if the given operation is side-effect free as are all of its
+/// nested operations.
+static bool isSideEffectFree(Operation *op) {
+ if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
+ // If the op has side-effects, it cannot be moved.
+ if (!memInterface.hasNoEffect())
+ return false;
+ // If the op does not have recursive side effects, then it can be moved.
+ if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
+ return true;
+ } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
+ // Otherwise, if the op does not implement the memory effect interface and
+ // it does not have recursive side effects, then it cannot be known that the
+ // op is moveable.
+ return false;
+ }
+
+ // Recurse into the regions and ensure that all nested ops can also be moved.
+ for (Region ®ion : op->getRegions())
+ for (Operation &op : region.getOps())
+ if (!isSideEffectFree(&op))
+ return false;
+ return true;
+}
+
void ControlFlowSink::runOnOperation() {
auto &domInfo = getAnalysis<DominanceInfo>();
getOperation()->walk([&](RegionBranchOpInterface branch) {
//===----------------------------------------------------------------------===//
#include "PassDetail.h"
+#include "mlir/IR/Builders.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Transforms/Passes.h"
-#include "mlir/Transforms/SideEffectUtils.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "licm"
using namespace mlir;
// Walk through all loops in a function in innermost-loop-first order. This
// way, we first LICM from the inner loop, and place the ops in
// the outer loop, which in turn can be further LICM'ed.
- getOperation()->walk(
- [&](LoopLikeOpInterface loopLike) { moveLoopInvariantCode(loopLike); });
+ getOperation()->walk([&](LoopLikeOpInterface loopLike) {
+ LLVM_DEBUG(loopLike.print(llvm::dbgs() << "\nOriginal loop:\n"));
+ moveLoopInvariantCode(loopLike);
+ });
}
std::unique_ptr<Pass> mlir::createLoopInvariantCodeMotionPass() {
FoldUtils.cpp
GreedyPatternRewriteDriver.cpp
InliningUtils.cpp
- LoopInvariantCodeMotionUtils.cpp
RegionUtils.cpp
- SideEffectUtils.cpp
ADDITIONAL_HEADER_DIRS
${MLIR_MAIN_INCLUDE_DIR}/mlir/Transforms
LINK_LIBS PUBLIC
MLIRAnalysis
- MLIRLoopLikeInterface
MLIRRewrite
)
/// Given a list of regions, find operations to sink and sink them. Return the
/// number of operations sunk.
- size_t sinkRegions(RegionRange regions);
+ size_t sinkRegions(ArrayRef<Region *> regions);
private:
/// Given a region and an op which dominates the region, returns true if all
}
}
-size_t Sinker::sinkRegions(RegionRange regions) {
+size_t Sinker::sinkRegions(ArrayRef<Region *> regions) {
for (Region *region : regions)
if (!region->empty())
sinkRegion(region);
}
size_t mlir::controlFlowSink(
- RegionRange regions, DominanceInfo &domInfo,
+ ArrayRef<Region *> regions, DominanceInfo &domInfo,
function_ref<bool(Operation *, Region *)> shouldMoveIntoRegion,
function_ref<void(Operation *, Region *)> moveIntoRegion) {
return Sinker(shouldMoveIntoRegion, moveIntoRegion, domInfo)
+++ /dev/null
-//===- LoopInvariantCodeMotionUtils.cpp - LICM Utils ------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the implementation of the core LICM algorithm.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
-#include "mlir/IR/Operation.h"
-#include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Transforms/SideEffectUtils.h"
-#include "llvm/Support/Debug.h"
-#include <queue>
-
-#define DEBUG_TYPE "licm"
-
-using namespace mlir;
-
-/// Checks whether the given op can be hoisted by checking that
-/// - the op and none of its contained operations depend on values inside of the
-/// loop (by means of calling definedOutside).
-/// - the op has no side-effects.
-static bool canBeHoisted(Operation *op,
- function_ref<bool(Value)> definedOutside) {
- // Do not move terminators.
- if (op->hasTrait<OpTrait::IsTerminator>())
- return false;
-
- // Walk the nested operations and check that all used values are either
- // defined outside of the loop or in a nested region, but not at the level of
- // the loop body.
- auto walkFn = [&](Operation *child) {
- for (Value operand : child->getOperands()) {
- // Ignore values defined in a nested region.
- if (op->isAncestor(operand.getParentRegion()->getParentOp()))
- continue;
- if (!definedOutside(operand))
- return WalkResult::interrupt();
- }
- return WalkResult::advance();
- };
- return !op->walk(walkFn).wasInterrupted();
-}
-
-size_t mlir::moveLoopInvariantCode(
- RegionRange regions,
- function_ref<bool(Value, Region *)> isDefinedOutsideRegion,
- function_ref<bool(Operation *, Region *)> shouldMoveOutOfRegion,
- function_ref<void(Operation *, Region *)> moveOutOfRegion) {
- size_t numMoved = 0;
-
- for (Region *region : regions) {
- LLVM_DEBUG(llvm::dbgs() << "Original loop:\n" << *region->getParentOp());
-
- std::queue<Operation *> worklist;
- // Add top-level operations in the loop body to the worklist.
- for (Operation &op : region->getOps())
- worklist.push(&op);
-
- auto definedOutside = [&](Value value) {
- return isDefinedOutsideRegion(value, region);
- };
-
- while (!worklist.empty()) {
- Operation *op = worklist.front();
- worklist.pop();
- // Skip ops that have already been moved. Check if the op can be hoisted.
- if (op->getParentRegion() != region)
- continue;
-
- LLVM_DEBUG(llvm::dbgs() << "Checking op: " << *op);
- if (!shouldMoveOutOfRegion(op, region) ||
- !canBeHoisted(op, definedOutside))
- continue;
-
- LLVM_DEBUG(llvm::dbgs() << "Moving loop-invariant op: " << *op);
- moveOutOfRegion(op, region);
- ++numMoved;
-
- // Since the op has been moved, we need to check its users within the
- // top-level of the loop body.
- for (Operation *user : op->getUsers())
- if (user->getParentRegion() == region)
- worklist.push(user);
- }
- }
-
- return numMoved;
-}
-
-size_t mlir::moveLoopInvariantCode(LoopLikeOpInterface loopLike) {
- return moveLoopInvariantCode(
- {&loopLike.getLoopBody()},
- [&](Value value, Region *) {
- return loopLike.isDefinedOutsideOfLoop(value);
- },
- [&](Operation *op, Region *) { return isSideEffectFree(op); },
- [&](Operation *op, Region *) { loopLike.moveOutOfLoop(op); });
-}
+++ /dev/null
-//===- SideEffectUtils.cpp - Side Effect Utils ------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Transforms/SideEffectUtils.h"
-#include "mlir/IR/Operation.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
-
-using namespace mlir;
-
-bool mlir::isSideEffectFree(Operation *op) {
- if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
- // If the op has side-effects, it cannot be moved.
- if (!memInterface.hasNoEffect())
- return false;
- // If the op does not have recursive side effects, then it can be moved.
- if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
- return true;
- } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
- // Otherwise, if the op does not implement the memory effect interface and
- // it does not have recursive side effects, then it cannot be known that the
- // op is moveable.
- return false;
- }
-
- // Recurse into the regions and ensure that all nested ops can also be moved.
- for (Region ®ion : op->getRegions())
- for (Operation &op : region.getOps())
- if (!isSideEffectFree(&op))
- return false;
- return true;
-}
deps = [
":ControlFlowInterfaces",
":IR",
- ":LoopLikeInterface",
":Rewrite",
":SideEffectInterfaces",
":Support",
projects / platform / upstream / llvm.git / commitdiff