LICM is refactored into a utility that is application on any region. The implementation is moved to Transform/Utils.
RegionRange(Arg &&arg)
: RegionRange(ArrayRef<std::unique_ptr<Region>>(std::forward<Arg>(arg))) {
}
+ template <typename Arg>
+ RegionRange(
+ Arg &&arg,
+ typename std::enable_if_t<
+ std::is_constructible<ArrayRef<Region *>, Arg>::value> * = nullptr)
+ : RegionRange(ArrayRef<Region *>(std::forward<Arg>(arg))) {}
RegionRange(ArrayRef<std::unique_ptr<Region>> regions);
RegionRange(ArrayRef<Region *> regions);
#include "mlir/IR/OpDefinition.h"
-//===----------------------------------------------------------------------===//
-// LoopLike Interfaces
-//===----------------------------------------------------------------------===//
-
/// Include the generated interface declarations.
#include "mlir/Interfaces/LoopLikeInterface.h.inc"
-//===----------------------------------------------------------------------===//
-// LoopLike Utilities
-//===----------------------------------------------------------------------===//
-
-namespace mlir {
-/// Move loop invariant code out of a `looplike` operation.
-void moveLoopInvariantCode(LoopLikeOpInterface looplike);
-} // namespace mlir
-
#endif // MLIR_INTERFACES_LOOPLIKEINTERFACE_H_
class Operation;
class Region;
class RegionBranchOpInterface;
+class RegionRange;
/// Given a list of regions, perform control flow sinking on them. For each
/// region, control-flow sinking moves operations that dominate the region but
///
/// Returns the number of operations sunk.
size_t
-controlFlowSink(ArrayRef<Region *> regions, DominanceInfo &domInfo,
+controlFlowSink(RegionRange regions, DominanceInfo &domInfo,
function_ref<bool(Operation *, Region *)> shouldMoveIntoRegion,
function_ref<void(Operation *, Region *)> moveIntoRegion);
--- /dev/null
+//===- LoopInvariantCodeMotionUtils.h - LICM Utils --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
+#define MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
+
+#include "mlir/Support/LLVM.h"
+
+namespace mlir {
+
+class LoopLikeOpInterface;
+class Operation;
+class Region;
+class RegionRange;
+class Value;
+
+/// Given a list of regions, perform loop-invariant code motion. An operation is
+/// loop-invariant if it depends only of values defined outside of the loop.
+/// LICM moves these operations out of the loop body so that they are not
+/// computed more than once.
+///
+/// Example:
+///
+/// ```mlir
+/// affine.for %arg0 = 0 to 10 {
+/// affine.for %arg1 = 0 to 10 {
+/// %v0 = arith.addi %arg0, %arg0 : i32
+/// %v1 = arith.addi %v0, %arg1 : i32
+/// }
+/// }
+/// ```
+///
+/// After LICM:
+///
+/// ```mlir
+/// affine.for %arg0 = 0 to 10 {
+/// %v0 = arith.addi %arg0, %arg0 : i32
+/// affine.for %arg1 = 0 to 10 {
+/// %v1 = arith.addi %v0, %arg1 : i32
+/// }
+/// }
+/// ```
+///
+/// Users must supply three callbacks.
+///
+/// - `isDefinedOutsideRegion` returns true if the given value is invariant with
+/// respect to the given region. A common implementation might be:
+/// `value.getParentRegion()->isProperAncestor(region)`.
+/// - `shouldMoveOutOfRegion` returns true if the provided operation can be
+/// moved of the given region, e.g. if it is side-effect free.
+/// - `moveOutOfRegion` moves the operation out of the given region. A common
+/// implementation might be: `op->moveBefore(region->getParentOp())`.
+///
+/// An operation is moved if all of its operands satisfy
+/// `isDefinedOutsideRegion` and it satisfies `shouldMoveOutOfRegion`.
+///
+/// Returns the number of operations moved.
+size_t moveLoopInvariantCode(
+ RegionRange regions,
+ function_ref<bool(Value, Region *)> isDefinedOutsideRegion,
+ function_ref<bool(Operation *, Region *)> shouldMoveOutOfRegion,
+ function_ref<void(Operation *, Region *)> moveOutOfRegion);
+
+/// Move side-effect free loop invariant code out of a loop-like op using
+/// methods provided by the interface.
+size_t moveLoopInvariantCode(LoopLikeOpInterface loopLike);
+
+} // end namespace mlir
+
+#endif // MLIR_TRANSFORMS_LOOPINVARIANTCODEMOTIONUTILS_H
--- /dev/null
+//===- SideEffectUtils.h - Side Effect Utils --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TRANSFORMS_SIDEFFECTUTILS_H
+#define MLIR_TRANSFORMS_SIDEFFECTUTILS_H
+
+namespace mlir {
+
+class Operation;
+
+/// Returns true if the given operation is side-effect free.
+///
+/// An operation is side-effect free if its implementation of
+/// `MemoryEffectOpInterface` indicates that it has no memory effects. For
+/// example, it may implement `NoSideEffect` in ODS. Alternatively, if the
+/// operation `HasRecursiveSideEffects`, then it is side-effect free if all of
+/// its nested operations are side-effect free.
+///
+/// If the operation has both, then it is side-effect free if both conditions
+/// are satisfied.
+bool isSideEffectFree(Operation *op);
+
+} // end namespace mlir
+
+#endif // MLIR_TRANSFORMS_SIDEFFECTUTILS_H
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Dominance.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Debug.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
#include "mlir/Transforms/Passes.h"
using namespace mlir;
add_mlir_interface_library(TilingInterface)
add_mlir_interface_library(VectorInterfaces)
add_mlir_interface_library(ViewLikeInterface)
-
-add_mlir_library(MLIRLoopLikeInterface
- LoopLikeInterface.cpp
-
- ADDITIONAL_HEADER_DIRS
- ${MLIR_MAIN_INCLUDE_DIR}/mlir/Interfaces
-
- DEPENDS
- MLIRLoopLikeInterfaceIncGen
-
- LINK_LIBS PUBLIC
- MLIRIR
- MLIRSideEffectInterfaces
- )
+add_mlir_interface_library(LoopLikeInterface)
//===----------------------------------------------------------------------===//
#include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/Support/Debug.h"
-#include <queue>
using namespace mlir;
-#define DEBUG_TYPE "loop-like"
-
-//===----------------------------------------------------------------------===//
-// LoopLike Interfaces
-//===----------------------------------------------------------------------===//
-
/// Include the definitions of the loop-like interfaces.
#include "mlir/Interfaces/LoopLikeInterface.cpp.inc"
-
-//===----------------------------------------------------------------------===//
-// LoopLike Utilities
-//===----------------------------------------------------------------------===//
-
-/// Returns true if the given operation is side-effect free as are all of its
-/// nested operations.
-///
-/// TODO: There is a duplicate function in ControlFlowSink. Move
-/// `moveLoopInvariantCode` to TransformUtils and then factor out this function.
-static bool isSideEffectFree(Operation *op) {
- if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
- // If the op has side-effects, it cannot be moved.
- if (!memInterface.hasNoEffect())
- return false;
- // If the op does not have recursive side effects, then it can be moved.
- if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
- return true;
- } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
- // Otherwise, if the op does not implement the memory effect interface and
- // it does not have recursive side effects, then it cannot be known that the
- // op is moveable.
- return false;
- }
-
- // Recurse into the regions and ensure that all nested ops can also be moved.
- for (Region ®ion : op->getRegions())
- for (Operation &op : region.getOps())
- if (!isSideEffectFree(&op))
- return false;
- return true;
-}
-
-/// Checks whether the given op can be hoisted by checking that
-/// - the op and none of its contained operations depend on values inside of the
-/// loop (by means of calling definedOutside).
-/// - the op has no side-effects.
-static bool canBeHoisted(Operation *op,
- function_ref<bool(Value)> definedOutside) {
- if (!isSideEffectFree(op))
- return false;
-
- // Do not move terminators.
- if (op->hasTrait<OpTrait::IsTerminator>())
- return false;
-
- // Walk the nested operations and check that all used values are either
- // defined outside of the loop or in a nested region, but not at the level of
- // the loop body.
- auto walkFn = [&](Operation *child) {
- for (Value operand : child->getOperands()) {
- // Ignore values defined in a nested region.
- if (op->isAncestor(operand.getParentRegion()->getParentOp()))
- continue;
- if (!definedOutside(operand))
- return WalkResult::interrupt();
- }
- return WalkResult::advance();
- };
- return !op->walk(walkFn).wasInterrupted();
-}
-
-void mlir::moveLoopInvariantCode(LoopLikeOpInterface looplike) {
- Region *loopBody = &looplike.getLoopBody();
-
- std::queue<Operation *> worklist;
- // Add top-level operations in the loop body to the worklist.
- for (Operation &op : loopBody->getOps())
- worklist.push(&op);
-
- auto definedOutside = [&](Value value) {
- return looplike.isDefinedOutsideOfLoop(value);
- };
-
- while (!worklist.empty()) {
- Operation *op = worklist.front();
- worklist.pop();
- // Skip ops that have already been moved. Check if the op can be hoisted.
- if (op->getParentRegion() != loopBody || !canBeHoisted(op, definedOutside))
- continue;
-
- looplike.moveOutOfLoop(op);
-
- // Since the op has been moved, we need to check its users within the
- // top-level of the loop body.
- for (Operation *user : op->getUsers())
- if (user->getParentRegion() == loopBody)
- worklist.push(user);
- }
-}
#include "mlir/Interfaces/SideEffectInterfaces.h"
#include "mlir/Transforms/ControlFlowSinkUtils.h"
#include "mlir/Transforms/Passes.h"
+#include "mlir/Transforms/SideEffectUtils.h"
using namespace mlir;
};
} // end anonymous namespace
-/// Returns true if the given operation is side-effect free as are all of its
-/// nested operations.
-static bool isSideEffectFree(Operation *op) {
- if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
- // If the op has side-effects, it cannot be moved.
- if (!memInterface.hasNoEffect())
- return false;
- // If the op does not have recursive side effects, then it can be moved.
- if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
- return true;
- } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
- // Otherwise, if the op does not implement the memory effect interface and
- // it does not have recursive side effects, then it cannot be known that the
- // op is moveable.
- return false;
- }
-
- // Recurse into the regions and ensure that all nested ops can also be moved.
- for (Region ®ion : op->getRegions())
- for (Operation &op : region.getOps())
- if (!isSideEffectFree(&op))
- return false;
- return true;
-}
-
void ControlFlowSink::runOnOperation() {
auto &domInfo = getAnalysis<DominanceInfo>();
getOperation()->walk([&](RegionBranchOpInterface branch) {
//===----------------------------------------------------------------------===//
#include "PassDetail.h"
-#include "mlir/IR/Builders.h"
#include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
#include "mlir/Transforms/Passes.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-
-#define DEBUG_TYPE "licm"
+#include "mlir/Transforms/SideEffectUtils.h"
using namespace mlir;
// Walk through all loops in a function in innermost-loop-first order. This
// way, we first LICM from the inner loop, and place the ops in
// the outer loop, which in turn can be further LICM'ed.
- getOperation()->walk([&](LoopLikeOpInterface loopLike) {
- LLVM_DEBUG(loopLike.print(llvm::dbgs() << "\nOriginal loop:\n"));
- moveLoopInvariantCode(loopLike);
- });
+ getOperation()->walk(
+ [&](LoopLikeOpInterface loopLike) { moveLoopInvariantCode(loopLike); });
}
std::unique_ptr<Pass> mlir::createLoopInvariantCodeMotionPass() {
FoldUtils.cpp
GreedyPatternRewriteDriver.cpp
InliningUtils.cpp
+ LoopInvariantCodeMotionUtils.cpp
RegionUtils.cpp
ADDITIONAL_HEADER_DIRS
/// Given a list of regions, find operations to sink and sink them. Return the
/// number of operations sunk.
- size_t sinkRegions(ArrayRef<Region *> regions);
+ size_t sinkRegions(RegionRange regions);
private:
/// Given a region and an op which dominates the region, returns true if all
}
}
-size_t Sinker::sinkRegions(ArrayRef<Region *> regions) {
+size_t Sinker::sinkRegions(RegionRange regions) {
for (Region *region : regions)
if (!region->empty())
sinkRegion(region);
}
size_t mlir::controlFlowSink(
- ArrayRef<Region *> regions, DominanceInfo &domInfo,
+ RegionRange regions, DominanceInfo &domInfo,
function_ref<bool(Operation *, Region *)> shouldMoveIntoRegion,
function_ref<void(Operation *, Region *)> moveIntoRegion) {
return Sinker(shouldMoveIntoRegion, moveIntoRegion, domInfo)
--- /dev/null
+//===- LoopInvariantCodeMotionUtils.cpp - LICM Utils ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the implementation of the core LICM algorithm.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Interfaces/LoopLikeInterface.h"
+#include "mlir/Transforms/SideEffectUtils.h"
+#include "llvm/Support/Debug.h"
+#include <queue>
+
+#define DEBUG_TYPE "licm"
+
+using namespace mlir;
+
+/// Checks whether the given op can be hoisted by checking that
+/// - the op and none of its contained operations depend on values inside of the
+/// loop (by means of calling definedOutside).
+/// - the op has no side-effects.
+static bool canBeHoisted(Operation *op,
+ function_ref<bool(Value)> definedOutside) {
+ // Do not move terminators.
+ if (op->hasTrait<OpTrait::IsTerminator>())
+ return false;
+
+ // Walk the nested operations and check that all used values are either
+ // defined outside of the loop or in a nested region, but not at the level of
+ // the loop body.
+ auto walkFn = [&](Operation *child) {
+ for (Value operand : child->getOperands()) {
+ // Ignore values defined in a nested region.
+ if (op->isAncestor(operand.getParentRegion()->getParentOp()))
+ continue;
+ if (!definedOutside(operand))
+ return WalkResult::interrupt();
+ }
+ return WalkResult::advance();
+ };
+ return !op->walk(walkFn).wasInterrupted();
+}
+
+size_t mlir::moveLoopInvariantCode(
+ RegionRange regions,
+ function_ref<bool(Value, Region *)> isDefinedOutsideRegion,
+ function_ref<bool(Operation *, Region *)> shouldMoveOutOfRegion,
+ function_ref<void(Operation *, Region *)> moveOutOfRegion) {
+ size_t numMoved = 0;
+
+ for (Region *region : regions) {
+ LLVM_DEBUG(llvm::dbgs() << "Original loop:\n" << *region->getParentOp());
+
+ std::queue<Operation *> worklist;
+ // Add top-level operations in the loop body to the worklist.
+ for (Operation &op : region->getOps())
+ worklist.push(&op);
+
+ auto definedOutside = [&](Value value) {
+ return isDefinedOutsideRegion(value, region);
+ };
+
+ while (!worklist.empty()) {
+ Operation *op = worklist.front();
+ worklist.pop();
+ // Skip ops that have already been moved. Check if the op can be hoisted.
+ if (op->getParentRegion() != region)
+ continue;
+
+ LLVM_DEBUG(llvm::dbgs() << "Checking op: " << *op);
+ if (!shouldMoveOutOfRegion(op, region) ||
+ !canBeHoisted(op, definedOutside))
+ continue;
+
+ LLVM_DEBUG(llvm::dbgs() << "Moving loop-invariant op: " << *op);
+ moveOutOfRegion(op, region);
+ ++numMoved;
+
+ // Since the op has been moved, we need to check its users within the
+ // top-level of the loop body.
+ for (Operation *user : op->getUsers())
+ if (user->getParentRegion() == region)
+ worklist.push(user);
+ }
+ }
+
+ return numMoved;
+}
+
+size_t mlir::moveLoopInvariantCode(LoopLikeOpInterface loopLike) {
+ return moveLoopInvariantCode(
+ &loopLike.getLoopBody(),
+ [&](Value value, Region *) {
+ return loopLike.isDefinedOutsideOfLoop(value);
+ },
+ [&](Operation *op, Region *) { return isSideEffectFree(op); },
+ [&](Operation *op, Region *) { loopLike.moveOutOfLoop(op); });
+}
--- /dev/null
+//===- SideEffectUtils.cpp - Side Effect Utils ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Transforms/SideEffectUtils.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+
+using namespace mlir;
+
+bool mlir::isSideEffectFree(Operation *op) {
+ if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(op)) {
+ // If the op has side-effects, it cannot be moved.
+ if (!memInterface.hasNoEffect())
+ return false;
+ // If the op does not have recursive side effects, then it can be moved.
+ if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>())
+ return true;
+ } else if (!op->hasTrait<OpTrait::HasRecursiveSideEffects>()) {
+ // Otherwise, if the op does not implement the memory effect interface and
+ // it does not have recursive side effects, then it cannot be known that the
+ // op is moveable.
+ return false;
+ }
+
+ // Recurse into the regions and ensure that all nested ops can also be moved.
+ for (Region ®ion : op->getRegions())
+ for (Operation &op : region.getOps())
+ if (!isSideEffectFree(&op))
+ return false;
+ return true;
+}
deps = [
":ControlFlowInterfaces",
":IR",
+ ":LoopLikeInterface",
":Rewrite",
":SideEffectInterfaces",
":Support",
projects / platform / upstream / llvm.git / commitdiff