From 50b2a113db197a97f60ad2aace8b7382dc9b8c31 Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Wed, 2 Nov 2022 08:52:29 +0100 Subject: [PATCH] Move "auto-init" instructions to the dominator of their users As a result of -ftrivial-auto-var-init, clang generates instructions to set alloca'd memory to a given pattern, right after the allocation site. In some cases, this (somehow costly) operation could be delayed, leading to conditional execution in some cases. This is not an uncommon situation: it happens ~500 times on the cPython code base, and much more on the LLVM codebase. The benefit greatly varies on the execution path, but it should not regress on performance. This is a recommit of cca01008cc31a891d0ec70aff2201b25d05d8f1b with MemorySSA update fixes. Differential Revision: https://reviews.llvm.org/D137707 --- llvm/include/llvm/Transforms/Utils/MoveAutoInit.h | 29 +++ llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassBuilderPipelines.cpp | 4 + llvm/lib/Passes/PassRegistry.def | 1 + llvm/lib/Transforms/Utils/CMakeLists.txt | 1 + llvm/lib/Transforms/Utils/MoveAutoInit.cpp | 220 +++++++++++++++++++++ llvm/test/Other/new-pm-defaults.ll | 1 + llvm/test/Other/new-pm-lto-defaults.ll | 1 + .../test/Other/new-pm-thinlto-postlink-defaults.ll | 1 + .../Other/new-pm-thinlto-postlink-pgo-defaults.ll | 1 + .../new-pm-thinlto-postlink-samplepgo-defaults.ll | 1 + llvm/test/Other/new-pm-thinlto-prelink-defaults.ll | 1 + .../Other/new-pm-thinlto-prelink-pgo-defaults.ll | 1 + .../new-pm-thinlto-prelink-samplepgo-defaults.ll | 1 + llvm/test/Transforms/MoveAutoInit/branch.ll | 41 ++++ llvm/test/Transforms/MoveAutoInit/clobber.ll | 100 ++++++++++ llvm/test/Transforms/MoveAutoInit/fence.ll | 70 +++++++ llvm/test/Transforms/MoveAutoInit/loop.ll | 102 ++++++++++ llvm/test/Transforms/MoveAutoInit/scalar.ll | 36 ++++ 19 files changed, 613 insertions(+) create mode 100644 llvm/include/llvm/Transforms/Utils/MoveAutoInit.h create mode 100644 llvm/lib/Transforms/Utils/MoveAutoInit.cpp create mode 100644 llvm/test/Transforms/MoveAutoInit/branch.ll create mode 100644 llvm/test/Transforms/MoveAutoInit/clobber.ll create mode 100644 llvm/test/Transforms/MoveAutoInit/fence.ll create mode 100644 llvm/test/Transforms/MoveAutoInit/loop.ll create mode 100644 llvm/test/Transforms/MoveAutoInit/scalar.ll diff --git a/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h b/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h new file mode 100644 index 0000000..980b55f --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/MoveAutoInit.h @@ -0,0 +1,29 @@ +//===- MoveAutoInit.h - Move insts marked as auto-init Pass --*- C++ -*-======// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass moves instructions marked as auto-init closer to their use if +// profitable, generally because it moves them under a guard, potentially +// skipping the overhead of the auto-init under some execution paths. +// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H +#define LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class MoveAutoInitPass : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_MOVEAUTOINIT_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index a04f8bb..ebfd537 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -247,6 +247,7 @@ #include "llvm/Transforms/Utils/LowerSwitch.h" #include "llvm/Transforms/Utils/Mem2Reg.h" #include "llvm/Transforms/Utils/MetaRenamer.h" +#include "llvm/Transforms/Utils/MoveAutoInit.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" #include "llvm/Transforms/Utils/PredicateInfo.h" #include "llvm/Transforms/Utils/RelLookupTableConverter.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index ed085b4..ec817c4 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -125,6 +125,7 @@ #include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h" #include "llvm/Transforms/Utils/Mem2Reg.h" +#include "llvm/Transforms/Utils/MoveAutoInit.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" #include "llvm/Transforms/Utils/RelLookupTableConverter.h" #include "llvm/Transforms/Utils/SimplifyCFGOptions.h" @@ -656,6 +657,8 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, FPM.addPass(MemCpyOptPass()); FPM.addPass(DSEPass()); + FPM.addPass(MoveAutoInitPass()); + FPM.addPass(createFunctionToLoopPassAdaptor( LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap, /*AllowSpeculation=*/true), @@ -1802,6 +1805,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, // Nuke dead stores. MainFPM.addPass(DSEPass()); + MainFPM.addPass(MoveAutoInitPass()); MainFPM.addPass(MergedLoadStoreMotionPass()); LoopPassManager LPM; diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 82592a1..2c62941 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -333,6 +333,7 @@ FUNCTION_PASS("mem2reg", PromotePass()) FUNCTION_PASS("memcpyopt", MemCpyOptPass()) FUNCTION_PASS("mergeicmps", MergeICmpsPass()) FUNCTION_PASS("mergereturn", UnifyFunctionExitNodesPass()) +FUNCTION_PASS("move-auto-init", MoveAutoInitPass()) FUNCTION_PASS("nary-reassociate", NaryReassociatePass()) FUNCTION_PASS("newgvn", NewGVNPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt index 6663ce6..a870071 100644 --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -58,6 +58,7 @@ add_llvm_component_library(LLVMTransformUtils MetaRenamer.cpp MisExpect.cpp ModuleUtils.cpp + MoveAutoInit.cpp NameAnonGlobals.cpp PredicateInfo.cpp PromoteMemoryToRegister.cpp diff --git a/llvm/lib/Transforms/Utils/MoveAutoInit.cpp b/llvm/lib/Transforms/Utils/MoveAutoInit.cpp new file mode 100644 index 0000000..ca8a587 --- /dev/null +++ b/llvm/lib/Transforms/Utils/MoveAutoInit.cpp @@ -0,0 +1,220 @@ +//===-- MoveAutoInit.cpp - move auto-init inst closer to their use site----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass moves instruction maked as auto-init closer to the basic block that +// use it, eventually removing it from some control path of the function. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/MoveAutoInit.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Analysis/MemorySSA.h" +#include "llvm/Analysis/MemorySSAUpdater.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Transforms/Utils.h" +#include "llvm/Transforms/Utils/LoopUtils.h" + +using namespace llvm; + +#define DEBUG_TYPE "move-auto-init" + +STATISTIC(NumMoved, "Number of instructions moved"); + +static cl::opt MoveAutoInitThreshold( + "move-auto-init-threshold", cl::Hidden, cl::init(128), + cl::desc("Maximum instructions to analyze per moved initialization")); + +static bool hasAutoInitMetadata(const Instruction &I) { + return I.hasMetadata(LLVMContext::MD_annotation) && + any_of(I.getMetadata(LLVMContext::MD_annotation)->operands(), + [](const MDOperand &Op) { + return cast(Op.get())->getString() == "auto-init"; + }); +} + +/// Finds a BasicBlock in the CFG where instruction `I` can be moved to while +/// not changing the Memory SSA ordering and being guarded by at least one +/// condition. +static BasicBlock *usersDominator(Instruction *I, DominatorTree &DT, + MemorySSA &MSSA) { + BasicBlock *CurrentDominator = nullptr; + MemoryLocation ML; + if (auto *MI = dyn_cast(I)) + ML = MemoryLocation::getForDest(MI); + else if (auto *SI = dyn_cast(I)) + ML = MemoryLocation::get(SI); + else + assert(false && "memory location set"); + + MemoryUseOrDef &IMA = *MSSA.getMemoryAccess(I); + BatchAAResults AA(MSSA.getAA()); + + SmallPtrSet Visited; + + auto AsMemoryAccess = [](User *U) { return cast(U); }; + SmallVector WorkList(map_range(IMA.users(), AsMemoryAccess)); + + while (!WorkList.empty()) { + MemoryAccess *MA = WorkList.pop_back_val(); + if (!Visited.insert(MA).second) + continue; + + if (Visited.size() > MoveAutoInitThreshold) + return nullptr; + + bool FoundClobberingUser = false; + if (auto *M = dyn_cast(MA)) { + Instruction *MI = M->getMemoryInst(); + + // If this memory instruction may not clobber `I`, we can skip it. + // LifetimeEnd is a valid user, but we do not want it in the user + // dominator. + if (AA.getModRefInfo(MI, ML) != ModRefInfo::NoModRef && + !MI->isLifetimeStartOrEnd() && MI != I) { + FoundClobberingUser = true; + CurrentDominator = CurrentDominator + ? DT.findNearestCommonDominator(CurrentDominator, + MI->getParent()) + : MI->getParent(); + } + } + if (!FoundClobberingUser) { + auto UsersAsMemoryAccesses = map_range(MA->users(), AsMemoryAccess); + append_range(WorkList, UsersAsMemoryAccesses); + } + } + return CurrentDominator; +} + +static bool runMoveAutoInit(Function &F, DominatorTree &DT, MemorySSA &MSSA) { + BasicBlock &EntryBB = F.getEntryBlock(); + SmallVector> JobList; + + // + // Compute movable instructions. + // + for (Instruction &I : EntryBB) { + if (!hasAutoInitMetadata(I)) + continue; + + assert(!I.isVolatile() && "auto init instructions cannot be volatile."); + + BasicBlock *UsersDominator = usersDominator(&I, DT, MSSA); + if (!UsersDominator) + continue; + + if (UsersDominator == &EntryBB) + continue; + + // Traverse the CFG to detect cycles `UsersDominator` would be part of. + SmallPtrSet TransitiveSuccessors; + SmallVector WorkList(successors(UsersDominator)); + bool HasCycle = false; + while (!WorkList.empty()) { + BasicBlock *CurrBB = WorkList.pop_back_val(); + if (CurrBB == UsersDominator) + // No early exit because we want to compute the full set of transitive + // successors. + HasCycle = true; + for (BasicBlock *Successor : successors(CurrBB)) { + if (!TransitiveSuccessors.insert(Successor).second) + continue; + WorkList.push_back(Successor); + } + } + + // Don't insert if that could create multiple execution of I, + // but we can insert it in the non back-edge predecessors, if it exists. + if (HasCycle) { + BasicBlock *UsersDominatorHead = UsersDominator; + while (BasicBlock *UniquePredecessor = + UsersDominatorHead->getUniquePredecessor()) + UsersDominatorHead = UniquePredecessor; + + if (UsersDominatorHead == &EntryBB) + continue; + + BasicBlock *DominatingPredecessor = nullptr; + for (BasicBlock *Pred : predecessors(UsersDominatorHead)) { + // If one of the predecessor of the dominator also transitively is a + // successor, moving to the dominator would do the inverse of loop + // hoisting, and we don't want that. + if (TransitiveSuccessors.count(Pred)) + continue; + + DominatingPredecessor = + DominatingPredecessor + ? DT.findNearestCommonDominator(DominatingPredecessor, Pred) + : Pred; + } + + if (!DominatingPredecessor || DominatingPredecessor == &EntryBB) + continue; + + UsersDominator = DominatingPredecessor; + } + + // CatchSwitchInst blocks can only have one instruction, so they are not + // good candidates for insertion. + while (isa(UsersDominator->getFirstInsertionPt())) { + for (BasicBlock *Pred : predecessors(UsersDominator)) + UsersDominator = DT.findNearestCommonDominator(UsersDominator, Pred); + } + + // We finally found a place where I can be moved while not introducing extra + // execution, and guarded by at least one condition. + if (UsersDominator != &EntryBB) + JobList.emplace_back(&I, UsersDominator); + } + + // + // Perform the actual substitution. + // + if (JobList.empty()) + return false; + + MemorySSAUpdater MSSAU(&MSSA); + + // Reverse insertion to respect relative order between instructions: + // if two instructions are moved from the same BB to the same BB, we insert + // the second one in the front, then the first on top of it. + for (auto &Job : reverse(JobList)) { + Job.first->moveBefore(&*Job.second->getFirstInsertionPt()); + MSSAU.moveToPlace(MSSA.getMemoryAccess(Job.first), Job.first->getParent(), + MemorySSA::InsertionPlace::Beginning); + } + + if (VerifyMemorySSA) + MSSA.verifyMemorySSA(); + + NumMoved += JobList.size(); + + return true; +} + +PreservedAnalyses MoveAutoInitPass::run(Function &F, + FunctionAnalysisManager &AM) { + + auto &DT = AM.getResult(F); + auto &MSSA = AM.getResult(F).getMSSA(); + if (!runMoveAutoInit(F, DT, MSSA)) + return PreservedAnalyses::all(); + + PreservedAnalyses PA; + PA.preserve(); + PA.preserve(); + PA.preserveSet(); + return PA; +} diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll index c2b44b7..f192389 100644 --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -206,6 +206,7 @@ ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll index 2e0cd56..f48264a 100644 --- a/llvm/test/Other/new-pm-lto-defaults.ll +++ b/llvm/test/Other/new-pm-lto-defaults.ll @@ -108,6 +108,7 @@ ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass on foo ; CHECK-O23SZ-NEXT: Running pass: DSEPass on foo ; CHECK-O23SZ-NEXT: Running analysis: PostDominatorTreeAnalysis on foo +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: MergedLoadStoreMotionPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass on foo ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass on foo diff --git a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll index 5f46a2b..580f22a 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-defaults.ll @@ -149,6 +149,7 @@ ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass on loop diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll index 36ce293..8e0c083 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -136,6 +136,7 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll index 746c4b6..b47d688 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -143,6 +143,7 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll index 0dca953..ea05961 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-defaults.ll @@ -147,6 +147,7 @@ ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass on loop diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll index fdaee98..59b578f 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -172,6 +172,7 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll index b187bbc..cbba246 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -137,6 +137,7 @@ ; CHECK-O-NEXT: Running pass: ADCEPass ; CHECK-O23SZ-NEXT: Running pass: MemCpyOptPass ; CHECK-O23SZ-NEXT: Running pass: DSEPass +; CHECK-O23SZ-NEXT: Running pass: MoveAutoInitPass on foo ; CHECK-O23SZ-NEXT: Running pass: LoopSimplifyPass ; CHECK-O23SZ-NEXT: Running pass: LCSSAPass ; CHECK-O23SZ-NEXT: Running pass: LICMPass diff --git a/llvm/test/Transforms/MoveAutoInit/branch.ll b/llvm/test/Transforms/MoveAutoInit/branch.ll new file mode 100644 index 0000000..6c51e33 --- /dev/null +++ b/llvm/test/Transforms/MoveAutoInit/branch.ll @@ -0,0 +1,41 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s + +@__const.foo.buffer = private unnamed_addr constant [8 x i32] [i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766, i32 -1431655766], align 16 + +define void @foo(i32 %x) { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[BUFFER:%.*]] = alloca [8 x i32], align 16 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[BUFFER]], ptr align 16 @__const.foo.buffer, i64 32, i1 false), !annotation !0 +; CHECK-NEXT: call void @dump(ptr [[BUFFER]]) +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: ret void +; + +entry: + %buffer = alloca [8 x i32], align 16 + call void @llvm.memcpy.p0.p0.i64(ptr align 16 %buffer, ptr align 16 @__const.foo.buffer, i64 32, i1 false), !annotation !0 + %tobool = icmp ne i32 %x, 0 + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + call void @dump(ptr %buffer) + br label %if.end + +if.end: ; preds = %if.then, %entry + ret void +} + + + +declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) + +declare void @dump(ptr) + +!0 = !{!"auto-init"} + diff --git a/llvm/test/Transforms/MoveAutoInit/clobber.ll b/llvm/test/Transforms/MoveAutoInit/clobber.ll new file mode 100644 index 0000000..0d70d85 --- /dev/null +++ b/llvm/test/Transforms/MoveAutoInit/clobber.ll @@ -0,0 +1,100 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; Checks that move-auto-init can move instruction passed unclobbering memory +; instructions. +; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define i32 @foo(i32 noundef %0, i32 noundef %1, i32 noundef %2) #0 { +; CHECK-LABEL: @foo( +; CHECK-NEXT: [[TMP4:%.*]] = alloca [100 x i8], align 16 +; CHECK-NEXT: [[TMP5:%.*]] = alloca [2 x i8], align 1 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [100 x i8], ptr [[TMP4]], i64 0, i64 0 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 100, ptr nonnull [[TMP6]]) #[[ATTR3:[0-9]+]] +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 0 +; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 2, ptr nonnull [[TMP7]]) #[[ATTR3]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 1 +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP1:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP9]], label [[TMP15:%.*]], label [[TMP10:%.*]] +; CHECK: 10: +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 dereferenceable(100) [[TMP6]], i8 -86, i64 100, i1 false), !annotation !0 +; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP0:%.*]] to i64 +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [100 x i8], ptr [[TMP4]], i64 0, i64 [[TMP11]] +; CHECK-NEXT: store i8 12, ptr [[TMP12]], align 1 +; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[TMP6]], align 16 +; CHECK-NEXT: [[TMP14:%.*]] = sext i8 [[TMP13]] to i32 +; CHECK-NEXT: br label [[TMP22:%.*]] +; CHECK: 15: +; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP2:%.*]], 0 +; CHECK-NEXT: br i1 [[TMP16]], label [[TMP22]], label [[TMP17:%.*]] +; CHECK: 17: +; CHECK-NEXT: store i8 -86, ptr [[TMP7]], align 1, !annotation !0 +; CHECK-NEXT: store i8 -86, ptr [[TMP8]], align 1, !annotation !0 +; CHECK-NEXT: [[TMP18:%.*]] = sext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 [[TMP18]] +; CHECK-NEXT: store i8 12, ptr [[TMP19]], align 1 +; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP7]], align 1 +; CHECK-NEXT: [[TMP21:%.*]] = sext i8 [[TMP20]] to i32 +; CHECK-NEXT: br label [[TMP22]] +; CHECK: 22: +; CHECK-NEXT: [[TMP23:%.*]] = phi i32 [ [[TMP14]], [[TMP10]] ], [ [[TMP21]], [[TMP17]] ], [ 0, [[TMP15]] ] +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 2, ptr nonnull [[TMP7]]) #[[ATTR3]] +; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 100, ptr nonnull [[TMP6]]) #[[ATTR3]] +; CHECK-NEXT: ret i32 [[TMP23]] +; + + %4 = alloca [100 x i8], align 16 + %5 = alloca [2 x i8], align 1 + %6 = getelementptr inbounds [100 x i8], [100 x i8]* %4, i64 0, i64 0 + call void @llvm.lifetime.start.p0i8(i64 100, i8* nonnull %6) #3 + ; This memset must move. + call void @llvm.memset.p0i8.i64(i8* noundef nonnull align 16 dereferenceable(100) %6, i8 -86, i64 100, i1 false), !annotation !0 + %7 = getelementptr inbounds [2 x i8], [2 x i8]* %5, i64 0, i64 0 + call void @llvm.lifetime.start.p0i8(i64 2, i8* nonnull %7) #3 + ; This store must move. + store i8 -86, i8* %7, align 1, !annotation !0 + %8 = getelementptr inbounds [2 x i8], [2 x i8]* %5, i64 0, i64 1 + ; This store must move. + store i8 -86, i8* %8, align 1, !annotation !0 + %9 = icmp eq i32 %1, 0 + br i1 %9, label %15, label %10 + +10: + %11 = sext i32 %0 to i64 + %12 = getelementptr inbounds [100 x i8], [100 x i8]* %4, i64 0, i64 %11 + store i8 12, i8* %12, align 1 + %13 = load i8, i8* %6, align 16 + %14 = sext i8 %13 to i32 + br label %22 + +15: + %16 = icmp eq i32 %2, 0 + br i1 %16, label %22, label %17 + +17: + %18 = sext i32 %0 to i64 + %19 = getelementptr inbounds [2 x i8], [2 x i8]* %5, i64 0, i64 %18 + store i8 12, i8* %19, align 1 + %20 = load i8, i8* %7, align 1 + %21 = sext i8 %20 to i32 + br label %22 + +22: + %23 = phi i32 [ %14, %10 ], [ %21, %17 ], [ 0, %15 ] + call void @llvm.lifetime.end.p0i8(i64 2, i8* nonnull %7) #3 + call void @llvm.lifetime.end.p0i8(i64 100, i8* nonnull %6) #3 + ret i32 %23 +} + +declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1 + +declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #2 + +declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1 + +attributes #0 = { mustprogress nofree nosync nounwind readnone uwtable willreturn } +attributes #1 = { argmemonly mustprogress nofree nosync nounwind willreturn } +attributes #2 = { argmemonly mustprogress nofree nounwind willreturn writeonly } +attributes #3 = { nounwind } + +!0 = !{!"auto-init"} diff --git a/llvm/test/Transforms/MoveAutoInit/fence.ll b/llvm/test/Transforms/MoveAutoInit/fence.ll new file mode 100644 index 0000000..bce9c13 --- /dev/null +++ b/llvm/test/Transforms/MoveAutoInit/fence.ll @@ -0,0 +1,70 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +; In that case, the store to %val happens before the fence and cannot go past +; it. +define void @foo(i32 %x) { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[VAL:%.*]] = alloca i32, align 4 +; CHECK-NEXT: store i32 -1431655766, ptr [[VAL]], align 4, !annotation !0 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0 +; CHECK-NEXT: fence acquire +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: call void @dump(ptr [[VAL]]) +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + %val = alloca i32, align 4 + store i32 -1431655766, ptr %val, align 4, !annotation !0 + %tobool = icmp ne i32 %x, 0 + fence acquire + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + call void @dump(ptr %val) + br label %if.end + +if.end: ; preds = %if.then, %entry + ret void +} + +; In that case, the store to %val happens after the fence and it is moved within +; the true branch as expected. +define void @bar(i32 %x) { +; CHECK-LABEL: @bar( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[VAL:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0 +; CHECK-NEXT: fence acquire +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: store i32 -1431655766, ptr [[VAL]], align 4, !annotation !0 +; CHECK-NEXT: call void @dump(ptr [[VAL]]) +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + %val = alloca i32, align 4 + %tobool = icmp ne i32 %x, 0 + fence acquire + store i32 -1431655766, ptr %val, align 4, !annotation !0 + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + call void @dump(ptr %val) + br label %if.end + +if.end: ; preds = %if.then, %entry + ret void +} + +declare void @dump(ptr) + +!0 = !{!"auto-init"} diff --git a/llvm/test/Transforms/MoveAutoInit/loop.ll b/llvm/test/Transforms/MoveAutoInit/loop.ll new file mode 100644 index 0000000..71153e5 --- /dev/null +++ b/llvm/test/Transforms/MoveAutoInit/loop.ll @@ -0,0 +1,102 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @foo(i32 %x) { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[BUFFER:%.*]] = alloca [80 x i32], align 16 +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 16 [[BUFFER]], i8 -86, i64 320, i1 false), !annotation !0 +; CHECK-NEXT: br label [[DO_BODY:%.*]] +; CHECK: do.body: +; CHECK-NEXT: [[X_ADDR_0:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[DEC:%.*]], [[DO_COND:%.*]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [80 x i32], ptr [[BUFFER]], i64 0, i64 0 +; CHECK-NEXT: call void @dump(ptr [[ARRAYIDX]]) +; CHECK-NEXT: br label [[DO_COND]] +; CHECK: do.cond: +; CHECK-NEXT: [[DEC]] = add nsw i32 [[X_ADDR_0]], -1 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[X_ADDR_0]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[DO_BODY]], label [[DO_END:%.*]] +; CHECK: do.end: +; CHECK-NEXT: ret void +; + +entry: + %buffer = alloca [80 x i32], align 16 + call void @llvm.memset.p0.i64(ptr align 16 %buffer, i8 -86, i64 320, i1 false), !annotation !0 + br label %do.body + +do.body: ; preds = %do.cond, %entry + %x.addr.0 = phi i32 [ %x, %entry ], [ %dec, %do.cond ] + %arrayidx = getelementptr inbounds [80 x i32], ptr %buffer, i64 0, i64 0 + call void @dump(ptr %arrayidx) + br label %do.cond + +do.cond: ; preds = %do.body + %dec = add nsw i32 %x.addr.0, -1 + %tobool = icmp ne i32 %x.addr.0, 0 + br i1 %tobool, label %do.body, label %do.end + +do.end: ; preds = %do.cond + ret void +} + +declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) + +declare void @dump(ptr ) + +define void @bar(i32 %x, i32 %y) { +; CHECK-LABEL: @bar( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[BUFFER:%.*]] = alloca [80 x i32], align 16 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[Y:%.*]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 16 [[BUFFER]], i8 -86, i64 320, i1 false), !annotation !0 +; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[X:%.*]], [[Y]] +; CHECK-NEXT: br label [[DO_BODY:%.*]] +; CHECK: do.body: +; CHECK-NEXT: [[X_ADDR_0:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[DEC:%.*]], [[DO_COND:%.*]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [80 x i32], ptr [[BUFFER]], i64 0, i64 0 +; CHECK-NEXT: call void @dump(ptr [[ARRAYIDX]]) +; CHECK-NEXT: br label [[DO_COND]] +; CHECK: do.cond: +; CHECK-NEXT: [[DEC]] = add nsw i32 [[X_ADDR_0]], -1 +; CHECK-NEXT: [[TOBOOL1:%.*]] = icmp ne i32 [[X_ADDR_0]], 0 +; CHECK-NEXT: br i1 [[TOBOOL1]], label [[DO_BODY]], label [[DO_END:%.*]] +; CHECK: do.end: +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: ret void +; + +entry: + %buffer = alloca [80 x i32], align 16 + call void @llvm.memset.p0.i64(ptr align 16 %buffer, i8 -86, i64 320, i1 false), !annotation !0 + %tobool = icmp ne i32 %y, 0 + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + %add = add nsw i32 %x, %y + br label %do.body + +do.body: ; preds = %do.cond, %if.then + %x.addr.0 = phi i32 [ %add, %if.then ], [ %dec, %do.cond ] + %arrayidx = getelementptr inbounds [80 x i32], ptr %buffer, i64 0, i64 0 + call void @dump(ptr %arrayidx) + br label %do.cond + +do.cond: ; preds = %do.body + %dec = add nsw i32 %x.addr.0, -1 + %tobool1 = icmp ne i32 %x.addr.0, 0 + br i1 %tobool1, label %do.body, label %do.end + +do.end: ; preds = %do.cond + br label %if.end + +if.end: ; preds = %do.end, %entry + ret void +} + +!0 = !{!"auto-init"} diff --git a/llvm/test/Transforms/MoveAutoInit/scalar.ll b/llvm/test/Transforms/MoveAutoInit/scalar.ll new file mode 100644 index 0000000..6929aeb --- /dev/null +++ b/llvm/test/Transforms/MoveAutoInit/scalar.ll @@ -0,0 +1,36 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -S -passes='move-auto-init' -verify-memoryssa | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @foo(i32 %x) { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[VAL:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[X:%.*]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: store i32 -1431655766, ptr [[VAL]], align 4, !annotation !0 +; CHECK-NEXT: call void @dump(ptr [[VAL]]) +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: ret void +; + +entry: + %val = alloca i32, align 4 + store i32 -1431655766, ptr %val, align 4, !annotation !0 + %tobool = icmp ne i32 %x, 0 + br i1 %tobool, label %if.then, label %if.end + +if.then: ; preds = %entry + call void @dump(ptr %val) + br label %if.end + +if.end: ; preds = %if.then, %entry + ret void +} + +declare void @dump(ptr) + +!0 = !{!"auto-init"} -- 2.7.4