From 19f73f0d1b8d4f4a7b7451fea33a35015fe0942d Mon Sep 17 00:00:00 2001 From: Francesco Petrogalli Date: Fri, 13 Dec 2019 19:15:29 +0000 Subject: [PATCH] Revert "[VectorUtils] Introduce the Vector Function Database (VFDatabase)." This reverts commit 0be81968a283fd4161cb9ac9748d5ed200926292. The VFDatabase needs some rework to be able to handle vectorization and subsequent scalarization of intrinsics in out-of-tree versions of the compiler. For more details, see the discussion in https://reviews.llvm.org/D67572. --- llvm/include/llvm/Analysis/TargetLibraryInfo.h | 6 +- llvm/include/llvm/Analysis/VectorUtils.h | 82 ++-------------------- llvm/lib/Analysis/LazyCallGraph.cpp | 8 +-- llvm/lib/Analysis/LoopAccessAnalysis.cpp | 2 +- llvm/lib/Analysis/VectorUtils.cpp | 1 - llvm/lib/Transforms/Utils/InjectTLIMappings.cpp | 7 -- llvm/lib/Transforms/Utils/ModuleUtils.cpp | 7 +- .../Vectorize/LoopVectorizationLegality.cpp | 5 +- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 34 +++++---- llvm/test/Other/opt-O2-pipeline.ll | 2 - llvm/test/Other/opt-O3-pipeline.ll | 2 - llvm/test/Other/opt-Os-pipeline.ll | 2 - llvm/unittests/Analysis/VectorFunctionABITest.cpp | 24 +------ 13 files changed, 31 insertions(+), 151 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h index 04c59bb..be6c8cd 100644 --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -341,16 +341,12 @@ public: FunctionAnalysisManager::Invalidator &) { return false; } + /// Returns the largest vectorization factor used in the list of /// vector functions. unsigned getWidestVF(StringRef ScalarF) const { return Impl->getWidestVF(ScalarF); } - - /// Check if the function "F" is listed in a library known to LLVM. - bool isKnownVectorFunctionInLibrary(StringRef F) const { - return this->isFunctionVectorizable(F); - } }; /// Analysis pass providing the \c TargetLibraryInfo. diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h index 984c769..0ca69be 100644 --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -16,7 +16,6 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopAccessAnalysis.h" -#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/CheckedArithmetic.h" @@ -117,10 +116,10 @@ struct VFShape { /// Holds the VFShape for a specific scalar to vector function mapping. struct VFInfo { - VFShape Shape; /// Classification of the vector function. - std::string ScalarName; /// Scalar Function Name. - std::string VectorName; /// Vector Function Name associated to this VFInfo. - VFISAKind ISA; /// Instruction Set Architecture. + VFShape Shape; // Classification of the vector function. + StringRef ScalarName; // Scalar Function Name. + StringRef VectorName; // Vector Function Name associated to this VFInfo. + VFISAKind ISA; // Instruction Set Architecture. // Comparison operator. bool operator==(const VFInfo &Other) const { @@ -168,79 +167,6 @@ void getVectorVariantNames(const CallInst &CI, SmallVectorImpl &VariantMappings); } // end namespace VFABI -/// The Vector Function Database. -/// -/// Helper class used to find the vector functions associated to a -/// scalar CallInst. -class VFDatabase { - /// The Module of the CallInst CI. - const Module *M; - /// List of vector functions descritors associated to the call - /// instruction. - const SmallVector ScalarToVectorMappings; - - /// Retreive the scalar-to-vector mappings associated to the rule of - /// a vector Function ABI. - static void getVFABIMappings(const CallInst &CI, - SmallVectorImpl &Mappings) { - const StringRef ScalarName = CI.getCalledFunction()->getName(); - const StringRef S = - CI.getAttribute(AttributeList::FunctionIndex, VFABI::MappingsAttrName) - .getValueAsString(); - if (S.empty()) - return; - - SmallVector ListOfStrings; - VFABI::getVectorVariantNames(CI, ListOfStrings); - for (const auto &MangledName : ListOfStrings) { - const Optional Shape = VFABI::tryDemangleForVFABI(MangledName); - // A match is found via scalar and vector names, and also by - // ensuring that the variant described in the attribute has a - // corresponding definition or declaration of the vector - // function in the Module M. - if (Shape.hasValue() && (Shape.getValue().ScalarName == ScalarName)) { - assert(CI.getModule()->getFunction(Shape.getValue().VectorName) && - "Vector function is missing."); - Mappings.push_back(Shape.getValue()); - } - } - } - -public: - /// Retrieve all the VFInfo instances associated to the CallInst CI. - static SmallVector getMappings(const CallInst &CI) { - SmallVector Ret; - - // Get mappings from the Vector Function ABI variants. - getVFABIMappings(CI, Ret); - - // Other non-VFABI variants should be retrieved here. - - return Ret; - } - - /// Constructor, requires a CallInst instance. - VFDatabase(CallInst &CI) - : M(CI.getModule()), ScalarToVectorMappings(VFDatabase::getMappings(CI)) { - } - /// \defgroup VFDatabase query interface. - /// - /// @{ - /// Retrieve the Function with VFShape \p Shape. - Function *getVectorizedFunction(const VFShape &Shape) const { - for (const auto &Info : ScalarToVectorMappings) - if (Info.Shape == Shape) - return M->getFunction(Info.VectorName); - - return nullptr; - } - /// Checks if a function is vectorizable with VFShape \p Shape. - bool isFunctionVectorizable(const VFShape &Shape) const { - return getVectorizedFunction(Shape) != nullptr; - } - /// @} -}; - template class ArrayRef; class DemandedBits; class GetElementPtrInst; diff --git a/llvm/lib/Analysis/LazyCallGraph.cpp b/llvm/lib/Analysis/LazyCallGraph.cpp index a9dd5e0..ef31c1e 100644 --- a/llvm/lib/Analysis/LazyCallGraph.cpp +++ b/llvm/lib/Analysis/LazyCallGraph.cpp @@ -15,7 +15,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" @@ -147,11 +146,8 @@ LLVM_DUMP_METHOD void LazyCallGraph::Node::dump() const { static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI) { LibFunc LF; - // Either this is a normal library function or a "vectorizable" - // function. Not using the VFDatabase here because this query - // is related only to libraries handled via the TLI. - return TLI.getLibFunc(F, LF) || - TLI.isKnownVectorFunctionInLibrary(F.getName()); + // Either this is a normal library function or a "vectorizable" function. + return TLI.getLibFunc(F, LF) || TLI.isFunctionVectorizable(F.getName()); } LazyCallGraph::LazyCallGraph( diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 05f6010..26fa5112 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1845,7 +1845,7 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, // If the function has an explicit vectorized counterpart, we can safely // assume that it can be vectorized. if (Call && !Call->isNoBuiltin() && Call->getCalledFunction() && - !VFDatabase::getMappings(*Call).empty()) + TLI->isFunctionVectorizable(Call->getCalledFunction()->getName())) continue; auto *Ld = dyn_cast(&I); diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index a83dda1..c45ab941 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -1174,7 +1174,6 @@ void VFABI::getVectorVariantNames( for (auto &S : SetVector(ListAttr.begin(), ListAttr.end())) { #ifndef NDEBUG - LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << S << "'\n"); Optional Info = VFABI::tryDemangleForVFABI(S); assert(Info.hasValue() && "Invalid name for a VFABI variant."); assert(CI.getModule()->getFunction(Info.getValue().VectorName) && diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp index 9e9fea9..9192e74 100644 --- a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp +++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp @@ -13,8 +13,6 @@ #include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/ADT/Statistic.h" -#include "llvm/Analysis/DemandedBits.h" -#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/InstIterator.h" #include "llvm/Transforms/Utils.h" @@ -170,11 +168,6 @@ void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesCFG(); AU.addRequired(); AU.addPreserved(); - AU.addPreserved(); - AU.addPreserved(); - AU.addPreserved(); - AU.addPreserved(); - AU.addPreserved(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp index 51c6450..b94f57e 100644 --- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp +++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp @@ -11,16 +11,14 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/ModuleUtils.h" -#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/Support/raw_ostream.h" -using namespace llvm; -#define DEBUG_TYPE "moduleutils" +using namespace llvm; static void appendToGlobalArray(const char *Array, Module &M, Function *F, int Priority, Constant *Data) { @@ -300,9 +298,8 @@ void VFABI::setVectorVariantNames( Module *M = CI->getModule(); #ifndef NDEBUG for (const std::string &VariantMapping : VariantMappings) { - LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << VariantMapping << "'\n"); Optional VI = VFABI::tryDemangleForVFABI(VariantMapping); - assert(VI.hasValue() && "Cannot add an invalid VFABI name."); + assert(VI.hasValue() && "Canno add an invalid VFABI name."); assert(M->getNamedValue(VI.getValue().VectorName) && "Cannot add variant to attribute: " "vector function declaration is missing."); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 0072a7d..3f943f4 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -670,7 +670,7 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { if (CI && !getVectorIntrinsicIDForCall(CI, TLI) && !isa(CI) && !(CI->getCalledFunction() && TLI && - !VFDatabase::getMappings(*CI).empty())) { + TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) { // If the call is a recognized math libary call, it is likely that // we can vectorize it given loosened floating-point constraints. LibFunc Func; @@ -685,8 +685,7 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { // but it's hard to provide meaningful yet generic advice. // Also, should this be guarded by allowExtraAnalysis() and/or be part // of the returned info from isFunctionVectorizable()? - reportVectorizationFailure( - "Found a non-intrinsic callsite", + reportVectorizationFailure("Found a non-intrinsic callsite", "library call cannot be vectorized. " "Try compiling with -fno-math-errno, -ffast-math, " "or similar flags", diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index e80a78b..be4bb93 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -134,7 +134,6 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" -#include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LoopUtils.h" #include "llvm/Transforms/Utils/LoopVersioning.h" @@ -1635,7 +1634,6 @@ struct LoopVectorize : public FunctionPass { AU.addRequired(); AU.addRequired(); AU.addRequired(); - AU.addRequired(); // We currently do not preserve loopinfo/dominator analyses with outer loop // vectorization. Until this is addressed, mark these analyses as preserved @@ -3228,6 +3226,7 @@ unsigned LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, unsigned VF, bool &NeedToScalarize) { Function *F = CI->getCalledFunction(); + StringRef FnName = CI->getCalledFunction()->getName(); Type *ScalarRetTy = CI->getType(); SmallVector Tys, ScalarTys; for (auto &ArgOp : CI->arg_operands()) @@ -3255,9 +3254,7 @@ unsigned LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, // If we can't emit a vector call for this function, then the currently found // cost is the cost we need to return. NeedToScalarize = true; - if (!TLI || CI->isNoBuiltin() || - !VFDatabase(*CI).isFunctionVectorizable( - VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/))) + if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin()) return Cost; // If the corresponding vector cost is cheaper, return its cost. @@ -4282,6 +4279,9 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) { Module *M = I.getParent()->getParent()->getParent(); auto *CI = cast(&I); + StringRef FnName = CI->getCalledFunction()->getName(); + Function *F = CI->getCalledFunction(); + Type *RetTy = ToVectorTy(CI->getType(), VF); SmallVector Tys; for (Value *ArgOperand : CI->arg_operands()) Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); @@ -4317,18 +4317,17 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) { TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF); VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl); } else { - // Use vector version of the function call. - const VFShape Shape = - VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/); -#ifndef NDEBUG - const SmallVector Infos = VFDatabase::getMappings(*CI); - assert(std::find_if(Infos.begin(), Infos.end(), - [&Shape](const VFInfo &Info) { - return Info.Shape == Shape; - }) != Infos.end() && - "Vector function shape is missing from the database."); -#endif - VectorF = VFDatabase(*CI).getVectorizedFunction(Shape); + // Use vector version of the library call. + StringRef VFnName = TLI->getVectorizedFunction(FnName, VF); + assert(!VFnName.empty() && "Vector function name is empty."); + VectorF = M->getFunction(VFnName); + if (!VectorF) { + // Generate a declaration + FunctionType *FTy = FunctionType::get(RetTy, Tys, false); + VectorF = + Function::Create(FTy, Function::ExternalLinkage, VFnName, M); + VectorF->copyAttributesFrom(F); + } } assert(VectorF && "Can't create vector function."); @@ -6357,7 +6356,6 @@ INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis) INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass) INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) -INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy) INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false) namespace llvm { diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll index a98781e3..d263050 100644 --- a/llvm/test/Other/opt-O2-pipeline.ll +++ b/llvm/test/Other/opt-O2-pipeline.ll @@ -223,11 +223,9 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter -; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis -; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll index 7600b47..82f6e8f 100644 --- a/llvm/test/Other/opt-O3-pipeline.ll +++ b/llvm/test/Other/opt-O3-pipeline.ll @@ -228,11 +228,9 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter -; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis -; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll index e12a36c..c690cc5 100644 --- a/llvm/test/Other/opt-Os-pipeline.ll +++ b/llvm/test/Other/opt-Os-pipeline.ll @@ -210,11 +210,9 @@ ; CHECK-NEXT: Lazy Branch Probability Analysis ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter -; CHECK-NEXT: Inject TLI Mappings ; CHECK-NEXT: Loop Vectorization ; CHECK-NEXT: Canonicalize natural loops ; CHECK-NEXT: Scalar Evolution Analysis -; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Function Alias Analysis Results ; CHECK-NEXT: Loop Access Analysis ; CHECK-NEXT: Lazy Branch Probability Analysis diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp index becfe51..6aaa38c 100644 --- a/llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -13,7 +13,7 @@ using namespace llvm; -// This test makes sure that the demangling method succeeds only on +// This test makes sure that the getFromVFABI method succeeds only on // valid values of the string. TEST(VectorFunctionABITests, OnlyValidNames) { // Incomplete string. @@ -91,8 +91,8 @@ protected: unsigned &VF = Info.Shape.VF; VFISAKind &ISA = Info.ISA; SmallVector &Parameters = Info.Shape.Parameters; - std::string &ScalarName = Info.ScalarName; - std::string &VectorName = Info.VectorName; + StringRef &ScalarName = Info.ScalarName; + StringRef &VectorName = Info.VectorName; bool &IsScalable = Info.Shape.IsScalable; // Invoke the parser. bool invokeParser(const StringRef MangledName) { @@ -243,12 +243,6 @@ TEST_F(VFABIParserTest, ISA) { EXPECT_EQ(ISA, VFISAKind::AVX512); } -TEST_F(VFABIParserTest, LLVM_ISA) { - EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin")); - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)")); - EXPECT_EQ(ISA, VFISAKind::LLVM); -} - TEST_F(VFABIParserTest, InvalidMask) { EXPECT_FALSE(invokeParser("_ZGVsK2v_sin")); } @@ -531,15 +525,3 @@ TEST_F(VFABIParserTest, LLVM_InternalISA) { EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)")); EXPECT_EQ(ISA, VFISAKind::LLVM); } - -TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) { - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)")); - EXPECT_EQ(VF, (unsigned)4); - EXPECT_FALSE(IsMasked()); - EXPECT_FALSE(IsScalable); - EXPECT_EQ(ISA, VFISAKind::LLVM); - EXPECT_EQ(Parameters.size(), (unsigned)2); - EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); - EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); - EXPECT_EQ(ScalarName, "llvm.pow.f32"); -} -- 2.7.4