From: Johannes Doerfert Date: Fri, 17 Apr 2020 23:03:04 +0000 (-0500) Subject: [Attributor][NFC] Encode IRPositions in the bits of a single pointer X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=d254b50b2b5b22368780c6003c419ffa1e23fa93;p=platform%2Fupstream%2Fllvm.git [Attributor][NFC] Encode IRPositions in the bits of a single pointer This reduces memory consumption for IRPositions by eliminating the vtable pointer and the `KindOrArgNo` integer. Since each abstract attribute has an associated IRPosition, the 12-16 bytes we save add up quickly. No functional change is intended. --- Single run of the Attributor module and then CGSCC pass (oldPM) for SPASS/clause.c (~10k LLVM-IR loc): Before: ``` calls to allocation functions: 469545 (260135/s) temporary memory allocations: 77137 (42735/s) peak heap memory consumption: 30.50MB peak RSS (including heaptrack overhead): 119.50MB total memory leaked: 269.07KB ``` After: ``` calls to allocation functions: 468999 (274108/s) temporary memory allocations: 77002 (45004/s) peak heap memory consumption: 28.83MB peak RSS (including heaptrack overhead): 118.05MB total memory leaked: 269.07KB ``` Difference: ``` calls to allocation functions: -546 (5808/s) temporary memory allocations: -135 (1436/s) peak heap memory consumption: -1.67MB peak RSS (including heaptrack overhead): 0B total memory leaked: 0B ``` --- CTMark 15 runs Metric: compile_time Program lhs rhs diff test-suite...:: CTMark/sqlite3/sqlite3.test 25.07 24.09 -3.9% test-suite...Mark/mafft/pairlocalalign.test 14.58 14.14 -3.0% test-suite...-typeset/consumer-typeset.test 21.78 21.58 -0.9% test-suite :: CTMark/SPASS/SPASS.test 21.95 22.03 0.4% test-suite :: CTMark/lencod/lencod.test 25.43 25.50 0.3% test-suite...ark/tramp3d-v4/tramp3d-v4.test 23.88 23.83 -0.2% test-suite...TMark/7zip/7zip-benchmark.test 60.24 60.11 -0.2% test-suite :: CTMark/kimwitu++/kc.test 15.69 15.69 -0.0% test-suite...:: CTMark/ClamAV/clamscan.test 25.43 25.42 -0.0% test-suite :: CTMark/Bullet/bullet.test 37.63 37.62 -0.0% Geomean difference -0.8% --- Reviewed By: lebedev.ri Differential Revision: https://reviews.llvm.org/D78722 --- diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h index 079d13f..4576966 100644 --- a/llvm/include/llvm/Transforms/IPO/Attributor.h +++ b/llvm/include/llvm/Transforms/IPO/Attributor.h @@ -153,29 +153,24 @@ enum class DepClassTy { /// are floating values that do not have a corresponding attribute list /// position. struct IRPosition { - virtual ~IRPosition() {} /// The positions we distinguish in the IR. - /// - /// The values are chosen such that the KindOrArgNo member has a value >= 0 - /// if it is an argument or call site argument while a value < 0 indicates the - /// respective kind of that value. - enum Kind : int { - IRP_INVALID = -6, ///< An invalid position. - IRP_FLOAT = -5, ///< A position that is not associated with a spot suitable - ///< for attributes. This could be any value or instruction. - IRP_RETURNED = -4, ///< An attribute for the function return value. - IRP_CALL_SITE_RETURNED = -3, ///< An attribute for a call site return value. - IRP_FUNCTION = -2, ///< An attribute for a function (scope). - IRP_CALL_SITE = -1, ///< An attribute for a call site (function scope). - IRP_ARGUMENT = 0, ///< An attribute for a function argument. - IRP_CALL_SITE_ARGUMENT = 1, ///< An attribute for a call site argument. + enum Kind : char { + IRP_INVALID, ///< An invalid position. + IRP_FLOAT, ///< A position that is not associated with a spot suitable + ///< for attributes. This could be any value or instruction. + IRP_RETURNED, ///< An attribute for the function return value. + IRP_CALL_SITE_RETURNED, ///< An attribute for a call site return value. + IRP_FUNCTION, ///< An attribute for a function (scope). + IRP_CALL_SITE, ///< An attribute for a call site (function scope). + IRP_ARGUMENT, ///< An attribute for a function argument. + IRP_CALL_SITE_ARGUMENT, ///< An attribute for a call site argument. }; /// Default constructor available to create invalid positions implicitly. All /// other positions need to be created explicitly through the appropriate /// static member function. - IRPosition() : AnchorVal(nullptr), KindOrArgNo(IRP_INVALID) { verify(); } + IRPosition() : Enc(nullptr, ENC_VALUE) { verify(); } /// Create a position describing the value of \p V. static const IRPosition value(const Value &V) { @@ -198,7 +193,7 @@ struct IRPosition { /// Create a position describing the argument \p Arg. static const IRPosition argument(const Argument &Arg) { - return IRPosition(const_cast(Arg), Kind(Arg.getArgNo())); + return IRPosition(const_cast(Arg), IRP_ARGUMENT); } /// Create a position describing the function scope of \p CB. @@ -214,7 +209,8 @@ struct IRPosition { /// Create a position describing the argument of \p CB at position \p ArgNo. static const IRPosition callsite_argument(const CallBase &CB, unsigned ArgNo) { - return IRPosition(const_cast(CB), Kind(ArgNo)); + return IRPosition(const_cast(CB.getArgOperandUse(ArgNo)), + IRP_CALL_SITE_ARGUMENT); } /// Create a position describing the argument of \p ACS at position \p ArgNo. @@ -242,9 +238,7 @@ struct IRPosition { return IRPosition::function(*IRP.getAssociatedFunction()); } - bool operator==(const IRPosition &RHS) const { - return (AnchorVal == RHS.AnchorVal) && (KindOrArgNo == RHS.KindOrArgNo); - } + bool operator==(const IRPosition &RHS) const { return Enc == RHS.Enc; } bool operator!=(const IRPosition &RHS) const { return !(*this == RHS); } /// Return the value this abstract attribute is anchored with. @@ -254,16 +248,21 @@ struct IRPosition { /// far, only the case for call site arguments as the value is not sufficient /// to pinpoint them. Instead, we can use the call site as an anchor. Value &getAnchorValue() const { - assert(KindOrArgNo != IRP_INVALID && - "Invalid position does not have an anchor value!"); - return *AnchorVal; + switch (getEncodingBits()) { + case ENC_VALUE: + case ENC_RETURNED_VALUE: + case ENC_FLOATING_FUNCTION: + return *getAsValuePtr(); + case ENC_CALL_SITE_ARGUMENT_USE: + return *(getAsUsePtr()->getUser()); + default: + llvm_unreachable("Unkown encoding!"); + }; } /// Return the associated function, if any. Function *getAssociatedFunction() const { - assert(KindOrArgNo != IRP_INVALID && - "Invalid position does not have an anchor scope!"); - if (auto *CB = dyn_cast(AnchorVal)) + if (auto *CB = dyn_cast(&getAnchorValue())) return CB->getCalledFunction(); return getAnchorScope(); } @@ -312,18 +311,14 @@ struct IRPosition { /// Return the value this abstract attribute is associated with. Value &getAssociatedValue() const { - assert(KindOrArgNo != IRP_INVALID && - "Invalid position does not have an associated value!"); - if (getArgNo() < 0 || isa(AnchorVal)) - return *AnchorVal; - assert(isa(AnchorVal) && "Expected a call base!"); - return *cast(AnchorVal)->getArgOperand(getArgNo()); + if (getArgNo() < 0 || isa(&getAnchorValue())) + return getAnchorValue(); + assert(isa(&getAnchorValue()) && "Expected a call base!"); + return *cast(&getAnchorValue())->getArgOperand(getArgNo()); } /// Return the type this abstract attribute is associated with. Type *getAssociatedType() const { - assert(KindOrArgNo != IRP_INVALID && - "Invalid position does not have an associated type!"); if (getPositionKind() == IRPosition::IRP_RETURNED) return getAssociatedFunction()->getReturnType(); return getAssociatedValue().getType(); @@ -331,7 +326,18 @@ struct IRPosition { /// Return the argument number of the associated value if it is an argument or /// call site argument, otherwise a negative value. - int getArgNo() const { return KindOrArgNo; } + int getArgNo() const { + switch (getPositionKind()) { + case IRPosition::IRP_ARGUMENT: + return cast(getAsValuePtr())->getArgNo(); + case IRPosition::IRP_CALL_SITE_ARGUMENT: { + Use &U = *getAsUsePtr(); + return cast(U.getUser())->getArgOperandNo(&U); + } + default: + return -1; + } + } /// Return the index in the attribute list for this position. unsigned getAttrIdx() const { @@ -347,7 +353,7 @@ struct IRPosition { return AttributeList::ReturnIndex; case IRPosition::IRP_ARGUMENT: case IRPosition::IRP_CALL_SITE_ARGUMENT: - return KindOrArgNo + AttributeList::FirstArgIndex; + return getArgNo() + AttributeList::FirstArgIndex; } llvm_unreachable( "There is no attribute index for a floating or invalid position!"); @@ -355,19 +361,23 @@ struct IRPosition { /// Return the associated position kind. Kind getPositionKind() const { - if (getArgNo() >= 0) { - assert(((isa(getAnchorValue()) && - isa(getAssociatedValue())) || - isa(getAnchorValue())) && - "Expected argument or call base due to argument number!"); - if (isa(getAnchorValue())) - return IRP_CALL_SITE_ARGUMENT; + char EncodingBits = getEncodingBits(); + if (EncodingBits == ENC_CALL_SITE_ARGUMENT_USE) + return IRP_CALL_SITE_ARGUMENT; + if (EncodingBits == ENC_FLOATING_FUNCTION) + return IRP_FLOAT; + + Value *V = getAsValuePtr(); + if (!V) + return IRP_INVALID; + if (isa(V)) return IRP_ARGUMENT; - } - - assert(KindOrArgNo < 0 && - "Expected (call site) arguments to never reach this point!"); - return Kind(KindOrArgNo); + if (isa(V)) + return isReturnPosition(EncodingBits) ? IRP_RETURNED : IRP_FUNCTION; + if (isa(V)) + return isReturnPosition(EncodingBits) ? IRP_CALL_SITE_RETURNED + : IRP_CALL_SITE; + return IRP_FLOAT; } /// TODO: Figure out if the attribute related helper functions should live @@ -435,14 +445,52 @@ struct IRPosition { static const IRPosition TombstoneKey; ///} + /// Conversion into a void * to allow reuse of pointer hashing. + operator void *() const { return Enc.getOpaqueValue(); } + private: /// Private constructor for special values only! - explicit IRPosition(int KindOrArgNo) - : AnchorVal(0), KindOrArgNo(KindOrArgNo) {} + explicit IRPosition(void *Ptr) { Enc.setFromOpaqueValue(Ptr); } /// IRPosition anchored at \p AnchorVal with kind/argument numbet \p PK. - explicit IRPosition(Value &AnchorVal, Kind PK) - : AnchorVal(&AnchorVal), KindOrArgNo(PK) { + explicit IRPosition(Value &AnchorVal, Kind PK) { + switch (PK) { + case IRPosition::IRP_INVALID: + llvm_unreachable("Cannot create invalid IRP with an anchor value!"); + break; + case IRPosition::IRP_FLOAT: + // Special case for floating functions. + if (isa(AnchorVal)) + Enc = {&AnchorVal, ENC_FLOATING_FUNCTION}; + else + Enc = {&AnchorVal, ENC_VALUE}; + break; + case IRPosition::IRP_FUNCTION: + case IRPosition::IRP_CALL_SITE: + Enc = {&AnchorVal, ENC_VALUE}; + break; + case IRPosition::IRP_RETURNED: + case IRPosition::IRP_CALL_SITE_RETURNED: + Enc = {&AnchorVal, ENC_RETURNED_VALUE}; + break; + case IRPosition::IRP_ARGUMENT: + Enc = {&AnchorVal, ENC_VALUE}; + break; + case IRPosition::IRP_CALL_SITE_ARGUMENT: + llvm_unreachable( + "Cannot create call site argument IRP with an anchor value!"); + break; + } + verify(); + } + + /// IRPosition for the use \p U. The position kind \p PK needs to be + /// IRP_CALL_SITE_ARGUMENT, the anchor value is the user, the associated value + /// the used value. + explicit IRPosition(Use &U, Kind PK) { + assert(PK == IRP_CALL_SITE_ARGUMENT && + "Use constructor is for call site arguments only!"); + Enc = {&U, ENC_CALL_SITE_ARGUMENT_USE}; verify(); } @@ -459,30 +507,65 @@ private: SmallVectorImpl &Attrs, Attributor &A) const; -protected: - /// The value this position is anchored at. - Value *AnchorVal; - - /// If AnchorVal is Argument or CallBase then this number should be - /// non-negative and it denotes the argument or call site argument index - /// respectively. Otherwise, it denotes the kind of this IRPosition according - /// to Kind above. - int KindOrArgNo; + /// Return the underlying pointer as Value *, valid for all positions but + /// IRP_CALL_SITE_ARGUMENT. + Value *getAsValuePtr() const { + assert(getEncodingBits() != ENC_CALL_SITE_ARGUMENT_USE && + "Not a value pointer!"); + return reinterpret_cast(Enc.getPointer()); + } + + /// Return the underlying pointer as Use *, valid only for + /// IRP_CALL_SITE_ARGUMENT positions. + Use *getAsUsePtr() const { + assert(getEncodingBits() == ENC_CALL_SITE_ARGUMENT_USE && + "Not a value pointer!"); + return reinterpret_cast(Enc.getPointer()); + } + + /// Return true if \p EncodingBits describe a returned or call site returned + /// position. + static bool isReturnPosition(char EncodingBits) { + return EncodingBits == ENC_RETURNED_VALUE; + } + + /// Return true if the encoding bits describe a returned or call site returned + /// position. + bool isReturnPosition() const { return isReturnPosition(getEncodingBits()); } + + /// The encoding of the IRPosition is a combination of a pointer and two + /// encoding bits. The values of the encoding bits are defined in the enum + /// below. The pointer is either a Value* (for the first three encoding bit + /// combinations) or Use* (for ENC_CALL_SITE_ARGUMENT_USE). + /// + ///{ + enum { + ENC_VALUE = 0b00, + ENC_RETURNED_VALUE = 0b01, + ENC_FLOATING_FUNCTION = 0b10, + ENC_CALL_SITE_ARGUMENT_USE = 0b11, + }; + + // Reserve the maximal amount of bits so there is no need to mask out the + // remaining ones. We will not encode anything else in the pointer anyway. + static constexpr int NumEncodingBits = + PointerLikeTypeTraits::NumLowBitsAvailable; + static_assert(NumEncodingBits >= 2, "At least two bits are required!"); + + /// The pointer with the encoding bits. + PointerIntPair Enc; + ///} + + /// Return the encoding bits. + char getEncodingBits() const { return Enc.getInt(); } }; /// Helper that allows IRPosition as a key in a DenseMap. -template <> struct DenseMapInfo { +template <> struct DenseMapInfo : DenseMapInfo { static inline IRPosition getEmptyKey() { return IRPosition::EmptyKey; } static inline IRPosition getTombstoneKey() { return IRPosition::TombstoneKey; } - static unsigned getHashValue(const IRPosition &IRP) { - return (DenseMapInfo::getHashValue(&IRP.getAnchorValue()) << 4) ^ - (unsigned(IRP.getArgNo())); - } - static bool isEqual(const IRPosition &LHS, const IRPosition &RHS) { - return LHS == RHS; - } }; /// A visitor class for IR positions. diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp index ba08061..11ef841 100644 --- a/llvm/lib/Transforms/IPO/Attributor.cpp +++ b/llvm/lib/Transforms/IPO/Attributor.cpp @@ -258,8 +258,9 @@ IRAttributeManifest::manifestAttrs(Attributor &A, const IRPosition &IRP, return HasChanged; } -const IRPosition IRPosition::EmptyKey(255); -const IRPosition IRPosition::TombstoneKey(256); +const IRPosition IRPosition::EmptyKey(DenseMapInfo::getEmptyKey()); +const IRPosition + IRPosition::TombstoneKey(DenseMapInfo::getTombstoneKey()); SubsumingPositionIterator::SubsumingPositionIterator(const IRPosition &IRP) { IRPositions.emplace_back(IRP); @@ -401,52 +402,60 @@ bool IRPosition::getAttrsFromAssumes(Attribute::AttrKind AK, void IRPosition::verify() { #ifdef EXPENSIVE_CHECKS - switch (KindOrArgNo) { - default: - assert(KindOrArgNo >= 0 && "Expected argument or call site argument!"); - assert((isa(AnchorVal) || isa(AnchorVal)) && - "Expected call base or argument for positive attribute index!"); - if (isa(AnchorVal)) { - assert(cast(AnchorVal)->getArgNo() == unsigned(getArgNo()) && - "Argument number mismatch!"); - assert(cast(AnchorVal) == &getAssociatedValue() && - "Associated value mismatch!"); - } else { - assert(cast(*AnchorVal).arg_size() > unsigned(getArgNo()) && - "Call site argument number mismatch!"); - assert(cast(*AnchorVal).getArgOperand(getArgNo()) == - &getAssociatedValue() && - "Associated value mismatch!"); - } - break; + switch (getPositionKind()) { case IRP_INVALID: - assert(!AnchorVal && "Expected no value for an invalid position!"); - break; + assert(!Enc.getOpaqueValue() && + "Expected a nullptr for an invalid position!"); + return; case IRP_FLOAT: assert((!isa(&getAssociatedValue()) && !isa(&getAssociatedValue())) && "Expected specialized kind for call base and argument values!"); - break; + return; case IRP_RETURNED: - assert(isa(AnchorVal) && + assert(isa(getAsValuePtr()) && "Expected function for a 'returned' position!"); - assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!"); - break; + assert(getAsValuePtr() == &getAssociatedValue() && + "Associated value mismatch!"); + return; case IRP_CALL_SITE_RETURNED: - assert((isa(AnchorVal)) && + assert((isa(getAsValuePtr())) && "Expected call base for 'call site returned' position!"); - assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!"); - break; + assert(getAsValuePtr() == &getAssociatedValue() && + "Associated value mismatch!"); + return; case IRP_CALL_SITE: - assert((isa(AnchorVal)) && + assert((isa(getAsValuePtr())) && "Expected call base for 'call site function' position!"); - assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!"); - break; + assert(getAsValuePtr() == &getAssociatedValue() && + "Associated value mismatch!"); + return; case IRP_FUNCTION: - assert(isa(AnchorVal) && + assert(isa(getAsValuePtr()) && "Expected function for a 'function' position!"); - assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!"); - break; + assert(getAsValuePtr() == &getAssociatedValue() && + "Associated value mismatch!"); + return; + case IRP_ARGUMENT: + assert(isa(getAsValuePtr()) && + "Expected argument for a 'argument' position!"); + assert(getAsValuePtr() == &getAssociatedValue() && + "Associated value mismatch!"); + return; + case IRP_CALL_SITE_ARGUMENT: { + Use *U = getAsUsePtr(); + assert(U && "Expected use for a 'call site argument' position!"); + assert(isa(U->getUser()) && + "Expected call base user for a 'call site argument' position!"); + assert(cast(U->getUser())->isArgOperand(U) && + "Expected call base argument operand for a 'call site argument' " + "position"); + assert(cast(U->getUser())->getArgOperandNo(U) == + unsigned(getArgNo()) && + "Argument number mismatch!"); + assert(U->get() == &getAssociatedValue() && "Associated value mismatch!"); + return; + } } #endif } diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 2c218e2..f14c0f0 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -906,8 +906,7 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) { // If the assumed unique return value is an argument, annotate it. if (auto *UniqueRVArg = dyn_cast(UniqueRV.getValue())) { // TODO: This should be handled differently! - this->AnchorVal = UniqueRVArg; - this->KindOrArgNo = UniqueRVArg->getArgNo(); + getIRPosition() = IRPosition::argument(*UniqueRVArg); Changed = IRAttribute::manifest(A); } else if (auto *RVC = dyn_cast(UniqueRV.getValue())) { // We can replace the returned value with the unique returned constant.