This changes adds attribute field for metadata of context profile. Currently we have an inline attribute that indicates whether the leaf frame corresponding to a context profile was inlined in previous build.
This will be used to help estimating inlining and be taken into account when trimming context. Changes for that in llvm-profgen will follow. It will also help tuning.
Differential Revision: https://reviews.llvm.org/D98823
enum class SecFuncMetadataFlags : uint32_t {
SecFlagInvalid = 0,
SecFlagIsProbeBased = (1 << 0),
+ SecFlagHasAttribute = (1 << 1)
};
// Verify section specific flag is used for the correct section.
MergedContext = 0x8 // Profile for context merged into base profile
};
+// Attribute of context associated with FunctionSamples
+enum ContextAttributeMask {
+ ContextNone = 0x0,
+ ContextWasInlined = 0x1, // Leaf of context was inlined in previous build
+ ContextShouldBeInlined = 0x2, // Leaf of context should be inlined
+};
+
// Sample context for FunctionSamples. It consists of the calling context,
// the function name and context state. Internally sample context is represented
// using StringRef, which is also the input for constructing a `SampleContext`.
// `_Z8funcLeafi`
class SampleContext {
public:
- SampleContext() : State(UnknownContext) {}
- SampleContext(StringRef ContextStr,
- ContextStateMask CState = UnknownContext) {
+ SampleContext() : State(UnknownContext), Attributes(ContextNone) {}
+ SampleContext(StringRef ContextStr, ContextStateMask CState = UnknownContext)
+ : Attributes(ContextNone) {
setContext(ContextStr, CState);
}
}
operator StringRef() const { return FullContext; }
+ bool hasAttribute(ContextAttributeMask A) { return Attributes & (uint32_t)A; }
+ void setAttribute(ContextAttributeMask A) { Attributes |= (uint32_t)A; }
+ uint32_t getAllAttributes() { return Attributes; }
+ void setAllAttributes(uint32_t A) { Attributes = A; }
bool hasState(ContextStateMask S) { return State & (uint32_t)S; }
void setState(ContextStateMask S) { State |= (uint32_t)S; }
void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
StringRef CallingContext;
// State of the associated sample profile
uint32_t State;
+ // Attribute of the associated sample profile
+ uint32_t Attributes;
};
class FunctionSamples;
// offsetA1[.discriminator]: number_of_samples [fn7:num fn8:num ... ]
// ...
// !CFGChecksum: num
+// !Attribute: flags
//
// This is a nested tree in which the indentation represents the nesting level
// of the inline stack. There are no blank lines in the file. And the spacing
//
// a. CFG Checksum (a.k.a. function hash):
// !CFGChecksum: 12345
+// b. CFG Checksum (see ContextAttributeMask):
+// !Atribute: 1
//
//
// Binary format
std::error_code readSecHdrTableEntry(uint32_t Idx);
std::error_code readSecHdrTable();
- std::error_code readFuncMetadata();
+ std::error_code readFuncMetadata(bool ProfileHasAttribute);
std::error_code readFuncOffsetTable();
std::error_code readFuncProfiles();
std::error_code readMD5NameTable();
/// Possible metadata:
/// - CFG Checksum information:
/// !CFGChecksum: 12345
+/// - CFG Checksum information:
+/// !Attributes: 1
/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
-static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash) {
- if (!Input.startswith("!CFGChecksum:"))
- return false;
+static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
+ uint32_t &Attributes) {
+ if (Input.startswith("!CFGChecksum:")) {
+ StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();
+ return !CFGInfo.getAsInteger(10, FunctionHash);
+ }
+
+ if (Input.startswith("!Attributes:")) {
+ StringRef Attrib = Input.substr(strlen("!Attributes:")).trim();
+ return !Attrib.getAsInteger(10, Attributes);
+ }
- StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();
- return !CFGInfo.getAsInteger(10, FunctionHash);
+ return false;
}
enum class LineType {
uint64_t &NumSamples, uint32_t &LineOffset,
uint32_t &Discriminator, StringRef &CalleeName,
DenseMap<StringRef, uint64_t> &TargetCountMap,
- uint64_t &FunctionHash) {
+ uint64_t &FunctionHash, uint32_t &Attributes) {
for (Depth = 0; Input[Depth] == ' '; Depth++)
;
if (Depth == 0)
if (Depth == 1 && Input[Depth] == '!') {
LineTy = LineType::Metadata;
- return parseMetadata(Input.substr(Depth), FunctionHash);
+ return parseMetadata(Input.substr(Depth), FunctionHash, Attributes);
}
size_t n1 = Input.find(':');
DenseMap<StringRef, uint64_t> TargetCountMap;
uint32_t Depth, LineOffset, Discriminator;
LineType LineTy;
- uint64_t FunctionHash;
+ uint64_t FunctionHash = 0;
+ uint32_t Attributes = 0;
if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
- Discriminator, FName, TargetCountMap, FunctionHash)) {
+ Discriminator, FName, TargetCountMap, FunctionHash,
+ Attributes)) {
reportError(LineIt.line_number(),
"Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
*LineIt);
}
case LineType::Metadata: {
FunctionSamples &FProfile = *InlineStack.back();
- FProfile.setFunctionHash(FunctionHash);
- ++ProbeProfileCount;
+ if (FunctionHash) {
+ FProfile.setFunctionHash(FunctionHash);
+ ++ProbeProfileCount;
+ }
+ if (Attributes)
+ FProfile.getContext().setAllAttributes(Attributes);
SeenMetadata = true;
break;
}
if (std::error_code EC = readFuncOffsetTable())
return EC;
break;
- case SecFuncMetadata:
+ case SecFuncMetadata: {
ProfileIsProbeBased =
hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased);
FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
- if (std::error_code EC = readFuncMetadata())
+ bool HasAttribute =
+ hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute);
+ if (std::error_code EC = readFuncMetadata(HasAttribute))
return EC;
break;
+ }
case SecProfileSymbolList:
if (std::error_code EC = readProfileSymbolList())
return EC;
return SampleProfileReaderBinary::readNameTable();
}
-std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
- if (!ProfileIsProbeBased)
- return sampleprof_error::success;
+std::error_code
+SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
while (Data < End) {
auto FName(readStringFromTable());
if (std::error_code EC = FName.getError())
return EC;
- auto Checksum = readNumber<uint64_t>();
- if (std::error_code EC = Checksum.getError())
- return EC;
-
SampleContext FContext(*FName);
- // No need to load metadata for profiles that are not loaded in the current
- // module.
- if (Profiles.count(FContext))
- Profiles[FContext].setFunctionHash(*Checksum);
+ bool ProfileInMap = Profiles.count(FContext);
+
+ if (ProfileIsProbeBased) {
+ auto Checksum = readNumber<uint64_t>();
+ if (std::error_code EC = Checksum.getError())
+ return EC;
+ if (ProfileInMap)
+ Profiles[FContext].setFunctionHash(*Checksum);
+ }
+
+ if (ProfileHasAttribute) {
+ auto Attributes = readNumber<uint32_t>();
+ if (std::error_code EC = Attributes.getError())
+ return EC;
+ if (ProfileInMap)
+ Profiles[FContext].getContext().setAllAttributes(*Attributes);
+ }
}
assert(Data == End && "More data is read than expected");
std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
const StringMap<FunctionSamples> &Profiles) {
- if (!FunctionSamples::ProfileIsProbeBased)
+ if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS)
return sampleprof_error::success;
auto &OS = *OutputStream;
for (const auto &Entry : Profiles) {
writeNameIdx(Entry.first());
- encodeULEB128(Entry.second.getFunctionHash(), OS);
+ if (FunctionSamples::ProfileIsProbeBased)
+ encodeULEB128(Entry.second.getFunctionHash(), OS);
+ if (FunctionSamples::ProfileIsCS)
+ encodeULEB128(Entry.second.getContext().getAllAttributes(), OS);
}
return sampleprof_error::success;
}
addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);
if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)
addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);
+ if (Type == SecFuncMetadata && FunctionSamples::ProfileIsCS)
+ addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute);
uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
switch (Type) {
OS.indent(Indent + 1);
OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
}
+ if (FunctionSamples::ProfileIsCS) {
+ OS.indent(Indent + 1);
+ OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
+ }
}
return sampleprof_error::success;
3: 287884
4: 287864 _Z3fibi:315608
15: 23
+ !Attributes: 0
[main:3.1 @ _Z5funcBi:1 @ _Z8funcLeafi]:500853:20
0: 15
1: 15
10: 23324
11: 23327 _Z3fibi:25228
15: 11
+ !Attributes: 1
[main]:154:0
2: 12
3: 18 _Z5funcAi:11
3.1: 18 _Z5funcBi:19
+ !Attributes: 0
[external:12 @ main]:154:12
2: 12
3: 10 _Z5funcAi:7
3.1: 10 _Z5funcBi:11
+ !Attributes: 0
[main:3.1 @ _Z5funcBi]:120:19
0: 19
1: 19 _Z8funcLeafi:20
3: 12
+ !Attributes: 1
[externalA:17 @ _Z5funcBi]:120:3
0: 3
1: 3
+ !Attributes: 0
[external:10 @ _Z5funcBi]:120:10
0: 10
1: 10
+ !Attributes: 0
[main:3 @ _Z5funcAi]:99:11
0: 10
1: 10 _Z8funcLeafi:11
3: 24
+ !Attributes: 0
5: 7 _Z3foov:5 _Z3barv:2
6: 6 _Z3barv:4 _Z3foov:2
!CFGChecksum: 563022570642068
+ !Attributes: 0
; CHECK-NEXT: 6: 15
; CHECK-NEXT: 8: 14 bar:14
; CHECK-NEXT: !CFGChecksum: 138950591924
-; CHECK-NEXT:[main:2 @ foo:8 @ bar]:28:14
+; CHECK:[main:2 @ foo:8 @ bar]:28:14
; CHECK-NEXT: 1: 14
; CHECK-NEXT: 2: 18446744073709551615
; CHECK-NEXT: 3: 18446744073709551615
; CHECK-NEXT: 7: 2 fb:2
; CHECK-NEXT: 8: 1 fa:1
; CHECK-NEXT: !CFGChecksum: 120515930909
+; CHECK-NEXT: !Attributes: 0
; CHECK-NEXT:[main:2 @ foo:5 @ fa:8 @ fa:7 @ fb:5 @ fb]:13:4
; CHECK-NEXT: 1: 4
; CHECK-NEXT: 2: 3
; CHECK-KEEP-COLD-NEXT: 5: 4 fb:4
; CHECK-KEEP-COLD-NEXT: 6: 3 fa:3
; CHECK-KEEP-COLD-NEXT: !CFGChecksum: 72617220756
+; CHECK-KEEP-COLD-NEXT: !Attributes: 0
; CHECK-KEEP-COLD-NEXT:[fa]:14:4
; CHECK-KEEP-COLD-NEXT: 1: 4
; CHECK-KEEP-COLD-NEXT: 3: 4
; CHECK-NEXT: 6: 15
; CHECK-NEXT: 8: 15 bar:15
; CHECK-NEXT: !CFGChecksum: 138950591924
-; CHECK-NEXT:[main:2 @ foo:8 @ bar]:30:15
+; CHECK:[main:2 @ foo:8 @ bar]:30:15
; CHECK-NEXT: 1: 15
; CHECK-NEXT: 2: 18446744073709551615
; CHECK-NEXT: 3: 18446744073709551615
std::shared_ptr<StringBasedCtxKey> FrameStack::getContextKey() {
std::shared_ptr<StringBasedCtxKey> KeyStr =
std::make_shared<StringBasedCtxKey>();
- KeyStr->Context = Binary->getExpandedContextStr(Stack);
+ KeyStr->Context =
+ Binary->getExpandedContextStr(Stack, KeyStr->WasLeafInlined);
if (KeyStr->Context.empty())
return nullptr;
KeyStr->genHashCode();
// String based context id
struct StringBasedCtxKey : public ContextKey {
std::string Context;
- StringBasedCtxKey() : ContextKey(CK_StringBased){};
+ bool WasLeafInlined;
+ StringBasedCtxKey() : ContextKey(CK_StringBased), WasLeafInlined(false){};
static bool classof(const ContextKey *K) {
return K->getKind() == CK_StringBased;
}
}
FunctionSamples &
-CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr) {
+CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr,
+ bool WasLeafInlined) {
auto Ret = ProfileMap.try_emplace(ContextStr, FunctionSamples());
if (Ret.second) {
SampleContext FContext(Ret.first->first(), RawContext);
+ if (WasLeafInlined)
+ FContext.setAttribute(ContextWasInlined);
FunctionSamples &FProfile = Ret.first->second;
FProfile.setContext(FContext);
}
StringRef ContextId(CtxKey->Context);
// Get or create function profile for the range
FunctionSamples &FunctionProfile =
- getFunctionProfileForContext(ContextId);
+ getFunctionProfileForContext(ContextId, CtxKey->WasLeafInlined);
// Fill in function body samples
populateFunctionBodySamples(FunctionProfile, CI.second.RangeCounter,
assert(Ret.second && "Must be a unique context");
SampleContext FContext(Ret.first->first(), RawContext);
FunctionSamples &FProfile = Ret.first->second;
+ FContext.setAllAttributes(FProfile.getContext().getAllAttributes());
FProfile.setName(FContext.getNameWithContext(true));
FProfile.setContext(FContext);
}
FunctionSamples &PseudoProbeCSProfileGenerator::getFunctionProfileForLeafProbe(
SmallVectorImpl<std::string> &ContextStrStack,
- const PseudoProbeFuncDesc *LeafFuncDesc) {
+ const PseudoProbeFuncDesc *LeafFuncDesc, bool WasLeafInlined) {
assert(ContextStrStack.size() && "Profile context must have the leaf frame");
// Compress the context string except for the leaf frame
std::string LeafFrame = ContextStrStack.back();
OContextStr << StringRef(LeafFrame).split(":").first.str();
FunctionSamples &FunctionProile =
- getFunctionProfileForContext(OContextStr.str());
+ getFunctionProfileForContext(OContextStr.str(), WasLeafInlined);
FunctionProile.setFunctionHash(LeafFuncDesc->FuncHash);
return FunctionProile;
}
// Explicitly copy the context for appending the leaf context
SmallVector<std::string, 16> ContextStrStackCopy(ContextStrStack.begin(),
ContextStrStack.end());
- Binary->getInlineContextForProbe(LeafProbe, ContextStrStackCopy);
- // Note that the context from probe doesn't include leaf frame,
- // hence we need to retrieve and append the leaf frame.
+ Binary->getInlineContextForProbe(LeafProbe, ContextStrStackCopy, true);
const auto *FuncDesc = Binary->getFuncDescForGUID(LeafProbe->GUID);
- ContextStrStackCopy.emplace_back(FuncDesc->FuncName + ":" +
- Twine(LeafProbe->Index).str());
- return getFunctionProfileForLeafProbe(ContextStrStackCopy, FuncDesc);
+ bool WasLeafInlined = LeafProbe->InlineTree->hasInlineSite();
+ return getFunctionProfileForLeafProbe(ContextStrStackCopy, FuncDesc,
+ WasLeafInlined);
}
} // end namespace sampleprof
protected:
// Lookup or create FunctionSamples for the context
- FunctionSamples &getFunctionProfileForContext(StringRef ContextId);
+ FunctionSamples &getFunctionProfileForContext(StringRef ContextId,
+ bool WasLeafInlined = false);
// Merge cold context profile whose total sample is below threshold
// into base profile.
void mergeAndTrimColdProfile(StringMap<FunctionSamples> &ProfileMap);
// Helper function to get FunctionSamples for the leaf inlined context
FunctionSamples &
getFunctionProfileForLeafProbe(SmallVectorImpl<std::string> &ContextStrStack,
- const PseudoProbeFuncDesc *LeafFuncDesc);
+ const PseudoProbeFuncDesc *LeafFuncDesc,
+ bool WasLeafInlined);
// Helper function to get FunctionSamples for the leaf probe
FunctionSamples &
getFunctionProfileForLeafProbe(SmallVectorImpl<std::string> &ContextStrStack,
Context2.begin(), Context2.begin() + Context2.size() - 1);
}
-std::string ProfiledBinary::getExpandedContextStr(
- const SmallVectorImpl<uint64_t> &Stack) const {
+std::string
+ProfiledBinary::getExpandedContextStr(const SmallVectorImpl<uint64_t> &Stack,
+ bool &WasLeafInlined) const {
std::string ContextStr;
SmallVector<std::string, 16> ContextVec;
// Process from frame root to leaf
// processing
if (ExpandedContext.empty())
return std::string();
+ // Set WasLeafInlined to the size of inlined frame count for the last
+ // address which is leaf
+ WasLeafInlined = (ExpandedContext.size() > 1);
for (const auto &Loc : ExpandedContext) {
ContextVec.push_back(getCallSite(Loc));
}
// Get the context string of the current stack with inline context filled in.
// It will search the disassembling info stored in Offset2LocStackMap. This is
// used as the key of function sample map
- std::string
- getExpandedContextStr(const SmallVectorImpl<uint64_t> &Stack) const;
+ std::string getExpandedContextStr(const SmallVectorImpl<uint64_t> &Stack,
+ bool &WasLeafInlined) const;
const PseudoProbe *getCallProbeForAddr(uint64_t Address) const {
return ProbeDecoder.getCallProbeForAddr(Address);