This change enables pseudo-probe-based sample counts to be consumed by the sample profile loader under the regular `-fprofile-sample-use` switch with minimal adjustments to the existing sample file formats. After the counts are imported, a probe helper, aka, a `PseudoProbeManager` object, is automatically launched to verify the CFG checksum of every function in the current compilation against the corresponding checksum from the profile. Mismatched checksums will cause a function profile to be slipped. A `SampleProfileProber` pass is scheduled before any of the `SampleProfileLoader` instances so that the CFG checksums as well as probe mappings are available during the profile loading time. The `PseudoProbeManager` object is set up right after the profile reading is done. In the future a CFG-based fuzzy matching could be done in `PseudoProbeManager`.
Samples will be applied only to pseudo probe instructions as well as probed callsites once the checksum verification goes through. Those instructions are processed in the same way that regular instructions would be processed in the line-number-based scenario. In other words, a function is processed in a regular way as if it was reduced to just containing pseudo probes (block probes and callsites).
**Adjustment to profile format **
A CFG checksum field is being added to the existing AutoFDO profile formats. So far only the text format and the extended binary format are supported. For the text format, a new line like
```
!CFGChecksum: 12345
```
is added to the end of the body sample lines. For the extended binary profile format, we introduce a metadata section to store the checksum map from function names to their CFG checksums.
Differential Revision: https://reviews.llvm.org/D92347
#ifndef LLVM_IR_PSEUDOPROBE_H
#define LLVM_IR_PSEUDOPROBE_H
+#include "llvm/ADT/Optional.h"
#include <cassert>
#include <cstdint>
namespace llvm {
+class Instruction;
+
constexpr const char *PseudoProbeDescMetadataName = "llvm.pseudo_probe_desc";
enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall };
return (Value >> 29) & 0x7;
}
};
+
+struct PseudoProbe {
+ uint32_t Id;
+ uint32_t Type;
+ uint32_t Attr;
+};
+
+Optional<PseudoProbe> extractProbe(const Instruction &Inst);
+
} // end namespace llvm
#endif // LLVM_IR_PSEUDOPROBE_H
ostream_seek_unsupported,
compress_failed,
uncompress_failed,
- zlib_unavailable
+ zlib_unavailable,
+ hash_mismatch
};
inline std::error_code make_error_code(sampleprof_error E) {
SecNameTable = 2,
SecProfileSymbolList = 3,
SecFuncOffsetTable = 4,
+ SecFuncMetadata = 5,
// marker for the first type of profile.
SecFuncProfileFirst = 32,
SecLBRProfile = SecFuncProfileFirst
return "ProfileSymbolListSection";
case SecFuncOffsetTable:
return "FuncOffsetTableSection";
+ case SecFuncMetadata:
+ return "FunctionMetadata";
case SecLBRProfile:
return "LBRProfileSection";
}
SecFlagPartial = (1 << 0)
};
+enum class SecFuncMetadataFlags : uint32_t {
+ SecFlagInvalid = 0,
+ SecFlagIsProbeBased = (1 << 0),
+};
+
// Verify section specific flag is used for the correct section.
template <class SecFlagType>
static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
case SecProfSummary:
IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();
break;
+ case SecFuncMetadata:
+ IsFlagLegal = std::is_same<SecFuncMetadataFlags, SecFlagType>();
+ break;
default:
break;
}
: sampleprof_error::success;
}
+ void setTotalSamples(uint64_t Num) { TotalSamples = Num; }
+
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) {
bool Overflowed;
TotalHeadSamples =
if (ProfileIsCS)
return 0;
return std::error_code();
+ // A missing counter for a probe likely means the probe was not executed.
+ // Treat it as a zero count instead of an unknown count to help edge
+ // weight inference.
+ if (FunctionSamples::ProfileIsProbeBased)
+ return 0;
+ return std::error_code();
} else {
return ret->second.getSamples();
}
return ret->second.getCallTargets();
}
+ /// Returns the call target map collected at a given location specified by \p
+ /// CallSite. If the location is not found in profile, return error.
+ ErrorOr<SampleRecord::CallTargetMap>
+ findCallTargetMapAt(const LineLocation &CallSite) const {
+ const auto &Ret = BodySamples.find(CallSite);
+ if (Ret == BodySamples.end())
+ return std::error_code();
+ return Ret->second.getCallTargets();
+ }
+
/// Return the function samples at the given callsite location.
FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) {
return CallsiteSamples[Loc];
Name = Other.getName();
if (!GUIDToFuncNameMap)
GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
+
+ if (FunctionHash == 0) {
+ // Set the function hash code for the target profile.
+ FunctionHash = Other.getFunctionHash();
+ } else if (FunctionHash != Other.getFunctionHash()) {
+ // The two profiles coming with different valid hash codes indicates
+ // either:
+ // 1. They are same-named static functions from different compilation
+ // units (without using -unique-internal-linkage-names), or
+ // 2. They are really the same function but from different compilations.
+ // Let's bail out in either case for now, which means one profile is
+ // dropped.
+ return sampleprof_error::hash_mismatch;
+ }
+
MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight));
MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight));
for (const auto &I : Other.getBodySamples()) {
/// Return the original function name.
StringRef getFuncName() const { return getFuncName(Name); }
+ void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; }
+
+ uint64_t getFunctionHash() const { return FunctionHash; }
+
/// Return the canonical name for a function, taking into account
/// suffix elision policy attributes.
static StringRef getCanonicalFnName(const Function &F) {
/// We assume that a single function will not exceed 65535 LOC.
static unsigned getOffset(const DILocation *DIL);
+ /// Returns a unique call site identifier for a given debug location of a call
+ /// instruction. This is wrapper of two scenarios, the probe-based profile and
+ /// regular profile, to hide implementation details from the sample loader and
+ /// the context tracker.
+ static LineLocation getCallSiteIdentifier(const DILocation *DIL);
+
/// Get the FunctionSamples of the inline instance where DIL originates
/// from.
///
const DILocation *DIL,
SampleProfileReaderItaniumRemapper *Remapper = nullptr) const;
+ static bool ProfileIsProbeBased;
+
static bool ProfileIsCS;
SampleContext &getContext() const { return Context; }
/// Mangled name of the function.
StringRef Name;
+ /// CFG hash value for the function.
+ uint64_t FunctionHash = 0;
+
/// Calling context for function profile
mutable SampleContext Context;
// offsetA[.discriminator]: fnA:num_of_total_samples
// offsetA1[.discriminator]: number_of_samples [fn7:num fn8:num ... ]
// ...
+// !CFGChecksum: num
//
-// This is a nested tree in which the identation represents the nesting level
+// This is a nested tree in which the indentation represents the nesting level
// of the inline stack. There are no blank lines in the file. And the spacing
// within a single line is fixed. Additional spaces will result in an error
// while reading the file.
// in the prologue of the function (second number). This head sample
// count provides an indicator of how frequently the function is invoked.
//
-// There are two types of lines in the function body.
+// There are three types of lines in the function body.
//
// * Sampled line represents the profile information of a source location.
// * Callsite line represents the profile information of a callsite.
+// * Metadata line represents extra metadata of the function.
//
// Each sampled line may contain several items. Some are optional (marked
// below):
// total number of samples collected for the inlined instance at this
// callsite
//
+// Metadata line can occur in lines with one indent only, containing extra
+// information for the top-level function. Furthermore, metadata can only
+// occur after all the body samples and callsite samples.
+// Each metadata line may contain a particular type of metadata, marked by
+// the starting characters annotated with !. We process each metadata line
+// independently, hence each metadata line has to form an independent piece
+// of information that does not require cross-line reference.
+// We support the following types of metadata:
+//
+// a. CFG Checksum (a.k.a. function hash):
+// !CFGChecksum: 12345
+//
//
// Binary format
// -------------
/// \brief Return the profile format.
SampleProfileFormat getFormat() const { return Format; }
- /// Whether input profile is fully context-sensitie
+ /// Whether input profile is based on pseudo probes.
+ bool profileIsProbeBased() const { return ProfileIsProbeBased; }
+
+ /// Whether input profile is fully context-sensitive
bool profileIsCS() const { return ProfileIsCS; }
virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() {
std::unique_ptr<SampleProfileReaderItaniumRemapper> Remapper;
+ /// \brief Whether samples are collected based on pseudo probes.
+ bool ProfileIsProbeBased = false;
+
bool ProfileIsCS = false;
/// \brief The format of sample.
std::error_code readSecHdrTableEntry();
std::error_code readSecHdrTable();
+ std::error_code readFuncMetadata();
std::error_code readFuncOffsetTable();
std::error_code readFuncProfiles();
std::error_code readMD5NameTable();
// Helper function to write name table.
virtual std::error_code writeNameTable() override;
+ std::error_code writeFuncMetadata(const StringMap<FunctionSamples> &Profiles);
+
// Functions to write various kinds of sections.
std::error_code
writeNameTableSection(const StringMap<FunctionSamples> &ProfileMap);
// SecFuncOffsetTable section is written after SecLBRProfile in the
// profile because FuncOffsetTable needs to be populated while section
// SecLBRProfile is written.
- SectionHdrLayout = {{SecProfSummary, 0, 0, 0},
- {SecNameTable, 0, 0, 0},
- {SecFuncOffsetTable, 0, 0, 0},
- {SecLBRProfile, 0, 0, 0},
- {SecProfileSymbolList, 0, 0, 0}};
+ SectionHdrLayout = {
+ {SecProfSummary, 0, 0, 0}, {SecNameTable, 0, 0, 0},
+ {SecFuncOffsetTable, 0, 0, 0}, {SecLBRProfile, 0, 0, 0},
+ {SecProfileSymbolList, 0, 0, 0}, {SecFuncMetadata, 0, 0, 0}};
};
virtual std::error_code
writeSections(const StringMap<FunctionSamples> &ProfileMap) override;
#include "llvm/ADT/DenseMap.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PseudoProbe.h"
+#include "llvm/ProfileData/SampleProf.h"
#include "llvm/Target/TargetMachine.h"
#include <unordered_map>
class Module;
+using namespace sampleprof;
using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>;
using InstructionIdMap = std::unordered_map<Instruction *, uint32_t>;
enum class PseudoProbeReservedId { Invalid = 0, Last = Invalid };
+class PseudoProbeDescriptor {
+ uint64_t FunctionGUID;
+ uint64_t FunctionHash;
+
+public:
+ PseudoProbeDescriptor(uint64_t GUID, uint64_t Hash)
+ : FunctionGUID(GUID), FunctionHash(Hash) {}
+ uint64_t getFunctionGUID() const { return FunctionGUID; }
+ uint64_t getFunctionHash() const { return FunctionHash; }
+};
+
+// This class serves sample counts correlation for SampleProfileLoader by
+// analyzing pseudo probes and their function descriptors injected by
+// SampleProfileProber.
+class PseudoProbeManager {
+ DenseMap<uint64_t, PseudoProbeDescriptor> GUIDToProbeDescMap;
+
+ const PseudoProbeDescriptor *getDesc(const Function &F) const;
+
+public:
+ PseudoProbeManager(const Module &M);
+ bool moduleIsProbed(const Module &M) const;
+ bool profileIsValid(const Function &F, const FunctionSamples &Samples) const;
+};
/// Sample profile pseudo prober.
///
PrintPasses.cpp
SafepointIRVerifier.cpp
ProfileSummary.cpp
+ PseudoProbe.cpp
Statepoint.cpp
StructuralHash.cpp
Type.cpp
--- /dev/null
+//===- PseudoProbe.cpp - Pseudo Probe Helpers -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the helpers to manipulate pseudo probe IR intrinsic
+// calls.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/PseudoProbe.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+
+using namespace llvm;
+
+namespace llvm {
+
+Optional<PseudoProbe> extractProbeFromDiscriminator(const Instruction &Inst) {
+ assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) &&
+ "Only call instructions should have pseudo probe encodes as their "
+ "Dwarf discriminators");
+ if (const DebugLoc &DLoc = Inst.getDebugLoc()) {
+ const DILocation *DIL = DLoc;
+ auto Discriminator = DIL->getDiscriminator();
+ if (DILocation::isPseudoProbeDiscriminator(Discriminator)) {
+ PseudoProbe Probe;
+ Probe.Id =
+ PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator);
+ Probe.Type =
+ PseudoProbeDwarfDiscriminator::extractProbeType(Discriminator);
+ Probe.Attr =
+ PseudoProbeDwarfDiscriminator::extractProbeAttributes(Discriminator);
+ return Probe;
+ }
+ }
+ return None;
+}
+
+Optional<PseudoProbe> extractProbe(const Instruction &Inst) {
+ if (const auto *II = dyn_cast<PseudoProbeInst>(&Inst)) {
+ PseudoProbe Probe;
+ Probe.Id = II->getIndex()->getZExtValue();
+ Probe.Type = (uint32_t)PseudoProbeType::Block;
+ Probe.Attr = II->getAttributes()->getZExtValue();
+ return Probe;
+ }
+
+ if (isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst))
+ return extractProbeFromDiscriminator(Inst);
+
+ return None;
+}
+} // namespace llvm
#include "llvm/ProfileData/SampleProf.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/PseudoProbe.h"
#include "llvm/ProfileData/SampleProfReader.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
namespace llvm {
namespace sampleprof {
SampleProfileFormat FunctionSamples::Format;
+bool FunctionSamples::ProfileIsProbeBased = false;
bool FunctionSamples::ProfileIsCS = false;
bool FunctionSamples::UseMD5;
} // namespace sampleprof
return "Uncompress failure";
case sampleprof_error::zlib_unavailable:
return "Zlib is unavailable";
+ case sampleprof_error::hash_mismatch:
+ return "Function hash mismatch";
}
llvm_unreachable("A value of sampleprof_error has no message.");
}
/// Print the samples collected for a function on stream \p OS.
void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
+ if (getFunctionHash())
+ OS << "CFG checksum " << getFunctionHash() << "\n";
+
OS << TotalSamples << ", " << TotalHeadSamples << ", " << BodySamples.size()
<< " sampled lines\n";
0xffff;
}
+LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL) {
+ if (FunctionSamples::ProfileIsProbeBased)
+ // In a pseudo-probe based profile, a callsite is simply represented by the
+ // ID of the probe associated with the call instruction. The probe ID is
+ // encoded in the Discriminator field of the call instruction's debug
+ // metadata.
+ return LineLocation(PseudoProbeDwarfDiscriminator::extractProbeIndex(
+ DIL->getDiscriminator()),
+ 0);
+ else
+ return LineLocation(FunctionSamples::getOffset(DIL),
+ DIL->getBaseDiscriminator());
+}
+
const FunctionSamples *FunctionSamples::findFunctionSamples(
const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const {
assert(DIL);
/// Returns true if line offset \p L is legal (only has 16 bits).
static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
+/// Parse \p Input that contains metadata.
+/// Possible metadata:
+/// - CFG Checksum information:
+/// !CFGChecksum: 12345
+/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
+static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash) {
+ if (!Input.startswith("!CFGChecksum:"))
+ return false;
+
+ StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();
+ return !CFGInfo.getAsInteger(10, FunctionHash);
+}
+
+enum class LineType {
+ CallSiteProfile,
+ BodyProfile,
+ Metadata,
+};
+
/// Parse \p Input as line sample.
///
/// \param Input input line.
-/// \param IsCallsite true if the line represents an inlined callsite.
+/// \param LineTy Type of this line.
/// \param Depth the depth of the inline stack.
/// \param NumSamples total samples of the line/inlined callsite.
/// \param LineOffset line offset to the start of the function.
/// \param Discriminator discriminator of the line.
/// \param TargetCountMap map from indirect call target to count.
+/// \param FunctionHash the function's CFG hash, used by pseudo probe.
///
/// returns true if parsing is successful.
-static bool ParseLine(const StringRef &Input, bool &IsCallsite, uint32_t &Depth,
+static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
uint64_t &NumSamples, uint32_t &LineOffset,
uint32_t &Discriminator, StringRef &CalleeName,
- DenseMap<StringRef, uint64_t> &TargetCountMap) {
+ DenseMap<StringRef, uint64_t> &TargetCountMap,
+ uint64_t &FunctionHash) {
for (Depth = 0; Input[Depth] == ' '; Depth++)
;
if (Depth == 0)
return false;
+ if (Depth == 1 && Input[Depth] == '!') {
+ LineTy = LineType::Metadata;
+ return parseMetadata(Input.substr(Depth), FunctionHash);
+ }
+
size_t n1 = Input.find(':');
StringRef Loc = Input.substr(Depth, n1 - Depth);
size_t n2 = Loc.find('.');
StringRef Rest = Input.substr(n1 + 2);
if (Rest[0] >= '0' && Rest[0] <= '9') {
- IsCallsite = false;
+ LineTy = LineType::BodyProfile;
size_t n3 = Rest.find(' ');
if (n3 == StringRef::npos) {
if (Rest.getAsInteger(10, NumSamples))
n3 = n4;
}
} else {
- IsCallsite = true;
+ LineTy = LineType::CallSiteProfile;
size_t n3 = Rest.find_last_of(':');
CalleeName = Rest.substr(0, n3);
if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples))
InlineCallStack InlineStack;
int CSProfileCount = 0;
int RegularProfileCount = 0;
+ uint32_t ProbeProfileCount = 0;
+
+ // SeenMetadata tracks whether we have processed metadata for the current
+ // top-level function profile.
+ bool SeenMetadata = false;
for (; !LineIt.is_at_eof(); ++LineIt) {
if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#')
"Expected 'mangled_name:NUM:NUM', found " + *LineIt);
return sampleprof_error::malformed;
}
+ SeenMetadata = false;
SampleContext FContext(FName);
if (FContext.hasContext())
++CSProfileCount;
uint64_t NumSamples;
StringRef FName;
DenseMap<StringRef, uint64_t> TargetCountMap;
- bool IsCallsite;
uint32_t Depth, LineOffset, Discriminator;
- if (!ParseLine(*LineIt, IsCallsite, Depth, NumSamples, LineOffset,
- Discriminator, FName, TargetCountMap)) {
+ LineType LineTy;
+ uint64_t FunctionHash;
+ if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
+ Discriminator, FName, TargetCountMap, FunctionHash)) {
reportError(LineIt.line_number(),
"Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
*LineIt);
return sampleprof_error::malformed;
}
- if (IsCallsite) {
- while (InlineStack.size() > Depth) {
- InlineStack.pop_back();
- }
+ if (SeenMetadata && LineTy != LineType::Metadata) {
+ // Metadata must be put at the end of a function profile.
+ reportError(LineIt.line_number(),
+ "Found non-metadata after metadata: " + *LineIt);
+ return sampleprof_error::malformed;
+ }
+ while (InlineStack.size() > Depth) {
+ InlineStack.pop_back();
+ }
+ switch (LineTy) {
+ case LineType::CallSiteProfile: {
FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
LineLocation(LineOffset, Discriminator))[std::string(FName)];
FSamples.setName(FName);
MergeResult(Result, FSamples.addTotalSamples(NumSamples));
InlineStack.push_back(&FSamples);
- } else {
+ break;
+ }
+ case LineType::BodyProfile: {
while (InlineStack.size() > Depth) {
InlineStack.pop_back();
}
}
MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
NumSamples));
+ break;
+ }
+ case LineType::Metadata: {
+ FunctionSamples &FProfile = *InlineStack.back();
+ FProfile.setFunctionHash(FunctionHash);
+ ++ProbeProfileCount;
+ SeenMetadata = true;
+ break;
+ }
}
}
}
assert((RegularProfileCount == 0 || CSProfileCount == 0) &&
"Cannot have both context-sensitive and regular profile");
ProfileIsCS = (CSProfileCount > 0);
+ assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) &&
+ "Cannot have both probe-based profiles and regular profiles");
+ ProfileIsProbeBased = (ProbeProfileCount > 0);
+ FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
if (Result == sampleprof_error::success)
computeSummary();
if (std::error_code EC = readFuncOffsetTable())
return EC;
break;
+ case SecFuncMetadata:
+ ProfileIsProbeBased =
+ hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased);
+ FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
+ if (std::error_code EC = readFuncMetadata())
+ return EC;
+ break;
case SecProfileSymbolList:
if (std::error_code EC = readProfileSymbolList())
return EC;
return SampleProfileReaderBinary::readNameTable();
}
+std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata() {
+ if (!ProfileIsProbeBased)
+ return sampleprof_error::success;
+ for (unsigned I = 0; I < Profiles.size(); ++I) {
+ auto FName(readStringFromTable());
+ if (std::error_code EC = FName.getError())
+ return EC;
+
+ auto Checksum = readNumber<uint64_t>();
+ if (std::error_code EC = Checksum.getError())
+ return EC;
+
+ Profiles[*FName].setFunctionHash(*Checksum);
+ }
+ return sampleprof_error::success;
+}
+
std::error_code SampleProfileReaderCompactBinary::readNameTable() {
auto Size = readNumber<uint64_t>();
if (std::error_code EC = Size.getError())
return sampleprof_error::success;
}
+std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(
+ const StringMap<FunctionSamples> &Profiles) {
+ if (!FunctionSamples::ProfileIsProbeBased)
+ return sampleprof_error::success;
+ auto &OS = *OutputStream;
+ for (const auto &Entry : Profiles) {
+ writeNameIdx(Entry.first());
+ encodeULEB128(Entry.second.getFunctionHash(), OS);
+ }
+ return sampleprof_error::success;
+}
+
std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
if (!UseMD5)
return SampleProfileWriterBinary::writeNameTable();
// The setting of SecFlagCompress should happen before markSectionStart.
if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
setToCompressSection(SecProfileSymbolList);
+ if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)
+ addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);
uint64_t SectionStart = markSectionStart(Type);
switch (Type) {
if (auto EC = writeFuncOffsetTable())
return EC;
break;
+ case SecFuncMetadata:
+ if (std::error_code EC = writeFuncMetadata(ProfileMap))
+ return EC;
+ break;
case SecProfileSymbolList:
if (auto EC = writeProfileSymbolListSection())
return EC;
return EC;
if (auto EC = writeOneSection(SecFuncOffsetTable, ProfileMap))
return EC;
+ if (auto EC = writeOneSection(SecFuncMetadata, ProfileMap))
+ return EC;
return sampleprof_error::success;
}
}
Indent -= 1;
+ if (Indent == 0) {
+ if (FunctionSamples::ProfileIsProbeBased) {
+ OS.indent(Indent + 1);
+ OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
+ }
+ }
+
return sampleprof_error::success;
}
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/SampleContextTracker.h"
+#include "llvm/Transforms/IPO/SampleProfileProbe.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
"Number of functions inlined with context sensitive profile");
STATISTIC(NumCSNotInlined,
"Number of functions not inlined with context sensitive profile");
+STATISTIC(NumMismatchedProfile,
+ "Number of functions with CFG mismatched profile");
+STATISTIC(NumMatchedProfile, "Number of functions with CFG matched profile");
// Command line option to specify the file to read samples from. This is
// mainly used for debugging.
unsigned getFunctionLoc(Function &F);
bool emitAnnotations(Function &F);
ErrorOr<uint64_t> getInstWeight(const Instruction &I);
+ ErrorOr<uint64_t> getProbeWeight(const Instruction &I);
ErrorOr<uint64_t> getBlockWeight(const BasicBlock *BB);
const FunctionSamples *findCalleeFunctionSamples(const CallBase &I) const;
std::vector<const FunctionSamples *>
// External inline advisor used to replay inline decision from remarks.
std::unique_ptr<ReplayInlineAdvisor> ExternalInlineAdvisor;
+
+ // A pseudo probe helper to correlate the imported sample counts.
+ std::unique_ptr<PseudoProbeManager> ProbeManager;
};
class SampleProfileLoaderLegacyPass : public ModulePass {
///
/// \returns the weight of \p Inst.
ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
+ if (FunctionSamples::ProfileIsProbeBased)
+ return getProbeWeight(Inst);
+
const DebugLoc &DLoc = Inst.getDebugLoc();
if (!DLoc)
return std::error_code();
return R;
}
+ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) {
+ assert(FunctionSamples::ProfileIsProbeBased &&
+ "Profile is not pseudo probe based");
+ Optional<PseudoProbe> Probe = extractProbe(Inst);
+ if (!Probe)
+ return std::error_code();
+
+ const FunctionSamples *FS = findFunctionSamples(Inst);
+ if (!FS)
+ return std::error_code();
+
+ // If a direct call/invoke instruction is inlined in profile
+ // (findCalleeFunctionSamples returns non-empty result), but not inlined here,
+ // it means that the inlined callsite has no sample, thus the call
+ // instruction should have 0 count.
+ if (const auto *CB = dyn_cast<CallBase>(&Inst))
+ if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
+ return 0;
+
+ const ErrorOr<uint64_t> &R = FS->findSamplesAt(Probe->Id, 0);
+ if (R) {
+ uint64_t Samples = R.get();
+ bool FirstMark = CoverageTracker.markSamplesUsed(FS, Probe->Id, 0, Samples);
+ if (FirstMark) {
+ ORE->emit([&]() {
+ OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "AppliedSamples", &Inst);
+ Remark << "Applied " << ore::NV("NumSamples", Samples);
+ Remark << " samples from profile (ProbeId=";
+ Remark << ore::NV("ProbeId", Probe->Id);
+ Remark << ")";
+ return Remark;
+ });
+ }
+
+ LLVM_DEBUG(dbgs() << " " << Probe->Id << ":" << Inst
+ << " - weight: " << R.get() << ")\n");
+ return Samples;
+ }
+ return R;
+}
+
/// Compute the weight of a basic block.
///
/// The weight of basic block \p BB is the maximum weight of all the
if (FS == nullptr)
return nullptr;
- return FS->findFunctionSamplesAt(LineLocation(FunctionSamples::getOffset(DIL),
- DIL->getBaseDiscriminator()),
+ return FS->findFunctionSamplesAt(FunctionSamples::getCallSiteIdentifier(DIL),
CalleeName, Reader->getRemapper());
}
if (FS == nullptr)
return R;
- uint32_t LineOffset = FunctionSamples::getOffset(DIL);
- uint32_t Discriminator = DIL->getBaseDiscriminator();
-
- auto T = FS->findCallTargetMapAt(LineOffset, Discriminator);
+ auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL);
+ auto T = FS->findCallTargetMapAt(CallSite);
Sum = 0;
if (T)
for (const auto &T_C : T.get())
Sum += T_C.second;
- if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(LineLocation(
- FunctionSamples::getOffset(DIL), DIL->getBaseDiscriminator()))) {
+ if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) {
if (M->empty())
return R;
for (const auto &NameFS : *M) {
/// \returns the FunctionSamples pointer to the inlined instance.
const FunctionSamples *
SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
+ if (FunctionSamples::ProfileIsProbeBased) {
+ Optional<PseudoProbe> Probe = extractProbe(Inst);
+ if (!Probe)
+ return nullptr;
+ }
+
const DILocation *DIL = Inst.getDebugLoc();
if (!DIL)
return Samples;
if (!DLoc)
continue;
const DILocation *DIL = DLoc;
- uint32_t LineOffset = FunctionSamples::getOffset(DIL);
- uint32_t Discriminator = DIL->getBaseDiscriminator();
-
const FunctionSamples *FS = findFunctionSamples(I);
if (!FS)
continue;
- auto T = FS->findCallTargetMapAt(LineOffset, Discriminator);
+ auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL);
+ auto T = FS->findCallTargetMapAt(CallSite);
if (!T || T.get().empty())
continue;
SmallVector<InstrProfValueData, 2> SortedCallTargets =
bool SampleProfileLoader::emitAnnotations(Function &F) {
bool Changed = false;
- if (getFunctionLoc(F) == 0)
- return false;
+ if (FunctionSamples::ProfileIsProbeBased) {
+ if (!ProbeManager->profileIsValid(F, *Samples)) {
+ LLVM_DEBUG(
+ dbgs() << "Profile is invalid due to CFG mismatch for Function "
+ << F.getName());
+ ++NumMismatchedProfile;
+ return false;
+ }
+ ++NumMatchedProfile;
+ } else {
+ if (getFunctionLoc(F) == 0)
+ return false;
- LLVM_DEBUG(dbgs() << "Line number for the first instruction in "
- << F.getName() << ": " << getFunctionLoc(F) << "\n");
+ LLVM_DEBUG(dbgs() << "Line number for the first instruction in "
+ << F.getName() << ": " << getFunctionLoc(F) << "\n");
+ }
DenseSet<GlobalValue::GUID> InlinedGUIDs;
Changed |= inlineHotFunctions(F, InlinedGUIDs);
std::make_unique<SampleContextTracker>(Reader->getProfiles());
}
+ // Load pseudo probe descriptors for probe-based function samples.
+ if (Reader->profileIsProbeBased()) {
+ ProbeManager = std::make_unique<PseudoProbeManager>(M);
+ if (!ProbeManager->moduleIsProbed(M)) {
+ const char *Msg =
+ "Pseudo-probe-based profile requires SampleProfileProbePass";
+ Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
+ return false;
+ }
+ }
+
return true;
}
STATISTIC(ArtificialDbgLine,
"Number of probes that have an artificial debug line");
+PseudoProbeManager::PseudoProbeManager(const Module &M) {
+ if (NamedMDNode *FuncInfo = M.getNamedMetadata(PseudoProbeDescMetadataName)) {
+ for (const auto *Operand : FuncInfo->operands()) {
+ const auto *MD = cast<MDNode>(Operand);
+ auto GUID =
+ mdconst::dyn_extract<ConstantInt>(MD->getOperand(0))->getZExtValue();
+ auto Hash =
+ mdconst::dyn_extract<ConstantInt>(MD->getOperand(1))->getZExtValue();
+ GUIDToProbeDescMap.try_emplace(GUID, PseudoProbeDescriptor(GUID, Hash));
+ }
+ }
+}
+
+const PseudoProbeDescriptor *
+PseudoProbeManager::getDesc(const Function &F) const {
+ auto I = GUIDToProbeDescMap.find(
+ Function::getGUID(FunctionSamples::getCanonicalFnName(F)));
+ return I == GUIDToProbeDescMap.end() ? nullptr : &I->second;
+}
+
+bool PseudoProbeManager::moduleIsProbed(const Module &M) const {
+ return M.getNamedMetadata(PseudoProbeDescMetadataName);
+}
+
+bool PseudoProbeManager::profileIsValid(const Function &F,
+ const FunctionSamples &Samples) const {
+ const auto *Desc = getDesc(F);
+ if (!Desc) {
+ LLVM_DEBUG(dbgs() << "Probe descriptor missing for Function " << F.getName()
+ << "\n");
+ return false;
+ } else {
+ if (Desc->getFunctionHash() != Samples.getFunctionHash()) {
+ LLVM_DEBUG(dbgs() << "Hash mismatch for Function " << F.getName()
+ << "\n");
+ return false;
+ }
+ }
+ return true;
+}
+
SampleProfileProber::SampleProfileProber(Function &Func,
const std::string &CurModuleUniqueId)
: F(&Func), CurModuleUniqueId(CurModuleUniqueId) {
--- /dev/null
+test:3200:0
+ 1: 100
+ 2: foo:1000
+ 1: 800
+ 3: bar:200
+ 2: 190
+ 4: baz:10
+ 2: 10
+ 3: foo1:1000
+ 1: 1000
+ 3: foo2:1000
+ 1: 1000 foo3:1000
+ !CFGChecksum: 562954248388607
+test_liveness:1000:0
+ 2: foo:1000
+ 1: bar:1000
+ 1: 2000 bar_dbg:1000 bar_available:1000
+ 2: 1000
+ !CFGChecksum: 281479271677951
\ No newline at end of file
--- /dev/null
+foo:3200:13
+ 1: 13
+ 2: 7
+ 3: 6
+ 4: 13
+ 5: 7 _Z3barv:2 _Z3foov:5
+ 6: 6 _Z3barv:4 _Z3foov:2
+ !CFGChecksum: 563022570642068
; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.prof -S | FileCheck %s
; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.compact.afdo -S | FileCheck %s
+; RUN: opt < %s -passes='pseudo-probe,thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/pseudo-probe-func-metadata.prof -S | FileCheck %s
; Tests whether the functions in the inline stack are added to the
; function_entry_count metadata.
--- /dev/null
+; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile.prof -pass-remarks=sample-profile -pass-remarks-output=%t.opt.yaml -S | FileCheck %s
+; RUN: FileCheck %s -check-prefix=YAML < %t.opt.yaml
+
+define dso_local i32 @foo(i32 %x, void (i32)* %f) #0 !dbg !4 {
+entry:
+ %retval = alloca i32, align 4
+ %x.addr = alloca i32, align 4
+ store i32 %x, i32* %x.addr, align 4
+ %0 = load i32, i32* %x.addr, align 4
+ %cmp = icmp eq i32 %0, 0
+ ; CHECK: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 1, i32 0)
+ br i1 %cmp, label %if.then, label %if.else
+ ; CHECK: br i1 %cmp, label %if.then, label %if.else, !prof ![[PD1:[0-9]+]]
+
+if.then:
+ ; CHECK: call {{.*}}, !dbg ![[#PROBE1:]], !prof ![[PROF1:[0-9]+]]
+ call void %f(i32 1)
+ ; CHECK: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 2, i32 0)
+ store i32 1, i32* %retval, align 4
+ br label %return
+
+if.else:
+ ; CHECK: call {{.*}}, !dbg ![[#PROBE2:]], !prof ![[PROF2:[0-9]+]]
+ call void %f(i32 2)
+ ; CHECK: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 3, i32 0)
+ store i32 2, i32* %retval, align 4
+ br label %return
+
+return:
+ ; CHECK: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 4, i32 0)
+ %1 = load i32, i32* %retval, align 4
+ ret i32 %1
+}
+
+attributes #0 = {"use-sample-profile"}
+
+; CHECK: ![[PD1]] = !{!"branch_weights", i32 8, i32 7}
+; CHECK: ![[#PROBE1]] = !DILocation(line: 0, scope: ![[#SCOPE1:]])
+;; A discriminator of 119537711 which is 0x400002f in hexdecimal, stands for an indirect call probe
+;; with an index of 5.
+; CHECK: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 67108911)
+; CHECK: ![[PROF1]] = !{!"VP", i32 0, i64 7, i64 9191153033785521275, i64 5, i64 -1069303473483922844, i64 2}
+; CHECK: ![[#PROBE2]] = !DILocation(line: 0, scope: ![[#SCOPE2:]])
+;; A discriminator of 119537719 which is 0x4000037 in hexdecimal, stands for an indirect call probe
+;; with an index of 6.
+; CHECK: ![[#SCOPE2]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 67108919)
+; CHECK: ![[PROF2]] = !{!"VP", i32 0, i64 6, i64 -1069303473483922844, i64 4, i64 9191153033785521275, i64 2}
+
+!llvm.module.flags = !{!9, !10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+
+; Checking to see if YAML file is generated and contains remarks
+;YAML: --- !Analysis
+;YAML-NEXT: Pass: sample-profile
+;YAML-NEXT: Name: AppliedSamples
+;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 }
+;YAML-NEXT: Function: foo
+;YAML-NEXT: Args:
+;YAML-NEXT: - String: 'Applied '
+;YAML-NEXT: - NumSamples: '13'
+;YAML-NEXT: - String: ' samples from profile (ProbeId='
+;YAML-NEXT: - ProbeId: '1'
+;YAML-NEXT: - String: ')'
+;YAML: --- !Analysis
+;YAML-NEXT: Pass: sample-profile
+;YAML-NEXT: Name: AppliedSamples
+;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 }
+;YAML-NEXT: Function: foo
+;YAML-NEXT: Args:
+;YAML-NEXT: - String: 'Applied '
+;YAML-NEXT: - NumSamples: '7'
+;YAML-NEXT: - String: ' samples from profile (ProbeId='
+;YAML-NEXT: - ProbeId: '5'
+;YAML-NEXT: - String: ')'
+;YAML: --- !Analysis
+;YAML-NEXT: Pass: sample-profile
+;YAML-NEXT: Name: AppliedSamples
+;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 }
+;YAML-NEXT: Function: foo
+;YAML-NEXT: Args:
+;YAML-NEXT: - String: 'Applied '
+;YAML-NEXT: - NumSamples: '7'
+;YAML-NEXT: - String: ' samples from profile (ProbeId='
+;YAML-NEXT: - ProbeId: '2'
+;YAML-NEXT: - String: ')'
+;YAML: --- !Analysis
+;YAML-NEXT: Pass: sample-profile
+;YAML-NEXT: Name: AppliedSamples
+;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 }
+;YAML-NEXT: Function: foo
+;YAML-NEXT: Args:
+;YAML-NEXT: - String: 'Applied '
+;YAML-NEXT: - NumSamples: '6'
+;YAML-NEXT: - String: ' samples from profile (ProbeId='
+;YAML-NEXT: - ProbeId: '6'
+;YAML-NEXT: - String: ')'
+;YAML: --- !Analysis
+;YAML-NEXT: Pass: sample-profile
+;YAML-NEXT: Name: AppliedSamples
+;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 }
+;YAML-NEXT: Function: foo
+;YAML-NEXT: Args:
+;YAML-NEXT: - String: 'Applied '
+;YAML-NEXT: - NumSamples: '6'
+;YAML-NEXT: - String: ' samples from profile (ProbeId='
+;YAML-NEXT: - ProbeId: '3'
+;YAML-NEXT: - String: ')'
+;YAML: --- !Analysis
+;YAML-NEXT: Pass: sample-profile
+;YAML-NEXT: Name: AppliedSamples
+;YAML-NEXT: DebugLoc: { File: test.c, Line: 0, Column: 0 }
+;YAML-NEXT: Function: foo
+;YAML-NEXT: Args:
+;YAML-NEXT: - String: 'Applied '
+;YAML-NEXT: - NumSamples: '13'
+;YAML-NEXT: - String: ' samples from profile (ProbeId='
+;YAML-NEXT: - ProbeId: '4'
+;YAML-NEXT: - String: ')'
--- /dev/null
+foo:3200:13
+ 1: 13
+ 2: 7
+ 3: 6
+ 4: 13
+ 5: 7 _Z3foov:5 _Z3barv:2
+ 6: 6 _Z3barv:4 _Z3foov:2
+ !CFGChecksum: 563022570642068
--- /dev/null
+# Tests for merge of probe-based profile files.
+
+RUN: llvm-profdata merge --sample --text %p/Inputs/pseudo-probe-profile.proftext -o - | FileCheck %s --check-prefix=MERGE1
+RUN: llvm-profdata merge --sample --extbinary %p/Inputs/pseudo-probe-profile.proftext -o %t && llvm-profdata merge --sample --text %t -o - | FileCheck %s --check-prefix=MERGE1
+MERGE1: foo:3200:13
+MERGE1: 1: 13
+MERGE1: 2: 7
+MERGE1: 3: 6
+MERGE1: 4: 13
+MERGE1: 5: 7 _Z3foov:5 _Z3barv:2
+MERGE1: 6: 6 _Z3barv:4 _Z3foov:2
+MERGE1: !CFGChecksum: 563022570642068
+
+RUN: llvm-profdata merge --sample --text %p/Inputs/pseudo-probe-profile.proftext %p/Inputs/pseudo-probe-profile.proftext -o - | FileCheck %s --check-prefix=MERGE2
+RUN: llvm-profdata merge --sample --extbinary %p/Inputs/pseudo-probe-profile.proftext %p/Inputs/pseudo-probe-profile.proftext -o %t && llvm-profdata merge --sample --text %t -o - | FileCheck %s --check-prefix=MERGE2
+MERGE2: foo:6400:26
+MERGE2: 1: 26
+MERGE2: 2: 14
+MERGE2: 3: 12
+MERGE2: 4: 26
+MERGE2: 5: 14 _Z3foov:10 _Z3barv:4
+MERGE2: 6: 12 _Z3barv:8 _Z3foov:4
+MERGE2: !CFGChecksum: 563022570642068
SmallVector<std::unique_ptr<sampleprof::SampleProfileReader>, 5> Readers;
LLVMContext Context;
sampleprof::ProfileSymbolList WriterList;
+ Optional<bool> ProfileIsProbeBased;
for (const auto &Input : Inputs) {
auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context);
if (std::error_code EC = ReaderOrErr.getError()) {
}
StringMap<FunctionSamples> &Profiles = Reader->getProfiles();
+ if (ProfileIsProbeBased &&
+ ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased)
+ exitWithError(
+ "cannot merge probe-based profile with non-probe-based profile");
+ ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased;
for (StringMap<FunctionSamples>::iterator I = Profiles.begin(),
E = Profiles.end();
I != E; ++I) {
exitWithErrorCode(EC, BaseFilename);
if (std::error_code EC = TestReader->read())
exitWithErrorCode(EC, TestFilename);
+ if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased())
+ exitWithError(
+ "cannot compare probe-based profile with non-probe-based profile");
// Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in
// profile summary.
projects / platform / upstream / llvm.git / commitdiff