void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) {
if (Start == Stop) return;
- if (NumModulesWithInline8bitCounters &&
- ModuleCounters[NumModulesWithInline8bitCounters-1].Start == Start) return;
- assert(NumModulesWithInline8bitCounters <
- sizeof(ModuleCounters) / sizeof(ModuleCounters[0]));
- ModuleCounters[NumModulesWithInline8bitCounters++] = {Start, Stop};
- NumInline8bitCounters += Stop - Start;
+ if (NumModules &&
+ Modules[NumModules - 1].Start() == Start)
+ return;
+ assert(NumModules <
+ sizeof(Modules) / sizeof(Modules[0]));
+ auto &M = Modules[NumModules++];
+ uint8_t *AlignedStart = RoundUpByPage(Start);
+ uint8_t *AlignedStop = RoundDownByPage(Stop);
+ size_t NumFullPages = AlignedStop > AlignedStart ?
+ (AlignedStop - AlignedStart) / PageSize() : 0;
+ bool NeedFirst = Start < AlignedStart || !NumFullPages;
+ bool NeedLast = Stop > AlignedStop && AlignedStop >= AlignedStart;
+ M.NumRegions = NumFullPages + NeedFirst + NeedLast;;
+ assert(M.NumRegions > 0);
+ M.Regions = new Module::Region[M.NumRegions];
+ assert(M.Regions);
+ size_t R = 0;
+ if (NeedFirst)
+ M.Regions[R++] = {Start, std::min(Stop, AlignedStart), true, false};
+ for (uint8_t *P = AlignedStart; P < AlignedStop; P += PageSize())
+ M.Regions[R++] = {P, P + PageSize(), true, true};
+ if (NeedLast)
+ M.Regions[R++] = {AlignedStop, Stop, true, false};
+ assert(R == M.NumRegions);
+ assert(M.Size() == (size_t)(Stop - Start));
+ assert(M.Stop() == Stop);
+ assert(M.Start() == Start);
+ NumInline8bitCounters += M.Size();
}
void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) {
}
void TracePC::PrintModuleInfo() {
- if (NumModulesWithInline8bitCounters) {
+ if (NumModules) {
Printf("INFO: Loaded %zd modules (%zd inline 8-bit counters): ",
- NumModulesWithInline8bitCounters, NumInline8bitCounters);
- for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++)
- Printf("%zd [%p, %p), ", ModuleCounters[i].Stop - ModuleCounters[i].Start,
- ModuleCounters[i].Start, ModuleCounters[i].Stop);
+ NumModules, NumInline8bitCounters);
+ for (size_t i = 0; i < NumModules; i++)
+ Printf("%zd [%p, %p), ", Modules[i].Size(), Modules[i].Start(),
+ Modules[i].Stop());
Printf("\n");
}
if (NumPCTables) {
if (NumPCsInPCTables) {
if (NumInline8bitCounters == NumPCsInPCTables) {
- for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) {
- uint8_t *Beg = ModuleCounters[i].Start;
- size_t Size = ModuleCounters[i].Stop - Beg;
- assert(Size ==
+ for (size_t i = 0; i < NumModules; i++) {
+ auto &M = Modules[i];
+ assert(M.Size() ==
(size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start));
- for (size_t j = 0; j < Size; j++)
- if (Beg[j])
- Observe(ModulePCTable[i].Start[j]);
+ for (size_t r = 0; r < M.NumRegions; r++) {
+ auto &R = M.Regions[r];
+ if (!R.Enabled) continue;
+ for (uint8_t *P = R.Start; P < R.Stop; P++)
+ if (*P)
+ Observe(ModulePCTable[i].Start[M.Idx(P)]);
+ }
}
}
}
void TracePC::SetFocusFunction(const std::string &FuncName) {
// This function should be called once.
- assert(FocusFunction.first > NumModulesWithInline8bitCounters);
+ assert(!FocusFunctionCounterPtr);
if (FuncName.empty())
return;
- for (size_t M = 0; M < NumModulesWithInline8bitCounters; M++) {
+ for (size_t M = 0; M < NumModules; M++) {
auto &PCTE = ModulePCTable[M];
size_t N = PCTE.Stop - PCTE.Start;
for (size_t I = 0; I < N; I++) {
Name = Name.substr(3, std::string::npos);
if (FuncName != Name) continue;
Printf("INFO: Focus function is set to '%s'\n", Name.c_str());
- FocusFunction = {M, I};
+ FocusFunctionCounterPtr = Modules[M].Start() + I;
return;
}
}
}
bool TracePC::ObservedFocusFunction() {
- size_t I = FocusFunction.first;
- size_t J = FocusFunction.second;
- if (I >= NumModulesWithInline8bitCounters)
- return false;
- auto &MC = ModuleCounters[I];
- size_t Size = MC.Stop - MC.Start;
- if (J >= Size)
- return false;
- return MC.Start[J] != 0;
+ return FocusFunctionCounterPtr && *FocusFunctionCounterPtr;
}
void TracePC::PrintCoverage() {
}
void TracePC::ClearInlineCounters() {
- for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) {
- uint8_t *Beg = ModuleCounters[i].Start;
- size_t Size = ModuleCounters[i].Stop - Beg;
- memset(Beg, 0, Size);
- }
+ IterateCounterRegions([](const Module::Region &R){
+ if (R.Enabled)
+ memset(R.Start, 0, R.Stop - R.Start);
+ });
}
ATTRIBUTE_NO_SANITIZE_ALL
bool DoPrintNewPCs = false;
size_t NumPrintNewFuncs = 0;
- struct { uint8_t *Start, *Stop; } ModuleCounters[4096];
- size_t NumModulesWithInline8bitCounters; // linker-initialized.
+ // Module represents the array of 8-bit counters split into regions
+ // such that every region, except maybe the first and the last one, is one
+ // full page.
+ struct Module {
+ struct Region {
+ uint8_t *Start, *Stop;
+ bool Enabled;
+ bool OneFullPage;
+ };
+ Region *Regions;
+ size_t NumRegions;
+ uint8_t *Start() { return Regions[0].Start; }
+ uint8_t *Stop() { return Regions[NumRegions - 1].Stop; }
+ size_t Size() { return Stop() - Start(); }
+ size_t Idx(uint8_t *P) {
+ assert(P >= Start() && P < Stop());
+ return P - Start();
+ }
+ };
+
+ Module Modules[4096];
+ size_t NumModules; // linker-initialized.
size_t NumInline8bitCounters;
+ template <class Callback>
+ void IterateCounterRegions(Callback CB) {
+ for (size_t m = 0; m < NumModules; m++)
+ for (size_t r = 0; r < Modules[m].NumRegions; r++)
+ CB(Modules[m].Regions[r]);
+ }
+
+
struct PCTableEntry {
uintptr_t PC, PCFlags;
};
Set<uintptr_t> ObservedPCs;
std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter.
- std::pair<size_t, size_t> FocusFunction = {-1, -1}; // Module and PC IDs.
+ uint8_t *FocusFunctionCounterPtr = nullptr;
ValueBitMap ValueProfileMap;
uintptr_t InitialStack;
template <class Callback>
// void Callback(size_t FirstFeature, size_t Idx, uint8_t Value);
ATTRIBUTE_NO_SANITIZE_ALL
-void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End,
+size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End,
size_t FirstFeature, Callback Handle8bitCounter) {
typedef uintptr_t LargeType;
const size_t Step = sizeof(LargeType) / sizeof(uint8_t);
for (; P < End; P++)
if (uint8_t V = *P)
Handle8bitCounter(FirstFeature, P - Begin, V);
+ return End - Begin;
}
// Given a non-zero Counter returns a number in the range [0,7].
size_t FirstFeature = 0;
- if (NumInline8bitCounters) {
- for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) {
- ForEachNonZeroByte(ModuleCounters[i].Start, ModuleCounters[i].Stop,
- FirstFeature, Handle8bitCounter);
- FirstFeature += 8 * (ModuleCounters[i].Stop - ModuleCounters[i].Start);
+ for (size_t i = 0; i < NumModules; i++) {
+ for (size_t r = 0; r < Modules[i].NumRegions; r++) {
+ if (!Modules[i].Regions[r].Enabled) continue;
+ FirstFeature += 8 * ForEachNonZeroByte(Modules[i].Regions[r].Start,
+ Modules[i].Regions[r].Stop,
+ FirstFeature, Handle8bitCounter);
}
}
- ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), FirstFeature,
- Handle8bitCounter);
- FirstFeature += (ExtraCountersEnd() - ExtraCountersBegin()) * 8;
+ FirstFeature +=
+ 8 * ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(),
+ FirstFeature, Handle8bitCounter);
if (UseValueProfileMask) {
ValueProfileMap.ForEach([&](size_t Idx) {