From f407c9ed1057c4093cd09c0f8cee16f68c63157e Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Wed, 13 Apr 2022 14:02:19 +0200 Subject: [PATCH] [clangd] Export preamble AST and serialized size as metrics Differential Revision: https://reviews.llvm.org/D123672 --- clang-tools-extra/clangd/Preamble.cpp | 37 ++++++++++++++++++++++++++------ clang-tools-extra/clangd/Preamble.h | 8 +++++++ clang-tools-extra/clangd/TUScheduler.cpp | 8 +++++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp index b3a3aec..0cc5b65 100644 --- a/clang-tools-extra/clangd/Preamble.cpp +++ b/clang-tools-extra/clangd/Preamble.cpp @@ -64,8 +64,9 @@ bool compileCommandsAreEqual(const tooling::CompileCommand &LHS, class CppFilePreambleCallbacks : public PreambleCallbacks { public: - CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback) - : File(File), ParsedCallback(ParsedCallback) {} + CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback, + PreambleBuildStats *Stats) + : File(File), ParsedCallback(ParsedCallback), Stats(Stats) {} IncludeStructure takeIncludes() { return std::move(Includes); } @@ -88,6 +89,25 @@ public: IsMainFileIncludeGuarded = CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded( MainFE); + + if (Stats) { + const ASTContext &AST = CI.getASTContext(); + Stats->BuildSize = AST.getASTAllocatedMemory(); + Stats->BuildSize += AST.getSideTableAllocatedMemory(); + Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory(); + Stats->BuildSize += AST.Selectors.getTotalMemory(); + + Stats->BuildSize += AST.getSourceManager().getContentCacheSize(); + Stats->BuildSize += AST.getSourceManager().getDataStructureSizes(); + Stats->BuildSize += + AST.getSourceManager().getMemoryBufferSizes().malloc_bytes; + + const Preprocessor &PP = CI.getPreprocessor(); + Stats->BuildSize += PP.getTotalMemory(); + if (PreprocessingRecord *PRec = PP.getPreprocessingRecord()) + Stats->BuildSize += PRec->getTotalMemory(); + Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory(); + } } void BeforeExecute(CompilerInstance &CI) override { @@ -135,6 +155,7 @@ private: std::unique_ptr IWYUHandler = nullptr; const clang::LangOptions *LangOpts = nullptr; const SourceManager *SourceMgr = nullptr; + PreambleBuildStats *Stats; }; // Represents directives other than includes, where basic textual information is @@ -456,7 +477,7 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, // to read back. We rely on dynamic index for the comments instead. CI.getPreprocessorOpts().WriteCommentListToPCH = false; - CppFilePreambleCallbacks CapturedInfo(FileName, PreambleCallback); + CppFilePreambleCallbacks CapturedInfo(FileName, PreambleCallback, Stats); auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory); llvm::SmallString<32> AbsFileName(FileName); VFS->makeAbsolute(AbsFileName); @@ -476,6 +497,12 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, // bodies. CI.getFrontendOpts().SkipFunctionBodies = false; + if (Stats != nullptr) { + Stats->TotalBuildTime = PreambleTimer.getTime(); + Stats->FileSystemTime = TimedFS->getTime(); + Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0; + } + if (BuiltPreamble) { vlog("Built preamble of size {0} for file {1} version {2} in {3} seconds", BuiltPreamble->getSize(), FileName, Inputs.Version, @@ -491,10 +518,6 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes(); Result->StatCache = std::move(StatCache); Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded(); - if (Stats != nullptr) { - Stats->TotalBuildTime = PreambleTimer.getTime(); - Stats->FileSystemTime = TimedFS->getTime(); - } return Result; } diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h index 33a3084..f108dd4 100644 --- a/clang-tools-extra/clangd/Preamble.h +++ b/clang-tools-extra/clangd/Preamble.h @@ -83,6 +83,14 @@ struct PreambleBuildStats { double TotalBuildTime; /// Time spent in filesystem operations during the build, in seconds. double FileSystemTime; + + /// Estimate of the memory used while building the preamble. + /// This memory has been released when buildPreamble returns. + /// For example, this includes the size of the in-memory AST (ASTContext). + size_t BuildSize; + /// The serialized size of the preamble. + /// This storage is needed while the preamble is used (but may be on disk). + size_t SerializedSize; }; /// Build a preamble for the new inputs unless an old one can be reused. diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp index 928ab56..7c5a52f 100644 --- a/clang-tools-extra/clangd/TUScheduler.cpp +++ b/clang-tools-extra/clangd/TUScheduler.cpp @@ -110,6 +110,11 @@ constexpr trace::Metric constexpr trace::Metric PreambleBuildFilesystemLatencyRatio( "preamble_fs_latency_ratio", trace::Metric::Distribution, "build_type"); +constexpr trace::Metric PreambleBuildSize("preamble_build_size", + trace::Metric::Distribution); +constexpr trace::Metric PreambleSerializedSize("preamble_serialized_size", + trace::Metric::Distribution); + void reportPreambleBuild(const PreambleBuildStats &Stats, bool IsFirstPreamble) { auto RecordWithLabel = [&Stats](llvm::StringRef Label) { @@ -122,6 +127,9 @@ void reportPreambleBuild(const PreambleBuildStats &Stats, static llvm::once_flag OnceFlag; llvm::call_once(OnceFlag, [&] { RecordWithLabel("first_build"); }); RecordWithLabel(IsFirstPreamble ? "first_build_for_file" : "rebuild"); + + PreambleBuildSize.record(Stats.BuildSize); + PreambleSerializedSize.record(Stats.SerializedSize); } class ASTWorker; -- 2.7.4