From e9eb7f0cb815641827fa3d031c38190c10900c14 Mon Sep 17 00:00:00 2001 From: Ilya Biryukov Date: Thu, 16 Nov 2017 16:25:18 +0000 Subject: [PATCH] [clangd] Use in-memory preambles in clangd. Reviewers: klimek, bkramer, sammccall Reviewed By: sammccall Subscribers: cfe-commits Differential Revision: https://reviews.llvm.org/D39843 llvm-svn: 318412 --- clang-tools-extra/clangd/ClangdLSPServer.cpp | 2 ++ clang-tools-extra/clangd/ClangdLSPServer.h | 2 +- clang-tools-extra/clangd/ClangdServer.cpp | 21 ++++++++-------- clang-tools-extra/clangd/ClangdServer.h | 5 ++++ clang-tools-extra/clangd/ClangdUnit.cpp | 13 ++++++---- clang-tools-extra/clangd/ClangdUnit.h | 3 +++ clang-tools-extra/clangd/ClangdUnitStore.cpp | 7 ++++-- clang-tools-extra/clangd/ClangdUnitStore.h | 12 ++++++---- clang-tools-extra/clangd/tool/ClangdMain.cpp | 28 ++++++++++++++++++++-- clang-tools-extra/test/clangd/completion.test | 1 + .../test/clangd/diagnostics-preamble.test | 1 + clang-tools-extra/unittests/clangd/ClangdTests.cpp | 27 ++++++++++++++++----- 12 files changed, 92 insertions(+), 30 deletions(-) diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp index 4eac624..80352ee 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -235,11 +235,13 @@ void ClangdLSPServer::onSwitchSourceHeader(Ctx C, } ClangdLSPServer::ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount, + bool StorePreamblesInMemory, bool SnippetCompletions, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir) : Out(Out), CDB(/*Logger=*/Out, std::move(CompileCommandsDir)), Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount, + StorePreamblesInMemory, clangd::CodeCompleteOptions( /*EnableSnippetsAndCodePatterns=*/SnippetCompletions), /*Logger=*/Out, ResourceDir) {} diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h index c2e494d..1c2ede9 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.h +++ b/clang-tools-extra/clangd/ClangdLSPServer.h @@ -31,7 +31,7 @@ public: /// loaded only from \p CompileCommandsDir. Otherwise, clangd will look /// for compile_commands.json in all parent directories of each file. ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount, - bool SnippetCompletions, + bool StorePreamblesInMemory, bool SnippetCompletions, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir); diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 18fd2b7..dfe94b3 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -169,17 +169,16 @@ ClangdScheduler::~ClangdScheduler() { Worker.join(); } -ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB, - DiagnosticsConsumer &DiagConsumer, - FileSystemProvider &FSProvider, - unsigned AsyncThreadsCount, - clangd::CodeCompleteOptions CodeCompleteOpts, - clangd::Logger &Logger, - llvm::Optional ResourceDir) +ClangdServer::ClangdServer( + GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, + FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, + bool StorePreamblesInMemory, clangd::CodeCompleteOptions CodeCompleteOpts, + clangd::Logger &Logger, llvm::Optional ResourceDir) : Logger(Logger), CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider), ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()), PCHs(std::make_shared()), + StorePreamblesInMemory(StorePreamblesInMemory), CodeCompleteOpts(CodeCompleteOpts), WorkScheduler(AsyncThreadsCount) {} void ClangdServer::setRootPath(PathRef RootPath) { @@ -193,8 +192,8 @@ std::future ClangdServer::addDocument(PathRef File, StringRef Contents) { DocVersion Version = DraftMgr.updateDraft(File, Contents); auto TaggedFS = FSProvider.getTaggedFileSystem(File); - std::shared_ptr Resources = - Units.getOrCreateFile(File, ResourceDir, CDB, PCHs, Logger); + std::shared_ptr Resources = Units.getOrCreateFile( + File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger); return scheduleReparseAndDiags(File, VersionedDraft{Version, Contents.str()}, std::move(Resources), std::move(TaggedFS)); } @@ -211,8 +210,8 @@ std::future ClangdServer::forceReparse(PathRef File) { "forceReparse() was called for non-added document"); auto TaggedFS = FSProvider.getTaggedFileSystem(File); - auto Recreated = Units.recreateFileIfCompileCommandChanged(File, ResourceDir, - CDB, PCHs, Logger); + auto Recreated = Units.recreateFileIfCompileCommandChanged( + File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger); // Note that std::future from this cleanup action is ignored. scheduleCancelRebuild(std::move(Recreated.RemovedFile)); diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h index 247fc64..bd95d00 100644 --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -205,10 +205,14 @@ public: /// worker thread. Therefore, instances of \p DiagConsumer must properly /// synchronize access to shared state. /// + /// \p StorePreamblesInMemory defines whether the Preambles generated by + /// clangd are stored in-memory or on disk. + /// /// Various messages are logged using \p Logger. ClangdServer(GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, + bool StorePreamblesInMemory, clangd::CodeCompleteOptions CodeCompleteOpts, clangd::Logger &Logger, llvm::Optional ResourceDir = llvm::None); @@ -325,6 +329,7 @@ private: // If set, this represents the workspace path. llvm::Optional RootPath; std::shared_ptr PCHs; + bool StorePreamblesInMemory; clangd::CodeCompleteOptions CodeCompleteOpts; /// Used to serialize diagnostic callbacks. /// FIXME(ibiryukov): get rid of an extra map and put all version counters diff --git a/clang-tools-extra/clangd/ClangdUnit.cpp b/clang-tools-extra/clangd/ClangdUnit.cpp index 2fbc7a2..e88680f 100644 --- a/clang-tools-extra/clangd/ClangdUnit.cpp +++ b/clang-tools-extra/clangd/ClangdUnit.cpp @@ -235,7 +235,7 @@ prepareCompilerInstance(std::unique_ptr CI, // NOTE: we use Buffer.get() when adding remapped files, so we have to make // sure it will be released if no error is emitted. if (Preamble) { - Preamble->AddImplicitPreamble(*CI, Buffer.get()); + Preamble->AddImplicitPreamble(*CI, VFS, Buffer.get()); } else { CI->getPreprocessorOpts().addRemappedFile( CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get()); @@ -1137,16 +1137,20 @@ PreambleData::PreambleData(PrecompiledPreamble Preamble, std::shared_ptr CppFile::Create(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger) { - return std::shared_ptr( - new CppFile(FileName, std::move(Command), std::move(PCHs), Logger)); + return std::shared_ptr(new CppFile(FileName, std::move(Command), + StorePreamblesInMemory, + std::move(PCHs), Logger)); } CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger) - : FileName(FileName), Command(std::move(Command)), RebuildCounter(0), + : FileName(FileName), Command(std::move(Command)), + StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0), RebuildInProgress(false), PCHs(std::move(PCHs)), Logger(Logger) { std::lock_guard Lock(Mutex); @@ -1290,6 +1294,7 @@ CppFile::deferRebuild(StringRef NewContents, CppFilePreambleCallbacks SerializedDeclsCollector; auto BuiltPreamble = PrecompiledPreamble::Build( *CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs, + /*StoreInMemory=*/That->StorePreamblesInMemory, SerializedDeclsCollector); if (BuiltPreamble) { diff --git a/clang-tools-extra/clangd/ClangdUnit.h b/clang-tools-extra/clangd/ClangdUnit.h index 9ce3bbc..6babf55 100644 --- a/clang-tools-extra/clangd/ClangdUnit.h +++ b/clang-tools-extra/clangd/ClangdUnit.h @@ -143,10 +143,12 @@ public: // deferRebuild will hold references to it. static std::shared_ptr Create(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger); private: CppFile(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger); public: @@ -222,6 +224,7 @@ private: Path FileName; tooling::CompileCommand Command; + bool StorePreamblesInMemory; /// Mutex protects all fields, declared below it, FileName and Command are not /// mutated. diff --git a/clang-tools-extra/clangd/ClangdUnitStore.cpp b/clang-tools-extra/clangd/ClangdUnitStore.cpp index 0c905c8..8814d7a 100644 --- a/clang-tools-extra/clangd/ClangdUnitStore.cpp +++ b/clang-tools-extra/clangd/ClangdUnitStore.cpp @@ -29,7 +29,8 @@ std::shared_ptr CppFileCollection::removeIfPresent(PathRef File) { CppFileCollection::RecreateResult CppFileCollection::recreateFileIfCompileCommandChanged( PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, - std::shared_ptr PCHs, clangd::Logger &Logger) { + bool StorePreamblesInMemory, std::shared_ptr PCHs, + clangd::Logger &Logger) { auto NewCommand = getCompileCommand(CDB, File, ResourceDir); std::lock_guard Lock(Mutex); @@ -40,13 +41,15 @@ CppFileCollection::recreateFileIfCompileCommandChanged( if (It == OpenedFiles.end()) { It = OpenedFiles .try_emplace(File, CppFile::Create(File, std::move(NewCommand), + StorePreamblesInMemory, std::move(PCHs), Logger)) .first; } else if (!compileCommandsAreEqual(It->second->getCompileCommand(), NewCommand)) { Result.RemovedFile = std::move(It->second); It->second = - CppFile::Create(File, std::move(NewCommand), std::move(PCHs), Logger); + CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory, + std::move(PCHs), Logger); } Result.FileInCollection = It->second; return Result; diff --git a/clang-tools-extra/clangd/ClangdUnitStore.h b/clang-tools-extra/clangd/ClangdUnitStore.h index cf7f1fb..f7901f8 100644 --- a/clang-tools-extra/clangd/ClangdUnitStore.h +++ b/clang-tools-extra/clangd/ClangdUnitStore.h @@ -25,9 +25,11 @@ class Logger; /// Thread-safe mapping from FileNames to CppFile. class CppFileCollection { public: - std::shared_ptr getOrCreateFile( - PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, - std::shared_ptr PCHs, clangd::Logger &Logger) { + std::shared_ptr + getOrCreateFile(PathRef File, PathRef ResourceDir, + GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory, + std::shared_ptr PCHs, + clangd::Logger &Logger) { std::lock_guard Lock(Mutex); auto It = OpenedFiles.find(File); @@ -36,6 +38,7 @@ public: It = OpenedFiles .try_emplace(File, CppFile::Create(File, std::move(Command), + StorePreamblesInMemory, std::move(PCHs), Logger)) .first; } @@ -58,7 +61,8 @@ public: /// will be returned in RecreateResult.RemovedFile. RecreateResult recreateFileIfCompileCommandChanged( PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, - std::shared_ptr PCHs, clangd::Logger &Logger); + bool StorePreamblesInMemory, std::shared_ptr PCHs, + clangd::Logger &Logger); std::shared_ptr getFile(PathRef File) { std::lock_guard Lock(Mutex); diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp index dc8969a..81512c2 100644 --- a/clang-tools-extra/clangd/tool/ClangdMain.cpp +++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp @@ -24,6 +24,10 @@ using namespace clang; using namespace clang::clangd; +namespace { +enum class PCHStorageFlag { Disk, Memory }; +} + static llvm::cl::opt CompileCommandsDir( "compile-commands-dir", llvm::cl::desc("Specify a path to look for compile_commands.json. If path " @@ -45,6 +49,15 @@ static llvm::cl::opt PrettyPrint("pretty", llvm::cl::desc("Pretty-print JSON output"), llvm::cl::init(false)); +static llvm::cl::opt PCHStorage( + "pch-storage", + llvm::cl::desc("Storing PCHs in memory increases memory usages, but may " + "improve performance"), + llvm::cl::values( + clEnumValN(PCHStorageFlag::Disk, "disk", "store PCHs on disk"), + clEnumValN(PCHStorageFlag::Memory, "memory", "store PCHs in memory")), + llvm::cl::init(PCHStorageFlag::Disk)); + static llvm::cl::opt RunSynchronously( "run-synchronously", llvm::cl::desc("Parse on main thread. If set, -j is ignored"), @@ -127,6 +140,16 @@ int main(int argc, char *argv[]) { CompileCommandsDirPath = CompileCommandsDir; } + bool StorePreamblesInMemory; + switch (PCHStorage) { + case PCHStorageFlag::Memory: + StorePreamblesInMemory = true; + break; + case PCHStorageFlag::Disk: + StorePreamblesInMemory = false; + break; + } + llvm::Optional ResourceDirRef = None; if (!ResourceDir.empty()) ResourceDirRef = ResourceDir; @@ -135,8 +158,9 @@ int main(int argc, char *argv[]) { llvm::sys::ChangeStdinToBinary(); // Initialize and run ClangdLSPServer. - ClangdLSPServer LSPServer(Out, WorkerThreadsCount, EnableSnippets, - ResourceDirRef, CompileCommandsDirPath); + ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory, + EnableSnippets, ResourceDirRef, + CompileCommandsDirPath); constexpr int NoShutdownRequestErrorCode = 1; llvm::set_thread_name("clangd.main"); return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode; diff --git a/clang-tools-extra/test/clangd/completion.test b/clang-tools-extra/test/clangd/completion.test index 40d1b52..e4f818f 100644 --- a/clang-tools-extra/test/clangd/completion.test +++ b/clang-tools-extra/test/clangd/completion.test @@ -1,4 +1,5 @@ # RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s +# RUN: clangd -pretty -run-synchronously -pch-storage=memory < %s | FileCheck -strict-whitespace %s # It is absolutely vital that this file has CRLF line endings. # Content-Length: 125 diff --git a/clang-tools-extra/test/clangd/diagnostics-preamble.test b/clang-tools-extra/test/clangd/diagnostics-preamble.test index befb809..6ba64a7 100644 --- a/clang-tools-extra/test/clangd/diagnostics-preamble.test +++ b/clang-tools-extra/test/clangd/diagnostics-preamble.test @@ -1,4 +1,5 @@ # RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s +# RUN: clangd -pretty -run-synchronously -pch-storage=memory < %s | FileCheck -strict-whitespace %s # It is absolutely vital that this file has CRLF line endings. # Content-Length: 125 diff --git a/clang-tools-extra/unittests/clangd/ClangdTests.cpp b/clang-tools-extra/unittests/clangd/ClangdTests.cpp index aac5fab..570e7d1 100644 --- a/clang-tools-extra/unittests/clangd/ClangdTests.cpp +++ b/clang-tools-extra/unittests/clangd/ClangdTests.cpp @@ -332,6 +332,7 @@ protected: ErrorCheckingDiagConsumer DiagConsumer; MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); for (const auto &FileWithContents : ExtraFiles) @@ -396,6 +397,7 @@ TEST_F(ClangdVFSTest, Reparse) { ErrorCheckingDiagConsumer DiagConsumer; MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -442,6 +444,7 @@ TEST_F(ClangdVFSTest, ReparseOnHeaderChange) { MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -490,7 +493,9 @@ TEST_F(ClangdVFSTest, CheckVersions) { MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); // Run ClangdServer synchronously. ClangdServer Server(CDB, DiagConsumer, FS, - /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(), + /*AsyncThreadsCount=*/0, + /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); auto FooCpp = getVirtualTestFilePath("foo.cpp"); @@ -524,7 +529,9 @@ TEST_F(ClangdVFSTest, SearchLibDir) { "-stdlib=libstdc++"}); // Run ClangdServer synchronously. ClangdServer Server(CDB, DiagConsumer, FS, - /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(), + /*AsyncThreadsCount=*/0, + /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); // Just a random gcc version string @@ -573,7 +580,9 @@ TEST_F(ClangdVFSTest, ForceReparseCompileCommand) { ErrorCheckingDiagConsumer DiagConsumer; MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, - /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(), + /*AsyncThreadsCount=*/0, + /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); // No need to sync reparses, because reparses are performed on the calling // thread to true. @@ -641,6 +650,7 @@ TEST_F(ClangdCompletionTest, CheckContentsOverride) { MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -701,7 +711,8 @@ TEST_F(ClangdCompletionTest, Limit) { clangd::CodeCompleteOptions Opts; Opts.Limit = 2; ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), - Opts, EmptyLogger::getInstance()); + /*StorePreamblesInMemory=*/true, Opts, + EmptyLogger::getInstance()); auto FooCpp = getVirtualTestFilePath("foo.cpp"); FS.Files[FooCpp] = ""; @@ -796,7 +807,8 @@ int test() { auto TestWithOpts = [&](clangd::CodeCompleteOptions Opts) { ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), - Opts, EmptyLogger::getInstance()); + /*StorePreamblesInMemory=*/true, Opts, + EmptyLogger::getInstance()); // No need to sync reparses here as there are no asserts on diagnostics (or // other async operations). Server.addDocument(FooCpp, GlobalCompletion.Text); @@ -989,6 +1001,7 @@ int d; { MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -1149,6 +1162,7 @@ TEST_F(ClangdVFSTest, CheckSourceHeaderSwitch) { MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -1275,7 +1289,8 @@ int d; std::move(StartSecondReparsePromise)); MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); - ClangdServer Server(CDB, DiagConsumer, FS, 4, clangd::CodeCompleteOptions(), + ClangdServer Server(CDB, DiagConsumer, FS, 4, /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); Server.addDocument(FooCpp, SourceContentsWithErrors); StartSecondReparse.wait(); -- 2.7.4