From 006d1915e2defce50c05de5ba83727814e2ac11c Mon Sep 17 00:00:00 2001 From: Kadir Cetinkaya Date: Thu, 18 Jul 2019 17:20:41 +0000 Subject: [PATCH] [clangd] BackgroundIndex stores shards to the closest project Summary: Changes persistance logic to store shards at the directory of closest CDB. Previously we were storing all shards to directory of the CDB that triggered indexing, it had its downsides. For example, if you had two TUs coming from a different CDB but depending on the same header foo.h, we will store the foo.h only for the first CDB, and it would be missing for the second and we would never persist it since it was actually present in the memory and persisted before. This patch still stores only a single copy of a shard, but makes the directory a function of the file name. So that the shard place will be unique even with multiple CDBs accessing the file. This directory is determined as the first directory containing a CDB in the file's parent directories, if no such directory exists we make use of the home directory. Reviewers: sammccall Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D64745 llvm-svn: 366467 --- clang-tools-extra/clangd/ClangdServer.cpp | 3 +- clang-tools-extra/clangd/index/Background.cpp | 58 ++++++++++------------ clang-tools-extra/clangd/index/Background.h | 22 ++++---- .../clangd/index/BackgroundIndexLoader.cpp | 26 +++++----- .../clangd/index/BackgroundIndexStorage.cpp | 40 ++++++++++++--- .../Inputs/background-index/definition.jsonrpc | 2 +- .../clangd/test/Inputs/background-index/foo.cpp | 2 +- .../background-index/sub_dir/compile_flags.txt | 0 .../Inputs/background-index/{ => sub_dir}/foo.h | 0 .../clangd/test/background-index.test | 4 +- 10 files changed, 88 insertions(+), 69 deletions(-) create mode 100644 clang-tools-extra/clangd/test/Inputs/background-index/sub_dir/compile_flags.txt rename clang-tools-extra/clangd/test/Inputs/background-index/{ => sub_dir}/foo.h (100%) diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 10949ef..7e77e18 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -127,7 +127,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, if (Opts.BackgroundIndex) { BackgroundIdx = llvm::make_unique( Context::current().clone(), FSProvider, CDB, - BackgroundIndexStorage::createDiskBackedStorageFactory()); + BackgroundIndexStorage::createDiskBackedStorageFactory( + [&CDB](llvm::StringRef File) { return CDB.getProjectInfo(File); })); AddIndex(BackgroundIdx.get()); } if (DynamicIdx) diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp index 6e514cd..309804e 100644 --- a/clang-tools-extra/clangd/index/Background.cpp +++ b/clang-tools-extra/clangd/index/Background.cpp @@ -181,8 +181,8 @@ BackgroundQueue::Task BackgroundIndex::changedFilesTask( std::mt19937(std::random_device{}())); std::vector Tasks; Tasks.reserve(NeedsReIndexing.size()); - for (auto &Elem : NeedsReIndexing) - Tasks.push_back(indexFileTask(std::move(Elem.first), Elem.second)); + for (auto &Cmd : NeedsReIndexing) + Tasks.push_back(indexFileTask(std::move(Cmd))); Queue.append(std::move(Tasks)); }); @@ -197,13 +197,12 @@ static llvm::StringRef filenameWithoutExtension(llvm::StringRef Path) { } BackgroundQueue::Task -BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd, - BackgroundIndexStorage *Storage) { - BackgroundQueue::Task T([this, Storage, Cmd] { +BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd) { + BackgroundQueue::Task T([this, Cmd] { // We can't use llvm::StringRef here since we are going to // move from Cmd during the call below. const std::string FileName = Cmd.Filename; - if (auto Error = index(std::move(Cmd), Storage)) + if (auto Error = index(std::move(Cmd))) elog("Indexing {0} failed: {1}", FileName, std::move(Error)); }); T.QueuePri = IndexFile; @@ -226,7 +225,7 @@ void BackgroundIndex::boostRelated(llvm::StringRef Path) { void BackgroundIndex::update( llvm::StringRef MainFile, IndexFileIn Index, const llvm::StringMap &ShardVersionsSnapshot, - BackgroundIndexStorage *IndexStorage, bool HadErrors) { + bool HadErrors) { // Partition symbols/references into files. struct File { llvm::DenseSet Symbols; @@ -310,22 +309,21 @@ void BackgroundIndex::update( // We need to store shards before updating the index, since the latter // consumes slabs. // FIXME: Also skip serializing the shard if it is already up-to-date. - if (IndexStorage) { - IndexFileOut Shard; - Shard.Symbols = SS.get(); - Shard.Refs = RS.get(); - Shard.Relations = RelS.get(); - Shard.Sources = IG.get(); - - // Only store command line hash for main files of the TU, since our - // current model keeps only one version of a header file. - if (Path == MainFile) - Shard.Cmd = Index.Cmd.getPointer(); - - if (auto Error = IndexStorage->storeShard(Path, Shard)) - elog("Failed to write background-index shard for file {0}: {1}", Path, - std::move(Error)); - } + BackgroundIndexStorage *IndexStorage = IndexStorageFactory(Path); + IndexFileOut Shard; + Shard.Symbols = SS.get(); + Shard.Refs = RS.get(); + Shard.Relations = RelS.get(); + Shard.Sources = IG.get(); + + // Only store command line hash for main files of the TU, since our + // current model keeps only one version of a header file. + if (Path == MainFile) + Shard.Cmd = Index.Cmd.getPointer(); + + if (auto Error = IndexStorage->storeShard(Path, Shard)) + elog("Failed to write background-index shard for file {0}: {1}", Path, + std::move(Error)); { std::lock_guard Lock(ShardVersionsMu); @@ -348,8 +346,7 @@ void BackgroundIndex::update( } } -llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd, - BackgroundIndexStorage *IndexStorage) { +llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) { trace::Span Tracer("BackgroundIndex"); SPAN_ATTACH(Tracer, "file", Cmd.Filename); auto AbsolutePath = getAbsolutePath(Cmd); @@ -443,8 +440,7 @@ llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd, for (auto &It : *Index.Sources) It.second.Flags |= IncludeGraphNode::SourceFlag::HadErrors; } - update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, IndexStorage, - HadErrors); + update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, HadErrors); Rebuilder.indexedTU(); return llvm::Error::success(); @@ -453,10 +449,9 @@ llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd, // Restores shards for \p MainFiles from index storage. Then checks staleness of // those shards and returns a list of TUs that needs to be indexed to update // staleness. -std::vector> +std::vector BackgroundIndex::loadProject(std::vector MainFiles) { - std::vector> - NeedsReIndexing; + std::vector NeedsReIndexing; Rebuilder.startLoading(); // Load shards for all of the mainfiles. @@ -517,8 +512,7 @@ BackgroundIndex::loadProject(std::vector MainFiles) { std::string ProjectRoot; if (auto PI = CDB.getProjectInfo(TU)) ProjectRoot = std::move(PI->SourceRoot); - BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot); - NeedsReIndexing.emplace_back(std::move(*Cmd), Storage); + NeedsReIndexing.emplace_back(std::move(*Cmd)); } return NeedsReIndexing; diff --git a/clang-tools-extra/clangd/index/Background.h b/clang-tools-extra/clangd/index/Background.h index a703b9b..7ce9e7b 100644 --- a/clang-tools-extra/clangd/index/Background.h +++ b/clang-tools-extra/clangd/index/Background.h @@ -50,15 +50,17 @@ public: virtual std::unique_ptr loadShard(llvm::StringRef ShardIdentifier) const = 0; - // The factory provides storage for each CDB. + // The factory provides storage for each File. // It keeps ownership of the storage instances, and should manage caching // itself. Factory must be threadsafe and never returns nullptr. - using Factory = - llvm::unique_function; + using Factory = llvm::unique_function; // Creates an Index Storage that saves shards into disk. Index storage uses - // CDBDirectory + ".clangd/index/" as the folder to save shards. - static Factory createDiskBackedStorageFactory(); + // CDBDirectory + ".clangd/index/" as the folder to save shards. CDBDirectory + // is the first directory containing a CDB in parent directories of a file, or + // user's home directory if none was found, e.g. standard library headers. + static Factory createDiskBackedStorageFactory( + std::function(PathRef)> GetProjectInfo); }; // A priority queue of tasks which can be run on (external) worker threads. @@ -158,15 +160,14 @@ private: /// information on IndexStorage. void update(llvm::StringRef MainFile, IndexFileIn Index, const llvm::StringMap &ShardVersionsSnapshot, - BackgroundIndexStorage *IndexStorage, bool HadErrors); + bool HadErrors); // configuration const FileSystemProvider &FSProvider; const GlobalCompilationDatabase &CDB; Context BackgroundContext; - llvm::Error index(tooling::CompileCommand, - BackgroundIndexStorage *IndexStorage); + llvm::Error index(tooling::CompileCommand); FileSymbols IndexedSymbols; BackgroundIndexRebuilder Rebuilder; @@ -175,13 +176,12 @@ private: BackgroundIndexStorage::Factory IndexStorageFactory; // Tries to load shards for the MainFiles and their dependencies. - std::vector> + std::vector loadProject(std::vector MainFiles); BackgroundQueue::Task changedFilesTask(const std::vector &ChangedFiles); - BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd, - BackgroundIndexStorage *Storage); + BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd); // from lowest to highest priority enum QueuePriority { diff --git a/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp b/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp index b5b33a2..128b0c5 100644 --- a/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp +++ b/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp @@ -38,8 +38,10 @@ llvm::Optional uriToAbsolutePath(llvm::StringRef URI, PathRef HintPath) { /// inverse dependency mapping. class BackgroundIndexLoader { public: + BackgroundIndexLoader(BackgroundIndexStorage::Factory &IndexStorageFactory) + : IndexStorageFactory(IndexStorageFactory) {} /// Load the shards for \p MainFile and all of its dependencies. - void load(PathRef MainFile, BackgroundIndexStorage *Storage); + void load(PathRef MainFile); /// Consumes the loader and returns all shards. std::vector takeResult() &&; @@ -49,18 +51,19 @@ private: /// Storage. Also returns paths for dependencies of \p StartSourceFile if it /// wasn't cached yet. std::pair> - loadShard(PathRef StartSourceFile, BackgroundIndexStorage *Storage); + loadShard(PathRef StartSourceFile); /// Cache for Storage lookups. llvm::StringMap LoadedShards; /// References are into the AbsolutePaths in LoadedShards. llvm::DenseMap FileToTU; + + BackgroundIndexStorage::Factory &IndexStorageFactory; }; std::pair> -BackgroundIndexLoader::loadShard(PathRef StartSourceFile, - BackgroundIndexStorage *Storage) { +BackgroundIndexLoader::loadShard(PathRef StartSourceFile) { auto It = LoadedShards.try_emplace(StartSourceFile); LoadedShard &LS = It.first->getValue(); std::vector Edges = {}; @@ -69,6 +72,7 @@ BackgroundIndexLoader::loadShard(PathRef StartSourceFile, return {LS, Edges}; LS.AbsolutePath = StartSourceFile.str(); + BackgroundIndexStorage *Storage = IndexStorageFactory(LS.AbsolutePath); auto Shard = Storage->loadShard(StartSourceFile); if (!Shard || !Shard->Sources) { vlog("Failed to load shard: {0}", StartSourceFile); @@ -96,8 +100,7 @@ BackgroundIndexLoader::loadShard(PathRef StartSourceFile, return {LS, Edges}; } -void BackgroundIndexLoader::load(PathRef MainFile, - BackgroundIndexStorage *Storage) { +void BackgroundIndexLoader::load(PathRef MainFile) { llvm::StringSet<> InQueue; // Following containers points to strings inside InQueue. std::queue ToVisit; @@ -108,7 +111,7 @@ void BackgroundIndexLoader::load(PathRef MainFile, PathRef SourceFile = ToVisit.front(); ToVisit.pop(); - auto ShardAndEdges = loadShard(SourceFile, Storage); + auto ShardAndEdges = loadShard(SourceFile); FileToTU[ShardAndEdges.first.AbsolutePath] = MainFile; for (PathRef Edge : ShardAndEdges.second) { auto It = InQueue.insert(Edge); @@ -136,15 +139,10 @@ std::vector loadIndexShards(llvm::ArrayRef MainFiles, BackgroundIndexStorage::Factory &IndexStorageFactory, const GlobalCompilationDatabase &CDB) { - BackgroundIndexLoader Loader; + BackgroundIndexLoader Loader(IndexStorageFactory); for (llvm::StringRef MainFile : MainFiles) { assert(llvm::sys::path::is_absolute(MainFile)); - - std::string ProjectRoot; - if (auto PI = CDB.getProjectInfo(MainFile)) - ProjectRoot = std::move(PI->SourceRoot); - BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot); - Loader.load(MainFile, Storage); + Loader.load(MainFile); } return std::move(Loader).takeResult(); } diff --git a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp index 80246b9..8b2c411 100644 --- a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp +++ b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp @@ -6,13 +6,21 @@ // //===----------------------------------------------------------------------===// +#include "GlobalCompilationDatabase.h" #include "Logger.h" +#include "Path.h" #include "index/Background.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" +#include namespace clang { namespace clangd { @@ -118,12 +126,21 @@ public: // Creates and owns IndexStorages for multiple CDBs. class DiskBackedIndexStorageManager { public: - DiskBackedIndexStorageManager() - : IndexStorageMapMu(llvm::make_unique()) {} + DiskBackedIndexStorageManager( + std::function(PathRef)> GetProjectInfo) + : IndexStorageMapMu(llvm::make_unique()), + GetProjectInfo(std::move(GetProjectInfo)) { + llvm::SmallString<128> HomeDir; + llvm::sys::path::home_directory(HomeDir); + this->HomeDir = HomeDir.str().str(); + } - // Creates or fetches to storage from cache for the specified CDB. - BackgroundIndexStorage *operator()(llvm::StringRef CDBDirectory) { + // Creates or fetches to storage from cache for the specified project. + BackgroundIndexStorage *operator()(PathRef File) { std::lock_guard Lock(*IndexStorageMapMu); + Path CDBDirectory = HomeDir; + if (auto PI = GetProjectInfo(File)) + CDBDirectory = PI->SourceRoot; auto &IndexStorage = IndexStorageMap[CDBDirectory]; if (!IndexStorage) IndexStorage = create(CDBDirectory); @@ -131,21 +148,28 @@ public: } private: - std::unique_ptr create(llvm::StringRef CDBDirectory) { - if (CDBDirectory.empty()) + std::unique_ptr create(PathRef CDBDirectory) { + if (CDBDirectory.empty()) { + elog("Tried to create storage for empty directory!"); return llvm::make_unique(); + } return llvm::make_unique(CDBDirectory); } + Path HomeDir; + llvm::StringMap> IndexStorageMap; std::unique_ptr IndexStorageMapMu; + + std::function(PathRef)> GetProjectInfo; }; } // namespace BackgroundIndexStorage::Factory -BackgroundIndexStorage::createDiskBackedStorageFactory() { - return DiskBackedIndexStorageManager(); +BackgroundIndexStorage::createDiskBackedStorageFactory( + std::function(PathRef)> GetProjectInfo) { + return DiskBackedIndexStorageManager(std::move(GetProjectInfo)); } } // namespace clangd diff --git a/clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc b/clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc index 933e779..de99072 100644 --- a/clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc +++ b/clang-tools-extra/clangd/test/Inputs/background-index/definition.jsonrpc @@ -18,7 +18,7 @@ "uri": "file://DIRECTORY/bar.cpp", "languageId": "cpp", "version": 1, - "text": "#include \"foo.h\"\nint main(){\nreturn foo();\n}" + "text": "#include \"sub_dir/foo.h\"\nint main(){\nreturn foo();\n}" } } } diff --git a/clang-tools-extra/clangd/test/Inputs/background-index/foo.cpp b/clang-tools-extra/clangd/test/Inputs/background-index/foo.cpp index c42ca4d..21fc589 100644 --- a/clang-tools-extra/clangd/test/Inputs/background-index/foo.cpp +++ b/clang-tools-extra/clangd/test/Inputs/background-index/foo.cpp @@ -1,2 +1,2 @@ -#include "foo.h" +#include "sub_dir/foo.h" int foo() { return 42; } diff --git a/clang-tools-extra/clangd/test/Inputs/background-index/sub_dir/compile_flags.txt b/clang-tools-extra/clangd/test/Inputs/background-index/sub_dir/compile_flags.txt new file mode 100644 index 0000000..e69de29 diff --git a/clang-tools-extra/clangd/test/Inputs/background-index/foo.h b/clang-tools-extra/clangd/test/Inputs/background-index/sub_dir/foo.h similarity index 100% rename from clang-tools-extra/clangd/test/Inputs/background-index/foo.h rename to clang-tools-extra/clangd/test/Inputs/background-index/sub_dir/foo.h diff --git a/clang-tools-extra/clangd/test/background-index.test b/clang-tools-extra/clangd/test/background-index.test index eabca2b..57118f2 100644 --- a/clang-tools-extra/clangd/test/background-index.test +++ b/clang-tools-extra/clangd/test/background-index.test @@ -5,7 +5,8 @@ # RUN: rm -rf %t # RUN: cp -r %S/Inputs/background-index %t # Need to embed the correct temp path in the actual JSON-RPC requests. -# RUN: sed -i -e "s|DIRECTORY|%t|" %t/* +# RUN: sed -i -e "s|DIRECTORY|%t|" %t/definition.jsonrpc +# RUN: sed -i -e "s|DIRECTORY|%t|" %t/compile_commands.json # We're editing bar.cpp, which includes foo.h. # foo() is declared in foo.h and defined in foo.cpp. @@ -14,6 +15,7 @@ # Test that the index is writing files in the expected location. # RUN: ls %t/.clangd/index/foo.cpp.*.idx +# RUN: ls %t/sub_dir/.clangd/index/foo.h.*.idx # Test the index is read from disk: delete code and restart clangd. # RUN: rm %t/foo.cpp -- 2.7.4