//===----------------------------------------------------------------------===//
#include "GlobalCompilationDatabase.h"
+#include "Config.h"
#include "FS.h"
#include "SourceCode.h"
#include "support/Logger.h"
#include "clang/Tooling/JSONCompilationDatabase.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallString.h"
DirectoryBasedGlobalCompilationDatabase::
DirectoryBasedGlobalCompilationDatabase(const Options &Opts)
: Opts(Opts), Broadcaster(std::make_unique<BroadcastThread>(*this)) {
- if (Opts.CompileCommandsDir)
- OnlyDirCache = std::make_unique<DirectoryCache>(*Opts.CompileCommandsDir);
+ if (!this->Opts.ContextProvider)
+ this->Opts.ContextProvider = [](llvm::StringRef) {
+ return Context::current().clone();
+ };
}
DirectoryBasedGlobalCompilationDatabase::
#endif
}
-static bool pathEqual(PathRef A, PathRef B) {
-#if defined(_WIN32) || defined(__APPLE__)
- return A.equals_lower(B);
-#else
- return A == B;
-#endif
-}
-
std::vector<DirectoryBasedGlobalCompilationDatabase::DirectoryCache *>
DirectoryBasedGlobalCompilationDatabase::getDirectoryCaches(
llvm::ArrayRef<llvm::StringRef> Dirs) const {
assert(llvm::sys::path::is_absolute(Request.FileName) &&
"path must be absolute");
+ std::string Storage;
+ std::vector<llvm::StringRef> SearchDirs;
+ if (Opts.CompileCommandsDir) // FIXME: unify this case with config.
+ SearchDirs = {Opts.CompileCommandsDir.getValue()};
+ else {
+ WithContext WithProvidedContext(Opts.ContextProvider(Request.FileName));
+ const auto &Spec = Config::current().CompileFlags.CDBSearch;
+ switch (Spec.Policy) {
+ case Config::CDBSearchSpec::NoCDBSearch:
+ return llvm::None;
+ case Config::CDBSearchSpec::FixedDir:
+ Storage = Spec.FixedCDBPath.getValue();
+ SearchDirs = {Storage};
+ break;
+ case Config::CDBSearchSpec::Ancestors:
+ // Traverse the canonical version to prevent false positives. i.e.:
+ // src/build/../a.cc can detect a CDB in /src/build if not
+ // canonicalized.
+ Storage = removeDots(Request.FileName);
+ actOnAllParentDirectories(Storage, [&](llvm::StringRef Dir) {
+ SearchDirs.push_back(Dir);
+ return false;
+ });
+ }
+ }
+
+ std::shared_ptr<const tooling::CompilationDatabase> CDB = nullptr;
bool ShouldBroadcast = false;
DirectoryCache *DirCache = nullptr;
- std::shared_ptr<const tooling::CompilationDatabase> CDB = nullptr;
- if (OnlyDirCache) {
- DirCache = OnlyDirCache.get();
- ShouldBroadcast = Request.ShouldBroadcast;
- CDB = DirCache->get(Opts.TFS, ShouldBroadcast, Request.FreshTime,
- Request.FreshTimeMissing);
- } else {
- // Traverse the canonical version to prevent false positives. i.e.:
- // src/build/../a.cc can detect a CDB in /src/build if not canonicalized.
- std::string CanonicalPath = removeDots(Request.FileName);
- std::vector<llvm::StringRef> SearchDirs;
- actOnAllParentDirectories(CanonicalPath, [&](PathRef Path) {
- SearchDirs.push_back(Path);
- return false;
- });
- for (DirectoryCache *Candidate : getDirectoryCaches(SearchDirs)) {
- bool CandidateShouldBroadcast = Request.ShouldBroadcast;
- if ((CDB = Candidate->get(Opts.TFS, CandidateShouldBroadcast,
- Request.FreshTime, Request.FreshTimeMissing))) {
- DirCache = Candidate;
- ShouldBroadcast = CandidateShouldBroadcast;
- break;
- }
+ for (DirectoryCache *Candidate : getDirectoryCaches(SearchDirs)) {
+ bool CandidateShouldBroadcast = Request.ShouldBroadcast;
+ if ((CDB = Candidate->get(Opts.TFS, CandidateShouldBroadcast,
+ Request.FreshTime, Request.FreshTimeMissing))) {
+ DirCache = Candidate;
+ ShouldBroadcast = CandidateShouldBroadcast;
+ break;
}
}
}
};
-void DirectoryBasedGlobalCompilationDatabase::BroadcastThread::process(
- const CDBLookupResult &T) {
- vlog("Broadcasting compilation database from {0}", T.PI.SourceRoot);
+// The DirBasedCDB associates each file with a specific CDB.
+// When a CDB is discovered, it may claim to describe files that we associate
+// with a different CDB. We do not want to broadcast discovery of these, and
+// trigger background indexing of them.
+//
+// We must filter the list, and check whether they are associated with this CDB.
+// This class attempts to do so efficiently.
+//
+// Roughly, it:
+// - loads the config for each file, and determines the relevant search path
+// - gathers all directories that are part of any search path
+// - (lazily) checks for a CDB in each such directory at most once
+// - walks the search path for each file and determines whether to include it.
+class DirectoryBasedGlobalCompilationDatabase::BroadcastThread::Filter {
+ llvm::StringRef ThisDir;
+ DirectoryBasedGlobalCompilationDatabase &Parent;
- std::vector<std::string> AllFiles = T.CDB->getAllFiles();
- // We assume CDB in CompileCommandsDir owns all of its entries, since we don't
- // perform any search in parent paths whenever it is set.
- if (Parent.OnlyDirCache) {
- assert(Parent.OnlyDirCache->Path == T.PI.SourceRoot &&
- "Trying to broadcast a CDB outside of CompileCommandsDir!");
- Parent.OnCommandChanged.broadcast(std::move(AllFiles));
- return;
+ // Keep track of all directories we might check for CDBs.
+ struct DirInfo {
+ DirectoryCache *Cache = nullptr;
+ enum { Unknown, Missing, TargetCDB, OtherCDB } State = Unknown;
+ DirInfo *Parent = nullptr;
+ };
+ llvm::StringMap<DirInfo> Dirs;
+
+ // A search path starts at a directory, and either includes ancestors or not.
+ using SearchPath = llvm::PointerIntPair<DirInfo *, 1>;
+
+ // Add all ancestor directories of FilePath to the tracked set.
+ // Returns the immediate parent of the file.
+ DirInfo *addParents(llvm::StringRef FilePath) {
+ DirInfo *Leaf = nullptr;
+ DirInfo *Child = nullptr;
+ actOnAllParentDirectories(FilePath, [&](llvm::StringRef Dir) {
+ auto &Info = Dirs[Dir];
+ // If this is the first iteration, then this node is the overall result.
+ if (!Leaf)
+ Leaf = &Info;
+ // Fill in the parent link from the previous iteration to this parent.
+ if (Child)
+ Child->Parent = &Info;
+ // Keep walking, whether we inserted or not, if parent link is missing.
+ // (If it's present, parent links must be present up to the root, so stop)
+ Child = &Info;
+ return Info.Parent != nullptr;
+ });
+ return Leaf;
}
- // Uniquify all parent directories of all files.
- llvm::StringMap<bool> DirectoryHasCDB;
- std::vector<llvm::StringRef> FileAncestors;
- for (llvm::StringRef File : AllFiles) {
- actOnAllParentDirectories(File, [&](PathRef Path) {
- auto It = DirectoryHasCDB.try_emplace(Path);
- // Already seen this path, and all of its parents.
- if (!It.second)
- return true;
+ // Populates DirInfo::Cache (and State, if it is TargetCDB).
+ void grabCaches() {
+ // Fast path out if there were no files, or CDB loading is off.
+ if (Dirs.empty())
+ return;
- FileAncestors.push_back(It.first->getKey());
- return pathEqual(Path, T.PI.SourceRoot);
- });
+ std::vector<llvm::StringRef> DirKeys;
+ std::vector<DirInfo *> DirValues;
+ DirKeys.reserve(Dirs.size() + 1);
+ DirValues.reserve(Dirs.size());
+ for (auto &E : Dirs) {
+ DirKeys.push_back(E.first());
+ DirValues.push_back(&E.second);
+ }
+
+ // Also look up the cache entry for the CDB we're broadcasting.
+ // Comparing DirectoryCache pointers is more robust than checking string
+ // equality, e.g. reuses the case-sensitivity handling.
+ DirKeys.push_back(ThisDir);
+ auto DirCaches = Parent.getDirectoryCaches(DirKeys);
+ const DirectoryCache *ThisCache = DirCaches.back();
+ DirCaches.pop_back();
+ DirKeys.pop_back();
+
+ for (unsigned I = 0; I < DirKeys.size(); ++I) {
+ DirValues[I]->Cache = DirCaches[I];
+ if (DirCaches[I] == ThisCache)
+ DirValues[I]->State = DirInfo::TargetCDB;
+ }
}
- // Work out which ones have CDBs in them.
- // Given that we know that CDBs have been moved/generated, don't trust caches.
- // (This should be rare, so it's OK to add a little latency).
- constexpr auto IgnoreCache = std::chrono::steady_clock::time_point::max();
- auto DirectoryCaches = Parent.getDirectoryCaches(FileAncestors);
- assert(DirectoryCaches.size() == FileAncestors.size());
- for (unsigned I = 0; I < DirectoryCaches.size(); ++I) {
- bool ShouldBroadcast = false;
- if (ShouldStop.load(std::memory_order_acquire)) {
- log("Giving up on broadcasting CDB, as we're shutting down");
- return;
+
+ // Should we include a file from this search path?
+ bool shouldInclude(SearchPath P) {
+ DirInfo *Info = P.getPointer();
+ if (!Info)
+ return false;
+ if (Info->State == DirInfo::Unknown) {
+ assert(Info->Cache && "grabCaches() should have filled this");
+ // Given that we know that CDBs have been moved/generated, don't trust
+ // caches. (This should be rare, so it's OK to add a little latency).
+ constexpr auto IgnoreCache = std::chrono::steady_clock::time_point::max();
+ // Don't broadcast CDBs discovered while broadcasting!
+ bool ShouldBroadcast = false;
+ bool Exists =
+ nullptr != Info->Cache->get(Parent.Opts.TFS, ShouldBroadcast,
+ /*FreshTime=*/IgnoreCache,
+ /*FreshTimeMissing=*/IgnoreCache);
+ Info->State = Exists ? DirInfo::OtherCDB : DirInfo::Missing;
}
- if (DirectoryCaches[I]->get(Parent.Opts.TFS, ShouldBroadcast,
- /*FreshTime=*/IgnoreCache,
- /*FreshTimeMissing=*/IgnoreCache))
- DirectoryHasCDB.find(FileAncestors[I])->setValue(true);
+ // If we have a CDB, include the file if it's the target CDB only.
+ if (Info->State != DirInfo::Missing)
+ return Info->State == DirInfo::TargetCDB;
+ // If we have no CDB and no relevant parent, don't include the file.
+ if (!P.getInt() || !Info->Parent)
+ return false;
+ // Walk up to the next parent.
+ return shouldInclude(SearchPath(Info->Parent, 1));
}
- std::vector<std::string> GovernedFiles;
- for (llvm::StringRef File : AllFiles) {
- // A file is governed by this CDB if lookup for the file would find it.
- // Independent of whether it has an entry for that file or not.
- actOnAllParentDirectories(File, [&](PathRef Path) {
- if (DirectoryHasCDB.lookup(Path)) {
- if (pathEqual(Path, T.PI.SourceRoot))
- // Make sure listeners always get a canonical path for the file.
- GovernedFiles.push_back(removeDots(File));
- // Stop as soon as we hit a CDB.
+public:
+ Filter(llvm::StringRef ThisDir,
+ DirectoryBasedGlobalCompilationDatabase &Parent)
+ : ThisDir(ThisDir), Parent(Parent) {}
+
+ std::vector<std::string> filter(std::vector<std::string> AllFiles,
+ std::atomic<bool> &ShouldStop) {
+ std::vector<std::string> Filtered;
+ // Allow for clean early-exit of the slow parts.
+ auto ExitEarly = [&] {
+ if (ShouldStop.load(std::memory_order_acquire)) {
+ log("Giving up on broadcasting CDB, as we're shutting down");
+ Filtered.clear();
return true;
}
return false;
- });
+ };
+ // Compute search path for each file.
+ std::vector<SearchPath> SearchPaths(AllFiles.size());
+ for (unsigned I = 0; I < AllFiles.size(); ++I) {
+ if (Parent.Opts.CompileCommandsDir) { // FIXME: unify with config
+ SearchPaths[I].setPointer(
+ &Dirs[Parent.Opts.CompileCommandsDir.getValue()]);
+ continue;
+ }
+ if (ExitEarly()) // loading config may be slow
+ return Filtered;
+ WithContext WithProvidedContent(Parent.Opts.ContextProvider(AllFiles[I]));
+ const Config::CDBSearchSpec &Spec =
+ Config::current().CompileFlags.CDBSearch;
+ switch (Spec.Policy) {
+ case Config::CDBSearchSpec::NoCDBSearch:
+ break;
+ case Config::CDBSearchSpec::Ancestors:
+ SearchPaths[I].setInt(/*Recursive=*/1);
+ SearchPaths[I].setPointer(addParents(AllFiles[I]));
+ break;
+ case Config::CDBSearchSpec::FixedDir:
+ SearchPaths[I].setPointer(&Dirs[Spec.FixedCDBPath.getValue()]);
+ break;
+ }
+ }
+ // Get the CDB cache for each dir on the search path, but don't load yet.
+ grabCaches();
+ // Now work out which files we want to keep, loading CDBs where needed.
+ for (unsigned I = 0; I < AllFiles.size(); ++I) {
+ if (ExitEarly()) // loading CDBs may be slow
+ return Filtered;
+ if (shouldInclude(SearchPaths[I]))
+ Filtered.push_back(std::move(AllFiles[I]));
+ }
+ return Filtered;
}
+};
- Parent.OnCommandChanged.broadcast(std::move(GovernedFiles));
+void DirectoryBasedGlobalCompilationDatabase::BroadcastThread::process(
+ const CDBLookupResult &T) {
+ vlog("Broadcasting compilation database from {0}", T.PI.SourceRoot);
+ std::vector<std::string> GovernedFiles =
+ Filter(T.PI.SourceRoot, Parent).filter(T.CDB->getAllFiles(), ShouldStop);
+ if (!GovernedFiles.empty())
+ Parent.OnCommandChanged.broadcast(std::move(GovernedFiles));
}
void DirectoryBasedGlobalCompilationDatabase::broadcastCDB(
#include "GlobalCompilationDatabase.h"
+#include "Config.h"
#include "Matchers.h"
#include "TestFS.h"
#include "support/Path.h"
llvm::formatv(CDBOuter, llvm::sys::path::convert_to_slash(testRoot()));
FS.Files[testPath("build/compile_commands.json")] =
llvm::formatv(CDBInner, llvm::sys::path::convert_to_slash(testRoot()));
+ FS.Files[testPath("foo/compile_flags.txt")] = "-DFOO";
// Note that gen2.cc goes missing with our following model, not sure this
// happens in practice though.
{
+ SCOPED_TRACE("Default ancestor scanning");
DirectoryBasedGlobalCompilationDatabase DB(FS);
std::vector<std::string> DiscoveredFiles;
auto Sub =
EXPECT_THAT(DiscoveredFiles, UnorderedElementsAre(EndsWith("gen.cc")));
}
- // With a custom compile commands dir.
{
+ SCOPED_TRACE("With config");
+ DirectoryBasedGlobalCompilationDatabase::Options Opts(FS);
+ Opts.ContextProvider = [&](llvm::StringRef Path) {
+ Config Cfg;
+ if (Path.endswith("a.cc")) {
+ // a.cc uses another directory's CDB, so it won't be discovered.
+ Cfg.CompileFlags.CDBSearch.Policy = Config::CDBSearchSpec::FixedDir;
+ Cfg.CompileFlags.CDBSearch.FixedCDBPath = testPath("foo");
+ } else if (Path.endswith("gen.cc")) {
+ // gen.cc has CDB search disabled, so it won't be discovered.
+ Cfg.CompileFlags.CDBSearch.Policy = Config::CDBSearchSpec::NoCDBSearch;
+ } else if (Path.endswith("gen2.cc")) {
+ // gen2.cc explicitly lists this directory, so it will be discovered.
+ Cfg.CompileFlags.CDBSearch.Policy = Config::CDBSearchSpec::FixedDir;
+ Cfg.CompileFlags.CDBSearch.FixedCDBPath = testRoot();
+ }
+ return Context::current().derive(Config::Key, std::move(Cfg));
+ };
+ DirectoryBasedGlobalCompilationDatabase DB(Opts);
+ std::vector<std::string> DiscoveredFiles;
+ auto Sub =
+ DB.watch([&DiscoveredFiles](const std::vector<std::string> Changes) {
+ DiscoveredFiles = Changes;
+ });
+
+ // Does not use the root CDB, so no broadcast.
+ auto Cmd = DB.getCompileCommand(testPath("build/../a.cc"));
+ ASSERT_TRUE(Cmd.hasValue());
+ EXPECT_THAT(Cmd->CommandLine, Contains("-DFOO")) << "a.cc uses foo/ CDB";
+ ASSERT_TRUE(DB.blockUntilIdle(timeoutSeconds(10)));
+ EXPECT_THAT(DiscoveredFiles, IsEmpty()) << "Root CDB not discovered yet";
+
+ // No special config for b.cc, so we trigger broadcast of the root CDB.
+ DB.getCompileCommand(testPath("b.cc"));
+ ASSERT_TRUE(DB.blockUntilIdle(timeoutSeconds(10)));
+ EXPECT_THAT(DiscoveredFiles, ElementsAre(testPath("build/gen2.cc")));
+ DiscoveredFiles.clear();
+
+ // No CDB search so no discovery/broadcast triggered for build/ CDB.
+ DB.getCompileCommand(testPath("build/gen.cc"));
+ ASSERT_TRUE(DB.blockUntilIdle(timeoutSeconds(10)));
+ EXPECT_THAT(DiscoveredFiles, IsEmpty());
+ }
+
+ {
+ SCOPED_TRACE("With custom compile commands dir");
DirectoryBasedGlobalCompilationDatabase::Options Opts(FS);
Opts.CompileCommandsDir = testRoot();
DirectoryBasedGlobalCompilationDatabase DB(Opts);
EXPECT_EQ(testPath("x"), Commands.getValue().Directory);
}
+MATCHER_P(hasArg, Flag, "") {
+ if (!arg.hasValue()) {
+ *result_listener << "command is null";
+ return false;
+ }
+ if (!llvm::is_contained(arg->CommandLine, Flag)) {
+ *result_listener << "flags are " << llvm::join(arg->CommandLine, " ");
+ return false;
+ }
+ return true;
+}
+
+TEST(GlobalCompilationDatabaseTest, Config) {
+ MockFS FS;
+ FS.Files[testPath("x/compile_flags.txt")] = "-DX";
+ FS.Files[testPath("x/y/z/compile_flags.txt")] = "-DZ";
+
+ Config::CDBSearchSpec Spec;
+ DirectoryBasedGlobalCompilationDatabase::Options Opts(FS);
+ Opts.ContextProvider = [&](llvm::StringRef Path) {
+ Config C;
+ C.CompileFlags.CDBSearch = Spec;
+ return Context::current().derive(Config::Key, std::move(C));
+ };
+ DirectoryBasedGlobalCompilationDatabase CDB(Opts);
+
+ // Default ancestor behavior.
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("foo.cc")));
+ EXPECT_THAT(CDB.getCompileCommand(testPath("x/foo.cc")), hasArg("-DX"));
+ EXPECT_THAT(CDB.getCompileCommand(testPath("x/y/foo.cc")), hasArg("-DX"));
+ EXPECT_THAT(CDB.getCompileCommand(testPath("x/y/z/foo.cc")), hasArg("-DZ"));
+
+ Spec.Policy = Config::CDBSearchSpec::NoCDBSearch;
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("foo.cc")));
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("x/foo.cc")));
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("x/y/foo.cc")));
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("x/y/z/foo.cc")));
+
+ Spec.Policy = Config::CDBSearchSpec::FixedDir;
+ Spec.FixedCDBPath = testPath("w"); // doesn't exist
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("foo.cc")));
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("x/foo.cc")));
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("x/y/foo.cc")));
+ EXPECT_FALSE(CDB.getCompileCommand(testPath("x/y/z/foo.cc")));
+
+ Spec.FixedCDBPath = testPath("x/y/z");
+ EXPECT_THAT(CDB.getCompileCommand(testPath("foo.cc")), hasArg("-DZ"));
+ EXPECT_THAT(CDB.getCompileCommand(testPath("x/foo.cc")), hasArg("-DZ"));
+ EXPECT_THAT(CDB.getCompileCommand(testPath("x/y/foo.cc")), hasArg("-DZ"));
+ EXPECT_THAT(CDB.getCompileCommand(testPath("x/y/z/foo.cc")), hasArg("-DZ"));
+}
+
TEST(GlobalCompilationDatabaseTest, NonCanonicalFilenames) {
OverlayCDB DB(nullptr);
std::vector<std::string> DiscoveredFiles;
projects / platform / upstream / llvm.git / commitdiff