return;
}
{
- std::lock_guard<std::mutex> Lock(Mutex);
- // If shutdown is issued, don't bother building.
- if (Done)
- return;
+ std::unique_lock<std::mutex> Lock(Mutex);
+ // If NextReq was requested with WantDiagnostics::Yes we cannot just drop
+ // that on the floor. Block until we start building it. This won't
+ // dead-lock as we are blocking the caller thread, while builds continue
+ // on preamble thread.
+ ReqCV.wait(Lock, [this] {
+ return !NextReq || NextReq->WantDiags != WantDiagnostics::Yes;
+ });
NextReq = std::move(Req);
}
// Let the worker thread know there's a request, notify_one is safe as there
friend class ASTWorkerHandle;
ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache, Semaphore &Barrier, bool RunSync,
- DebouncePolicy UpdateDebounce, bool StorePreamblesInMemory,
- ParsingCallbacks &Callbacks);
+ const TUScheduler::Options &Opts, ParsingCallbacks &Callbacks);
public:
/// Create a new ASTWorker and return a handle to it.
/// is null, all requests will be processed on the calling thread
/// synchronously instead. \p Barrier is acquired when processing each
/// request, it is used to limit the number of actively running threads.
- static ASTWorkerHandle
- create(PathRef FileName, const GlobalCompilationDatabase &CDB,
- TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
- Semaphore &Barrier, DebouncePolicy UpdateDebounce,
- bool StorePreamblesInMemory, ParsingCallbacks &Callbacks);
+ static ASTWorkerHandle create(PathRef FileName,
+ const GlobalCompilationDatabase &CDB,
+ TUScheduler::ASTCache &IdleASTs,
+ AsyncTaskRunner *Tasks, Semaphore &Barrier,
+ const TUScheduler::Options &Opts,
+ ParsingCallbacks &Callbacks);
~ASTWorker();
void update(ParseInputs Inputs, WantDiagnostics);
std::queue<Request> PreambleRequests; /* GUARDED_BY(Mutex) */
llvm::Optional<Request> CurrentRequest; /* GUARDED_BY(Mutex) */
mutable std::condition_variable RequestsCV;
+ Notification ReceivedPreamble;
/// Guards the callback that publishes results of AST-related computations
/// (diagnostics, highlightings) and file statuses.
std::mutex PublishMu;
std::shared_ptr<ASTWorker> Worker;
};
-ASTWorkerHandle
-ASTWorker::create(PathRef FileName, const GlobalCompilationDatabase &CDB,
- TUScheduler::ASTCache &IdleASTs, AsyncTaskRunner *Tasks,
- Semaphore &Barrier, DebouncePolicy UpdateDebounce,
- bool StorePreamblesInMemory, ParsingCallbacks &Callbacks) {
- std::shared_ptr<ASTWorker> Worker(
- new ASTWorker(FileName, CDB, IdleASTs, Barrier, /*RunSync=*/!Tasks,
- UpdateDebounce, StorePreamblesInMemory, Callbacks));
+ASTWorkerHandle ASTWorker::create(PathRef FileName,
+ const GlobalCompilationDatabase &CDB,
+ TUScheduler::ASTCache &IdleASTs,
+ AsyncTaskRunner *Tasks, Semaphore &Barrier,
+ const TUScheduler::Options &Opts,
+ ParsingCallbacks &Callbacks) {
+ std::shared_ptr<ASTWorker> Worker(new ASTWorker(
+ FileName, CDB, IdleASTs, Barrier, /*RunSync=*/!Tasks, Opts, Callbacks));
if (Tasks) {
Tasks->runAsync("ASTWorker:" + llvm::sys::path::filename(FileName),
[Worker]() { Worker->run(); });
ASTWorker::ASTWorker(PathRef FileName, const GlobalCompilationDatabase &CDB,
TUScheduler::ASTCache &LRUCache, Semaphore &Barrier,
- bool RunSync, DebouncePolicy UpdateDebounce,
- bool StorePreamblesInMemory, ParsingCallbacks &Callbacks)
- : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce),
+ bool RunSync, const TUScheduler::Options &Opts,
+ ParsingCallbacks &Callbacks)
+ : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(Opts.UpdateDebounce),
FileName(FileName), CDB(CDB), Callbacks(Callbacks), Barrier(Barrier),
Done(false), Status(FileName, Callbacks),
- PreamblePeer(FileName, Callbacks, StorePreamblesInMemory,
- // FIXME: Run PreamblePeer asynchronously once ast patching
- // is available.
- /*RunSync=*/true, Status, *this) {
+ PreamblePeer(FileName, Callbacks, Opts.StorePreamblesInMemory,
+ RunSync || !Opts.AsyncPreambleBuilds, Status, *this) {
// Set a fallback command because compile command can be accessed before
// `Inputs` is initialized. Other fields are only used after initialization
// from client inputs.
PreamblePeer.update(std::move(Invocation), std::move(Inputs),
std::move(CompilerInvocationDiags), WantDiags);
+ // Block until first preamble is ready, as patching an empty preamble would
+ // imply rebuilding it from scratch.
+ // This isn't the natural place to block, rather where the preamble would be
+ // consumed. But that's too late, we'd be running on the worker thread with
+ // the PreambleTask scheduled and so we'd deadlock.
+ ReceivedPreamble.wait();
return;
};
startTask(TaskName, std::move(Task), WantDiags, TUScheduler::NoInvalidation);
};
if (RunSync) {
Task();
+ ReceivedPreamble.notify();
return;
}
{
steady_clock::now(), Context::current().clone(),
llvm::None, TUScheduler::NoInvalidation, nullptr});
}
+ ReceivedPreamble.notify();
RequestsCV.notify_all();
}
Done = true;
}
// We are no longer going to build any preambles, let the waiters know that.
+ ReceivedPreamble.notify();
BuiltFirstPreamble.notify();
PreamblePeer.stop();
Status.stop();
}
bool ASTWorker::blockUntilIdle(Deadline Timeout) const {
- std::unique_lock<std::mutex> Lock(Mutex);
- return wait(Lock, RequestsCV, Timeout, [&] {
- return PreambleRequests.empty() && Requests.empty() && !CurrentRequest;
- });
+ auto WaitUntilASTWorkerIsIdle = [&] {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ return wait(Lock, RequestsCV, Timeout, [&] {
+ return PreambleRequests.empty() && Requests.empty() && !CurrentRequest;
+ });
+ };
+ // Make sure ASTWorker has processed all requests, which might issue new
+ // updates to PreamblePeer.
+ WaitUntilASTWorkerIsIdle();
+ // Now that ASTWorker processed all requests, ensure PreamblePeer has served
+ // all update requests. This might create new PreambleRequests for the
+ // ASTWorker.
+ PreamblePeer.blockUntilIdle(Timeout);
+ assert(Requests.empty() &&
+ "No new normal tasks can be scheduled concurrently with "
+ "blockUntilIdle(): ASTWorker isn't threadsafe");
+ // Finally make sure ASTWorker has processed all of the preamble updates.
+ return WaitUntilASTWorkerIsIdle();
}
// Render a TUAction to a user-facing string representation.
TUScheduler::TUScheduler(const GlobalCompilationDatabase &CDB,
const Options &Opts,
std::unique_ptr<ParsingCallbacks> Callbacks)
- : CDB(CDB), StorePreamblesInMemory(Opts.StorePreamblesInMemory),
+ : CDB(CDB), Opts(Opts),
Callbacks(Callbacks ? move(Callbacks)
: std::make_unique<ParsingCallbacks>()),
Barrier(Opts.AsyncThreadsCount),
IdleASTs(
- std::make_unique<ASTCache>(Opts.RetentionPolicy.MaxRetainedASTs)),
- UpdateDebounce(Opts.UpdateDebounce) {
+ std::make_unique<ASTCache>(Opts.RetentionPolicy.MaxRetainedASTs)) {
if (0 < Opts.AsyncThreadsCount) {
PreambleTasks.emplace();
WorkerThreads.emplace();
bool NewFile = FD == nullptr;
if (!FD) {
// Create a new worker to process the AST-related tasks.
- ASTWorkerHandle Worker = ASTWorker::create(
- File, CDB, *IdleASTs,
- WorkerThreads ? WorkerThreads.getPointer() : nullptr, Barrier,
- UpdateDebounce, StorePreamblesInMemory, *Callbacks);
+ ASTWorkerHandle Worker =
+ ASTWorker::create(File, CDB, *IdleASTs,
+ WorkerThreads ? WorkerThreads.getPointer() : nullptr,
+ Barrier, Opts, *Callbacks);
FD = std::unique_ptr<FileData>(
new FileData{Inputs.Contents, std::move(Worker)});
} else {
FS.Files[FooCpp] = SourceContents;
Server.addDocument(FooCpp, SourceContents);
- auto DumpParse1 = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for diagnostics";
+ auto DumpParse1 = dumpASTWithoutMemoryLocs(Server, FooCpp);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
Server.addDocument(FooCpp, "");
- auto DumpParseEmpty = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for diagnostics";
+ auto DumpParseEmpty = dumpASTWithoutMemoryLocs(Server, FooCpp);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
Server.addDocument(FooCpp, SourceContents);
- auto DumpParse2 = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for diagnostics";
+ auto DumpParse2 = dumpASTWithoutMemoryLocs(Server, FooCpp);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
EXPECT_EQ(DumpParse1, DumpParse2);
FS.Files[FooCpp] = SourceContents;
Server.addDocument(FooCpp, SourceContents);
- auto DumpParse1 = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for diagnostics";
+ auto DumpParse1 = dumpASTWithoutMemoryLocs(Server, FooCpp);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
FS.Files[FooH] = "";
Server.addDocument(FooCpp, SourceContents);
- auto DumpParseDifferent = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for diagnostics";
+ auto DumpParseDifferent = dumpASTWithoutMemoryLocs(Server, FooCpp);
EXPECT_TRUE(DiagConsumer.hadErrorInLastDiags());
FS.Files[FooH] = "int a;";
Server.addDocument(FooCpp, SourceContents);
- auto DumpParse2 = dumpASTWithoutMemoryLocs(Server, FooCpp);
ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for diagnostics";
+ auto DumpParse2 = dumpASTWithoutMemoryLocs(Server, FooCpp);
EXPECT_FALSE(DiagConsumer.hadErrorInLastDiags());
EXPECT_EQ(DumpParse1, DumpParse2);
#include "support/Threading.h"
#include "clang/Basic/DiagnosticDriver.h"
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdint>
+#include <memory>
+#include <string>
#include <utility>
namespace clang {
int TotalASTReads = 0;
int TotalPreambleReads = 0;
int TotalUpdates = 0;
+ llvm::StringMap<int> LatestDiagVersion;
// Run TUScheduler and collect some stats.
{
auto Inputs = getInputs(File, Contents.str());
{
WithContextValue WithNonce(NonceKey, ++Nonce);
- Inputs.Version = std::to_string(Nonce);
+ Inputs.Version = std::to_string(UpdateI);
updateWithDiags(
S, File, Inputs, WantDiagnostics::Auto,
- [File, Nonce, &Mut, &TotalUpdates](std::vector<Diag>) {
+ [File, Nonce, Version(Inputs.Version), &Mut, &TotalUpdates,
+ &LatestDiagVersion](std::vector<Diag>) {
EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
std::lock_guard<std::mutex> Lock(Mut);
++TotalUpdates;
EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
+ // Make sure Diags are for a newer version.
+ auto It = LatestDiagVersion.try_emplace(File, -1);
+ const int PrevVersion = It.first->second;
+ int CurVersion;
+ ASSERT_TRUE(llvm::to_integer(Version, CurVersion, 10));
+ EXPECT_LT(PrevVersion, CurVersion);
+ It.first->getValue() = CurVersion;
});
}
{
} // TUScheduler destructor waits for all operations to finish.
std::lock_guard<std::mutex> Lock(Mut);
- EXPECT_EQ(TotalUpdates, FilesCount * UpdatesPerFile);
+ // Updates might get coalesced in preamble thread and result in dropping
+ // diagnostics for intermediate snapshots.
+ EXPECT_GE(TotalUpdates, FilesCount);
+ EXPECT_LE(TotalUpdates, FilesCount * UpdatesPerFile);
+ // We should receive diags for last update.
+ for (const auto &Entry : LatestDiagVersion)
+ EXPECT_EQ(Entry.second, UpdatesPerFile - 1);
EXPECT_EQ(TotalASTReads, FilesCount * UpdatesPerFile);
EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile);
}
EXPECT_NEAR(25, Compute({}), 0.01) << "no history -> max";
}
+TEST_F(TUSchedulerTests, AsyncPreambleThread) {
+ // Blocks preamble thread while building preamble with \p BlockVersion until
+ // \p N is notified.
+ class BlockPreambleThread : public ParsingCallbacks {
+ public:
+ BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)
+ : BlockVersion(BlockVersion), N(N) {}
+ void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
+ std::shared_ptr<clang::Preprocessor> PP,
+ const CanonicalIncludes &) override {
+ if (Version == BlockVersion)
+ N.wait();
+ }
+
+ private:
+ llvm::StringRef BlockVersion;
+ Notification &N;
+ };
+
+ static constexpr llvm::StringLiteral InputsV0 = "v0";
+ static constexpr llvm::StringLiteral InputsV1 = "v1";
+ Notification Ready;
+ TUScheduler S(CDB, optsForTest(),
+ std::make_unique<BlockPreambleThread>(InputsV1, Ready));
+
+ Path File = testPath("foo.cpp");
+ auto PI = getInputs(File, "");
+ PI.Version = InputsV0.str();
+ S.update(File, PI, WantDiagnostics::Auto);
+ S.blockUntilIdle(timeoutSeconds(10));
+
+ // Block preamble builds.
+ PI.Version = InputsV1.str();
+ // Issue second update which will block preamble thread.
+ S.update(File, PI, WantDiagnostics::Auto);
+
+ Notification RunASTAction;
+ // Issue an AST read, which shouldn't be blocked and see latest version of the
+ // file.
+ S.runWithAST("test", File, [&](Expected<InputsAndAST> AST) {
+ ASSERT_TRUE(bool(AST));
+ // Make sure preamble is built with stale inputs, but AST was built using
+ // new ones.
+ EXPECT_THAT(AST->AST.preambleVersion(), InputsV0);
+ EXPECT_THAT(AST->Inputs.Version, InputsV1.str());
+ RunASTAction.notify();
+ });
+ RunASTAction.wait();
+ Ready.notify();
+}
+
} // namespace
} // namespace clangd
} // namespace clang