#include "clang/Lex/Lexer.h"
#include "clang/Lex/Preprocessor.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/AlignOf.h"
#include "llvm/Support/MD5.h"
#include <memory>
#include <system_error>
/// reuse the PCH for the subsequent runs. Use BuildPreamble to create PCH and
/// CanReusePreamble + AddImplicitPreamble to make use of it.
class PrecompiledPreamble {
- class TempPCHFile;
+ class PCHStorage;
struct PreambleFileHash;
public:
///
/// \param PCHContainerOps An instance of PCHContainerOperations.
///
+ /// \param StoreInMemory Store PCH in memory. If false, PCH will be stored in
+ /// a temporary file.
+ ///
/// \param Callbacks A set of callbacks to be executed when building
/// the preamble.
static llvm::ErrorOr<PrecompiledPreamble>
const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
std::shared_ptr<PCHContainerOperations> PCHContainerOps,
- PreambleCallbacks &Callbacks);
+ bool StoreInMemory, PreambleCallbacks &Callbacks);
PrecompiledPreamble(PrecompiledPreamble &&) = default;
PrecompiledPreamble &operator=(PrecompiledPreamble &&) = default;
/// PreambleBounds used to build the preamble.
PreambleBounds getBounds() const;
- /// The temporary file path at which the preamble PCH was placed.
- StringRef GetPCHPath() const { return PCHFile.getFilePath(); }
-
/// Check whether PrecompiledPreamble can be reused for the new contents(\p
/// MainFileBuffer) of the main file.
bool CanReuse(const CompilerInvocation &Invocation,
vfs::FileSystem *VFS) const;
/// Changes options inside \p CI to use PCH from this preamble. Also remaps
- /// main file to \p MainFileBuffer.
+ /// main file to \p MainFileBuffer and updates \p VFS to ensure the preamble
+ /// is accessible.
void AddImplicitPreamble(CompilerInvocation &CI,
+ IntrusiveRefCntPtr<vfs::FileSystem> &VFS,
llvm::MemoryBuffer *MainFileBuffer) const;
private:
- PrecompiledPreamble(TempPCHFile PCHFile, std::vector<char> PreambleBytes,
+ PrecompiledPreamble(PCHStorage Storage, std::vector<char> PreambleBytes,
bool PreambleEndsAtStartOfLine,
llvm::StringMap<PreambleFileHash> FilesInPreamble);
llvm::Optional<std::string> FilePath;
};
+ class InMemoryPreamble {
+ public:
+ std::string Data;
+ };
+
+ class PCHStorage {
+ public:
+ enum class Kind { Empty, InMemory, TempFile };
+
+ PCHStorage() = default;
+ PCHStorage(TempPCHFile File);
+ PCHStorage(InMemoryPreamble Memory);
+
+ PCHStorage(const PCHStorage &) = delete;
+ PCHStorage &operator=(const PCHStorage &) = delete;
+
+ PCHStorage(PCHStorage &&Other);
+ PCHStorage &operator=(PCHStorage &&Other);
+
+ ~PCHStorage();
+
+ Kind getKind() const;
+
+ TempPCHFile &asFile();
+ const TempPCHFile &asFile() const;
+
+ InMemoryPreamble &asMemory();
+ const InMemoryPreamble &asMemory() const;
+
+ private:
+ void destroy();
+ void setEmpty();
+
+ private:
+ Kind StorageKind = Kind::Empty;
+ llvm::AlignedCharArrayUnion<TempPCHFile, InMemoryPreamble> Storage = {};
+ };
+
/// Data used to determine if a file used in the preamble has been changed.
struct PreambleFileHash {
/// All files have size set.
}
};
- /// Manages the lifetime of temporary file that stores a PCH.
- TempPCHFile PCHFile;
+ /// Sets up the PreprocessorOptions and changes VFS, so that PCH stored in \p
+ /// Storage is accessible to clang. This method is an implementation detail of
+ /// AddImplicitPreamble.
+ static void setupPreambleStorage(const PCHStorage &Storage,
+ PreprocessorOptions &PreprocessorOpts,
+ IntrusiveRefCntPtr<vfs::FileSystem> &VFS);
+
+ /// Manages the memory buffer or temporary file that stores the PCH.
+ PCHStorage Storage;
/// Keeps track of the files that were used when computing the
/// preamble, with both their buffer size and their modification time.
///
}
}
-static IntrusiveRefCntPtr<vfs::FileSystem> createVFSOverlayForPreamblePCH(
- StringRef PCHFilename,
- IntrusiveRefCntPtr<vfs::FileSystem> RealFS,
- IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
- // We want only the PCH file from the real filesystem to be available,
- // so we create an in-memory VFS with just that and overlay it on top.
- auto Buf = RealFS->getBufferForFile(PCHFilename);
- if (!Buf)
- return VFS;
- IntrusiveRefCntPtr<vfs::InMemoryFileSystem>
- PCHFS(new vfs::InMemoryFileSystem());
- PCHFS->addFile(PCHFilename, 0, std::move(*Buf));
- IntrusiveRefCntPtr<vfs::OverlayFileSystem>
- Overlay(new vfs::OverlayFileSystem(VFS));
- Overlay->pushOverlay(PCHFS);
- return Overlay;
-}
-
/// Parse the source file into a translation unit using the given compiler
/// invocation, replacing the current translation unit.
///
if (!Invocation)
return true;
+ auto CCInvocation = std::make_shared<CompilerInvocation>(*Invocation);
+ if (OverrideMainBuffer) {
+ assert(Preamble &&
+ "No preamble was built, but OverrideMainBuffer is not null");
+ IntrusiveRefCntPtr<vfs::FileSystem> OldVFS = VFS;
+ Preamble->AddImplicitPreamble(*CCInvocation, VFS, OverrideMainBuffer.get());
+ if (OldVFS != VFS && FileMgr) {
+ assert(OldVFS == FileMgr->getVirtualFileSystem() &&
+ "VFS passed to Parse and VFS in FileMgr are different");
+ FileMgr = new FileManager(FileMgr->getFileSystemOpts(), VFS);
+ }
+ }
+
// Create the compiler instance to use for building the AST.
std::unique_ptr<CompilerInstance> Clang(
new CompilerInstance(std::move(PCHContainerOps)));
Clang->setVirtualFileSystem(VFS);
}
- // Make sure we can access the PCH file even if we're using a VFS
- if (!VFS && FileMgr)
- VFS = FileMgr->getVirtualFileSystem();
- IntrusiveRefCntPtr<vfs::FileSystem> RealFS = vfs::getRealFileSystem();
- if (OverrideMainBuffer && VFS && RealFS && VFS != RealFS &&
- !VFS->exists(Preamble->GetPCHPath())) {
- // We have a slight inconsistency here -- we're using the VFS to
- // read files, but the PCH was generated in the real file system.
- VFS = createVFSOverlayForPreamblePCH(Preamble->GetPCHPath(), RealFS, VFS);
- if (FileMgr) {
- FileMgr = new FileManager(FileMgr->getFileSystemOpts(), VFS);
- Clang->setFileManager(FileMgr.get());
- }
- else {
- Clang->setVirtualFileSystem(VFS);
- }
- }
-
// Recover resources if we crash before exiting this method.
llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance>
CICleanup(Clang.get());
- Clang->setInvocation(std::make_shared<CompilerInvocation>(*Invocation));
+ Clang->setInvocation(CCInvocation);
OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].getFile();
// Set up diagnostics, capturing any diagnostics that would
// If the main file has been overridden due to the use of a preamble,
// make that override happen and introduce the preamble.
if (OverrideMainBuffer) {
- assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null");
- Preamble->AddImplicitPreamble(Clang->getInvocation(), OverrideMainBuffer.get());
-
// The stored diagnostic has the old source manager in it; update
// the locations to refer into the new source manager. Since we've
// been careful to make sure that the source manager's state
llvm::ErrorOr<PrecompiledPreamble> NewPreamble = PrecompiledPreamble::Build(
PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, VFS,
- PCHContainerOps, Callbacks);
+ PCHContainerOps, /*StoreInMemory=*/false, Callbacks);
if (NewPreamble) {
Preamble = std::move(*NewPreamble);
PreambleRebuildCounter = 1;
// If the main file has been overridden due to the use of a preamble,
// make that override happen and introduce the preamble.
if (OverrideMainBuffer) {
- assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null");
- Preamble->AddImplicitPreamble(Clang->getInvocation(), OverrideMainBuffer.get());
+ assert(Preamble &&
+ "No preamble was built, but OverrideMainBuffer is not null");
+
+ auto VFS = FileMgr.getVirtualFileSystem();
+ Preamble->AddImplicitPreamble(Clang->getInvocation(), VFS,
+ OverrideMainBuffer.get());
+ // FIXME: there is no way to update VFS if it was changed by
+ // AddImplicitPreamble as FileMgr is accepted as a parameter by this method.
+ // We use on-disk preambles instead and rely on FileMgr's VFS to ensure the
+ // PCH files are always readable.
OwnedBuffers.push_back(OverrideMainBuffer.release());
} else {
PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
#include "clang/Serialization/ASTWriter.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSet.h"
+#include "llvm/Config/config.h"
#include "llvm/Support/CrashRecoveryContext.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/MutexGuard.h"
#include "llvm/Support/Process.h"
+#include <utility>
+
using namespace clang;
namespace {
+StringRef getInMemoryPreamblePath() {
+#if defined(LLVM_ON_UNIX)
+ return "/__clang_tmp/___clang_inmemory_preamble___";
+#elif defined(LLVM_ON_WIN32)
+ return "C:\\__clang_tmp\\___clang_inmemory_preamble___";
+#else
+#warning "Unknown platform. Defaulting to UNIX-style paths for in-memory PCHs"
+ return "/__clang_tmp/___clang_inmemory_preamble___";
+#endif
+}
+
+IntrusiveRefCntPtr<vfs::FileSystem>
+createVFSOverlayForPreamblePCH(StringRef PCHFilename,
+ std::unique_ptr<llvm::MemoryBuffer> PCHBuffer,
+ IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+ // We want only the PCH file from the real filesystem to be available,
+ // so we create an in-memory VFS with just that and overlay it on top.
+ IntrusiveRefCntPtr<vfs::InMemoryFileSystem> PCHFS(
+ new vfs::InMemoryFileSystem());
+ PCHFS->addFile(PCHFilename, 0, std::move(PCHBuffer));
+ IntrusiveRefCntPtr<vfs::OverlayFileSystem> Overlay(
+ new vfs::OverlayFileSystem(VFS));
+ Overlay->pushOverlay(PCHFS);
+ return Overlay;
+}
+
/// Keeps a track of files to be deleted in destructor.
class TemporaryFiles {
public:
class PrecompilePreambleAction : public ASTFrontendAction {
public:
- PrecompilePreambleAction(PreambleCallbacks &Callbacks)
- : Callbacks(Callbacks) {}
+ PrecompilePreambleAction(std::string *InMemStorage,
+ PreambleCallbacks &Callbacks)
+ : InMemStorage(InMemStorage), Callbacks(Callbacks) {}
std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
StringRef InFile) override;
friend class PrecompilePreambleConsumer;
bool HasEmittedPreamblePCH = false;
+ std::string *InMemStorage;
PreambleCallbacks &Callbacks;
};
std::unique_ptr<ASTConsumer>
PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
-
StringRef InFile) {
std::string Sysroot;
- std::string OutputFile;
- std::unique_ptr<raw_ostream> OS =
- GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot,
- OutputFile);
+ if (!GeneratePCHAction::ComputeASTConsumerArguments(CI, Sysroot))
+ return nullptr;
+
+ std::unique_ptr<llvm::raw_ostream> OS;
+ if (InMemStorage) {
+ OS = llvm::make_unique<llvm::raw_string_ostream>(*InMemStorage);
+ } else {
+ std::string OutputFile;
+ OS = GeneratePCHAction::CreateOutputFile(CI, InFile, OutputFile);
+ }
if (!OS)
return nullptr;
const CompilerInvocation &Invocation,
const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
- std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps, bool StoreInMemory,
PreambleCallbacks &Callbacks) {
assert(VFS && "VFS is null");
PreprocessorOptions &PreprocessorOpts =
PreambleInvocation->getPreprocessorOpts();
- // Create a temporary file for the precompiled preamble. In rare
- // circumstances, this can fail.
- llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
- PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
- if (!PreamblePCHFile)
- return BuildPreambleError::CouldntCreateTempFile;
+ llvm::Optional<TempPCHFile> TempFile;
+ if (!StoreInMemory) {
+ // Create a temporary file for the precompiled preamble. In rare
+ // circumstances, this can fail.
+ llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
+ PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
+ if (!PreamblePCHFile)
+ return BuildPreambleError::CouldntCreateTempFile;
+ TempFile = std::move(*PreamblePCHFile);
+ }
+
+ PCHStorage Storage = StoreInMemory ? PCHStorage(InMemoryPreamble())
+ : PCHStorage(std::move(*TempFile));
// Save the preamble text for later; we'll need to compare against it for
// subsequent reparses.
// Tell the compiler invocation to generate a temporary precompiled header.
FrontendOpts.ProgramAction = frontend::GeneratePCH;
- // FIXME: Generate the precompiled header into memory?
- FrontendOpts.OutputFile = PreamblePCHFile->getFilePath();
+ FrontendOpts.OutputFile = StoreInMemory ? getInMemoryPreamblePath()
+ : Storage.asFile().getFilePath();
PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
PreprocessorOpts.PrecompiledPreambleBytes.second = false;
// Inform preprocessor to record conditional stack when building the preamble.
}
std::unique_ptr<PrecompilePreambleAction> Act;
- Act.reset(new PrecompilePreambleAction(Callbacks));
+ Act.reset(new PrecompilePreambleAction(
+ StoreInMemory ? &Storage.asMemory().Data : nullptr, Callbacks));
if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
return BuildPreambleError::BeginSourceFileFailed;
}
}
- return PrecompiledPreamble(
- std::move(*PreamblePCHFile), std::move(PreambleBytes),
- PreambleEndsAtStartOfLine, std::move(FilesInPreamble));
+ return PrecompiledPreamble(std::move(Storage), std::move(PreambleBytes),
+ PreambleEndsAtStartOfLine,
+ std::move(FilesInPreamble));
}
PreambleBounds PrecompiledPreamble::getBounds() const {
}
void PrecompiledPreamble::AddImplicitPreamble(
- CompilerInvocation &CI, llvm::MemoryBuffer *MainFileBuffer) const {
+ CompilerInvocation &CI, IntrusiveRefCntPtr<vfs::FileSystem> &VFS,
+ llvm::MemoryBuffer *MainFileBuffer) const {
+ assert(VFS && "VFS must not be null");
+
auto &PreprocessorOpts = CI.getPreprocessorOpts();
+ // Remap main file to point to MainFileBuffer.
+ auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
+ PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
+
// Configure ImpicitPCHInclude.
PreprocessorOpts.PrecompiledPreambleBytes.first = PreambleBytes.size();
PreprocessorOpts.PrecompiledPreambleBytes.second = PreambleEndsAtStartOfLine;
- PreprocessorOpts.ImplicitPCHInclude = PCHFile.getFilePath();
PreprocessorOpts.DisablePCHValidation = true;
- // Remap main file to point to MainFileBuffer.
- auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
- PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
+ setupPreambleStorage(Storage, PreprocessorOpts, VFS);
}
PrecompiledPreamble::PrecompiledPreamble(
- TempPCHFile PCHFile, std::vector<char> PreambleBytes,
+ PCHStorage Storage, std::vector<char> PreambleBytes,
bool PreambleEndsAtStartOfLine,
llvm::StringMap<PreambleFileHash> FilesInPreamble)
- : PCHFile(std::move(PCHFile)), FilesInPreamble(std::move(FilesInPreamble)),
+ : Storage(std::move(Storage)), FilesInPreamble(std::move(FilesInPreamble)),
PreambleBytes(std::move(PreambleBytes)),
- PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {}
+ PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {
+ assert(this->Storage.getKind() != PCHStorage::Kind::Empty);
+}
llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile() {
// that we would never get a race condition in a multi-threaded setting (i.e.,
// multiple threads getting the same temporary path).
int FD;
- auto EC = llvm::sys::fs::createTemporaryFile(Prefix, Suffix, /*ref*/ FD,
- /*ref*/ File);
+ auto EC = llvm::sys::fs::createTemporaryFile(Prefix, Suffix, FD, File);
if (EC)
return EC;
// We only needed to make sure the file exists, close the file right away.
return *FilePath;
}
+PrecompiledPreamble::PCHStorage::PCHStorage(TempPCHFile File)
+ : StorageKind(Kind::TempFile) {
+ new (&asFile()) TempPCHFile(std::move(File));
+}
+
+PrecompiledPreamble::PCHStorage::PCHStorage(InMemoryPreamble Memory)
+ : StorageKind(Kind::InMemory) {
+ new (&asMemory()) InMemoryPreamble(std::move(Memory));
+}
+
+PrecompiledPreamble::PCHStorage::PCHStorage(PCHStorage &&Other) : PCHStorage() {
+ *this = std::move(Other);
+}
+
+PrecompiledPreamble::PCHStorage &PrecompiledPreamble::PCHStorage::
+operator=(PCHStorage &&Other) {
+ destroy();
+
+ StorageKind = Other.StorageKind;
+ switch (StorageKind) {
+ case Kind::Empty:
+ // do nothing;
+ break;
+ case Kind::TempFile:
+ new (&asFile()) TempPCHFile(std::move(Other.asFile()));
+ break;
+ case Kind::InMemory:
+ new (&asMemory()) InMemoryPreamble(std::move(Other.asMemory()));
+ break;
+ }
+
+ Other.setEmpty();
+ return *this;
+}
+
+PrecompiledPreamble::PCHStorage::~PCHStorage() { destroy(); }
+
+PrecompiledPreamble::PCHStorage::Kind
+PrecompiledPreamble::PCHStorage::getKind() const {
+ return StorageKind;
+}
+
+PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::PCHStorage::asFile() {
+ assert(getKind() == Kind::TempFile);
+ return *reinterpret_cast<TempPCHFile *>(Storage.buffer);
+}
+
+const PrecompiledPreamble::TempPCHFile &
+PrecompiledPreamble::PCHStorage::asFile() const {
+ return const_cast<PCHStorage *>(this)->asFile();
+}
+
+PrecompiledPreamble::InMemoryPreamble &
+PrecompiledPreamble::PCHStorage::asMemory() {
+ assert(getKind() == Kind::InMemory);
+ return *reinterpret_cast<InMemoryPreamble *>(Storage.buffer);
+}
+
+const PrecompiledPreamble::InMemoryPreamble &
+PrecompiledPreamble::PCHStorage::asMemory() const {
+ return const_cast<PCHStorage *>(this)->asMemory();
+}
+
+void PrecompiledPreamble::PCHStorage::destroy() {
+ switch (StorageKind) {
+ case Kind::Empty:
+ return;
+ case Kind::TempFile:
+ asFile().~TempPCHFile();
+ return;
+ case Kind::InMemory:
+ asMemory().~InMemoryPreamble();
+ return;
+ }
+}
+
+void PrecompiledPreamble::PCHStorage::setEmpty() {
+ destroy();
+ StorageKind = Kind::Empty;
+}
+
PrecompiledPreamble::PreambleFileHash
PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
time_t ModTime) {
return Result;
}
+void PrecompiledPreamble::setupPreambleStorage(
+ const PCHStorage &Storage, PreprocessorOptions &PreprocessorOpts,
+ IntrusiveRefCntPtr<vfs::FileSystem> &VFS) {
+ if (Storage.getKind() == PCHStorage::Kind::TempFile) {
+ const TempPCHFile &PCHFile = Storage.asFile();
+ PreprocessorOpts.ImplicitPCHInclude = PCHFile.getFilePath();
+
+ // Make sure we can access the PCH file even if we're using a VFS
+ IntrusiveRefCntPtr<vfs::FileSystem> RealFS = vfs::getRealFileSystem();
+ auto PCHPath = PCHFile.getFilePath();
+ if (VFS == RealFS || VFS->exists(PCHPath))
+ return;
+ auto Buf = RealFS->getBufferForFile(PCHPath);
+ if (!Buf) {
+ // We can't read the file even from RealFS, this is clearly an error,
+ // but we'll just leave the current VFS as is and let clang's code
+ // figure out what to do with missing PCH.
+ return;
+ }
+
+ // We have a slight inconsistency here -- we're using the VFS to
+ // read files, but the PCH was generated in the real file system.
+ VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(*Buf), VFS);
+ } else {
+ assert(Storage.getKind() == PCHStorage::Kind::InMemory);
+ // For in-memory preamble, we have to provide a VFS overlay that makes it
+ // accessible.
+ StringRef PCHPath = getInMemoryPreamblePath();
+ PreprocessorOpts.ImplicitPCHInclude = PCHPath;
+
+ // FIMXE(ibiryukov): Preambles can be large. We should allow shared access
+ // to the preamble data instead of copying it here.
+ auto Buf = llvm::MemoryBuffer::getMemBufferCopy(Storage.asMemory().Data);
+ VFS = createVFSOverlayForPreamblePCH(PCHPath, std::move(Buf), VFS);
+ }
+}
+
void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}