return;
auto AddStream = [&](size_t Task) {
- return std::make_unique<NativeObjectStream>(std::move(OS));
+ return std::make_unique<CachedFileStream>(std::move(OS));
};
lto::Config Conf;
if (CGOpts.SaveTempsFilePrefix != "") {
// The /lldltocache option specifies the path to a directory in which to cache
// native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->ltoCache.empty())
cache =
check(localCache("ThinLTO", "Thin", config->ltoCache,
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->thinLTOCacheDir.empty())
cache =
check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
if (!bitcodeFiles.empty())
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
// The -cache_path_lto option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->thinLTOCacheDir.empty())
cache =
check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->thinLTOCacheDir.empty())
cache =
check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>(
const Config &C, ModuleSummaryIndex &CombinedIndex,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache)>;
+ AddStreamFn AddStream, FileCache Cache)>;
/// This ThinBackend runs the individual backend jobs in-process.
/// The default value means to use one job per hardware core (not hyper-thread).
///
/// The client will receive at most one callback (via either AddStream or
/// Cache) for each task identifier.
- Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr);
+ Error run(AddStreamFn AddStream, FileCache Cache = nullptr);
/// Static method that returns a list of libcall symbols that can be generated
/// by LTO but might not be visible from bitcode symbol table.
const SymbolResolution *&ResI, const SymbolResolution *ResE);
Error runRegularLTO(AddStreamFn AddStream);
- Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
+ Error runThinLTO(AddStreamFn AddStream, FileCache Cache,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols);
Error checkPartiallySplit();
-//===- Caching.h - LLVM File Cache Handling Configuration -------*- C++ -*-===//
+//===- Caching.h - LLVM Local File Cache ------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
//
-// This file defines the localCache function, which allows clients to add a
-// filesystem cache. This is used by ThinLTO.
+// This file defines the CachedFileStream and the localCache function, which
+// simplifies caching files on the local filesystem in a directory whose
+// contents are managed by a CachePruningPolicy.
//
//===----------------------------------------------------------------------===//
#define LLVM_SUPPORT_CACHING_H
#include "llvm/Support/Error.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/thread.h"
namespace llvm {
-/// This class wraps an output stream for a native object. Most clients should
-/// just be able to return an instance of this base class from the stream
-/// callback, but if a client needs to perform some action after the stream is
-/// written to, that can be done by deriving from this class and overriding the
-/// destructor.
-class NativeObjectStream {
+class MemoryBuffer;
+
+/// This class wraps an output stream for a file. Most clients should just be
+/// able to return an instance of this base class from the stream callback, but
+/// if a client needs to perform some action after the stream is written to,
+/// that can be done by deriving from this class and overriding the destructor.
+class CachedFileStream {
public:
- NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS)
- : OS(std::move(OS)) {}
+ CachedFileStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {}
std::unique_ptr<raw_pwrite_stream> OS;
- virtual ~NativeObjectStream() = default;
+ virtual ~CachedFileStream() = default;
};
-/// This type defines the callback to add a native object that is generated on
-/// the fly.
+/// This type defines the callback to add a file that is generated on the fly.
///
/// Stream callbacks must be thread safe.
using AddStreamFn =
- std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>;
+ std::function<Expected<std::unique_ptr<CachedFileStream>>(unsigned Task)>;
-/// This is the type of a native object cache. To request an item from the
-/// cache, pass a unique string as the Key. For hits, the cached file will be
-/// added to the link and this function will return AddStreamFn(). For misses,
-/// the cache will return a stream callback which must be called at most once to
-/// produce content for the stream. The native object stream produced by the
-/// stream callback will add the file to the link after the stream is written
-/// to.
+/// This is the type of a file cache. To request an item from the cache, pass a
+/// unique string as the Key. For hits, the cached file will be added to the
+/// link and this function will return AddStreamFn(). For misses, the cache will
+/// return a stream callback which must be called at most once to produce
+/// content for the stream. The file stream produced by the stream callback will
+/// add the file to the link after the stream is written to.
///
/// Clients generally look like this:
///
/// if (AddStreamFn AddStream = Cache(Task, Key))
/// ProduceContent(AddStream);
-using NativeObjectCache =
- std::function<AddStreamFn(unsigned Task, StringRef Key)>;
+using FileCache =
+ std::function<Expected<AddStreamFn>(unsigned Task, StringRef Key)>;
-/// This type defines the callback to add a pre-existing native object file
-/// (e.g. in a cache).
+/// This type defines the callback to add a pre-existing file (e.g. in a cache).
///
/// Buffer callbacks must be thread safe.
using AddBufferFn =
/// the cache directory if it does not already exist. The cache name appears in
/// error messages for errors during caching. The temporary file prefix is used
/// in the temporary file naming scheme used when writing files atomically.
-Expected<NativeObjectCache> localCache(Twine CacheNameRef,
- Twine TempFilePrefixRef,
- Twine CacheDirectoryPathRef,
- AddBufferFn AddBuffer);
+Expected<FileCache> localCache(Twine CacheNameRef, Twine TempFilePrefixRef,
+ Twine CacheDirectoryPathRef,
+ AddBufferFn AddBuffer);
} // namespace llvm
#endif
return Error::success();
}
-Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
+Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
// Compute "dead" symbols, we don't want to import/export these!
DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
class InProcessThinBackend : public ThinBackendProc {
ThreadPool BackendThreadPool;
AddStreamFn AddStream;
- NativeObjectCache Cache;
+ FileCache Cache;
std::set<GlobalValue::GUID> CfiFunctionDefs;
std::set<GlobalValue::GUID> CfiFunctionDecls;
const Config &Conf, ModuleSummaryIndex &CombinedIndex,
ThreadPoolStrategy ThinLTOParallelism,
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache)
+ AddStreamFn AddStream, FileCache Cache)
: ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
Cache(std::move(Cache)) {
}
Error runThinLTOBackendThread(
- AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,
- BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
+ AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
+ ModuleSummaryIndex &CombinedIndex,
const FunctionImporter::ImportMapTy &ImportList,
const FunctionImporter::ExportSetTy &ExportList,
const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
CfiFunctionDecls);
- if (AddStreamFn CacheAddStream = Cache(Task, Key))
+ Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key);
+ if (Error Err = CacheAddStreamOrErr.takeError())
+ return Err;
+ AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
+ if (CacheAddStream)
return RunThinBackend(CacheAddStream);
return Error::success();
ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism) {
return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache) {
+ AddStreamFn AddStream, FileCache Cache) {
return std::make_unique<InProcessThinBackend>(
Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream,
Cache);
raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache) {
+ AddStreamFn AddStream, FileCache Cache) {
return std::make_unique<WriteIndexesThinBackend>(
Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
};
}
-Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
+Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
timeTraceProfilerBegin("ThinLink", StringRef(""));
auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
EC.message());
}
- auto Stream = AddStream(Task);
+ Expected<std::unique_ptr<CachedFileStream>> StreamOrErr = AddStream(Task);
+ if (Error Err = StreamOrErr.takeError())
+ report_fatal_error(std::move(Err));
+ std::unique_ptr<CachedFileStream> &Stream = *StreamOrErr;
legacy::PassManager CodeGenPasses;
CodeGenPasses.add(
createImmutableModuleSummaryIndexWrapperPass(&CombinedIndex));
// make unique temp output file to put generated code
SmallString<128> Filename;
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o");
int FD;
if (EC)
emitError(EC.message());
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<llvm::raw_fd_ostream>(FD, true));
};
-//===-Caching.cpp - LLVM File Cache Handling ------------------------------===//
+//===-Caching.cpp - LLVM Local File Cache ---------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
//
-// This file implements the Caching used by ThinLTO.
+// This file implements the localCache function, which simplifies creating,
+// adding to, and querying a local file system cache. localCache takes care of
+// periodically pruning older files from the cache using a CachePruningPolicy.
//
//===----------------------------------------------------------------------===//
#include "llvm/Support/Caching.h"
-#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
-#include "llvm/Support/Process.h"
-#include "llvm/Support/raw_ostream.h"
#if !defined(_MSC_VER) && !defined(__MINGW32__)
#include <unistd.h>
using namespace llvm;
-Expected<NativeObjectCache> llvm::localCache(Twine CacheNameRef,
- Twine TempFilePrefixRef,
- Twine CacheDirectoryPathRef,
- AddBufferFn AddBuffer) {
+Expected<FileCache> llvm::localCache(Twine CacheNameRef,
+ Twine TempFilePrefixRef,
+ Twine CacheDirectoryPathRef,
+ AddBufferFn AddBuffer) {
if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPathRef))
return errorCodeToError(EC);
TempFilePrefixRef.toVector(TempFilePrefix);
CacheDirectoryPathRef.toVector(CacheDirectoryPath);
- return [=](unsigned Task, StringRef Key) -> AddStreamFn {
+ return [=](unsigned Task, StringRef Key) -> Expected<AddStreamFn> {
// This choice of file name allows the cache to be pruned (see pruneCache()
// in include/llvm/Support/CachePruning.h).
SmallString<64> EntryPath;
// Since the file is probably being deleted we handle it in the same way as
// if the file did not exist at all.
if (EC != errc::no_such_file_or_directory && EC != errc::permission_denied)
- report_fatal_error(Twine("Failed to open cache file ") + EntryPath +
- ": " + EC.message() + "\n");
+ return createStringError(EC, Twine("Failed to open cache file ") +
+ EntryPath + ": " + EC.message() + "\n");
- // This native object stream is responsible for commiting the resulting
- // file to the cache and calling AddBuffer to add it to the link.
- struct CacheStream : NativeObjectStream {
+ // This file stream is responsible for commiting the resulting file to the
+ // cache and calling AddBuffer to add it to the link.
+ struct CacheStream : CachedFileStream {
AddBufferFn AddBuffer;
sys::fs::TempFile TempFile;
std::string EntryPath;
CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer,
sys::fs::TempFile TempFile, std::string EntryPath,
unsigned Task)
- : NativeObjectStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
+ : CachedFileStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)),
Task(Task) {}
~CacheStream() {
+ // TODO: Manually commit rather than using non-trivial destructor,
+ // allowing to replace report_fatal_errors with a return Error.
+
// Make sure the stream is closed before committing it.
OS.reset();
}
};
- return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ return [=](size_t Task) -> Expected<std::unique_ptr<CachedFileStream>> {
// Write to a temporary to avoid race condition
SmallString<64> TempFilenameModel;
sys::path::append(TempFilenameModel, CacheDirectoryPath,
TempFilePrefix + "-%%%%%%.tmp.o");
Expected<sys::fs::TempFile> Temp = sys::fs::TempFile::create(
TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write);
- if (!Temp) {
- errs() << "Error: " << toString(Temp.takeError()) << "\n";
- report_fatal_error(CacheName + ": Can't get a temporary file");
- }
+ if (!Temp)
+ return createStringError(errc::io_error,
+ toString(Temp.takeError()) + ": " + CacheName +
+ ": Can't get a temporary file");
// This CacheStream will move the temporary file into the cache when done.
return std::make_unique<CacheStream>(
size_t MaxTasks = Lto->getMaxTasks();
std::vector<std::pair<SmallString<128>, bool>> Files(MaxTasks);
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
Files[Task].second = !SaveTemps;
int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<llvm::raw_fd_ostream>(FD, true));
};
*AddStream(Task)->OS << MB->getBuffer();
};
- NativeObjectCache Cache;
+ FileCache Cache;
if (!options::cache_dir.empty())
Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
error("writing merged module failed.");
}
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
std::string PartFilename = OutputFilename;
if (Parallelism != 1)
PartFilename += "." + utostr(Task);
std::make_unique<raw_fd_ostream>(PartFilename, EC, sys::fs::OF_None);
if (EC)
error("error opening the file '" + PartFilename + "': " + EC.message());
- return std::make_unique<NativeObjectStream>(std::move(S));
+ return std::make_unique<CachedFileStream>(std::move(S));
};
if (!CodeGen.compileOptimized(AddStream, Parallelism))
if (HasErrors)
return 1;
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
std::string Path = OutputFilename + "." + utostr(Task);
std::error_code EC;
auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None);
check(EC, Path);
- return std::make_unique<NativeObjectStream>(std::move(S));
+ return std::make_unique<CachedFileStream>(std::move(S));
};
auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
*AddStream(Task)->OS << MB->getBuffer();
};
- NativeObjectCache Cache;
+ FileCache Cache;
if (!CacheDir.empty())
Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
"failed to create cache");
projects / platform / upstream / llvm.git / commitdiff