#ifndef LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H
#define LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H
-//#include "CloneSubModule.h"
#include "IndirectionUtils.h"
#include "LambdaResolver.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
-#include "llvm/Transforms/Utils/Cloning.h"
#include <list>
-#include <set>
-
-#include "llvm/Support/Debug.h"
namespace llvm {
namespace orc {
/// @brief Compile-on-demand layer.
///
-/// When a module is added to this layer a stub is created for each of its
-/// function definitions. The stubs and other global values are immediately
-/// added to the layer below. When a stub is called it triggers the extraction
-/// of the function body from the original module. The extracted body is then
-/// compiled and executed.
+/// Modules added to this layer have their calls indirected, and are then
+/// broken up into a set of single-function modules, each of which is added
+/// to the layer below in a singleton set. The lower layer can be any layer that
+/// accepts IR module sets.
+///
+/// It is expected that this layer will frequently be used on top of a
+/// LazyEmittingLayer. The combination of the two ensures that each function is
+/// compiled only when it is first called.
template <typename BaseLayerT, typename CompileCallbackMgrT>
class CompileOnDemandLayer {
private:
-
- // Utility class for MapValue. Only materializes declarations for global
- // variables.
- class GlobalDeclMaterializer : public ValueMaterializer {
- public:
- GlobalDeclMaterializer(Module &Dst) : Dst(Dst) {}
- Value* materializeValueFor(Value *V) final {
- if (auto *GV = dyn_cast<GlobalVariable>(V))
- return cloneGlobalVariableDecl(Dst, *GV);
- else if (auto *F = dyn_cast<Function>(V))
- return cloneFunctionDecl(Dst, *F);
- // Else.
- return nullptr;
- }
+ /// @brief Lookup helper that provides compatibility with the classic
+ /// static-compilation symbol resolution process.
+ ///
+ /// The CompileOnDemand (COD) layer splits modules up into multiple
+ /// sub-modules, each held in its own llvm::Module instance, in order to
+ /// support lazy compilation. When a module that contains private symbols is
+ /// broken up symbol linkage changes may be required to enable access to
+ /// "private" data that now resides in a different llvm::Module instance. To
+ /// retain expected symbol resolution behavior for clients of the COD layer,
+ /// the CODScopedLookup class uses a two-tiered lookup system to resolve
+ /// symbols. Lookup first scans sibling modules that were split from the same
+ /// original module (logical-module scoped lookup), then scans all other
+ /// modules that have been added to the lookup scope (logical-dylib scoped
+ /// lookup).
+ class CODScopedLookup {
private:
- Module &Dst;
- };
+ typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT;
+ typedef std::vector<BaseLayerModuleSetHandleT> SiblingHandlesList;
+ typedef std::list<SiblingHandlesList> PseudoDylibModuleSetHandlesList;
- typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT;
- class UncompiledPartition;
-
- // Logical module.
- //
- // This struct contains the handles for the global values and stubs (which
- // cover the external symbols of the original module), plus the handes for
- // each of the extracted partitions. These handleds are used for lookup (only
- // the globals/stubs module is searched) and memory management. The actual
- // searching and resource management are handled by the LogicalDylib that owns
- // the LogicalModule.
- struct LogicalModule {
- std::unique_ptr<Module> SrcM;
- BaseLayerModuleSetHandleT GVsAndStubsHandle;
- std::vector<BaseLayerModuleSetHandleT> ImplHandles;
- };
-
- // Logical dylib.
- //
- // This class handles symbol resolution and resource management for a set of
- // modules that were added together as a logical dylib.
- //
- // A logical dylib contains one-or-more LogicalModules plus a set of
- // UncompiledPartitions. LogicalModules support symbol resolution and resource
- // management for for code that has already been emitted. UncompiledPartitions
- // represent code that has not yet been compiled.
- class LogicalDylib {
- private:
- friend class UncompiledPartition;
- typedef std::list<LogicalModule> LogicalModuleList;
public:
+ /// @brief Handle for a logical module.
+ typedef typename PseudoDylibModuleSetHandlesList::iterator LMHandle;
- typedef unsigned UncompiledPartitionID;
- typedef typename LogicalModuleList::iterator LMHandle;
-
- // Construct a logical dylib.
- LogicalDylib(CompileOnDemandLayer &CODLayer) : CODLayer(CODLayer) { }
+ /// @brief Construct a scoped lookup.
+ CODScopedLookup(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {}
- // Delete this logical dylib, release logical module resources.
- virtual ~LogicalDylib() {
- releaseLogicalModuleResources();
- }
-
- // Get a reference to the containing layer.
- CompileOnDemandLayer& getCODLayer() { return CODLayer; }
+ virtual ~CODScopedLookup() {}
- // Get a reference to the base layer.
- BaseLayerT& getBaseLayer() { return CODLayer.BaseLayer; }
-
- // Start a new context for a single logical module.
+ /// @brief Start a new context for a single logical module.
LMHandle createLogicalModule() {
- LogicalModules.push_back(LogicalModule());
- return std::prev(LogicalModules.end());
+ Handles.push_back(SiblingHandlesList());
+ return std::prev(Handles.end());
}
- // Set the global-values-and-stubs module handle for this logical module.
- void setGVsAndStubsHandle(LMHandle LMH, BaseLayerModuleSetHandleT H) {
- LMH->GVsAndStubsHandle = H;
- }
-
- // Return the global-values-and-stubs module handle for this logical module.
- BaseLayerModuleSetHandleT getGVsAndStubsHandle(LMHandle LMH) {
- return LMH->GVsAndStubsHandle;
- }
-
- // Add a handle to a module containing lazy function bodies to the given
- // logical module.
+ /// @brief Add a concrete Module's handle to the given logical Module's
+ /// lookup scope.
void addToLogicalModule(LMHandle LMH, BaseLayerModuleSetHandleT H) {
- LMH->ImplHandles.push_back(H);
+ LMH->push_back(H);
}
- // Create an UncompiledPartition attached to this LogicalDylib.
- UncompiledPartition& createUncompiledPartition(LMHandle LMH,
- std::shared_ptr<Module> SrcM);
-
- // Take ownership of the given UncompiledPartition from the logical dylib.
- std::unique_ptr<UncompiledPartition>
- takeUPOwnership(UncompiledPartitionID ID);
+ /// @brief Remove a logical Module from the CODScopedLookup entirely.
+ void removeLogicalModule(LMHandle LMH) { Handles.erase(LMH); }
- // Look up a symbol in this context.
- JITSymbol findSymbolInternally(LMHandle LMH, const std::string &Name) {
- if (auto Symbol = getBaseLayer().findSymbolIn(LMH->GVsAndStubsHandle,
- Name, false))
+ /// @brief Look up a symbol in this context.
+ JITSymbol findSymbol(LMHandle LMH, const std::string &Name) {
+ if (auto Symbol = findSymbolIn(LMH, Name))
return Symbol;
- for (auto I = LogicalModules.begin(), E = LogicalModules.end(); I != E;
- ++I)
+ for (auto I = Handles.begin(), E = Handles.end(); I != E; ++I)
if (I != LMH)
- if (auto Symbol = getBaseLayer().findSymbolIn(I->GVsAndStubsHandle,
- Name, false))
+ if (auto Symbol = findSymbolIn(I, Name))
return Symbol;
return nullptr;
}
- JITSymbol findSymbol(const std::string &Name, bool ExportedSymbolsOnly) {
- for (auto &LM : LogicalModules)
- if (auto Symbol = getBaseLayer().findSymbolIn(LM.GVsAndStubsHandle,
- Name,
- ExportedSymbolsOnly))
- return Symbol;
- return nullptr;
- }
-
- // Find an external symbol (via the user supplied SymbolResolver).
+ /// @brief Find an external symbol (via the user supplied SymbolResolver).
virtual RuntimeDyld::SymbolInfo
- findSymbolExternally(const std::string &Name) const = 0;
+ externalLookup(const std::string &Name) const = 0;
private:
- void releaseLogicalModuleResources() {
- for (auto I = LogicalModules.begin(), E = LogicalModules.end(); I != E;
- ++I) {
- getBaseLayer().removeModuleSet(I->GVsAndStubsHandle);
- for (auto H : I->ImplHandles)
- getBaseLayer().removeModuleSet(H);
- }
+ JITSymbol findSymbolIn(LMHandle LMH, const std::string &Name) {
+ for (auto H : *LMH)
+ if (auto Symbol = BaseLayer.findSymbolIn(H, Name, false))
+ return Symbol;
+ return nullptr;
}
- CompileOnDemandLayer &CODLayer;
- LogicalModuleList LogicalModules;
- std::vector<std::unique_ptr<UncompiledPartition>> UncompiledPartitions;
+ BaseLayerT &BaseLayer;
+ PseudoDylibModuleSetHandlesList Handles;
};
template <typename ResolverPtrT>
- class LogicalDylibImpl : public LogicalDylib {
+ class CODScopedLookupImpl : public CODScopedLookup {
public:
- LogicalDylibImpl(CompileOnDemandLayer &CODLayer, ResolverPtrT Resolver)
- : LogicalDylib(CODLayer), Resolver(std::move(Resolver)) {}
+ CODScopedLookupImpl(BaseLayerT &BaseLayer, ResolverPtrT Resolver)
+ : CODScopedLookup(BaseLayer), Resolver(std::move(Resolver)) {}
RuntimeDyld::SymbolInfo
- findSymbolExternally(const std::string &Name) const override {
+ externalLookup(const std::string &Name) const override {
return Resolver->findSymbol(Name);
}
};
template <typename ResolverPtrT>
- static std::unique_ptr<LogicalDylib>
- createLogicalDylib(CompileOnDemandLayer &CODLayer,
- ResolverPtrT Resolver) {
- typedef LogicalDylibImpl<ResolverPtrT> Impl;
- return llvm::make_unique<Impl>(CODLayer, std::move(Resolver));
+ static std::shared_ptr<CODScopedLookup>
+ createCODScopedLookup(BaseLayerT &BaseLayer,
+ ResolverPtrT Resolver) {
+ typedef CODScopedLookupImpl<ResolverPtrT> Impl;
+ return std::make_shared<Impl>(BaseLayer, std::move(Resolver));
}
- // Uncompiled partition.
- //
- // Represents one as-yet uncompiled portion of a module.
- class UncompiledPartition {
- public:
-
- struct PartitionEntry {
- PartitionEntry(Function *F, TargetAddress CallbackID)
- : F(F), CallbackID(CallbackID) {}
- Function *F;
- TargetAddress CallbackID;
- };
-
- typedef std::vector<PartitionEntry> PartitionEntryList;
-
- // Creates an uncompiled partition with the list of functions that make up
- // this partition.
- UncompiledPartition(LogicalDylib &LD, typename LogicalDylib::LMHandle LMH,
- std::shared_ptr<Module> SrcM)
- : LD(LD), LMH(LMH), SrcM(std::move(SrcM)), ID(~0U) {}
-
- ~UncompiledPartition() {
- // FIXME: When we want to support threaded lazy compilation we'll need to
- // lock the callback manager here.
- auto &CCMgr = LD.getCODLayer().CompileCallbackMgr;
- for (auto PEntry : PartitionEntries)
- CCMgr.releaseCompileCallback(PEntry.CallbackID);
- }
-
- // Set the ID for this partition.
- void setID(typename LogicalDylib::UncompiledPartitionID ID) {
- this->ID = ID;
- }
-
- // Set the function set and callbacks for this partition.
- void setPartitionEntries(PartitionEntryList PartitionEntries) {
- this->PartitionEntries = std::move(PartitionEntries);
- }
+ typedef typename BaseLayerT::ModuleSetHandleT BaseLayerModuleSetHandleT;
+ typedef std::vector<BaseLayerModuleSetHandleT> BaseLayerModuleSetHandleListT;
- // Handle a compile callback for the function at index FnIdx.
- TargetAddress compile(unsigned FnIdx) {
- // Take ownership of self. This will ensure we delete the partition and
- // free all its resources once we're done compiling.
- std::unique_ptr<UncompiledPartition> This = LD.takeUPOwnership(ID);
-
- // Release all other compile callbacks for this partition.
- // We skip the callback for this function because that's the one that
- // called us, and the callback manager will already have removed it.
- auto &CCMgr = LD.getCODLayer().CompileCallbackMgr;
- for (unsigned I = 0; I < PartitionEntries.size(); ++I)
- if (I != FnIdx)
- CCMgr.releaseCompileCallback(PartitionEntries[I].CallbackID);
-
- // Grab the name of the function being called here.
- Function *F = PartitionEntries[FnIdx].F;
- std::string CalledFnName = Mangle(F->getName(), SrcM->getDataLayout());
-
- // Extract the function and add it to the base layer.
- auto PartitionImplH = emitPartition();
- LD.addToLogicalModule(LMH, PartitionImplH);
-
- // Update body pointers.
- // FIXME: When we start supporting remote lazy jitting this will need to
- // be replaced with a user-supplied callback for updating the
- // remote pointers.
- TargetAddress CalledAddr = 0;
- for (unsigned I = 0; I < PartitionEntries.size(); ++I) {
- auto F = PartitionEntries[I].F;
- std::string FName(F->getName());
- auto FnBodySym =
- LD.getBaseLayer().findSymbolIn(PartitionImplH,
- Mangle(FName, SrcM->getDataLayout()),
- false);
- auto FnPtrSym =
- LD.getBaseLayer().findSymbolIn(LD.getGVsAndStubsHandle(LMH),
- Mangle(FName + "$orc_addr",
- SrcM->getDataLayout()),
- false);
- assert(FnBodySym && "Couldn't find function body.");
- assert(FnPtrSym && "Couldn't find function body pointer.");
-
- auto FnBodyAddr = FnBodySym.getAddress();
- void *FnPtrAddr = reinterpret_cast<void*>(
- static_cast<uintptr_t>(FnPtrSym.getAddress()));
-
- // If this is the function we're calling record the address so we can
- // return it from this function.
- if (I == FnIdx)
- CalledAddr = FnBodyAddr;
-
- memcpy(FnPtrAddr, &FnBodyAddr, sizeof(uintptr_t));
- }
+ struct ModuleSetInfo {
+ // Symbol lookup - just one for the whole module set.
+ std::shared_ptr<CODScopedLookup> Lookup;
- // Finally, clear the partition structure so we don't try to
- // double-release the callbacks in the UncompiledPartition destructor.
- PartitionEntries.clear();
+ // Logical module handles.
+ std::vector<typename CODScopedLookup::LMHandle> LMHandles;
- return CalledAddr;
- }
+ // List of vectors of module set handles:
+ // One vector per logical module - each vector holds the handles for the
+ // exploded modules for that logical module in the base layer.
+ BaseLayerModuleSetHandleListT BaseLayerModuleSetHandles;
- private:
+ ModuleSetInfo(std::shared_ptr<CODScopedLookup> Lookup)
+ : Lookup(std::move(Lookup)) {}
- BaseLayerModuleSetHandleT emitPartition() {
- // Create the module.
- std::string NewName(SrcM->getName());
- for (auto &PEntry : PartitionEntries) {
- NewName += ".";
- NewName += PEntry.F->getName();
- }
- auto PM = llvm::make_unique<Module>(NewName, SrcM->getContext());
- PM->setDataLayout(SrcM->getDataLayout());
- ValueToValueMapTy VMap;
- GlobalDeclMaterializer GDM(*PM);
-
- // Create decls in the new module.
- for (auto &PEntry : PartitionEntries)
- cloneFunctionDecl(*PM, *PEntry.F, &VMap);
-
- // Move the function bodies.
- for (auto &PEntry : PartitionEntries)
- moveFunctionBody(*PEntry.F, VMap);
-
- // Create memory manager and symbol resolver.
- auto MemMgr = llvm::make_unique<SectionMemoryManager>();
- auto Resolver = createLambdaResolver(
- [this](const std::string &Name) {
- if (auto Symbol = LD.findSymbolInternally(LMH, Name))
- return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
- Symbol.getFlags());
- return LD.findSymbolExternally(Name);
- },
- [this](const std::string &Name) {
- if (auto Symbol = LD.findSymbolInternally(LMH, Name))
- return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
- Symbol.getFlags());
- return RuntimeDyld::SymbolInfo(nullptr);
- });
- std::vector<std::unique_ptr<Module>> PartMSet;
- PartMSet.push_back(std::move(PM));
- return LD.getBaseLayer().addModuleSet(std::move(PartMSet),
- std::move(MemMgr),
- std::move(Resolver));
+ void releaseResources(BaseLayerT &BaseLayer) {
+ for (auto LMH : LMHandles)
+ Lookup->removeLogicalModule(LMH);
+ for (auto H : BaseLayerModuleSetHandles)
+ BaseLayer.removeModuleSet(H);
}
-
- LogicalDylib &LD;
- typename LogicalDylib::LMHandle LMH;
- std::shared_ptr<Module> SrcM;
- typename LogicalDylib::UncompiledPartitionID ID;
- PartitionEntryList PartitionEntries;
};
- typedef std::list<std::unique_ptr<LogicalDylib>> LogicalDylibList;
+ typedef std::list<ModuleSetInfo> ModuleSetInfoListT;
public:
/// @brief Handle to a set of loaded modules.
- typedef typename LogicalDylibList::iterator ModuleSetHandleT;
+ typedef typename ModuleSetInfoListT::iterator ModuleSetHandleT;
/// @brief Construct a compile-on-demand layer instance.
CompileOnDemandLayer(BaseLayerT &BaseLayer, CompileCallbackMgrT &CallbackMgr)
assert(MemMgr == nullptr &&
"User supplied memory managers not supported with COD yet.");
- LogicalDylibs.push_back(createLogicalDylib(*this, std::move(Resolver)));
+ // Create a lookup context and ModuleSetInfo for this module set.
+ // For the purposes of symbol resolution the set Ms will be treated as if
+ // the modules it contained had been linked together as a dylib.
+ auto DylibLookup = createCODScopedLookup(BaseLayer, std::move(Resolver));
+ ModuleSetHandleT H =
+ ModuleSetInfos.insert(ModuleSetInfos.end(), ModuleSetInfo(DylibLookup));
+ ModuleSetInfo &MSI = ModuleSetInfos.back();
// Process each of the modules in this module set.
- for (auto &M : Ms) {
- std::vector<std::vector<Function*>> Partitioning;
- for (auto &F : *M) {
- if (F.isDeclaration())
- continue;
- Partitioning.push_back(std::vector<Function*>());
- Partitioning.back().push_back(&F);
- }
- addLogicalModule(*LogicalDylibs.back(),
- std::shared_ptr<Module>(std::move(M)),
- std::move(Partitioning));
- }
+ for (auto &M : Ms)
+ partitionAndAdd(*M, MSI);
- return std::prev(LogicalDylibs.end());
+ return H;
}
/// @brief Remove the module represented by the given handle.
/// This will remove all modules in the layers below that were derived from
/// the module represented by H.
void removeModuleSet(ModuleSetHandleT H) {
- LogicalDylibs.erase(H);
+ H->releaseResources(BaseLayer);
+ ModuleSetInfos.erase(H);
}
/// @brief Search for the given named symbol.
/// below this one.
JITSymbol findSymbolIn(ModuleSetHandleT H, const std::string &Name,
bool ExportedSymbolsOnly) {
- return (*H)->findSymbol(Name, ExportedSymbolsOnly);
+
+ for (auto &BH : H->BaseLayerModuleSetHandles) {
+ if (auto Symbol = BaseLayer.findSymbolIn(BH, Name, ExportedSymbolsOnly))
+ return Symbol;
+ }
+ return nullptr;
}
private:
- void addLogicalModule(LogicalDylib &LD, std::shared_ptr<Module> SrcM,
- std::vector<std::vector<Function*>> Partitions) {
-
- // Bump the linkage and rename any anonymous/privote members in SrcM to
- // ensure that everything will resolve properly after we partition SrcM.
- makeAllSymbolsExternallyAccessible(*SrcM);
-
- // Create a logical module handle for SrcM within the logical dylib.
- auto LMH = LD.createLogicalModule();
-
- // Create the GVs-and-stubs module.
- auto GVsAndStubsM = llvm::make_unique<Module>(
- (SrcM->getName() + ".globals_and_stubs").str(),
- SrcM->getContext());
- GVsAndStubsM->setDataLayout(SrcM->getDataLayout());
- ValueToValueMapTy VMap;
-
- // Process partitions and create stubs.
- // We create the stubs before copying the global variables as we know the
- // stubs won't refer to any globals (they only refer to their implementation
- // pointer) so there's no ordering/value-mapping issues.
- for (auto& Partition : Partitions) {
- auto &UP = LD.createUncompiledPartition(LMH, SrcM);
- typename UncompiledPartition::PartitionEntryList PartitionEntries;
- for (auto &F : Partition) {
- assert(!F->isDeclaration() &&
- "Partition should only contain definitions");
- unsigned FnIdx = PartitionEntries.size();
- auto CCI = CompileCallbackMgr.getCompileCallback(SrcM->getContext());
- PartitionEntries.push_back(
- typename UncompiledPartition::PartitionEntry(F, CCI.getAddress()));
- Function *StubF = cloneFunctionDecl(*GVsAndStubsM, *F, &VMap);
- GlobalVariable *FnBodyPtr =
- createImplPointer(*StubF->getType(), *StubF->getParent(),
- StubF->getName() + "$orc_addr",
- createIRTypedAddress(*StubF->getFunctionType(),
- CCI.getAddress()));
- makeStub(*StubF, *FnBodyPtr);
- CCI.setCompileAction([&UP, FnIdx]() { return UP.compile(FnIdx); });
+ void partitionAndAdd(Module &M, ModuleSetInfo &MSI) {
+ const char *AddrSuffix = "$orc_addr";
+ const char *BodySuffix = "$orc_body";
+
+ // We're going to break M up into a bunch of sub-modules, but we want
+ // internal linkage symbols to still resolve sensibly. CODScopedLookup
+ // provides the "logical module" concept to make this work, so create a
+ // new logical module for M.
+ auto DylibLookup = MSI.Lookup;
+ auto LogicalModule = DylibLookup->createLogicalModule();
+ MSI.LMHandles.push_back(LogicalModule);
+
+ // Partition M into a "globals and stubs" module, a "common symbols" module,
+ // and a list of single-function modules.
+ auto PartitionedModule = fullyPartition(M);
+ auto StubsModule = std::move(PartitionedModule.GlobalVars);
+ auto CommonsModule = std::move(PartitionedModule.Commons);
+ auto FunctionModules = std::move(PartitionedModule.Functions);
+
+ // Emit the commons stright away.
+ auto CommonHandle = addModule(std::move(CommonsModule), MSI, LogicalModule);
+ BaseLayer.emitAndFinalize(CommonHandle);
+
+ // Map of definition names to callback-info data structures. We'll use
+ // this to build the compile actions for the stubs below.
+ typedef std::map<std::string,
+ typename CompileCallbackMgrT::CompileCallbackInfo>
+ StubInfoMap;
+ StubInfoMap StubInfos;
+
+ // Now we need to take each of the extracted Modules and add them to
+ // base layer. Each Module will be added individually to make sure they
+ // can be compiled separately, and each will get its own lookaside
+ // memory manager that will resolve within this logical module first.
+ for (auto &SubM : FunctionModules) {
+
+ // Keep track of the stubs we create for this module so that we can set
+ // their compile actions.
+ std::vector<typename StubInfoMap::iterator> NewStubInfos;
+
+ // Search for function definitions and insert stubs into the stubs
+ // module.
+ for (auto &F : *SubM) {
+ if (F.isDeclaration())
+ continue;
+
+ std::string Name = F.getName();
+ Function *Proto = StubsModule->getFunction(Name);
+ assert(Proto && "Failed to clone function decl into stubs module.");
+ auto CallbackInfo =
+ CompileCallbackMgr.getCompileCallback(Proto->getContext());
+ GlobalVariable *FunctionBodyPointer =
+ createImplPointer(*Proto->getType(), *Proto->getParent(),
+ Name + AddrSuffix,
+ createIRTypedAddress(*Proto->getFunctionType(),
+ CallbackInfo.getAddress()));
+ makeStub(*Proto, *FunctionBodyPointer);
+
+ F.setName(Name + BodySuffix);
+ F.setVisibility(GlobalValue::HiddenVisibility);
+
+ auto KV = std::make_pair(std::move(Name), std::move(CallbackInfo));
+ NewStubInfos.push_back(StubInfos.insert(StubInfos.begin(), KV));
+ }
+
+ auto H = addModule(std::move(SubM), MSI, LogicalModule);
+
+ // Set the compile actions for this module:
+ for (auto &KVPair : NewStubInfos) {
+ std::string BodyName = Mangle(KVPair->first + BodySuffix,
+ M.getDataLayout());
+ auto &CCInfo = KVPair->second;
+ CCInfo.setCompileAction(
+ [=](){
+ return BaseLayer.findSymbolIn(H, BodyName, false).getAddress();
+ });
}
- UP.setPartitionEntries(std::move(PartitionEntries));
}
- // Now clone the global variable declarations.
- GlobalDeclMaterializer GDMat(*GVsAndStubsM);
- for (auto &GV : SrcM->globals())
- if (!GV.isDeclaration())
- cloneGlobalVariableDecl(*GVsAndStubsM, GV, &VMap);
-
- // Then clone the initializers.
- for (auto &GV : SrcM->globals())
- if (!GV.isDeclaration())
- moveGlobalVariableInitializer(GV, VMap, &GDMat);
-
- // Build a resolver for the stubs module and add it to the base layer.
- auto GVsAndStubsResolver = createLambdaResolver(
- [&LD](const std::string &Name) {
- if (auto Symbol = LD.findSymbol(Name, false))
+ // Ok - we've processed all the partitioned modules. Now add the
+ // stubs/globals module and set the update actions.
+ auto StubsH =
+ addModule(std::move(StubsModule), MSI, LogicalModule);
+
+ for (auto &KVPair : StubInfos) {
+ std::string AddrName = Mangle(KVPair.first + AddrSuffix,
+ M.getDataLayout());
+ auto &CCInfo = KVPair.second;
+ CCInfo.setUpdateAction(
+ getLocalFPUpdater(BaseLayer, StubsH, AddrName));
+ }
+ }
+
+ // Add the given Module to the base layer using a memory manager that will
+ // perform the appropriate scoped lookup (i.e. will look first with in the
+ // module from which it was extracted, then into the set to which that module
+ // belonged, and finally externally).
+ BaseLayerModuleSetHandleT addModule(
+ std::unique_ptr<Module> M,
+ ModuleSetInfo &MSI,
+ typename CODScopedLookup::LMHandle LogicalModule) {
+
+ // Add this module to the JIT with a memory manager that uses the
+ // DylibLookup to resolve symbols.
+ std::vector<std::unique_ptr<Module>> MSet;
+ MSet.push_back(std::move(M));
+
+ auto DylibLookup = MSI.Lookup;
+ auto Resolver =
+ createLambdaResolver(
+ [=](const std::string &Name) {
+ if (auto Symbol = DylibLookup->findSymbol(LogicalModule, Name))
return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
Symbol.getFlags());
- return LD.findSymbolExternally(Name);
+ return DylibLookup->externalLookup(Name);
},
- [&LD](const std::string &Name) {
- return RuntimeDyld::SymbolInfo(nullptr);
+ [=](const std::string &Name) -> RuntimeDyld::SymbolInfo {
+ if (auto Symbol = DylibLookup->findSymbol(LogicalModule, Name))
+ return RuntimeDyld::SymbolInfo(Symbol.getAddress(),
+ Symbol.getFlags());
+ return nullptr;
});
- std::vector<std::unique_ptr<Module>> GVsAndStubsMSet;
- GVsAndStubsMSet.push_back(std::move(GVsAndStubsM));
- auto GVsAndStubsH =
- BaseLayer.addModuleSet(std::move(GVsAndStubsMSet),
- llvm::make_unique<SectionMemoryManager>(),
- std::move(GVsAndStubsResolver));
- LD.setGVsAndStubsHandle(LMH, GVsAndStubsH);
+ BaseLayerModuleSetHandleT H =
+ BaseLayer.addModuleSet(std::move(MSet),
+ make_unique<SectionMemoryManager>(),
+ std::move(Resolver));
+ // Add this module to the logical module lookup.
+ DylibLookup->addToLogicalModule(LogicalModule, H);
+ MSI.BaseLayerModuleSetHandles.push_back(H);
+
+ return H;
}
static std::string Mangle(StringRef Name, const DataLayout &DL) {
BaseLayerT &BaseLayer;
CompileCallbackMgrT &CompileCallbackMgr;
- LogicalDylibList LogicalDylibs;
+ ModuleSetInfoListT ModuleSetInfos;
};
-template <typename BaseLayerT, typename CompileCallbackMgrT>
-typename CompileOnDemandLayer<BaseLayerT, CompileCallbackMgrT>::
- UncompiledPartition&
-CompileOnDemandLayer<BaseLayerT, CompileCallbackMgrT>::LogicalDylib::
- createUncompiledPartition(LMHandle LMH, std::shared_ptr<Module> SrcM) {
- UncompiledPartitions.push_back(
- llvm::make_unique<UncompiledPartition>(*this, LMH, std::move(SrcM)));
- UncompiledPartitions.back()->setID(UncompiledPartitions.size() - 1);
- return *UncompiledPartitions.back();
-}
-
-template <typename BaseLayerT, typename CompileCallbackMgrT>
-std::unique_ptr<typename CompileOnDemandLayer<BaseLayerT, CompileCallbackMgrT>::
- UncompiledPartition>
-CompileOnDemandLayer<BaseLayerT, CompileCallbackMgrT>::LogicalDylib::
- takeUPOwnership(UncompiledPartitionID ID) {
-
- std::swap(UncompiledPartitions[ID], UncompiledPartitions.back());
- UncompiledPartitions[ID]->setID(ID);
- auto UP = std::move(UncompiledPartitions.back());
- UncompiledPartitions.pop_back();
- return UP;
-}
-
} // End namespace orc.
} // End namespace llvm.
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Mangler.h"
#include "llvm/IR/Module.h"
-#include "llvm/Transforms/Utils/ValueMapper.h"
#include <sstream>
namespace llvm {
public:
typedef std::function<TargetAddress()> CompileFtor;
+ typedef std::function<void(TargetAddress)> UpdateFtor;
/// @brief Handle to a newly created compile callback. Can be used to get an
/// IR constant representing the address of the trampoline, and to set
- /// the compile action for the callback.
+ /// the compile and update actions for the callback.
class CompileCallbackInfo {
public:
- CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile)
- : Addr(Addr), Compile(Compile) {}
+ CompileCallbackInfo(TargetAddress Addr, CompileFtor &Compile,
+ UpdateFtor &Update)
+ : Addr(Addr), Compile(Compile), Update(Update) {}
TargetAddress getAddress() const { return Addr; }
void setCompileAction(CompileFtor Compile) {
this->Compile = std::move(Compile);
}
+ void setUpdateAction(UpdateFtor Update) {
+ this->Update = std::move(Update);
+ }
private:
TargetAddress Addr;
CompileFtor &Compile;
+ UpdateFtor &Update;
};
/// @brief Construct a JITCompileCallbackManagerBase.
/// @brief Execute the callback for the given trampoline id. Called by the JIT
/// to compile functions on demand.
- TargetAddress executeCompileCallback(TargetAddress TrampolineAddr) {
- auto I = ActiveTrampolines.find(TrampolineAddr);
+ TargetAddress executeCompileCallback(TargetAddress TrampolineID) {
+ TrampolineMapT::iterator I = ActiveTrampolines.find(TrampolineID);
// FIXME: Also raise an error in the Orc error-handler when we finally have
// one.
if (I == ActiveTrampolines.end())
// Moving the trampoline ID back to the available list first means there's at
// least one available trampoline if the compile action triggers a request for
// a new one.
- auto Compile = std::move(I->second);
+ AvailableTrampolines.push_back(I->first);
+ auto CallbackHandler = std::move(I->second);
ActiveTrampolines.erase(I);
- AvailableTrampolines.push_back(TrampolineAddr);
- if (auto Addr = Compile())
+ if (auto Addr = CallbackHandler.Compile()) {
+ CallbackHandler.Update(Addr);
return Addr;
-
+ }
return ErrorHandlerAddress;
}
- /// @brief Reserve a compile callback.
+ /// @brief Get/create a compile callback with the given signature.
virtual CompileCallbackInfo getCompileCallback(LLVMContext &Context) = 0;
- /// @brief Get a CompileCallbackInfo for an existing callback.
- CompileCallbackInfo getCompileCallbackInfo(TargetAddress TrampolineAddr) {
- auto I = ActiveTrampolines.find(TrampolineAddr);
- assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
- return CompileCallbackInfo(I->first, I->second);
- }
+protected:
- /// @brief Release a compile callback.
- ///
- /// Note: Callbacks are auto-released after they execute. This method should
- /// only be called to manually release a callback that is not going to
- /// execute.
- void releaseCompileCallback(TargetAddress TrampolineAddr) {
- auto I = ActiveTrampolines.find(TrampolineAddr);
- assert(I != ActiveTrampolines.end() && "Not an active trampoline.");
- ActiveTrampolines.erase(I);
- AvailableTrampolines.push_back(TrampolineAddr);
- }
+ struct CallbackHandler {
+ CompileFtor Compile;
+ UpdateFtor Update;
+ };
-protected:
TargetAddress ErrorHandlerAddress;
unsigned NumTrampolinesPerBlock;
- typedef std::map<TargetAddress, CompileFtor> TrampolineMapT;
+ typedef std::map<TargetAddress, CallbackHandler> TrampolineMapT;
TrampolineMapT ActiveTrampolines;
std::vector<TargetAddress> AvailableTrampolines;
};
/// @brief Get/create a compile callback with the given signature.
CompileCallbackInfo getCompileCallback(LLVMContext &Context) final {
TargetAddress TrampolineAddr = getAvailableTrampolineAddr(Context);
- auto &Compile = this->ActiveTrampolines[TrampolineAddr];
- return CompileCallbackInfo(TrampolineAddr, Compile);
+ auto &CallbackHandler =
+ this->ActiveTrampolines[TrampolineAddr];
+
+ return CompileCallbackInfo(TrampolineAddr, CallbackHandler.Compile,
+ CallbackHandler.Update);
}
private:
TargetAddress ResolverBlockAddr;
};
+/// @brief Get an update functor that updates the value of a named function
+/// pointer.
+template <typename JITLayerT>
+JITCompileCallbackManagerBase::UpdateFtor
+getLocalFPUpdater(JITLayerT &JIT, typename JITLayerT::ModuleSetHandleT H,
+ std::string Name) {
+ // FIXME: Move-capture Name once we can use C++14.
+ return [=,&JIT](TargetAddress Addr) {
+ auto FPSym = JIT.findSymbolIn(H, Name, true);
+ assert(FPSym && "Cannot find function pointer to update.");
+ void *FPAddr = reinterpret_cast<void*>(
+ static_cast<uintptr_t>(FPSym.getAddress()));
+ memcpy(FPAddr, &Addr, sizeof(uintptr_t));
+ };
+ }
+
/// @brief Build a function pointer of FunctionType with the given constant
/// address.
///
/// indirect call using the given function pointer.
void makeStub(Function &F, GlobalVariable &ImplPointer);
-/// @brief Raise linkage types and rename as necessary to ensure that all
-/// symbols are accessible for other modules.
-///
-/// This should be called before partitioning a module to ensure that the
-/// partitions retain access to each other's symbols.
-void makeAllSymbolsExternallyAccessible(Module &M);
+typedef std::map<Module*, DenseSet<const GlobalValue*>> ModulePartitionMap;
-/// @brief Clone a function declaration into a new module.
-///
-/// This function can be used as the first step towards creating a callback
-/// stub (see makeStub), or moving a function body (see moveFunctionBody).
-///
-/// If the VMap argument is non-null, a mapping will be added between F and
-/// the new declaration, and between each of F's arguments and the new
-/// declaration's arguments. This map can then be passed in to moveFunction to
-/// move the function body if required. Note: When moving functions between
-/// modules with these utilities, all decls should be cloned (and added to a
-/// single VMap) before any bodies are moved. This will ensure that references
-/// between functions all refer to the versions in the new module.
-Function* cloneFunctionDecl(Module &Dst, const Function &F,
- ValueToValueMapTy *VMap = nullptr);
-
-/// @brief Move the body of function 'F' to a cloned function declaration in a
-/// different module (See related cloneFunctionDecl).
-///
-/// If the target function declaration is not supplied via the NewF parameter
-/// then it will be looked up via the VMap.
-///
-/// This will delete the body of function 'F' from its original parent module,
-/// but leave its declaration.
-void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
- ValueMaterializer *Materializer = nullptr,
- Function *NewF = nullptr);
-
-/// @brief Clone a global variable declaration into a new module.
-GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
- ValueToValueMapTy *VMap = nullptr);
-
-/// @brief Move global variable GV from its parent module to cloned global
-/// declaration in a different module.
-///
-/// If the target global declaration is not supplied via the NewGV parameter
-/// then it will be looked up via the VMap.
-///
-/// This will delete the initializer of GV from its original parent module,
-/// but leave its declaration.
-void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
- ValueToValueMapTy &VMap,
- ValueMaterializer *Materializer = nullptr,
- GlobalVariable *NewGV = nullptr);
+/// @brief Extract subsections of a Module into the given Module according to
+/// the given ModulePartitionMap.
+void partition(Module &M, const ModulePartitionMap &PMap);
+
+/// @brief Struct for trivial "complete" partitioning of a module.
+class FullyPartitionedModule {
+public:
+ std::unique_ptr<Module> GlobalVars;
+ std::unique_ptr<Module> Commons;
+ std::vector<std::unique_ptr<Module>> Functions;
+
+ FullyPartitionedModule() = default;
+ FullyPartitionedModule(FullyPartitionedModule &&S)
+ : GlobalVars(std::move(S.GlobalVars)), Commons(std::move(S.Commons)),
+ Functions(std::move(S.Functions)) {}
+};
+
+/// @brief Extract every function in M into a separate module.
+FullyPartitionedModule fullyPartition(Module &M);
} // End namespace orc.
} // End namespace llvm.
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/ExecutionEngine/Orc/CloneSubModule.h"
#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/IRBuilder.h"
-#include "llvm/Transforms/Utils/Cloning.h"
#include <set>
#include <sstream>
const Twine &Name, Constant *Initializer) {
if (!Initializer)
Initializer = Constant::getNullValue(&PT);
- auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
- Initializer, Name, nullptr,
- GlobalValue::NotThreadLocal, 0, true);
- IP->setVisibility(GlobalValue::HiddenVisibility);
- return IP;
+ return new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
+ Initializer, Name, nullptr,
+ GlobalValue::NotThreadLocal, 0, true);
}
void makeStub(Function &F, GlobalVariable &ImplPointer) {
CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs);
Call->setTailCall();
Call->setAttributes(F.getAttributes());
- if (F.getReturnType()->isVoidTy())
- Builder.CreateRetVoid();
- else
- Builder.CreateRet(Call);
+ Builder.CreateRet(Call);
}
// Utility class for renaming global values and functions during partitioning.
DenseMap<const Value*, std::string> Names;
};
-static void raiseVisibilityOnValue(GlobalValue &V, GlobalRenamer &R) {
- if (V.hasLocalLinkage()) {
- if (R.needsRenaming(V))
- V.setName(R.getRename(V));
- V.setLinkage(GlobalValue::ExternalLinkage);
- V.setVisibility(GlobalValue::HiddenVisibility);
+void partition(Module &M, const ModulePartitionMap &PMap) {
+
+ GlobalRenamer Renamer;
+
+ for (auto &KVPair : PMap) {
+
+ auto ExtractGlobalVars =
+ [&](GlobalVariable &New, const GlobalVariable &Orig,
+ ValueToValueMapTy &VMap) {
+ if (KVPair.second.count(&Orig)) {
+ copyGVInitializer(New, Orig, VMap);
+ }
+ if (New.hasLocalLinkage()) {
+ if (Renamer.needsRenaming(New))
+ New.setName(Renamer.getRename(Orig));
+ New.setLinkage(GlobalValue::ExternalLinkage);
+ New.setVisibility(GlobalValue::HiddenVisibility);
+ }
+ assert(!Renamer.needsRenaming(New) && "Invalid global name.");
+ };
+
+ auto ExtractFunctions =
+ [&](Function &New, const Function &Orig, ValueToValueMapTy &VMap) {
+ if (KVPair.second.count(&Orig))
+ copyFunctionBody(New, Orig, VMap);
+ if (New.hasLocalLinkage()) {
+ if (Renamer.needsRenaming(New))
+ New.setName(Renamer.getRename(Orig));
+ New.setLinkage(GlobalValue::ExternalLinkage);
+ New.setVisibility(GlobalValue::HiddenVisibility);
+ }
+ assert(!Renamer.needsRenaming(New) && "Invalid function name.");
+ };
+
+ CloneSubModule(*KVPair.first, M, ExtractGlobalVars, ExtractFunctions,
+ false);
}
- V.setUnnamedAddr(false);
- assert(!R.needsRenaming(V) && "Invalid global name.");
}
-void makeAllSymbolsExternallyAccessible(Module &M) {
- GlobalRenamer Renamer;
+FullyPartitionedModule fullyPartition(Module &M) {
+ FullyPartitionedModule MP;
- for (auto &F : M)
- raiseVisibilityOnValue(F, Renamer);
+ ModulePartitionMap PMap;
- for (auto &GV : M.globals())
- raiseVisibilityOnValue(GV, Renamer);
-}
+ for (auto &F : M) {
-Function* cloneFunctionDecl(Module &Dst, const Function &F,
- ValueToValueMapTy *VMap) {
- assert(F.getParent() != &Dst && "Can't copy decl over existing function.");
- Function *NewF =
- Function::Create(cast<FunctionType>(F.getType()->getElementType()),
- F.getLinkage(), F.getName(), &Dst);
- NewF->copyAttributesFrom(&F);
-
- if (VMap) {
- (*VMap)[&F] = NewF;
- auto NewArgI = NewF->arg_begin();
- for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
- ++ArgI, ++NewArgI)
- (*VMap)[ArgI] = NewArgI;
+ if (F.isDeclaration())
+ continue;
+
+ std::string NewModuleName = (M.getName() + "." + F.getName()).str();
+ MP.Functions.push_back(
+ llvm::make_unique<Module>(NewModuleName, M.getContext()));
+ MP.Functions.back()->setDataLayout(M.getDataLayout());
+ PMap[MP.Functions.back().get()].insert(&F);
}
- return NewF;
-}
+ MP.GlobalVars =
+ llvm::make_unique<Module>((M.getName() + ".globals_and_stubs").str(),
+ M.getContext());
+ MP.GlobalVars->setDataLayout(M.getDataLayout());
-void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
- ValueMaterializer *Materializer,
- Function *NewF) {
- assert(!OrigF.isDeclaration() && "Nothing to move");
- if (!NewF)
- NewF = cast<Function>(VMap[&OrigF]);
- else
- assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
- assert(NewF && "Function mapping missing from VMap.");
- assert(NewF->getParent() != OrigF.getParent() &&
- "moveFunctionBody should only be used to move bodies between "
- "modules.");
-
- SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
- CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns,
- "", nullptr, nullptr, Materializer);
- OrigF.deleteBody();
-}
+ MP.Commons =
+ llvm::make_unique<Module>((M.getName() + ".commons").str(), M.getContext());
+ MP.Commons->setDataLayout(M.getDataLayout());
-GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
- ValueToValueMapTy *VMap) {
- assert(GV.getParent() != &Dst && "Can't copy decl over existing global var.");
- GlobalVariable *NewGV = new GlobalVariable(
- Dst, GV.getType()->getElementType(), GV.isConstant(),
- GV.getLinkage(), nullptr, GV.getName(), nullptr,
- GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
- NewGV->copyAttributesFrom(&GV);
- if (VMap)
- (*VMap)[&GV] = NewGV;
- return NewGV;
-}
+ // Make sure there's at least an empty set for the stubs map or we'll fail
+ // to clone anything for it (including the decls).
+ PMap[MP.GlobalVars.get()] = ModulePartitionMap::mapped_type();
+ for (auto &GV : M.globals())
+ if (GV.getLinkage() == GlobalValue::CommonLinkage)
+ PMap[MP.Commons.get()].insert(&GV);
+ else
+ PMap[MP.GlobalVars.get()].insert(&GV);
+
+ partition(M, PMap);
-void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
- ValueToValueMapTy &VMap,
- ValueMaterializer *Materializer,
- GlobalVariable *NewGV) {
- assert(OrigGV.hasInitializer() && "Nothing to move");
- if (!NewGV)
- NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
- else
- assert(VMap[&OrigGV] == NewGV &&
- "Incorrect global variable mapping in VMap.");
- assert(NewGV->getParent() != OrigGV.getParent() &&
- "moveGlobalVariable should only be used to move initializers between "
- "modules");
-
- NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
- nullptr, Materializer));
+ return MP;
}
} // End namespace orc.
projects / platform / upstream / llvm.git / commitdiff