From bb5f97e3ad10a0f8a62560890e5a87b4bc2c00bd Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Wed, 14 Jul 2021 21:09:36 +1000 Subject: [PATCH] [ORC][ORC-RT] Introduce ORC-runtime based MachO-Platform. Adds support for MachO static initializers/deinitializers and eh-frame registration via the ORC runtime. This commit introduces cooperative support code into the ORC runtime and ORC LLVM libraries (especially the MachOPlatform class) to support macho runtime features for JIT'd code. This commit introduces support for static initializers, static destructors (via cxa_atexit interposition), and eh-frame registration. Near-future commits will add support for MachO native thread-local variables, and language runtime registration (e.g. for Objective-C and Swift). The llvm-jitlink tool is updated to use the ORC runtime where available, and regression tests for the new MachOPlatform support are added to compiler-rt. Notable changes on the ORC runtime side: 1. The new macho_platform.h / macho_platform.cpp files contain the bulk of the runtime-side support. This includes eh-frame registration; jit versions of dlopen, dlsym, and dlclose; a cxa_atexit interpose to record static destructors, and an '__orc_rt_macho_run_program' function that defines running a JIT'd MachO program in terms of the jit- dlopen/dlsym/dlclose functions. 2. Replaces JITTargetAddress (and casting operations) with ExecutorAddress (copied from LLVM) to improve type-safety of address management. 3. Adds serialization support for ExecutorAddress and unordered_map types to the runtime-side Simple Packed Serialization code. 4. Adds orc-runtime regression tests to ensure that static initializers and cxa-atexit interposes work as expected. Notable changes on the LLVM side: 1. The MachOPlatform class is updated to: 1.1. Load the ORC runtime into the ExecutionSession. 1.2. Set up standard aliases for macho-specific runtime functions. E.g. ___cxa_atexit -> ___orc_rt_macho_cxa_atexit. 1.3. Install the MachOPlatformPlugin to scrape LinkGraphs for information needed to support MachO features (e.g. eh-frames, mod-inits), and communicate this information to the runtime. 1.4. Provide entry-points that the runtime can call to request initializers, perform symbol lookup, and request deinitialiers (the latter is implemented as an empty placeholder as macho object deinits are rarely used). 1.5. Create a MachO header object for each JITDylib (defining the __mh_header and __dso_handle symbols). 2. The llvm-jitlink tool (and llvm-jitlink-executor) are updated to use the runtime when available. 3. A `lookupInitSymbolsAsync` method is added to the Platform base class. This can be used to issue an async lookup for initializer symbols. The existing `lookupInitSymbols` method is retained (the GenericIRPlatform code is still using it), but is deprecated and will be removed soon. 4. JIT-dispatch support code is added to ExecutorProcessControl. The JIT-dispatch system allows handlers in the JIT process to be associated with 'tag' symbols in the executor, and allows the executor to make remote procedure calls back to the JIT process (via __orc_rt_jit_dispatch) using those tags. The primary use case is ORC runtime code that needs to call bakc to handlers in orc::Platform subclasses. E.g. __orc_rt_macho_jit_dlopen calling back to MachOPlatform::rt_getInitializers using __orc_rt_macho_get_initializers_tag. (The system is generic however, and could be used by non-runtime code). The new ExecutorProcessControl::JITDispatchInfo struct provides the address (in the executor) of the jit-dispatch function and a jit-dispatch context object, and implementations of the dispatch function are added to SelfExecutorProcessControl and OrcRPCExecutorProcessControl. 5. OrcRPCTPCServer is updated to support JIT-dispatch calls over ORC-RPC. 6. Serialization support for StringMap is added to the LLVM-side Simple Packed Serialization code. 7. A JITLink::allocateBuffer operation is introduced to allocate writable memory attached to the graph. This is used by the MachO header synthesis code, and will be generically useful for other clients who want to create new graph content from scratch. --- compiler-rt/lib/orc/CMakeLists.txt | 5 + compiler-rt/lib/orc/common.h | 18 - compiler-rt/lib/orc/executor_address.h | 208 +++++ compiler-rt/lib/orc/log_error_to_stderr.cpp | 19 + compiler-rt/lib/orc/macho_platform.cpp | 475 ++++++++++++ compiler-rt/lib/orc/macho_platform.h | 129 ++++ compiler-rt/lib/orc/run_program_wrapper.cpp | 51 ++ compiler-rt/lib/orc/simple_packed_serialization.h | 41 +- compiler-rt/lib/orc/wrapper_function_utils.h | 4 +- .../orc/TestCases/Darwin/x86-64/placeholder_test.S | 13 - .../TestCases/Darwin/x86-64/trivial-cxa-atexit.S | 38 + .../Darwin/x86-64/trivial-static-initializer.S | 37 + .../include/llvm/ExecutionEngine/JITLink/JITLink.h | 6 + llvm/include/llvm/ExecutionEngine/Orc/Core.h | 10 + .../ExecutionEngine/Orc/ExecutorProcessControl.h | 36 +- .../llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h | 69 ++ .../llvm/ExecutionEngine/Orc/MachOPlatform.h | 295 ++++++-- .../Orc/OrcRPCExecutorProcessControl.h | 38 +- .../Orc/Shared/SimplePackedSerialization.h | 1 + .../Orc/Shared/WrapperFunctionUtils.h | 6 +- .../Orc/TargetProcess/OrcRPCTPCServer.h | 126 +++- llvm/lib/ExecutionEngine/Orc/Core.cpp | 43 ++ .../ExecutionEngine/Orc/ExecutorProcessControl.cpp | 29 + llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp | 839 +++++++++++++++------ .../llvm-jitlink-executor.cpp | 2 +- llvm/tools/llvm-jitlink/llvm-jitlink.cpp | 231 ++++-- llvm/tools/llvm-jitlink/llvm-jitlink.h | 2 +- .../Orc/SimplePackedSerializationTest.cpp | 7 + 28 files changed, 2323 insertions(+), 455 deletions(-) create mode 100644 compiler-rt/lib/orc/executor_address.h create mode 100644 compiler-rt/lib/orc/log_error_to_stderr.cpp create mode 100644 compiler-rt/lib/orc/macho_platform.cpp create mode 100644 compiler-rt/lib/orc/macho_platform.h create mode 100644 compiler-rt/lib/orc/run_program_wrapper.cpp delete mode 100644 compiler-rt/test/orc/TestCases/Darwin/x86-64/placeholder_test.S create mode 100644 compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-cxa-atexit.S create mode 100644 compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-static-initializer.S create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt index 417033a..0a83787 100644 --- a/compiler-rt/lib/orc/CMakeLists.txt +++ b/compiler-rt/lib/orc/CMakeLists.txt @@ -3,6 +3,9 @@ # ORC runtime library implementation files. set(ORC_SOURCES extensible_rtti.cpp + log_error_to_stderr.cpp + macho_platform.cpp + run_program_wrapper.cpp ) # Implementation files for all ORC architectures. @@ -18,7 +21,9 @@ set(ORC_IMPL_HEADERS compiler.h endianness.h error.h + executor_address.h extensible_rtti.h + macho_platform.h simple_packed_serialization.h stl_extras.h wrapper_function_utils.h diff --git a/compiler-rt/lib/orc/common.h b/compiler-rt/lib/orc/common.h index 62ff12a..59c1a54 100644 --- a/compiler-rt/lib/orc/common.h +++ b/compiler-rt/lib/orc/common.h @@ -40,22 +40,4 @@ __orc_rt_jit_dispatch(__orc_rt_Opaque *DispatchCtx, const void *FnTag, const char *Data, size_t Size) __attribute__((weak_import)); -namespace __orc_rt { - -/// Must be kept in sync with JITSymbol.h -using JITTargetAddress = uint64_t; - -/// Cast from JITTargetAddress to pointer. -template T jitTargetAddressToPointer(JITTargetAddress Addr) { - static_assert(std::is_pointer::value, "T must be a pointer type"); - return reinterpret_cast(static_cast(Addr)); -} - -/// Cast from pointer to JITTargetAddress. -template JITTargetAddress pointerToJITTargetAddress(T *Ptr) { - return static_cast(reinterpret_cast(Ptr)); -} - -} // end namespace __orc_rt - #endif // ORC_RT_COMMON_H diff --git a/compiler-rt/lib/orc/executor_address.h b/compiler-rt/lib/orc/executor_address.h new file mode 100644 index 0000000..cfe985b --- /dev/null +++ b/compiler-rt/lib/orc/executor_address.h @@ -0,0 +1,208 @@ +//===------ ExecutorAddress.h - Executing process address -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Represents an address in the executing program. +// +// This file was derived from +// llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_EXECUTOR_ADDRESS_H +#define ORC_RT_EXECUTOR_ADDRESS_H + +#include "adt.h" +#include "simple_packed_serialization.h" + +#include +#include + +namespace __orc_rt { + +/// Represents the difference between two addresses in the executor process. +class ExecutorAddrDiff { +public: + ExecutorAddrDiff() = default; + explicit ExecutorAddrDiff(uint64_t Value) : Value(Value) {} + + uint64_t getValue() const { return Value; } + +private: + int64_t Value = 0; +}; + +/// Represents an address in the executor process. +class ExecutorAddress { +public: + ExecutorAddress() = default; + explicit ExecutorAddress(uint64_t Addr) : Addr(Addr) {} + + /// Create an ExecutorAddress from the given pointer. + /// Warning: This should only be used when JITing in-process. + template static ExecutorAddress fromPtr(T *Value) { + return ExecutorAddress( + static_cast(reinterpret_cast(Value))); + } + + /// Cast this ExecutorAddress to a pointer of the given type. + /// Warning: This should only be esude when JITing in-process. + template T toPtr() const { + static_assert(std::is_pointer::value, "T must be a pointer type"); + uintptr_t IntPtr = static_cast(Addr); + assert(IntPtr == Addr && + "JITTargetAddress value out of range for uintptr_t"); + return reinterpret_cast(IntPtr); + } + + uint64_t getValue() const { return Addr; } + void setValue(uint64_t Addr) { this->Addr = Addr; } + bool isNull() const { return Addr == 0; } + + explicit operator bool() const { return Addr != 0; } + + friend bool operator==(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr == RHS.Addr; + } + + friend bool operator!=(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr != RHS.Addr; + } + + friend bool operator<(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr < RHS.Addr; + } + + friend bool operator<=(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr <= RHS.Addr; + } + + friend bool operator>(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr > RHS.Addr; + } + + friend bool operator>=(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return LHS.Addr >= RHS.Addr; + } + + ExecutorAddress &operator++() { + ++Addr; + return *this; + } + ExecutorAddress &operator--() { + --Addr; + return *this; + } + ExecutorAddress operator++(int) { return ExecutorAddress(Addr++); } + ExecutorAddress operator--(int) { return ExecutorAddress(Addr++); } + + ExecutorAddress &operator+=(const ExecutorAddrDiff Delta) { + Addr += Delta.getValue(); + return *this; + } + + ExecutorAddress &operator-=(const ExecutorAddrDiff Delta) { + Addr -= Delta.getValue(); + return *this; + } + +private: + uint64_t Addr = 0; +}; + +/// Subtracting two addresses yields an offset. +inline ExecutorAddrDiff operator-(const ExecutorAddress &LHS, + const ExecutorAddress &RHS) { + return ExecutorAddrDiff(LHS.getValue() - RHS.getValue()); +} + +/// Adding an offset and an address yields an address. +inline ExecutorAddress operator+(const ExecutorAddress &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddress(LHS.getValue() + RHS.getValue()); +} + +/// Adding an address and an offset yields an address. +inline ExecutorAddress operator+(const ExecutorAddrDiff &LHS, + const ExecutorAddress &RHS) { + return ExecutorAddress(LHS.getValue() + RHS.getValue()); +} + +/// Represents an address range in the exceutor process. +struct ExecutorAddressRange { + ExecutorAddressRange() = default; + ExecutorAddressRange(ExecutorAddress StartAddress, ExecutorAddress EndAddress) + : StartAddress(StartAddress), EndAddress(EndAddress) {} + + bool empty() const { return StartAddress == EndAddress; } + ExecutorAddrDiff size() const { return EndAddress - StartAddress; } + + template span toSpan() const { + assert(size().getValue() % sizeof(T) == 0 && + "AddressRange is not a multiple of sizeof(T)"); + return span(StartAddress.toPtr(), size().getValue() / sizeof(T)); + } + + ExecutorAddress StartAddress; + ExecutorAddress EndAddress; +}; + +/// SPS serializatior for ExecutorAddress. +template <> class SPSSerializationTraits { +public: + static size_t size(const ExecutorAddress &EA) { + return SPSArgList::size(EA.getValue()); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddress &EA) { + return SPSArgList::serialize(BOB, EA.getValue()); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddress &EA) { + uint64_t Tmp; + if (!SPSArgList::deserialize(BIB, Tmp)) + return false; + EA = ExecutorAddress(Tmp); + return true; + } +}; + +using SPSExecutorAddressRange = + SPSTuple; + +/// Serialization traits for address ranges. +template <> +class SPSSerializationTraits { +public: + static size_t size(const ExecutorAddressRange &Value) { + return SPSArgList::size( + Value.StartAddress, Value.EndAddress); + } + + static bool serialize(SPSOutputBuffer &BOB, + const ExecutorAddressRange &Value) { + return SPSArgList::serialize( + BOB, Value.StartAddress, Value.EndAddress); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddressRange &Value) { + return SPSArgList::deserialize( + BIB, Value.StartAddress, Value.EndAddress); + } +}; + +using SPSExecutorAddressRangeSequence = SPSSequence; + +} // End namespace __orc_rt + +#endif // ORC_RT_EXECUTOR_ADDRESS_H diff --git a/compiler-rt/lib/orc/log_error_to_stderr.cpp b/compiler-rt/lib/orc/log_error_to_stderr.cpp new file mode 100644 index 0000000..4fabbc0 --- /dev/null +++ b/compiler-rt/lib/orc/log_error_to_stderr.cpp @@ -0,0 +1,19 @@ +//===-- log_error_to_stderr.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "compiler.h" + +#include + +ORC_RT_INTERFACE void __orc_rt_log_error_to_stderr(const char *ErrMsg) { + fprintf(stderr, "orc runtime error: %s\n", ErrMsg); +} diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp new file mode 100644 index 0000000..cd6fbcf --- /dev/null +++ b/compiler-rt/lib/orc/macho_platform.cpp @@ -0,0 +1,475 @@ +//===- macho_platform.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code required to load the rest of the MachO runtime. +// +//===----------------------------------------------------------------------===// + +#include "macho_platform.h" +#include "common.h" +#include "error.h" +#include "wrapper_function_utils.h" + +#include +#include +#include +#include +#include + +using namespace __orc_rt; +using namespace __orc_rt::macho; + +// Declare function tags for functions in the JIT process. +extern "C" char __orc_rt_macho_get_initializers_tag = 0; +extern "C" char __orc_rt_macho_get_deinitializers_tag = 0; +extern "C" char __orc_rt_macho_symbol_lookup_tag = 0; + +// eh-frame registration functions. +// We expect these to be available for all processes. +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +namespace { + +template +void walkEHFrameSection(span EHFrameSection, + HandleFDEFn HandleFDE) { + const char *CurCFIRecord = EHFrameSection.data(); + uint64_t Size = *reinterpret_cast(CurCFIRecord); + + while (CurCFIRecord != EHFrameSection.end() && Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast(OffsetField); + + if (Offset != 0) + HandleFDE(CurCFIRecord); + + CurCFIRecord += Size; + Size = *reinterpret_cast(CurCFIRecord); + } +} + +Error validatePointerSectionExtent(const char *SectionName, + const ExecutorAddressRange &SE) { + if (SE.size().getValue() % sizeof(uintptr_t)) { + std::ostringstream ErrMsg; + ErrMsg << std::hex << "Size of " << SectionName << " 0x" + << SE.StartAddress.getValue() << " -- 0x" << SE.EndAddress.getValue() + << " is not a pointer multiple"; + return make_error(ErrMsg.str()); + } + return Error::success(); +} + +Error runModInits(const std::vector &ModInitsSections, + const MachOJITDylibInitializers &MOJDIs) { + + for (const auto &ModInits : ModInitsSections) { + if (auto Err = validatePointerSectionExtent("__mod_inits", ModInits)) + return Err; + + using InitFunc = void (*)(); + for (auto *Init : ModInits.toSpan()) + (*Init)(); + } + + return Error::success(); +} + +class MachOPlatformRuntimeState { +private: + struct AtExitEntry { + void (*Func)(void *); + void *Arg; + }; + + using AtExitsVector = std::vector; + + struct PerJITDylibState { + void *Header = nullptr; + size_t RefCount = 0; + bool AllowReinitialization = false; + AtExitsVector AtExits; + }; + +public: + static void initialize(); + static MachOPlatformRuntimeState &get(); + static void destroy(); + + MachOPlatformRuntimeState() = default; + + // Delete copy and move constructors. + MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete; + MachOPlatformRuntimeState & + operator=(const MachOPlatformRuntimeState &) = delete; + MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; + MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; + + Error registerObjectSections(MachOPerObjectSectionsToRegister POSR); + Error deregisterObjectSections(MachOPerObjectSectionsToRegister POSR); + + const char *dlerror(); + void *dlopen(string_view Name, int Mode); + int dlclose(void *DSOHandle); + void *dlsym(void *DSOHandle, string_view Symbol); + + int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); + void runAtExits(void *DSOHandle); + +private: + PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); + PerJITDylibState *getJITDylibStateByName(string_view Path); + PerJITDylibState &getOrCreateJITDylibState(MachOJITDylibInitializers &MOJDIs); + + Expected lookupSymbolInJITDylib(void *DSOHandle, + string_view Symbol); + + Expected + getJITDylibInitializersByName(string_view Path); + Expected dlopenInitialize(string_view Path, int Mode); + Error initializeJITDylib(MachOJITDylibInitializers &MOJDIs); + + static MachOPlatformRuntimeState *MOPS; + + using InitSectionHandler = + Error (*)(const std::vector &Sections, + const MachOJITDylibInitializers &MOJDIs); + const std::vector> InitSections = { + {"__DATA,__mod_init_func", runModInits}}; + + // FIXME: Move to thread-state. + std::string DLFcnError; + + std::recursive_mutex JDStatesMutex; + std::unordered_map JDStates; + std::unordered_map JDNameToHeader; +}; + +MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr; + +void MachOPlatformRuntimeState::initialize() { + assert(!MOPS && "MachOPlatformRuntimeState should be null"); + MOPS = new MachOPlatformRuntimeState(); +} + +MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() { + assert(MOPS && "MachOPlatformRuntimeState not initialized"); + return *MOPS; +} + +void MachOPlatformRuntimeState::destroy() { + assert(MOPS && "MachOPlatformRuntimeState not initialized"); + delete MOPS; +} + +Error MachOPlatformRuntimeState::registerObjectSections( + MachOPerObjectSectionsToRegister POSR) { + if (POSR.EHFrameSection.StartAddress) + walkEHFrameSection(POSR.EHFrameSection.toSpan(), + __register_frame); + + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterObjectSections( + MachOPerObjectSectionsToRegister POSR) { + if (POSR.EHFrameSection.StartAddress) + walkEHFrameSection(POSR.EHFrameSection.toSpan(), + __deregister_frame); + + return Error::success(); +} + +const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } + +void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) { + std::lock_guard Lock(JDStatesMutex); + + // Use fast path if all JITDylibs are already loaded and don't require + // re-running initializers. + if (auto *JDS = getJITDylibStateByName(Path)) { + if (!JDS->AllowReinitialization) { + ++JDS->RefCount; + return JDS->Header; + } + } + + auto H = dlopenInitialize(Path, Mode); + if (!H) { + DLFcnError = toString(H.takeError()); + return nullptr; + } + + return *H; +} + +int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { + runAtExits(DSOHandle); + return 0; +} + +void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) { + auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); + if (!Addr) { + DLFcnError = toString(Addr.takeError()); + return 0; + } + + return Addr->toPtr(); +} + +int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, + void *DSOHandle) { + // FIXME: Handle out-of-memory errors, returning -1 if OOM. + std::lock_guard Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); + assert(JDS && "JITDylib state not initialized"); + JDS->AtExits.push_back({F, Arg}); + return 0; +} + +void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) { + // FIXME: Should atexits be allowed to run concurrently with access to + // JDState? + AtExitsVector V; + { + std::lock_guard Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); + assert(JDS && "JITDlybi state not initialized"); + std::swap(V, JDS->AtExits); + } + + while (!V.empty()) { + auto &AE = V.back(); + AE.Func(AE.Arg); + V.pop_back(); + } +} + +MachOPlatformRuntimeState::PerJITDylibState * +MachOPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { + auto I = JDStates.find(DSOHandle); + if (I == JDStates.end()) + return nullptr; + return &I->second; +} + +MachOPlatformRuntimeState::PerJITDylibState * +MachOPlatformRuntimeState::getJITDylibStateByName(string_view Name) { + // FIXME: Avoid creating string copy here. + auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); + if (I == JDNameToHeader.end()) + return nullptr; + void *H = I->second; + auto J = JDStates.find(H); + assert(J != JDStates.end() && + "JITDylib has name map entry but no header map entry"); + return &J->second; +} + +MachOPlatformRuntimeState::PerJITDylibState & +MachOPlatformRuntimeState::getOrCreateJITDylibState( + MachOJITDylibInitializers &MOJDIs) { + void *Header = MOJDIs.MachOHeaderAddress.toPtr(); + + auto &JDS = JDStates[Header]; + + // If this entry hasn't been created yet. + if (!JDS.Header) { + assert(!JDNameToHeader.count(MOJDIs.Name) && + "JITDylib has header map entry but no name map entry"); + JDNameToHeader[MOJDIs.Name] = Header; + JDS.Header = Header; + } + + return JDS; +} + +Expected +MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, + string_view Sym) { + Expected Result((ExecutorAddress())); + if (auto Err = WrapperFunction( + SPSExecutorAddress, + SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag, Result, + ExecutorAddress::fromPtr(DSOHandle), Sym)) + return std::move(Err); + return Result; +} + +Expected +MachOPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) { + Expected Result( + (MachOJITDylibInitializerSequence())); + std::string PathStr(Path.data(), Path.size()); + if (auto Err = + WrapperFunction( + SPSString)>::call(&__orc_rt_macho_get_initializers_tag, Result, + Path)) + return std::move(Err); + return Result; +} + +Expected MachOPlatformRuntimeState::dlopenInitialize(string_view Path, + int Mode) { + // Either our JITDylib wasn't loaded, or it or one of its dependencies allows + // reinitialization. We need to call in to the JIT to see if there's any new + // work pending. + auto InitSeq = getJITDylibInitializersByName(Path); + if (!InitSeq) + return InitSeq.takeError(); + + // Init sequences should be non-empty. + if (InitSeq->empty()) + return make_error( + "__orc_rt_macho_get_initializers returned an " + "empty init sequence"); + + // Otherwise register and run initializers for each JITDylib. + for (auto &MOJDIs : *InitSeq) + if (auto Err = initializeJITDylib(MOJDIs)) + return std::move(Err); + + // Return the header for the last item in the list. + auto *JDS = getJITDylibStateByHeaderAddr( + InitSeq->back().MachOHeaderAddress.toPtr()); + assert(JDS && "Missing state entry for JD"); + return JDS->Header; +} + +Error MachOPlatformRuntimeState::initializeJITDylib( + MachOJITDylibInitializers &MOJDIs) { + + auto &JDS = getOrCreateJITDylibState(MOJDIs); + ++JDS.RefCount; + + for (auto &KV : InitSections) { + const auto &Name = KV.first; + const auto &Handler = KV.second; + // FIXME: Remove copy once we have C++17. + auto I = MOJDIs.InitSections.find(to_string(Name)); + if (I != MOJDIs.InitSections.end()) { + if (auto Err = Handler(I->second, MOJDIs)) + return Err; + } + } + + return Error::success(); +} + +} // end anonymous namespace + +//------------------------------------------------------------------------------ +// JIT entry points +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) { + MachOPlatformRuntimeState::initialize(); + return WrapperFunctionResult().release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { + MachOPlatformRuntimeState::destroy(); + return WrapperFunctionResult().release(); +} + +/// Wrapper function for registering metadata on a per-object basis. +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_register_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](MachOPerObjectSectionsToRegister &POSR) { + return MachOPlatformRuntimeState::get().registerObjectSections( + std::move(POSR)); + }) + .release(); +} + +/// Wrapper for releasing per-object metadat. +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_deregister_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](MachOPerObjectSectionsToRegister &POSR) { + return MachOPlatformRuntimeState::get().deregisterObjectSections( + std::move(POSR)); + }) + .release(); +} + +//------------------------------------------------------------------------------ +// cxa_atexit support +//------------------------------------------------------------------------------ + +int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle) { + return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle); +} + +void __orc_rt_macho_cxa_finalize(void *dso_handle) { + MachOPlatformRuntimeState::get().runAtExits(dso_handle); +} + +//------------------------------------------------------------------------------ +// JIT'd dlfcn alternatives. +//------------------------------------------------------------------------------ + +const char *__orc_rt_macho_jit_dlerror() { + return MachOPlatformRuntimeState::get().dlerror(); +} + +void *__orc_rt_macho_jit_dlopen(const char *path, int mode) { + return MachOPlatformRuntimeState::get().dlopen(path, mode); +} + +int __orc_rt_macho_jit_dlclose(void *dso_handle) { + return MachOPlatformRuntimeState::get().dlclose(dso_handle); +} + +void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) { + return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol); +} + +//------------------------------------------------------------------------------ +// MachO Run Program +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName, + const char *EntrySymbolName, + int argc, char *argv[]) { + using MainTy = int (*)(int, char *[]); + + void *H = __orc_rt_macho_jit_dlopen(JITDylibName, + __orc_rt::macho::ORC_RT_RTLD_LAZY); + if (!H) { + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + return -1; + } + + auto *Main = + reinterpret_cast(__orc_rt_macho_jit_dlsym(H, EntrySymbolName)); + + if (!Main) { + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + return -1; + } + + int Result = Main(argc, argv); + + if (__orc_rt_macho_jit_dlclose(H) == -1) + __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); + + return Result; +} diff --git a/compiler-rt/lib/orc/macho_platform.h b/compiler-rt/lib/orc/macho_platform.h new file mode 100644 index 0000000..b4abb50 --- /dev/null +++ b/compiler-rt/lib/orc/macho_platform.h @@ -0,0 +1,129 @@ +//===- macho_platform.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC Runtime support for Darwin dynamic loading features. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_MACHO_PLATFORM_H +#define ORC_RT_MACHO_PLATFORM_H + +#include "common.h" +#include "executor_address.h" + +// Atexit functions. +ORC_RT_INTERFACE int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, + void *dso_handle); +ORC_RT_INTERFACE void __orc_rt_macho_cxa_finalize(void *dso_handle); + +// dlfcn functions. +ORC_RT_INTERFACE const char *__orc_rt_macho_jit_dlerror(); +ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlopen(const char *path, int mode); +ORC_RT_INTERFACE int __orc_rt_macho_jit_dlclose(void *dso_handle); +ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlsym(void *dso_handle, + const char *symbol); + +namespace __orc_rt { +namespace macho { + +struct MachOPerObjectSectionsToRegister { + ExecutorAddressRange EHFrameSection; +}; + +struct MachOJITDylibInitializers { + using SectionList = std::vector; + + MachOJITDylibInitializers() = default; + MachOJITDylibInitializers(std::string Name, + ExecutorAddress MachOHeaderAddress) + : Name(std::move(Name)), + MachOHeaderAddress(std::move(MachOHeaderAddress)) {} + + std::string Name; + ExecutorAddress MachOHeaderAddress; + + std::unordered_map InitSections; +}; + +class MachOJITDylibDeinitializers {}; + +using MachOJITDylibInitializerSequence = std::vector; + +using MachOJITDylibDeinitializerSequence = + std::vector; + +enum dlopen_mode : int { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 +}; + +} // end namespace macho + +using SPSMachOPerObjectSectionsToRegister = SPSTuple; + +template <> +class SPSSerializationTraits { + +public: + static size_t size(const macho::MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::size( + MOPOSR.EHFrameSection); + } + + static bool serialize(SPSOutputBuffer &OB, + const macho::MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize( + OB, MOPOSR.EHFrameSection); + } + + static bool deserialize(SPSInputBuffer &IB, + macho::MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize( + IB, MOPOSR.EHFrameSection); + } +}; + +using SPSNamedExecutorAddressRangeSequenceMap = + SPSSequence>; + +using SPSMachOJITDylibInitializers = + SPSTuple; + +using SPSMachOJITDylibInitializerSequence = + SPSSequence; + +/// Serialization traits for MachOJITDylibInitializers. +template <> +class SPSSerializationTraits { +public: + static size_t size(const macho::MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const macho::MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + macho::MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } +}; + +} // end namespace __orc_rt + +#endif // ORC_RT_MACHO_PLATFORM_H diff --git a/compiler-rt/lib/orc/run_program_wrapper.cpp b/compiler-rt/lib/orc/run_program_wrapper.cpp new file mode 100644 index 0000000..d0f8853 --- /dev/null +++ b/compiler-rt/lib/orc/run_program_wrapper.cpp @@ -0,0 +1,51 @@ +//===- run_program_wrapper.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of the ORC runtime support library. +// +//===----------------------------------------------------------------------===// + +#include "adt.h" +#include "common.h" +#include "wrapper_function_utils.h" + +#include + +using namespace __orc_rt; + +extern "C" int64_t __orc_rt_run_program(const char *JITDylibName, + const char *EntrySymbolName, int argc, + char *argv[]); + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_run_program_wrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction)>:: + handle(ArgData, ArgSize, + [](const std::string &JITDylibName, + const std::string &EntrySymbolName, + const std::vector &Args) { + std::vector> ArgVStorage; + ArgVStorage.reserve(Args.size()); + for (auto &Arg : Args) { + ArgVStorage.push_back( + std::make_unique(Arg.size() + 1)); + memcpy(ArgVStorage.back().get(), Arg.data(), Arg.size()); + ArgVStorage.back()[Arg.size()] = '\0'; + } + std::vector ArgV; + ArgV.reserve(ArgVStorage.size() + 1); + for (auto &ArgStorage : ArgVStorage) + ArgV.push_back(ArgStorage.get()); + ArgV.push_back(nullptr); + return __orc_rt_run_program(JITDylibName.c_str(), + EntrySymbolName.c_str(), + ArgV.size() - 1, ArgV.data()); + }) + .release(); +} diff --git a/compiler-rt/lib/orc/simple_packed_serialization.h b/compiler-rt/lib/orc/simple_packed_serialization.h index a6e90f5..b561a19 100644 --- a/compiler-rt/lib/orc/simple_packed_serialization.h +++ b/compiler-rt/lib/orc/simple_packed_serialization.h @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -85,6 +86,7 @@ public: bool skip(size_t Size) { if (Size > Remaining) return false; + Buffer += Size; Remaining -= Size; return true; } @@ -170,17 +172,11 @@ public: } }; -// Any empty placeholder suitable as a substitute for void when deserializing +/// Any empty placeholder suitable as a substitute for void when deserializing class SPSEmpty {}; -/// SPS tag type for target addresses. -/// -/// SPSTagTargetAddresses should be serialized as a uint64_t value. -class SPSTagTargetAddress; - -template <> -class SPSSerializationTraits - : public SPSSerializationTraits {}; +/// Represents an address in the executor. +class SPSExecutorAddress {}; /// SPS tag type for tuples. /// @@ -290,6 +286,33 @@ public: } }; +/// Trivial std::unordered_map -> SPSSequence> +/// serialization. +template +class TrivialSPSSequenceSerialization, + std::unordered_map> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence> -> std::unordered_map +/// deserialization. +template +class TrivialSPSSequenceDeserialization, + std::unordered_map> { +public: + static constexpr bool available = true; + + using element_type = std::pair; + + static void reserve(std::unordered_map &M, uint64_t Size) { + M.reserve(Size); + } + static bool append(std::unordered_map &M, element_type E) { + return M.insert(std::move(E)).second; + } +}; + /// 'Trivial' sequence serialization: Sequence is serialized as a uint64_t size /// followed by a for-earch loop over the elements of the sequence to serialize /// each of them. diff --git a/compiler-rt/lib/orc/wrapper_function_utils.h b/compiler-rt/lib/orc/wrapper_function_utils.h index cf027e7..0c6973d 100644 --- a/compiler-rt/lib/orc/wrapper_function_utils.h +++ b/compiler-rt/lib/orc/wrapper_function_utils.h @@ -305,9 +305,9 @@ public: detail::ResultDeserializer::makeSafe(Result); if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch_ctx)) - return make_error("__orc_jtjit_dispatch_ctx not set"); + return make_error("__orc_rt_jit_dispatch_ctx not set"); if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch)) - return make_error("__orc_jtjit_dispatch not set"); + return make_error("__orc_rt_jit_dispatch not set"); auto ArgBuffer = detail::serializeViaSPSToWrapperFunctionResult>( diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/placeholder_test.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/placeholder_test.S deleted file mode 100644 index a794e10..0000000 --- a/compiler-rt/test/orc/TestCases/Darwin/x86-64/placeholder_test.S +++ /dev/null @@ -1,13 +0,0 @@ -// RUN: %clang -c -o %t %s -// RUN: %llvm_jitlink %t - - .section __TEXT,__text,regular,pure_instructions - .build_version macos, 11, 0 sdk_version 11, 3 - - .globl _main - .p2align 4, 0x90 -_main: - xorl %eax, %eax - retq - -.subsections_via_symbols diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-cxa-atexit.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-cxa-atexit.S new file mode 100644 index 0000000..f04af5e --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-cxa-atexit.S @@ -0,0 +1,38 @@ +// Test that the runtime correctly interposes ___cxa_atexit. +// +// RUN: %clang -c -o %t %s +// RUN: %llvm_jitlink %t + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 11, 0 sdk_version 11, 3 + +// OnExit destructor resets the test result override to zero. + .globl __ZN6OnExitD1Ev + .weak_def_can_be_hidden __ZN6OnExitD1Ev + .p2align 4, 0x90 +__ZN6OnExitD1Ev: + xorl %edi, %edi + jmp _llvm_jitlink_setTestResultOverride + +// main registers the atexit and sets the test result to one. + .globl _main + .p2align 4, 0x90 +_main: + pushq %rbp + movq %rsp, %rbp + + movq __ZN6OnExitD1Ev@GOTPCREL(%rip), %rdi + leaq _onExit(%rip), %rsi + leaq ___dso_handle(%rip), %rdx + callq ___cxa_atexit + + movl $1, %edi + callq _llvm_jitlink_setTestResultOverride + xorl %eax, %eax + popq %rbp + retq + + .globl _onExit +.zerofill __DATA,__common,_onExit,1,0 + +.subsections_via_symbols diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-static-initializer.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-static-initializer.S new file mode 100644 index 0000000..a90c67c --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-static-initializer.S @@ -0,0 +1,37 @@ +// Test that basic MachO static initializers work. The main function in this +// test returns the value of 'x', which is initially 1 in the data section, +// and reset to 0 if the _static_init function is run. If the static initializer +// does not run then main will return 1, causing the test to be treated as a +// failure. +// +// RUN: %clang -c -o %t %s +// RUN: %llvm_jitlink %t + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 11, 0 + +# main returns the value of 'x', which is defined as 1 in the data section.. + .globl _main + .p2align 4, 0x90 +_main: + movl _x(%rip), %eax + retq + +# static initializer sets the value of 'x' to zero. + .section __TEXT,__StaticInit,regular,pure_instructions + .p2align 4, 0x90 +_static_init: + movl $0, _x(%rip) + retq + + .section __DATA,__data + .globl _x + .p2align 2 +_x: + .long 1 + + .section __DATA,__mod_init_func,mod_init_funcs + .p2align 3 + .quad _static_init + +.subsections_via_symbols diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index 15b5286..81d8dbe 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -916,6 +916,12 @@ public: const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); } + /// Allocate a mutable buffer of the given size using the LinkGraph's + /// allocator. + MutableArrayRef allocateBuffer(size_t Size) { + return {Allocator.Allocate(Size), Size}; + } + /// Allocate a copy of the given string using the LinkGraph's allocator. /// This can be useful when renaming symbols or adding new content to the /// graph. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index 9219e28..159553b5 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -1224,9 +1224,19 @@ public: /// A utility function for looking up initializer symbols. Performs a blocking /// lookup for the given symbols in each of the given JITDylibs. + /// + /// Note: This function is deprecated and will be removed in the near future. static Expected> lookupInitSymbols(ExecutionSession &ES, const DenseMap &InitSyms); + + /// Performs an async lookup for the the given symbols in each of the given + /// JITDylibs, calling the given handler with the compound result map once + /// all lookups have completed. + static void + lookupInitSymbolsAsync(unique_function OnComplete, + ExecutionSession &ES, + const DenseMap &InitSyms); }; /// Represents an abstract task for ORC to run. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h index 566637e..f8d6192 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h @@ -18,6 +18,7 @@ #include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" #include "llvm/Support/DynamicLibrary.h" @@ -113,6 +114,13 @@ public: const SymbolLookupSet &Symbols; }; + /// Contains the address of the dispatch function and context that the ORC + /// runtime can use to call functions in the JIT. + struct JITDispatchInfo { + ExecutorAddress JITDispatchFunctionAddress; + ExecutorAddress JITDispatchContextAddress; + }; + virtual ~ExecutorProcessControl(); /// Intern a symbol name in the SymbolStringPool. @@ -127,6 +135,9 @@ public: /// Get the page size for the target process. unsigned getPageSize() const { return PageSize; } + /// Get the JIT dispatch function and context address for the executor. + const JITDispatchInfo &getJITDispatchInfo() const { return JDI; } + /// Return a MemoryAccess object for the target process. MemoryAccess &getMemoryAccess() const { return *MemAccess; } @@ -198,14 +209,17 @@ public: /// Run a wrapper function using SPS to serialize the arguments and /// deserialize the results. - template - Error runSPSWrapper(JITTargetAddress WrapperFnAddr, RetT &RetVal, - const ArgTs &...Args) { + /// + /// If SPSSignature is a non-void function signature then the second argument + /// (the first in the Args list) should be a reference to a return value. + template + Error runSPSWrapper(JITTargetAddress WrapperFnAddr, + WrapperCallArgTs &&...WrapperCallArgs) { return shared::WrapperFunction::call( [this, WrapperFnAddr](const char *ArgData, size_t ArgSize) { return runWrapper(WrapperFnAddr, ArrayRef(ArgData, ArgSize)); }, - RetVal, Args...); + std::forward(WrapperCallArgs)...); } /// Wrap a handler that takes concrete argument types (and a sender for a @@ -223,6 +237,15 @@ public: }; } + template + static AsyncWrapperFunction + wrapAsyncWithSPS(ClassT *Instance, void (ClassT::*Method)(MethodArgTs...)) { + return wrapAsyncWithSPS( + [Instance, Method](MethodArgTs &&...MethodArgs) { + (Instance->*Method)(std::forward(MethodArgs)...); + }); + } + /// For each symbol name, associate the AsyncWrapperFunction implementation /// value with the address of that symbol. /// @@ -250,6 +273,7 @@ protected: std::shared_ptr SSP; Triple TargetTriple; unsigned PageSize = 0; + JITDispatchInfo JDI; MemoryAccess *MemAccess = nullptr; jitlink::JITLinkMemoryManager *MemMgr = nullptr; @@ -318,6 +342,10 @@ private: void writeBuffers(ArrayRef Ws, WriteResultFn OnWriteComplete) override; + static shared::detail::CWrapperFunctionResult + jitDispatchViaWrapperFunctionManager(void *Ctx, const void *FnTag, + const char *Data, size_t Size); + std::unique_ptr OwnedMemMgr; char GlobalManglingPrefix = 0; std::vector> DynamicLibraries; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h b/llvm/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h new file mode 100644 index 0000000..f3d616d --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h @@ -0,0 +1,69 @@ +//===-- LLVMSPSSerializers.h - SPS serialization for LLVM types -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// SPS Serialization for common LLVM types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H +#define LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" + +namespace llvm { +namespace orc { +namespace shared { + +template +class SPSSerializationTraits>, + StringMap> { +public: + static size_t size(const StringMap &M) { + size_t Sz = SPSArgList::size(static_cast(M.size())); + for (auto &E : M) + Sz += SPSArgList::size(E.first(), E.second); + return Sz; + } + + static bool serialize(SPSOutputBuffer &OB, const StringMap &M) { + if (!SPSArgList::serialize(OB, static_cast(M.size()))) + return false; + + for (auto &E : M) + if (!SPSArgList::serialize(OB, E.first(), E.second)) + return false; + + return true; + } + + static bool deserialize(SPSInputBuffer &IB, StringMap &M) { + uint64_t Size; + assert(M.empty() && "M already contains elements"); + + if (!SPSArgList::deserialize(IB, Size)) + return false; + + while (Size--) { + StringRef S; + ValueT V; + if (!SPSArgList::deserialize(IB, S, V)) + return false; + if (!M.insert(std::make_pair(S, V)).second) + return false; + } + + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 10d0f90..50f26b3 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -15,7 +15,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/Orc/Core.h" -#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" @@ -26,83 +27,106 @@ namespace llvm { namespace orc { +struct MachOPerObjectSectionsToRegister { + ExecutorAddressRange EHFrameSection; +}; -class MachOJITDylibInitializers { -public: - using RawPointerSectionList = std::vector; - - void setObjCImageInfoAddr(JITTargetAddress ObjCImageInfoAddr) { - this->ObjCImageInfoAddr = ObjCImageInfoAddr; - } - - void addModInitsSection(ExecutorAddressRange ModInit) { - ModInitSections.push_back(std::move(ModInit)); - } - - const RawPointerSectionList &getModInitsSections() const { - return ModInitSections; - } - - void addObjCSelRefsSection(ExecutorAddressRange ObjCSelRefs) { - ObjCSelRefsSections.push_back(std::move(ObjCSelRefs)); - } - - const RawPointerSectionList &getObjCSelRefsSections() const { - return ObjCSelRefsSections; - } +struct MachOJITDylibInitializers { + using SectionList = std::vector; - void addObjCClassListSection(ExecutorAddressRange ObjCClassList) { - ObjCClassListSections.push_back(std::move(ObjCClassList)); - } - - const RawPointerSectionList &getObjCClassListSections() const { - return ObjCClassListSections; - } + MachOJITDylibInitializers(std::string Name, + ExecutorAddress MachOHeaderAddress) + : Name(std::move(Name)), + MachOHeaderAddress(std::move(MachOHeaderAddress)) {} - void runModInits() const; - void registerObjCSelectors() const; - Error registerObjCClasses() const; + std::string Name; + ExecutorAddress MachOHeaderAddress; -private: - - JITTargetAddress ObjCImageInfoAddr; - RawPointerSectionList ModInitSections; - RawPointerSectionList ObjCSelRefsSections; - RawPointerSectionList ObjCClassListSections; + StringMap InitSections; }; class MachOJITDylibDeinitializers {}; +using MachOJITDylibInitializerSequence = std::vector; + +using MachOJITDylibDeinitializerSequence = + std::vector; + /// Mediates between MachO initialization and ExecutionSession state. class MachOPlatform : public Platform { public: - using InitializerSequence = - std::vector>; - - using DeinitializerSequence = - std::vector>; - - MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - std::unique_ptr StandardSymbolsObject); + /// Try to create a MachOPlatform instance, adding the ORC runtime to the + /// given JITDylib. + /// + /// The ORC runtime requires access to a number of symbols in libc++, and + /// requires access to symbols in libobjc, and libswiftCore to support + /// Objective-C and Swift code. It is up to the caller to ensure that the + /// requried symbols can be referenced by code added to PlatformJD. The + /// standard way to achieve this is to first attach dynamic library search + /// generators for either the given process, or for the specific required + /// libraries, to PlatformJD, then to create the platform instance: + /// + /// \code{.cpp} + /// auto &PlatformJD = ES.createBareJITDylib("stdlib"); + /// PlatformJD.addGenerator( + /// ExitOnErr(EPCDynamicLibrarySearchGenerator + /// ::GetForTargetProcess(EPC))); + /// ES.setPlatform( + /// ExitOnErr(MachOPlatform::Create(ES, ObjLayer, EPC, PlatformJD, + /// "/path/to/orc/runtime"))); + /// \endcode + /// + /// Alternatively, these symbols could be added to another JITDylib that + /// PlatformJD links against. + /// + /// Clients are also responsible for ensuring that any JIT'd code that + /// depends on runtime functions (including any code using TLV or static + /// destructors) can reference the runtime symbols. This is usually achieved + /// by linking any JITDylibs containing regular code against + /// PlatformJD. + /// + /// By default, MachOPlatform will add the set of aliases returned by the + /// standardPlatformAliases function. This includes both required aliases + /// (e.g. __cxa_atexit -> __orc_rt_macho_cxa_atexit for static destructor + /// support), and optional aliases that provide JIT versions of common + /// functions (e.g. dlopen -> __orc_rt_macho_jit_dlopen). Clients can + /// override these defaults by passing a non-None value for the + /// RuntimeAliases function, in which case the client is responsible for + /// setting up all aliases (including the required ones). + static Expected> + Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + const char *OrcRuntimePath, + Optional RuntimeAliases = None); ExecutionSession &getExecutionSession() const { return ES; } + ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + ExecutorProcessControl &getExecutorProcessControl() const { return EPC; } Error setupJITDylib(JITDylib &JD) override; Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) override; Error notifyRemoving(ResourceTracker &RT) override; - Expected getInitializerSequence(JITDylib &JD); + /// Returns an AliasMap containing the default aliases for the MachOPlatform. + /// This can be modified by clients when constructing the platform to add + /// or remove aliases. + static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES); - Expected getDeinitializerSequence(JITDylib &JD); + /// Returns the array of required CXX aliases. + static ArrayRef> requiredCXXAliases(); + + /// Returns the array of standard runtime utility aliases for MachO. + static ArrayRef> + standardRuntimeUtilityAliases(); private: - // This ObjectLinkingLayer plugin scans JITLink graphs for __mod_init_func, - // __objc_classlist and __sel_ref sections and records their extents so that - // they can be run in the target process. - class InitScraperPlugin : public ObjectLinkingLayer::Plugin { + // The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO + // platform features including initializers, exceptions, TLV, and language + // runtime registration. + class MachOPlatformPlugin : public ObjectLinkingLayer::Plugin { public: - InitScraperPlugin(MachOPlatform &MP) : MP(MP) {} + MachOPlatformPlugin(MachOPlatform &MP) : MP(MP) {} void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &G, @@ -128,36 +152,173 @@ private: using InitSymbolDepMap = DenseMap; - void preserveInitSectionIfPresent(JITLinkSymbolSet &Symbols, - jitlink::LinkGraph &G, - StringRef SectionName); + void addInitializerSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + void addMachOHeaderSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); - Error processObjCImageInfo(jitlink::LinkGraph &G, + void addEHSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + Error preserveInitSections(jitlink::LinkGraph &G, MaterializationResponsibility &MR); - std::mutex InitScraperMutex; + Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD); + + std::mutex PluginMutex; MachOPlatform &MP; - DenseMap> ObjCImageInfos; InitSymbolDepMap InitSymbolDeps; }; - void registerInitInfo(JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, - ExecutorAddressRange ModInits, - ExecutorAddressRange ObjCSelRefs, - ExecutorAddressRange ObjCClassList); + using SendInitializerSequenceFn = + unique_function)>; + + using SendDeinitializerSequenceFn = + unique_function)>; + + using SendSymbolAddressFn = unique_function)>; + + static bool supportedTarget(const Triple &TT); + + MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, + Error &Err); + + // Associate MachOPlatform JIT-side runtime support functions with handlers. + Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); + + void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult, + JITDylib &JD, + std::vector DFSLinkOrder); + + void getInitializersLookupPhase(SendInitializerSequenceFn SendResult, + JITDylib &JD); + + void rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName); + + void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddress Handle); + + void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddress Handle, + StringRef SymbolName); + + // Records the addresses of runtime symbols used by the platform. + Error bootstrapMachORuntime(JITDylib &PlatformJD); + + Error registerInitInfo(JITDylib &JD, + ArrayRef InitSections); + + Error registerPerObjectSections(const MachOPerObjectSectionsToRegister &POSR); ExecutionSession &ES; ObjectLinkingLayer &ObjLinkingLayer; - std::unique_ptr StandardSymbolsObject; + ExecutorProcessControl &EPC; + + SymbolStringPtr MachOHeaderStartSymbol; + std::atomic RuntimeBootstrapped{false}; + + ExecutorAddress orc_rt_macho_platform_bootstrap; + ExecutorAddress orc_rt_macho_platform_shutdown; + ExecutorAddress orc_rt_macho_register_object_sections; DenseMap RegisteredInitSymbols; // InitSeqs gets its own mutex to avoid locking the whole session when // aggregating data from the jitlink. - std::mutex InitSeqsMutex; + std::mutex PlatformMutex; DenseMap InitSeqs; + std::vector BootstrapPOSRs; + + DenseMap HeaderAddrToJITDylib; +}; + +namespace shared { + +using SPSMachOPerObjectSectionsToRegister = SPSTuple; + +template <> +class SPSSerializationTraits { + +public: + static size_t size(const MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::size( + MOPOSR.EHFrameSection); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize( + OB, MOPOSR.EHFrameSection); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize( + IB, MOPOSR.EHFrameSection); + } +}; + +using SPSNamedExecutorAddressRangeSequenceMap = + SPSSequence>; + +using SPSMachOJITDylibInitializers = + SPSTuple; + +using SPSMachOJITDylibInitializerSequence = + SPSSequence; + +/// Serialization traits for MachOJITDylibInitializers. +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } +}; + +using SPSMachOJITDylibDeinitializers = SPSEmpty; + +using SPSMachOJITDylibDeinitializerSequence = + SPSSequence; + +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOJITDylibDeinitializers &MOJDDs) { return 0; } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibDeinitializers &MOJDDs) { + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibDeinitializers &MOJDDs) { + MOJDDs = MachOJITDylibDeinitializers(); + return true; + } }; +} // end namespace shared } // end namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h index 69e37f9..fedd79c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h @@ -283,7 +283,11 @@ public: OrcRPCExecutorProcessControlBase(std::shared_ptr SSP, RPCEndpointT &EP, ErrorReporter ReportError) : ExecutorProcessControl(std::move(SSP)), - ReportError(std::move(ReportError)), EP(EP) {} + ReportError(std::move(ReportError)), EP(EP) { + using ThisT = OrcRPCExecutorProcessControlBase; + EP.template addAsyncHandler(*this, + &ThisT::runWrapperInJIT); + } void reportError(Error Err) { ReportError(std::move(Err)); } @@ -396,20 +400,32 @@ protected: /// Subclasses must call this during construction to initialize the /// TargetTriple and PageSize members. Error initializeORCRPCEPCBase() { - if (auto TripleOrErr = EP.template callB()) - TargetTriple = Triple(*TripleOrErr); - else - return TripleOrErr.takeError(); - - if (auto PageSizeOrErr = EP.template callB()) - PageSize = *PageSizeOrErr; - else - return PageSizeOrErr.takeError(); + if (auto EPI = EP.template callB()) { + this->TargetTriple = Triple(EPI->Triple); + this->PageSize = PageSize; + this->JDI = {ExecutorAddress(EPI->DispatchFuncAddr), + ExecutorAddress(EPI->DispatchCtxAddr)}; + return Error::success(); + } else + return EPI.takeError(); + } +private: + Error runWrapperInJIT( + std::function)> SendResult, + JITTargetAddress FunctionTag, std::vector ArgBuffer) { + + runJITSideWrapperFunction( + [this, SendResult = std::move(SendResult)]( + Expected R) { + if (auto Err = SendResult(std::move(R))) + ReportError(std::move(Err)); + }, + FunctionTag, + {reinterpret_cast(ArgBuffer.data()), ArgBuffer.size()}); return Error::success(); } -private: ErrorReporter ReportError; RPCEndpointT &EP; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h index a2ad846..854f109 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h @@ -85,6 +85,7 @@ public: bool skip(size_t Size) { if (Size > Remaining) return false; + Buffer += Size; Remaining -= Size; return true; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h index ceaea1d..2f14a1c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h @@ -298,7 +298,7 @@ public: SendWFR(ResultSerializer::serialize(std::move(Result))); }; - callAsync(std::forward(H), std::move(SendResult), Args, + callAsync(std::forward(H), std::move(SendResult), std::move(Args), ArgIndices{}); } @@ -314,9 +314,9 @@ private: typename ArgTupleT, std::size_t... I> static void callAsync(HandlerT &&H, SerializeAndSendResultT &&SerializeAndSendResult, - ArgTupleT &Args, std::index_sequence) { + ArgTupleT Args, std::index_sequence) { return std::forward(H)(std::move(SerializeAndSendResult), - std::get(Args)...); + std::move(std::get(Args))...); } }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h index 458947c..96e4341 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h @@ -41,6 +41,13 @@ enum WireProtectionFlags : uint8_t { LLVM_MARK_AS_BITMASK_ENUM(WPF_Exec) }; +struct ExecutorProcessInfo { + std::string Triple; + unsigned PageSize; + JITTargetAddress DispatchFuncAddr; + JITTargetAddress DispatchCtxAddr; +}; + /// Convert from sys::Memory::ProtectionFlags inline WireProtectionFlags toWireProtectionFlags(sys::Memory::ProtectionFlags PF) { @@ -95,6 +102,41 @@ using ReleaseOrFinalizeMemRequest = namespace shared { +template <> class SerializationTypeName { +public: + static const char *getName() { return "WrapperFunctionResult"; } +}; + +template +class SerializationTraits< + ChannelT, WrapperFunctionResult, WrapperFunctionResult, + std::enable_if_t::value>> { +public: + static Error serialize(ChannelT &C, const WrapperFunctionResult &E) { + if (auto Err = serializeSeq(C, static_cast(E.size()))) + return Err; + if (E.size() == 0) + return Error::success(); + return C.appendBytes(E.data(), E.size()); + } + + static Error deserialize(ChannelT &C, WrapperFunctionResult &E) { + uint64_t Size; + if (auto Err = deserializeSeq(C, Size)) + return Err; + + WrapperFunctionResult Tmp; + char *Data = WrapperFunctionResult::allocate(Tmp, Size); + + if (auto Err = C.readBytes(Data, Size)) + return Err; + + E = std::move(Tmp); + + return Error::success(); + } +}; + template <> class SerializationTypeName { public: static const char *getName() { return "UInt8Write"; } @@ -136,9 +178,9 @@ public: static const char *getName() { return "ReleaseOrFinalizeMemRequestElement"; } }; -template <> class SerializationTypeName { +template <> class SerializationTypeName { public: - static const char *getName() { return "WrapperFunctionResult"; } + static const char *getName() { return "ExecutorProcessInfo"; } }; template @@ -234,26 +276,17 @@ public: }; template -class SerializationTraits< - ChannelT, shared::WrapperFunctionResult, shared::WrapperFunctionResult, - std::enable_if_t::value>> { +class SerializationTraits { public: - static Error serialize(ChannelT &C, const shared::WrapperFunctionResult &E) { - if (auto Err = serializeSeq(C, static_cast(E.size()))) - return Err; - if (E.size() == 0) - return Error::success(); - return C.appendBytes(E.data(), E.size()); + static Error serialize(ChannelT &C, + const orcrpctpc::ExecutorProcessInfo &EPI) { + return serializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr, + EPI.DispatchCtxAddr); } - static Error deserialize(ChannelT &C, shared::WrapperFunctionResult &E) { - - uint64_t Size; - if (auto Err = deserializeSeq(C, Size)) - return Err; - - char *DataPtr = shared::WrapperFunctionResult::allocate(E, Size); - return C.readBytes(DataPtr, E.size()); + static Error deserialize(ChannelT &C, orcrpctpc::ExecutorProcessInfo &EPI) { + return deserializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr, + EPI.DispatchCtxAddr); } }; @@ -265,15 +298,11 @@ using RemoteSymbolLookupSet = std::vector>; using RemoteLookupRequest = std::pair; -class GetTargetTriple - : public shared::RPCFunction { -public: - static const char *getName() { return "GetTargetTriple"; } -}; - -class GetPageSize : public shared::RPCFunction { +class GetExecutorProcessInfo + : public shared::RPCFunction { public: - static const char *getName() { return "GetPageSize"; } + static const char *getName() { return "GetJITDispatchInfo"; } }; class ReserveMem @@ -349,7 +378,7 @@ public: class RunMain : public shared::RPCFunction Args)> { public: static const char *getName() { return "RunMain"; } @@ -372,18 +401,18 @@ public: /// TargetProcessControl for a process connected via an ORC RPC Endpoint. template class OrcRPCTPCServer { +private: + using ThisT = OrcRPCTPCServer; + public: /// Create an OrcRPCTPCServer from the given endpoint. OrcRPCTPCServer(RPCEndpointT &EP) : EP(EP) { - using ThisT = OrcRPCTPCServer; TripleStr = sys::getProcessTriple(); PageSize = sys::Process::getPageSizeEstimate(); - EP.template addHandler(*this, - &ThisT::getTargetTriple); - EP.template addHandler(*this, &ThisT::getPageSize); - + EP.template addHandler( + *this, &ThisT::getExecutorProcessInfo); EP.template addHandler(*this, &ThisT::reserveMemory); EP.template addHandler(*this, &ThisT::finalizeMemory); @@ -428,9 +457,34 @@ public: return Error::success(); } + Expected + runWrapperInJIT(JITTargetAddress FunctionId, ArrayRef ArgBuffer) { + return EP.template callB( + FunctionId, + ArrayRef(reinterpret_cast(ArgBuffer.data()), + ArgBuffer.size())); + } + private: - std::string getTargetTriple() { return TripleStr; } - uint64_t getPageSize() { return PageSize; } + static shared::detail::CWrapperFunctionResult + jitDispatchViaOrcRPCTPCServer(void *Ctx, const void *FnTag, const char *Data, + size_t Size) { + assert(Ctx && "Attempt to dispatch with null context ptr"); + auto R = static_cast(Ctx)->runWrapperInJIT( + pointerToJITTargetAddress(FnTag), {Data, Size}); + if (!R) { + auto ErrMsg = toString(R.takeError()); + return shared::WrapperFunctionResult::createOutOfBandError(ErrMsg.data()) + .release(); + } + return R->release(); + } + + orcrpctpc::ExecutorProcessInfo getExecutorProcessInfo() { + return {TripleStr, static_cast(PageSize), + pointerToJITTargetAddress(jitDispatchViaOrcRPCTPCServer), + pointerToJITTargetAddress(this)}; + } template static void handleWriteUInt(const std::vector &Ws) { @@ -569,7 +623,7 @@ private: return Result; } - int32_t runMain(JITTargetAddress MainFnAddr, + int64_t runMain(JITTargetAddress MainFnAddr, const std::vector &Args) { Optional ProgramNameOverride; if (ProgramName) diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index f6ebfc3..c613c57 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -1750,6 +1750,49 @@ Expected> Platform::lookupInitSymbols( return std::move(CompoundResult); } +void Platform::lookupInitSymbolsAsync( + unique_function OnComplete, ExecutionSession &ES, + const DenseMap &InitSyms) { + + class TriggerOnComplete { + public: + using OnCompleteFn = unique_function; + TriggerOnComplete(OnCompleteFn OnComplete) + : OnComplete(std::move(OnComplete)) {} + ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); } + void reportResult(Error Err) { + std::lock_guard Lock(ResultMutex); + LookupResult = joinErrors(std::move(LookupResult), std::move(Err)); + } + + private: + std::mutex ResultMutex; + Error LookupResult{Error::success()}; + OnCompleteFn OnComplete; + }; + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + auto TOC = std::make_shared(std::move(OnComplete)); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [TOC](Expected Result) { + TOC->reportResult(Result.takeError()); + }, + NoDependenciesToRegister); + } +} + void Task::anchor() {} void MaterializationTask::printDescription(raw_ostream &OS) { diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 12fa42c..9442eab 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -14,6 +14,8 @@ #include "llvm/Support/Host.h" #include "llvm/Support/Process.h" +#define DEBUG_TYPE "orc" + namespace llvm { namespace orc { @@ -47,6 +49,10 @@ Error ExecutorProcessControl::associateJITSideWrapperFunctions( "AsyncWrapperFunction implementation missing"); TagToFunc[KV.second.getAddress()] = std::make_shared(std::move(I->second)); + LLVM_DEBUG({ + dbgs() << "Associated function tag \"" << *KV.first << "\" (" + << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n"; + }); } return Error::success(); } @@ -84,6 +90,8 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( this->PageSize = PageSize; this->MemMgr = OwnedMemMgr.get(); this->MemAccess = this; + this->JDI = {ExecutorAddress::fromPtr(jitDispatchViaWrapperFunctionManager), + ExecutorAddress::fromPtr(this)}; if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; } @@ -198,5 +206,26 @@ void SelfExecutorProcessControl::writeBuffers( OnWriteComplete(Error::success()); } +shared::detail::CWrapperFunctionResult +SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( + void *Ctx, const void *FnTag, const char *Data, size_t Size) { + + LLVM_DEBUG({ + dbgs() << "jit-dispatch call with tag " << FnTag << " and " << Size + << " byte payload.\n"; + }); + + std::promise ResultP; + auto ResultF = ResultP.get_future(); + static_cast(Ctx)->runJITSideWrapperFunction( + [ResultP = + std::move(ResultP)](shared::WrapperFunctionResult Result) mutable { + ResultP.set_value(std::move(Result)); + }, + pointerToJITTargetAddress(FnTag), {Data, Size}); + + return ResultF.get().release(); +} + } // end namespace orc } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 0fe4f00..686c3e8 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -9,27 +9,182 @@ #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "orc" +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace { + +class MachOHeaderMaterializationUnit : public MaterializationUnit { +public: + MachOHeaderMaterializationUnit(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) + : MaterializationUnit(createHeaderSymbols(MOP, HeaderStartSymbol), + HeaderStartSymbol), + MOP(MOP) {} + + StringRef getName() const override { return "MachOHeaderMU"; } + + void materialize(std::unique_ptr R) override { + unsigned PointerSize; + support::endianness Endianness; + + switch (MOP.getExecutorProcessControl().getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + auto G = std::make_unique( + "", MOP.getExecutorProcessControl().getTargetTriple(), + PointerSize, Endianness, jitlink::getGenericEdgeKindName); + auto &HeaderSection = G->createSection("__header", sys::Memory::MF_READ); + auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); + + // Init symbol is header-start symbol. + G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(), + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + for (auto &HS : AdditionalHeaderSymbols) + G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} + +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; + + static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { + {"___mh_executable_header", 0}}; + + static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + MachO::mach_header_64 Hdr; + Hdr.magic = MachO::MH_MAGIC_64; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + Hdr.cputype = MachO::CPU_TYPE_ARM64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + case Triple::x86_64: + Hdr.cputype = MachO::CPU_TYPE_X86_64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + Hdr.filetype = MachO::MH_DYLIB; // Custom file type? + Hdr.ncmds = 0; + Hdr.sizeofcmds = 0; + Hdr.flags = 0; + Hdr.reserved = 0; + + if (G.getEndianness() != support::endian::system_endianness()) + MachO::swapStruct(Hdr); + + auto HeaderContent = G.allocateString( + StringRef(reinterpret_cast(&Hdr), sizeof(Hdr))); + + return G.createContentBlock(HeaderSection, HeaderContent, 0, 8, 0); + } + + static SymbolFlagsMap + createHeaderSymbols(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + for (auto &HS : AdditionalHeaderSymbols) + HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = + JITSymbolFlags::Exported; + + return HeaderSymbolFlags; + } + + MachOPlatform &MOP; +}; + +constexpr MachOHeaderMaterializationUnit::HeaderSymbol + MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; + +StringRef EHFrameSectionName = "__TEXT,__eh_frame"; +StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; + +StringRef InitSectionNames[] = {ModInitFuncSectionName}; + +} // end anonymous namespace + namespace llvm { namespace orc { -MachOPlatform::MachOPlatform( - ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - std::unique_ptr StandardSymbolsObject) - : ES(ES), ObjLinkingLayer(ObjLinkingLayer), - StandardSymbolsObject(std::move(StandardSymbolsObject)) { - ObjLinkingLayer.addPlugin(std::make_unique(*this)); +Expected> +MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + const char *OrcRuntimePath, + Optional RuntimeAliases) { + + // If the target is not supported then bail out immediately. + if (!supportedTarget(EPC.getTargetTriple())) + return make_error("Unsupported MachOPlatform triple: " + + EPC.getTargetTriple().str(), + inconvertibleErrorCode()); + + // Create default aliases if the caller didn't supply any. + if (!RuntimeAliases) + RuntimeAliases = standardPlatformAliases(ES); + + // Define the aliases. + if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) + return std::move(Err); + + // Add JIT-dispatch function support symbols. + if (auto Err = PlatformJD.define(absoluteSymbols( + {{ES.intern("___orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunctionAddress.getValue(), + JITSymbolFlags::Exported}}, + {ES.intern("___orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContextAddress.getValue(), + JITSymbolFlags::Exported}}}))) + return std::move(Err); + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( + ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // Create the instance. + Error Err = Error::success(); + auto P = std::unique_ptr( + new MachOPlatform(ES, ObjLinkingLayer, EPC, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), Err)); + if (Err) + return std::move(Err); + return std::move(P); } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - auto ObjBuffer = MemoryBuffer::getMemBuffer( - StandardSymbolsObject->getMemBufferRef(), false); - return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); + return JD.define(std::make_unique( + *this, MachOHeaderStartSymbol)); } Error MachOPlatform::notifyAdding(ResourceTracker &RT, @@ -52,58 +207,117 @@ Error MachOPlatform::notifyRemoving(ResourceTracker &RT) { llvm_unreachable("Not supported yet"); } -Expected -MachOPlatform::getInitializerSequence(JITDylib &JD) { +static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, + ArrayRef> AL) { + for (auto &KV : AL) { + auto AliasName = ES.intern(KV.first); + assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); + Aliases[std::move(AliasName)] = {ES.intern(KV.second), + JITSymbolFlags::Exported}; + } +} - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Building initializer sequence for " - << JD.getName() << "\n"; - }); +SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) { + SymbolAliasMap Aliases; + addAliases(ES, Aliases, requiredCXXAliases()); + addAliases(ES, Aliases, standardRuntimeUtilityAliases()); + return Aliases; +} - std::vector DFSLinkOrder; +ArrayRef> +MachOPlatform::requiredCXXAliases() { + static const std::pair RequiredCXXAliases[] = { + {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}}; - while (true) { + return RequiredCXXAliases; +} - DenseMap NewInitSymbols; +ArrayRef> +MachOPlatform::standardRuntimeUtilityAliases() { + static const std::pair + StandardRuntimeUtilityAliases[] = { + {"___orc_rt_run_program", "___orc_rt_macho_run_program"}, + {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}}; - ES.runSessionLocked([&]() { - DFSLinkOrder = JD.getDFSLinkOrder(); + return StandardRuntimeUtilityAliases; +} - for (auto &InitJD : DFSLinkOrder) { - auto RISItr = RegisteredInitSymbols.find(InitJD.get()); - if (RISItr != RegisteredInitSymbols.end()) { - NewInitSymbols[InitJD.get()] = std::move(RISItr->second); - RegisteredInitSymbols.erase(RISItr); - } - } - }); +bool MachOPlatform::supportedTarget(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } +} - if (NewInitSymbols.empty()) - break; +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, Error &Err) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), EPC(EPC), + MachOHeaderStartSymbol(ES.intern("___dso_handle")) { + ErrorAsOutParameter _(&Err); - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Issuing lookups for new init symbols: " - "(lookup may require multiple rounds)\n"; - for (auto &KV : NewInitSymbols) - dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; - }); + ObjLinkingLayer.addPlugin(std::make_unique(*this)); + + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); - // Outside the lock, issue the lookup. - if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) - ; // Nothing to do in the success case. - else - return R.takeError(); + // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; } - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Init symbol lookup complete, building init " - "sequence\n"; - }); + RegisteredInitSymbols[&PlatformJD].add( + MachOHeaderStartSymbol, SymbolLookupFlags::WeaklyReferencedSymbol); + + // Associate wrapper function tags with JIT-side function implementations. + if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { + Err = std::move(E2); + return; + } - // Lock again to collect the initializers. - InitializerSequence FullInitSeq; + // Lookup addresses of runtime functions callable by the platform, + // call the platform bootstrap function to initialize the platform-state + // object in the executor. + if (auto E2 = bootstrapMachORuntime(PlatformJD)) { + Err = std::move(E2); + return; + } +} + +Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { + ExecutorProcessControl::WrapperFunctionAssociationMap WFs; + + using GetInitializersSPSSig = + SPSExpected(SPSString); + WFs[ES.intern("___orc_rt_macho_get_initializers_tag")] = + EPC.wrapAsyncWithSPS( + this, &MachOPlatform::rt_getInitializers); + + using GetDeinitializersSPSSig = + SPSExpected(SPSExecutorAddress); + WFs[ES.intern("___orc_rt_macho_get_deinitializers_tag")] = + EPC.wrapAsyncWithSPS( + this, &MachOPlatform::rt_getDeinitializers); + + using LookupSymbolSPSSig = + SPSExpected(SPSExecutorAddress, SPSString); + WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] = + EPC.wrapAsyncWithSPS(this, + &MachOPlatform::rt_lookupSymbol); + + return EPC.associateJITSideWrapperFunctions(PlatformJD, std::move(WFs)); +} + +void MachOPlatform::getInitializersBuildSequencePhase( + SendInitializerSequenceFn SendResult, JITDylib &JD, + std::vector DFSLinkOrder) { + MachOJITDylibInitializerSequence FullInitSeq; { - std::lock_guard Lock(InitSeqsMutex); + std::lock_guard Lock(PlatformMutex); for (auto &InitJD : reverse(DFSLinkOrder)) { LLVM_DEBUG({ dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() @@ -111,160 +325,267 @@ MachOPlatform::getInitializerSequence(JITDylib &JD) { }); auto ISItr = InitSeqs.find(InitJD.get()); if (ISItr != InitSeqs.end()) { - FullInitSeq.emplace_back(InitJD.get(), std::move(ISItr->second)); + FullInitSeq.emplace_back(std::move(ISItr->second)); InitSeqs.erase(ISItr); } } } - return FullInitSeq; + SendResult(std::move(FullInitSeq)); } -Expected -MachOPlatform::getDeinitializerSequence(JITDylib &JD) { - std::vector DFSLinkOrder = JD.getDFSLinkOrder(); - - DeinitializerSequence FullDeinitSeq; - { - std::lock_guard Lock(InitSeqsMutex); - for (auto &DeinitJD : DFSLinkOrder) { - FullDeinitSeq.emplace_back(DeinitJD.get(), MachOJITDylibDeinitializers()); +void MachOPlatform::getInitializersLookupPhase( + SendInitializerSequenceFn SendResult, JITDylib &JD) { + + auto DFSLinkOrder = JD.getDFSLinkOrder(); + DenseMap NewInitSymbols; + ES.runSessionLocked([&]() { + for (auto &InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD.get()); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD.get()] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } } + }); + + // If there are no further init symbols to look up then move on to the next + // phase. + if (NewInitSymbols.empty()) { + getInitializersBuildSequencePhase(std::move(SendResult), JD, + std::move(DFSLinkOrder)); + return; } - return FullDeinitSeq; + // Otherwise issue a lookup and re-run this phase when it completes. + lookupInitSymbolsAsync( + [this, SendResult = std::move(SendResult), &JD](Error Err) mutable { + if (Err) + SendResult(std::move(Err)); + else + getInitializersLookupPhase(std::move(SendResult), JD); + }, + ES, std::move(NewInitSymbols)); } -void MachOPlatform::registerInitInfo(JITDylib &JD, - JITTargetAddress ObjCImageInfoAddr, - ExecutorAddressRange ModInits, - ExecutorAddressRange ObjCSelRefs, - ExecutorAddressRange ObjCClassList) { - std::lock_guard Lock(InitSeqsMutex); +void MachOPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getInitializers(\"" << JDName << "\")\n"; + }); - auto &InitSeq = InitSeqs[&JD]; + JITDylib *JD = ES.getJITDylibByName(JDName); + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n"; + }); + SendResult(make_error("No JITDylib named " + JDName, + inconvertibleErrorCode())); + return; + } - InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); + getInitializersLookupPhase(std::move(SendResult), *JD); +} - if (ModInits.StartAddress) - InitSeq.addModInitsSection(std::move(ModInits)); +void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddress Handle) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getDeinitializers(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); - if (ObjCSelRefs.StartAddress) - InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); + JITDylib *JD = nullptr; - if (ObjCClassList.StartAddress) - InitSeq.addObjCClassListSection(std::move(ObjCClassList)); -} + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } -static Expected getSectionExtent(jitlink::LinkGraph &G, - StringRef SectionName) { - auto *Sec = G.findSectionByName(SectionName); - if (!Sec) - return ExecutorAddressRange(); - jitlink::SectionRange R(*Sec); - if (R.getSize() % G.getPointerSize() != 0) - return make_error(SectionName + " section size is not a " - "multiple of the pointer size", - inconvertibleErrorCode()); - return ExecutorAddressRange(ExecutorAddress(R.getStart()), - ExecutorAddress(R.getEnd())); + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + SendResult(MachOJITDylibDeinitializerSequence()); } -void MachOPlatform::InitScraperPlugin::modifyPassConfig( - MaterializationResponsibility &MR, jitlink::LinkGraph &LG, - jitlink::PassConfiguration &Config) { +void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, + ExecutorAddress Handle, + StringRef SymbolName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_lookupSymbol(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); - if (!MR.getInitializerSymbol()) + JITDylib *JD = nullptr; + + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); return; + } + + // FIXME: Proper mangling. + auto MangledName = ("_" + SymbolName).str(); + ES.lookup( + LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, + SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready, + [SendResult = std::move(SendResult)](Expected Result) mutable { + if (Result) { + assert(Result->size() == 1 && "Unexpected result map count"); + SendResult(ExecutorAddress(Result->begin()->second.getAddress())); + } else + SendResult(Result.takeError()); + }, + NoDependenciesToRegister); +} - Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { - JITLinkSymbolSet InitSectionSyms; - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__mod_init_func"); - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__objc_selrefs"); - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__objc_classlist"); +Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { - if (!InitSectionSyms.empty()) { - std::lock_guard Lock(InitScraperMutex); - InitSymbolDeps[&MR] = std::move(InitSectionSyms); - } + std::pair Symbols[] = { + {"___orc_rt_macho_platform_bootstrap", &orc_rt_macho_platform_bootstrap}, + {"___orc_rt_macho_platform_shutdown", &orc_rt_macho_platform_shutdown}, + {"___orc_rt_macho_register_object_sections", + &orc_rt_macho_register_object_sections}}; + + SymbolLookupSet RuntimeSymbols; + std::vector> AddrsToRecord; + for (const auto &KV : Symbols) { + auto Name = ES.intern(KV.first); + RuntimeSymbols.add(Name); + AddrsToRecord.push_back({std::move(Name), KV.second}); + } + + auto RuntimeSymbolAddrs = ES.lookup( + {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols); + if (!RuntimeSymbolAddrs) + return RuntimeSymbolAddrs.takeError(); + + for (const auto &KV : AddrsToRecord) { + auto &Name = KV.first; + assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); + KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); + } + + if (auto Err = + EPC.runSPSWrapper(orc_rt_macho_platform_bootstrap.getValue())) + return Err; - if (auto Err = processObjCImageInfo(G, MR)) + // FIXME: Ordering is fuzzy here. We're probably best off saying + // "behavior is undefined if code that uses the runtime is added before + // the platform constructor returns", then move all this to the constructor. + RuntimeBootstrapped = true; + std::vector DeferredPOSRs; + { + std::lock_guard Lock(PlatformMutex); + DeferredPOSRs = std::move(BootstrapPOSRs); + } + + for (auto &D : DeferredPOSRs) + if (auto Err = registerPerObjectSections(D)) return Err; - return Error::success(); - }); + return Error::success(); +} + +Error MachOPlatform::registerInitInfo( + JITDylib &JD, ArrayRef InitSections) { - Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib()]( - jitlink::LinkGraph &G) -> Error { - ExecutorAddressRange ModInits, ObjCSelRefs, ObjCClassList; + std::unique_lock Lock(PlatformMutex); - JITTargetAddress ObjCImageInfoAddr = 0; - if (auto *ObjCImageInfoSec = - G.findSectionByName("__DATA,__objc_image_info")) { - if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) - ObjCImageInfoAddr = Addr; + MachOJITDylibInitializers *InitSeq = nullptr; + { + auto I = InitSeqs.find(&JD); + if (I == InitSeqs.end()) { + // If there's no init sequence entry yet then we need to look up the + // header symbol to force creation of one. + Lock.unlock(); + + auto SearchOrder = + JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; }); + if (auto Err = ES.lookup(SearchOrder, MachOHeaderStartSymbol).takeError()) + return Err; + + Lock.lock(); + I = InitSeqs.find(&JD); + assert(I != InitSeqs.end() && + "Entry missing after header symbol lookup?"); } + InitSeq = &I->second; + } - // Record __mod_init_func. - if (auto ModInitsOrErr = getSectionExtent(G, "__DATA,__mod_init_func")) - ModInits = std::move(*ModInitsOrErr); - else - return ModInitsOrErr.takeError(); - - // Record __objc_selrefs. - if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__DATA,__objc_selrefs")) - ObjCSelRefs = std::move(*ObjCSelRefsOrErr); - else - return ObjCSelRefsOrErr.takeError(); - - // Record __objc_classlist. - if (auto ObjCClassListOrErr = - getSectionExtent(G, "__DATA,__objc_classlist")) - ObjCClassList = std::move(*ObjCClassListOrErr); - else - return ObjCClassListOrErr.takeError(); - - // Dump the scraped inits. - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; - dbgs() << " __objc_selrefs: "; - auto NumObjCSelRefs = ObjCSelRefs.size().getValue() / sizeof(uintptr_t); - if (NumObjCSelRefs) - dbgs() << NumObjCSelRefs << " pointer(s) at " - << formatv("{0:x16}", ObjCSelRefs.StartAddress.getValue()) - << "\n"; - else - dbgs() << "none\n"; - - dbgs() << " __objc_classlist: "; - auto NumObjCClasses = ObjCClassList.size().getValue() / sizeof(uintptr_t); - if (NumObjCClasses) - dbgs() << NumObjCClasses << " pointer(s) at " - << formatv("{0:x16}", ObjCClassList.StartAddress.getValue()) - << "\n"; - else - dbgs() << "none\n"; - - dbgs() << " __mod_init_func: "; - auto NumModInits = ModInits.size().getValue() / sizeof(uintptr_t); - if (NumModInits) - dbgs() << NumModInits << " pointer(s) at " - << formatv("{0:x16}", ModInits.StartAddress.getValue()) << "\n"; - else - dbgs() << "none\n"; - }); + for (auto *Sec : InitSections) { + // FIXME: Avoid copy here. + jitlink::SectionRange R(*Sec); + InitSeq->InitSections[Sec->getName()].push_back( + {ExecutorAddress(R.getStart()), ExecutorAddress(R.getEnd())}); + } - MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), - std::move(ObjCSelRefs), std::move(ObjCClassList)); + return Error::success(); +} - return Error::success(); - }); +Error MachOPlatform::registerPerObjectSections( + const MachOPerObjectSectionsToRegister &POSR) { + + if (!orc_rt_macho_register_object_sections) + return make_error("Attempting to register per-object " + "sections, but runtime support has not " + "been loaded yet", + inconvertibleErrorCode()); + + Error ErrResult = Error::success(); + if (auto Err = EPC.runSPSWrapper( + orc_rt_macho_register_object_sections.getValue(), ErrResult, POSR)) + return Err; + return ErrResult; +} + +void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, + jitlink::PassConfiguration &Config) { + + // If the initializer symbol is the MachOHeader start symbol then just add + // the macho header support passes. + if (MR.getInitializerSymbol() == MP.MachOHeaderStartSymbol) { + addMachOHeaderSupportPasses(MR, Config); + // The header materialization unit doesn't require any other support, so we + // can bail out early. + return; + } + + // If the object contains initializers then add passes to record them. + if (MR.getInitializerSymbol()) + addInitializerSupportPasses(MR, Config); + + // Add passes for eh-frame support. + addEHSupportPasses(MR, Config); } ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap -MachOPlatform::InitScraperPlugin::getSyntheticSymbolDependencies( +MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( MaterializationResponsibility &MR) { - std::lock_guard Lock(InitScraperMutex); + std::lock_guard Lock(PluginMutex); auto I = InitSymbolDeps.find(&MR); if (I != InitSymbolDeps.end()) { SyntheticSymbolDependenciesMap Result; @@ -275,93 +596,135 @@ MachOPlatform::InitScraperPlugin::getSyntheticSymbolDependencies( return SyntheticSymbolDependenciesMap(); } -void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( - JITLinkSymbolSet &Symbols, jitlink::LinkGraph &G, StringRef SectionName) { - if (auto *Sec = G.findSectionByName(SectionName)) { - auto SecBlocks = Sec->blocks(); - if (!llvm::empty(SecBlocks)) - Symbols.insert( - &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); - } +void MachOPlatform::MachOPlatformPlugin::addInitializerSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + /// Preserve init sections. + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + return preserveInitSections(G, MR); + }); + + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerInitSections(G, JD); + }); } -Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( - jitlink::LinkGraph &G, MaterializationResponsibility &MR) { +void MachOPlatform::MachOPlatformPlugin::addMachOHeaderSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { - // If there's an ObjC imagine info then either - // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In - // this case we name and record it. - // OR - // (2) We already have a recorded __objc_imageinfo for this JITDylib, - // in which case we just verify it. - auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); - if (!ObjCImageInfo) + Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *MP.MachOHeaderStartSymbol; + }); + assert(I != G.defined_symbols().end() && + "Missing MachO header start symbol"); + { + std::lock_guard Lock(MP.PlatformMutex); + JITTargetAddress HeaderAddr = (*I)->getAddress(); + MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); + MP.InitSeqs.insert( + std::make_pair(&JD, MachOJITDylibInitializers( + JD.getName(), ExecutorAddress(HeaderAddr)))); + } return Error::success(); + }); +} - auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); +void MachOPlatform::MachOPlatformPlugin::addEHSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { - // Check that the section is not empty if present. - if (llvm::empty(ObjCImageInfoBlocks)) - return make_error("Empty __objc_imageinfo section in " + - G.getName(), - inconvertibleErrorCode()); + // Add a pass to register the final addresses of the eh-frame sections + // with the runtime. + Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { + MachOPerObjectSectionsToRegister POSR; - // Check that there's only one block in the section. - if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) - return make_error("Multiple blocks in __objc_imageinfo " - "section in " + - G.getName(), - inconvertibleErrorCode()); + if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + jitlink::SectionRange R(*EHFrameSection); + if (!R.empty()) + POSR.EHFrameSection = {ExecutorAddress(R.getStart()), + ExecutorAddress(R.getEnd())}; + } - // Check that the __objc_imageinfo section is unreferenced. - // FIXME: We could optimize this check if Symbols had a ref-count. - for (auto &Sec : G.sections()) { - if (&Sec != ObjCImageInfo) - for (auto *B : Sec.blocks()) - for (auto &E : B->edges()) - if (E.getTarget().isDefined() && - &E.getTarget().getBlock().getSection() == ObjCImageInfo) - return make_error("__objc_imageinfo is referenced " - "within file " + - G.getName(), - inconvertibleErrorCode()); + if (POSR.EHFrameSection.StartAddress) { + + // If we're still bootstrapping the runtime then just record this + // frame for now. + if (!MP.RuntimeBootstrapped) { + std::lock_guard Lock(MP.PlatformMutex); + MP.BootstrapPOSRs.push_back(POSR); + return Error::success(); + } + + // Otherwise register it immediately. + if (auto Err = MP.registerPerObjectSections(POSR)) + return Err; + } + + return Error::success(); + }); +} + +Error MachOPlatform::MachOPlatformPlugin::preserveInitSections( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + JITLinkSymbolSet InitSectionSymbols; + for (auto &InitSectionName : InitSectionNames) { + // Skip non-init sections. + auto *InitSection = G.findSectionByName(InitSectionName); + if (!InitSection) + continue; + + // Make a pass over live symbols in the section: those blocks are already + // preserved. + DenseSet AlreadyLiveBlocks; + for (auto &Sym : InitSection->symbols()) { + auto &B = Sym->getBlock(); + if (Sym->isLive() && Sym->getOffset() == 0 && + Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) { + InitSectionSymbols.insert(Sym); + AlreadyLiveBlocks.insert(&B); + } + } + + // Add anonymous symbols to preserve any not-already-preserved blocks. + for (auto *B : InitSection->blocks()) + if (!AlreadyLiveBlocks.count(B)) + InitSectionSymbols.insert( + &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true)); } - auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); - auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); - auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness()); - auto Flags = - support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); - - // Lock the mutex while we verify / update the ObjCImageInfos map. - std::lock_guard Lock(InitScraperMutex); - - auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); - if (ObjCImageInfoItr != ObjCImageInfos.end()) { - // We've already registered an __objc_imageinfo section. Verify the - // content of this new section matches, then delete it. - if (ObjCImageInfoItr->second.first != Version) - return make_error( - "ObjC version in " + G.getName() + - " does not match first registered version", - inconvertibleErrorCode()); - if (ObjCImageInfoItr->second.second != Flags) - return make_error("ObjC flags in " + G.getName() + - " do not match first registered flags", - inconvertibleErrorCode()); - - // __objc_imageinfo is valid. Delete the block. - for (auto *S : ObjCImageInfo->symbols()) - G.removeDefinedSymbol(*S); - G.removeBlock(ObjCImageInfoBlock); - } else { - // We haven't registered an __objc_imageinfo section yet. Register and - // move on. The section should already be marked no-dead-strip. - ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + if (!InitSectionSymbols.empty()) { + std::lock_guard Lock(PluginMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); } return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::registerInitSections( + jitlink::LinkGraph &G, JITDylib &JD) { + + SmallVector InitSections; + + for (auto InitSectionName : InitSectionNames) + if (auto *Sec = G.findSectionByName(InitSectionName)) + InitSections.push_back(Sec); + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; + for (auto *Sec : InitSections) { + jitlink::SectionRange R(*Sec); + dbgs() << " " << Sec->getName() << ": " + << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; + } + }); + + return MP.registerInitInfo(JD, InitSections); +} + } // End namespace orc. } // End namespace llvm. diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp index 9a92581..7f197a5 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp @@ -141,7 +141,7 @@ int main(int argc, char *argv[]) { ExitOnErr.setBanner(std::string(argv[0]) + ":"); using JITLinkExecutorEndpoint = - shared::MultiThreadedRPCEndpoint; + shared::SingleThreadedRPCEndpoint; shared::registerStringError(); diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp index f829831..ec91597 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -20,6 +20,7 @@ #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/MC/MCAsmInfo.h" @@ -159,6 +160,15 @@ static cl::opt OutOfProcessExecutorConnect( "oop-executor-connect", cl::desc("Connect to an out-of-process executor via TCP")); +// TODO: Default to false if compiler-rt is not built. +static cl::opt UseOrcRuntime("use-orc-runtime", + cl::desc("Do not required/load ORC runtime"), + cl::init(true)); + +static cl::opt + OrcRuntimePath("orc-runtime-path", cl::desc("Add orc runtime to session"), + cl::init("")); + ExitOnError ExitOnErr; LLVM_ATTRIBUTE_USED void linkComponents() { @@ -167,6 +177,14 @@ LLVM_ATTRIBUTE_USED void linkComponents() { << (void *)&llvm_orc_registerJITLoaderGDBWrapper; } +static bool UseTestResultOverride = false; +static int64_t TestResultOverride = 0; + +extern "C" void llvm_jitlink_setTestResultOverride(int64_t Value) { + TestResultOverride = Value; + UseTestResultOverride = true; +} + namespace llvm { static raw_ostream & @@ -588,6 +606,31 @@ Error LLVMJITLinkObjectLinkingLayer::add(ResourceTrackerSP RT, return JD.define(std::move(MU), std::move(RT)); } +static Error loadProcessSymbols(Session &S) { + auto FilterMainEntryPoint = + [EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) { + return Name != EPName; + }; + S.MainJD->addGenerator( + ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess( + *S.EPC, std::move(FilterMainEntryPoint)))); + + return Error::success(); +} + +static Error loadDylibs(Session &S) { + LLVM_DEBUG(dbgs() << "Loading dylibs...\n"); + for (const auto &Dylib : Dylibs) { + LLVM_DEBUG(dbgs() << " " << Dylib << "\n"); + auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str()); + if (!G) + return G.takeError(); + S.MainJD->addGenerator(std::move(*G)); + } + + return Error::success(); +} + Expected> LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor() { #ifndef LLVM_ON_UNIX @@ -796,40 +839,46 @@ Expected> Session::Create(Triple TT) { if (!PageSize) return PageSize.takeError(); - /// If -oop-executor is passed then launch the executor. std::unique_ptr EPC; if (OutOfProcessExecutor.getNumOccurrences()) { + /// If -oop-executor is passed then launch the executor. if (auto REPC = LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor()) EPC = std::move(*REPC); else return REPC.takeError(); } else if (OutOfProcessExecutorConnect.getNumOccurrences()) { + /// If -oop-executor-connect is passed then connect to the executor. if (auto REPC = LLVMJITLinkRemoteExecutorProcessControl::ConnectToExecutor()) EPC = std::move(*REPC); else return REPC.takeError(); - } else + } else { + /// Otherwise use SelfExecutorProcessControl to target the current process. EPC = std::make_unique( std::make_shared(), std::move(TT), *PageSize, createMemoryManager()); + } Error Err = Error::success(); std::unique_ptr S(new Session(std::move(EPC), Err)); - if (Err) - return std::move(Err); - return std::move(S); + + // FIXME: Errors destroy the session, leaving the SymbolStringPtrs dangling, + // so just exit here. We could fix this by having errors keep the pool alive. + ExitOnErr(std::move(Err)); + return S; } Session::~Session() { if (auto Err = ES.endSession()) ES.reportError(std::move(Err)); + if (auto Err = EPC->disconnect()) + ES.reportError(std::move(Err)); } -// FIXME: Move to createJITDylib if/when we start using Platform support in -// llvm-jitlink. Session::Session(std::unique_ptr EPC, Error &Err) - : EPC(std::move(EPC)), ObjLayer(*this, this->EPC->getMemMgr()) { + : EPC(std::move(EPC)), ES(this->EPC->getSymbolStringPool()), + ObjLayer(*this, this->EPC->getMemMgr()) { /// Local ObjectLinkingLayer::Plugin class to forward modifyPassConfig to the /// Session. @@ -863,7 +912,21 @@ Session::Session(std::unique_ptr EPC, Error &Err) return; } - if (!NoExec && !this->EPC->getTargetTriple().isOSWindows()) { + if (!NoProcessSymbols) + ExitOnErr(loadProcessSymbols(*this)); + ExitOnErr(loadDylibs(*this)); + + // Set up the platform. + if (this->EPC->getTargetTriple().isOSBinFormatMachO() && UseOrcRuntime) { + if (auto P = MachOPlatform::Create(ES, ObjLayer, *this->EPC, *MainJD, + OrcRuntimePath.c_str())) + ES.setPlatform(std::move(*P)); + else { + Err = P.takeError(); + return; + } + } else if (!NoExec && !this->EPC->getTargetTriple().isOSWindows() && + !this->EPC->getTargetTriple().isOSBinFormatMachO()) { ObjLayer.addPlugin(std::make_unique( ES, ExitOnErr(EPCEHFrameRegistrar::Create(*this->EPC)))); ObjLayer.addPlugin(std::make_unique( @@ -1044,18 +1107,41 @@ static Triple getFirstFileTriple() { return FirstTT; } -static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { - // Set the entry point name if not specified. - if (EntryPointName.empty()) { - if (TT.getObjectFormat() == Triple::MachO) - EntryPointName = "_main"; - else - EntryPointName = "main"; +static bool isOrcRuntimeSupported(const Triple &TT) { + switch (TT.getObjectFormat()) { + case Triple::MachO: + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } + default: + return false; } +} + +static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { + + // If we're in noexec mode and the user didn't explicitly specify + // -use-orc-runtime then don't use it. + if (NoExec && UseOrcRuntime.getNumOccurrences() == 0) + UseOrcRuntime = false; // -noexec and --args should not be used together. if (NoExec && !InputArgv.empty()) - outs() << "Warning: --args passed to -noexec run will be ignored.\n"; + errs() << "Warning: --args passed to -noexec run will be ignored.\n"; + + // Turn off UseOrcRuntime on platforms where it's not supported. + if (UseOrcRuntime && !isOrcRuntimeSupported(TT)) { + errs() << "Warning: Orc runtime not available for target platform. " + "Use -use-orc-runtime=false to suppress this warning.\n"; + UseOrcRuntime = false; + } + + // Set the entry point name if not specified. + if (EntryPointName.empty()) + EntryPointName = TT.getObjectFormat() == Triple::MachO ? "_main" : "main"; // If -slab-address is passed, require -slab-allocate and -noexec if (SlabAddress != ~0ULL) { @@ -1080,35 +1166,27 @@ static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { SmallString<256> OOPExecutorPath(sys::fs::getMainExecutable( ArgV0, reinterpret_cast(&sanitizeArguments))); sys::path::remove_filename(OOPExecutorPath); - if (OOPExecutorPath.back() != '/') - OOPExecutorPath += '/'; - OOPExecutorPath += "llvm-jitlink-executor"; + sys::path::append(OOPExecutorPath, "llvm-jitlink-executor"); OutOfProcessExecutor = OOPExecutorPath.str().str(); } - return Error::success(); -} - -static Error loadProcessSymbols(Session &S) { - auto FilterMainEntryPoint = - [EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) { - return Name != EPName; - }; - S.MainJD->addGenerator( - ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess( - *S.EPC, std::move(FilterMainEntryPoint)))); - - return Error::success(); -} - -static Error loadDylibs(Session &S) { - for (const auto &Dylib : Dylibs) { - auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str()); - if (!G) - return G.takeError(); - S.MainJD->addGenerator(std::move(*G)); + // If we're loading the Orc runtime then determine the path for it. + if (UseOrcRuntime) { + if (OrcRuntimePath.empty()) { + SmallString<256> DefaultOrcRuntimePath(sys::fs::getMainExecutable( + ArgV0, reinterpret_cast(&sanitizeArguments))); + sys::path::remove_filename( + DefaultOrcRuntimePath); // remove 'llvm-jitlink' + while (!DefaultOrcRuntimePath.empty() && + DefaultOrcRuntimePath.back() == '/') + DefaultOrcRuntimePath.pop_back(); + if (DefaultOrcRuntimePath.endswith("bin")) + sys::path::remove_filename(DefaultOrcRuntimePath); // remove 'bin' + sys::path::append(DefaultOrcRuntimePath, + "lib/clang/13.0.0/lib/darwin/libclang_rt.orc_osx.a"); + OrcRuntimePath = DefaultOrcRuntimePath.str().str(); + } } - return Error::success(); } @@ -1227,6 +1305,11 @@ static Error loadObjects(Session &S) { static Error runChecks(Session &S) { + if (CheckFiles.empty()) + return Error::success(); + + LLVM_DEBUG(dbgs() << "Running checks...\n"); + auto TripleName = S.EPC->getTargetTriple().str(); std::string ErrorStr; const Target *TheTarget = TargetRegistry::lookupTarget(TripleName, ErrorStr); @@ -1310,9 +1393,15 @@ static Error runChecks(Session &S) { } static void dumpSessionStats(Session &S) { + if (!ShowSizes) + return; + if (UseOrcRuntime) + outs() << "Note: Session stats include runtime and entry point lookup, but " + "not JITDylib initialization/deinitialization.\n"; if (ShowSizes) - outs() << "Total size of all blocks before pruning: " << S.SizeBeforePruning - << "\nTotal size of all blocks after fixups: " << S.SizeAfterFixups + outs() << " Total size of all blocks before pruning: " + << S.SizeBeforePruning + << "\n Total size of all blocks after fixups: " << S.SizeAfterFixups << "\n"; } @@ -1320,6 +1409,34 @@ static Expected getMainEntryPoint(Session &S) { return S.ES.lookup(S.JDSearchOrder, EntryPointName); } +static Expected getOrcRuntimeEntryPoint(Session &S) { + std::string RuntimeEntryPoint = "__orc_rt_run_program_wrapper"; + if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO) + RuntimeEntryPoint = '_' + RuntimeEntryPoint; + return S.ES.lookup(S.JDSearchOrder, RuntimeEntryPoint); +} + +static Expected runWithRuntime(Session &S, + JITTargetAddress EntryPointAddress) { + StringRef DemangledEntryPoint = EntryPointName; + if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO && + DemangledEntryPoint.front() == '_') + DemangledEntryPoint = DemangledEntryPoint.drop_front(); + using SPSRunProgramSig = + int64_t(SPSString, SPSString, SPSSequence); + int64_t Result; + if (auto Err = S.EPC->runSPSWrapper( + EntryPointAddress, Result, S.MainJD->getName(), DemangledEntryPoint, + static_cast &>(InputArgv))) + return std::move(Err); + return Result; +} + +static Expected runWithoutRuntime(Session &S, + JITTargetAddress EntryPointAddress) { + return S.EPC->runAsMain(EntryPointAddress, InputArgv); +} + namespace { struct JITLinkTimers { TimerGroup JITLinkTG{"llvm-jitlink timers", "timers for llvm-jitlink phases"}; @@ -1352,21 +1469,23 @@ int main(int argc, char *argv[]) { ExitOnErr(loadObjects(*S)); } - if (!NoProcessSymbols) - ExitOnErr(loadProcessSymbols(*S)); - ExitOnErr(loadDylibs(*S)); - if (PhonyExternals) addPhonyExternalsGenerator(*S); - if (ShowInitialExecutionSessionState) S->ES.dump(outs()); JITEvaluatedSymbol EntryPoint = 0; { TimeRegion TR(Timers ? &Timers->LinkTimer : nullptr); + // Find the entry-point function unconditionally, since we want to force + // it to be materialized to collect stats. EntryPoint = ExitOnErr(getMainEntryPoint(*S)); + + // If we're running with the ORC runtime then replace the entry-point + // with the __orc_rt_run_program symbol. + if (UseOrcRuntime) + EntryPoint = ExitOnErr(getOrcRuntimeEntryPoint(*S)); } if (ShowAddrs) @@ -1381,12 +1500,20 @@ int main(int argc, char *argv[]) { int Result = 0; { + LLVM_DEBUG(dbgs() << "Running \"" << EntryPointName << "\"...\n"); TimeRegion TR(Timers ? &Timers->RunTimer : nullptr); - Result = ExitOnErr(S->EPC->runAsMain(EntryPoint.getAddress(), InputArgv)); + if (UseOrcRuntime) + Result = ExitOnErr(runWithRuntime(*S, EntryPoint.getAddress())); + else + Result = ExitOnErr(runWithoutRuntime(*S, EntryPoint.getAddress())); } - ExitOnErr(S->ES.endSession()); - ExitOnErr(S->EPC->disconnect()); + // Destroy the session. + S.reset(); + + // If the executing code set a test result override then use that. + if (UseTestResultOverride) + Result = TestResultOverride; return Result; } diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.h b/llvm/tools/llvm-jitlink/llvm-jitlink.h index 750e543..5050b2f 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink.h +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.h @@ -111,7 +111,7 @@ private: struct Session { std::unique_ptr EPC; orc::ExecutionSession ES; - orc::JITDylib *MainJD; + orc::JITDylib *MainJD = nullptr; LLVMJITLinkObjectLinkingLayer ObjLayer; std::vector JDSearchOrder; diff --git a/llvm/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp index 5c784c1..f4a6b45 100644 --- a/llvm/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h" #include "gtest/gtest.h" using namespace llvm; @@ -158,3 +159,9 @@ TEST(SimplePackedSerializationTest, ArgListSerialization) { EXPECT_EQ(Arg2, ArgOut2); EXPECT_EQ(Arg3, ArgOut3); } + +TEST(SimplePackedSerialization, StringMap) { + StringMap M({{"A", 1}, {"B", 2}}); + blobSerializationRoundTrip>, + StringMap>(M); +} -- 2.7.4