diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine')
8 files changed, 753 insertions, 3 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp index 43b9c2ba400b..dc07eaeaf615 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp @@ -524,4 +524,4 @@ COFFLinkGraphBuilder::exportCOMDATSymbol(COFFSymbolIndex SymIndex, } } // namespace jitlink -} // namespace llvm
\ No newline at end of file +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp b/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp index 0fc366bf505f..2a60d8206f63 100644 --- a/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.cpp @@ -107,7 +107,7 @@ Error DWARFRecordSectionSplitter::processBlock( } uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; - auto &NewBlock = G.splitBlock(B, BlockSize); + auto &NewBlock = G.splitBlock(B, BlockSize, &Cache); (void)NewBlock; LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); } diff --git a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index c60f4b3b263c..70a3c404d836 100644 --- a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -48,6 +48,10 @@ JITTargetMachineBuilder::createTargetMachine() { if (!TheTarget) return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + if (!TheTarget->hasJIT()) + return make_error<StringError>("Target has no JIT support", + inconvertibleErrorCode()); + auto *TM = TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(), Options, RM, CM, OptLevel, /*JIT*/ true); diff --git a/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp new file mode 100644 index 000000000000..c2e7baabb994 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp @@ -0,0 +1,135 @@ +//=== MapperJITLinkMemoryManager.cpp - Memory management with MemoryMapper ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/Process.h" + +#include <limits> + +using namespace llvm::jitlink; + +namespace llvm { +namespace orc { + +class MapperJITLinkMemoryManager::InFlightAlloc + : public JITLinkMemoryManager::InFlightAlloc { +public: + InFlightAlloc(MapperJITLinkMemoryManager &Parent, LinkGraph &G, + ExecutorAddr AllocAddr, + std::vector<MemoryMapper::AllocInfo::SegInfo> Segs) + : Parent(Parent), G(G), AllocAddr(AllocAddr), Segs(std::move(Segs)) {} + + void finalize(OnFinalizedFunction OnFinalize) override { + MemoryMapper::AllocInfo AI; + AI.MappingBase = AllocAddr; + + std::swap(AI.Segments, Segs); + std::swap(AI.Actions, G.allocActions()); + + Parent.Mapper->initialize(AI, [&](Expected<ExecutorAddr> Result) { + if (!Result) { + OnFinalize(Result.takeError()); + return; + } + + OnFinalize(FinalizedAlloc(*Result)); + }); + } + + void abandon(OnAbandonedFunction OnFinalize) override { + Parent.Mapper->release({AllocAddr}, std::move(OnFinalize)); + } + +private: + MapperJITLinkMemoryManager &Parent; + LinkGraph &G; + ExecutorAddr AllocAddr; + std::vector<MemoryMapper::AllocInfo::SegInfo> Segs; +}; + +MapperJITLinkMemoryManager::MapperJITLinkMemoryManager( + std::unique_ptr<MemoryMapper> Mapper) + : Mapper(std::move(Mapper)) {} + +void MapperJITLinkMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, + OnAllocatedFunction OnAllocated) { + BasicLayout BL(G); + + // find required address space + auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(Mapper->getPageSize()); + if (!SegsSizes) { + OnAllocated(SegsSizes.takeError()); + return; + } + + // Check if total size fits in address space + if (SegsSizes->total() > std::numeric_limits<size_t>::max()) { + OnAllocated(make_error<JITLinkError>( + formatv("Total requested size {:x} for graph {} exceeds address space", + SegsSizes->total(), G.getName()))); + return; + } + + Mapper->reserve( + SegsSizes->total(), + [this, &G, BL = std::move(BL), OnAllocated = std::move(OnAllocated)]( + Expected<ExecutorAddrRange> Result) mutable { + if (!Result) { + return OnAllocated(Result.takeError()); + } + + auto NextSegAddr = Result->Start; + + std::vector<MemoryMapper::AllocInfo::SegInfo> SegInfos; + + for (auto &KV : BL.segments()) { + auto &AG = KV.first; + auto &Seg = KV.second; + + auto TotalSize = Seg.ContentSize + Seg.ZeroFillSize; + + Seg.Addr = NextSegAddr; + Seg.WorkingMem = Mapper->prepare(NextSegAddr, TotalSize); + + NextSegAddr += alignTo(TotalSize, Mapper->getPageSize()); + + MemoryMapper::AllocInfo::SegInfo SI; + SI.Offset = Seg.Addr - Result->Start; + SI.ContentSize = Seg.ContentSize; + SI.ZeroFillSize = Seg.ZeroFillSize; + SI.Prot = (toSysMemoryProtectionFlags(AG.getMemProt())); + SI.WorkingMem = Seg.WorkingMem; + + SegInfos.push_back(SI); + } + + if (auto Err = BL.apply()) { + OnAllocated(std::move(Err)); + return; + } + + OnAllocated(std::make_unique<InFlightAlloc>(*this, G, Result->Start, + std::move(SegInfos))); + }); +} + +void MapperJITLinkMemoryManager::deallocate( + std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) { + std::vector<ExecutorAddr> Bases; + Bases.reserve(Allocs.size()); + for (auto &FA : Allocs) { + Bases.push_back(FA.getAddress()); + FA.release(); + } + Mapper->release(Bases, std::move(OnDeallocated)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp index 8b3fbd7117e2..ca3f64b8a409 100644 --- a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -8,11 +8,33 @@ #include "llvm/ExecutionEngine/Orc/MemoryMapper.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/WindowsError.h" + +#if defined(LLVM_ON_UNIX) +#include <fcntl.h> +#include <sys/mman.h> +#include <unistd.h> +#elif defined(_WIN32) +#include <windows.h> +#endif + namespace llvm { namespace orc { MemoryMapper::~MemoryMapper() {} +InProcessMemoryMapper::InProcessMemoryMapper(size_t PageSize) + : PageSize(PageSize) {} + +Expected<std::unique_ptr<InProcessMemoryMapper>> +InProcessMemoryMapper::Create() { + auto PageSize = sys::Process::getPageSize(); + if (!PageSize) + return PageSize.takeError(); + return std::make_unique<InProcessMemoryMapper>(*PageSize); +} + void InProcessMemoryMapper::reserve(size_t NumBytes, OnReservedFunction OnReserved) { std::error_code EC; @@ -147,6 +169,238 @@ InProcessMemoryMapper::~InProcessMemoryMapper() { cantFail(F.get()); } +// SharedMemoryMapper + +SharedMemoryMapper::SharedMemoryMapper(ExecutorProcessControl &EPC, + SymbolAddrs SAs, size_t PageSize) + : EPC(EPC), SAs(SAs), PageSize(PageSize) {} + +Expected<std::unique_ptr<SharedMemoryMapper>> +SharedMemoryMapper::Create(ExecutorProcessControl &EPC, SymbolAddrs SAs) { + auto PageSize = sys::Process::getPageSize(); + if (!PageSize) + return PageSize.takeError(); + + return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize); +} + +void SharedMemoryMapper::reserve(size_t NumBytes, + OnReservedFunction OnReserved) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>( + SAs.Reserve, + [this, NumBytes, OnReserved = std::move(OnReserved)]( + Error SerializationErr, + Expected<std::pair<ExecutorAddr, std::string>> Result) mutable { + if (SerializationErr) { + cantFail(Result.takeError()); + return OnReserved(std::move(SerializationErr)); + } + + if (!Result) + return OnReserved(Result.takeError()); + + ExecutorAddr RemoteAddr; + std::string SharedMemoryName; + std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result); + + void *LocalAddr = nullptr; + +#if defined(LLVM_ON_UNIX) + + int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700); + if (SharedMemoryFile < 0) { + return OnReserved(errorCodeToError( + std::error_code(errno, std::generic_category()))); + } + + // this prevents other processes from accessing it by name + shm_unlink(SharedMemoryName.c_str()); + + LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED, + SharedMemoryFile, 0); + if (LocalAddr == MAP_FAILED) { + return OnReserved(errorCodeToError( + std::error_code(errno, std::generic_category()))); + } + + close(SharedMemoryFile); + +#elif defined(_WIN32) + + std::wstring WideSharedMemoryName(SharedMemoryName.begin(), + SharedMemoryName.end()); + HANDLE SharedMemoryFile = OpenFileMappingW( + FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str()); + if (!SharedMemoryFile) + return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); + + LocalAddr = + MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0); + if (!LocalAddr) { + CloseHandle(SharedMemoryFile); + return OnReserved(errorCodeToError(mapWindowsError(GetLastError()))); + } + + CloseHandle(SharedMemoryFile); + +#endif + { + std::lock_guard<std::mutex> Lock(Mutex); + Reservations.insert({RemoteAddr, {LocalAddr, NumBytes}}); + } + + OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes)); + }, + SAs.Instance, static_cast<uint64_t>(NumBytes)); + +#else + OnReserved(make_error<StringError>( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode())); +#endif +} + +char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { + auto R = Reservations.upper_bound(Addr); + assert(R != Reservations.begin() && "Attempt to prepare unknown range"); + R--; + + ExecutorAddrDiff Offset = Addr - R->first; + + return static_cast<char *>(R->second.LocalAddr) + Offset; +} + +void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, + OnInitializedFunction OnInitialized) { + auto Reservation = Reservations.find(AI.MappingBase); + assert(Reservation != Reservations.end() && + "Attempt to initialize unreserved range"); + + tpctypes::SharedMemoryFinalizeRequest FR; + + AI.Actions.swap(FR.Actions); + + FR.Segments.reserve(AI.Segments.size()); + + for (auto Segment : AI.Segments) { + char *Base = + static_cast<char *>(Reservation->second.LocalAddr) + Segment.Offset; + std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); + + tpctypes::SharedMemorySegFinalizeRequest SegReq; + SegReq.Prot = tpctypes::toWireProtectionFlags( + static_cast<sys::Memory::ProtectionFlags>(Segment.Prot)); + SegReq.Addr = AI.MappingBase + Segment.Offset; + SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; + + FR.Segments.push_back(SegReq); + } + + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>( + SAs.Initialize, + [OnInitialized = std::move(OnInitialized)]( + Error SerializationErr, Expected<ExecutorAddr> Result) mutable { + if (SerializationErr) { + cantFail(Result.takeError()); + return OnInitialized(std::move(SerializationErr)); + } + + OnInitialized(std::move(Result)); + }, + SAs.Instance, AI.MappingBase, std::move(FR)); +} + +void SharedMemoryMapper::deinitialize( + ArrayRef<ExecutorAddr> Allocations, + MemoryMapper::OnDeinitializedFunction OnDeinitialized) { + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>( + SAs.Deinitialize, + [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr, + Error Result) mutable { + if (SerializationErr) { + cantFail(std::move(Result)); + return OnDeinitialized(std::move(SerializationErr)); + } + + OnDeinitialized(std::move(Result)); + }, + SAs.Instance, Allocations); +} + +void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, + OnReleasedFunction OnReleased) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + Error Err = Error::success(); + + { + std::lock_guard<std::mutex> Lock(Mutex); + + for (auto Base : Bases) { + +#if defined(LLVM_ON_UNIX) + + if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0) + Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( + errno, std::generic_category()))); + +#elif defined(_WIN32) + + if (!UnmapViewOfFile(Reservations[Base].LocalAddr)) + joinErrors(std::move(Err), + errorCodeToError(mapWindowsError(GetLastError()))); + +#endif + + Reservations.erase(Base); + } + } + + EPC.callSPSWrapperAsync< + rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>( + SAs.Release, + [OnReleased = std::move(OnReleased), + Err = std::move(Err)](Error SerializationErr, Error Result) mutable { + if (SerializationErr) { + cantFail(std::move(Result)); + return OnReleased( + joinErrors(std::move(Err), std::move(SerializationErr))); + } + + return OnReleased(joinErrors(std::move(Err), std::move(Result))); + }, + SAs.Instance, Bases); +#else + OnReleased(make_error<StringError>( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode())); +#endif +} + +SharedMemoryMapper::~SharedMemoryMapper() { + std::vector<ExecutorAddr> ReservationAddrs; + if (!Reservations.empty()) { + std::lock_guard<std::mutex> Lock(Mutex); + { + ReservationAddrs.reserve(Reservations.size()); + for (const auto &R : Reservations) { + ReservationAddrs.push_back(R.first); + } + } + } + + std::promise<MSVCPError> P; + auto F = P.get_future(); + release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); + // FIXME: Release can actually fail. The error should be propagated. + // Meanwhile, a better option is to explicitly call release(). + cantFail(F.get()); +} + } // namespace orc } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp index 5eae33121eb9..dfdd846c46a7 100644 --- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -18,6 +18,7 @@ const char *SimpleExecutorDylibManagerOpenWrapperName = "__llvm_orc_SimpleExecutorDylibManager_open_wrapper"; const char *SimpleExecutorDylibManagerLookupWrapperName = "__llvm_orc_SimpleExecutorDylibManager_lookup_wrapper"; + const char *SimpleExecutorMemoryManagerInstanceName = "__llvm_orc_SimpleExecutorMemoryManager_Instance"; const char *SimpleExecutorMemoryManagerReserveWrapperName = @@ -26,6 +27,18 @@ const char *SimpleExecutorMemoryManagerFinalizeWrapperName = "__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper"; const char *SimpleExecutorMemoryManagerDeallocateWrapperName = "__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper"; + +const char *ExecutorSharedMemoryMapperServiceInstanceName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Instance"; +const char *ExecutorSharedMemoryMapperServiceReserveWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Reserve"; +const char *ExecutorSharedMemoryMapperServiceInitializeWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Initialize"; +const char *ExecutorSharedMemoryMapperServiceDeinitializeWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Deinitialize"; +const char *ExecutorSharedMemoryMapperServiceReleaseWrapperName = + "__llvm_orc_ExecutorSharedMemoryMapperService_Release"; + const char *MemoryWriteUInt8sWrapperName = "__llvm_orc_bootstrap_mem_write_uint8s_wrapper"; const char *MemoryWriteUInt16sWrapperName = @@ -36,10 +49,12 @@ const char *MemoryWriteUInt64sWrapperName = "__llvm_orc_bootstrap_mem_write_uint64s_wrapper"; const char *MemoryWriteBuffersWrapperName = "__llvm_orc_bootstrap_mem_write_buffers_wrapper"; + const char *RegisterEHFrameSectionWrapperName = "__llvm_orc_bootstrap_register_ehframe_section_wrapper"; const char *DeregisterEHFrameSectionWrapperName = "__llvm_orc_bootstrap_deregister_ehframe_section_wrapper"; + const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper"; } // end namespace rt diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp new file mode 100644 index 000000000000..6c9f099061ae --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp @@ -0,0 +1,341 @@ +//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h" + +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/WindowsError.h" + +#include <sstream> + +#if defined(LLVM_ON_UNIX) +#include <errno.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <unistd.h> +#endif + +#if defined(_WIN32) +static DWORD getWindowsProtectionFlags(unsigned Flags) { + switch (Flags & llvm::sys::Memory::MF_RWE_MASK) { + case llvm::sys::Memory::MF_READ: + return PAGE_READONLY; + case llvm::sys::Memory::MF_WRITE: + // Note: PAGE_WRITE is not supported by VirtualProtect + return PAGE_READWRITE; + case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE: + return PAGE_READWRITE; + case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_EXEC: + return PAGE_EXECUTE_READ; + case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE | + llvm::sys::Memory::MF_EXEC: + return PAGE_EXECUTE_READWRITE; + case llvm::sys::Memory::MF_EXEC: + return PAGE_EXECUTE; + default: + llvm_unreachable("Illegal memory protection flag specified!"); + } + // Provide a default return value as required by some compilers. + return PAGE_NOACCESS; +} +#endif + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +Expected<std::pair<ExecutorAddr, std::string>> +ExecutorSharedMemoryMapperService::reserve(uint64_t Size) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + +#if defined(LLVM_ON_UNIX) + + std::string SharedMemoryName; + { + std::stringstream SharedMemoryNameStream; + SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_' + << (++SharedMemoryCount); + SharedMemoryName = SharedMemoryNameStream.str(); + } + + int SharedMemoryFile = + shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700); + if (SharedMemoryFile < 0) + return errorCodeToError(std::error_code(errno, std::generic_category())); + + // by default size is 0 + if (ftruncate(SharedMemoryFile, Size) < 0) + return errorCodeToError(std::error_code(errno, std::generic_category())); + + void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0); + if (Addr == MAP_FAILED) + return errorCodeToError(std::error_code(errno, std::generic_category())); + + close(SharedMemoryFile); + +#elif defined(_WIN32) + + std::string SharedMemoryName; + { + std::stringstream SharedMemoryNameStream; + SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_' + << (++SharedMemoryCount); + SharedMemoryName = SharedMemoryNameStream.str(); + } + + std::wstring WideSharedMemoryName(SharedMemoryName.begin(), + SharedMemoryName.end()); + HANDLE SharedMemoryFile = CreateFileMappingW( + INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32, + Size & 0xffffffff, WideSharedMemoryName.c_str()); + if (!SharedMemoryFile) + return errorCodeToError(mapWindowsError(GetLastError())); + + void *Addr = MapViewOfFile(SharedMemoryFile, + FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0); + if (!Addr) { + CloseHandle(SharedMemoryFile); + return errorCodeToError(mapWindowsError(GetLastError())); + } + +#endif + + { + std::lock_guard<std::mutex> Lock(Mutex); + Reservations[Addr].Size = Size; +#if defined(_WIN32) + Reservations[Addr].SharedMemoryFile = SharedMemoryFile; +#endif + } + + return std::make_pair(ExecutorAddr::fromPtr(Addr), + std::move(SharedMemoryName)); +#else + return make_error<StringError>( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode()); +#endif +} + +Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( + ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + + ExecutorAddr MinAddr(~0ULL); + + // Contents are already in place + for (auto &Segment : FR.Segments) { + if (Segment.Addr < MinAddr) + MinAddr = Segment.Addr; + +#if defined(LLVM_ON_UNIX) + + int NativeProt = 0; + if (Segment.Prot & tpctypes::WPF_Read) + NativeProt |= PROT_READ; + if (Segment.Prot & tpctypes::WPF_Write) + NativeProt |= PROT_WRITE; + if (Segment.Prot & tpctypes::WPF_Exec) + NativeProt |= PROT_EXEC; + + if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt)) + return errorCodeToError(std::error_code(errno, std::generic_category())); + +#elif defined(_WIN32) + + DWORD NativeProt = + getWindowsProtectionFlags(fromWireProtectionFlags(Segment.Prot)); + + if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt, + &NativeProt)) + return errorCodeToError(mapWindowsError(GetLastError())); + +#endif + + if (Segment.Prot & tpctypes::WPF_Exec) + sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(), + Segment.Size); + } + + // Run finalization actions and get deinitlization action list. + auto DeinitializeActions = shared::runFinalizeActions(FR.Actions); + if (!DeinitializeActions) { + return DeinitializeActions.takeError(); + } + + { + std::lock_guard<std::mutex> Lock(Mutex); + Allocations[MinAddr].DeinitializationActions = + std::move(*DeinitializeActions); + Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr); + } + + return MinAddr; + +#else + return make_error<StringError>( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode()); +#endif +} + +Error ExecutorSharedMemoryMapperService::deinitialize( + const std::vector<ExecutorAddr> &Bases) { + Error AllErr = Error::success(); + + { + std::lock_guard<std::mutex> Lock(Mutex); + + for (auto Base : Bases) { + if (Error Err = shared::runDeallocActions( + Allocations[Base].DeinitializationActions)) { + AllErr = joinErrors(std::move(AllErr), std::move(Err)); + } + + Allocations.erase(Base); + } + } + + return AllErr; +} + +Error ExecutorSharedMemoryMapperService::release( + const std::vector<ExecutorAddr> &Bases) { +#if defined(LLVM_ON_UNIX) || defined(_WIN32) + Error Err = Error::success(); + + for (auto Base : Bases) { + std::vector<ExecutorAddr> AllocAddrs; + size_t Size; + +#if defined(_WIN32) + HANDLE SharedMemoryFile; +#endif + + { + std::lock_guard<std::mutex> Lock(Mutex); + auto &R = Reservations[Base.toPtr<void *>()]; + Size = R.Size; + +#if defined(_WIN32) + SharedMemoryFile = R.SharedMemoryFile; +#endif + + AllocAddrs.swap(R.Allocations); + } + + // deinitialize sub allocations + if (Error E = deinitialize(AllocAddrs)) + Err = joinErrors(std::move(Err), std::move(E)); + +#if defined(LLVM_ON_UNIX) + + if (munmap(Base.toPtr<void *>(), Size) != 0) + Err = joinErrors(std::move(Err), errorCodeToError(std::error_code( + errno, std::generic_category()))); + +#elif defined(_WIN32) + + if (!UnmapViewOfFile(Base.toPtr<void *>())) + Err = joinErrors(std::move(Err), + errorCodeToError(mapWindowsError(GetLastError()))); + + CloseHandle(SharedMemoryFile); + +#endif + + std::lock_guard<std::mutex> Lock(Mutex); + Reservations.erase(Base.toPtr<void *>()); + } + + return Err; +#else + return make_error<StringError>( + "SharedMemoryMapper is not supported on this platform yet", + inconvertibleErrorCode()); +#endif +} + +Error ExecutorSharedMemoryMapperService::shutdown() { + std::vector<ExecutorAddr> ReservationAddrs; + if (!Reservations.empty()) { + std::lock_guard<std::mutex> Lock(Mutex); + { + ReservationAddrs.reserve(Reservations.size()); + for (const auto &R : Reservations) { + ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); + } + } + } + return release(ReservationAddrs); + + return Error::success(); +} + +void ExecutorSharedMemoryMapperService::addBootstrapSymbols( + StringMap<ExecutorAddr> &M) { + M[rt::ExecutorSharedMemoryMapperServiceInstanceName] = + ExecutorAddr::fromPtr(this); + M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] = + ExecutorAddr::fromPtr(&reserveWrapper); + M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] = + ExecutorAddr::fromPtr(&initializeWrapper); + M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] = + ExecutorAddr::fromPtr(&deinitializeWrapper); + M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] = + ExecutorAddr::fromPtr(&releaseWrapper); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::reserve)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::initialize)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::deinitialize)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &ExecutorSharedMemoryMapperService::release)) + .release(); +} + +} // namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp index 7cadf3bb51a7..c848dd65fa7e 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp @@ -126,7 +126,8 @@ Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) { inconvertibleErrorCode())); char *Mem = Seg.Addr.toPtr<char *>(); - memcpy(Mem, Seg.Content.data(), Seg.Content.size()); + if (!Seg.Content.empty()) + memcpy(Mem, Seg.Content.data(), Seg.Content.size()); memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size()); assert(Seg.Size <= std::numeric_limits<size_t>::max()); if (auto EC = sys::Memory::protectMappedMemory( |