diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2021-06-13 19:31:46 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2021-06-13 19:37:19 +0000 |
commit | e8d8bef961a50d4dc22501cde4fb9fb0be1b2532 (patch) | |
tree | 94f04805f47bb7c59ae29690d8952b6074fff602 /contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink | |
parent | bb130ff39747b94592cb26d71b7cb097b9a4ea6b (diff) | |
parent | b60736ec1405bb0a8dd40989f67ef4c93da068ab (diff) | |
download | src-e8d8bef961a50d4dc22501cde4fb9fb0be1b2532.tar.gz src-e8d8bef961a50d4dc22501cde4fb9fb0be1b2532.zip |
Merge llvm-project main llvmorg-12-init-17869-g8e464dd76bef
This updates llvm, clang, compiler-rt, libc++, libunwind, lld, lldb and
openmp to llvmorg-12-init-17869-g8e464dd76bef, the last commit before the
upstream release/12.x branch was created.
PR: 255570
MFC after: 6 weeks
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink')
13 files changed, 902 insertions, 455 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index f1114e92c360..3602601287f4 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -10,6 +10,8 @@ #include "EHFrameSupportImpl.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/Config/config.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/Support/DynamicLibrary.h" #define DEBUG_TYPE "jitlink" @@ -117,10 +119,10 @@ Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, } EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName, - Edge::Kind FDEToCIE, Edge::Kind FDEToPCBegin, - Edge::Kind FDEToLSDA) - : EHFrameSectionName(EHFrameSectionName), FDEToCIE(FDEToCIE), - FDEToPCBegin(FDEToPCBegin), FDEToLSDA(FDEToLSDA) {} + unsigned PointerSize, Edge::Kind Delta64, + Edge::Kind Delta32, Edge::Kind NegDelta32) + : EHFrameSectionName(EHFrameSectionName), PointerSize(PointerSize), + Delta64(Delta64), Delta32(Delta32), NegDelta32(NegDelta32) {} Error EHFrameEdgeFixer::operator()(LinkGraph &G) { auto *EHFrame = G.findSectionByName(EHFrameSectionName); @@ -133,6 +135,11 @@ Error EHFrameEdgeFixer::operator()(LinkGraph &G) { return Error::success(); } + // Check that we support the graph's pointer size. + if (G.getPointerSize() != 4 && G.getPointerSize() != 8) + return make_error<JITLinkError>( + "EHFrameEdgeFixer only supports 32 and 64 bit targets"); + LLVM_DEBUG({ dbgs() << "EHFrameEdgeFixer: Processing " << EHFrameSectionName << "...\n"; }); @@ -257,7 +264,6 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, size_t RecordOffset, size_t RecordLength, size_t CIEDeltaFieldOffset) { - using namespace dwarf; LLVM_DEBUG(dbgs() << " Record is CIE\n"); @@ -328,11 +334,12 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, uint8_t LSDAPointerEncoding; if (auto Err = RecordReader.readInteger(LSDAPointerEncoding)) return Err; - if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + if (!isSupportedPointerEncoding(LSDAPointerEncoding)) return make_error<JITLinkError>( "Unsupported LSDA pointer encoding " + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + formatv("{0:x16}", CIESymbol.getAddress())); + CIEInfo.LSDAPointerEncoding = LSDAPointerEncoding; break; } case 'P': { @@ -340,7 +347,8 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, if (auto Err = RecordReader.readInteger(PersonalityPointerEncoding)) return Err; if (PersonalityPointerEncoding != - (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4)) + (dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | + dwarf::DW_EH_PE_sdata4)) return make_error<JITLinkError>( "Unspported personality pointer " "encoding " + @@ -355,12 +363,12 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, uint8_t FDEPointerEncoding; if (auto Err = RecordReader.readInteger(FDEPointerEncoding)) return Err; - if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + if (!isSupportedPointerEncoding(FDEPointerEncoding)) return make_error<JITLinkError>( - "Unsupported FDE address pointer " - "encoding " + + "Unsupported FDE pointer encoding " + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + formatv("{0:x16}", CIESymbol.getAddress())); + CIEInfo.FDEPointerEncoding = FDEPointerEncoding; break; } default: @@ -417,7 +425,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, else return CIEInfoOrErr.takeError(); assert(CIEInfo->CIESymbol && "CIEInfo has no CIE symbol set"); - B.addEdge(FDEToCIE, RecordOffset + CIEDeltaFieldOffset, + B.addEdge(NegDelta32, RecordOffset + CIEDeltaFieldOffset, *CIEInfo->CIESymbol, 0); } else { LLVM_DEBUG({ @@ -444,11 +452,13 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, JITTargetAddress PCBeginFieldOffset = RecordReader.getOffset(); auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset); if (PCEdgeItr == BlockEdges.end()) { - auto PCBeginDelta = readAbsolutePointer(PC.G, RecordReader); - if (!PCBeginDelta) - return PCBeginDelta.takeError(); - JITTargetAddress PCBegin = - RecordAddress + PCBeginFieldOffset + *PCBeginDelta; + auto PCBeginPtrInfo = + readEncodedPointer(CIEInfo->FDEPointerEncoding, + RecordAddress + PCBeginFieldOffset, RecordReader); + if (!PCBeginPtrInfo) + return PCBeginPtrInfo.takeError(); + JITTargetAddress PCBegin = PCBeginPtrInfo->first; + Edge::Kind PCBeginEdgeKind = PCBeginPtrInfo->second; LLVM_DEBUG({ dbgs() << " Adding edge at " << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) @@ -457,7 +467,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, auto PCBeginSym = getOrCreateSymbol(PC, PCBegin); if (!PCBeginSym) return PCBeginSym.takeError(); - B.addEdge(FDEToPCBegin, RecordOffset + PCBeginFieldOffset, *PCBeginSym, + B.addEdge(PCBeginEdgeKind, RecordOffset + PCBeginFieldOffset, *PCBeginSym, 0); PCBeginBlock = &PCBeginSym->getBlock(); } else { @@ -479,38 +489,42 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, " points at external block"); } PCBeginBlock = &EI.Target->getBlock(); - if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + if (auto Err = RecordReader.skip( + getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) return Err; } // Add a keep-alive edge from the FDE target to the FDE to ensure that the // FDE is kept alive if its target is. assert(PCBeginBlock && "PC-begin block not recorded"); + LLVM_DEBUG({ + dbgs() << " Adding keep-alive edge from target at " + << formatv("{0:x16}", PCBeginBlock->getAddress()) << " to FDE at " + << formatv("{0:x16}", RecordAddress) << "\n"; + }); PCBeginBlock->addEdge(Edge::KeepAlive, 0, FDESymbol, 0); } // Skip over the PC range size field. - if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + if (auto Err = RecordReader.skip( + getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) return Err; if (CIEInfo->FDEsHaveLSDAField) { uint64_t AugmentationDataSize; if (auto Err = RecordReader.readULEB128(AugmentationDataSize)) return Err; - if (AugmentationDataSize != PC.G.getPointerSize()) - return make_error<JITLinkError>( - "Unexpected FDE augmentation data size (expected " + - Twine(PC.G.getPointerSize()) + ", got " + - Twine(AugmentationDataSize) + ") for FDE at " + - formatv("{0:x16}", RecordAddress)); JITTargetAddress LSDAFieldOffset = RecordReader.getOffset(); auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset); if (LSDAEdgeItr == BlockEdges.end()) { - auto LSDADelta = readAbsolutePointer(PC.G, RecordReader); - if (!LSDADelta) - return LSDADelta.takeError(); - JITTargetAddress LSDA = RecordAddress + LSDAFieldOffset + *LSDADelta; + auto LSDAPointerInfo = + readEncodedPointer(CIEInfo->LSDAPointerEncoding, + RecordAddress + LSDAFieldOffset, RecordReader); + if (!LSDAPointerInfo) + return LSDAPointerInfo.takeError(); + JITTargetAddress LSDA = LSDAPointerInfo->first; + Edge::Kind LSDAEdgeKind = LSDAPointerInfo->second; auto LSDASym = getOrCreateSymbol(PC, LSDA); if (!LSDASym) return LSDASym.takeError(); @@ -519,7 +533,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) << " to LSDA at " << formatv("{0:x16}", LSDA) << "\n"; }); - B.addEdge(FDEToLSDA, RecordOffset + LSDAFieldOffset, *LSDASym, 0); + B.addEdge(LSDAEdgeKind, RecordOffset + LSDAFieldOffset, *LSDASym, 0); } else { LLVM_DEBUG({ auto &EI = LSDAEdgeItr->second; @@ -530,7 +544,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, dbgs() << " + " << formatv("{0:x16}", EI.Addend); dbgs() << "\n"; }); - if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + if (auto Err = RecordReader.skip(AugmentationDataSize)) return Err; } } else { @@ -581,23 +595,110 @@ EHFrameEdgeFixer::parseAugmentationString(BinaryStreamReader &RecordReader) { return std::move(AugInfo); } -Expected<JITTargetAddress> -EHFrameEdgeFixer::readAbsolutePointer(LinkGraph &G, - BinaryStreamReader &RecordReader) { +bool EHFrameEdgeFixer::isSupportedPointerEncoding(uint8_t PointerEncoding) { + using namespace dwarf; + + // We only support PC-rel for now. + if ((PointerEncoding & 0x70) != DW_EH_PE_pcrel) + return false; + + // readEncodedPointer does not handle indirect. + if (PointerEncoding & DW_EH_PE_indirect) + return false; + + // Supported datatypes. + switch (PointerEncoding & 0xf) { + case DW_EH_PE_absptr: + case DW_EH_PE_udata4: + case DW_EH_PE_udata8: + case DW_EH_PE_sdata4: + case DW_EH_PE_sdata8: + return true; + } + + return false; +} + +unsigned EHFrameEdgeFixer::getPointerEncodingDataSize(uint8_t PointerEncoding) { + using namespace dwarf; + + assert(isSupportedPointerEncoding(PointerEncoding) && + "Unsupported pointer encoding"); + switch (PointerEncoding & 0xf) { + case DW_EH_PE_absptr: + return PointerSize; + case DW_EH_PE_udata4: + case DW_EH_PE_sdata4: + return 4; + case DW_EH_PE_udata8: + case DW_EH_PE_sdata8: + return 8; + default: + llvm_unreachable("Unsupported encoding"); + } +} + +Expected<std::pair<JITTargetAddress, Edge::Kind>> +EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, + JITTargetAddress PointerFieldAddress, + BinaryStreamReader &RecordReader) { static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), "Result must be able to hold a uint64_t"); + assert(isSupportedPointerEncoding(PointerEncoding) && + "Unsupported pointer encoding"); + + using namespace dwarf; + + // Isolate data type, remap absptr to udata4 or udata8. This relies on us + // having verified that the graph uses 32-bit or 64-bit pointers only at the + // start of this pass. + uint8_t EffectiveType = PointerEncoding & 0xf; + if (EffectiveType == DW_EH_PE_absptr) + EffectiveType = (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; + JITTargetAddress Addr; - if (G.getPointerSize() == 8) { - if (auto Err = RecordReader.readInteger(Addr)) + Edge::Kind PointerEdgeKind; + switch (EffectiveType) { + case DW_EH_PE_udata4: { + uint32_t Val; + if (auto Err = RecordReader.readInteger(Val)) return std::move(Err); - } else if (G.getPointerSize() == 4) { - uint32_t Addr32; - if (auto Err = RecordReader.readInteger(Addr32)) + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta32; + break; + } + case DW_EH_PE_udata8: { + uint64_t Val; + if (auto Err = RecordReader.readInteger(Val)) + return std::move(Err); + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta64; + break; + } + case DW_EH_PE_sdata4: { + int32_t Val; + if (auto Err = RecordReader.readInteger(Val)) + return std::move(Err); + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta32; + break; + } + case DW_EH_PE_sdata8: { + int64_t Val; + if (auto Err = RecordReader.readInteger(Val)) return std::move(Err); - Addr = Addr32; - } else - llvm_unreachable("Pointer size is not 32-bit or 64-bit"); - return Addr; + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta64; + break; + } + } + + if (PointerEdgeKind == Edge::Invalid) + return make_error<JITLinkError>( + "Unspported edge kind for encoded pointer at " + + formatv("{0:x}", PointerFieldAddress)); + + return std::make_pair(Addr, Delta64); } Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, @@ -629,146 +730,21 @@ Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, return PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); } -// Determine whether we can register EH tables. -#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ - !(defined(_AIX) && defined(__ibmxl__)) && !defined(__SEH__) && \ - !defined(__USING_SJLJ_EXCEPTIONS__)) -#define HAVE_EHTABLE_SUPPORT 1 -#else -#define HAVE_EHTABLE_SUPPORT 0 -#endif - -#if HAVE_EHTABLE_SUPPORT -extern "C" void __register_frame(const void *); -extern "C" void __deregister_frame(const void *); - -Error registerFrameWrapper(const void *P) { - __register_frame(P); - return Error::success(); -} - -Error deregisterFrameWrapper(const void *P) { - __deregister_frame(P); - return Error::success(); -} - -#else - -// The building compiler does not have __(de)register_frame but -// it may be found at runtime in a dynamically-loaded library. -// For example, this happens when building LLVM with Visual C++ -// but using the MingW runtime. -static Error registerFrameWrapper(const void *P) { - static void((*RegisterFrame)(const void *)) = 0; - - if (!RegisterFrame) - *(void **)&RegisterFrame = - llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame"); - - if (RegisterFrame) { - RegisterFrame(P); - return Error::success(); - } - - return make_error<JITLinkError>("could not register eh-frame: " - "__register_frame function not found"); -} - -static Error deregisterFrameWrapper(const void *P) { - static void((*DeregisterFrame)(const void *)) = 0; - - if (!DeregisterFrame) - *(void **)&DeregisterFrame = - llvm::sys::DynamicLibrary::SearchForAddressOfSymbol( - "__deregister_frame"); - - if (DeregisterFrame) { - DeregisterFrame(P); - return Error::success(); - } - - return make_error<JITLinkError>("could not deregister eh-frame: " - "__deregister_frame function not found"); -} -#endif - -#ifdef __APPLE__ - -template <typename HandleFDEFn> -Error walkAppleEHFrameSection(const char *const SectionStart, - size_t SectionSize, - HandleFDEFn HandleFDE) { - const char *CurCFIRecord = SectionStart; - const char *End = SectionStart + SectionSize; - uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); - - while (CurCFIRecord != End && Size != 0) { - const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); - if (Size == 0xffffffff) - Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; - else - Size += 4; - uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); - - LLVM_DEBUG({ - dbgs() << "Registering eh-frame section:\n"; - dbgs() << "Processing " << (Offset ? "FDE" : "CIE") << " @" - << (void *)CurCFIRecord << ": ["; - for (unsigned I = 0; I < Size; ++I) - dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); - dbgs() << " ]\n"; - }); - - if (Offset != 0) - if (auto Err = HandleFDE(CurCFIRecord)) - return Err; - - CurCFIRecord += Size; - - Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); - } - - return Error::success(); -} - -#endif // __APPLE__ - -Error registerEHFrameSection(const void *EHFrameSectionAddr, - size_t EHFrameSectionSize) { -#ifdef __APPLE__ - // On Darwin __register_frame has to be called for each FDE entry. - return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), - EHFrameSectionSize, - registerFrameWrapper); -#else - // On Linux __register_frame takes a single argument: - // a pointer to the start of the .eh_frame section. - - // How can it find the end? Because crtendS.o is linked - // in and it has an .eh_frame section with four zero chars. - return registerFrameWrapper(EHFrameSectionAddr); -#endif -} - -Error deregisterEHFrameSection(const void *EHFrameSectionAddr, - size_t EHFrameSectionSize) { -#ifdef __APPLE__ - return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), - EHFrameSectionSize, - deregisterFrameWrapper); -#else - return deregisterFrameWrapper(EHFrameSectionAddr); -#endif -} - EHFrameRegistrar::~EHFrameRegistrar() {} -InProcessEHFrameRegistrar &InProcessEHFrameRegistrar::getInstance() { - static InProcessEHFrameRegistrar Instance; - return Instance; +Error InProcessEHFrameRegistrar::registerEHFrames( + JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { + return orc::registerEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); } -InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {} +Error InProcessEHFrameRegistrar::deregisterEHFrames( + JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { + return orc::deregisterEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); +} LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index a8cd32c664dc..5e68e72ba18d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -40,8 +40,9 @@ private: /// edges. class EHFrameEdgeFixer { public: - EHFrameEdgeFixer(StringRef EHFrameSectionName, Edge::Kind FDEToCIE, - Edge::Kind FDEToPCBegin, Edge::Kind FDEToLSDA); + EHFrameEdgeFixer(StringRef EHFrameSectionName, unsigned PointerSize, + Edge::Kind Delta64, Edge::Kind Delta32, + Edge::Kind NegDelta32); Error operator()(LinkGraph &G); private: @@ -57,6 +58,8 @@ private: CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} Symbol *CIESymbol = nullptr; bool FDEsHaveLSDAField = false; + uint8_t FDEPointerEncoding = 0; + uint8_t LSDAPointerEncoding = 0; }; struct EdgeTarget { @@ -96,14 +99,21 @@ private: Expected<AugmentationInfo> parseAugmentationString(BinaryStreamReader &RecordReader); - Expected<JITTargetAddress> - readAbsolutePointer(LinkGraph &G, BinaryStreamReader &RecordReader); + + static bool isSupportedPointerEncoding(uint8_t PointerEncoding); + unsigned getPointerEncodingDataSize(uint8_t PointerEncoding); + Expected<std::pair<JITTargetAddress, Edge::Kind>> + readEncodedPointer(uint8_t PointerEncoding, + JITTargetAddress PointerFieldAddress, + BinaryStreamReader &RecordReader); + Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, JITTargetAddress Addr); StringRef EHFrameSectionName; - Edge::Kind FDEToCIE; - Edge::Kind FDEToPCBegin; - Edge::Kind FDEToLSDA; + unsigned PointerSize; + Edge::Kind Delta64; + Edge::Kind Delta32; + Edge::Kind NegDelta32; }; } // end namespace jitlink diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp index 6160583b13fe..27eb7d576e2d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp @@ -15,6 +15,7 @@ #include "llvm/BinaryFormat/ELF.h" #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "llvm/Object/ELF.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Format.h" #include "llvm/Support/MemoryBuffer.h" @@ -27,24 +28,63 @@ using namespace llvm; namespace llvm { namespace jitlink { -void jitLink_ELF(std::unique_ptr<JITLinkContext> Ctx) { +Expected<uint16_t> readTargetMachineArch(StringRef Buffer) { + const char *Data = Buffer.data(); - // We don't want to do full ELF validation here. We just verify it is elf'ish. - // Probably should parse into an elf header when we support more than x86 :) - - StringRef Data = Ctx->getObjectBuffer().getBuffer(); - if (Data.size() < llvm::ELF::EI_MAG3 + 1) { - Ctx->notifyFailed(make_error<JITLinkError>("Truncated ELF buffer")); - return; + if (Data[ELF::EI_DATA] == ELF::ELFDATA2LSB) { + if (Data[ELF::EI_CLASS] == ELF::ELFCLASS64) { + if (auto File = llvm::object::ELF64LEFile::create(Buffer)) { + return File->getHeader().e_machine; + } else { + return File.takeError(); + } + } else if (Data[ELF::EI_CLASS] == ELF::ELFCLASS32) { + if (auto File = llvm::object::ELF32LEFile::create(Buffer)) { + return File->getHeader().e_machine; + } else { + return File.takeError(); + } + } } - if (!memcmp(Data.data(), llvm::ELF::ElfMagic, strlen(llvm::ELF::ElfMagic))) { - if (Data.data()[llvm::ELF::EI_CLASS] == ELF::ELFCLASS64) { - return jitLink_ELF_x86_64(std::move(Ctx)); - } + return ELF::EM_NONE; +} + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) { + StringRef Buffer = ObjectBuffer.getBuffer(); + if (Buffer.size() < ELF::EI_MAG3 + 1) + return make_error<JITLinkError>("Truncated ELF buffer"); + + if (memcmp(Buffer.data(), ELF::ElfMagic, strlen(ELF::ElfMagic)) != 0) + return make_error<JITLinkError>("ELF magic not valid"); + + Expected<uint16_t> TargetMachineArch = readTargetMachineArch(Buffer); + if (!TargetMachineArch) + return TargetMachineArch.takeError(); + + switch (*TargetMachineArch) { + case ELF::EM_X86_64: + return createLinkGraphFromELFObject_x86_64(std::move(ObjectBuffer)); + default: + return make_error<JITLinkError>( + "Unsupported target machine architecture in ELF object " + + ObjectBuffer.getBufferIdentifier()); } +} - Ctx->notifyFailed(make_error<JITLinkError>("ELF magic not valid")); +void link_ELF(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + switch (G->getTargetTriple().getArch()) { + case Triple::x86_64: + link_ELF_x86_64(std::move(G), std::move(Ctx)); + return; + default: + Ctx->notifyFailed(make_error<JITLinkError>( + "Unsupported target machine architecture in ELF link graph " + + G->getName())); + return; + } } } // end namespace jitlink diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 505f03590b6b..2a6b3eb19ded 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -11,16 +11,204 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" -#include "JITLinkGeneric.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" + +#include "BasicGOTAndStubsBuilder.h" +#include "EHFrameSupportImpl.h" +#include "JITLinkGeneric.h" #define DEBUG_TYPE "jitlink" using namespace llvm; using namespace llvm::jitlink; +using namespace llvm::jitlink::ELF_x86_64_Edges; + +namespace { + +class ELF_x86_64_GOTAndStubsBuilder + : public BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder> { +public: + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + + ELF_x86_64_GOTAndStubsBuilder(LinkGraph &G) + : BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder>(G) {} + + bool isGOTEdge(Edge &E) const { + return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + } + + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { + assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && + "Not a GOT edge?"); + // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is + // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to + // check for GOT optimization opportunities in the + // optimizeMachO_x86_64_GOTAndStubs pass below. + if (E.getKind() == PCRel32GOT) + E.setKind(PCRel32); + + E.setTarget(GOTEntry); + // Leave the edge addend as-is. + } + + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == Branch32 && !E.getTarget().isDefined(); + } + + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, -4); + return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); + } + + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + + // Set the edge kind to Branch32ToStub. We will use this to check for stub + // optimization opportunities in the optimize ELF_x86_64_GOTAndStubs pass + // below. + E.setKind(Branch32ToStub); + E.setTarget(Stub); + } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); + return *GOTSection; + } + + Section &getStubsSection() { + if (!StubsSection) { + auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt); + } + return *StubsSection; + } + + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast<const char *>(StubContent), + sizeof(StubContent)); + } + + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const char *const DwarfSectionNames[] = { +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ + ELF_NAME, +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION +}; + +} // namespace + +const uint8_t ELF_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t ELF_x86_64_GOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; static const char *CommonSectionName = "__common"; +static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) + for (auto &E : B->edges()) + if (E.getKind() == PCRel32GOTLoad) { + // Replace GOT load with LEA only for MOVQ instructions. + constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; + if (E.getOffset() < 3 || + strncmp(B->getContent().data() + E.getOffset() - 3, + reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0) + continue; + + auto &GOTBlock = E.getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT entry block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT entry should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + // Change the edge kind as we don't go through GOT anymore. This is + // for formal correctness only. Technically, the two relocation kinds + // are resolved the same way. + E.setKind(PCRel32); + E.setTarget(GOTTarget); + auto *BlockData = reinterpret_cast<uint8_t *>( + const_cast<char *>(B->getContent().data())); + BlockData[E.getOffset() - 2] = 0x8d; + LLVM_DEBUG({ + dbgs() << " Replaced GOT load wih LEA:\n "; + printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } else if (E.getKind() == Branch32ToStub) { + auto &StubBlock = E.getTarget().getBlock(); + assert(StubBlock.getSize() == + sizeof(ELF_x86_64_GOTAndStubsBuilder::StubContent) && + "Stub block should be stub sized"); + assert(StubBlock.edges_size() == 1 && + "Stub block should only have one outgoing edge"); + + auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT block should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + E.setKind(Branch32); + E.setTarget(GOTTarget); + LLVM_DEBUG({ + dbgs() << " Replaced stub branch with direct branch:\n "; + printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + + return Error::success(); +} + +static bool isDwarfSection(StringRef SectionName) { + for (auto &DwarfSectionName : DwarfSectionNames) + if (SectionName == DwarfSectionName) + return true; + return false; +} namespace llvm { namespace jitlink { @@ -35,7 +223,8 @@ private: // Find a better way using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr; // For now we just assume - std::map<int32_t, Symbol *> JITSymbolTable; + using SymbolMap = std::map<int32_t, Symbol *>; + SymbolMap JITSymbolTable; Section &getCommonSection() { if (!CommonSection) { @@ -51,6 +240,16 @@ private: switch (Type) { case ELF::R_X86_64_PC32: return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; + case ELF::R_X86_64_PC64: + return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64; + case ELF::R_X86_64_64: + return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64; + case ELF::R_X86_64_GOTPCREL: + case ELF::R_X86_64_GOTPCRELX: + case ELF::R_X86_64_REX_GOTPCRELX: + return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad; + case ELF::R_X86_64_PLT32: + return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; } return make_error<JITLinkError>("Unsupported x86-64 relocation:" + formatv("{0:d}", Type)); @@ -62,7 +261,7 @@ private: object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections; SymbolTable SymTab; - bool isRelocatable() { return Obj.getHeader()->e_type == llvm::ELF::ET_REL; } + bool isRelocatable() { return Obj.getHeader().e_type == llvm::ELF::ET_REL; } support::endianness getEndianness(const object::ELFFile<object::ELF64LE> &Obj) { @@ -71,7 +270,7 @@ private: // This could also just become part of a template unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) { - return Obj.getHeader()->getFileClass() == ELF::ELFCLASS64 ? 8 : 4; + return Obj.getHeader().getFileClass() == ELF::ELFCLASS64 ? 8 : 4; } // We don't technically need this right now @@ -95,16 +294,12 @@ private: auto StrTabSec = Obj.getSection(SecRef.sh_link); if (!StrTabSec) return StrTabSec.takeError(); - auto StringTable = Obj.getStringTable(*StrTabSec); + auto StringTable = Obj.getStringTable(**StrTabSec); if (!StringTable) return StringTable.takeError(); for (auto SymRef : *Symbols) { Optional<StringRef> Name; - uint64_t Size = 0; - - // FIXME: Read size. - (void)Size; if (auto NameOrErr = SymRef.getName(*StringTable)) Name = *NameOrErr; @@ -112,16 +307,13 @@ private: return NameOrErr.takeError(); LLVM_DEBUG({ - dbgs() << " "; - if (!Name) - dbgs() << "<anonymous symbol>"; - else - dbgs() << *Name; - dbgs() << ": value = " << formatv("{0:x16}", SymRef.getValue()) + dbgs() << " value = " << formatv("{0:x16}", SymRef.getValue()) << ", type = " << formatv("{0:x2}", SymRef.getType()) - << ", binding = " << SymRef.getBinding() - << ", size =" << Size; - dbgs() << "\n"; + << ", binding = " << formatv("{0:x2}", SymRef.getBinding()) + << ", size = " + << formatv("{0:x16}", static_cast<uint64_t>(SymRef.st_size)) + << ", info = " << formatv("{0:x2}", SymRef.st_info) + << " :" << (Name ? *Name : "<anonymous symbol>") << "\n"; }); } } @@ -131,9 +323,19 @@ private: Error createNormalizedSections() { LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); for (auto &SecRef : sections) { - auto Name = Obj.getSectionName(&SecRef); + auto Name = Obj.getSectionName(SecRef); if (!Name) return Name.takeError(); + + // Skip Dwarf sections. + if (isDwarfSection(*Name)) { + LLVM_DEBUG({ + dbgs() << *Name + << " is a debug section: No graph section will be created.\n"; + }); + continue; + } + sys::Memory::ProtectionFlags Prot; if (SecRef.sh_flags & ELF::SHF_EXECINSTR) { Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | @@ -147,8 +349,8 @@ private: uint64_t Flags = SecRef.sh_flags; uint64_t Alignment = SecRef.sh_addralign; const char *Data = nullptr; - // TODO: figure out what it is that has 0 size no name and address - // 0000-0000 + // for now we just use this to skip the "undefined" section, probably need + // to revist if (Size == 0) continue; @@ -158,13 +360,13 @@ private: LLVM_DEBUG({ dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- " << formatv("{0:x16}", Address + Size) << ", align: " << Alignment - << " Flags:" << Flags << "\n"; + << " Flags: " << formatv("{0:x}", Flags) << "\n"; }); if (SecRef.sh_type != ELF::SHT_NOBITS) { // .sections() already checks that the data is not beyond the end of // file - auto contents = Obj.getSectionContentsAsArray<char>(&SecRef); + auto contents = Obj.getSectionContentsAsArray<char>(SecRef); if (!contents) return contents.takeError(); @@ -178,6 +380,9 @@ private: if (SecRef.sh_type == ELF::SHT_SYMTAB) // TODO: Dynamic? SymTab = SecRef; + } else { + auto &Section = G->createSection(*Name, Prot); + G->createZeroFillBlock(Section, Size, Address, Alignment, 0); } } @@ -196,21 +401,34 @@ private: return make_error<llvm::StringError>("Shouldn't have REL in x64", llvm::inconvertibleErrorCode()); - auto RelSectName = Obj.getSectionName(&SecRef); + auto RelSectName = Obj.getSectionName(SecRef); if (!RelSectName) return RelSectName.takeError(); - // Deal with .eh_frame later - if (*RelSectName == StringRef(".rela.eh_frame")) - continue; + + LLVM_DEBUG({ + dbgs() << "Adding relocations from section " << *RelSectName << "\n"; + }); auto UpdateSection = Obj.getSection(SecRef.sh_info); if (!UpdateSection) return UpdateSection.takeError(); - auto UpdateSectionName = Obj.getSectionName(*UpdateSection); + auto UpdateSectionName = Obj.getSectionName(**UpdateSection); if (!UpdateSectionName) return UpdateSectionName.takeError(); + // Don't process relocations for debug sections. + if (isDwarfSection(*UpdateSectionName)) { + LLVM_DEBUG({ + dbgs() << " Target is dwarf section " << *UpdateSectionName + << ". Skipping.\n"; + }); + continue; + } else + LLVM_DEBUG({ + dbgs() << " For target section " << *UpdateSectionName << "\n"; + }); + auto JITSection = G->findSectionByName(*UpdateSectionName); if (!JITSection) return make_error<llvm::StringError>( @@ -218,7 +436,7 @@ private: *UpdateSectionName, llvm::inconvertibleErrorCode()); - auto Relocations = Obj.relas(&SecRef); + auto Relocations = Obj.relas(SecRef); if (!Relocations) return Relocations.takeError(); @@ -229,13 +447,22 @@ private: dbgs() << "Relocation Type: " << Type << "\n" << "Name: " << Obj.getRelocationTypeName(Type) << "\n"; }); - - auto Symbol = Obj.getRelocationSymbol(&Rela, &SymTab); + auto SymbolIndex = Rela.getSymbol(false); + auto Symbol = Obj.getRelocationSymbol(Rela, &SymTab); if (!Symbol) return Symbol.takeError(); auto BlockToFix = *(JITSection->blocks().begin()); - auto TargetSymbol = JITSymbolTable[(*Symbol)->st_shndx]; + auto *TargetSymbol = JITSymbolTable[SymbolIndex]; + + if (!TargetSymbol) { + return make_error<llvm::StringError>( + "Could not find symbol at given index, did you add it to " + "JITSymbolTable? index: " + std::to_string(SymbolIndex) + + ", shndx: " + std::to_string((*Symbol)->st_shndx) + + " Size of table: " + std::to_string(JITSymbolTable.size()), + llvm::inconvertibleErrorCode()); + } uint64_t Addend = Rela.r_addend; JITTargetAddress FixupAddress = (*UpdateSection)->sh_addr + Rela.r_offset; @@ -251,8 +478,8 @@ private: LLVM_DEBUG({ Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - // TODO a mapping of KIND => type then call getRelocationTypeName4 - printEdge(dbgs(), *BlockToFix, GE, StringRef("")); + printEdge(dbgs(), *BlockToFix, GE, + getELFX86RelocationKindName(*Kind)); dbgs() << "\n"; }); BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), @@ -284,25 +511,31 @@ private: auto StrTabSec = Obj.getSection(SecRef.sh_link); if (!StrTabSec) return StrTabSec.takeError(); - auto StringTable = Obj.getStringTable(*StrTabSec); + auto StringTable = Obj.getStringTable(**StrTabSec); if (!StringTable) return StringTable.takeError(); - auto Name = Obj.getSectionName(&SecRef); + auto Name = Obj.getSectionName(SecRef); if (!Name) return Name.takeError(); + + LLVM_DEBUG(dbgs() << "Processing symbol section " << *Name << ":\n"); + auto Section = G->findSectionByName(*Name); if (!Section) - return make_error<llvm::StringError>("Could not find a section", + return make_error<llvm::StringError>("Could not find a section " + + *Name, llvm::inconvertibleErrorCode()); // we only have one for now auto blocks = Section->blocks(); if (blocks.empty()) return make_error<llvm::StringError>("Section has no block", llvm::inconvertibleErrorCode()); - + int SymbolIndex = -1; for (auto SymRef : *Symbols) { + ++SymbolIndex; auto Type = SymRef.getType(); - if (Type == ELF::STT_NOTYPE || Type == ELF::STT_FILE) + + if (Type == ELF::STT_FILE || SymbolIndex == 0) continue; // these should do it for now // if(Type != ELF::STT_NOTYPE && @@ -312,68 +545,119 @@ private: // Type != ELF::STT_COMMON) { // continue; // } - std::pair<Linkage, Scope> bindings; auto Name = SymRef.getName(*StringTable); // I am not sure on If this is going to hold as an invariant. Revisit. if (!Name) return Name.takeError(); - // TODO: weak and hidden - if (SymRef.isExternal()) - bindings = {Linkage::Strong, Scope::Default}; - else - bindings = {Linkage::Strong, Scope::Local}; + + if (SymRef.isCommon()) { + // Symbols in SHN_COMMON refer to uninitialized data. The st_value + // field holds alignment constraints. + Symbol &S = + G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, + SymRef.st_size, SymRef.getValue(), false); + JITSymbolTable[SymbolIndex] = &S; + continue; + } + + // Map Visibility and Binding to Scope and Linkage: + Linkage L = Linkage::Strong; + Scope S = Scope::Default; + + switch (SymRef.getBinding()) { + case ELF::STB_LOCAL: + S = Scope::Local; + break; + case ELF::STB_GLOBAL: + // Nothing to do here. + break; + case ELF::STB_WEAK: + L = Linkage::Weak; + break; + default: + return make_error<StringError>("Unrecognized symbol binding for " + + *Name, + inconvertibleErrorCode()); + } + + switch (SymRef.getVisibility()) { + case ELF::STV_DEFAULT: + case ELF::STV_PROTECTED: + // FIXME: Make STV_DEFAULT symbols pre-emptible? This probably needs + // Orc support. + // Otherwise nothing to do here. + break; + case ELF::STV_HIDDEN: + // Default scope -> Hidden scope. No effect on local scope. + if (S == Scope::Default) + S = Scope::Hidden; + break; + case ELF::STV_INTERNAL: + return make_error<StringError>("Unrecognized symbol visibility for " + + *Name, + inconvertibleErrorCode()); + } if (SymRef.isDefined() && - (Type == ELF::STT_FUNC || Type == ELF::STT_OBJECT)) { + (Type == ELF::STT_FUNC || Type == ELF::STT_OBJECT || + Type == ELF::STT_SECTION)) { auto DefinedSection = Obj.getSection(SymRef.st_shndx); if (!DefinedSection) return DefinedSection.takeError(); - auto sectName = Obj.getSectionName(*DefinedSection); + auto sectName = Obj.getSectionName(**DefinedSection); if (!sectName) return Name.takeError(); + // Skip debug section symbols. + if (isDwarfSection(*sectName)) + continue; + auto JitSection = G->findSectionByName(*sectName); if (!JitSection) return make_error<llvm::StringError>( - "Could not find a section", llvm::inconvertibleErrorCode()); + "Could not find the JitSection " + *sectName, + llvm::inconvertibleErrorCode()); auto bs = JitSection->blocks(); if (bs.empty()) return make_error<llvm::StringError>( "Section has no block", llvm::inconvertibleErrorCode()); - auto B = *bs.begin(); - LLVM_DEBUG({ dbgs() << " " << *Name << ": "; }); - - auto &S = G->addDefinedSymbol( - *B, SymRef.getValue(), *Name, SymRef.st_size, bindings.first, - bindings.second, SymRef.getType() == ELF::STT_FUNC, false); - JITSymbolTable[SymRef.st_shndx] = &S; - } - //TODO: The following has to be implmented. + auto *B = *bs.begin(); + LLVM_DEBUG({ dbgs() << " " << *Name << " at index " << SymbolIndex << "\n"; }); + if (SymRef.getType() == ELF::STT_SECTION) + *Name = *sectName; + auto &Sym = G->addDefinedSymbol( + *B, SymRef.getValue(), *Name, SymRef.st_size, L, S, + SymRef.getType() == ELF::STT_FUNC, false); + JITSymbolTable[SymbolIndex] = &Sym; + } else if (SymRef.isUndefined() && SymRef.isExternal()) { + auto &Sym = G->addExternalSymbol(*Name, SymRef.st_size, L); + JITSymbolTable[SymbolIndex] = &Sym; + } else + LLVM_DEBUG({ + dbgs() + << "Not creating graph symbol for normalized symbol at index " + << SymbolIndex << ", \"" << *Name << "\"\n"; + }); + + // TODO: The following has to be implmented. // leaving commented out to save time for future patchs /* G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size, Linkage::Strong, Scope::Default, false); - - if(SymRef.isCommon()) { - G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, 0, - SymRef.getValue(), false); - } - - - //G->addExternalSymbol(*Name, SymRef.st_size, Linkage::Strong); - */ + */ } } return Error::success(); } public: - ELFLinkGraphBuilder_x86_64(std::string filename, + ELFLinkGraphBuilder_x86_64(StringRef FileName, const object::ELFFile<object::ELF64LE> &Obj) - : G(std::make_unique<LinkGraph>(filename, getPointerSize(Obj), - getEndianness(Obj))), + : G(std::make_unique<LinkGraph>(FileName.str(), + Triple("x86_64-unknown-linux"), + getPointerSize(Obj), getEndianness(Obj))), Obj(Obj) {} Expected<std::unique_ptr<LinkGraph>> buildGraph() { @@ -409,55 +693,121 @@ class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { public: ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) - : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - StringRef getEdgeKindName(Edge::Kind R) const override { return StringRef(); } - - Expected<std::unique_ptr<LinkGraph>> - buildGraph(MemoryBufferRef ObjBuffer) override { - auto ELFObj = object::ObjectFile::createELFObjectFile(ObjBuffer); - if (!ELFObj) - return ELFObj.takeError(); - - auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); - std::string fileName(ELFObj->get()->getFileName()); - return ELFLinkGraphBuilder_x86_64(std::move(fileName), - *ELFObjFile.getELFFile()) - .buildGraph(); + StringRef getEdgeKindName(Edge::Kind R) const override { + return getELFX86RelocationKindName(R); + } + + static Error targetOutOfRangeError(const Block &B, const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, B, E, getELFX86RelocationKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); } Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { using namespace ELF_x86_64_Edges; + using namespace llvm::support; char *FixupPtr = BlockWorkingMem + E.getOffset(); JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { - + case ELFX86RelocationKind::Branch32: + case ELFX86RelocationKind::Branch32ToStub: case ELFX86RelocationKind::PCRel32: + case ELFX86RelocationKind::PCRel32GOTLoad: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + break; + } + case ELFX86RelocationKind::Pointer64: { + int64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case ELFX86RelocationKind::Delta64: { int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; - // verify - *(support::little32_t *)FixupPtr = Value; + *(little64_t *)FixupPtr = Value; break; } + } return Error::success(); } }; -void jitLink_ELF_x86_64(std::unique_ptr<JITLinkContext> Ctx) { +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { + LLVM_DEBUG({ + dbgs() << "Building jitlink graph for new input " + << ObjectBuffer.getBufferIdentifier() << "...\n"; + }); + + auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); + return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), + ELFObjFile.getELFFile()) + .buildGraph(); +} + +void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { PassConfiguration Config; - Triple TT("x86_64-linux"); - // Construct a JITLinker and run the link function. - // Add a mark-live pass. - if (auto MarkLive = Ctx->getMarkLivePass(TT)) - Config.PrePrunePasses.push_back(std::move(MarkLive)); - else - Config.PrePrunePasses.push_back(markAllSymbolsLive); - - if (auto Err = Ctx->modifyPassConfig(TT, Config)) + + if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { + + Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); + + // Construct a JITLinker and run the link function. + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { + ELF_x86_64_GOTAndStubsBuilder(G).run(); + return Error::success(); + }); + + // Add GOT/Stubs optimizer pass. + Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); + } + + if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) return Ctx->notifyFailed(std::move(Err)); - ELFJITLinker_x86_64::link(std::move(Ctx), std::move(Config)); + ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); +} +StringRef getELFX86RelocationKindName(Edge::Kind R) { + switch (R) { + case PCRel32: + return "PCRel32"; + case Pointer64: + return "Pointer64"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case Branch32: + return "Branch32"; + case Branch32ToStub: + return "Branch32ToStub"; + } + return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); } } // end namespace jitlink } // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 5105ec495148..93dfba9c759b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -64,7 +64,7 @@ const char *getGenericEdgeKindName(Edge::Kind K) { case Edge::KeepAlive: return "Keep-Alive"; default: - llvm_unreachable("Unrecognized relocation kind"); + return "<Unrecognized edge kind>"; } } @@ -93,6 +93,7 @@ const char *getScopeName(Scope S) { raw_ostream &operator<<(raw_ostream &OS, const Block &B) { return OS << formatv("{0:x16}", B.getAddress()) << " -- " << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": " + << "size = " << formatv("{0:x}", B.getSize()) << ", " << (B.isZeroFill() ? "zero-fill" : "content") << ", align = " << B.getAlignment() << ", align-ofs = " << B.getAlignmentOffset() @@ -126,10 +127,10 @@ raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { break; } OS << (Sym.isLive() ? '+' : '-') - << ", size = " << formatv("{0:x8}", Sym.getSize()) + << ", size = " << formatv("{0:x}", Sym.getSize()) << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " (" << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + " - << formatv("{0:x8}", Sym.getOffset()); + << formatv("{0:x}", Sym.getOffset()); if (Sym.isDefined()) OS << " " << Sym.getBlock().getSection().getName(); OS << ")>"; @@ -139,8 +140,33 @@ raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { void printEdge(raw_ostream &OS, const Block &B, const Edge &E, StringRef EdgeKindName) { OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": " - << formatv("{0:x16}", B.getAddress()) << " + " << E.getOffset() << " -- " - << EdgeKindName << " -> " << E.getTarget() << " + " << E.getAddend(); + << formatv("{0:x16}", B.getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName << " -> "; + + auto &TargetSym = E.getTarget(); + if (TargetSym.hasName()) + OS << TargetSym.getName(); + else { + auto &TargetBlock = TargetSym.getBlock(); + auto &TargetSec = TargetBlock.getSection(); + JITTargetAddress SecAddress = ~JITTargetAddress(0); + for (auto *B : TargetSec.blocks()) + if (B->getAddress() < SecAddress) + SecAddress = B->getAddress(); + + JITTargetAddress SecDelta = TargetSym.getAddress() - SecAddress; + OS << formatv("{0:x16}", TargetSym.getAddress()) << " (section " + << TargetSec.getName(); + if (SecDelta) + OS << " + " << formatv("{0:x}", SecDelta); + OS << " / block " << formatv("{0:x16}", TargetBlock.getAddress()); + if (TargetSym.getOffset()) + OS << " + " << formatv("{0:x}", TargetSym.getOffset()); + OS << ")"; + } + + if (E.getAddend() != 0) + OS << " + " << E.getAddend(); } Section::~Section() { @@ -296,15 +322,27 @@ Error markAllSymbolsLive(LinkGraph &G) { return Error::success(); } -void jitLink(std::unique_ptr<JITLinkContext> Ctx) { - auto Magic = identify_magic(Ctx->getObjectBuffer().getBuffer()); +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) { + auto Magic = identify_magic(ObjectBuffer.getBuffer()); switch (Magic) { case file_magic::macho_object: - return jitLink_MachO(std::move(Ctx)); + return createLinkGraphFromMachOObject(std::move(ObjectBuffer)); case file_magic::elf_relocatable: - return jitLink_ELF(std::move(Ctx)); + return createLinkGraphFromELFObject(std::move(ObjectBuffer)); + default: + return make_error<JITLinkError>("Unsupported file format"); + }; +} + +void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { + switch (G->getTargetTriple().getObjectFormat()) { + case Triple::MachO: + return link_MachO(std::move(G), std::move(Ctx)); + case Triple::ELF: + return link_ELF(std::move(G), std::move(Ctx)); default: - Ctx->notifyFailed(make_error<JITLinkError>("Unsupported file format")); + Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format")); }; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 1d76a49939dc..7a5e014f223d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -24,15 +24,6 @@ JITLinkerBase::~JITLinkerBase() {} void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { - LLVM_DEBUG({ dbgs() << "Building jitlink graph for new input...\n"; }); - - // Build the link graph. - if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) - G = std::move(*GraphOrErr); - else - return Ctx->notifyFailed(GraphOrErr.takeError()); - assert(G && "Graph should have been created by buildGraph above"); - LLVM_DEBUG({ dbgs() << "Starting link phase 1 for graph " << G->getName() << "\n"; }); @@ -64,10 +55,22 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { if (auto Err = allocateSegments(Layout)) return Ctx->notifyFailed(std::move(Err)); + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() + << "\" before post-allocation passes:\n"; + dumpGraph(dbgs()); + }); + + // Run post-allocation passes. + if (auto Err = runPasses(Passes.PostAllocationPasses)) + return Ctx->notifyFailed(std::move(Err)); + // Notify client that the defined symbols have been assigned addresses. LLVM_DEBUG( { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; }); - Ctx->notifyResolved(*G); + + if (auto Err = Ctx->notifyResolved(*G)) + return Ctx->notifyFailed(std::move(Err)); auto ExternalSymbols = getExternalSymbolNames(); @@ -117,11 +120,11 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() - << "\" before post-allocation passes:\n"; + << "\" before pre-fixup passes:\n"; dumpGraph(dbgs()); }); - if (auto Err = runPasses(Passes.PostAllocationPasses)) + if (auto Err = runPasses(Passes.PreFixupPasses)) return deallocateAndBailOut(std::move(Err)); LLVM_DEBUG({ @@ -261,7 +264,8 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { } LLVM_DEBUG(dbgs() << " }\n"); - if (auto AllocOrErr = Ctx->getMemoryManager().allocate(Segments)) + if (auto AllocOrErr = + Ctx->getMemoryManager().allocate(Ctx->getJITLinkDylib(), Segments)) Alloc = std::move(*AllocOrErr); else return AllocOrErr.takeError(); @@ -332,12 +336,6 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { dbgs() << " " << Sym->getName() << ": " << formatv("{0:x16}", Sym->getAddress()) << "\n"; }); - assert(llvm::all_of(G->external_symbols(), - [](Symbol *Sym) { - return Sym->getAddress() != 0 || - Sym->getLinkage() == Linkage::Weak; - }) && - "All strong external symbols should have been resolved by now"); } void JITLinkerBase::copyBlockContentToWorkingMemory( @@ -445,16 +443,19 @@ void prune(LinkGraph &G) { VisitedBlocks.insert(&B); for (auto &E : Sym->getBlock().edges()) { - if (E.getTarget().isDefined() && !E.getTarget().isLive()) { - E.getTarget().setLive(true); + // If the edge target is a defined symbol that is being newly marked live + // then add it to the worklist. + if (E.getTarget().isDefined() && !E.getTarget().isLive()) Worklist.push_back(&E.getTarget()); - } + + // Mark the target live. + E.getTarget().setLive(true); } } - // Collect all the symbols to remove, then remove them. + // Collect all defined symbols to remove, then remove them. { - LLVM_DEBUG(dbgs() << "Dead-stripping symbols:\n"); + LLVM_DEBUG(dbgs() << "Dead-stripping defined symbols:\n"); std::vector<Symbol *> SymbolsToRemove; for (auto *Sym : G.defined_symbols()) if (!Sym->isLive()) @@ -477,6 +478,19 @@ void prune(LinkGraph &G) { G.removeBlock(*B); } } + + // Collect all external symbols to remove, then remove them. + { + LLVM_DEBUG(dbgs() << "Removing unused external symbols:\n"); + std::vector<Symbol *> SymbolsToRemove; + for (auto *Sym : G.external_symbols()) + if (!Sym->isLive()) + SymbolsToRemove.push_back(Sym); + for (auto *Sym : SymbolsToRemove) { + LLVM_DEBUG(dbgs() << " " << *Sym << "...\n"); + G.removeExternalSymbol(*Sym); + } + } } } // end namespace jitlink diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index 87e5e8bbc98d..1d28f5006b2b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -32,9 +32,11 @@ namespace jitlink { /// remaining linker work) to allow them to be performed asynchronously. class JITLinkerBase { public: - JITLinkerBase(std::unique_ptr<JITLinkContext> Ctx, PassConfiguration Passes) - : Ctx(std::move(Ctx)), Passes(std::move(Passes)) { + JITLinkerBase(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration Passes) + : Ctx(std::move(Ctx)), G(std::move(G)), Passes(std::move(Passes)) { assert(this->Ctx && "Ctx can not be null"); + assert(this->G && "G can not be null"); } virtual ~JITLinkerBase(); @@ -50,8 +52,7 @@ protected: using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>; // Phase 1: - // 1.1: Build link graph - // 1.2: Run pre-prune passes + // 1.1: Run pre-prune passes // 1.2: Prune graph // 1.3: Run post-prune passes // 1.4: Sort blocks into segments @@ -72,11 +73,6 @@ protected: // 3.1: Call OnFinalized callback, handing off allocation. void linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err); - // Build a graph from the given object buffer. - // To be implemented by the client. - virtual Expected<std::unique_ptr<LinkGraph>> - buildGraph(MemoryBufferRef ObjBuffer) = 0; - // For debug dumping of the link graph. virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; @@ -113,8 +109,8 @@ private: void dumpGraph(raw_ostream &OS); std::unique_ptr<JITLinkContext> Ctx; - PassConfiguration Passes; std::unique_ptr<LinkGraph> G; + PassConfiguration Passes; std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc; }; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index 68ec9d79af9b..fbbb29e9164a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -17,7 +17,8 @@ JITLinkMemoryManager::~JITLinkMemoryManager() = default; JITLinkMemoryManager::Allocation::~Allocation() = default; Expected<std::unique_ptr<JITLinkMemoryManager::Allocation>> -InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { +InProcessMemoryManager::allocate(const JITLinkDylib *JD, + const SegmentsRequestMap &Request) { using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp index b3e45868ab22..e9327df6da41 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp @@ -27,39 +27,29 @@ using namespace llvm; namespace llvm { namespace jitlink { -void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) { - - // We don't want to do full MachO validation here. Just parse enough of the - // header to find out what MachO linker to use. - - StringRef Data = Ctx->getObjectBuffer().getBuffer(); - if (Data.size() < 4) { - StringRef BufferName = Ctx->getObjectBuffer().getBufferIdentifier(); - Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer \"" + - BufferName + "\"")); - return; - } +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer) { + StringRef Data = ObjectBuffer.getBuffer(); + if (Data.size() < 4) + return make_error<JITLinkError>("Truncated MachO buffer \"" + + ObjectBuffer.getBufferIdentifier() + "\""); uint32_t Magic; memcpy(&Magic, Data.data(), sizeof(uint32_t)); LLVM_DEBUG({ dbgs() << "jitLink_MachO: magic = " << format("0x%08" PRIx32, Magic) - << ", identifier = \"" - << Ctx->getObjectBuffer().getBufferIdentifier() << "\"\n"; + << ", identifier = \"" << ObjectBuffer.getBufferIdentifier() + << "\"\n"; }); - if (Magic == MachO::MH_MAGIC || Magic == MachO::MH_CIGAM) { - Ctx->notifyFailed( - make_error<JITLinkError>("MachO 32-bit platforms not supported")); - return; - } else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { + if (Magic == MachO::MH_MAGIC || Magic == MachO::MH_CIGAM) + return make_error<JITLinkError>("MachO 32-bit platforms not supported"); + else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { - if (Data.size() < sizeof(MachO::mach_header_64)) { - StringRef BufferName = Ctx->getObjectBuffer().getBufferIdentifier(); - Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer \"" + - BufferName + "\"")); - return; - } + if (Data.size() < sizeof(MachO::mach_header_64)) + return make_error<JITLinkError>("Truncated MachO buffer \"" + + ObjectBuffer.getBufferIdentifier() + + "\""); // Read the CPU type from the header. uint32_t CPUType; @@ -74,15 +64,27 @@ void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) { switch (CPUType) { case MachO::CPU_TYPE_ARM64: - return jitLink_MachO_arm64(std::move(Ctx)); + return createLinkGraphFromMachOObject_arm64(std::move(ObjectBuffer)); case MachO::CPU_TYPE_X86_64: - return jitLink_MachO_x86_64(std::move(Ctx)); + return createLinkGraphFromMachOObject_x86_64(std::move(ObjectBuffer)); } + return make_error<JITLinkError>("MachO-64 CPU type not valid"); + } else + return make_error<JITLinkError>("Unrecognized MachO magic value"); +} + +void link_MachO(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + + switch (G->getTargetTriple().getArch()) { + case Triple::aarch64: + return link_MachO_arm64(std::move(G), std::move(Ctx)); + case Triple::x86_64: + return link_MachO_x86_64(std::move(G), std::move(Ctx)); + default: Ctx->notifyFailed(make_error<JITLinkError>("MachO-64 CPU type not valid")); return; } - - Ctx->notifyFailed(make_error<JITLinkError>("MachO magic not valid")); } } // end namespace jitlink diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index fa3f403b717c..4602154eb579 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -45,10 +45,12 @@ Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() { return std::move(G); } -MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj) +MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, + Triple TT) : Obj(Obj), G(std::make_unique<LinkGraph>(std::string(Obj.getFileName()), - getPointerSize(Obj), getEndianness(Obj))) {} + std::move(TT), getPointerSize(Obj), + getEndianness(Obj))) {} void MachOLinkGraphBuilder::addCustomSectionParser( StringRef SectionName, SectionParserFunction Parser) { @@ -64,10 +66,8 @@ Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) { } Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) { - if (Type & MachO::N_PEXT) - return Scope::Hidden; if (Type & MachO::N_EXT) { - if (Name.startswith("l")) + if ((Type & MachO::N_PEXT) || Name.startswith("l")) return Scope::Hidden; else return Scope::Default; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index dd3bcf27494c..26e6859de91d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -16,10 +16,10 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Object/MachO.h" #include "EHFrameSupportImpl.h" #include "JITLinkGeneric.h" -#include "llvm/Object/MachO.h" #include <list> @@ -81,7 +81,7 @@ protected: using SectionParserFunction = std::function<Error(NormalizedSection &S)>; - MachOLinkGraphBuilder(const object::MachOObjectFile &Obj); + MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT); LinkGraph &getGraph() const { return *G; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 463845a5b8cb..8366e9658539 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -26,7 +26,7 @@ namespace { class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj), + : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin")), NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: @@ -148,10 +148,11 @@ private: else return ToSymbolOrErr.takeError(); } else { - if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue)) - ToSymbol = &*ToSymbolOrErr; - else - return ToSymbolOrErr.takeError(); + auto ToSymbolSec = findSectionByIndex(UnsignedRI.r_symbolnum - 1); + if (!ToSymbolSec) + return ToSymbolSec.takeError(); + ToSymbol = getSymbolByAddress(ToSymbolSec->Address); + assert(ToSymbol && "No symbol for section"); FixupValue -= ToSymbol->getAddress(); } @@ -181,6 +182,8 @@ private: using namespace support; auto &Obj = getObject(); + LLVM_DEBUG(dbgs() << "Processing relocations:\n"); + for (auto &S : Obj.sections()) { JITTargetAddress SectionAddress = S.getAddress(); @@ -199,8 +202,8 @@ private: getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); if (!NSec.GraphSection) { LLVM_DEBUG({ - dbgs() << "Skipping relocations for MachO section " << NSec.SegName - << "/" << NSec.SectName + dbgs() << " Skipping relocations for MachO section " + << NSec.SegName << "/" << NSec.SectName << " which has no associated graph section\n"; }); continue; @@ -221,9 +224,10 @@ private: JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; LLVM_DEBUG({ - dbgs() << "Processing " << getMachOARM64RelocationKindName(*Kind) - << " relocation at " << format("0x%016" PRIx64, FixupAddress) - << "\n"; + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + dbgs() << " " << NSec.SectName << " + " + << formatv("{0:x8}", RI.r_address) << ":\n"; }); // Find the block that the fixup points to. @@ -252,7 +256,7 @@ private: // If this is an Addend relocation then process it and move to the // paired reloc. - Addend = RI.r_symbolnum; + Addend = SignExtend64(RI.r_symbolnum, 24); if (RelItr == RelEnd) return make_error<JITLinkError>("Unpaired Addend reloc at " + @@ -268,11 +272,12 @@ private: return make_error<JITLinkError>( "Invalid relocation pair: Addend + " + getMachOARM64RelocationKindName(*Kind)); - else - LLVM_DEBUG({ - dbgs() << " pair is " << getMachOARM64RelocationKindName(*Kind) - << "`\n"; - }); + + LLVM_DEBUG({ + dbgs() << " Addend: value = " << formatv("{0:x6}", Addend) + << ", pair is " << getMachOARM64RelocationKindName(*Kind) + << "\n"; + }); // Find the address of the value to fix up. JITTargetAddress PairedFixupAddress = @@ -335,6 +340,11 @@ private: TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + uint32_t EncodedAddend = (Instr & 0x003FFC00) >> 10; + if (EncodedAddend != 0) + return make_error<JITLinkError>("GOTPAGEOFF12 target has non-zero " + "encoded addend"); break; } case GOTPageOffset12: { @@ -377,6 +387,7 @@ private: } LLVM_DEBUG({ + dbgs() << " "; Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); printEdge(dbgs(), *BlockToFix, GE, @@ -490,22 +501,15 @@ class MachOJITLinker_arm64 : public JITLinker<MachOJITLinker_arm64> { public: MachOJITLinker_arm64(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) - : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: StringRef getEdgeKindName(Edge::Kind R) const override { return getMachOARM64RelocationKindName(R); } - Expected<std::unique_ptr<LinkGraph>> - buildGraph(MemoryBufferRef ObjBuffer) override { - auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); - if (!MachOObj) - return MachOObj.takeError(); - return MachOLinkGraphBuilder_arm64(**MachOObj).buildGraph(); - } - static Error targetOutOfRangeError(const Block &B, const Edge &E) { std::string ErrMsg; { @@ -518,23 +522,17 @@ private: } static unsigned getPageOffset12Shift(uint32_t Instr) { - constexpr uint32_t LDRLiteralMask = 0x3ffffc00; + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + constexpr uint32_t Vec128Mask = 0x04800000; - // Check for a GPR LDR immediate with a zero embedded literal. - // If found, the top two bits contain the shift. - if ((Instr & LDRLiteralMask) == 0x39400000) - return Instr >> 30; + if ((Instr & LoadStoreImm12Mask) == 0x39000000) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; - // Check for a Neon LDR immediate of size 64-bit or less with a zero - // embedded literal. If found, the top two bits contain the shift. - if ((Instr & LDRLiteralMask) == 0x3d400000) - return Instr >> 30; - - // Check for a Neon LDR immediate of size 128-bit with a zero embedded - // literal. - constexpr uint32_t SizeBitsMask = 0xc0000000; - if ((Instr & (LDRLiteralMask | SizeBitsMask)) == 0x3dc00000) - return 4; + return ImplicitShift; + } return 0; } @@ -581,10 +579,12 @@ private: } case Page21: case GOTPage21: { - assert(E.getAddend() == 0 && "PAGE21/GOTPAGE21 with non-zero addend"); + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); uint64_t TargetPage = - E.getTarget().getAddress() & ~static_cast<uint64_t>(4096 - 1); - uint64_t PCPage = B.getAddress() & ~static_cast<uint64_t>(4096 - 1); + (E.getTarget().getAddress() + E.getAddend()) & + ~static_cast<uint64_t>(4096 - 1); + uint64_t PCPage = FixupAddress & ~static_cast<uint64_t>(4096 - 1); int64_t PageDelta = TargetPage - PCPage; if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) @@ -600,8 +600,8 @@ private: break; } case PageOffset12: { - assert(E.getAddend() == 0 && "PAGEOFF12 with non-zero addend"); - uint64_t TargetOffset = E.getTarget().getAddress() & 0xfff; + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()) & 0xfff; uint32_t RawInstr = *(ulittle32_t *)FixupPtr; unsigned ImmShift = getPageOffset12Shift(RawInstr); @@ -674,13 +674,22 @@ private: uint64_t NullValue = 0; }; -void jitLink_MachO_arm64(std::unique_ptr<JITLinkContext> Ctx) { +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer) { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOLinkGraphBuilder_arm64(**MachOObj).buildGraph(); +} + +void link_MachO_arm64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; - Triple TT("arm64-apple-ios"); - if (Ctx->shouldAddDefaultTargetPasses(TT)) { + if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { // Add a mark-live pass. - if (auto MarkLive = Ctx->getMarkLivePass(TT)) + if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) Config.PrePrunePasses.push_back(std::move(MarkLive)); else Config.PrePrunePasses.push_back(markAllSymbolsLive); @@ -692,11 +701,11 @@ void jitLink_MachO_arm64(std::unique_ptr<JITLinkContext> Ctx) { }); } - if (auto Err = Ctx->modifyPassConfig(TT, Config)) + if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) return Ctx->notifyFailed(std::move(Err)); // Construct a JITLinker and run the link function. - MachOJITLinker_arm64::link(std::move(Ctx), std::move(Config)); + MachOJITLinker_arm64::link(std::move(Ctx), std::move(G), std::move(Config)); } StringRef getMachOARM64RelocationKindName(Edge::Kind R) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index a91bc3b6033c..bde4a19e71ba 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -26,7 +26,7 @@ namespace { class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj) {} + : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin")) {} private: static Expected<MachOX86RelocationKind> @@ -150,10 +150,11 @@ private: else return ToSymbolOrErr.takeError(); } else { - if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue)) - ToSymbol = &*ToSymbolOrErr; - else - return ToSymbolOrErr.takeError(); + auto ToSymbolSec = findSectionByIndex(UnsignedRI.r_symbolnum - 1); + if (!ToSymbolSec) + return ToSymbolSec.takeError(); + ToSymbol = getSymbolByAddress(ToSymbolSec->Address); + assert(ToSymbol && "No symbol for section"); FixupValue -= ToSymbol->getAddress(); } @@ -183,6 +184,8 @@ private: using namespace support; auto &Obj = getObject(); + LLVM_DEBUG(dbgs() << "Processing relocations:\n"); + for (auto &S : Obj.sections()) { JITTargetAddress SectionAddress = S.getAddress(); @@ -201,8 +204,8 @@ private: getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); if (!NSec.GraphSection) { LLVM_DEBUG({ - dbgs() << "Skipping relocations for MachO section " << NSec.SegName - << "/" << NSec.SectName + dbgs() << " Skipping relocations for MachO section " + << NSec.SegName << "/" << NSec.SectName << " which has no associated graph section\n"; }); continue; @@ -224,8 +227,10 @@ private: JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; LLVM_DEBUG({ - dbgs() << "Processing relocation at " - << format("0x%016" PRIx64, FixupAddress) << "\n"; + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + dbgs() << " " << NSec.SectName << " + " + << formatv("{0:x8}", RI.r_address) << ":\n"; }); // Find the block that the fixup points to. @@ -334,12 +339,16 @@ private: assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } + case PCRel32TLV: + return make_error<JITLinkError>( + "MachO TLV relocations not yet supported"); default: llvm_unreachable("Special relocation kind should not appear in " "mach-o file"); } LLVM_DEBUG({ + dbgs() << " "; Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); printEdge(dbgs(), *BlockToFix, GE, @@ -539,22 +548,15 @@ class MachOJITLinker_x86_64 : public JITLinker<MachOJITLinker_x86_64> { public: MachOJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) - : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: StringRef getEdgeKindName(Edge::Kind R) const override { return getMachOX86RelocationKindName(R); } - Expected<std::unique_ptr<LinkGraph>> - buildGraph(MemoryBufferRef ObjBuffer) override { - auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); - if (!MachOObj) - return MachOObj.takeError(); - return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph(); - } - static Error targetOutOfRangeError(const Block &B, const Edge &E) { std::string ErrMsg; { @@ -651,18 +653,27 @@ private: uint64_t NullValue = 0; }; -void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject_x86_64(MemoryBufferRef ObjectBuffer) { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph(); +} + +void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; - Triple TT("x86_64-apple-macosx"); - if (Ctx->shouldAddDefaultTargetPasses(TT)) { + if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { // Add eh-frame passses. Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame")); - Config.PrePrunePasses.push_back( - EHFrameEdgeFixer("__eh_frame", NegDelta32, Delta64, Delta64)); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + "__eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); // Add a mark-live pass. - if (auto MarkLive = Ctx->getMarkLivePass(TT)) + if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) Config.PrePrunePasses.push_back(std::move(MarkLive)); else Config.PrePrunePasses.push_back(markAllSymbolsLive); @@ -674,14 +685,14 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { }); // Add GOT/Stubs optimizer pass. - Config.PostAllocationPasses.push_back(optimizeMachO_x86_64_GOTAndStubs); + Config.PreFixupPasses.push_back(optimizeMachO_x86_64_GOTAndStubs); } - if (auto Err = Ctx->modifyPassConfig(TT, Config)) + if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) return Ctx->notifyFailed(std::move(Err)); // Construct a JITLinker and run the link function. - MachOJITLinker_x86_64::link(std::move(Ctx), std::move(Config)); + MachOJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); } StringRef getMachOX86RelocationKindName(Edge::Kind R) { |