diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine/JITLink')
18 files changed, 475 insertions, 320 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h index 8ae3bc2bf61d..159880e4b152 100644 --- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h +++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h @@ -52,13 +52,13 @@ public: auto &SR = getSectionRange(*D.Sec); if (D.IsStart) { if (SR.empty()) - G.makeAbsolute(*Sym, 0); + G.makeAbsolute(*Sym, orc::ExecutorAddr()); else G.makeDefined(*Sym, *SR.getFirstBlock(), 0, 0, Linkage::Strong, Scope::Local, false); } else { if (SR.empty()) - G.makeAbsolute(*Sym, 0); + G.makeAbsolute(*Sym, orc::ExecutorAddr()); else G.makeDefined(*Sym, *SR.getLastBlock(), SR.getLastBlock()->getSize(), 0, Linkage::Strong, diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 4d7d5ce26668..2ae193595fc0 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -65,10 +65,7 @@ Error EHFrameSplitter::operator()(LinkGraph &G) { Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache) { - LLVM_DEBUG({ - dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) - << "\n"; - }); + LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); // eh-frame should not contain zero-fill blocks. if (B.isZeroFill()) @@ -400,7 +397,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, BlockEdgeMap &BlockEdges) { LLVM_DEBUG(dbgs() << " Record is FDE\n"); - JITTargetAddress RecordAddress = B.getAddress() + RecordOffset; + orc::ExecutorAddr RecordAddress = B.getAddress() + RecordOffset; auto RecordContent = B.getContent().slice(RecordOffset, RecordLength); BinaryStreamReader RecordReader( @@ -418,8 +415,9 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, { // Process the CIE pointer field. auto CIEEdgeItr = BlockEdges.find(RecordOffset + CIEDeltaFieldOffset); - JITTargetAddress CIEAddress = - RecordAddress + CIEDeltaFieldOffset - CIEDelta; + orc::ExecutorAddr CIEAddress = + RecordAddress + orc::ExecutorAddrDiff(CIEDeltaFieldOffset) - + orc::ExecutorAddrDiff(CIEDelta); if (CIEEdgeItr == BlockEdges.end()) { LLVM_DEBUG({ @@ -456,7 +454,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, { // Process the PC-Begin field. Block *PCBeginBlock = nullptr; - JITTargetAddress PCBeginFieldOffset = RecordReader.getOffset(); + orc::ExecutorAddrDiff PCBeginFieldOffset = RecordReader.getOffset(); auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset); if (PCEdgeItr == BlockEdges.end()) { auto PCBeginPtrInfo = @@ -464,12 +462,12 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, RecordAddress + PCBeginFieldOffset, RecordReader); if (!PCBeginPtrInfo) return PCBeginPtrInfo.takeError(); - JITTargetAddress PCBegin = PCBeginPtrInfo->first; + orc::ExecutorAddr PCBegin = PCBeginPtrInfo->first; Edge::Kind PCBeginEdgeKind = PCBeginPtrInfo->second; LLVM_DEBUG({ dbgs() << " Adding edge at " - << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) - << " to PC at " << formatv("{0:x16}", PCBegin) << "\n"; + << (RecordAddress + PCBeginFieldOffset) << " to PC at " + << formatv("{0:x16}", PCBegin) << "\n"; }); auto PCBeginSym = getOrCreateSymbol(PC, PCBegin); if (!PCBeginSym) @@ -522,7 +520,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, if (auto Err = RecordReader.readULEB128(AugmentationDataSize)) return Err; - JITTargetAddress LSDAFieldOffset = RecordReader.getOffset(); + orc::ExecutorAddrDiff LSDAFieldOffset = RecordReader.getOffset(); auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset); if (LSDAEdgeItr == BlockEdges.end()) { auto LSDAPointerInfo = @@ -530,7 +528,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, RecordAddress + LSDAFieldOffset, RecordReader); if (!LSDAPointerInfo) return LSDAPointerInfo.takeError(); - JITTargetAddress LSDA = LSDAPointerInfo->first; + orc::ExecutorAddr LSDA = LSDAPointerInfo->first; Edge::Kind LSDAEdgeKind = LSDAPointerInfo->second; auto LSDASym = getOrCreateSymbol(PC, LSDA); if (!LSDASym) @@ -645,12 +643,10 @@ unsigned EHFrameEdgeFixer::getPointerEncodingDataSize(uint8_t PointerEncoding) { } } -Expected<std::pair<JITTargetAddress, Edge::Kind>> +Expected<std::pair<orc::ExecutorAddr, Edge::Kind>> EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, - JITTargetAddress PointerFieldAddress, + orc::ExecutorAddr PointerFieldAddress, BinaryStreamReader &RecordReader) { - static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), - "Result must be able to hold a uint64_t"); assert(isSupportedPointerEncoding(PointerEncoding) && "Unsupported pointer encoding"); @@ -663,7 +659,7 @@ EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, if (EffectiveType == DW_EH_PE_absptr) EffectiveType = (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; - JITTargetAddress Addr; + orc::ExecutorAddr Addr; Edge::Kind PointerEdgeKind = Edge::Invalid; switch (EffectiveType) { case DW_EH_PE_udata4: { @@ -709,7 +705,7 @@ EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, } Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, - JITTargetAddress Addr) { + orc::ExecutorAddr Addr) { Symbol *CanonicalSym = nullptr; auto UpdateCanonicalSym = [&](Symbol *Sym) { @@ -753,8 +749,9 @@ Error EHFrameNullTerminator::operator()(LinkGraph &G) { << EHFrameSectionName << "\n"; }); - auto &NullTerminatorBlock = G.createContentBlock( - *EHFrame, NullTerminatorBlockContent, 0xfffffffffffffffc, 1, 0); + auto &NullTerminatorBlock = + G.createContentBlock(*EHFrame, NullTerminatorBlockContent, + orc::ExecutorAddr(~uint64_t(4)), 1, 0); G.addAnonymousSymbol(NullTerminatorBlock, 0, 4, false, true); return Error::success(); } @@ -762,17 +759,15 @@ Error EHFrameNullTerminator::operator()(LinkGraph &G) { EHFrameRegistrar::~EHFrameRegistrar() {} Error InProcessEHFrameRegistrar::registerEHFrames( - JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { - return orc::registerEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr), - EHFrameSectionSize); + orc::ExecutorAddrRange EHFrameSection) { + return orc::registerEHFrameSection(EHFrameSection.Start.toPtr<void *>(), + EHFrameSection.size()); } Error InProcessEHFrameRegistrar::deregisterEHFrames( - JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { - return orc::deregisterEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr), - EHFrameSectionSize); + orc::ExecutorAddrRange EHFrameSection) { + return orc::deregisterEHFrameSection(EHFrameSection.Start.toPtr<void *>(), + EHFrameSection.size()); } LinkGraphPassFunction @@ -789,14 +784,14 @@ createEHFrameRecorderPass(const Triple &TT, StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error { // Search for a non-empty eh-frame and record the address of the first // symbol in it. - JITTargetAddress Addr = 0; + orc::ExecutorAddr Addr; size_t Size = 0; if (auto *S = G.findSectionByName(EHFrameSectionName)) { auto R = SectionRange(*S); Addr = R.getStart(); Size = R.getSize(); } - if (Addr == 0 && Size != 0) + if (!Addr && Size != 0) return make_error<JITLinkError>( StringRef(EHFrameSectionName) + " section can not have zero address with non-zero size"); diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index b4c4b0f7b097..ef4b47b9aa28 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -71,12 +71,12 @@ private: }; using BlockEdgeMap = DenseMap<Edge::OffsetT, EdgeTarget>; - using CIEInfosMap = DenseMap<JITTargetAddress, CIEInformation>; + using CIEInfosMap = DenseMap<orc::ExecutorAddr, CIEInformation>; struct ParseContext { ParseContext(LinkGraph &G) : G(G) {} - Expected<CIEInformation *> findCIEInfo(JITTargetAddress Address) { + Expected<CIEInformation *> findCIEInfo(orc::ExecutorAddr Address) { auto I = CIEInfos.find(Address); if (I == CIEInfos.end()) return make_error<JITLinkError>("No CIE found at address " + @@ -102,12 +102,13 @@ private: static bool isSupportedPointerEncoding(uint8_t PointerEncoding); unsigned getPointerEncodingDataSize(uint8_t PointerEncoding); - Expected<std::pair<JITTargetAddress, Edge::Kind>> + Expected<std::pair<orc::ExecutorAddr, Edge::Kind>> readEncodedPointer(uint8_t PointerEncoding, - JITTargetAddress PointerFieldAddress, + orc::ExecutorAddr PointerFieldAddress, BinaryStreamReader &RecordReader); - Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, JITTargetAddress Addr); + Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, + orc::ExecutorAddr Addr); StringRef EHFrameSectionName; unsigned PointerSize; diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h index f9101d71dfa8..2ab7ed61f71b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h @@ -77,14 +77,14 @@ protected: return Obj.getHeader().e_type == llvm::ELF::ET_REL; } - void setGraphSection(ELFSectionIndex SecIndex, Section &Sec) { - assert(!GraphSections.count(SecIndex) && "Duplicate section at index"); - GraphSections[SecIndex] = &Sec; + void setGraphBlock(ELFSectionIndex SecIndex, Block *B) { + assert(!GraphBlocks.count(SecIndex) && "Duplicate section at index"); + GraphBlocks[SecIndex] = B; } - Section *getGraphSection(ELFSectionIndex SecIndex) { - auto I = GraphSections.find(SecIndex); - if (I == GraphSections.end()) + Block *getGraphBlock(ELFSectionIndex SecIndex) { + auto I = GraphBlocks.find(SecIndex); + if (I == GraphBlocks.end()) return nullptr; return I->second; } @@ -139,9 +139,9 @@ protected: const typename ELFFile::Elf_Shdr *SymTabSec = nullptr; StringRef SectionStringTab; - // Maps ELF section indexes to LinkGraph Sections. - // Only SHF_ALLOC sections will have graph sections. - DenseMap<ELFSectionIndex, Section *> GraphSections; + // Maps ELF section indexes to LinkGraph Blocks. + // Only SHF_ALLOC sections will have graph blocks. + DenseMap<ELFSectionIndex, Block *> GraphBlocks; DenseMap<ELFSymbolIndex, Symbol *> GraphSymbols; DenseMap<const typename ELFFile::Elf_Shdr *, ArrayRef<typename ELFFile::Elf_Word>> @@ -316,18 +316,27 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() { else Prot = MemProt::Read | MemProt::Write; - auto &GraphSec = G->createSection(*Name, Prot); + // Look for existing sections first. + auto *GraphSec = G->findSectionByName(*Name); + if (!GraphSec) + GraphSec = &G->createSection(*Name, Prot); + assert(GraphSec->getMemProt() == Prot && "MemProt should match"); + + Block *B = nullptr; if (Sec.sh_type != ELF::SHT_NOBITS) { auto Data = Obj.template getSectionContentsAsArray<char>(Sec); if (!Data) return Data.takeError(); - G->createContentBlock(GraphSec, *Data, Sec.sh_addr, Sec.sh_addralign, 0); + B = &G->createContentBlock(*GraphSec, *Data, + orc::ExecutorAddr(Sec.sh_addr), + Sec.sh_addralign, 0); } else - G->createZeroFillBlock(GraphSec, Sec.sh_size, Sec.sh_addr, - Sec.sh_addralign, 0); + B = &G->createZeroFillBlock(*GraphSec, Sec.sh_size, + orc::ExecutorAddr(Sec.sh_addr), + Sec.sh_addralign, 0); - setGraphSection(SecIndex, GraphSec); + setGraphBlock(SecIndex, B); } return Error::success(); @@ -393,9 +402,9 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { // Handle common symbols specially. if (Sym.isCommon()) { - Symbol &GSym = - G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, - Sym.st_size, Sym.getValue(), false); + Symbol &GSym = G->addCommonSymbol(*Name, Scope::Default, + getCommonSection(), orc::ExecutorAddr(), + Sym.st_size, Sym.getValue(), false); setGraphSymbol(SymIndex, GSym); continue; } @@ -425,28 +434,24 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { return NdxOrErr.takeError(); Shndx = *NdxOrErr; } - if (auto *GraphSec = getGraphSection(Shndx)) { - Block *B = nullptr; - { - auto Blocks = GraphSec->blocks(); - assert(Blocks.begin() != Blocks.end() && "No blocks for section"); - assert(std::next(Blocks.begin()) == Blocks.end() && - "Multiple blocks for section"); - B = *Blocks.begin(); - } - + if (auto *B = getGraphBlock(Shndx)) { LLVM_DEBUG({ dbgs() << " " << SymIndex << ": Creating defined graph symbol for ELF symbol \"" << *Name << "\"\n"; }); - if (Sym.getType() == ELF::STT_SECTION) - *Name = GraphSec->getName(); - + // In RISCV, temporary symbols (Used to generate dwarf, eh_frame + // sections...) will appear in object code's symbol table, and LLVM does + // not use names on these temporary symbols (RISCV gnu toolchain uses + // names on these temporary symbols). If the symbol is unnamed, add an + // anonymous symbol. auto &GSym = - G->addDefinedSymbol(*B, Sym.getValue(), *Name, Sym.st_size, L, S, - Sym.getType() == ELF::STT_FUNC, false); + Name->empty() + ? G->addAnonymousSymbol(*B, Sym.getValue(), Sym.st_size, + false, false) + : G->addDefinedSymbol(*B, Sym.getValue(), *Name, Sym.st_size, L, + S, Sym.getType() == ELF::STT_FUNC, false); setGraphSymbol(SymIndex, GSym); } } else if (Sym.isUndefined() && Sym.isExternal()) { @@ -498,8 +503,8 @@ Error ELFLinkGraphBuilder<ELFT>::forEachRelocation( } // Lookup the link-graph node corresponding to the target section name. - Section *GraphSect = G->findSectionByName(*Name); - if (!GraphSect) + auto *BlockToFix = getGraphBlock(RelSect.sh_info); + if (!BlockToFix) return make_error<StringError>( "Refencing a section that wasn't added to the graph: " + *Name, inconvertibleErrorCode()); @@ -510,7 +515,7 @@ Error ELFLinkGraphBuilder<ELFT>::forEachRelocation( // Let the callee process relocation entries one by one. for (const typename ELFT::Rela &R : *RelEntries) - if (Error Err = Func(R, **FixupSection, *GraphSect)) + if (Error Err = Func(R, **FixupSection, *BlockToFix)) return Err; LLVM_DEBUG(dbgs() << "\n"); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index dc183dfddfae..dd3eb97c21a0 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -16,6 +16,7 @@ #include "llvm/BinaryFormat/ELF.h" #include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/MathExtras.h" #define DEBUG_TYPE "jitlink" @@ -41,16 +42,17 @@ private: char *BlockWorkingMem = B.getAlreadyMutableContent().data(); char *FixupPtr = BlockWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + auto FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { case aarch64::R_AARCH64_CALL26: { - assert((FixupAddress & 0x3) == 0 && "Call-inst is not 32-bit aligned"); + assert((FixupAddress.getValue() & 0x3) == 0 && + "Call-inst is not 32-bit aligned"); int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); if (static_cast<uint64_t>(Value) & 0x3) return make_error<JITLinkError>("Call target is not 32-bit aligned"); - if (!fitsRangeSignedInt<27>(Value)) + if (!isInt<28>(Value)) return makeTargetOutOfRangeError(G, B, E); uint32_t RawInstr = *(little32_t *)FixupPtr; @@ -64,10 +66,6 @@ private: } return Error::success(); } - - template <uint8_t Bits> static bool fitsRangeSignedInt(int64_t Value) { - return Value >= -(1ll << Bits) && Value < (1ll << Bits); - } }; template <typename ELFT> @@ -100,7 +98,7 @@ private: Error addSingleRelocation(const typename ELFT::Rela &Rel, const typename ELFT::Shdr &FixupSect, - Section &GraphSection) { + Block &BlockToFix) { using Base = ELFLinkGraphBuilder<ELFT>; uint32_t SymbolIndex = Rel.getSymbol(false); @@ -123,17 +121,17 @@ private: return Kind.takeError(); int64_t Addend = Rel.r_addend; - Block *BlockToFix = *(GraphSection.blocks().begin()); - JITTargetAddress FixupAddress = FixupSect.sh_addr + Rel.r_offset; - Edge::OffsetT Offset = FixupAddress - BlockToFix->getAddress(); + orc::ExecutorAddr FixupAddress = + orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; + Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); Edge GE(*Kind, Offset, *GraphSymbol, Addend); LLVM_DEBUG({ dbgs() << " "; - printEdge(dbgs(), *BlockToFix, GE, aarch64::getEdgeKindName(*Kind)); + printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(*Kind)); dbgs() << "\n"; }); - BlockToFix->addEdge(std::move(GE)); + BlockToFix.addEdge(std::move(GE)); return Error::success(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp index b057788ce3ef..f83001417e94 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -19,6 +19,7 @@ #include "llvm/ExecutionEngine/JITLink/riscv.h" #include "llvm/Object/ELF.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" #define DEBUG_TYPE "jitlink" using namespace llvm; @@ -44,15 +45,16 @@ public: bool isGOTEdgeToFix(Edge &E) const { return E.getKind() == R_RISCV_GOT_HI20; } Symbol &createGOTEntry(Symbol &Target) { - Block &GOTBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), 0, G.getPointerSize(), 0); + Block &GOTBlock = + G.createContentBlock(getGOTSection(), getGOTEntryBlockContent(), + orc::ExecutorAddr(), G.getPointerSize(), 0); GOTBlock.addEdge(isRV64() ? R_RISCV_64 : R_RISCV_32, 0, Target, 0); return G.addAnonymousSymbol(GOTBlock, 0, G.getPointerSize(), false, false); } Symbol &createPLTStub(Symbol &Target) { - Block &StubContentBlock = - G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 4, 0); + Block &StubContentBlock = G.createContentBlock( + getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 4, 0); auto &GOTEntrySymbol = getGOTEntry(Target); StubContentBlock.addEdge(R_RISCV_CALL, 0, GOTEntrySymbol, 0); return G.addAnonymousSymbol(StubContentBlock, 0, StubEntrySize, true, @@ -134,13 +136,13 @@ static Expected<const Edge &> getRISCVPCRelHi20(const Edge &E) { const Symbol &Sym = E.getTarget(); const Block &B = Sym.getBlock(); - JITTargetAddress Offset = Sym.getOffset(); + orc::ExecutorAddrDiff Offset = Sym.getOffset(); struct Comp { - bool operator()(const Edge &Lhs, JITTargetAddress Offset) { + bool operator()(const Edge &Lhs, orc::ExecutorAddrDiff Offset) { return Lhs.getOffset() < Offset; } - bool operator()(JITTargetAddress Offset, const Edge &Rhs) { + bool operator()(orc::ExecutorAddrDiff Offset, const Edge &Rhs) { return Offset < Rhs.getOffset(); } }; @@ -157,8 +159,24 @@ static Expected<const Edge &> getRISCVPCRelHi20(const Edge &E) { "No HI20 PCREL relocation type be found for LO12 PCREL relocation type"); } -static uint32_t extractBits(uint64_t Num, unsigned High, unsigned Low) { - return (Num & ((1ULL << (High + 1)) - 1)) >> Low; +static uint32_t extractBits(uint32_t Num, unsigned Low, unsigned Size) { + return (Num & (((1ULL << (Size + 1)) - 1) << Low)) >> Low; +} + +inline Error checkAlignment(llvm::orc::ExecutorAddr loc, uint64_t v, int n, + const Edge &E) { + if (v & (n - 1)) + return make_error<JITLinkError>("0x" + llvm::utohexstr(loc.getValue()) + + " improper alignment for relocation " + + formatv("{0:d}", E.getKind()) + ": 0x" + + llvm::utohexstr(v) + " is not aligned to " + + Twine(n) + " bytes"); + return Error::success(); +} + +static inline bool isInRangeForImmS32(int64_t Value) { + return (Value >= std::numeric_limits<int32_t>::min() && + Value <= std::numeric_limits<int32_t>::max()); } class ELFJITLinker_riscv : public JITLinker<ELFJITLinker_riscv> { @@ -176,27 +194,47 @@ private: char *BlockWorkingMem = B.getAlreadyMutableContent().data(); char *FixupPtr = BlockWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { case R_RISCV_32: { - int64_t Value = E.getTarget().getAddress() + E.getAddend(); + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); break; } case R_RISCV_64: { - int64_t Value = E.getTarget().getAddress() + E.getAddend(); + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); break; } + case R_RISCV_BRANCH: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + Error AlignmentIssue = checkAlignment(FixupAddress, Value, 2, E); + if (AlignmentIssue) { + return AlignmentIssue; + } + int64_t Lo = Value & 0xFFF; + uint32_t Imm31_25 = extractBits(Lo, 5, 6) << 25 | extractBits(Lo, 12, 1) + << 31; + uint32_t Imm11_7 = extractBits(Lo, 1, 4) << 8 | extractBits(Lo, 11, 1) + << 7; + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm31_25 | Imm11_7; + break; + } case R_RISCV_HI20: { - int64_t Value = E.getTarget().getAddress() + E.getAddend(); - int32_t Hi = (Value + 0x800) & 0xFFFFF000; + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + int64_t Hi = Value + 0x800; + if (LLVM_UNLIKELY(!isInRangeForImmS32(Hi))) + return makeTargetOutOfRangeError(G, B, E); uint32_t RawInstr = *(little32_t *)FixupPtr; - *(little32_t *)FixupPtr = (RawInstr & 0xFFF) | static_cast<uint32_t>(Hi); + *(little32_t *)FixupPtr = + (RawInstr & 0xFFF) | (static_cast<uint32_t>(Hi & 0xFFFFF000)); break; } case R_RISCV_LO12_I: { - int64_t Value = E.getTarget().getAddress() + E.getAddend(); + // FIXME: We assume that R_RISCV_HI20 is present in object code and pairs + // with current relocation R_RISCV_LO12_I. So here may need a check. + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); int32_t Lo = Value & 0xFFF; uint32_t RawInstr = *(little32_t *)FixupPtr; *(little32_t *)FixupPtr = @@ -205,23 +243,32 @@ private: } case R_RISCV_CALL: { int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; - int32_t Hi = (Value + 0x800) & 0xFFFFF000; + int64_t Hi = Value + 0x800; + if (LLVM_UNLIKELY(!isInRangeForImmS32(Hi))) + return makeTargetOutOfRangeError(G, B, E); int32_t Lo = Value & 0xFFF; uint32_t RawInstrAuipc = *(little32_t *)FixupPtr; uint32_t RawInstrJalr = *(little32_t *)(FixupPtr + 4); - *(little32_t *)FixupPtr = RawInstrAuipc | static_cast<uint32_t>(Hi); + *(little32_t *)FixupPtr = + RawInstrAuipc | (static_cast<uint32_t>(Hi & 0xFFFFF000)); *(little32_t *)(FixupPtr + 4) = RawInstrJalr | (static_cast<uint32_t>(Lo) << 20); break; } case R_RISCV_PCREL_HI20: { int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; - int32_t Hi = (Value + 0x800) & 0xFFFFF000; + int64_t Hi = Value + 0x800; + if (LLVM_UNLIKELY(!isInRangeForImmS32(Hi))) + return makeTargetOutOfRangeError(G, B, E); uint32_t RawInstr = *(little32_t *)FixupPtr; - *(little32_t *)FixupPtr = (RawInstr & 0xFFF) | static_cast<uint32_t>(Hi); + *(little32_t *)FixupPtr = + (RawInstr & 0xFFF) | (static_cast<uint32_t>(Hi & 0xFFFFF000)); break; } case R_RISCV_PCREL_LO12_I: { + // FIXME: We assume that R_RISCV_PCREL_HI20 is present in object code and + // pairs with current relocation R_RISCV_PCREL_LO12_I. So here may need a + // check. auto RelHI20 = getRISCVPCRelHi20(E); if (!RelHI20) return RelHI20.takeError(); @@ -234,17 +281,117 @@ private: break; } case R_RISCV_PCREL_LO12_S: { + // FIXME: We assume that R_RISCV_PCREL_HI20 is present in object code and + // pairs with current relocation R_RISCV_PCREL_LO12_S. So here may need a + // check. auto RelHI20 = getRISCVPCRelHi20(E); int64_t Value = RelHI20->getTarget().getAddress() + RelHI20->getAddend() - E.getTarget().getAddress(); int64_t Lo = Value & 0xFFF; - uint32_t Imm31_25 = extractBits(Lo, 11, 5) << 25; - uint32_t Imm11_7 = extractBits(Lo, 4, 0) << 7; + uint32_t Imm31_25 = extractBits(Lo, 5, 7) << 25; + uint32_t Imm11_7 = extractBits(Lo, 0, 5) << 7; uint32_t RawInstr = *(little32_t *)FixupPtr; *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm31_25 | Imm11_7; break; } + case R_RISCV_ADD64: { + int64_t Value = (E.getTarget().getAddress() + + support::endian::read64le(reinterpret_cast<const void *>( + FixupAddress.getValue())) + + E.getAddend()) + .getValue(); + *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); + break; + } + case R_RISCV_ADD32: { + int64_t Value = (E.getTarget().getAddress() + + support::endian::read32le(reinterpret_cast<const void *>( + FixupAddress.getValue())) + + E.getAddend()) + .getValue(); + *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); + break; + } + case R_RISCV_ADD16: { + int64_t Value = (E.getTarget().getAddress() + + support::endian::read16le(reinterpret_cast<const void *>( + FixupAddress.getValue())) + + E.getAddend()) + .getValue(); + *(little16_t *)FixupPtr = static_cast<uint32_t>(Value); + break; + } + case R_RISCV_ADD8: { + int64_t Value = + (E.getTarget().getAddress() + + *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) + + E.getAddend()) + .getValue(); + *FixupPtr = static_cast<uint8_t>(Value); + break; + } + case R_RISCV_SUB64: { + int64_t Value = support::endian::read64le(reinterpret_cast<const void *>( + FixupAddress.getValue())) - + E.getTarget().getAddress().getValue() - E.getAddend(); + *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); + break; + } + case R_RISCV_SUB32: { + int64_t Value = support::endian::read32le(reinterpret_cast<const void *>( + FixupAddress.getValue())) - + E.getTarget().getAddress().getValue() - E.getAddend(); + *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); + break; + } + case R_RISCV_SUB16: { + int64_t Value = support::endian::read16le(reinterpret_cast<const void *>( + FixupAddress.getValue())) - + E.getTarget().getAddress().getValue() - E.getAddend(); + *(little16_t *)FixupPtr = static_cast<uint32_t>(Value); + break; + } + case R_RISCV_SUB8: { + int64_t Value = + *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) - + E.getTarget().getAddress().getValue() - E.getAddend(); + *FixupPtr = static_cast<uint8_t>(Value); + break; + } + case R_RISCV_SET6: { + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + uint32_t RawData = *(little32_t *)FixupPtr; + int64_t Word6 = Value & 0x3f; + *(little32_t *)FixupPtr = (RawData & 0xffffffc0) | Word6; + break; + } + case R_RISCV_SET8: { + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + uint32_t RawData = *(little32_t *)FixupPtr; + int64_t Word8 = Value & 0xff; + *(little32_t *)FixupPtr = (RawData & 0xffffff00) | Word8; + break; + } + case R_RISCV_SET16: { + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + uint32_t RawData = *(little32_t *)FixupPtr; + int64_t Word16 = Value & 0xffff; + *(little32_t *)FixupPtr = (RawData & 0xffff0000) | Word16; + break; + } + case R_RISCV_SET32: { + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + int64_t Word32 = Value & 0xffffffff; + *(little32_t *)FixupPtr = Word32; + break; + } + case R_RISCV_32_PCREL: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + int64_t Word32 = Value & 0xffffffff; + *(little32_t *)FixupPtr = Word32; + break; + } } return Error::success(); } @@ -261,6 +408,8 @@ private: return EdgeKind_riscv::R_RISCV_32; case ELF::R_RISCV_64: return EdgeKind_riscv::R_RISCV_64; + case ELF::R_RISCV_BRANCH: + return EdgeKind_riscv::R_RISCV_BRANCH; case ELF::R_RISCV_HI20: return EdgeKind_riscv::R_RISCV_HI20; case ELF::R_RISCV_LO12_I: @@ -277,6 +426,32 @@ private: return EdgeKind_riscv::R_RISCV_GOT_HI20; case ELF::R_RISCV_CALL_PLT: return EdgeKind_riscv::R_RISCV_CALL_PLT; + case ELF::R_RISCV_ADD64: + return EdgeKind_riscv::R_RISCV_ADD64; + case ELF::R_RISCV_ADD32: + return EdgeKind_riscv::R_RISCV_ADD32; + case ELF::R_RISCV_ADD16: + return EdgeKind_riscv::R_RISCV_ADD16; + case ELF::R_RISCV_ADD8: + return EdgeKind_riscv::R_RISCV_ADD8; + case ELF::R_RISCV_SUB64: + return EdgeKind_riscv::R_RISCV_SUB64; + case ELF::R_RISCV_SUB32: + return EdgeKind_riscv::R_RISCV_SUB32; + case ELF::R_RISCV_SUB16: + return EdgeKind_riscv::R_RISCV_SUB16; + case ELF::R_RISCV_SUB8: + return EdgeKind_riscv::R_RISCV_SUB8; + case ELF::R_RISCV_SET6: + return EdgeKind_riscv::R_RISCV_SET6; + case ELF::R_RISCV_SET8: + return EdgeKind_riscv::R_RISCV_SET8; + case ELF::R_RISCV_SET16: + return EdgeKind_riscv::R_RISCV_SET16; + case ELF::R_RISCV_SET32: + return EdgeKind_riscv::R_RISCV_SET32; + case ELF::R_RISCV_32_PCREL: + return EdgeKind_riscv::R_RISCV_32_PCREL; } return make_error<JITLinkError>("Unsupported riscv relocation:" + @@ -298,7 +473,7 @@ private: Error addSingleRelocation(const typename ELFT::Rela &Rel, const typename ELFT::Shdr &FixupSect, - Section &GraphSection) { + Block &BlockToFix) { using Base = ELFLinkGraphBuilder<ELFT>; uint32_t SymbolIndex = Rel.getSymbol(false); @@ -321,17 +496,16 @@ private: return Kind.takeError(); int64_t Addend = Rel.r_addend; - Block *BlockToFix = *(GraphSection.blocks().begin()); - JITTargetAddress FixupAddress = FixupSect.sh_addr + Rel.r_offset; - Edge::OffsetT Offset = FixupAddress - BlockToFix->getAddress(); + auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; + Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); Edge GE(*Kind, Offset, *GraphSymbol, Addend); LLVM_DEBUG({ dbgs() << " "; - printEdge(dbgs(), *BlockToFix, GE, riscv::getEdgeKindName(*Kind)); + printEdge(dbgs(), BlockToFix, GE, riscv::getEdgeKindName(*Kind)); dbgs() << "\n"; }); - BlockToFix->addEdge(std::move(GE)); + BlockToFix.addEdge(std::move(GE)); return Error::success(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 27d8833ae19e..79d2cdbb30f1 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -59,8 +59,8 @@ public: // the TLS Info entry's key value will be written by the fixTLVSectionByName // pass, so create mutable content. auto &TLSInfoEntry = G.createMutableContentBlock( - getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), 0, 8, - 0); + getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), + orc::ExecutorAddr(), 8, 0); TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0); return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); } @@ -172,7 +172,7 @@ private: Error addSingleRelocation(const typename ELFT::Rela &Rel, const typename ELFT::Shdr &FixupSection, - Section &GraphSection) { + Block &BlockToFix) { using Base = ELFLinkGraphBuilder<ELFT>; uint32_t SymbolIndex = Rel.getSymbol(false); @@ -248,17 +248,16 @@ private: } } - Block *BlockToFix = *(GraphSection.blocks().begin()); - JITTargetAddress FixupAddress = FixupSection.sh_addr + Rel.r_offset; - Edge::OffsetT Offset = FixupAddress - BlockToFix->getAddress(); + auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset; + Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); Edge GE(Kind, Offset, *GraphSymbol, Addend); LLVM_DEBUG({ dbgs() << " "; - printEdge(dbgs(), *BlockToFix, GE, x86_64::getEdgeKindName(Kind)); + printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind)); dbgs() << "\n"; }); - BlockToFix->addEdge(std::move(GE)); + BlockToFix.addEdge(std::move(GE)); return Error::success(); } @@ -322,8 +321,9 @@ private: // If there's no defined symbol then create one. SectionRange SR(*GOTSection); if (SR.empty()) - GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0, - Linkage::Strong, Scope::Local, true); + GOTSymbol = + &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0, + Linkage::Strong, Scope::Local, true); else GOTSymbol = &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 51dcc1c35fad..78a603cfed17 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -90,8 +90,8 @@ const char *getScopeName(Scope S) { } raw_ostream &operator<<(raw_ostream &OS, const Block &B) { - return OS << formatv("{0:x16}", B.getAddress()) << " -- " - << formatv("{0:x8}", B.getAddress() + B.getSize()) << ": " + return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize()) + << ": " << "size = " << formatv("{0:x8}", B.getSize()) << ", " << (B.isZeroFill() ? "zero-fill" : "content") << ", align = " << B.getAlignment() @@ -100,9 +100,8 @@ raw_ostream &operator<<(raw_ostream &OS, const Block &B) { } raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { - OS << formatv("{0:x16}", Sym.getAddress()) << " (" - << (Sym.isDefined() ? "block" : "addressable") << " + " - << formatv("{0:x8}", Sym.getOffset()) + OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable") + << " + " << formatv("{0:x8}", Sym.getOffset()) << "): size: " << formatv("{0:x8}", Sym.getSize()) << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage())) << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", " @@ -113,9 +112,9 @@ raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { void printEdge(raw_ostream &OS, const Block &B, const Edge &E, StringRef EdgeKindName) { - OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": " - << formatv("{0:x16}", B.getAddress()) << " + " - << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName << " -> "; + OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress() + << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName + << " -> "; auto &TargetSym = E.getTarget(); if (TargetSym.hasName()) @@ -123,17 +122,16 @@ void printEdge(raw_ostream &OS, const Block &B, const Edge &E, else { auto &TargetBlock = TargetSym.getBlock(); auto &TargetSec = TargetBlock.getSection(); - JITTargetAddress SecAddress = ~JITTargetAddress(0); + orc::ExecutorAddr SecAddress(~uint64_t(0)); for (auto *B : TargetSec.blocks()) if (B->getAddress() < SecAddress) SecAddress = B->getAddress(); - JITTargetAddress SecDelta = TargetSym.getAddress() - SecAddress; - OS << formatv("{0:x16}", TargetSym.getAddress()) << " (section " - << TargetSec.getName(); + orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress; + OS << TargetSym.getAddress() << " (section " << TargetSec.getName(); if (SecDelta) OS << " + " << formatv("{0:x}", SecDelta); - OS << " / block " << formatv("{0:x16}", TargetBlock.getAddress()); + OS << " / block " << TargetBlock.getAddress(); if (TargetSym.getOffset()) OS << " + " << formatv("{0:x}", TargetSym.getOffset()); OS << ")"; @@ -265,7 +263,7 @@ void LinkGraph::dump(raw_ostream &OS) { }); for (auto *B : SortedBlocks) { - OS << " block " << formatv("{0:x16}", B->getAddress()) + OS << " block " << B->getAddress() << " size = " << formatv("{0:x8}", B->getSize()) << ", align = " << B->getAlignment() << ", alignment-offset = " << B->getAlignmentOffset(); @@ -290,9 +288,8 @@ void LinkGraph::dump(raw_ostream &OS) { return LHS.getOffset() < RHS.getOffset(); }); for (auto &E : SortedEdges) { - OS << " " << formatv("{0:x16}", B->getFixupAddress(E)) - << " (block + " << formatv("{0:x8}", E.getOffset()) - << "), addend = "; + OS << " " << B->getFixupAddress(E) << " (block + " + << formatv("{0:x8}", E.getOffset()) << "), addend = "; if (E.getAddend() >= 0) OS << formatv("+{0:x8}", E.getAddend()); else @@ -315,16 +312,14 @@ void LinkGraph::dump(raw_ostream &OS) { OS << "Absolute symbols:\n"; if (!llvm::empty(absolute_symbols())) { for (auto *Sym : absolute_symbols()) - OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym - << "\n"; + OS << " " << Sym->getAddress() << ": " << *Sym << "\n"; } else OS << " none\n"; OS << "\nExternal symbols:\n"; if (!llvm::empty(external_symbols())) { for (auto *Sym : external_symbols()) - OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym - << "\n"; + OS << " " << Sym->getAddress() << ": " << *Sym << "\n"; } else OS << " none\n"; } @@ -370,10 +365,13 @@ Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B, Section &Sec = B.getSection(); ErrStream << "In graph " << G.getName() << ", section " << Sec.getName() << ": relocation target "; - if (E.getTarget().hasName()) - ErrStream << "\"" << E.getTarget().getName() << "\" "; - ErrStream << "at address " << formatv("{0:x}", E.getTarget().getAddress()); - ErrStream << " is out of range of " << G.getEdgeKindName(E.getKind()) + if (E.getTarget().hasName()) { + ErrStream << "\"" << E.getTarget().getName() << "\""; + } else + ErrStream << E.getTarget().getBlock().getSection().getName() << " + " + << formatv("{0:x}", E.getOffset()); + ErrStream << " at address " << formatv("{0:x}", E.getTarget().getAddress()) + << " is out of range of " << G.getEdgeKindName(E.getKind()) << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " ("; Symbol *BestSymbolForBlock = nullptr; diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 706688aba4ec..35ee050c8566 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -192,7 +192,7 @@ JITLinkContext::LookupMap JITLinkerBase::getExternalSymbolNames() const { // Identify unresolved external symbols. JITLinkContext::LookupMap UnresolvedExternals; for (auto *Sym : G->external_symbols()) { - assert(Sym->getAddress() == 0 && + assert(!Sym->getAddress() && "External has already been assigned an address"); assert(Sym->getName() != StringRef() && Sym->getName() != "" && "Externals must be named"); @@ -209,11 +209,12 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { for (auto *Sym : G->external_symbols()) { assert(Sym->getOffset() == 0 && "External symbol is not at the start of its addressable block"); - assert(Sym->getAddress() == 0 && "Symbol already resolved"); + assert(!Sym->getAddress() && "Symbol already resolved"); assert(!Sym->isDefined() && "Symbol being resolved is already defined"); auto ResultI = Result.find(Sym->getName()); if (ResultI != Result.end()) - Sym->getAddressable().setAddress(ResultI->second.getAddress()); + Sym->getAddressable().setAddress( + orc::ExecutorAddr(ResultI->second.getAddress())); else assert(Sym->getLinkage() == Linkage::Weak && "Failed to resolve non-weak reference"); @@ -223,7 +224,7 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { dbgs() << "Externals after applying lookup result:\n"; for (auto *Sym : G->external_symbols()) dbgs() << " " << Sym->getName() << ": " - << formatv("{0:x16}", Sym->getAddress()) << "\n"; + << formatv("{0:x16}", Sym->getAddress().getValue()) << "\n"; }); } diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index e4fdda0783a4..1095fa5ce701 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -19,9 +19,6 @@ #define DEBUG_TYPE "jitlink" namespace llvm { - -class MemoryBufferRef; - namespace jitlink { /// Base class for a JIT linker. @@ -161,4 +158,4 @@ void prune(LinkGraph &G); #undef DEBUG_TYPE // "jitlink" -#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H +#endif // LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index 831b9b26d2fd..9315ac4f6120 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -15,63 +15,12 @@ using namespace llvm; -namespace { - -// FIXME: Remove this copy of CWrapperFunctionResult as soon as JITLink can -// depend on shared utils from Orc. - -// Must be kept in-sync with compiler-rt/lib/orc/c-api.h. -union CWrapperFunctionResultDataUnion { - char *ValuePtr; - char Value[sizeof(ValuePtr)]; -}; - -// Must be kept in-sync with compiler-rt/lib/orc/c-api.h. -typedef struct { - CWrapperFunctionResultDataUnion Data; - size_t Size; -} CWrapperFunctionResult; - -Error toError(CWrapperFunctionResult R) { - bool HasError = false; - std::string ErrMsg; - if (R.Size) { - bool Large = R.Size > sizeof(CWrapperFunctionResultDataUnion); - char *Content = Large ? R.Data.ValuePtr : R.Data.Value; - if (Content[0]) { - HasError = true; - constexpr unsigned StrStart = 1 + sizeof(uint64_t); - ErrMsg.resize(R.Size - StrStart); - memcpy(&ErrMsg[0], Content + StrStart, R.Size - StrStart); - } - if (Large) - free(R.Data.ValuePtr); - } else if (R.Data.ValuePtr) { - HasError = true; - ErrMsg = R.Data.ValuePtr; - free(R.Data.ValuePtr); - } - - if (HasError) - return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); - return Error::success(); -} -} // namespace - namespace llvm { namespace jitlink { JITLinkMemoryManager::~JITLinkMemoryManager() = default; JITLinkMemoryManager::InFlightAlloc::~InFlightAlloc() = default; -static Error runAllocAction(JITLinkMemoryManager::AllocActionCall &C) { - using WrapperFnTy = CWrapperFunctionResult (*)(const void *, size_t); - auto *Fn = jitTargetAddressToPointer<WrapperFnTy>(C.FnAddr); - - return toError(Fn(jitTargetAddressToPointer<const void *>(C.CtxAddr), - static_cast<size_t>(C.CtxSize))); -} - BasicLayout::BasicLayout(LinkGraph &G) : G(G) { for (auto &Sec : G.sections()) { @@ -189,7 +138,7 @@ Error BasicLayout::apply() { return Error::success(); } -JITLinkMemoryManager::AllocActions &BasicLayout::graphAllocActions() { +orc::shared::AllocActions &BasicLayout::graphAllocActions() { return G.allocActions(); } @@ -209,7 +158,7 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, std::make_unique<LinkGraph>("", Triple(), 0, support::native, nullptr); AllocGroupSmallMap<Block *> ContentBlocks; - JITTargetAddress NextAddr = 0x100000; + orc::ExecutorAddr NextAddr(0x100000); for (auto &KV : Segments) { auto &AG = KV.first; auto &Seg = KV.second; @@ -222,7 +171,8 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, Sec.setMemDeallocPolicy(AG.getMemDeallocPolicy()); if (Seg.ContentSize != 0) { - NextAddr = alignTo(NextAddr, Seg.ContentAlign); + NextAddr = + orc::ExecutorAddr(alignTo(NextAddr.getValue(), Seg.ContentAlign)); auto &B = G->createMutableContentBlock(Sec, G->allocateBuffer(Seg.ContentSize), NextAddr, Seg.ContentAlign.value(), 0); @@ -297,19 +247,11 @@ public: } // Run finalization actions. - // FIXME: Roll back previous successful actions on failure. - std::vector<AllocActionCall> DeallocActions; - DeallocActions.reserve(G.allocActions().size()); - for (auto &ActPair : G.allocActions()) { - if (ActPair.Finalize.FnAddr) - if (auto Err = runAllocAction(ActPair.Finalize)) { - OnFinalized(std::move(Err)); - return; - } - if (ActPair.Dealloc.FnAddr) - DeallocActions.push_back(ActPair.Dealloc); + auto DeallocActions = runFinalizeActions(G.allocActions()); + if (!DeallocActions) { + OnFinalized(DeallocActions.takeError()); + return; } - G.allocActions().clear(); // Release the finalize segments slab. if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments)) { @@ -319,7 +261,7 @@ public: // Continue with finalized allocation. OnFinalized(MemMgr.createFinalizedAlloc(std::move(StandardSegments), - std::move(DeallocActions))); + std::move(*DeallocActions))); } void abandon(OnAbandonedFunction OnAbandoned) override { @@ -428,8 +370,8 @@ void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, static_cast<size_t>(SegsSizes->FinalizeSegs)}; } - auto NextStandardSegAddr = pointerToJITTargetAddress(StandardSegsMem.base()); - auto NextFinalizeSegAddr = pointerToJITTargetAddress(FinalizeSegsMem.base()); + auto NextStandardSegAddr = orc::ExecutorAddr::fromPtr(StandardSegsMem.base()); + auto NextFinalizeSegAddr = orc::ExecutorAddr::fromPtr(FinalizeSegsMem.base()); LLVM_DEBUG({ dbgs() << "InProcessMemoryManager allocated:\n"; @@ -456,7 +398,7 @@ void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, ? NextStandardSegAddr : NextFinalizeSegAddr; - Seg.WorkingMem = jitTargetAddressToPointer<char *>(SegAddr); + Seg.WorkingMem = SegAddr.toPtr<char *>(); Seg.Addr = SegAddr; SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize); @@ -475,13 +417,12 @@ void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, void InProcessMemoryManager::deallocate(std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) { std::vector<sys::MemoryBlock> StandardSegmentsList; - std::vector<std::vector<AllocActionCall>> DeallocActionsList; + std::vector<std::vector<orc::shared::WrapperFunctionCall>> DeallocActionsList; { std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex); for (auto &Alloc : Allocs) { - auto *FA = - jitTargetAddressToPointer<FinalizedAllocInfo *>(Alloc.release()); + auto *FA = Alloc.release().toPtr<FinalizedAllocInfo *>(); StandardSegmentsList.push_back(std::move(FA->StandardSegments)); if (!FA->DeallocActions.empty()) DeallocActionsList.push_back(std::move(FA->DeallocActions)); @@ -498,7 +439,7 @@ void InProcessMemoryManager::deallocate(std::vector<FinalizedAlloc> Allocs, /// Run any deallocate calls. while (!DeallocActions.empty()) { - if (auto Err = runAllocAction(DeallocActions.back())) + if (auto Err = DeallocActions.back().runWithSPSRetErrorMerged()) DeallocErr = joinErrors(std::move(DeallocErr), std::move(Err)); DeallocActions.pop_back(); } @@ -517,12 +458,12 @@ void InProcessMemoryManager::deallocate(std::vector<FinalizedAlloc> Allocs, JITLinkMemoryManager::FinalizedAlloc InProcessMemoryManager::createFinalizedAlloc( sys::MemoryBlock StandardSegments, - std::vector<AllocActionCall> DeallocActions) { + std::vector<orc::shared::WrapperFunctionCall> DeallocActions) { std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex); auto *FA = FinalizedAllocInfos.Allocate<FinalizedAllocInfo>(); new (FA) FinalizedAllocInfo( {std::move(StandardSegments), std::move(DeallocActions)}); - return FinalizedAlloc(pointerToJITTargetAddress(FA)); + return FinalizedAlloc(orc::ExecutorAddr::fromPtr(FA)); } } // end namespace jitlink diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index d588b63d9e88..62574604458c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -134,7 +134,7 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { memcpy(&NSec.SegName, Sec64.segname, 16); NSec.SegName[16] = '\0'; - NSec.Address = Sec64.addr; + NSec.Address = orc::ExecutorAddr(Sec64.addr); NSec.Size = Sec64.size; NSec.Alignment = 1ULL << Sec64.align; NSec.Flags = Sec64.flags; @@ -147,7 +147,7 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { memcpy(&NSec.SegName, Sec32.segname, 16); NSec.SegName[16] = '\0'; - NSec.Address = Sec32.addr; + NSec.Address = orc::ExecutorAddr(Sec32.addr); NSec.Size = Sec32.size; NSec.Alignment = 1ULL << Sec32.align; NSec.Flags = Sec32.flags; @@ -287,7 +287,8 @@ Error MachOLinkGraphBuilder::createNormalizedSymbols() { if (!NSec) return NSec.takeError(); - if (Value < NSec->Address || Value > NSec->Address + NSec->Size) + if (orc::ExecutorAddr(Value) < NSec->Address || + orc::ExecutorAddr(Value) > NSec->Address + NSec->Size) return make_error<JITLinkError>("Address " + formatv("{0:x}", Value) + " for symbol " + *Name + " does not fall within section"); @@ -311,8 +312,9 @@ Error MachOLinkGraphBuilder::createNormalizedSymbols() { } void MachOLinkGraphBuilder::addSectionStartSymAndBlock( - unsigned SecIndex, Section &GraphSec, uint64_t Address, const char *Data, - uint64_t Size, uint32_t Alignment, bool IsLive) { + unsigned SecIndex, Section &GraphSec, orc::ExecutorAddr Address, + const char *Data, orc::ExecutorAddrDiff Size, uint32_t Alignment, + bool IsLive) { Block &B = Data ? G->createContentBlock(GraphSec, ArrayRef<char>(Data, Size), Address, Alignment, 0) @@ -346,7 +348,8 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { return make_error<JITLinkError>("Anonymous common symbol at index " + Twine(KV.first)); NSym.GraphSymbol = &G->addCommonSymbol( - *NSym.Name, NSym.S, getCommonSection(), 0, NSym.Value, + *NSym.Name, NSym.S, getCommonSection(), orc::ExecutorAddr(), + orc::ExecutorAddrDiff(NSym.Value), 1ull << MachO::GET_COMM_ALIGN(NSym.Desc), NSym.Desc & MachO::N_NO_DEAD_STRIP); } else { @@ -364,8 +367,8 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { return make_error<JITLinkError>("Anonymous absolute symbol at index " + Twine(KV.first)); NSym.GraphSymbol = &G->addAbsoluteSymbol( - *NSym.Name, NSym.Value, 0, Linkage::Strong, Scope::Default, - NSym.Desc & MachO::N_NO_DEAD_STRIP); + *NSym.Name, orc::ExecutorAddr(NSym.Value), 0, Linkage::Strong, + Scope::Default, NSym.Desc & MachO::N_NO_DEAD_STRIP); break; case MachO::N_SECT: SecIndexToSymbols[NSym.Sect - 1].push_back(&NSym); @@ -468,13 +471,13 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { // If the section is non-empty but there is no symbol covering the start // address then add an anonymous one. - if (SecNSymStack.back()->Value != NSec.Address) { - auto AnonBlockSize = SecNSymStack.back()->Value - NSec.Address; + if (orc::ExecutorAddr(SecNSymStack.back()->Value) != NSec.Address) { + auto AnonBlockSize = + orc::ExecutorAddr(SecNSymStack.back()->Value) - NSec.Address; LLVM_DEBUG({ dbgs() << " Section start not covered by symbol. " - << "Creating anonymous block to cover [ " - << formatv("{0:x16}", NSec.Address) << " -- " - << formatv("{0:x16}", NSec.Address + AnonBlockSize) << " ]\n"; + << "Creating anonymous block to cover [ " << NSec.Address + << " -- " << (NSec.Address + AnonBlockSize) << " ]\n"; }); addSectionStartSymAndBlock(SecIndex, *NSec.GraphSection, NSec.Address, NSec.Data, AnonBlockSize, NSec.Alignment, @@ -496,12 +499,12 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { } // BlockNSyms now contains the block symbols in reverse canonical order. - JITTargetAddress BlockStart = BlockSyms.front()->Value; - JITTargetAddress BlockEnd = SecNSymStack.empty() - ? NSec.Address + NSec.Size - : SecNSymStack.back()->Value; - JITTargetAddress BlockOffset = BlockStart - NSec.Address; - JITTargetAddress BlockSize = BlockEnd - BlockStart; + auto BlockStart = orc::ExecutorAddr(BlockSyms.front()->Value); + orc::ExecutorAddr BlockEnd = + SecNSymStack.empty() ? NSec.Address + NSec.Size + : orc::ExecutorAddr(SecNSymStack.back()->Value); + orc::ExecutorAddrDiff BlockOffset = BlockStart - NSec.Address; + orc::ExecutorAddrDiff BlockSize = BlockEnd - BlockStart; LLVM_DEBUG({ dbgs() << " Creating block for " << formatv("{0:x16}", BlockStart) @@ -521,8 +524,8 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { BlockStart, NSec.Alignment, BlockStart % NSec.Alignment); - Optional<JITTargetAddress> LastCanonicalAddr; - JITTargetAddress SymEnd = BlockEnd; + Optional<orc::ExecutorAddr> LastCanonicalAddr; + auto SymEnd = BlockEnd; while (!BlockSyms.empty()) { auto &NSym = *BlockSyms.back(); BlockSyms.pop_back(); @@ -530,9 +533,9 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { bool SymLive = (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; - auto &Sym = createStandardGraphSymbol(NSym, B, SymEnd - NSym.Value, - SectionIsText, SymLive, - LastCanonicalAddr != NSym.Value); + auto &Sym = createStandardGraphSymbol( + NSym, B, SymEnd - orc::ExecutorAddr(NSym.Value), SectionIsText, + SymLive, LastCanonicalAddr != orc::ExecutorAddr(NSym.Value)); if (LastCanonicalAddr != Sym.getAddress()) { if (LastCanonicalAddr) @@ -568,11 +571,12 @@ Symbol &MachOLinkGraphBuilder::createStandardGraphSymbol(NormalizedSymbol &NSym, dbgs() << "\n"; }); - auto &Sym = NSym.Name ? G->addDefinedSymbol(B, NSym.Value - B.getAddress(), - *NSym.Name, Size, NSym.L, NSym.S, - IsText, IsNoDeadStrip) - : G->addAnonymousSymbol(B, NSym.Value - B.getAddress(), - Size, IsText, IsNoDeadStrip); + auto SymOffset = orc::ExecutorAddr(NSym.Value) - B.getAddress(); + auto &Sym = + NSym.Name + ? G->addDefinedSymbol(B, SymOffset, *NSym.Name, Size, NSym.L, NSym.S, + IsText, IsNoDeadStrip) + : G->addAnonymousSymbol(B, SymOffset, Size, IsText, IsNoDeadStrip); NSym.GraphSymbol = &Sym; if (IsCanonical) @@ -635,12 +639,12 @@ Error MachOLinkGraphBuilder::graphifyCStringSection( bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP; bool SectionIsText = NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS; - JITTargetAddress BlockStart = 0; + orc::ExecutorAddrDiff BlockStart = 0; // Scan section for null characters. for (size_t I = 0; I != NSec.Size; ++I) if (NSec.Data[I] == '\0') { - JITTargetAddress BlockEnd = I + 1; + orc::ExecutorAddrDiff BlockEnd = I + 1; size_t BlockSize = BlockEnd - BlockStart; // Create a block for this null terminated string. auto &B = G->createContentBlock(*NSec.GraphSection, @@ -654,7 +658,8 @@ Error MachOLinkGraphBuilder::graphifyCStringSection( }); // If there's no symbol at the start of this block then create one. - if (NSyms.empty() || NSyms.back()->Value != B.getAddress()) { + if (NSyms.empty() || + orc::ExecutorAddr(NSyms.back()->Value) != B.getAddress()) { auto &S = G->addAnonymousSymbol(B, 0, BlockSize, false, false); setCanonicalSymbol(NSec, S); LLVM_DEBUG({ @@ -666,18 +671,19 @@ Error MachOLinkGraphBuilder::graphifyCStringSection( } // Process any remaining symbols that point into this block. - JITTargetAddress LastCanonicalAddr = B.getAddress() + BlockEnd; - while (!NSyms.empty() && - NSyms.back()->Value < (B.getAddress() + BlockSize)) { + auto LastCanonicalAddr = B.getAddress() + BlockEnd; + while (!NSyms.empty() && orc::ExecutorAddr(NSyms.back()->Value) < + B.getAddress() + BlockSize) { auto &NSym = *NSyms.back(); - size_t SymSize = (B.getAddress() + BlockSize) - NSyms.back()->Value; + size_t SymSize = (B.getAddress() + BlockSize) - + orc::ExecutorAddr(NSyms.back()->Value); bool SymLive = (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; bool IsCanonical = false; - if (LastCanonicalAddr != NSym.Value) { + if (LastCanonicalAddr != orc::ExecutorAddr(NSym.Value)) { IsCanonical = true; - LastCanonicalAddr = NSym.Value; + LastCanonicalAddr = orc::ExecutorAddr(NSym.Value); } createStandardGraphSymbol(NSym, B, SymSize, SectionIsText, SymLive, @@ -785,7 +791,7 @@ Error CompactUnwindSplitter::operator()(LinkGraph &G) { E.getTarget().getName() + " is an external symbol"); auto &TgtBlock = E.getTarget().getBlock(); auto &CURecSym = - G.addAnonymousSymbol(CURec, 0, CURecordSize, 0, false); + G.addAnonymousSymbol(CURec, 0, CURecordSize, false, false); TgtBlock.addEdge(Edge::KeepAlive, 0, CURecSym, 0); AddedKeepAlive = true; } else if (E.getOffset() != PersonalityEdgeOffset && diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index d29732ebdba8..2951a8533098 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -71,13 +71,13 @@ protected: public: char SectName[17]; char SegName[17]; - uint64_t Address = 0; + orc::ExecutorAddr Address; uint64_t Size = 0; uint64_t Alignment = 0; uint32_t Flags = 0; const char *Data = nullptr; Section *GraphSection = nullptr; - std::map<JITTargetAddress, Symbol *> CanonicalSymbols; + std::map<orc::ExecutorAddr, Symbol *> CanonicalSymbols; }; using SectionParserFunction = std::function<Error(NormalizedSection &S)>; @@ -137,7 +137,7 @@ protected: /// Returns the symbol with the highest address not greater than the search /// address, or null if no such symbol exists. Symbol *getSymbolByAddress(NormalizedSection &NSec, - JITTargetAddress Address) { + orc::ExecutorAddr Address) { auto I = NSec.CanonicalSymbols.upper_bound(Address); if (I == NSec.CanonicalSymbols.begin()) return nullptr; @@ -147,7 +147,7 @@ protected: /// Returns the symbol with the highest address not greater than the search /// address, or an error if no such symbol exists. Expected<Symbol &> findSymbolByAddress(NormalizedSection &NSec, - JITTargetAddress Address) { + orc::ExecutorAddr Address) { auto *Sym = getSymbolByAddress(NSec, Address); if (Sym) if (Address <= Sym->getAddress() + Sym->getSize()) @@ -193,9 +193,9 @@ private: Section &getCommonSection(); void addSectionStartSymAndBlock(unsigned SecIndex, Section &GraphSec, - uint64_t Address, const char *Data, - uint64_t Size, uint32_t Alignment, - bool IsLive); + orc::ExecutorAddr Address, const char *Data, + orc::ExecutorAddrDiff Size, + uint32_t Alignment, bool IsLive); Error createNormalizedSections(); Error createNormalizedSymbols(); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index f2a029d35cd5..3ca2e40c7263 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -109,7 +109,7 @@ private: Expected<PairRelocInfo> parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, const MachO::relocation_info &SubRI, - JITTargetAddress FixupAddress, const char *FixupContent, + orc::ExecutorAddr FixupAddress, const char *FixupContent, object::relocation_iterator &UnsignedRelItr, object::relocation_iterator &RelEnd) { using namespace support; @@ -162,7 +162,7 @@ private: return ToSymbolSec.takeError(); ToSymbol = getSymbolByAddress(*ToSymbolSec, ToSymbolSec->Address); assert(ToSymbol && "No symbol for section"); - FixupValue -= ToSymbol->getAddress(); + FixupValue -= ToSymbol->getAddress().getValue(); } MachOARM64RelocationKind DeltaKind; @@ -195,7 +195,7 @@ private: for (auto &S : Obj.sections()) { - JITTargetAddress SectionAddress = S.getAddress(); + orc::ExecutorAddr SectionAddress(S.getAddress()); // Skip relocations virtual sections. if (S.isVirtual()) { @@ -234,7 +234,8 @@ private: return Kind.takeError(); // Find the address of the value to fix up. - JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + orc::ExecutorAddr FixupAddress = + SectionAddress + (uint32_t)RI.r_address; LLVM_DEBUG({ dbgs() << " " << NSec->SectName << " + " << formatv("{0:x8}", RI.r_address) << ":\n"; @@ -249,7 +250,7 @@ private: BlockToFix = &SymbolToFixOrErr->getBlock(); } - if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > + if (FixupAddress + orc::ExecutorAddrDiff(1ULL << RI.r_length) > BlockToFix->getAddress() + BlockToFix->getContent().size()) return make_error<JITLinkError>( "Relocation content extends past end of fixup block"); @@ -290,7 +291,7 @@ private: }); // Find the address of the value to fix up. - JITTargetAddress PairedFixupAddress = + orc::ExecutorAddr PairedFixupAddress = SectionAddress + (uint32_t)RI.r_address; if (PairedFixupAddress != FixupAddress) return make_error<JITLinkError>("Paired relocation points at " @@ -324,7 +325,7 @@ private: Addend = *(const ulittle64_t *)FixupContent; break; case Pointer64Anon: { - JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + orc::ExecutorAddr TargetAddress(*(const ulittle64_t *)FixupContent); auto TargetNSec = findSectionByIndex(RI.r_symbolnum - 1); if (!TargetNSec) return TargetNSec.takeError(); @@ -435,7 +436,7 @@ public: Symbol &createGOTEntry(Symbol &Target) { auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + getGOTSection(), getGOTEntryBlockContent(), orc::ExecutorAddr(), 8, 0); GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); } @@ -457,8 +458,8 @@ public: } Symbol &createPLTStub(Symbol &Target) { - auto &StubContentBlock = - G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); + auto &StubContentBlock = G.createContentBlock( + getStubsSection(), getStubBlockContent(), orc::ExecutorAddr(), 1, 0); // Re-use GOT entries for stub targets. auto &GOTEntrySymbol = getGOTEntry(Target); StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0); @@ -474,7 +475,7 @@ public: private: Section &getGOTSection() { if (!GOTSection) - GOTSection = &G.createSection("$__GOT", MemProt::Read); + GOTSection = &G.createSection("$__GOT", MemProt::Read | MemProt::Exec); return *GOTSection; } @@ -545,11 +546,12 @@ private: char *BlockWorkingMem = B.getAlreadyMutableContent().data(); char *FixupPtr = BlockWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { case Branch26: { - assert((FixupAddress & 0x3) == 0 && "Branch-inst is not 32-bit aligned"); + assert((FixupAddress.getValue() & 0x3) == 0 && + "Branch-inst is not 32-bit aligned"); int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); @@ -569,7 +571,7 @@ private: break; } case Pointer32: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); if (Value > std::numeric_limits<uint32_t>::max()) return makeTargetOutOfRangeError(G, B, E); *(ulittle32_t *)FixupPtr = Value; @@ -577,7 +579,7 @@ private: } case Pointer64: case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); *(ulittle64_t *)FixupPtr = Value; break; } @@ -587,9 +589,10 @@ private: assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && "GOTPAGE21 with non-zero addend"); uint64_t TargetPage = - (E.getTarget().getAddress() + E.getAddend()) & - ~static_cast<uint64_t>(4096 - 1); - uint64_t PCPage = FixupAddress & ~static_cast<uint64_t>(4096 - 1); + (E.getTarget().getAddress().getValue() + E.getAddend()) & + ~static_cast<uint64_t>(4096 - 1); + uint64_t PCPage = + FixupAddress.getValue() & ~static_cast<uint64_t>(4096 - 1); int64_t PageDelta = TargetPage - PCPage; if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) @@ -606,7 +609,7 @@ private: } case PageOffset12: { uint64_t TargetOffset = - (E.getTarget().getAddress() + E.getAddend()) & 0xfff; + (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff; uint32_t RawInstr = *(ulittle32_t *)FixupPtr; unsigned ImmShift = getPageOffset12Shift(RawInstr); @@ -627,7 +630,7 @@ private: assert((RawInstr & 0xfffffc00) == 0xf9400000 && "RawInstr isn't a 64-bit LDR immediate"); - uint32_t TargetOffset = E.getTarget().getAddress() & 0xfff; + uint32_t TargetOffset = E.getTarget().getAddress().getValue() & 0xfff; assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); uint32_t EncodedImm = (TargetOffset >> 3) << 10; uint32_t FixedInstr = RawInstr | EncodedImm; @@ -635,7 +638,8 @@ private: break; } case LDRLiteral19: { - assert((FixupAddress & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert((FixupAddress.getValue() & 0x3) == 0 && + "LDR is not 32-bit aligned"); assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); uint32_t RawInstr = *(ulittle32_t *)FixupPtr; assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); @@ -705,6 +709,13 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back( CompactUnwindSplitter("__LD,__compact_unwind")); + // Add eh-frame passses. + // FIXME: Prune eh-frames for which compact-unwind is available once + // we support compact-unwind registration with libunwind. + Config.PrePrunePasses.push_back(EHFrameSplitter("__TEXT,__eh_frame")); + Config.PrePrunePasses.push_back( + EHFrameEdgeFixer("__TEXT,__eh_frame", 8, Delta64, Delta32, NegDelta32)); + // Add an in-place GOT/Stubs pass. Config.PostPrunePasses.push_back( PerGraphGOTAndPLTStubsBuilder_MachO_arm64::asPass); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index a4fcd3b9a5f5..82afaa3aa3c5 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -119,7 +119,7 @@ private: // returns the edge kind and addend to be used. Expected<PairRelocInfo> parsePairRelocation( Block &BlockToFix, MachONormalizedRelocationType SubtractorKind, - const MachO::relocation_info &SubRI, JITTargetAddress FixupAddress, + const MachO::relocation_info &SubRI, orc::ExecutorAddr FixupAddress, const char *FixupContent, object::relocation_iterator &UnsignedRelItr, object::relocation_iterator &RelEnd) { using namespace support; @@ -172,7 +172,7 @@ private: return ToSymbolSec.takeError(); ToSymbol = getSymbolByAddress(*ToSymbolSec, ToSymbolSec->Address); assert(ToSymbol && "No symbol for section"); - FixupValue -= ToSymbol->getAddress(); + FixupValue -= ToSymbol->getAddress().getValue(); } Edge::Kind DeltaKind; @@ -206,7 +206,7 @@ private: for (auto &S : Obj.sections()) { - JITTargetAddress SectionAddress = S.getAddress(); + orc::ExecutorAddr SectionAddress(S.getAddress()); // Skip relocations virtual sections. if (S.isVirtual()) { @@ -241,7 +241,7 @@ private: MachO::relocation_info RI = getRelocationInfo(RelItr); // Find the address of the value to fix up. - JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + auto FixupAddress = SectionAddress + (uint32_t)RI.r_address; LLVM_DEBUG({ dbgs() << " " << NSec->SectName << " + " @@ -257,7 +257,7 @@ private: BlockToFix = &SymbolToFixOrErr->getBlock(); } - if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > + if (FixupAddress + orc::ExecutorAddrDiff(1ULL << RI.r_length) > BlockToFix->getAddress() + BlockToFix->getContent().size()) return make_error<JITLinkError>( "Relocation extends past end of fixup block"); @@ -343,7 +343,7 @@ private: Kind = x86_64::Pointer64; break; case MachOPointer64Anon: { - JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + orc::ExecutorAddr TargetAddress(*(const ulittle64_t *)FixupContent); auto TargetNSec = findSectionByIndex(RI.r_symbolnum - 1); if (!TargetNSec) return TargetNSec.takeError(); @@ -367,8 +367,8 @@ private: Kind = x86_64::Delta32; break; case MachOPCRel32Anon: { - JITTargetAddress TargetAddress = - FixupAddress + 4 + *(const little32_t *)FixupContent; + orc::ExecutorAddr TargetAddress(FixupAddress + 4 + + *(const little32_t *)FixupContent); auto TargetNSec = findSectionByIndex(RI.r_symbolnum - 1); if (!TargetNSec) return TargetNSec.takeError(); @@ -384,10 +384,10 @@ private: case MachOPCRel32Minus1Anon: case MachOPCRel32Minus2Anon: case MachOPCRel32Minus4Anon: { - JITTargetAddress Delta = - 4 + static_cast<JITTargetAddress>( + orc::ExecutorAddrDiff Delta = + 4 + orc::ExecutorAddrDiff( 1ULL << (*MachORelocKind - MachOPCRel32Minus1Anon)); - JITTargetAddress TargetAddress = + orc::ExecutorAddr TargetAddress = FixupAddress + Delta + *(const little32_t *)FixupContent; auto TargetNSec = findSectionByIndex(RI.r_symbolnum - 1); if (!TargetNSec) diff --git a/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h index 6e9df9c75a65..6e325f92bafb 100644 --- a/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h @@ -47,16 +47,16 @@ public: if (impl().isGOTEdgeToFix(E)) { LLVM_DEBUG({ dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) - << " edge at " << formatv("{0:x}", B->getFixupAddress(E)) - << " (" << formatv("{0:x}", B->getAddress()) << " + " + << " edge at " << B->getFixupAddress(E) << " (" + << B->getAddress() << " + " << formatv("{0:x}", E.getOffset()) << ")\n"; }); impl().fixGOTEdge(E, getGOTEntry(E.getTarget())); } else if (impl().isExternalBranchEdge(E)) { LLVM_DEBUG({ dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) - << " edge at " << formatv("{0:x}", B->getFixupAddress(E)) - << " (" << formatv("{0:x}", B->getAddress()) << " + " + << " edge at " << B->getFixupAddress(E) << " (" + << B->getAddress() << " + " << formatv("{0:x}", E.getOffset()) << ")\n"; }); impl().fixPLTEdge(E, getPLTStub(E.getTarget())); diff --git a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp index 6b73ff95a3b0..3ce2cf10a24c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp @@ -24,6 +24,8 @@ const char *getEdgeKindName(Edge::Kind K) { return "R_RISCV_32"; case R_RISCV_64: return "R_RISCV_64"; + case R_RISCV_BRANCH: + return "R_RISCV_BRANCH"; case R_RISCV_HI20: return "R_RISCV_HI20"; case R_RISCV_LO12_I: @@ -36,6 +38,32 @@ const char *getEdgeKindName(Edge::Kind K) { return "R_RISCV_PCREL_LO12_S"; case R_RISCV_CALL: return "R_RISCV_CALL"; + case R_RISCV_32_PCREL: + return "R_RISCV_32_PCREL"; + case R_RISCV_ADD64: + return "R_RISCV_ADD64"; + case R_RISCV_ADD32: + return "R_RISCV_ADD32"; + case R_RISCV_ADD16: + return "R_RISCV_ADD16"; + case R_RISCV_ADD8: + return "R_RISCV_ADD8"; + case R_RISCV_SUB64: + return "R_RISCV_SUB64"; + case R_RISCV_SUB32: + return "R_RISCV_SUB32"; + case R_RISCV_SUB16: + return "R_RISCV_SUB16"; + case R_RISCV_SUB8: + return "R_RISCV_SUB8"; + case R_RISCV_SET6: + return "R_RISCV_SET6"; + case R_RISCV_SET8: + return "R_RISCV_SET8"; + case R_RISCV_SET16: + return "R_RISCV_SET16"; + case R_RISCV_SET32: + return "R_RISCV_SET32"; } return getGenericEdgeKindName(K); } diff --git a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp index 48521280059d..df9979b47e88 100644 --- a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp @@ -95,10 +95,10 @@ Error optimizeGOTAndStubAccesses(LinkGraph &G) { assert(GOTEntryBlock.edges_size() == 1 && "GOT entry should only have one outgoing edge"); auto &GOTTarget = GOTEntryBlock.edges().begin()->getTarget(); - JITTargetAddress TargetAddr = GOTTarget.getAddress(); - JITTargetAddress EdgeAddr = B->getFixupAddress(E); + orc::ExecutorAddr TargetAddr = GOTTarget.getAddress(); + orc::ExecutorAddr EdgeAddr = B->getFixupAddress(E); int64_t Displacement = TargetAddr - EdgeAddr + 4; - bool TargetInRangeForImmU32 = isInRangeForImmU32(TargetAddr); + bool TargetInRangeForImmU32 = isInRangeForImmU32(TargetAddr.getValue()); bool DisplacementInRangeForImmS32 = isInRangeForImmS32(Displacement); // If both of the Target and displacement is out of range, then @@ -165,8 +165,8 @@ Error optimizeGOTAndStubAccesses(LinkGraph &G) { "GOT block should only have one outgoing edge"); auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); - JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); - JITTargetAddress TargetAddr = GOTTarget.getAddress(); + orc::ExecutorAddr EdgeAddr = B->getAddress() + E.getOffset(); + orc::ExecutorAddr TargetAddr = GOTTarget.getAddress(); int64_t Displacement = TargetAddr - EdgeAddr + 4; if (isInRangeForImmS32(Displacement)) { |