diff options
Diffstat (limited to 'lib/Bitcode/Writer/BitcodeWriter.cpp')
| -rw-r--r-- | lib/Bitcode/Writer/BitcodeWriter.cpp | 2225 |
1 files changed, 1481 insertions, 744 deletions
diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index a899a0cc3ee4..dcb8b58cd7b3 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -11,12 +11,12 @@ // //===----------------------------------------------------------------------===// -#include "llvm/Bitcode/ReaderWriter.h" #include "ValueEnumerator.h" -#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -24,20 +24,20 @@ #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/UseListOrder.h" #include "llvm/IR/ValueSymbolTable.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Program.h" +#include "llvm/Support/SHA1.h" #include "llvm/Support/raw_ostream.h" #include <cctype> #include <map> using namespace llvm; +namespace { /// These are manifest constants used by the bitcode writer. They do not need to /// be kept in sync with the reader, but need to be consistent within this file. enum { @@ -64,7 +64,455 @@ enum { FUNCTION_INST_GEP_ABBREV, }; -static unsigned GetEncodedCastOpcode(unsigned Opcode) { +/// Abstract class to manage the bitcode writing, subclassed for each bitcode +/// file type. Owns the BitstreamWriter, and includes the main entry point for +/// writing. +class BitcodeWriter { +protected: + /// Pointer to the buffer allocated by caller for bitcode writing. + const SmallVectorImpl<char> &Buffer; + + /// The stream created and owned by the BitodeWriter. + BitstreamWriter Stream; + + /// Saves the offset of the VSTOffset record that must eventually be + /// backpatched with the offset of the actual VST. + uint64_t VSTOffsetPlaceholder = 0; + +public: + /// Constructs a BitcodeWriter object, and initializes a BitstreamRecord, + /// writing to the provided \p Buffer. + BitcodeWriter(SmallVectorImpl<char> &Buffer) + : Buffer(Buffer), Stream(Buffer) {} + + virtual ~BitcodeWriter() = default; + + /// Main entry point to write the bitcode file, which writes the bitcode + /// header and will then invoke the virtual writeBlocks() method. + void write(); + +private: + /// Derived classes must implement this to write the corresponding blocks for + /// that bitcode file type. + virtual void writeBlocks() = 0; + +protected: + bool hasVSTOffsetPlaceholder() { return VSTOffsetPlaceholder != 0; } + void writeValueSymbolTableForwardDecl(); + void writeBitcodeHeader(); +}; + +/// Class to manage the bitcode writing for a module. +class ModuleBitcodeWriter : public BitcodeWriter { + /// The Module to write to bitcode. + const Module &M; + + /// Enumerates ids for all values in the module. + ValueEnumerator VE; + + /// Optional per-module index to write for ThinLTO. + const ModuleSummaryIndex *Index; + + /// True if a module hash record should be written. + bool GenerateHash; + + /// The start bit of the module block, for use in generating a module hash + uint64_t BitcodeStartBit = 0; + + /// Map that holds the correspondence between GUIDs in the summary index, + /// that came from indirect call profiles, and a value id generated by this + /// class to use in the VST and summary block records. + std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap; + + /// Tracks the last value id recorded in the GUIDToValueMap. + unsigned GlobalValueId; + +public: + /// Constructs a ModuleBitcodeWriter object for the given Module, + /// writing to the provided \p Buffer. + ModuleBitcodeWriter(const Module *M, SmallVectorImpl<char> &Buffer, + bool ShouldPreserveUseListOrder, + const ModuleSummaryIndex *Index, bool GenerateHash) + : BitcodeWriter(Buffer), M(*M), VE(*M, ShouldPreserveUseListOrder), + Index(Index), GenerateHash(GenerateHash) { + // Save the start bit of the actual bitcode, in case there is space + // saved at the start for the darwin header above. The reader stream + // will start at the bitcode, and we need the offset of the VST + // to line up. + BitcodeStartBit = Stream.GetCurrentBitNo(); + + // Assign ValueIds to any callee values in the index that came from + // indirect call profiles and were recorded as a GUID not a Value* + // (which would have been assigned an ID by the ValueEnumerator). + // The starting ValueId is just after the number of values in the + // ValueEnumerator, so that they can be emitted in the VST. + GlobalValueId = VE.getValues().size(); + if (Index) + for (const auto &GUIDSummaryLists : *Index) + // Examine all summaries for this GUID. + for (auto &Summary : GUIDSummaryLists.second) + if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) + // For each call in the function summary, see if the call + // is to a GUID (which means it is for an indirect call, + // otherwise we would have a Value for it). If so, synthesize + // a value id. + for (auto &CallEdge : FS->calls()) + if (CallEdge.first.isGUID()) + assignValueId(CallEdge.first.getGUID()); + } + +private: + /// Main entry point for writing a module to bitcode, invoked by + /// BitcodeWriter::write() after it writes the header. + void writeBlocks() override; + + /// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the + /// current llvm version, and a record for the epoch number. + void writeIdentificationBlock(); + + /// Emit the current module to the bitstream. + void writeModule(); + + uint64_t bitcodeStartBit() { return BitcodeStartBit; } + + void writeStringRecord(unsigned Code, StringRef Str, unsigned AbbrevToUse); + void writeAttributeGroupTable(); + void writeAttributeTable(); + void writeTypeTable(); + void writeComdats(); + void writeModuleInfo(); + void writeValueAsMetadata(const ValueAsMetadata *MD, + SmallVectorImpl<uint64_t> &Record); + void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + unsigned createDILocationAbbrev(); + void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record, + unsigned &Abbrev); + unsigned createGenericDINodeAbbrev(); + void writeGenericDINode(const GenericDINode *N, + SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev); + void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIEnumerator(const DIEnumerator *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIDerivedType(const DIDerivedType *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDICompositeType(const DICompositeType *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDISubroutineType(const DISubroutineType *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDICompileUnit(const DICompileUnit *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDISubprogram(const DISubprogram *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDILexicalBlock(const DILexicalBlock *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDILexicalBlockFile(const DILexicalBlockFile *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDITemplateTypeParameter(const DITemplateTypeParameter *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDITemplateValueParameter(const DITemplateValueParameter *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDIGlobalVariable(const DIGlobalVariable *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + void writeDILocalVariable(const DILocalVariable *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIExpression(const DIExpression *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIObjCProperty(const DIObjCProperty *N, + SmallVectorImpl<uint64_t> &Record, unsigned Abbrev); + void writeDIImportedEntity(const DIImportedEntity *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev); + unsigned createNamedMetadataAbbrev(); + void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record); + unsigned createMetadataStringsAbbrev(); + void writeMetadataStrings(ArrayRef<const Metadata *> Strings, + SmallVectorImpl<uint64_t> &Record); + void writeMetadataRecords(ArrayRef<const Metadata *> MDs, + SmallVectorImpl<uint64_t> &Record); + void writeModuleMetadata(); + void writeFunctionMetadata(const Function &F); + void writeFunctionMetadataAttachment(const Function &F); + void writeGlobalVariableMetadataAttachment(const GlobalVariable &GV); + void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record, + const GlobalObject &GO); + void writeModuleMetadataKinds(); + void writeOperandBundleTags(); + void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal); + void writeModuleConstants(); + bool pushValueAndType(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals); + void writeOperandBundles(ImmutableCallSite CS, unsigned InstID); + void pushValue(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals); + void pushValueSigned(const Value *V, unsigned InstID, + SmallVectorImpl<uint64_t> &Vals); + void writeInstruction(const Instruction &I, unsigned InstID, + SmallVectorImpl<unsigned> &Vals); + void writeValueSymbolTable( + const ValueSymbolTable &VST, bool IsModuleLevel = false, + DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex = nullptr); + void writeUseList(UseListOrder &&Order); + void writeUseListBlock(const Function *F); + void + writeFunction(const Function &F, + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex); + void writeBlockInfo(); + void writePerModuleFunctionSummaryRecord(SmallVector<uint64_t, 64> &NameVals, + GlobalValueSummary *Summary, + unsigned ValueID, + unsigned FSCallsAbbrev, + unsigned FSCallsProfileAbbrev, + const Function &F); + void writeModuleLevelReferences(const GlobalVariable &V, + SmallVector<uint64_t, 64> &NameVals, + unsigned FSModRefsAbbrev); + void writePerModuleGlobalValueSummary(); + void writeModuleHash(size_t BlockStartPos); + + void assignValueId(GlobalValue::GUID ValGUID) { + GUIDToValueIdMap[ValGUID] = ++GlobalValueId; + } + unsigned getValueId(GlobalValue::GUID ValGUID) { + const auto &VMI = GUIDToValueIdMap.find(ValGUID); + assert(VMI != GUIDToValueIdMap.end()); + return VMI->second; + } + // Helper to get the valueId for the type of value recorded in VI. + unsigned getValueId(ValueInfo VI) { + if (VI.isGUID()) + return getValueId(VI.getGUID()); + return VE.getValueID(VI.getValue()); + } + std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } +}; + +/// Class to manage the bitcode writing for a combined index. +class IndexBitcodeWriter : public BitcodeWriter { + /// The combined index to write to bitcode. + const ModuleSummaryIndex &Index; + + /// When writing a subset of the index for distributed backends, client + /// provides a map of modules to the corresponding GUIDs/summaries to write. + std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex; + + /// Map that holds the correspondence between the GUID used in the combined + /// index and a value id generated by this class to use in references. + std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap; + + /// Tracks the last value id recorded in the GUIDToValueMap. + unsigned GlobalValueId = 0; + +public: + /// Constructs a IndexBitcodeWriter object for the given combined index, + /// writing to the provided \p Buffer. When writing a subset of the index + /// for a distributed backend, provide a \p ModuleToSummariesForIndex map. + IndexBitcodeWriter(SmallVectorImpl<char> &Buffer, + const ModuleSummaryIndex &Index, + std::map<std::string, GVSummaryMapTy> + *ModuleToSummariesForIndex = nullptr) + : BitcodeWriter(Buffer), Index(Index), + ModuleToSummariesForIndex(ModuleToSummariesForIndex) { + // Assign unique value ids to all summaries to be written, for use + // in writing out the call graph edges. Save the mapping from GUID + // to the new global value id to use when writing those edges, which + // are currently saved in the index in terms of GUID. + for (const auto &I : *this) + GUIDToValueIdMap[I.first] = ++GlobalValueId; + } + + /// The below iterator returns the GUID and associated summary. + typedef std::pair<GlobalValue::GUID, GlobalValueSummary *> GVInfo; + + /// Iterator over the value GUID and summaries to be written to bitcode, + /// hides the details of whether they are being pulled from the entire + /// index or just those in a provided ModuleToSummariesForIndex map. + class iterator + : public llvm::iterator_facade_base<iterator, std::forward_iterator_tag, + GVInfo> { + /// Enables access to parent class. + const IndexBitcodeWriter &Writer; + + // Iterators used when writing only those summaries in a provided + // ModuleToSummariesForIndex map: + + /// Points to the last element in outer ModuleToSummariesForIndex map. + std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesBack; + /// Iterator on outer ModuleToSummariesForIndex map. + std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesIter; + /// Iterator on an inner global variable summary map. + GVSummaryMapTy::iterator ModuleGVSummariesIter; + + // Iterators used when writing all summaries in the index: + + /// Points to the last element in the Index outer GlobalValueMap. + const_gvsummary_iterator IndexSummariesBack; + /// Iterator on outer GlobalValueMap. + const_gvsummary_iterator IndexSummariesIter; + /// Iterator on an inner GlobalValueSummaryList. + GlobalValueSummaryList::const_iterator IndexGVSummariesIter; + + public: + /// Construct iterator from parent \p Writer and indicate if we are + /// constructing the end iterator. + iterator(const IndexBitcodeWriter &Writer, bool IsAtEnd) : Writer(Writer) { + // Set up the appropriate set of iterators given whether we are writing + // the full index or just a subset. + // Can't setup the Back or inner iterators if the corresponding map + // is empty. This will be handled specially in operator== as well. + if (Writer.ModuleToSummariesForIndex && + !Writer.ModuleToSummariesForIndex->empty()) { + for (ModuleSummariesBack = Writer.ModuleToSummariesForIndex->begin(); + std::next(ModuleSummariesBack) != + Writer.ModuleToSummariesForIndex->end(); + ModuleSummariesBack++) + ; + ModuleSummariesIter = !IsAtEnd + ? Writer.ModuleToSummariesForIndex->begin() + : ModuleSummariesBack; + ModuleGVSummariesIter = !IsAtEnd ? ModuleSummariesIter->second.begin() + : ModuleSummariesBack->second.end(); + } else if (!Writer.ModuleToSummariesForIndex && + Writer.Index.begin() != Writer.Index.end()) { + for (IndexSummariesBack = Writer.Index.begin(); + std::next(IndexSummariesBack) != Writer.Index.end(); + IndexSummariesBack++) + ; + IndexSummariesIter = + !IsAtEnd ? Writer.Index.begin() : IndexSummariesBack; + IndexGVSummariesIter = !IsAtEnd ? IndexSummariesIter->second.begin() + : IndexSummariesBack->second.end(); + } + } + + /// Increment the appropriate set of iterators. + iterator &operator++() { + // First the inner iterator is incremented, then if it is at the end + // and there are more outer iterations to go, the inner is reset to + // the start of the next inner list. + if (Writer.ModuleToSummariesForIndex) { + ++ModuleGVSummariesIter; + if (ModuleGVSummariesIter == ModuleSummariesIter->second.end() && + ModuleSummariesIter != ModuleSummariesBack) { + ++ModuleSummariesIter; + ModuleGVSummariesIter = ModuleSummariesIter->second.begin(); + } + } else { + ++IndexGVSummariesIter; + if (IndexGVSummariesIter == IndexSummariesIter->second.end() && + IndexSummariesIter != IndexSummariesBack) { + ++IndexSummariesIter; + IndexGVSummariesIter = IndexSummariesIter->second.begin(); + } + } + return *this; + } + + /// Access the <GUID,GlobalValueSummary*> pair corresponding to the current + /// outer and inner iterator positions. + GVInfo operator*() { + if (Writer.ModuleToSummariesForIndex) + return std::make_pair(ModuleGVSummariesIter->first, + ModuleGVSummariesIter->second); + return std::make_pair(IndexSummariesIter->first, + IndexGVSummariesIter->get()); + } + + /// Checks if the iterators are equal, with special handling for empty + /// indexes. + bool operator==(const iterator &RHS) const { + if (Writer.ModuleToSummariesForIndex) { + // First ensure that both are writing the same subset. + if (Writer.ModuleToSummariesForIndex != + RHS.Writer.ModuleToSummariesForIndex) + return false; + // Already determined above that maps are the same, so if one is + // empty, they both are. + if (Writer.ModuleToSummariesForIndex->empty()) + return true; + // Ensure the ModuleGVSummariesIter are iterating over the same + // container before checking them below. + if (ModuleSummariesIter != RHS.ModuleSummariesIter) + return false; + return ModuleGVSummariesIter == RHS.ModuleGVSummariesIter; + } + // First ensure RHS also writing the full index, and that both are + // writing the same full index. + if (RHS.Writer.ModuleToSummariesForIndex || + &Writer.Index != &RHS.Writer.Index) + return false; + // Already determined above that maps are the same, so if one is + // empty, they both are. + if (Writer.Index.begin() == Writer.Index.end()) + return true; + // Ensure the IndexGVSummariesIter are iterating over the same + // container before checking them below. + if (IndexSummariesIter != RHS.IndexSummariesIter) + return false; + return IndexGVSummariesIter == RHS.IndexGVSummariesIter; + } + }; + + /// Obtain the start iterator over the summaries to be written. + iterator begin() { return iterator(*this, /*IsAtEnd=*/false); } + /// Obtain the end iterator over the summaries to be written. + iterator end() { return iterator(*this, /*IsAtEnd=*/true); } + +private: + /// Main entry point for writing a combined index to bitcode, invoked by + /// BitcodeWriter::write() after it writes the header. + void writeBlocks() override; + + void writeIndex(); + void writeModStrings(); + void writeCombinedValueSymbolTable(); + void writeCombinedGlobalValueSummary(); + + /// Indicates whether the provided \p ModulePath should be written into + /// the module string table, e.g. if full index written or if it is in + /// the provided subset. + bool doIncludeModule(StringRef ModulePath) { + return !ModuleToSummariesForIndex || + ModuleToSummariesForIndex->count(ModulePath); + } + + bool hasValueId(GlobalValue::GUID ValGUID) { + const auto &VMI = GUIDToValueIdMap.find(ValGUID); + return VMI != GUIDToValueIdMap.end(); + } + unsigned getValueId(GlobalValue::GUID ValGUID) { + const auto &VMI = GUIDToValueIdMap.find(ValGUID); + // If this GUID doesn't have an entry, assign one. + if (VMI == GUIDToValueIdMap.end()) { + GUIDToValueIdMap[ValGUID] = ++GlobalValueId; + return GlobalValueId; + } else { + return VMI->second; + } + } + std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; } +}; +} // end anonymous namespace + +static unsigned getEncodedCastOpcode(unsigned Opcode) { switch (Opcode) { default: llvm_unreachable("Unknown cast instruction!"); case Instruction::Trunc : return bitc::CAST_TRUNC; @@ -83,7 +531,7 @@ static unsigned GetEncodedCastOpcode(unsigned Opcode) { } } -static unsigned GetEncodedBinaryOpcode(unsigned Opcode) { +static unsigned getEncodedBinaryOpcode(unsigned Opcode) { switch (Opcode) { default: llvm_unreachable("Unknown binary instruction!"); case Instruction::Add: @@ -107,7 +555,7 @@ static unsigned GetEncodedBinaryOpcode(unsigned Opcode) { } } -static unsigned GetEncodedRMWOperation(AtomicRMWInst::BinOp Op) { +static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) { switch (Op) { default: llvm_unreachable("Unknown RMW operation!"); case AtomicRMWInst::Xchg: return bitc::RMW_XCHG; @@ -124,20 +572,20 @@ static unsigned GetEncodedRMWOperation(AtomicRMWInst::BinOp Op) { } } -static unsigned GetEncodedOrdering(AtomicOrdering Ordering) { +static unsigned getEncodedOrdering(AtomicOrdering Ordering) { switch (Ordering) { - case NotAtomic: return bitc::ORDERING_NOTATOMIC; - case Unordered: return bitc::ORDERING_UNORDERED; - case Monotonic: return bitc::ORDERING_MONOTONIC; - case Acquire: return bitc::ORDERING_ACQUIRE; - case Release: return bitc::ORDERING_RELEASE; - case AcquireRelease: return bitc::ORDERING_ACQREL; - case SequentiallyConsistent: return bitc::ORDERING_SEQCST; + case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC; + case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED; + case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC; + case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE; + case AtomicOrdering::Release: return bitc::ORDERING_RELEASE; + case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL; + case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST; } llvm_unreachable("Invalid ordering"); } -static unsigned GetEncodedSynchScope(SynchronizationScope SynchScope) { +static unsigned getEncodedSynchScope(SynchronizationScope SynchScope) { switch (SynchScope) { case SingleThread: return bitc::SYNCHSCOPE_SINGLETHREAD; case CrossThread: return bitc::SYNCHSCOPE_CROSSTHREAD; @@ -145,8 +593,8 @@ static unsigned GetEncodedSynchScope(SynchronizationScope SynchScope) { llvm_unreachable("Invalid synch scope"); } -static void WriteStringRecord(unsigned Code, StringRef Str, - unsigned AbbrevToUse, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeStringRecord(unsigned Code, StringRef Str, + unsigned AbbrevToUse) { SmallVector<unsigned, 64> Vals; // Code: [strchar x N] @@ -164,6 +612,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { switch (Kind) { case Attribute::Alignment: return bitc::ATTR_KIND_ALIGNMENT; + case Attribute::AllocSize: + return bitc::ATTR_KIND_ALLOC_SIZE; case Attribute::AlwaysInline: return bitc::ATTR_KIND_ALWAYS_INLINE; case Attribute::ArgMemOnly: @@ -254,8 +704,14 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_SANITIZE_THREAD; case Attribute::SanitizeMemory: return bitc::ATTR_KIND_SANITIZE_MEMORY; + case Attribute::SwiftError: + return bitc::ATTR_KIND_SWIFT_ERROR; + case Attribute::SwiftSelf: + return bitc::ATTR_KIND_SWIFT_SELF; case Attribute::UWTable: return bitc::ATTR_KIND_UW_TABLE; + case Attribute::WriteOnly: + return bitc::ATTR_KIND_WRITEONLY; case Attribute::ZExt: return bitc::ATTR_KIND_Z_EXT; case Attribute::EndAttrKinds: @@ -267,8 +723,7 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { llvm_unreachable("Trying to encode unknown attribute"); } -static void WriteAttributeGroupTable(const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeAttributeGroupTable() { const std::vector<AttributeSet> &AttrGrps = VE.getAttributeGroups(); if (AttrGrps.empty()) return; @@ -315,8 +770,7 @@ static void WriteAttributeGroupTable(const ValueEnumerator &VE, Stream.ExitBlock(); } -static void WriteAttributeTable(const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeAttributeTable() { const std::vector<AttributeSet> &Attrs = VE.getAttributes(); if (Attrs.empty()) return; @@ -336,7 +790,7 @@ static void WriteAttributeTable(const ValueEnumerator &VE, } /// WriteTypeTable - Write out the type table for a module. -static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeTypeTable() { const ValueEnumerator::TypeList &TypeList = VE.getTypes(); Stream.EnterSubblock(bitc::TYPE_BLOCK_ID_NEW, 4 /*count from # abbrevs */); @@ -464,8 +918,8 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) { // Emit the name if it is present. if (!ST->getName().empty()) - WriteStringRecord(bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), - StructNameAbbrev, Stream); + writeStringRecord(bitc::TYPE_CODE_STRUCT_NAME, ST->getName(), + StructNameAbbrev); } break; } @@ -496,8 +950,8 @@ static void WriteTypeTable(const ValueEnumerator &VE, BitstreamWriter &Stream) { Stream.ExitBlock(); } -static unsigned getEncodedLinkage(const GlobalValue &GV) { - switch (GV.getLinkage()) { +static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) { + switch (Linkage) { case GlobalValue::ExternalLinkage: return 0; case GlobalValue::WeakAnyLinkage: @@ -524,6 +978,24 @@ static unsigned getEncodedLinkage(const GlobalValue &GV) { llvm_unreachable("Invalid linkage"); } +static unsigned getEncodedLinkage(const GlobalValue &GV) { + return getEncodedLinkage(GV.getLinkage()); +} + +// Decode the flags for GlobalValue in the summary +static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { + uint64_t RawFlags = 0; + + RawFlags |= Flags.HasSection; // bool + + // Linkage don't need to be remapped at that time for the summary. Any future + // change to the getEncodedLinkage() function will need to be taken into + // account here as well. + RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits + + return RawFlags; +} + static unsigned getEncodedVisibility(const GlobalValue &GV) { switch (GV.getVisibility()) { case GlobalValue::DefaultVisibility: return 0; @@ -569,13 +1041,22 @@ static unsigned getEncodedComdatSelectionKind(const Comdat &C) { llvm_unreachable("Invalid selection kind"); } -static void writeComdats(const ValueEnumerator &VE, BitstreamWriter &Stream) { - SmallVector<uint16_t, 64> Vals; +static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) { + switch (GV.getUnnamedAddr()) { + case GlobalValue::UnnamedAddr::None: return 0; + case GlobalValue::UnnamedAddr::Local: return 2; + case GlobalValue::UnnamedAddr::Global: return 1; + } + llvm_unreachable("Invalid unnamed_addr"); +} + +void ModuleBitcodeWriter::writeComdats() { + SmallVector<unsigned, 64> Vals; for (const Comdat *C : VE.getComdats()) { // COMDAT: [selection_kind, name] Vals.push_back(getEncodedComdatSelectionKind(*C)); size_t Size = C->getName().size(); - assert(isUInt<16>(Size)); + assert(isUInt<32>(Size)); Vals.push_back(Size); for (char Chr : C->getName()) Vals.push_back((unsigned char)Chr); @@ -586,12 +1067,8 @@ static void writeComdats(const ValueEnumerator &VE, BitstreamWriter &Stream) { /// Write a record that will eventually hold the word offset of the /// module-level VST. For now the offset is 0, which will be backpatched -/// after the real VST is written. Returns the bit offset to backpatch. -static uint64_t WriteValueSymbolTableForwardDecl(const ValueSymbolTable &VST, - BitstreamWriter &Stream) { - if (VST.empty()) - return 0; - +/// after the real VST is written. Saves the bit offset to backpatch. +void BitcodeWriter::writeValueSymbolTableForwardDecl() { // Write a placeholder value in for the offset of the real VST, // which is written after the function blocks so that it can include // the offset of each function. The placeholder offset will be @@ -608,27 +1085,44 @@ static uint64_t WriteValueSymbolTableForwardDecl(const ValueSymbolTable &VST, uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0}; Stream.EmitRecordWithAbbrev(VSTOffsetAbbrev, Vals); - // Compute and return the bit offset to the placeholder, which will be + // Compute and save the bit offset to the placeholder, which will be // patched when the real VST is written. We can simply subtract the 32-bit // fixed size from the current bit number to get the location to backpatch. - return Stream.GetCurrentBitNo() - 32; + VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32; +} + +enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 }; + +/// Determine the encoding to use for the given string name and length. +static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { + bool isChar6 = true; + for (const char *C = Str, *E = C + StrLen; C != E; ++C) { + if (isChar6) + isChar6 = BitCodeAbbrevOp::isChar6(*C); + if ((unsigned char)*C & 128) + // don't bother scanning the rest. + return SE_Fixed8; + } + if (isChar6) + return SE_Char6; + else + return SE_Fixed7; } /// Emit top-level description of module, including target triple, inline asm, /// descriptors for global variables, and function prototype info. /// Returns the bit offset to backpatch with the location of the real VST. -static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeModuleInfo() { // Emit various pieces of data attached to a module. - if (!M->getTargetTriple().empty()) - WriteStringRecord(bitc::MODULE_CODE_TRIPLE, M->getTargetTriple(), - 0/*TODO*/, Stream); - const std::string &DL = M->getDataLayoutStr(); + if (!M.getTargetTriple().empty()) + writeStringRecord(bitc::MODULE_CODE_TRIPLE, M.getTargetTriple(), + 0 /*TODO*/); + const std::string &DL = M.getDataLayoutStr(); if (!DL.empty()) - WriteStringRecord(bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/, Stream); - if (!M->getModuleInlineAsm().empty()) - WriteStringRecord(bitc::MODULE_CODE_ASM, M->getModuleInlineAsm(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_DATALAYOUT, DL, 0 /*TODO*/); + if (!M.getModuleInlineAsm().empty()) + writeStringRecord(bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(), + 0 /*TODO*/); // Emit information about sections and GC, computing how many there are. Also // compute the maximum alignment value. @@ -636,27 +1130,27 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, std::map<std::string, unsigned> GCMap; unsigned MaxAlignment = 0; unsigned MaxGlobalType = 0; - for (const GlobalValue &GV : M->globals()) { + for (const GlobalValue &GV : M.globals()) { MaxAlignment = std::max(MaxAlignment, GV.getAlignment()); MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType())); if (GV.hasSection()) { // Give section names unique ID's. unsigned &Entry = SectionMap[GV.getSection()]; if (!Entry) { - WriteStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), + 0 /*TODO*/); Entry = SectionMap.size(); } } } - for (const Function &F : *M) { + for (const Function &F : M) { MaxAlignment = std::max(MaxAlignment, F.getAlignment()); if (F.hasSection()) { // Give section names unique ID's. unsigned &Entry = SectionMap[F.getSection()]; if (!Entry) { - WriteStringRecord(bitc::MODULE_CODE_SECTIONNAME, F.getSection(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_SECTIONNAME, F.getSection(), + 0 /*TODO*/); Entry = SectionMap.size(); } } @@ -664,8 +1158,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, // Same for GC names. unsigned &Entry = GCMap[F.getGC()]; if (!Entry) { - WriteStringRecord(bitc::MODULE_CODE_GCNAME, F.getGC(), - 0/*TODO*/, Stream); + writeStringRecord(bitc::MODULE_CODE_GCNAME, F.getGC(), 0 /*TODO*/); Entry = GCMap.size(); } } @@ -673,7 +1166,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, // Emit abbrev for globals, now that we know # sections and max alignment. unsigned SimpleGVarAbbrev = 0; - if (!M->global_empty()) { + if (!M.global_empty()) { // Add an abbrev for common globals with no visibility or thread localness. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR)); @@ -702,7 +1195,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, // Emit the global variable information. SmallVector<unsigned, 64> Vals; - for (const GlobalVariable &GV : M->globals()) { + for (const GlobalVariable &GV : M.globals()) { unsigned AbbrevToUse = 0; // GLOBALVAR: [type, isconst, initid, @@ -718,12 +1211,13 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, Vals.push_back(GV.hasSection() ? SectionMap[GV.getSection()] : 0); if (GV.isThreadLocal() || GV.getVisibility() != GlobalValue::DefaultVisibility || - GV.hasUnnamedAddr() || GV.isExternallyInitialized() || + GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None || + GV.isExternallyInitialized() || GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass || GV.hasComdat()) { Vals.push_back(getEncodedVisibility(GV)); Vals.push_back(getEncodedThreadLocalMode(GV)); - Vals.push_back(GV.hasUnnamedAddr()); + Vals.push_back(getEncodedUnnamedAddr(GV)); Vals.push_back(GV.isExternallyInitialized()); Vals.push_back(getEncodedDLLStorageClass(GV)); Vals.push_back(GV.hasComdat() ? VE.getComdatID(GV.getComdat()) : 0); @@ -736,7 +1230,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, } // Emit the function proto information. - for (const Function &F : *M) { + for (const Function &F : M) { // FUNCTION: [type, callingconv, isproto, linkage, paramattrs, alignment, // section, visibility, gc, unnamed_addr, prologuedata, // dllstorageclass, comdat, prefixdata, personalityfn] @@ -749,7 +1243,7 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, Vals.push_back(F.hasSection() ? SectionMap[F.getSection()] : 0); Vals.push_back(getEncodedVisibility(F)); Vals.push_back(F.hasGC() ? GCMap[F.getGC()] : 0); - Vals.push_back(F.hasUnnamedAddr()); + Vals.push_back(getEncodedUnnamedAddr(F)); Vals.push_back(F.hasPrologueData() ? (VE.getValueID(F.getPrologueData()) + 1) : 0); Vals.push_back(getEncodedDLLStorageClass(F)); @@ -765,8 +1259,9 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, } // Emit the alias information. - for (const GlobalAlias &A : M->aliases()) { - // ALIAS: [alias type, aliasee val#, linkage, visibility] + for (const GlobalAlias &A : M.aliases()) { + // ALIAS: [alias type, aliasee val#, linkage, visibility, dllstorageclass, + // threadlocal, unnamed_addr] Vals.push_back(VE.getTypeID(A.getValueType())); Vals.push_back(A.getType()->getAddressSpace()); Vals.push_back(VE.getValueID(A.getAliasee())); @@ -774,33 +1269,56 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE, Vals.push_back(getEncodedVisibility(A)); Vals.push_back(getEncodedDLLStorageClass(A)); Vals.push_back(getEncodedThreadLocalMode(A)); - Vals.push_back(A.hasUnnamedAddr()); + Vals.push_back(getEncodedUnnamedAddr(A)); unsigned AbbrevToUse = 0; Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals, AbbrevToUse); Vals.clear(); } - // Write a record indicating the number of module-level metadata IDs - // This is needed because the ids of metadata are assigned implicitly - // based on their ordering in the bitcode, with the function-level - // metadata ids starting after the module-level metadata ids. For - // function importing where we lazy load the metadata as a postpass, - // we want to avoid parsing the module-level metadata before parsing - // the imported functions. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_METADATA_VALUES)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - unsigned MDValsAbbrev = Stream.EmitAbbrev(Abbv); - Vals.push_back(VE.numMDs()); - Stream.EmitRecord(bitc::MODULE_CODE_METADATA_VALUES, Vals, MDValsAbbrev); - Vals.clear(); + // Emit the ifunc information. + for (const GlobalIFunc &I : M.ifuncs()) { + // IFUNC: [ifunc type, address space, resolver val#, linkage, visibility] + Vals.push_back(VE.getTypeID(I.getValueType())); + Vals.push_back(I.getType()->getAddressSpace()); + Vals.push_back(VE.getValueID(I.getResolver())); + Vals.push_back(getEncodedLinkage(I)); + Vals.push_back(getEncodedVisibility(I)); + Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals); + Vals.clear(); + } + + // Emit the module's source file name. + { + StringEncoding Bits = getStringEncoding(M.getSourceFileName().data(), + M.getSourceFileName().size()); + BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); + if (Bits == SE_Char6) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); + else if (Bits == SE_Fixed7) + AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7); + + // MODULE_CODE_SOURCE_FILENAME: [namechar x N] + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(AbbrevOpToUse); + unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv); + + for (const auto P : M.getSourceFileName()) + Vals.push_back((unsigned char)P); - uint64_t VSTOffsetPlaceholder = - WriteValueSymbolTableForwardDecl(M->getValueSymbolTable(), Stream); - return VSTOffsetPlaceholder; + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_SOURCE_FILENAME, Vals, FilenameAbbrev); + Vals.clear(); + } + + // If we have a VST, write the VSTOFFSET record placeholder. + if (M.getValueSymbolTable().empty()) + return; + writeValueSymbolTableForwardDecl(); } -static uint64_t GetOptimizationFlags(const Value *V) { +static uint64_t getOptimizationFlags(const Value *V) { uint64_t Flags = 0; if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(V)) { @@ -827,10 +1345,8 @@ static uint64_t GetOptimizationFlags(const Value *V) { return Flags; } -static void WriteValueAsMetadata(const ValueAsMetadata *MD, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record) { +void ModuleBitcodeWriter::writeValueAsMetadata( + const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) { // Mimic an MDNode with a value as one operand. Value *V = MD->getValue(); Record.push_back(VE.getTypeID(V->getType())); @@ -839,9 +1355,9 @@ static void WriteValueAsMetadata(const ValueAsMetadata *MD, Record.clear(); } -static void WriteMDTuple(const MDTuple *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { Metadata *MD = N->getOperand(i); assert(!(MD && isa<LocalAsMetadata>(MD)) && @@ -854,10 +1370,25 @@ static void WriteMDTuple(const MDTuple *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDILocation(const DILocation *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +unsigned ModuleBitcodeWriter::createDILocationAbbrev() { + // Assume the column is usually under 128, and always output the inlined-at + // location (it's never more expensive than building an array size 1). + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + return Stream.EmitAbbrev(Abbv); +} + +void ModuleBitcodeWriter::writeDILocation(const DILocation *N, + SmallVectorImpl<uint64_t> &Record, + unsigned &Abbrev) { + if (!Abbrev) + Abbrev = createDILocationAbbrev(); + Record.push_back(N->isDistinct()); Record.push_back(N->getLine()); Record.push_back(N->getColumn()); @@ -868,11 +1399,26 @@ static void WriteDILocation(const DILocation *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteGenericDINode(const GenericDINode *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() { + // Assume the column is usually under 128, and always output the inlined-at + // location (it's never more expensive than building an array size 1). + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); + return Stream.EmitAbbrev(Abbv); +} + +void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N, + SmallVectorImpl<uint64_t> &Record, + unsigned &Abbrev) { + if (!Abbrev) + Abbrev = createGenericDINodeAbbrev(); + Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(0); // Per-tag version field; unused for now. @@ -889,10 +1435,9 @@ static uint64_t rotateSign(int64_t I) { return I < 0 ? ~(U << 1) : U << 1; } -static void WriteDISubrange(const DISubrange *N, const ValueEnumerator &, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getCount()); Record.push_back(rotateSign(N->getLowerBound())); @@ -901,10 +1446,9 @@ static void WriteDISubrange(const DISubrange *N, const ValueEnumerator &, Record.clear(); } -static void WriteDIEnumerator(const DIEnumerator *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(rotateSign(N->getValue())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -913,10 +1457,9 @@ static void WriteDIEnumerator(const DIEnumerator *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIBasicType(const DIBasicType *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -928,11 +1471,9 @@ static void WriteDIBasicType(const DIBasicType *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIDerivedType(const DIDerivedType *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -950,12 +1491,11 @@ static void WriteDIDerivedType(const DIDerivedType *N, Record.clear(); } -static void WriteDICompositeType(const DICompositeType *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { - Record.push_back(N->isDistinct()); +void ModuleBitcodeWriter::writeDICompositeType( + const DICompositeType *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + const unsigned IsNotUsedInOldTypeRef = 0x2; + Record.push_back(IsNotUsedInOldTypeRef | (unsigned)N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -976,22 +1516,22 @@ static void WriteDICompositeType(const DICompositeType *N, Record.clear(); } -static void WriteDISubroutineType(const DISubroutineType *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { - Record.push_back(N->isDistinct()); +void ModuleBitcodeWriter::writeDISubroutineType( + const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + const unsigned HasNoOldTypeRefs = 0x2; + Record.push_back(HasNoOldTypeRefs | (unsigned)N->isDistinct()); Record.push_back(N->getFlags()); Record.push_back(VE.getMetadataOrNullID(N->getTypeArray().get())); + Record.push_back(N->getCC()); Stream.EmitRecord(bitc::METADATA_SUBROUTINE_TYPE, Record, Abbrev); Record.clear(); } -static void WriteDIFile(const DIFile *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIFile(const DIFile *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawFilename())); Record.push_back(VE.getMetadataOrNullID(N->getRawDirectory())); @@ -1000,11 +1540,9 @@ static void WriteDIFile(const DIFile *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDICompileUnit(const DICompileUnit *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { assert(N->isDistinct() && "Expected distinct compile units"); Record.push_back(/* IsDistinct */ true); Record.push_back(N->getSourceLanguage()); @@ -1017,7 +1555,7 @@ static void WriteDICompileUnit(const DICompileUnit *N, Record.push_back(N->getEmissionKind()); Record.push_back(VE.getMetadataOrNullID(N->getEnumTypes().get())); Record.push_back(VE.getMetadataOrNullID(N->getRetainedTypes().get())); - Record.push_back(VE.getMetadataOrNullID(N->getSubprograms().get())); + Record.push_back(/* subprograms */ 0); Record.push_back(VE.getMetadataOrNullID(N->getGlobalVariables().get())); Record.push_back(VE.getMetadataOrNullID(N->getImportedEntities().get())); Record.push_back(N->getDWOId()); @@ -1027,11 +1565,11 @@ static void WriteDICompileUnit(const DICompileUnit *N, Record.clear(); } -static void WriteDISubprogram(const DISubprogram *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { - Record.push_back(N->isDistinct()); +void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { + uint64_t HasUnitFlag = 1 << 1; + Record.push_back(N->isDistinct() | HasUnitFlag); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getRawLinkageName())); @@ -1046,19 +1584,19 @@ static void WriteDISubprogram(const DISubprogram *N, const ValueEnumerator &VE, Record.push_back(N->getVirtualIndex()); Record.push_back(N->getFlags()); Record.push_back(N->isOptimized()); + Record.push_back(VE.getMetadataOrNullID(N->getRawUnit())); Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get())); Record.push_back(VE.getMetadataOrNullID(N->getDeclaration())); Record.push_back(VE.getMetadataOrNullID(N->getVariables().get())); + Record.push_back(N->getThisAdjustment()); Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev); Record.clear(); } -static void WriteDILexicalBlock(const DILexicalBlock *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1069,11 +1607,9 @@ static void WriteDILexicalBlock(const DILexicalBlock *N, Record.clear(); } -static void WriteDILexicalBlockFile(const DILexicalBlockFile *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDILexicalBlockFile( + const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1083,10 +1619,9 @@ static void WriteDILexicalBlockFile(const DILexicalBlockFile *N, Record.clear(); } -static void WriteDINamespace(const DINamespace *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1097,9 +1632,9 @@ static void WriteDINamespace(const DINamespace *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIMacro(const DIMacro *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getMacinfoType()); Record.push_back(N->getLine()); @@ -1110,10 +1645,9 @@ static void WriteDIMacro(const DIMacro *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIMacroFile(const DIMacroFile *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getMacinfoType()); Record.push_back(N->getLine()); @@ -1124,9 +1658,9 @@ static void WriteDIMacroFile(const DIMacroFile *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDIModule(const DIModule *N, const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIModule(const DIModule *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); for (auto &I : N->operands()) Record.push_back(VE.getMetadataOrNullID(I)); @@ -1135,11 +1669,9 @@ static void WriteDIModule(const DIModule *N, const ValueEnumerator &VE, Record.clear(); } -static void WriteDITemplateTypeParameter(const DITemplateTypeParameter *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDITemplateTypeParameter( + const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getType())); @@ -1148,11 +1680,9 @@ static void WriteDITemplateTypeParameter(const DITemplateTypeParameter *N, Record.clear(); } -static void WriteDITemplateValueParameter(const DITemplateValueParameter *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDITemplateValueParameter( + const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1163,11 +1693,9 @@ static void WriteDITemplateValueParameter(const DITemplateValueParameter *N, Record.clear(); } -static void WriteDIGlobalVariable(const DIGlobalVariable *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIGlobalVariable( + const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1184,11 +1712,9 @@ static void WriteDIGlobalVariable(const DIGlobalVariable *N, Record.clear(); } -static void WriteDILocalVariable(const DILocalVariable *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDILocalVariable( + const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); @@ -1202,10 +1728,9 @@ static void WriteDILocalVariable(const DILocalVariable *N, Record.clear(); } -static void WriteDIExpression(const DIExpression *N, const ValueEnumerator &, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.reserve(N->getElements().size() + 1); Record.push_back(N->isDistinct()); @@ -1215,11 +1740,9 @@ static void WriteDIExpression(const DIExpression *N, const ValueEnumerator &, Record.clear(); } -static void WriteDIObjCProperty(const DIObjCProperty *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N, + SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getFile())); @@ -1233,11 +1756,9 @@ static void WriteDIObjCProperty(const DIObjCProperty *N, Record.clear(); } -static void WriteDIImportedEntity(const DIImportedEntity *N, - const ValueEnumerator &VE, - BitstreamWriter &Stream, - SmallVectorImpl<uint64_t> &Record, - unsigned Abbrev) { +void ModuleBitcodeWriter::writeDIImportedEntity( + const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record, + unsigned Abbrev) { Record.push_back(N->isDistinct()); Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getScope())); @@ -1249,71 +1770,87 @@ static void WriteDIImportedEntity(const DIImportedEntity *N, Record.clear(); } -static void WriteModuleMetadata(const Module *M, - const ValueEnumerator &VE, - BitstreamWriter &Stream) { - const auto &MDs = VE.getMDs(); - if (MDs.empty() && M->named_metadata_empty()) +unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() { + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); + return Stream.EmitAbbrev(Abbv); +} + +void ModuleBitcodeWriter::writeNamedMetadata( + SmallVectorImpl<uint64_t> &Record) { + if (M.named_metadata_empty()) return; - Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); + unsigned Abbrev = createNamedMetadataAbbrev(); + for (const NamedMDNode &NMD : M.named_metadata()) { + // Write name. + StringRef Str = NMD.getName(); + Record.append(Str.bytes_begin(), Str.bytes_end()); + Stream.EmitRecord(bitc::METADATA_NAME, Record, Abbrev); + Record.clear(); - unsigned MDSAbbrev = 0; - if (VE.hasMDString()) { - // Abbrev for METADATA_STRING. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRING)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - MDSAbbrev = Stream.EmitAbbrev(Abbv); + // Write named metadata operands. + for (const MDNode *N : NMD.operands()) + Record.push_back(VE.getMetadataID(N)); + Stream.EmitRecord(bitc::METADATA_NAMED_NODE, Record, 0); + Record.clear(); } +} - // Initialize MDNode abbreviations. -#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0; -#include "llvm/IR/Metadata.def" +unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() { + BitCodeAbbrev *Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_STRINGS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + return Stream.EmitAbbrev(Abbv); +} - if (VE.hasDILocation()) { - // Abbrev for METADATA_LOCATION. - // - // Assume the column is usually under 128, and always output the inlined-at - // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - DILocationAbbrev = Stream.EmitAbbrev(Abbv); - } +/// Write out a record for MDString. +/// +/// All the metadata strings in a metadata block are emitted in a single +/// record. The sizes and strings themselves are shoved into a blob. +void ModuleBitcodeWriter::writeMetadataStrings( + ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) { + if (Strings.empty()) + return; - if (VE.hasGenericDINode()) { - // Abbrev for METADATA_GENERIC_DEBUG. - // - // Assume the column is usually under 128, and always output the inlined-at - // location (it's never more expensive than building an array size 1). - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); - GenericDINodeAbbrev = Stream.EmitAbbrev(Abbv); - } + // Start the record with the number of strings. + Record.push_back(bitc::METADATA_STRINGS); + Record.push_back(Strings.size()); - unsigned NameAbbrev = 0; - if (!M->named_metadata_empty()) { - // Abbrev for METADATA_NAME. - BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_NAME)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - NameAbbrev = Stream.EmitAbbrev(Abbv); + // Emit the sizes of the strings in the blob. + SmallString<256> Blob; + { + BitstreamWriter W(Blob); + for (const Metadata *MD : Strings) + W.EmitVBR(cast<MDString>(MD)->getLength(), 6); + W.FlushToWord(); } - SmallVector<uint64_t, 64> Record; + // Add the offset to the strings to the record. + Record.push_back(Blob.size()); + + // Add the strings to the blob. + for (const Metadata *MD : Strings) + Blob.append(cast<MDString>(MD)->getString()); + + // Emit the final record. + Stream.EmitRecordWithBlob(createMetadataStringsAbbrev(), Record, Blob); + Record.clear(); +} + +void ModuleBitcodeWriter::writeMetadataRecords( + ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record) { + if (MDs.empty()) + return; + + // Initialize MDNode abbreviations. +#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0; +#include "llvm/IR/Metadata.def" + for (const Metadata *MD : MDs) { if (const MDNode *N = dyn_cast<MDNode>(MD)) { assert(N->isResolved() && "Expected forward references to be resolved"); @@ -1323,82 +1860,79 @@ static void WriteModuleMetadata(const Module *M, llvm_unreachable("Invalid MDNode subclass"); #define HANDLE_MDNODE_LEAF(CLASS) \ case Metadata::CLASS##Kind: \ - Write##CLASS(cast<CLASS>(N), VE, Stream, Record, CLASS##Abbrev); \ + write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \ continue; #include "llvm/IR/Metadata.def" } } - if (const auto *MDC = dyn_cast<ConstantAsMetadata>(MD)) { - WriteValueAsMetadata(MDC, VE, Stream, Record); - continue; - } - const MDString *MDS = cast<MDString>(MD); - // Code: [strchar x N] - Record.append(MDS->bytes_begin(), MDS->bytes_end()); - - // Emit the finished record. - Stream.EmitRecord(bitc::METADATA_STRING, Record, MDSAbbrev); - Record.clear(); + writeValueAsMetadata(cast<ValueAsMetadata>(MD), Record); } +} - // Write named metadata. - for (const NamedMDNode &NMD : M->named_metadata()) { - // Write name. - StringRef Str = NMD.getName(); - Record.append(Str.bytes_begin(), Str.bytes_end()); - Stream.EmitRecord(bitc::METADATA_NAME, Record, NameAbbrev); - Record.clear(); +void ModuleBitcodeWriter::writeModuleMetadata() { + if (!VE.hasMDs() && M.named_metadata_empty()) + return; - // Write named metadata operands. - for (const MDNode *N : NMD.operands()) - Record.push_back(VE.getMetadataID(N)); - Stream.EmitRecord(bitc::METADATA_NAMED_NODE, Record, 0); - Record.clear(); - } + Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); + SmallVector<uint64_t, 64> Record; + writeMetadataStrings(VE.getMDStrings(), Record); + writeMetadataRecords(VE.getNonMDStrings(), Record); + writeNamedMetadata(Record); + + auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) { + SmallVector<uint64_t, 4> Record; + Record.push_back(VE.getValueID(&GO)); + pushGlobalMetadataAttachment(Record, GO); + Stream.EmitRecord(bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Record); + }; + for (const Function &F : M) + if (F.isDeclaration() && F.hasMetadata()) + AddDeclAttachedMetadata(F); + // FIXME: Only store metadata for declarations here, and move data for global + // variable definitions to a separate block (PR28134). + for (const GlobalVariable &GV : M.globals()) + if (GV.hasMetadata()) + AddDeclAttachedMetadata(GV); Stream.ExitBlock(); } -static void WriteFunctionLocalMetadata(const Function &F, - const ValueEnumerator &VE, - BitstreamWriter &Stream) { - bool StartedMetadataBlock = false; +void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) { + if (!VE.hasMDs()) + return; + + Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); SmallVector<uint64_t, 64> Record; - const SmallVectorImpl<const LocalAsMetadata *> &MDs = - VE.getFunctionLocalMDs(); - for (unsigned i = 0, e = MDs.size(); i != e; ++i) { - assert(MDs[i] && "Expected valid function-local metadata"); - if (!StartedMetadataBlock) { - Stream.EnterSubblock(bitc::METADATA_BLOCK_ID, 3); - StartedMetadataBlock = true; - } - WriteValueAsMetadata(MDs[i], VE, Stream, Record); - } + writeMetadataStrings(VE.getMDStrings(), Record); + writeMetadataRecords(VE.getNonMDStrings(), Record); + Stream.ExitBlock(); +} - if (StartedMetadataBlock) - Stream.ExitBlock(); +void ModuleBitcodeWriter::pushGlobalMetadataAttachment( + SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) { + // [n x [id, mdnode]] + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; + GO.getAllMetadata(MDs); + for (const auto &I : MDs) { + Record.push_back(I.first); + Record.push_back(VE.getMetadataID(I.second)); + } } -static void WriteMetadataAttachment(const Function &F, - const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) { Stream.EnterSubblock(bitc::METADATA_ATTACHMENT_ID, 3); SmallVector<uint64_t, 64> Record; - // Write metadata attachments - // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]] - SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; - F.getAllMetadata(MDs); - if (!MDs.empty()) { - for (const auto &I : MDs) { - Record.push_back(I.first); - Record.push_back(VE.getMetadataID(I.second)); - } + if (F.hasMetadata()) { + pushGlobalMetadataAttachment(Record, F); Stream.EmitRecord(bitc::METADATA_ATTACHMENT, Record, 0); Record.clear(); } + // Write metadata attachments + // METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]] + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { MDs.clear(); @@ -1420,13 +1954,13 @@ static void WriteMetadataAttachment(const Function &F, Stream.ExitBlock(); } -static void WriteModuleMetadataStore(const Module *M, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeModuleMetadataKinds() { SmallVector<uint64_t, 64> Record; // Write metadata kinds // METADATA_KIND - [n x [id, name]] SmallVector<StringRef, 8> Names; - M->getMDKindNames(Names); + M.getMDKindNames(Names); if (Names.empty()) return; @@ -1444,7 +1978,7 @@ static void WriteModuleMetadataStore(const Module *M, BitstreamWriter &Stream) { Stream.ExitBlock(); } -static void WriteOperandBundleTags(const Module *M, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeOperandBundleTags() { // Write metadata kinds // // OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG @@ -1452,7 +1986,7 @@ static void WriteOperandBundleTags(const Module *M, BitstreamWriter &Stream) { // OPERAND_BUNDLE_TAG - [strchr x N] SmallVector<StringRef, 8> Tags; - M->getOperandBundleTags(Tags); + M.getOperandBundleTags(Tags); if (Tags.empty()) return; @@ -1478,9 +2012,8 @@ static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) { Vals.push_back((-V << 1) | 1); } -static void WriteConstants(unsigned FirstVal, unsigned LastVal, - const ValueEnumerator &VE, - BitstreamWriter &Stream, bool isGlobal) { +void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, + bool isGlobal) { if (FirstVal == LastVal) return; Stream.EnterSubblock(bitc::CONSTANTS_BLOCK_ID, 4); @@ -1635,8 +2168,7 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal, Record.push_back( CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue()); } - } else if (isa<ConstantArray>(C) || isa<ConstantStruct>(C) || - isa<ConstantVector>(C)) { + } else if (isa<ConstantAggregate>(C)) { Code = bitc::CST_CODE_AGGREGATE; for (const Value *Op : C->operands()) Record.push_back(VE.getValueID(Op)); @@ -1646,17 +2178,17 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal, default: if (Instruction::isCast(CE->getOpcode())) { Code = bitc::CST_CODE_CE_CAST; - Record.push_back(GetEncodedCastOpcode(CE->getOpcode())); + Record.push_back(getEncodedCastOpcode(CE->getOpcode())); Record.push_back(VE.getTypeID(C->getOperand(0)->getType())); Record.push_back(VE.getValueID(C->getOperand(0))); AbbrevToUse = CONSTANTS_CE_CAST_Abbrev; } else { assert(CE->getNumOperands() == 2 && "Unknown constant expr!"); Code = bitc::CST_CODE_CE_BINOP; - Record.push_back(GetEncodedBinaryOpcode(CE->getOpcode())); + Record.push_back(getEncodedBinaryOpcode(CE->getOpcode())); Record.push_back(VE.getValueID(C->getOperand(0))); Record.push_back(VE.getValueID(C->getOperand(1))); - uint64_t Flags = GetOptimizationFlags(CE); + uint64_t Flags = getOptimizationFlags(CE); if (Flags != 0) Record.push_back(Flags); } @@ -1735,21 +2267,20 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal, Stream.ExitBlock(); } -static void WriteModuleConstants(const ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeModuleConstants() { const ValueEnumerator::ValueList &Vals = VE.getValues(); // Find the first constant to emit, which is the first non-globalvalue value. // We know globalvalues have been emitted by WriteModuleInfo. for (unsigned i = 0, e = Vals.size(); i != e; ++i) { if (!isa<GlobalValue>(Vals[i].first)) { - WriteConstants(i, Vals.size(), VE, Stream, true); + writeConstants(i, Vals.size(), true); return; } } } -/// PushValueAndType - The file has to encode both the value and type id for +/// pushValueAndType - The file has to encode both the value and type id for /// many values, because we need to know what type to create for forward /// references. However, most operands are not forward references, so this type /// field is not needed. @@ -1757,9 +2288,8 @@ static void WriteModuleConstants(const ValueEnumerator &VE, /// This function adds V's value ID to Vals. If the value ID is higher than the /// instruction ID, then it is a forward reference, and it also includes the /// type ID. The value ID that is written is encoded relative to the InstID. -static bool PushValueAndType(const Value *V, unsigned InstID, - SmallVectorImpl<unsigned> &Vals, - ValueEnumerator &VE) { +bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals) { unsigned ValID = VE.getValueID(V); // Make encoding relative to the InstID. Vals.push_back(InstID - ValID); @@ -1770,8 +2300,8 @@ static bool PushValueAndType(const Value *V, unsigned InstID, return false; } -static void WriteOperandBundles(BitstreamWriter &Stream, ImmutableCallSite CS, - unsigned InstID, ValueEnumerator &VE) { +void ModuleBitcodeWriter::writeOperandBundles(ImmutableCallSite CS, + unsigned InstID) { SmallVector<unsigned, 64> Record; LLVMContext &C = CS.getInstruction()->getContext(); @@ -1780,34 +2310,32 @@ static void WriteOperandBundles(BitstreamWriter &Stream, ImmutableCallSite CS, Record.push_back(C.getOperandBundleTagID(Bundle.getTagName())); for (auto &Input : Bundle.Inputs) - PushValueAndType(Input, InstID, Record, VE); + pushValueAndType(Input, InstID, Record); Stream.EmitRecord(bitc::FUNC_CODE_OPERAND_BUNDLE, Record); Record.clear(); } } -/// pushValue - Like PushValueAndType, but where the type of the value is +/// pushValue - Like pushValueAndType, but where the type of the value is /// omitted (perhaps it was already encoded in an earlier operand). -static void pushValue(const Value *V, unsigned InstID, - SmallVectorImpl<unsigned> &Vals, - ValueEnumerator &VE) { +void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID, + SmallVectorImpl<unsigned> &Vals) { unsigned ValID = VE.getValueID(V); Vals.push_back(InstID - ValID); } -static void pushValueSigned(const Value *V, unsigned InstID, - SmallVectorImpl<uint64_t> &Vals, - ValueEnumerator &VE) { +void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID, + SmallVectorImpl<uint64_t> &Vals) { unsigned ValID = VE.getValueID(V); int64_t diff = ((int32_t)InstID - (int32_t)ValID); emitSignedInt64(Vals, diff); } /// WriteInstruction - Emit an instruction to the specified stream. -static void WriteInstruction(const Instruction &I, unsigned InstID, - ValueEnumerator &VE, BitstreamWriter &Stream, - SmallVectorImpl<unsigned> &Vals) { +void ModuleBitcodeWriter::writeInstruction(const Instruction &I, + unsigned InstID, + SmallVectorImpl<unsigned> &Vals) { unsigned Code = 0; unsigned AbbrevToUse = 0; VE.setInstructionID(&I); @@ -1815,18 +2343,18 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, default: if (Instruction::isCast(I.getOpcode())) { Code = bitc::FUNC_CODE_INST_CAST; - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) AbbrevToUse = FUNCTION_INST_CAST_ABBREV; Vals.push_back(VE.getTypeID(I.getType())); - Vals.push_back(GetEncodedCastOpcode(I.getOpcode())); + Vals.push_back(getEncodedCastOpcode(I.getOpcode())); } else { assert(isa<BinaryOperator>(I) && "Unknown instruction!"); Code = bitc::FUNC_CODE_INST_BINOP; - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) AbbrevToUse = FUNCTION_INST_BINOP_ABBREV; - pushValue(I.getOperand(1), InstID, Vals, VE); - Vals.push_back(GetEncodedBinaryOpcode(I.getOpcode())); - uint64_t Flags = GetOptimizationFlags(&I); + pushValue(I.getOperand(1), InstID, Vals); + Vals.push_back(getEncodedBinaryOpcode(I.getOpcode())); + uint64_t Flags = getOptimizationFlags(&I); if (Flags != 0) { if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV) AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV; @@ -1842,55 +2370,55 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(GEPInst.isInBounds()); Vals.push_back(VE.getTypeID(GEPInst.getSourceElementType())); for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) - PushValueAndType(I.getOperand(i), InstID, Vals, VE); + pushValueAndType(I.getOperand(i), InstID, Vals); break; } case Instruction::ExtractValue: { Code = bitc::FUNC_CODE_INST_EXTRACTVAL; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); const ExtractValueInst *EVI = cast<ExtractValueInst>(&I); Vals.append(EVI->idx_begin(), EVI->idx_end()); break; } case Instruction::InsertValue: { Code = bitc::FUNC_CODE_INST_INSERTVAL; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - PushValueAndType(I.getOperand(1), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValueAndType(I.getOperand(1), InstID, Vals); const InsertValueInst *IVI = cast<InsertValueInst>(&I); Vals.append(IVI->idx_begin(), IVI->idx_end()); break; } case Instruction::Select: Code = bitc::FUNC_CODE_INST_VSELECT; - PushValueAndType(I.getOperand(1), InstID, Vals, VE); - pushValue(I.getOperand(2), InstID, Vals, VE); - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(1), InstID, Vals); + pushValue(I.getOperand(2), InstID, Vals); + pushValueAndType(I.getOperand(0), InstID, Vals); break; case Instruction::ExtractElement: Code = bitc::FUNC_CODE_INST_EXTRACTELT; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - PushValueAndType(I.getOperand(1), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValueAndType(I.getOperand(1), InstID, Vals); break; case Instruction::InsertElement: Code = bitc::FUNC_CODE_INST_INSERTELT; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - pushValue(I.getOperand(1), InstID, Vals, VE); - PushValueAndType(I.getOperand(2), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValue(I.getOperand(1), InstID, Vals); + pushValueAndType(I.getOperand(2), InstID, Vals); break; case Instruction::ShuffleVector: Code = bitc::FUNC_CODE_INST_SHUFFLEVEC; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - pushValue(I.getOperand(1), InstID, Vals, VE); - pushValue(I.getOperand(2), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValue(I.getOperand(1), InstID, Vals); + pushValue(I.getOperand(2), InstID, Vals); break; case Instruction::ICmp: case Instruction::FCmp: { // compare returning Int1Ty or vector of Int1Ty Code = bitc::FUNC_CODE_INST_CMP2; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); - pushValue(I.getOperand(1), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); + pushValue(I.getOperand(1), InstID, Vals); Vals.push_back(cast<CmpInst>(I).getPredicate()); - uint64_t Flags = GetOptimizationFlags(&I); + uint64_t Flags = getOptimizationFlags(&I); if (Flags != 0) Vals.push_back(Flags); break; @@ -1903,11 +2431,11 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, if (NumOperands == 0) AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV; else if (NumOperands == 1) { - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV; } else { for (unsigned i = 0, e = NumOperands; i != e; ++i) - PushValueAndType(I.getOperand(i), InstID, Vals, VE); + pushValueAndType(I.getOperand(i), InstID, Vals); } } break; @@ -1918,7 +2446,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(VE.getValueID(II.getSuccessor(0))); if (II.isConditional()) { Vals.push_back(VE.getValueID(II.getSuccessor(1))); - pushValue(II.getCondition(), InstID, Vals, VE); + pushValue(II.getCondition(), InstID, Vals); } } break; @@ -1927,7 +2455,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Code = bitc::FUNC_CODE_INST_SWITCH; const SwitchInst &SI = cast<SwitchInst>(I); Vals.push_back(VE.getTypeID(SI.getCondition()->getType())); - pushValue(SI.getCondition(), InstID, Vals, VE); + pushValue(SI.getCondition(), InstID, Vals); Vals.push_back(VE.getValueID(SI.getDefaultDest())); for (SwitchInst::ConstCaseIt Case : SI.cases()) { Vals.push_back(VE.getValueID(Case.getCaseValue())); @@ -1939,7 +2467,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Code = bitc::FUNC_CODE_INST_INDIRECTBR; Vals.push_back(VE.getTypeID(I.getOperand(0)->getType())); // Encode the address operand as relative, but not the basic blocks. - pushValue(I.getOperand(0), InstID, Vals, VE); + pushValue(I.getOperand(0), InstID, Vals); for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) Vals.push_back(VE.getValueID(I.getOperand(i))); break; @@ -1950,7 +2478,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, FunctionType *FTy = II->getFunctionType(); if (II->hasOperandBundles()) - WriteOperandBundles(Stream, II, InstID, VE); + writeOperandBundles(II, InstID); Code = bitc::FUNC_CODE_INST_INVOKE; @@ -1959,28 +2487,28 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(VE.getValueID(II->getNormalDest())); Vals.push_back(VE.getValueID(II->getUnwindDest())); Vals.push_back(VE.getTypeID(FTy)); - PushValueAndType(Callee, InstID, Vals, VE); + pushValueAndType(Callee, InstID, Vals); // Emit value #'s for the fixed parameters. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) - pushValue(I.getOperand(i), InstID, Vals, VE); // fixed param. + pushValue(I.getOperand(i), InstID, Vals); // fixed param. // Emit type/value pairs for varargs params. if (FTy->isVarArg()) { for (unsigned i = FTy->getNumParams(), e = I.getNumOperands()-3; i != e; ++i) - PushValueAndType(I.getOperand(i), InstID, Vals, VE); // vararg + pushValueAndType(I.getOperand(i), InstID, Vals); // vararg } break; } case Instruction::Resume: Code = bitc::FUNC_CODE_INST_RESUME; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); break; case Instruction::CleanupRet: { Code = bitc::FUNC_CODE_INST_CLEANUPRET; const auto &CRI = cast<CleanupReturnInst>(I); - pushValue(CRI.getCleanupPad(), InstID, Vals, VE); + pushValue(CRI.getCleanupPad(), InstID, Vals); if (CRI.hasUnwindDest()) Vals.push_back(VE.getValueID(CRI.getUnwindDest())); break; @@ -1988,7 +2516,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, case Instruction::CatchRet: { Code = bitc::FUNC_CODE_INST_CATCHRET; const auto &CRI = cast<CatchReturnInst>(I); - pushValue(CRI.getCatchPad(), InstID, Vals, VE); + pushValue(CRI.getCatchPad(), InstID, Vals); Vals.push_back(VE.getValueID(CRI.getSuccessor())); break; } @@ -1997,19 +2525,19 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, const auto &FuncletPad = cast<FuncletPadInst>(I); Code = isa<CatchPadInst>(FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD : bitc::FUNC_CODE_INST_CLEANUPPAD; - pushValue(FuncletPad.getParentPad(), InstID, Vals, VE); + pushValue(FuncletPad.getParentPad(), InstID, Vals); unsigned NumArgOperands = FuncletPad.getNumArgOperands(); Vals.push_back(NumArgOperands); for (unsigned Op = 0; Op != NumArgOperands; ++Op) - PushValueAndType(FuncletPad.getArgOperand(Op), InstID, Vals, VE); + pushValueAndType(FuncletPad.getArgOperand(Op), InstID, Vals); break; } case Instruction::CatchSwitch: { Code = bitc::FUNC_CODE_INST_CATCHSWITCH; const auto &CatchSwitch = cast<CatchSwitchInst>(I); - pushValue(CatchSwitch.getParentPad(), InstID, Vals, VE); + pushValue(CatchSwitch.getParentPad(), InstID, Vals); unsigned NumHandlers = CatchSwitch.getNumHandlers(); Vals.push_back(NumHandlers); @@ -2034,7 +2562,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, SmallVector<uint64_t, 128> Vals64; Vals64.push_back(VE.getTypeID(PN.getType())); for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) { - pushValueSigned(PN.getIncomingValue(i), InstID, Vals64, VE); + pushValueSigned(PN.getIncomingValue(i), InstID, Vals64); Vals64.push_back(VE.getValueID(PN.getIncomingBlock(i))); } // Emit a Vals64 vector and exit. @@ -2054,7 +2582,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(LandingPadInst::Catch); else Vals.push_back(LandingPadInst::Filter); - PushValueAndType(LP.getClause(I), InstID, Vals, VE); + pushValueAndType(LP.getClause(I), InstID, Vals); } break; } @@ -2071,8 +2599,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, assert(AlignRecord < 1 << 5 && "alignment greater than 1 << 64"); AlignRecord |= AI.isUsedWithInAlloca() << 5; AlignRecord |= 1 << 6; - // Reserve bit 7 for SwiftError flag. - // AlignRecord |= AI.isSwiftError() << 7; + AlignRecord |= AI.isSwiftError() << 7; Vals.push_back(AlignRecord); break; } @@ -2080,18 +2607,18 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, case Instruction::Load: if (cast<LoadInst>(I).isAtomic()) { Code = bitc::FUNC_CODE_INST_LOADATOMIC; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); + pushValueAndType(I.getOperand(0), InstID, Vals); } else { Code = bitc::FUNC_CODE_INST_LOAD; - if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE)) // ptr + if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr AbbrevToUse = FUNCTION_INST_LOAD_ABBREV; } Vals.push_back(VE.getTypeID(I.getType())); Vals.push_back(Log2_32(cast<LoadInst>(I).getAlignment())+1); Vals.push_back(cast<LoadInst>(I).isVolatile()); if (cast<LoadInst>(I).isAtomic()) { - Vals.push_back(GetEncodedOrdering(cast<LoadInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope(cast<LoadInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering())); + Vals.push_back(getEncodedSynchScope(cast<LoadInst>(I).getSynchScope())); } break; case Instruction::Store: @@ -2099,57 +2626,57 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Code = bitc::FUNC_CODE_INST_STOREATOMIC; else Code = bitc::FUNC_CODE_INST_STORE; - PushValueAndType(I.getOperand(1), InstID, Vals, VE); // ptrty + ptr - PushValueAndType(I.getOperand(0), InstID, Vals, VE); // valty + val + pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr + pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val Vals.push_back(Log2_32(cast<StoreInst>(I).getAlignment())+1); Vals.push_back(cast<StoreInst>(I).isVolatile()); if (cast<StoreInst>(I).isAtomic()) { - Vals.push_back(GetEncodedOrdering(cast<StoreInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope(cast<StoreInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<StoreInst>(I).getOrdering())); + Vals.push_back(getEncodedSynchScope(cast<StoreInst>(I).getSynchScope())); } break; case Instruction::AtomicCmpXchg: Code = bitc::FUNC_CODE_INST_CMPXCHG; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); // ptrty + ptr - PushValueAndType(I.getOperand(1), InstID, Vals, VE); // cmp. - pushValue(I.getOperand(2), InstID, Vals, VE); // newval. + pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr + pushValueAndType(I.getOperand(1), InstID, Vals); // cmp. + pushValue(I.getOperand(2), InstID, Vals); // newval. Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile()); - Vals.push_back(GetEncodedOrdering( - cast<AtomicCmpXchgInst>(I).getSuccessOrdering())); - Vals.push_back(GetEncodedSynchScope( - cast<AtomicCmpXchgInst>(I).getSynchScope())); - Vals.push_back(GetEncodedOrdering( - cast<AtomicCmpXchgInst>(I).getFailureOrdering())); + Vals.push_back( + getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getSuccessOrdering())); + Vals.push_back( + getEncodedSynchScope(cast<AtomicCmpXchgInst>(I).getSynchScope())); + Vals.push_back( + getEncodedOrdering(cast<AtomicCmpXchgInst>(I).getFailureOrdering())); Vals.push_back(cast<AtomicCmpXchgInst>(I).isWeak()); break; case Instruction::AtomicRMW: Code = bitc::FUNC_CODE_INST_ATOMICRMW; - PushValueAndType(I.getOperand(0), InstID, Vals, VE); // ptrty + ptr - pushValue(I.getOperand(1), InstID, Vals, VE); // val. - Vals.push_back(GetEncodedRMWOperation( - cast<AtomicRMWInst>(I).getOperation())); + pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr + pushValue(I.getOperand(1), InstID, Vals); // val. + Vals.push_back( + getEncodedRMWOperation(cast<AtomicRMWInst>(I).getOperation())); Vals.push_back(cast<AtomicRMWInst>(I).isVolatile()); - Vals.push_back(GetEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope( - cast<AtomicRMWInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<AtomicRMWInst>(I).getOrdering())); + Vals.push_back( + getEncodedSynchScope(cast<AtomicRMWInst>(I).getSynchScope())); break; case Instruction::Fence: Code = bitc::FUNC_CODE_INST_FENCE; - Vals.push_back(GetEncodedOrdering(cast<FenceInst>(I).getOrdering())); - Vals.push_back(GetEncodedSynchScope(cast<FenceInst>(I).getSynchScope())); + Vals.push_back(getEncodedOrdering(cast<FenceInst>(I).getOrdering())); + Vals.push_back(getEncodedSynchScope(cast<FenceInst>(I).getSynchScope())); break; case Instruction::Call: { const CallInst &CI = cast<CallInst>(I); FunctionType *FTy = CI.getFunctionType(); if (CI.hasOperandBundles()) - WriteOperandBundles(Stream, &CI, InstID, VE); + writeOperandBundles(&CI, InstID); Code = bitc::FUNC_CODE_INST_CALL; Vals.push_back(VE.getAttributeID(CI.getAttributes())); - unsigned Flags = GetOptimizationFlags(&I); + unsigned Flags = getOptimizationFlags(&I); Vals.push_back(CI.getCallingConv() << bitc::CALL_CCONV | unsigned(CI.isTailCall()) << bitc::CALL_TAIL | unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL | @@ -2160,7 +2687,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.push_back(Flags); Vals.push_back(VE.getTypeID(FTy)); - PushValueAndType(CI.getCalledValue(), InstID, Vals, VE); // Callee + pushValueAndType(CI.getCalledValue(), InstID, Vals); // Callee // Emit value #'s for the fixed parameters. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) { @@ -2168,21 +2695,21 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, if (FTy->getParamType(i)->isLabelTy()) Vals.push_back(VE.getValueID(CI.getArgOperand(i))); else - pushValue(CI.getArgOperand(i), InstID, Vals, VE); // fixed param. + pushValue(CI.getArgOperand(i), InstID, Vals); // fixed param. } // Emit type/value pairs for varargs params. if (FTy->isVarArg()) { for (unsigned i = FTy->getNumParams(), e = CI.getNumArgOperands(); i != e; ++i) - PushValueAndType(CI.getArgOperand(i), InstID, Vals, VE); // varargs + pushValueAndType(CI.getArgOperand(i), InstID, Vals); // varargs } break; } case Instruction::VAArg: Code = bitc::FUNC_CODE_INST_VAARG; Vals.push_back(VE.getTypeID(I.getOperand(0)->getType())); // valistty - pushValue(I.getOperand(0), InstID, Vals, VE); // valist. + pushValue(I.getOperand(0), InstID, Vals); // valist. Vals.push_back(VE.getTypeID(I.getType())); // restype. break; } @@ -2191,49 +2718,27 @@ static void WriteInstruction(const Instruction &I, unsigned InstID, Vals.clear(); } -enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 }; - -/// Determine the encoding to use for the given string name and length. -static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { - bool isChar6 = true; - for (const char *C = Str, *E = C + StrLen; C != E; ++C) { - if (isChar6) - isChar6 = BitCodeAbbrevOp::isChar6(*C); - if ((unsigned char)*C & 128) - // don't bother scanning the rest. - return SE_Fixed8; - } - if (isChar6) - return SE_Char6; - else - return SE_Fixed7; -} - -/// Emit names for globals/functions etc. The VSTOffsetPlaceholder, -/// BitcodeStartBit and FunctionIndex are only passed for the module-level -/// VST, where we are including a function bitcode index and need to -/// backpatch the VST forward declaration record. -static void WriteValueSymbolTable( - const ValueSymbolTable &VST, const ValueEnumerator &VE, - BitstreamWriter &Stream, uint64_t VSTOffsetPlaceholder = 0, - uint64_t BitcodeStartBit = 0, - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> *FunctionIndex = - nullptr) { +/// Emit names for globals/functions etc. \p IsModuleLevel is true when +/// we are writing the module-level VST, where we are including a function +/// bitcode index and need to backpatch the VST forward declaration record. +void ModuleBitcodeWriter::writeValueSymbolTable( + const ValueSymbolTable &VST, bool IsModuleLevel, + DenseMap<const Function *, uint64_t> *FunctionToBitcodeIndex) { if (VST.empty()) { - // WriteValueSymbolTableForwardDecl should have returned early as + // writeValueSymbolTableForwardDecl should have returned early as // well. Ensure this handling remains in sync by asserting that // the placeholder offset is not set. - assert(VSTOffsetPlaceholder == 0); + assert(!IsModuleLevel || !hasVSTOffsetPlaceholder()); return; } - if (VSTOffsetPlaceholder > 0) { + if (IsModuleLevel && hasVSTOffsetPlaceholder()) { // Get the offset of the VST we are writing, and backpatch it into // the VST forward declaration record. uint64_t VSTOffset = Stream.GetCurrentBitNo(); // The BitcodeStartBit was the stream offset of the actual bitcode // (e.g. excluding any initial darwin header). - VSTOffset -= BitcodeStartBit; + VSTOffset -= bitcodeStartBit(); assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); } @@ -2245,8 +2750,9 @@ static void WriteValueSymbolTable( unsigned FnEntry8BitAbbrev; unsigned FnEntry7BitAbbrev; unsigned FnEntry6BitAbbrev; - if (VSTOffsetPlaceholder > 0) { - // 8-bit fixed-width VST_FNENTRY function strings. + unsigned GUIDEntryAbbrev; + if (IsModuleLevel && hasVSTOffsetPlaceholder()) { + // 8-bit fixed-width VST_CODE_FNENTRY function strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id @@ -2255,7 +2761,7 @@ static void WriteValueSymbolTable( Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); FnEntry8BitAbbrev = Stream.EmitAbbrev(Abbv); - // 7-bit fixed width VST_FNENTRY function strings. + // 7-bit fixed width VST_CODE_FNENTRY function strings. Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id @@ -2264,7 +2770,7 @@ static void WriteValueSymbolTable( Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); FnEntry7BitAbbrev = Stream.EmitAbbrev(Abbv); - // 6-bit char6 VST_FNENTRY function strings. + // 6-bit char6 VST_CODE_FNENTRY function strings. Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id @@ -2272,11 +2778,19 @@ static void WriteValueSymbolTable( Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); FnEntry6BitAbbrev = Stream.EmitAbbrev(Abbv); + + // FIXME: Change the name of this record as it is now used by + // the per-module index as well. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid + GUIDEntryAbbrev = Stream.EmitAbbrev(Abbv); } // FIXME: Set up the abbrev, we know how many values there are! // FIXME: We know if the type names can use 7-bit ascii. - SmallVector<unsigned, 64> NameVals; + SmallVector<uint64_t, 64> NameVals; for (const ValueName &Name : VST) { // Figure out the encoding to use for the name. @@ -2295,9 +2809,9 @@ static void WriteValueSymbolTable( F = dyn_cast<Function>(GA->getBaseObject()); } - // VST_ENTRY: [valueid, namechar x N] - // VST_FNENTRY: [valueid, funcoffset, namechar x N] - // VST_BBENTRY: [bbid, namechar x N] + // VST_CODE_ENTRY: [valueid, namechar x N] + // VST_CODE_FNENTRY: [valueid, funcoffset, namechar x N] + // VST_CODE_BBENTRY: [bbid, namechar x N] unsigned Code; if (isa<BasicBlock>(Name.getValue())) { Code = bitc::VST_CODE_BBENTRY; @@ -2307,14 +2821,12 @@ static void WriteValueSymbolTable( // Must be the module-level VST, where we pass in the Index and // have a VSTOffsetPlaceholder. The function-level VST should not // contain any Function symbols. - assert(FunctionIndex); - assert(VSTOffsetPlaceholder > 0); + assert(FunctionToBitcodeIndex); + assert(hasVSTOffsetPlaceholder()); // Save the word offset of the function (from the start of the // actual bitcode written to the stream). - assert(FunctionIndex->count(F) == 1); - uint64_t BitcodeIndex = - (*FunctionIndex)[F]->bitcodeIndex() - BitcodeStartBit; + uint64_t BitcodeIndex = (*FunctionToBitcodeIndex)[F] - bitcodeStartBit(); assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); NameVals.push_back(BitcodeIndex / 32); @@ -2339,71 +2851,51 @@ static void WriteValueSymbolTable( Stream.EmitRecord(Code, NameVals, AbbrevToUse); NameVals.clear(); } + // Emit any GUID valueIDs created for indirect call edges into the + // module-level VST. + if (IsModuleLevel && hasVSTOffsetPlaceholder()) + for (const auto &GI : valueIds()) { + NameVals.push_back(GI.second); + NameVals.push_back(GI.first); + Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, + GUIDEntryAbbrev); + NameVals.clear(); + } Stream.ExitBlock(); } /// Emit function names and summary offsets for the combined index /// used by ThinLTO. -static void WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index, - BitstreamWriter &Stream) { +void IndexBitcodeWriter::writeCombinedValueSymbolTable() { + assert(hasVSTOffsetPlaceholder() && "Expected non-zero VSTOffsetPlaceholder"); + // Get the offset of the VST we are writing, and backpatch it into + // the VST forward declaration record. + uint64_t VSTOffset = Stream.GetCurrentBitNo(); + assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned"); + Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32); + Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); - // 8-bit fixed-width VST_COMBINED_FNENTRY function strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - unsigned FnEntry8BitAbbrev = Stream.EmitAbbrev(Abbv); - - // 7-bit fixed width VST_COMBINED_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - unsigned FnEntry7BitAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid + unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv); - // 6-bit char6 VST_COMBINED_FNENTRY function strings. - Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - unsigned FnEntry6BitAbbrev = Stream.EmitAbbrev(Abbv); - - // FIXME: We know if the type names can use 7-bit ascii. - SmallVector<unsigned, 64> NameVals; + SmallVector<uint64_t, 64> NameVals; + for (const auto &GVI : valueIds()) { + // VST_CODE_COMBINED_ENTRY: [valueid, refguid] + NameVals.push_back(GVI.second); + NameVals.push_back(GVI.first); - for (const auto &FII : Index) { - for (const auto &FI : FII.getValue()) { - NameVals.push_back(FI->bitcodeIndex()); - - StringRef FuncName = FII.first(); - - // Figure out the encoding to use for the name. - StringEncoding Bits = getStringEncoding(FuncName.data(), FuncName.size()); - - // VST_COMBINED_FNENTRY: [funcsumoffset, namechar x N] - unsigned AbbrevToUse = FnEntry8BitAbbrev; - if (Bits == SE_Char6) - AbbrevToUse = FnEntry6BitAbbrev; - else if (Bits == SE_Fixed7) - AbbrevToUse = FnEntry7BitAbbrev; - - for (const auto P : FuncName) - NameVals.push_back((unsigned char)P); - - // Emit the finished record. - Stream.EmitRecord(bitc::VST_CODE_COMBINED_FNENTRY, NameVals, AbbrevToUse); - NameVals.clear(); - } + // Emit the finished record. + Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, EntryAbbrev); + NameVals.clear(); } Stream.ExitBlock(); } -static void WriteUseList(ValueEnumerator &VE, UseListOrder &&Order, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) { assert(Order.Shuffle.size() >= 2 && "Shuffle too small"); unsigned Code; if (isa<BasicBlock>(Order.V)) @@ -2416,8 +2908,7 @@ static void WriteUseList(ValueEnumerator &VE, UseListOrder &&Order, Stream.EmitRecord(Code, Record); } -static void WriteUseListBlock(const Function *F, ValueEnumerator &VE, - BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeUseListBlock(const Function *F) { assert(VE.shouldPreserveUseListOrder() && "Expected to be preserving use-list order"); @@ -2430,39 +2921,19 @@ static void WriteUseListBlock(const Function *F, ValueEnumerator &VE, Stream.EnterSubblock(bitc::USELIST_BLOCK_ID, 3); while (hasMore()) { - WriteUseList(VE, std::move(VE.UseListOrders.back()), Stream); + writeUseList(std::move(VE.UseListOrders.back())); VE.UseListOrders.pop_back(); } Stream.ExitBlock(); } -/// \brief Save information for the given function into the function index. -/// -/// At a minimum this saves the bitcode index of the function record that -/// was just written. However, if we are emitting function summary information, -/// for example for ThinLTO, then a \a FunctionSummary object is created -/// to hold the provided summary information. -static void SaveFunctionInfo( - const Function &F, - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex, - unsigned NumInsts, uint64_t BitcodeIndex, bool EmitFunctionSummary) { - std::unique_ptr<FunctionSummary> FuncSummary; - if (EmitFunctionSummary) { - FuncSummary = llvm::make_unique<FunctionSummary>(NumInsts); - FuncSummary->setLocalFunction(F.hasLocalLinkage()); - } - FunctionIndex[&F] = - llvm::make_unique<FunctionInfo>(BitcodeIndex, std::move(FuncSummary)); -} - /// Emit a function body to the module stream. -static void WriteFunction( - const Function &F, ValueEnumerator &VE, BitstreamWriter &Stream, - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex, - bool EmitFunctionSummary) { +void ModuleBitcodeWriter::writeFunction( + const Function &F, + DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) { // Save the bitcode index of the start of this function block for recording // in the VST. - uint64_t BitcodeIndex = Stream.GetCurrentBitNo(); + FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo(); Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4); VE.incorporateFunction(F); @@ -2478,10 +2949,10 @@ static void WriteFunction( // If there are function-local constants, emit them now. unsigned CstStart, CstEnd; VE.getFunctionConstantRange(CstStart, CstEnd); - WriteConstants(CstStart, CstEnd, VE, Stream, false); + writeConstants(CstStart, CstEnd, false); // If there is function-local metadata, emit it now. - WriteFunctionLocalMetadata(F, VE, Stream); + writeFunctionMetadata(F); // Keep a running idea of what the instruction ID is. unsigned InstID = CstEnd; @@ -2489,16 +2960,11 @@ static void WriteFunction( bool NeedsMetadataAttachment = F.hasMetadata(); DILocation *LastDL = nullptr; - unsigned NumInsts = 0; - // Finally, emit all the instructions, in order. for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I) { - WriteInstruction(*I, InstID, VE, Stream, Vals); - - if (!isa<DbgInfoIntrinsic>(I)) - ++NumInsts; + writeInstruction(*I, InstID, Vals); if (!I->getType()->isVoidTy()) ++InstID; @@ -2528,65 +2994,62 @@ static void WriteFunction( } // Emit names for all the instructions etc. - WriteValueSymbolTable(F.getValueSymbolTable(), VE, Stream); + writeValueSymbolTable(F.getValueSymbolTable()); if (NeedsMetadataAttachment) - WriteMetadataAttachment(F, VE, Stream); + writeFunctionMetadataAttachment(F); if (VE.shouldPreserveUseListOrder()) - WriteUseListBlock(&F, VE, Stream); + writeUseListBlock(&F); VE.purgeFunction(); Stream.ExitBlock(); - - SaveFunctionInfo(F, FunctionIndex, NumInsts, BitcodeIndex, - EmitFunctionSummary); } // Emit blockinfo, which defines the standard abbreviations etc. -static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeBlockInfo() { // We only want to emit block info records for blocks that have multiple // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK. // Other blocks can define their abbrevs inline. Stream.EnterBlockInfoBlock(2); - { // 8-bit fixed-width VST_ENTRY/VST_BBENTRY strings. + { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_ENTRY_8_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_ENTRY_8_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } - { // 7-bit fixed width VST_ENTRY strings. + { // 7-bit fixed width VST_CODE_ENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_ENTRY_7_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_ENTRY_7_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } - { // 6-bit char6 VST_ENTRY strings. + { // 6-bit char6 VST_CODE_ENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_ENTRY_6_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_ENTRY_6_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } - { // 6-bit char6 VST_BBENTRY strings. + { // 6-bit char6 VST_CODE_BBENTRY strings. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); - if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, - Abbv) != VST_BBENTRY_6_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) != + VST_BBENTRY_6_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2597,8 +3060,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, VE.computeBitsRequiredForTypeIndicies())); - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_SETTYPE_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_SETTYPE_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2606,8 +3069,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_INTEGER)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_INTEGER_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_INTEGER_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2619,15 +3082,15 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { VE.computeBitsRequiredForTypeIndicies())); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_CE_CAST_Abbrev) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_CE_CAST_Abbrev) llvm_unreachable("Unexpected abbrev ordering!"); } { // NULL abbrev for CONSTANTS_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_NULL)); - if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, - Abbv) != CONSTANTS_NULL_Abbrev) + if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) != + CONSTANTS_NULL_Abbrev) llvm_unreachable("Unexpected abbrev ordering!"); } @@ -2641,8 +3104,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { VE.computeBitsRequiredForTypeIndicies())); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_LOAD_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_LOAD_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP abbrev for FUNCTION_BLOCK. @@ -2651,8 +3114,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_BINOP_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_BINOP_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK. @@ -2662,8 +3125,8 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7)); // flags - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_BINOP_FLAGS_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_BINOP_FLAGS_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_CAST abbrev for FUNCTION_BLOCK. @@ -2673,31 +3136,31 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty VE.computeBitsRequiredForTypeIndicies())); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_CAST_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_CAST_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_RET abbrev for FUNCTION_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_RET_VOID_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_RET_VOID_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_RET abbrev for FUNCTION_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_RET_VAL_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_RET_VAL_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE)); - if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, - Abbv) != FUNCTION_INST_UNREACHABLE_ABBREV) + if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) != + FUNCTION_INST_UNREACHABLE_ABBREV) llvm_unreachable("Unexpected abbrev ordering!"); } { @@ -2718,8 +3181,7 @@ static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) { /// Write the module path strings, currently only used when generating /// a combined index file. -static void WriteModStrings(const FunctionInfoIndex &I, - BitstreamWriter &Stream) { +void IndexBitcodeWriter::writeModStrings() { Stream.EnterSubblock(bitc::MODULE_STRTAB_BLOCK_ID, 3); // TODO: See which abbrev sizes we actually need to emit @@ -2748,8 +3210,20 @@ static void WriteModStrings(const FunctionInfoIndex &I, Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv); - SmallVector<unsigned, 64> NameVals; - for (const StringMapEntry<uint64_t> &MPSE : I.modPathStringEntries()) { + // Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::MST_CODE_HASH)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + unsigned AbbrevHash = Stream.EmitAbbrev(Abbv); + + SmallVector<unsigned, 64> Vals; + for (const auto &MPSE : Index.modulePaths()) { + if (!doIncludeModule(MPSE.getKey())) + continue; StringEncoding Bits = getStringEncoding(MPSE.getKey().data(), MPSE.getKey().size()); unsigned AbbrevToUse = Abbrev8Bit; @@ -2758,116 +3232,355 @@ static void WriteModStrings(const FunctionInfoIndex &I, else if (Bits == SE_Fixed7) AbbrevToUse = Abbrev7Bit; - NameVals.push_back(MPSE.getValue()); + Vals.push_back(MPSE.getValue().first); for (const auto P : MPSE.getKey()) - NameVals.push_back((unsigned char)P); + Vals.push_back((unsigned char)P); // Emit the finished record. - Stream.EmitRecord(bitc::MST_CODE_ENTRY, NameVals, AbbrevToUse); - NameVals.clear(); + Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse); + + Vals.clear(); + // Emit an optional hash for the module now + auto &Hash = MPSE.getValue().second; + bool AllZero = true; // Detect if the hash is empty, and do not generate it + for (auto Val : Hash) { + if (Val) + AllZero = false; + Vals.push_back(Val); + } + if (!AllZero) { + // Emit the hash record. + Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash); + } + + Vals.clear(); } Stream.ExitBlock(); } // Helper to emit a single function summary record. -static void WritePerModuleFunctionSummaryRecord( - SmallVector<unsigned, 64> &NameVals, FunctionSummary *FS, unsigned ValueID, - unsigned FSAbbrev, BitstreamWriter &Stream) { - assert(FS); +void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord( + SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary, + unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, + const Function &F) { NameVals.push_back(ValueID); - NameVals.push_back(FS->isLocalFunction()); + + FunctionSummary *FS = cast<FunctionSummary>(Summary); + NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); + NameVals.push_back(FS->refs().size()); + + unsigned SizeBeforeRefs = NameVals.size(); + for (auto &RI : FS->refs()) + NameVals.push_back(VE.getValueID(RI.getValue())); + // Sort the refs for determinism output, the vector returned by FS->refs() has + // been initialized from a DenseSet. + std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); + + std::vector<FunctionSummary::EdgeTy> Calls = FS->calls(); + std::sort(Calls.begin(), Calls.end(), + [this](const FunctionSummary::EdgeTy &L, + const FunctionSummary::EdgeTy &R) { + return getValueId(L.first) < getValueId(R.first); + }); + bool HasProfileData = F.getEntryCount().hasValue(); + for (auto &ECI : Calls) { + NameVals.push_back(getValueId(ECI.first)); + assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite"); + NameVals.push_back(ECI.second.CallsiteCount); + if (HasProfileData) + NameVals.push_back(ECI.second.ProfileCount); + } + + unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); + unsigned Code = + (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE); // Emit the finished record. - Stream.EmitRecord(bitc::FS_CODE_PERMODULE_ENTRY, NameVals, FSAbbrev); + Stream.EmitRecord(Code, NameVals, FSAbbrev); NameVals.clear(); } -/// Emit the per-module function summary section alongside the rest of +// Collect the global value references in the given variable's initializer, +// and emit them in a summary record. +void ModuleBitcodeWriter::writeModuleLevelReferences( + const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals, + unsigned FSModRefsAbbrev) { + // Only interested in recording variable defs in the summary. + if (V.isDeclaration()) + return; + NameVals.push_back(VE.getValueID(&V)); + NameVals.push_back(getEncodedGVSummaryFlags(V)); + auto *Summary = Index->getGlobalValueSummary(V); + GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary); + + unsigned SizeBeforeRefs = NameVals.size(); + for (auto &RI : VS->refs()) + NameVals.push_back(VE.getValueID(RI.getValue())); + // Sort the refs for determinism output, the vector returned by FS->refs() has + // been initialized from a DenseSet. + std::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); + + Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); + NameVals.clear(); +} + +// Current version for the summary. +// This is bumped whenever we introduce changes in the way some record are +// interpreted, like flags for instance. +static const uint64_t INDEX_VERSION = 1; + +/// Emit the per-module summary section alongside the rest of /// the module's bitcode. -static void WritePerModuleFunctionSummary( - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex, - const Module *M, const ValueEnumerator &VE, BitstreamWriter &Stream) { - Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3); +void ModuleBitcodeWriter::writePerModuleGlobalValueSummary() { + if (Index->begin() == Index->end()) + return; - // Abbrev for FS_CODE_PERMODULE_ENTRY. + Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4); + + Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + + // Abbrev for FS_PERMODULE. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_PERMODULE_ENTRY)); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // islocal + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - unsigned FSAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_PERMODULE_PROFILE. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount, profilecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); - SmallVector<unsigned, 64> NameVals; - for (auto &I : FunctionIndex) { - // Skip anonymous functions. We will emit a function summary for - // any aliases below. - if (!I.first->hasName()) + // Abbrev for FS_ALIAS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + + SmallVector<uint64_t, 64> NameVals; + // Iterate over the list of functions instead of the Index to + // ensure the ordering is stable. + for (const Function &F : M) { + if (F.isDeclaration()) continue; + // Summary emission does not support anonymous functions, they have to + // renamed using the anonymous function renaming pass. + if (!F.hasName()) + report_fatal_error("Unexpected anonymous function when writing summary"); - WritePerModuleFunctionSummaryRecord( - NameVals, I.second->functionSummary(), - VE.getValueID(M->getValueSymbolTable().lookup(I.first->getName())), - FSAbbrev, Stream); + auto *Summary = Index->getGlobalValueSummary(F); + writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), + FSCallsAbbrev, FSCallsProfileAbbrev, F); } - for (const GlobalAlias &A : M->aliases()) { - if (!A.getBaseObject()) - continue; - const Function *F = dyn_cast<Function>(A.getBaseObject()); - if (!F || F->isDeclaration()) - continue; + // Capture references from GlobalVariable initializers, which are outside + // of a function scope. + for (const GlobalVariable &G : M.globals()) + writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev); - assert(FunctionIndex.count(F) == 1); - WritePerModuleFunctionSummaryRecord( - NameVals, FunctionIndex[F]->functionSummary(), - VE.getValueID(M->getValueSymbolTable().lookup(A.getName())), FSAbbrev, - Stream); + for (const GlobalAlias &A : M.aliases()) { + auto *Aliasee = A.getBaseObject(); + if (!Aliasee->hasName()) + // Nameless function don't have an entry in the summary, skip it. + continue; + auto AliasId = VE.getValueID(&A); + auto AliaseeId = VE.getValueID(Aliasee); + NameVals.push_back(AliasId); + NameVals.push_back(getEncodedGVSummaryFlags(A)); + NameVals.push_back(AliaseeId); + Stream.EmitRecord(bitc::FS_ALIAS, NameVals, FSAliasAbbrev); + NameVals.clear(); } Stream.ExitBlock(); } -/// Emit the combined function summary section into the combined index -/// file. -static void WriteCombinedFunctionSummary(const FunctionInfoIndex &I, - BitstreamWriter &Stream) { - Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3); +/// Emit the combined summary section into the combined index file. +void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { + Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3); + Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); - // Abbrev for FS_CODE_COMBINED_ENTRY. + // Abbrev for FS_COMBINED. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_COMBINED_ENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount - unsigned FSAbbrev = Stream.EmitAbbrev(Abbv); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_COMBINED_PROFILE. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid, callsitecount, profilecount) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); + + // Abbrev for FS_COMBINED_ALIAS. + Abbv = new BitCodeAbbrev(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); + + // The aliases are emitted as a post-pass, and will point to the value + // id of the aliasee. Save them in a vector for post-processing. + SmallVector<AliasSummary *, 64> Aliases; - SmallVector<unsigned, 64> NameVals; - for (const auto &FII : I) { - for (auto &FI : FII.getValue()) { - FunctionSummary *FS = FI->functionSummary(); - assert(FS); + // Save the value id for each summary for alias emission. + DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap; - NameVals.push_back(I.getModuleId(FS->modulePath())); - NameVals.push_back(FS->instCount()); + SmallVector<uint64_t, 64> NameVals; - // Record the starting offset of this summary entry for use - // in the VST entry. Add the current code size since the - // reader will invoke readRecord after the abbrev id read. - FI->setBitcodeIndex(Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth()); + // For local linkage, we also emit the original name separately + // immediately after the record. + auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) { + if (!GlobalValue::isLocalLinkage(S.linkage())) + return; + NameVals.push_back(S.getOriginalName()); + Stream.EmitRecord(bitc::FS_COMBINED_ORIGINAL_NAME, NameVals); + NameVals.clear(); + }; + + for (const auto &I : *this) { + GlobalValueSummary *S = I.second; + assert(S); + + assert(hasValueId(I.first)); + unsigned ValueId = getValueId(I.first); + SummaryToValueIdMap[S] = ValueId; + + if (auto *AS = dyn_cast<AliasSummary>(S)) { + // Will process aliases as a post-pass because the reader wants all + // global to be loaded first. + Aliases.push_back(AS); + continue; + } + + if (auto *VS = dyn_cast<GlobalVarSummary>(S)) { + NameVals.push_back(ValueId); + NameVals.push_back(Index.getModuleId(VS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); + for (auto &RI : VS->refs()) { + NameVals.push_back(getValueId(RI.getGUID())); + } // Emit the finished record. - Stream.EmitRecord(bitc::FS_CODE_COMBINED_ENTRY, NameVals, FSAbbrev); + Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); NameVals.clear(); + MaybeEmitOriginalName(*S); + continue; + } + + auto *FS = cast<FunctionSummary>(S); + NameVals.push_back(ValueId); + NameVals.push_back(Index.getModuleId(FS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); + NameVals.push_back(FS->instCount()); + NameVals.push_back(FS->refs().size()); + + for (auto &RI : FS->refs()) { + NameVals.push_back(getValueId(RI.getGUID())); } + + bool HasProfileData = false; + for (auto &EI : FS->calls()) { + HasProfileData |= EI.second.ProfileCount != 0; + if (HasProfileData) + break; + } + + for (auto &EI : FS->calls()) { + // If this GUID doesn't have a value id, it doesn't have a function + // summary and we don't need to record any calls to it. + if (!hasValueId(EI.first.getGUID())) + continue; + NameVals.push_back(getValueId(EI.first.getGUID())); + assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite"); + NameVals.push_back(EI.second.CallsiteCount); + if (HasProfileData) + NameVals.push_back(EI.second.ProfileCount); + } + + unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); + unsigned Code = + (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED); + + // Emit the finished record. + Stream.EmitRecord(Code, NameVals, FSAbbrev); + NameVals.clear(); + MaybeEmitOriginalName(*S); + } + + for (auto *AS : Aliases) { + auto AliasValueId = SummaryToValueIdMap[AS]; + assert(AliasValueId); + NameVals.push_back(AliasValueId); + NameVals.push_back(Index.getModuleId(AS->modulePath())); + NameVals.push_back(getEncodedGVSummaryFlags(AS->flags())); + auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()]; + assert(AliaseeValueId); + NameVals.push_back(AliaseeValueId); + + // Emit the finished record. + Stream.EmitRecord(bitc::FS_COMBINED_ALIAS, NameVals, FSAliasAbbrev); + NameVals.clear(); + MaybeEmitOriginalName(*AS); } Stream.ExitBlock(); } -// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the -// current llvm version, and a record for the epoch number. -static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) { +void ModuleBitcodeWriter::writeIdentificationBlock() { Stream.EnterSubblock(bitc::IDENTIFICATION_BLOCK_ID, 5); // Write the "user readable" string identifying the bitcode producer @@ -2876,8 +3589,8 @@ static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Char6)); auto StringAbbrev = Stream.EmitAbbrev(Abbv); - WriteStringRecord(bitc::IDENTIFICATION_CODE_STRING, - "LLVM" LLVM_VERSION_STRING, StringAbbrev, Stream); + writeStringRecord(bitc::IDENTIFICATION_CODE_STRING, + "LLVM" LLVM_VERSION_STRING, StringAbbrev); // Write the epoch version Abbv = new BitCodeAbbrev(); @@ -2889,71 +3602,114 @@ static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) { Stream.ExitBlock(); } -/// WriteModule - Emit the specified module to the bitstream. -static void WriteModule(const Module *M, BitstreamWriter &Stream, - bool ShouldPreserveUseListOrder, - uint64_t BitcodeStartBit, bool EmitFunctionSummary) { +void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) { + // Emit the module's hash. + // MODULE_CODE_HASH: [5*i32] + SHA1 Hasher; + Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos], + Buffer.size() - BlockStartPos)); + auto Hash = Hasher.result(); + SmallVector<uint64_t, 20> Vals; + auto LShift = [&](unsigned char Val, unsigned Amount) + -> uint64_t { return ((uint64_t)Val) << Amount; }; + for (int Pos = 0; Pos < 20; Pos += 4) { + uint32_t SubHash = LShift(Hash[Pos + 0], 24); + SubHash |= LShift(Hash[Pos + 1], 16) | LShift(Hash[Pos + 2], 8) | + (unsigned)(unsigned char)Hash[Pos + 3]; + Vals.push_back(SubHash); + } + + // Emit the finished record. + Stream.EmitRecord(bitc::MODULE_CODE_HASH, Vals); +} + +void BitcodeWriter::write() { + // Emit the file header first. + writeBitcodeHeader(); + + writeBlocks(); +} + +void ModuleBitcodeWriter::writeBlocks() { + writeIdentificationBlock(); + writeModule(); +} + +void IndexBitcodeWriter::writeBlocks() { + // Index contains only a single outer (module) block. + writeIndex(); +} + +void ModuleBitcodeWriter::writeModule() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); + size_t BlockStartPos = Buffer.size(); SmallVector<unsigned, 1> Vals; unsigned CurVersion = 1; Vals.push_back(CurVersion); Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); - // Analyze the module, enumerating globals, functions, etc. - ValueEnumerator VE(*M, ShouldPreserveUseListOrder); - // Emit blockinfo, which defines the standard abbreviations etc. - WriteBlockInfo(VE, Stream); + writeBlockInfo(); // Emit information about attribute groups. - WriteAttributeGroupTable(VE, Stream); + writeAttributeGroupTable(); // Emit information about parameter attributes. - WriteAttributeTable(VE, Stream); + writeAttributeTable(); // Emit information describing all of the types in the module. - WriteTypeTable(VE, Stream); + writeTypeTable(); - writeComdats(VE, Stream); + writeComdats(); // Emit top-level description of module, including target triple, inline asm, // descriptors for global variables, and function prototype info. - uint64_t VSTOffsetPlaceholder = WriteModuleInfo(M, VE, Stream); + writeModuleInfo(); // Emit constants. - WriteModuleConstants(VE, Stream); + writeModuleConstants(); - // Emit metadata. - WriteModuleMetadata(M, VE, Stream); + // Emit metadata kind names. + writeModuleMetadataKinds(); // Emit metadata. - WriteModuleMetadataStore(M, Stream); + writeModuleMetadata(); // Emit module-level use-lists. if (VE.shouldPreserveUseListOrder()) - WriteUseListBlock(nullptr, VE, Stream); + writeUseListBlock(nullptr); - WriteOperandBundleTags(M, Stream); + writeOperandBundleTags(); // Emit function bodies. - DenseMap<const Function *, std::unique_ptr<FunctionInfo>> FunctionIndex; - for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F) + DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex; + for (Module::const_iterator F = M.begin(), E = M.end(); F != E; ++F) if (!F->isDeclaration()) - WriteFunction(*F, VE, Stream, FunctionIndex, EmitFunctionSummary); + writeFunction(*F, FunctionToBitcodeIndex); // Need to write after the above call to WriteFunction which populates // the summary information in the index. - if (EmitFunctionSummary) - WritePerModuleFunctionSummary(FunctionIndex, M, VE, Stream); + if (Index) + writePerModuleGlobalValueSummary(); - WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream, - VSTOffsetPlaceholder, BitcodeStartBit, &FunctionIndex); + writeValueSymbolTable(M.getValueSymbolTable(), + /* IsModuleLevel */ true, &FunctionToBitcodeIndex); + + if (GenerateHash) { + writeModuleHash(BlockStartPos); + } Stream.ExitBlock(); } -/// EmitDarwinBCHeader - If generating a bc file on darwin, we have to emit a +static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer, + uint32_t &Position) { + support::endian::write32le(&Buffer[Position], Value); + Position += 4; +} + +/// If generating a bc file on darwin, we have to emit a /// header and trailer to make it compatible with the system archiver. To do /// this we emit the following header, and then emit a trailer that pads the /// file out to be a multiple of 16 bytes. @@ -2966,18 +3722,7 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream, /// uint32_t CPUType; // CPU specifier. /// ... potentially more later ... /// }; -enum { - DarwinBCSizeFieldOffset = 3*4, // Offset to bitcode_size. - DarwinBCHeaderSize = 5*4 -}; - -static void WriteInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer, - uint32_t &Position) { - support::endian::write32le(&Buffer[Position], Value); - Position += 4; -} - -static void EmitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, +static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, const Triple &TT) { unsigned CPUType = ~0U; @@ -3005,18 +3750,18 @@ static void EmitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, CPUType = DARWIN_CPU_TYPE_ARM; // Traditional Bitcode starts after header. - assert(Buffer.size() >= DarwinBCHeaderSize && + assert(Buffer.size() >= BWH_HeaderSize && "Expected header size to be reserved"); - unsigned BCOffset = DarwinBCHeaderSize; - unsigned BCSize = Buffer.size()-DarwinBCHeaderSize; + unsigned BCOffset = BWH_HeaderSize; + unsigned BCSize = Buffer.size() - BWH_HeaderSize; // Write the magic and version. unsigned Position = 0; - WriteInt32ToBuffer(0x0B17C0DE , Buffer, Position); - WriteInt32ToBuffer(0 , Buffer, Position); // Version. - WriteInt32ToBuffer(BCOffset , Buffer, Position); - WriteInt32ToBuffer(BCSize , Buffer, Position); - WriteInt32ToBuffer(CPUType , Buffer, Position); + writeInt32ToBuffer(0x0B17C0DE, Buffer, Position); + writeInt32ToBuffer(0, Buffer, Position); // Version. + writeInt32ToBuffer(BCOffset, Buffer, Position); + writeInt32ToBuffer(BCSize, Buffer, Position); + writeInt32ToBuffer(CPUType, Buffer, Position); // If the file is not a multiple of 16 bytes, insert dummy padding. while (Buffer.size() & 15) @@ -3024,7 +3769,7 @@ static void EmitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer, } /// Helper to write the header common to all bitcode files. -static void WriteBitcodeHeader(BitstreamWriter &Stream) { +void BitcodeWriter::writeBitcodeHeader() { // Emit the file header. Stream.Emit((unsigned)'B', 8); Stream.Emit((unsigned)'C', 8); @@ -3038,55 +3783,30 @@ static void WriteBitcodeHeader(BitstreamWriter &Stream) { /// stream. void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, - bool EmitFunctionSummary) { + const ModuleSummaryIndex *Index, + bool GenerateHash) { SmallVector<char, 0> Buffer; Buffer.reserve(256*1024); // If this is darwin or another generic macho target, reserve space for the // header. Triple TT(M->getTargetTriple()); - if (TT.isOSDarwin()) - Buffer.insert(Buffer.begin(), DarwinBCHeaderSize, 0); + if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) + Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); // Emit the module into the buffer. - { - BitstreamWriter Stream(Buffer); - // Save the start bit of the actual bitcode, in case there is space - // saved at the start for the darwin header above. The reader stream - // will start at the bitcode, and we need the offset of the VST - // to line up. - uint64_t BitcodeStartBit = Stream.GetCurrentBitNo(); - - // Emit the file header. - WriteBitcodeHeader(Stream); - - WriteIdentificationBlock(M, Stream); - - // Emit the module. - WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit, - EmitFunctionSummary); - } + ModuleBitcodeWriter ModuleWriter(M, Buffer, ShouldPreserveUseListOrder, Index, + GenerateHash); + ModuleWriter.write(); - if (TT.isOSDarwin()) - EmitDarwinBCHeaderAndTrailer(Buffer, TT); + if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) + emitDarwinBCHeaderAndTrailer(Buffer, TT); // Write the generated bitstream to "Out". Out.write((char*)&Buffer.front(), Buffer.size()); } -// Write the specified function summary index to the given raw output stream, -// where it will be written in a new bitcode block. This is used when -// writing the combined index file for ThinLTO. -void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index, - raw_ostream &Out) { - SmallVector<char, 0> Buffer; - Buffer.reserve(256 * 1024); - - BitstreamWriter Stream(Buffer); - - // Emit the bitcode header. - WriteBitcodeHeader(Stream); - +void IndexBitcodeWriter::writeIndex() { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); SmallVector<unsigned, 1> Vals; @@ -3094,17 +3814,34 @@ void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index, Vals.push_back(CurVersion); Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals); + // If we have a VST, write the VSTOFFSET record placeholder. + writeValueSymbolTableForwardDecl(); + // Write the module paths in the combined index. - WriteModStrings(Index, Stream); + writeModStrings(); - // Write the function summary combined index records. - WriteCombinedFunctionSummary(Index, Stream); + // Write the summary combined index records. + writeCombinedGlobalValueSummary(); // Need a special VST writer for the combined index (we don't have a // real VST and real values when this is invoked). - WriteCombinedValueSymbolTable(Index, Stream); + writeCombinedValueSymbolTable(); Stream.ExitBlock(); +} + +// Write the specified module summary index to the given raw output stream, +// where it will be written in a new bitcode block. This is used when +// writing the combined index file for ThinLTO. When writing a subset of the +// index for a distributed backend, provide a \p ModuleToSummariesForIndex map. +void llvm::WriteIndexToFile( + const ModuleSummaryIndex &Index, raw_ostream &Out, + std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) { + SmallVector<char, 0> Buffer; + Buffer.reserve(256 * 1024); + + IndexBitcodeWriter IndexWriter(Buffer, Index, ModuleToSummariesForIndex); + IndexWriter.write(); Out.write((char *)&Buffer.front(), Buffer.size()); } |