diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen')
38 files changed, 684 insertions, 560 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/Analysis.cpp b/contrib/llvm-project/llvm/lib/CodeGen/Analysis.cpp index e8fef505e43d..cdf5586766da 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/Analysis.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/Analysis.cpp @@ -585,7 +585,7 @@ bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I, // goes, they shouldn't affect whether the call is a tail call. for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable, Attribute::DereferenceableOrNull, Attribute::NoAlias, - Attribute::NonNull}) { + Attribute::NonNull, Attribute::NoUndef}) { CallerAttrs.removeAttribute(Attr); CalleeAttrs.removeAttribute(Attr); } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 4f3f798fe6f8..3e8e190eecc3 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1647,8 +1647,18 @@ void AsmPrinter::emitGlobalAlias(Module &M, const GlobalAlias &GA) { // Set the symbol type to function if the alias has a function type. // This affects codegen when the aliasee is not a function. - if (IsFunction) + if (IsFunction) { OutStreamer->emitSymbolAttribute(Name, MCSA_ELF_TypeFunction); + if (TM.getTargetTriple().isOSBinFormatCOFF()) { + OutStreamer->BeginCOFFSymbolDef(Name); + OutStreamer->EmitCOFFSymbolStorageClass( + GA.hasLocalLinkage() ? COFF::IMAGE_SYM_CLASS_STATIC + : COFF::IMAGE_SYM_CLASS_EXTERNAL); + OutStreamer->EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION + << COFF::SCT_COMPLEX_TYPE_SHIFT); + OutStreamer->EndCOFFSymbolDef(); + } + } emitVisibility(Name, GA.getVisibility()); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DIE.cpp index 1a0256f30d41..396322c4979d 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DIE.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DIE.cpp @@ -314,8 +314,7 @@ unsigned DIE::computeOffsetsAndAbbrevs(const dwarf::FormParams &FormParams, //===----------------------------------------------------------------------===// // DIEUnit Implementation //===----------------------------------------------------------------------===// -DIEUnit::DIEUnit(dwarf::Tag UnitTag) - : Die(UnitTag), Section(nullptr), Offset(0) { +DIEUnit::DIEUnit(dwarf::Tag UnitTag) : Die(UnitTag) { Die.Owner = this; assert((UnitTag == dwarf::DW_TAG_compile_unit || UnitTag == dwarf::DW_TAG_skeleton_unit || diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp index e36b7e2ae885..63343d2519f9 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp @@ -33,8 +33,7 @@ #include "llvm/Target/TargetOptions.h" using namespace llvm; -DwarfCFIExceptionBase::DwarfCFIExceptionBase(AsmPrinter *A) - : EHStreamer(A), shouldEmitCFI(false), hasEmittedCFISections(false) {} +DwarfCFIExceptionBase::DwarfCFIExceptionBase(AsmPrinter *A) : EHStreamer(A) {} void DwarfCFIExceptionBase::markFunctionEnd() { endFragment(); @@ -52,8 +51,7 @@ void DwarfCFIExceptionBase::endFragment() { } DwarfCFIException::DwarfCFIException(AsmPrinter *A) - : DwarfCFIExceptionBase(A), shouldEmitPersonality(false), - forceEmitPersonality(false), shouldEmitLSDA(false) {} + : DwarfCFIExceptionBase(A) {} DwarfCFIException::~DwarfCFIException() {} diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index 680b9586228f..609b568f28be 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -3367,8 +3367,7 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU, // Fast path if we're building some type units and one has already used the // address pool we know we're going to throw away all this work anyway, so // don't bother building dependent types. - if (!TypeUnitsUnderConstruction.empty() && - (AddrPool.hasBeenUsed() || SeenLocalType)) + if (!TypeUnitsUnderConstruction.empty() && AddrPool.hasBeenUsed()) return; auto Ins = TypeSignatures.insert(std::make_pair(CTy, 0)); @@ -3379,7 +3378,6 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU, bool TopLevelType = TypeUnitsUnderConstruction.empty(); AddrPool.resetUsedFlag(); - SeenLocalType = false; auto OwnedUnit = std::make_unique<DwarfTypeUnit>(CU, Asm, this, &InfoHolder, getDwoLineTable(CU)); @@ -3423,7 +3421,7 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU, // Types referencing entries in the address table cannot be placed in type // units. - if (AddrPool.hasBeenUsed() || SeenLocalType) { + if (AddrPool.hasBeenUsed()) { // Remove all the types built while building this type. // This is pessimistic as some of these types might not be dependent on @@ -3451,18 +3449,14 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU, DwarfDebug::NonTypeUnitContext::NonTypeUnitContext(DwarfDebug *DD) : DD(DD), - TypeUnitsUnderConstruction(std::move(DD->TypeUnitsUnderConstruction)), - AddrPoolUsed(DD->AddrPool.hasBeenUsed()), - SeenLocalType(DD->SeenLocalType) { + TypeUnitsUnderConstruction(std::move(DD->TypeUnitsUnderConstruction)), AddrPoolUsed(DD->AddrPool.hasBeenUsed()) { DD->TypeUnitsUnderConstruction.clear(); DD->AddrPool.resetUsedFlag(); - DD->SeenLocalType = false; } DwarfDebug::NonTypeUnitContext::~NonTypeUnitContext() { DD->TypeUnitsUnderConstruction = std::move(TypeUnitsUnderConstruction); DD->AddrPool.resetUsedFlag(AddrPoolUsed); - DD->SeenLocalType = SeenLocalType; } DwarfDebug::NonTypeUnitContext DwarfDebug::enterNonTypeUnitContext() { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h index 0043000652e8..4e1a1b1e068d 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -433,7 +433,6 @@ private: DenseMap<const DIStringType *, unsigned> StringTypeLocMap; AddressPool AddrPool; - bool SeenLocalType = false; /// Accelerator tables. AccelTable<DWARF5AccelTableData> AccelDebugNames; @@ -672,7 +671,6 @@ public: DwarfDebug *DD; decltype(DwarfDebug::TypeUnitsUnderConstruction) TypeUnitsUnderConstruction; bool AddrPoolUsed; - bool SeenLocalType; friend class DwarfDebug; NonTypeUnitContext(DwarfDebug *DD); public: @@ -681,7 +679,6 @@ public: }; NonTypeUnitContext enterNonTypeUnitContext(); - void seenLocalType() { SeenLocalType = true; } /// Add a label so that arange data can be generated for it. void addArangeLabel(SymbolCU SCU) { ArangeLabels.push_back(SCU); } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfException.h b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfException.h index 4defa8a30855..e5cda4739fde 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfException.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfException.h @@ -26,9 +26,9 @@ protected: DwarfCFIExceptionBase(AsmPrinter *A); /// Per-function flag to indicate if frame CFI info should be emitted. - bool shouldEmitCFI; + bool shouldEmitCFI = false; /// Per-module flag to indicate if .cfi_section has beeen emitted. - bool hasEmittedCFISections; + bool hasEmittedCFISections = false; void markFunctionEnd() override; void endFragment() override; @@ -36,13 +36,13 @@ protected: class LLVM_LIBRARY_VISIBILITY DwarfCFIException : public DwarfCFIExceptionBase { /// Per-function flag to indicate if .cfi_personality should be emitted. - bool shouldEmitPersonality; + bool shouldEmitPersonality = false; /// Per-function flag to indicate if .cfi_personality must be emitted. - bool forceEmitPersonality; + bool forceEmitPersonality = false; /// Per-function flag to indicate if .cfi_lsda should be emitted. - bool shouldEmitLSDA; + bool shouldEmitLSDA = false; public: //===--------------------------------------------------------------------===// diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp index ee932d105107..fe438102ee98 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp @@ -287,9 +287,17 @@ bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI, // expression representing a value, rather than a location. if ((!isParameterValue() && !isMemoryLocation() && !HasComplexExpression) || isEntryValue()) { + auto FragmentInfo = ExprCursor.getFragmentInfo(); + unsigned RegSize = 0; for (auto &Reg : DwarfRegs) { + RegSize += Reg.SubRegSize; if (Reg.DwarfRegNo >= 0) addReg(Reg.DwarfRegNo, Reg.Comment); + if (FragmentInfo) + if (RegSize > FragmentInfo->SizeInBits) + // If the register is larger than the current fragment stop + // once the fragment is covered. + break; addOpPiece(Reg.SubRegSize); } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp index 15d90c54adfc..5a2bd479f277 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp @@ -89,8 +89,7 @@ bool DIEDwarfExpression::isFrameRegister(const TargetRegisterInfo &TRI, DwarfUnit::DwarfUnit(dwarf::Tag UnitTag, const DICompileUnit *Node, AsmPrinter *A, DwarfDebug *DW, DwarfFile *DWU) - : DIEUnit(UnitTag), CUNode(Node), Asm(A), DD(DW), DU(DWU), - IndexTyDie(nullptr) {} + : DIEUnit(UnitTag), CUNode(Node), Asm(A), DD(DW), DU(DWU) {} DwarfTypeUnit::DwarfTypeUnit(DwarfCompileUnit &CU, AsmPrinter *A, DwarfDebug *DW, DwarfFile *DWU, @@ -597,8 +596,10 @@ DIE *DwarfUnit::createTypeDIE(const DIScope *Context, DIE &ContextDIE, // Skip updating the accelerator tables since this is not the full type. if (MDString *TypeId = CTy->getRawIdentifier()) DD->addDwarfTypeUnitType(getCU(), TypeId->getString(), TyDIE, CTy); - else + else { + auto X = DD->enterNonTypeUnitContext(); finishNonUnitTypeDIE(TyDIE, CTy); + } return &TyDIE; } constructTypeDIE(TyDIE, CTy); @@ -1852,23 +1853,5 @@ void DwarfTypeUnit::finishNonUnitTypeDIE(DIE& D, const DICompositeType *CTy) { addString(D, dwarf::DW_AT_name, Name); if (Name.startswith("_STN") || !Name.contains('<')) addTemplateParams(D, CTy->getTemplateParams()); - // If the type is in an anonymous namespace, we can't reference it from a TU - // (since the type would be CU local and the TU doesn't specify which TU has - // the appropriate type definition) - so flag this emission as such and skip - // the rest of the emission now since we're going to throw out all this work - // and put the outer/referencing type in the CU instead. - // FIXME: Probably good to generalize this to a DICompositeType flag populated - // by the frontend, then we could use that to have types that can have - // decl+def merged by LTO but where the definition still doesn't go in a type - // unit because the type has only one definition. - for (DIScope *S = CTy->getScope(); S; S = S->getScope()) { - if (auto *NS = dyn_cast<DINamespace>(S)) { - if (NS->getName().empty()) { - DD->seenLocalType(); - break; - } - } - } - auto X = DD->enterNonTypeUnitContext(); getCU().createTypeDIE(CTy); } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h index 330f3bacca43..48d63d126701 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h @@ -51,7 +51,7 @@ protected: DwarfFile *DU; /// An anonymous type for index type. Owned by DIEUnit. - DIE *IndexTyDie; + DIE *IndexTyDie = nullptr; /// Tracks the mapping of unit level debug information variables to debug /// information entries. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp b/contrib/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp index 28f24e5ea908..c888adeafca5 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -3446,7 +3446,7 @@ private: bool AllAddrModesTrivial = true; /// Common Type for all different fields in addressing modes. - Type *CommonType; + Type *CommonType = nullptr; /// SimplifyQuery for simplifyInstruction utility. const SimplifyQuery &SQ; @@ -3456,7 +3456,7 @@ private: public: AddressingModeCombiner(const SimplifyQuery &_SQ, Value *OriginalValue) - : CommonType(nullptr), SQ(_SQ), Original(OriginalValue) {} + : SQ(_SQ), Original(OriginalValue) {} /// Get the combined AddrMode const ExtAddrMode &getAddrMode() const { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/EarlyIfConversion.cpp b/contrib/llvm-project/llvm/lib/CodeGen/EarlyIfConversion.cpp index 0b5469b02637..6a0da4dad3c1 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/EarlyIfConversion.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/EarlyIfConversion.cpp @@ -111,12 +111,11 @@ public: /// Information about each phi in the Tail block. struct PHIInfo { MachineInstr *PHI; - unsigned TReg, FReg; + unsigned TReg = 0, FReg = 0; // Latencies from Cond+Branch, TReg, and FReg to DstReg. - int CondCycles, TCycles, FCycles; + int CondCycles = 0, TCycles = 0, FCycles = 0; - PHIInfo(MachineInstr *phi) - : PHI(phi), TReg(0), FReg(0), CondCycles(0), TCycles(0), FCycles(0) {} + PHIInfo(MachineInstr *phi) : PHI(phi) {} }; SmallVector<PHIInfo, 8> PHIs; diff --git a/contrib/llvm-project/llvm/lib/CodeGen/ExpandMemCmp.cpp b/contrib/llvm-project/llvm/lib/CodeGen/ExpandMemCmp.cpp index d0c2b8c267ff..60ee1812ee2c 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/ExpandMemCmp.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/ExpandMemCmp.cpp @@ -70,8 +70,8 @@ class MemCmpExpansion { CallInst *const CI; ResultBlock ResBlock; const uint64_t Size; - unsigned MaxLoadSize; - uint64_t NumLoadsNonOneByte; + unsigned MaxLoadSize = 0; + uint64_t NumLoadsNonOneByte = 0; const uint64_t NumLoadsPerBlockForZeroCmp; std::vector<BasicBlock *> LoadCmpBlocks; BasicBlock *EndBlock; @@ -219,8 +219,7 @@ MemCmpExpansion::MemCmpExpansion( const TargetTransformInfo::MemCmpExpansionOptions &Options, const bool IsUsedForZeroCmp, const DataLayout &TheDataLayout, DomTreeUpdater *DTU) - : CI(CI), Size(Size), MaxLoadSize(0), NumLoadsNonOneByte(0), - NumLoadsPerBlockForZeroCmp(Options.NumLoadsPerBlock), + : CI(CI), Size(Size), NumLoadsPerBlockForZeroCmp(Options.NumLoadsPerBlock), IsUsedForZeroCmp(IsUsedForZeroCmp), DL(TheDataLayout), DTU(DTU), Builder(CI) { assert(Size > 0 && "zero blocks"); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegacyLegalizerInfo.cpp b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegacyLegalizerInfo.cpp index 727d33fe4a40..6271a4514c27 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegacyLegalizerInfo.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/LegacyLegalizerInfo.cpp @@ -64,7 +64,7 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, LegacyLegalizeAction Action) { return OS; } -LegacyLegalizerInfo::LegacyLegalizerInfo() : TablesInitialized(false) { +LegacyLegalizerInfo::LegacyLegalizerInfo() { // Set defaults. // FIXME: these two (G_ANYEXT and G_TRUNC?) can be legalized to the // fundamental load/store Jakob proposed. Once loads & stores are supported. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp b/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp index 681e2f3dc848..1b20d1da20ad 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/IfConversion.cpp @@ -1211,11 +1211,11 @@ bool IfConverter::FeasibilityAnalysis(BBInfo &BBI, void IfConverter::AnalyzeBlock( MachineBasicBlock &MBB, std::vector<std::unique_ptr<IfcvtToken>> &Tokens) { struct BBState { - BBState(MachineBasicBlock &MBB) : MBB(&MBB), SuccsAnalyzed(false) {} + BBState(MachineBasicBlock &MBB) : MBB(&MBB) {} MachineBasicBlock *MBB; /// This flag is true if MBB's successors have been analyzed. - bool SuccsAnalyzed; + bool SuccsAnalyzed = false; }; // Push MBB to the stack. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/contrib/llvm-project/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp index 2ee9379cb286..230c6846dde2 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp @@ -656,10 +656,10 @@ public: }; /// Basic-block the load instructions are within - BasicBlock *BB; + BasicBlock *BB = nullptr; /// Pointer value of all participation load instructions - Value *PV; + Value *PV = nullptr; /// Participating load instructions std::set<LoadInst *> LIs; @@ -668,7 +668,7 @@ public: std::set<Instruction *> Is; /// Final shuffle-vector instruction - ShuffleVectorInst *SVI; + ShuffleVectorInst *SVI = nullptr; /// Information of the offset for each vector element ElementInfo *EI; @@ -676,8 +676,7 @@ public: /// Vector Type FixedVectorType *const VTy; - VectorInfo(FixedVectorType *VTy) - : BB(nullptr), PV(nullptr), SVI(nullptr), VTy(VTy) { + VectorInfo(FixedVectorType *VTy) : VTy(VTy) { EI = new ElementInfo[VTy->getNumElements()]; } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp index 8a190e769941..0eb6100230bd 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp @@ -274,6 +274,13 @@ public: // Map of the preferred location for each value. DenseMap<ValueIDNum, LocIdx> ValueToLoc; + + // Initialized the preferred-location map with illegal locations, to be + // filled in later. + for (auto &VLoc : VLocs) + if (VLoc.second.Kind == DbgValue::Def) + ValueToLoc.insert({VLoc.second.ID, LocIdx::MakeIllegalLoc()}); + ActiveMLocs.reserve(VLocs.size()); ActiveVLocs.reserve(VLocs.size()); @@ -285,21 +292,20 @@ public: ValueIDNum &VNum = MLocs[Idx.asU64()]; VarLocs.push_back(VNum); - // Short-circuit unnecessary preferred location update. - if (VLocs.empty()) + // Is there a variable that wants a location for this value? If not, skip. + auto VIt = ValueToLoc.find(VNum); + if (VIt == ValueToLoc.end()) continue; - auto it = ValueToLoc.find(VNum); + LocIdx CurLoc = VIt->second; // In order of preference, pick: // * Callee saved registers, // * Other registers, // * Spill slots. - if (it == ValueToLoc.end() || MTracker->isSpill(it->second) || - (!isCalleeSaved(it->second) && isCalleeSaved(Idx.asU64()))) { + if (CurLoc.isIllegal() || MTracker->isSpill(CurLoc) || + (!isCalleeSaved(CurLoc) && isCalleeSaved(Idx.asU64()))) { // Insert, or overwrite if insertion failed. - auto PrefLocRes = ValueToLoc.insert(std::make_pair(VNum, Idx)); - if (!PrefLocRes.second) - PrefLocRes.first->second = Idx; + VIt->second = Idx; } } @@ -314,7 +320,7 @@ public: // If the value has no location, we can't make a variable location. const ValueIDNum &Num = Var.second.ID; auto ValuesPreferredLoc = ValueToLoc.find(Num); - if (ValuesPreferredLoc == ValueToLoc.end()) { + if (ValuesPreferredLoc->second.isIllegal()) { // If it's a def that occurs in this block, register it as a // use-before-def to be resolved as we step through the block. if (Num.getBlock() == (unsigned)MBB.getNumber() && !Num.isPHI()) @@ -1374,18 +1380,20 @@ void InstrRefBasedLDV::transferRegisterDef(MachineInstr &MI) { // Look for any clobbers performed by a register mask. Only test locations // that are actually being tracked. - for (auto L : MTracker->locations()) { - // Stack locations can't be clobbered by regmasks. - if (MTracker->isSpill(L.Idx)) - continue; + if (!RegMaskPtrs.empty()) { + for (auto L : MTracker->locations()) { + // Stack locations can't be clobbered by regmasks. + if (MTracker->isSpill(L.Idx)) + continue; - Register Reg = MTracker->LocIdxToLocID[L.Idx]; - if (IgnoreSPAlias(Reg)) - continue; + Register Reg = MTracker->LocIdxToLocID[L.Idx]; + if (IgnoreSPAlias(Reg)) + continue; - for (auto *MO : RegMaskPtrs) - if (MO->clobbersPhysReg(Reg)) - TTracker->clobberMloc(L.Idx, MI.getIterator(), false); + for (auto *MO : RegMaskPtrs) + if (MO->clobbersPhysReg(Reg)) + TTracker->clobberMloc(L.Idx, MI.getIterator(), false); + } } // Tell TTracker about any folded stack store. @@ -2212,40 +2220,6 @@ void InstrRefBasedLDV::buildMLocValueMap( // redundant PHIs. } -// Boilerplate for feeding MachineBasicBlocks into IDF calculator. Provide -// template specialisations for graph traits and a successor enumerator. -namespace llvm { -template <> struct GraphTraits<MachineBasicBlock> { - using NodeRef = MachineBasicBlock *; - using ChildIteratorType = MachineBasicBlock::succ_iterator; - - static NodeRef getEntryNode(MachineBasicBlock *BB) { return BB; } - static ChildIteratorType child_begin(NodeRef N) { return N->succ_begin(); } - static ChildIteratorType child_end(NodeRef N) { return N->succ_end(); } -}; - -template <> struct GraphTraits<const MachineBasicBlock> { - using NodeRef = const MachineBasicBlock *; - using ChildIteratorType = MachineBasicBlock::const_succ_iterator; - - static NodeRef getEntryNode(const MachineBasicBlock *BB) { return BB; } - static ChildIteratorType child_begin(NodeRef N) { return N->succ_begin(); } - static ChildIteratorType child_end(NodeRef N) { return N->succ_end(); } -}; - -using MachineDomTreeBase = DomTreeBase<MachineBasicBlock>::NodeType; -using MachineDomTreeChildGetter = - typename IDFCalculatorDetail::ChildrenGetterTy<MachineDomTreeBase, false>; - -namespace IDFCalculatorDetail { -template <> -typename MachineDomTreeChildGetter::ChildrenTy -MachineDomTreeChildGetter::get(const NodeRef &N) { - return {N->succ_begin(), N->succ_end()}; -} -} // namespace IDFCalculatorDetail -} // namespace llvm - void InstrRefBasedLDV::BlockPHIPlacement( const SmallPtrSetImpl<MachineBasicBlock *> &AllBlocks, const SmallPtrSetImpl<MachineBasicBlock *> &DefBlocks, @@ -2253,8 +2227,7 @@ void InstrRefBasedLDV::BlockPHIPlacement( // Apply IDF calculator to the designated set of location defs, storing // required PHIs into PHIBlocks. Uses the dominator tree stored in the // InstrRefBasedLDV object. - IDFCalculatorDetail::ChildrenGetterTy<MachineDomTreeBase, false> foo; - IDFCalculatorBase<MachineDomTreeBase, false> IDF(DomTree->getBase(), foo); + IDFCalculatorBase<MachineBasicBlock, false> IDF(DomTree->getBase()); IDF.setLiveInBlocks(AllBlocks); IDF.setDefiningBlocks(DefBlocks); @@ -2465,8 +2438,71 @@ bool InstrRefBasedLDV::vlocJoin( } } -void InstrRefBasedLDV::buildVLocValueMap(const DILocation *DILoc, - const SmallSet<DebugVariable, 4> &VarsWeCareAbout, +void InstrRefBasedLDV::getBlocksForScope( + const DILocation *DILoc, + SmallPtrSetImpl<const MachineBasicBlock *> &BlocksToExplore, + const SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks) { + // Get the set of "normal" in-lexical-scope blocks. + LS.getMachineBasicBlocks(DILoc, BlocksToExplore); + + // VarLoc LiveDebugValues tracks variable locations that are defined in + // blocks not in scope. This is something we could legitimately ignore, but + // lets allow it for now for the sake of coverage. + BlocksToExplore.insert(AssignBlocks.begin(), AssignBlocks.end()); + + // Storage for artificial blocks we intend to add to BlocksToExplore. + DenseSet<const MachineBasicBlock *> ToAdd; + + // To avoid needlessly dropping large volumes of variable locations, propagate + // variables through aritifical blocks, i.e. those that don't have any + // instructions in scope at all. To accurately replicate VarLoc + // LiveDebugValues, this means exploring all artificial successors too. + // Perform a depth-first-search to enumerate those blocks. + for (auto *MBB : BlocksToExplore) { + // Depth-first-search state: each node is a block and which successor + // we're currently exploring. + SmallVector<std::pair<const MachineBasicBlock *, + MachineBasicBlock::const_succ_iterator>, + 8> + DFS; + + // Find any artificial successors not already tracked. + for (auto *succ : MBB->successors()) { + if (BlocksToExplore.count(succ)) + continue; + if (!ArtificialBlocks.count(succ)) + continue; + ToAdd.insert(succ); + DFS.push_back({succ, succ->succ_begin()}); + } + + // Search all those blocks, depth first. + while (!DFS.empty()) { + const MachineBasicBlock *CurBB = DFS.back().first; + MachineBasicBlock::const_succ_iterator &CurSucc = DFS.back().second; + // Walk back if we've explored this blocks successors to the end. + if (CurSucc == CurBB->succ_end()) { + DFS.pop_back(); + continue; + } + + // If the current successor is artificial and unexplored, descend into + // it. + if (!ToAdd.count(*CurSucc) && ArtificialBlocks.count(*CurSucc)) { + ToAdd.insert(*CurSucc); + DFS.push_back({*CurSucc, (*CurSucc)->succ_begin()}); + continue; + } + + ++CurSucc; + } + }; + + BlocksToExplore.insert(ToAdd.begin(), ToAdd.end()); +} + +void InstrRefBasedLDV::buildVLocValueMap( + const DILocation *DILoc, const SmallSet<DebugVariable, 4> &VarsWeCareAbout, SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks, LiveInsT &Output, ValueIDNum **MOutLocs, ValueIDNum **MInLocs, SmallVectorImpl<VLocTracker> &AllTheVLocs) { @@ -2490,74 +2526,7 @@ void InstrRefBasedLDV::buildVLocValueMap(const DILocation *DILoc, return BBToOrder[A] < BBToOrder[B]; }; - LS.getMachineBasicBlocks(DILoc, BlocksToExplore); - - // A separate container to distinguish "blocks we're exploring" versus - // "blocks that are potentially in scope. See comment at start of vlocJoin. - SmallPtrSet<const MachineBasicBlock *, 8> InScopeBlocks = BlocksToExplore; - - // VarLoc LiveDebugValues tracks variable locations that are defined in - // blocks not in scope. This is something we could legitimately ignore, but - // lets allow it for now for the sake of coverage. - BlocksToExplore.insert(AssignBlocks.begin(), AssignBlocks.end()); - - // We also need to propagate variable values through any artificial blocks - // that immediately follow blocks in scope. - DenseSet<const MachineBasicBlock *> ToAdd; - - // Helper lambda: For a given block in scope, perform a depth first search - // of all the artificial successors, adding them to the ToAdd collection. - auto AccumulateArtificialBlocks = - [this, &ToAdd, &BlocksToExplore, - &InScopeBlocks](const MachineBasicBlock *MBB) { - // Depth-first-search state: each node is a block and which successor - // we're currently exploring. - SmallVector<std::pair<const MachineBasicBlock *, - MachineBasicBlock::const_succ_iterator>, - 8> - DFS; - - // Find any artificial successors not already tracked. - for (auto *succ : MBB->successors()) { - if (BlocksToExplore.count(succ) || InScopeBlocks.count(succ)) - continue; - if (!ArtificialBlocks.count(succ)) - continue; - ToAdd.insert(succ); - DFS.push_back(std::make_pair(succ, succ->succ_begin())); - } - - // Search all those blocks, depth first. - while (!DFS.empty()) { - const MachineBasicBlock *CurBB = DFS.back().first; - MachineBasicBlock::const_succ_iterator &CurSucc = DFS.back().second; - // Walk back if we've explored this blocks successors to the end. - if (CurSucc == CurBB->succ_end()) { - DFS.pop_back(); - continue; - } - - // If the current successor is artificial and unexplored, descend into - // it. - if (!ToAdd.count(*CurSucc) && ArtificialBlocks.count(*CurSucc)) { - ToAdd.insert(*CurSucc); - DFS.push_back(std::make_pair(*CurSucc, (*CurSucc)->succ_begin())); - continue; - } - - ++CurSucc; - } - }; - - // Search in-scope blocks and those containing a DBG_VALUE from this scope - // for artificial successors. - for (auto *MBB : BlocksToExplore) - AccumulateArtificialBlocks(MBB); - for (auto *MBB : InScopeBlocks) - AccumulateArtificialBlocks(MBB); - - BlocksToExplore.insert(ToAdd.begin(), ToAdd.end()); - InScopeBlocks.insert(ToAdd.begin(), ToAdd.end()); + getBlocksForScope(DILoc, BlocksToExplore, AssignBlocks); // Single block scope: not interesting! No propagation at all. Note that // this could probably go above ArtificialBlocks without damage, but @@ -2628,7 +2597,15 @@ void InstrRefBasedLDV::buildVLocValueMap(const DILocation *DILoc, SmallVector<MachineBasicBlock *, 32> PHIBlocks; - // Request the set of PHIs we should insert for this variable. + // Request the set of PHIs we should insert for this variable. If there's + // only one value definition, things are very simple. + if (DefBlocks.size() == 1) { + placePHIsForSingleVarDefinition(MutBlocksToExplore, *DefBlocks.begin(), + AllTheVLocs, Var, Output); + continue; + } + + // Otherwise: we need to place PHIs through SSA and propagate values. BlockPHIPlacement(MutBlocksToExplore, DefBlocks, PHIBlocks); // Insert PHIs into the per-block live-in tables for this variable. @@ -2769,6 +2746,39 @@ void InstrRefBasedLDV::buildVLocValueMap(const DILocation *DILoc, BlocksToExplore.clear(); } +void InstrRefBasedLDV::placePHIsForSingleVarDefinition( + const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks, + MachineBasicBlock *AssignMBB, SmallVectorImpl<VLocTracker> &AllTheVLocs, + const DebugVariable &Var, LiveInsT &Output) { + // If there is a single definition of the variable, then working out it's + // value everywhere is very simple: it's every block dominated by the + // definition. At the dominance frontier, the usual algorithm would: + // * Place PHIs, + // * Propagate values into them, + // * Find there's no incoming variable value from the other incoming branches + // of the dominance frontier, + // * Specify there's no variable value in blocks past the frontier. + // This is a common case, hence it's worth special-casing it. + + // Pick out the variables value from the block transfer function. + VLocTracker &VLocs = AllTheVLocs[AssignMBB->getNumber()]; + auto ValueIt = VLocs.Vars.find(Var); + const DbgValue &Value = ValueIt->second; + + // Assign the variable value to entry to each dominated block that's in scope. + // Skip the definition block -- it's assigned the variable value in the middle + // of the block somewhere. + for (auto *ScopeBlock : InScopeBlocks) { + if (!DomTree->properlyDominates(AssignMBB, ScopeBlock)) + continue; + + Output[ScopeBlock->getNumber()].push_back({Var, Value}); + } + + // All blocks that aren't dominated have no live-in value, thus no variable + // value will be given to them. +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void InstrRefBasedLDV::dump_mloc_transfer( const MLocTransferMap &mloc_transfer) const { @@ -2806,39 +2816,7 @@ void InstrRefBasedLDV::emitLocations( } } - // Go through all the transfers recorded in the TransferTracker -- this is - // both the live-ins to a block, and any movements of values that happen - // in the middle. - for (const auto &P : TTracker->Transfers) { - // We have to insert DBG_VALUEs in a consistent order, otherwise they - // appear in DWARF in different orders. Use the order that they appear - // when walking through each block / each instruction, stored in - // AllVarsNumbering. - SmallVector<std::pair<unsigned, MachineInstr *>> Insts; - for (MachineInstr *MI : P.Insts) { - DebugVariable Var(MI->getDebugVariable(), MI->getDebugExpression(), - MI->getDebugLoc()->getInlinedAt()); - Insts.emplace_back(AllVarsNumbering.find(Var)->second, MI); - } - llvm::sort(Insts, - [](const auto &A, const auto &B) { return A.first < B.first; }); - - // Insert either before or after the designated point... - if (P.MBB) { - MachineBasicBlock &MBB = *P.MBB; - for (const auto &Pair : Insts) - MBB.insert(P.Pos, Pair.second); - } else { - // Terminators, like tail calls, can clobber things. Don't try and place - // transfers after them. - if (P.Pos->isTerminator()) - continue; - - MachineBasicBlock &MBB = *P.Pos->getParent(); - for (const auto &Pair : Insts) - MBB.insertAfterBundle(P.Pos, Pair.second); - } - } + emitTransfers(AllVarsNumbering); } void InstrRefBasedLDV::initialSetup(MachineFunction &MF) { @@ -2883,6 +2861,45 @@ void InstrRefBasedLDV::initialSetup(MachineFunction &MF) { #endif } +bool InstrRefBasedLDV::emitTransfers( + DenseMap<DebugVariable, unsigned> &AllVarsNumbering) { + // Go through all the transfers recorded in the TransferTracker -- this is + // both the live-ins to a block, and any movements of values that happen + // in the middle. + for (const auto &P : TTracker->Transfers) { + // We have to insert DBG_VALUEs in a consistent order, otherwise they + // appear in DWARF in different orders. Use the order that they appear + // when walking through each block / each instruction, stored in + // AllVarsNumbering. + SmallVector<std::pair<unsigned, MachineInstr *>> Insts; + for (MachineInstr *MI : P.Insts) { + DebugVariable Var(MI->getDebugVariable(), MI->getDebugExpression(), + MI->getDebugLoc()->getInlinedAt()); + Insts.emplace_back(AllVarsNumbering.find(Var)->second, MI); + } + llvm::sort(Insts, + [](const auto &A, const auto &B) { return A.first < B.first; }); + + // Insert either before or after the designated point... + if (P.MBB) { + MachineBasicBlock &MBB = *P.MBB; + for (const auto &Pair : Insts) + MBB.insert(P.Pos, Pair.second); + } else { + // Terminators, like tail calls, can clobber things. Don't try and place + // transfers after them. + if (P.Pos->isTerminator()) + continue; + + MachineBasicBlock &MBB = *P.Pos->getParent(); + for (const auto &Pair : Insts) + MBB.insertAfterBundle(P.Pos, Pair.second); + } + } + + return TTracker->Transfers.size() != 0; +} + /// Calculate the liveness information for the given machine function and /// extend ranges across basic blocks. bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF, @@ -2989,14 +3006,14 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF, DenseMap<DebugVariable, unsigned> AllVarsNumbering; // Map from one LexicalScope to all the variables in that scope. - DenseMap<const LexicalScope *, SmallSet<DebugVariable, 4>> ScopeToVars; + ScopeToVarsT ScopeToVars; - // Map from One lexical scope to all blocks in that scope. - DenseMap<const LexicalScope *, SmallPtrSet<MachineBasicBlock *, 4>> - ScopeToBlocks; + // Map from One lexical scope to all blocks where assignments happen for + // that scope. + ScopeToAssignBlocksT ScopeToAssignBlocks; - // Store a DILocation that describes a scope. - DenseMap<const LexicalScope *, const DILocation *> ScopeToDILocation; + // Store map of DILocations that describes scopes. + ScopeToDILocT ScopeToDILocation; // To mirror old LiveDebugValues, enumerate variables in RPOT order. Otherwise // the order is unimportant, it just has to be stable. @@ -3016,7 +3033,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF, AllVarsNumbering.insert(std::make_pair(Var, AllVarsNumbering.size())); ScopeToVars[Scope].insert(Var); - ScopeToBlocks[Scope].insert(VTracker->MBB); + ScopeToAssignBlocks[Scope].insert(VTracker->MBB); ScopeToDILocation[Scope] = ScopeLoc; ++VarAssignCount; } @@ -3040,7 +3057,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF, // a map of variables to values in SavedLiveIns. for (auto &P : ScopeToVars) { buildVLocValueMap(ScopeToDILocation[P.first], P.second, - ScopeToBlocks[P.first], SavedLiveIns, MOutLocs, MInLocs, + ScopeToAssignBlocks[P.first], SavedLiveIns, MOutLocs, MInLocs, vlocs); } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h index 9e9c0ce394fd..e7383209c027 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h @@ -779,6 +779,17 @@ public: /// Used as the result type for the variable value dataflow problem. using LiveInsT = SmallVector<SmallVector<VarAndLoc, 8>, 8>; + /// Mapping from lexical scopes to a DILocation in that scope. + using ScopeToDILocT = DenseMap<const LexicalScope *, const DILocation *>; + + /// Mapping from lexical scopes to variables in that scope. + using ScopeToVarsT = DenseMap<const LexicalScope *, SmallSet<DebugVariable, 4>>; + + /// Mapping from lexical scopes to blocks where variables in that scope are + /// assigned. Such blocks aren't necessarily "in" the lexical scope, it's + /// just a block where an assignment happens. + using ScopeToAssignBlocksT = DenseMap<const LexicalScope *, SmallPtrSet<MachineBasicBlock *, 4>>; + private: MachineDominatorTree *DomTree; const TargetRegisterInfo *TRI; @@ -816,7 +827,7 @@ private: /// Blocks which are artificial, i.e. blocks which exclusively contain /// instructions without DebugLocs, or with line 0 locations. - SmallPtrSet<const MachineBasicBlock *, 16> ArtificialBlocks; + SmallPtrSet<MachineBasicBlock *, 16> ArtificialBlocks; // Mapping of blocks to and from their RPOT order. DenseMap<unsigned int, MachineBasicBlock *> OrderToBB; @@ -958,6 +969,15 @@ private: ValueIDNum **MInLocs, SmallVectorImpl<MLocTransferMap> &MLocTransfer); + /// Propagate variable values to blocks in the common case where there's + /// only one value assigned to the variable. This function has better + /// performance as it doesn't have to find the dominance frontier between + /// different assignments. + void placePHIsForSingleVarDefinition( + const SmallPtrSetImpl<MachineBasicBlock *> &InScopeBlocks, + MachineBasicBlock *MBB, SmallVectorImpl<VLocTracker> &AllTheVLocs, + const DebugVariable &Var, LiveInsT &Output); + /// Calculate the iterated-dominance-frontier for a set of defs, using the /// existing LLVM facilities for this. Works for a single "value" or /// machine/variable location. @@ -979,6 +999,19 @@ private: SmallPtrSet<const MachineBasicBlock *, 16> &Visited, ValueIDNum **OutLocs, ValueIDNum *InLocs); + /// Produce a set of blocks that are in the current lexical scope. This means + /// those blocks that contain instructions "in" the scope, blocks where + /// assignments to variables in scope occur, and artificial blocks that are + /// successors to any of the earlier blocks. See https://llvm.org/PR48091 for + /// more commentry on what "in scope" means. + /// \p DILoc A location in the scope that we're fetching blocks for. + /// \p Output Set to put in-scope-blocks into. + /// \p AssignBlocks Blocks known to contain assignments of variables in scope. + void + getBlocksForScope(const DILocation *DILoc, + SmallPtrSetImpl<const MachineBasicBlock *> &Output, + const SmallPtrSetImpl<MachineBasicBlock *> &AssignBlocks); + /// Solve the variable value dataflow problem, for a single lexical scope. /// Uses the algorithm from the file comment to resolve control flow joins /// using PHI placement and value propagation. Reads the locations of machine @@ -1029,6 +1062,12 @@ private: DenseMap<DebugVariable, unsigned> &AllVarsNumbering, const TargetPassConfig &TPC); + /// Take collections of DBG_VALUE instructions stored in TTracker, and + /// install them into their output blocks. Preserves a stable order of + /// DBG_VALUEs produced (which would otherwise cause nondeterminism) through + /// the AllVarsNumbering order. + bool emitTransfers(DenseMap<DebugVariable, unsigned> &AllVarsNumbering); + /// Boilerplate computation of some initial sets, artifical blocks and /// RPOT block ordering. void initialSetup(MachineFunction &MF); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp index b4dd41bbb810..42a0967bce3f 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp @@ -329,7 +329,7 @@ private: EntryValueKind, EntryValueBackupKind, EntryValueCopyBackupKind - } EVKind; + } EVKind = EntryValueLocKind::NonEntryValueKind; /// The value location. Stored separately to avoid repeatedly /// extracting it from MI. @@ -397,8 +397,7 @@ private: VarLoc(const MachineInstr &MI, LexicalScopes &LS) : Var(MI.getDebugVariable(), MI.getDebugExpression(), MI.getDebugLoc()->getInlinedAt()), - Expr(MI.getDebugExpression()), MI(MI), - EVKind(EntryValueLocKind::NonEntryValueKind) { + Expr(MI.getDebugExpression()), MI(MI) { assert(MI.isDebugValue() && "not a DBG_VALUE"); assert((MI.isDebugValueList() || MI.getNumOperands() == 4) && "malformed DBG_VALUE"); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MLRegallocEvictAdvisor.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MLRegallocEvictAdvisor.cpp index a74c57690640..33782c755eb0 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/MLRegallocEvictAdvisor.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/MLRegallocEvictAdvisor.cpp @@ -220,6 +220,19 @@ void resetInputs(MLModelRunner &Runner) { #undef _RESET } +// Per-live interval components that get aggregated into the feature values that +// will be passed to the evaluator. +struct LIFeatureComponents { + double R = 0; + double W = 0; + double RW = 0; + double IndVarUpdates = 0; + double HintWeights = 0.0; + int64_t NrDefsAndUses = 0; + float HottestBlockFreq = 0.0; + bool IsRemat = false; +}; + using CandidateRegList = std::array<std::pair<MCRegister, bool>, NumberOfInterferences>; using FeaturesListNormalizer = std::array<float, FeatureIDs::FeatureCount>; @@ -227,8 +240,8 @@ using FeaturesListNormalizer = std::array<float, FeatureIDs::FeatureCount>; /// The ML evictor (commonalities between release and development mode) class MLEvictAdvisor : public RegAllocEvictionAdvisor { public: - MLEvictAdvisor(const MachineFunction &MF, const RAGreedy &RA, - MLModelRunner *Runner, const MachineBlockFrequencyInfo &MBFI, + MLEvictAdvisor(MachineFunction &MF, const RAGreedy &RA, MLModelRunner *Runner, + const MachineBlockFrequencyInfo &MBFI, const MachineLoopInfo &Loops); protected: @@ -277,6 +290,9 @@ private: FixedRegisters); } + const LIFeatureComponents + getLIFeatureComponents(const LiveInterval &LI) const; + // Hold on to a default advisor for: // 1) the implementation of canEvictHintInterference, because we didn't learn // that nuance yet; @@ -319,7 +335,7 @@ private: } std::unique_ptr<RegAllocEvictionAdvisor> - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override { + getAdvisor(MachineFunction &MF, const RAGreedy &RA) override { if (!Runner) Runner = std::make_unique<ReleaseModeModelRunner<RegallocEvictModel>>( MF.getFunction().getContext(), FeatureNames, DecisionName); @@ -364,7 +380,7 @@ static const std::vector<TensorSpec> TrainingInputFeatures{ class DevelopmentModeEvictAdvisor : public MLEvictAdvisor { public: - DevelopmentModeEvictAdvisor(const MachineFunction &MF, const RAGreedy &RA, + DevelopmentModeEvictAdvisor(MachineFunction &MF, const RAGreedy &RA, MLModelRunner *Runner, const MachineBlockFrequencyInfo &MBFI, const MachineLoopInfo &Loops, Logger *Log) @@ -420,7 +436,7 @@ private: } std::unique_ptr<RegAllocEvictionAdvisor> - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override { + getAdvisor(MachineFunction &MF, const RAGreedy &RA) override { LLVMContext &Ctx = MF.getFunction().getContext(); if (ModelUnderTraining.empty() && TrainingLog.empty()) { Ctx.emitError("Regalloc development mode should be requested with at " @@ -480,7 +496,7 @@ float MLEvictAdvisor::getInitialQueueSize(const MachineFunction &MF) { return Ret; } -MLEvictAdvisor::MLEvictAdvisor(const MachineFunction &MF, const RAGreedy &RA, +MLEvictAdvisor::MLEvictAdvisor(MachineFunction &MF, const RAGreedy &RA, MLModelRunner *Runner, const MachineBlockFrequencyInfo &MBFI, const MachineLoopInfo &Loops) @@ -615,16 +631,15 @@ MCRegister MLEvictAdvisor::tryFindEvictionCandidate( for (auto I = Order.begin(), E = Order.getOrderLimitEnd(OrderLimit); I != E; ++I, ++Pos) { MCRegister PhysReg = *I; - Regs[Pos] = std::make_pair(PhysReg, true); + assert(!Regs[Pos].second); assert(PhysReg); if (!canAllocatePhysReg(CostPerUseLimit, PhysReg)) { - Regs[Pos].second = false; continue; } if (loadInterferenceFeatures(VirtReg, PhysReg, I.isHint(), FixedRegisters, Largest, Pos)) { ++Available; - Regs[Pos].second = true; + Regs[Pos] = std::make_pair(PhysReg, true); } } if (Available == 0) { @@ -632,6 +647,7 @@ MCRegister MLEvictAdvisor::tryFindEvictionCandidate( assert(!MustFindEviction); return MCRegister::NoRegister; } + const size_t ValidPosLimit = Pos; // If we must find eviction, the candidate should be masked out of the // decision making process. Regs[CandidateVirtRegPos].second = !MustFindEviction; @@ -665,9 +681,55 @@ MCRegister MLEvictAdvisor::tryFindEvictionCandidate( assert(!MustFindEviction); return MCRegister::NoRegister; } + assert(CandidatePos < ValidPosLimit); + (void)ValidPosLimit; return Regs[CandidatePos].first; } +const LIFeatureComponents +MLEvictAdvisor::getLIFeatureComponents(const LiveInterval &LI) const { + LIFeatureComponents Ret; + SmallPtrSet<MachineInstr *, 8> Visited; + const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); + + for (MachineRegisterInfo::reg_instr_nodbg_iterator + I = MRI->reg_instr_nodbg_begin(LI.reg()), + E = MRI->reg_instr_nodbg_end(); + I != E;) { + MachineInstr *MI = &*(I++); + + ++Ret.NrDefsAndUses; + if (!Visited.insert(MI).second) + continue; + + if (MI->isIdentityCopy() || MI->isImplicitDef()) + continue; + + bool Reads, Writes; + std::tie(Reads, Writes) = MI->readsWritesVirtualRegister(LI.reg()); + + float Freq = MBFI.getBlockFreqRelativeToEntryBlock(MI->getParent()); + Ret.HottestBlockFreq = std::max(Freq, Ret.HottestBlockFreq); + + Ret.R += (Reads && !Writes) * Freq; + Ret.W += (!Reads && Writes) * Freq; + Ret.RW += (Reads && Writes) * Freq; + + auto *MBB = MI->getParent(); + auto *Loop = Loops.getLoopFor(MBB); + bool IsExiting = Loop ? Loop->isLoopExiting(MBB) : false; + + if (Writes && IsExiting && LIS->isLiveOutOfMBB(LI, MBB)) + Ret.IndVarUpdates += Freq; + + if (MI->isCopy() && VirtRegAuxInfo::copyHint(MI, LI.reg(), TRI, *MRI)) + Ret.HintWeights += Freq; + } + Ret.IsRemat = VirtRegAuxInfo::isRematerializable( + LI, *LIS, *VRM, *MF.getSubtarget().getInstrInfo()); + return Ret; +} + // Overall, this currently mimics what we do for weight calculation, but instead // of accummulating the various features, we keep them separate. void MLEvictAdvisor::extractFeatures( @@ -676,11 +738,11 @@ void MLEvictAdvisor::extractFeatures( int64_t IsHint, int64_t LocalIntfsCount, float NrUrgent) const { int64_t NrDefsAndUses = 0; int64_t NrBrokenHints = 0; - float R = 0; - float W = 0; - float RW = 0; - float IndVarUpdates = 0; - float HintWeights = 0.0; + double R = 0.0; + double W = 0.0; + double RW = 0.0; + double IndVarUpdates = 0.0; + double HintWeights = 0.0; float StartBBFreq = 0.0; float EndBBFreq = 0.0; float HottestBlockFreq = 0.0; @@ -707,46 +769,19 @@ void MLEvictAdvisor::extractFeatures( if (LI.endIndex() > EndSI) EndSI = LI.endIndex(); - - SmallPtrSet<MachineInstr *, 8> Visited; - const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); + const LIFeatureComponents LIFC = getLIFeatureComponents(LI); NrBrokenHints += VRM->hasPreferredPhys(LI.reg()); - for (MachineRegisterInfo::reg_instr_nodbg_iterator - I = MRI->reg_instr_nodbg_begin(LI.reg()), - E = MRI->reg_instr_nodbg_end(); - I != E;) { - MachineInstr *MI = &*(I++); + NrDefsAndUses += LIFC.NrDefsAndUses; + HottestBlockFreq = std::max(HottestBlockFreq, LIFC.HottestBlockFreq); + R += LIFC.R; + W += LIFC.W; + RW += LIFC.RW; - ++NrDefsAndUses; - if (!Visited.insert(MI).second) - continue; + IndVarUpdates += LIFC.IndVarUpdates; - if (MI->isIdentityCopy() || MI->isImplicitDef()) - continue; - - bool Reads, Writes; - std::tie(Reads, Writes) = MI->readsWritesVirtualRegister(LI.reg()); - - float Freq = MBFI.getBlockFreqRelativeToEntryBlock(MI->getParent()); - if (Freq > HottestBlockFreq) - HottestBlockFreq = Freq; - R += (Reads && !Writes) * Freq; - W += (!Reads && Writes) * Freq; - RW += (Reads && Writes) * Freq; - - auto *MBB = MI->getParent(); - auto *Loop = Loops.getLoopFor(MBB); - bool IsExiting = Loop ? Loop->isLoopExiting(MBB) : false; - - if (Writes && IsExiting && LIS->isLiveOutOfMBB(LI, MBB)) - IndVarUpdates += Freq; - - if (MI->isCopy() && VirtRegAuxInfo::copyHint(MI, LI.reg(), TRI, *MRI)) - HintWeights += Freq; - } - NrRematerializable += VirtRegAuxInfo::isRematerializable( - LI, *LIS, *VRM, *MF.getSubtarget().getInstrInfo()); + HintWeights += LIFC.HintWeights; + NrRematerializable += LIFC.IsRemat; } size_t Size = 0; if (!Intervals.empty()) { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleInfo.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleInfo.cpp index 50cbb14e926e..31d4fc7d02bf 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleInfo.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleInfo.cpp @@ -400,12 +400,14 @@ bool MachineModuleInfoWrapperPass::doInitialization(Module &M) { // FIXME: Do this for new pass manager. LLVMContext &Ctx = M.getContext(); MMI.getContext().setDiagnosticHandler( - [&Ctx](const SMDiagnostic &SMD, bool IsInlineAsm, const SourceMgr &SrcMgr, - std::vector<const MDNode *> &LocInfos) { + [&Ctx, &M](const SMDiagnostic &SMD, bool IsInlineAsm, + const SourceMgr &SrcMgr, + std::vector<const MDNode *> &LocInfos) { unsigned LocCookie = 0; if (IsInlineAsm) LocCookie = getLocCookie(SMD, SrcMgr, LocInfos); - Ctx.diagnose(DiagnosticInfoSrcMgr(SMD, IsInlineAsm, LocCookie)); + Ctx.diagnose( + DiagnosticInfoSrcMgr(SMD, M.getName(), IsInlineAsm, LocCookie)); }); MMI.DbgInfoAvailable = !M.debug_compile_units().empty(); return false; diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleSlotTracker.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleSlotTracker.cpp index e4da179efcc4..aa63411df965 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleSlotTracker.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineModuleSlotTracker.cpp @@ -66,8 +66,7 @@ MachineModuleSlotTracker::MachineModuleSlotTracker( const MachineFunction *MF, bool ShouldInitializeAllMetadata) : ModuleSlotTracker(MF->getFunction().getParent(), ShouldInitializeAllMetadata), - TheFunction(MF->getFunction()), TheMMI(MF->getMMI()), MDNStartSlot(0), - MDNEndSlot(0) { + TheFunction(MF->getFunction()), TheMMI(MF->getMMI()) { setProcessHook([this](AbstractSlotTrackerStorage *AST, const Module *M, bool ShouldInitializeAllMetadata) { this->processMachineModule(AST, M, ShouldInitializeAllMetadata); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineRegisterInfo.cpp index 19bf87d3e290..1a4ad53ddf81 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/MachineRegisterInfo.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineRegisterInfo.cpp @@ -43,8 +43,7 @@ void MachineRegisterInfo::Delegate::anchor() {} MachineRegisterInfo::MachineRegisterInfo(MachineFunction *MF) : MF(MF), TracksSubRegLiveness(MF->getSubtarget().enableSubRegLiveness() && - EnableSubRegLiveness), - IsUpdatedCSRsInitialized(false) { + EnableSubRegLiveness) { unsigned NumRegs = getTargetRegisterInfo()->getNumRegs(); VRegInfo.reserve(256); RegAllocHints.reserve(256); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineVerifier.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineVerifier.cpp index 005d4ad1a328..c9d3e473062b 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1909,7 +1909,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { const Register Reg = MO->getReg(); if (!Reg) return; - if (MRI->tracksLiveness() && !MI->isDebugValue()) + if (MRI->tracksLiveness() && !MI->isDebugInstr()) checkLiveness(MO, MONum); // Verify the consistency of tied operands. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/PostRASchedulerList.cpp b/contrib/llvm-project/llvm/lib/CodeGen/PostRASchedulerList.cpp index d7cd0a583cee..aac46cb22084 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/PostRASchedulerList.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/PostRASchedulerList.cpp @@ -139,7 +139,7 @@ namespace { /// /// This is the instruction number from the top of the current block, not /// the SlotIndex. It is only used by the AntiDepBreaker. - unsigned EndIndex; + unsigned EndIndex = 0; public: SchedulePostRATDList( @@ -206,7 +206,7 @@ SchedulePostRATDList::SchedulePostRATDList( const RegisterClassInfo &RCI, TargetSubtargetInfo::AntiDepBreakMode AntiDepMode, SmallVectorImpl<const TargetRegisterClass *> &CriticalPathRCs) - : ScheduleDAGInstrs(MF, &MLI), AA(AA), EndIndex(0) { + : ScheduleDAGInstrs(MF, &MLI), AA(AA) { const InstrItineraryData *InstrItins = MF.getSubtarget().getInstrItineraryData(); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp index 87df7bb4a689..fc5d1104a999 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp @@ -25,7 +25,7 @@ using namespace llvm; static cl::opt<RegAllocEvictionAdvisorAnalysis::AdvisorMode> Mode( - "regalloc-enable-advisor", cl::Hidden, + "regalloc-enable-advisor", cl::Hidden, cl::ZeroOrMore, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values( @@ -66,7 +66,7 @@ public: private: std::unique_ptr<RegAllocEvictionAdvisor> - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override { + getAdvisor(MachineFunction &MF, const RAGreedy &RA) override { return std::make_unique<DefaultEvictionAdvisor>(MF, RA); } bool doInitialization(Module &M) override { @@ -113,7 +113,7 @@ StringRef RegAllocEvictionAdvisorAnalysis::getPassName() const { llvm_unreachable("Unknown advisor kind"); } -RegAllocEvictionAdvisor::RegAllocEvictionAdvisor(const MachineFunction &MF, +RegAllocEvictionAdvisor::RegAllocEvictionAdvisor(MachineFunction &MF, const RAGreedy &RA) : MF(MF), RA(RA), Matrix(RA.getInterferenceMatrix()), LIS(RA.getLiveIntervals()), VRM(RA.getVirtRegMap()), @@ -122,3 +122,178 @@ RegAllocEvictionAdvisor::RegAllocEvictionAdvisor(const MachineFunction &MF, EnableLocalReassign(EnableLocalReassignment || MF.getSubtarget().enableRALocalReassignment( MF.getTarget().getOptLevel())) {} + +/// shouldEvict - determine if A should evict the assigned live range B. The +/// eviction policy defined by this function together with the allocation order +/// defined by enqueue() decides which registers ultimately end up being split +/// and spilled. +/// +/// Cascade numbers are used to prevent infinite loops if this function is a +/// cyclic relation. +/// +/// @param A The live range to be assigned. +/// @param IsHint True when A is about to be assigned to its preferred +/// register. +/// @param B The live range to be evicted. +/// @param BreaksHint True when B is already assigned to its preferred register. +bool DefaultEvictionAdvisor::shouldEvict(LiveInterval &A, bool IsHint, + LiveInterval &B, + bool BreaksHint) const { + bool CanSplit = RA.getExtraInfo().getStage(B) < RS_Spill; + + // Be fairly aggressive about following hints as long as the evictee can be + // split. + if (CanSplit && IsHint && !BreaksHint) + return true; + + if (A.weight() > B.weight()) { + LLVM_DEBUG(dbgs() << "should evict: " << B << " w= " << B.weight() << '\n'); + return true; + } + return false; +} + +/// canEvictHintInterference - return true if the interference for VirtReg +/// on the PhysReg, which is VirtReg's hint, can be evicted in favor of VirtReg. +bool DefaultEvictionAdvisor::canEvictHintInterference( + LiveInterval &VirtReg, MCRegister PhysReg, + const SmallVirtRegSet &FixedRegisters) const { + EvictionCost MaxCost; + MaxCost.setBrokenHints(1); + return canEvictInterferenceBasedOnCost(VirtReg, PhysReg, true, MaxCost, + FixedRegisters); +} + +/// canEvictInterferenceBasedOnCost - Return true if all interferences between +/// VirtReg and PhysReg can be evicted. +/// +/// @param VirtReg Live range that is about to be assigned. +/// @param PhysReg Desired register for assignment. +/// @param IsHint True when PhysReg is VirtReg's preferred register. +/// @param MaxCost Only look for cheaper candidates and update with new cost +/// when returning true. +/// @returns True when interference can be evicted cheaper than MaxCost. +bool DefaultEvictionAdvisor::canEvictInterferenceBasedOnCost( + LiveInterval &VirtReg, MCRegister PhysReg, bool IsHint, + EvictionCost &MaxCost, const SmallVirtRegSet &FixedRegisters) const { + // It is only possible to evict virtual register interference. + if (Matrix->checkInterference(VirtReg, PhysReg) > LiveRegMatrix::IK_VirtReg) + return false; + + bool IsLocal = VirtReg.empty() || LIS->intervalIsInOneMBB(VirtReg); + + // Find VirtReg's cascade number. This will be unassigned if VirtReg was never + // involved in an eviction before. If a cascade number was assigned, deny + // evicting anything with the same or a newer cascade number. This prevents + // infinite eviction loops. + // + // This works out so a register without a cascade number is allowed to evict + // anything, and it can be evicted by anything. + unsigned Cascade = RA.getExtraInfo().getCascadeOrCurrentNext(VirtReg.reg()); + + EvictionCost Cost; + for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { + LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); + // If there is 10 or more interferences, chances are one is heavier. + const auto &Interferences = Q.interferingVRegs(10); + if (Interferences.size() >= 10) + return false; + + // Check if any interfering live range is heavier than MaxWeight. + for (LiveInterval *Intf : reverse(Interferences)) { + assert(Register::isVirtualRegister(Intf->reg()) && + "Only expecting virtual register interference from query"); + + // Do not allow eviction of a virtual register if we are in the middle + // of last-chance recoloring and this virtual register is one that we + // have scavenged a physical register for. + if (FixedRegisters.count(Intf->reg())) + return false; + + // Never evict spill products. They cannot split or spill. + if (RA.getExtraInfo().getStage(*Intf) == RS_Done) + return false; + // Once a live range becomes small enough, it is urgent that we find a + // register for it. This is indicated by an infinite spill weight. These + // urgent live ranges get to evict almost anything. + // + // Also allow urgent evictions of unspillable ranges from a strictly + // larger allocation order. + bool Urgent = + !VirtReg.isSpillable() && + (Intf->isSpillable() || + RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(VirtReg.reg())) < + RegClassInfo.getNumAllocatableRegs( + MRI->getRegClass(Intf->reg()))); + // Only evict older cascades or live ranges without a cascade. + unsigned IntfCascade = RA.getExtraInfo().getCascade(Intf->reg()); + if (Cascade <= IntfCascade) { + if (!Urgent) + return false; + // We permit breaking cascades for urgent evictions. It should be the + // last resort, though, so make it really expensive. + Cost.BrokenHints += 10; + } + // Would this break a satisfied hint? + bool BreaksHint = VRM->hasPreferredPhys(Intf->reg()); + // Update eviction cost. + Cost.BrokenHints += BreaksHint; + Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight()); + // Abort if this would be too expensive. + if (!(Cost < MaxCost)) + return false; + if (Urgent) + continue; + // Apply the eviction policy for non-urgent evictions. + if (!shouldEvict(VirtReg, IsHint, *Intf, BreaksHint)) + return false; + // If !MaxCost.isMax(), then we're just looking for a cheap register. + // Evicting another local live range in this case could lead to suboptimal + // coloring. + if (!MaxCost.isMax() && IsLocal && LIS->intervalIsInOneMBB(*Intf) && + (!EnableLocalReassign || !canReassign(*Intf, PhysReg))) { + return false; + } + } + } + MaxCost = Cost; + return true; +} + +MCRegister DefaultEvictionAdvisor::tryFindEvictionCandidate( + LiveInterval &VirtReg, const AllocationOrder &Order, + uint8_t CostPerUseLimit, const SmallVirtRegSet &FixedRegisters) const { + // Keep track of the cheapest interference seen so far. + EvictionCost BestCost; + BestCost.setMax(); + MCRegister BestPhys; + auto MaybeOrderLimit = getOrderLimit(VirtReg, Order, CostPerUseLimit); + if (!MaybeOrderLimit) + return MCRegister::NoRegister; + unsigned OrderLimit = *MaybeOrderLimit; + + // When we are just looking for a reduced cost per use, don't break any + // hints, and only evict smaller spill weights. + if (CostPerUseLimit < uint8_t(~0u)) { + BestCost.BrokenHints = 0; + BestCost.MaxWeight = VirtReg.weight(); + } + + for (auto I = Order.begin(), E = Order.getOrderLimitEnd(OrderLimit); I != E; + ++I) { + MCRegister PhysReg = *I; + assert(PhysReg); + if (!canAllocatePhysReg(CostPerUseLimit, PhysReg) || + !canEvictInterferenceBasedOnCost(VirtReg, PhysReg, false, BestCost, + FixedRegisters)) + continue; + + // Best so far. + BestPhys = PhysReg; + + // Stop if the hint can be used. + if (I.isHint()) + break; + } + return BestPhys; +} diff --git a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h index 33e03aed81a7..1f40386db8da 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h @@ -115,7 +115,7 @@ public: bool isUnusedCalleeSavedReg(MCRegister PhysReg) const; protected: - RegAllocEvictionAdvisor(const MachineFunction &MF, const RAGreedy &RA); + RegAllocEvictionAdvisor(MachineFunction &MF, const RAGreedy &RA); Register canReassign(LiveInterval &VirtReg, Register PrevReg) const; @@ -173,7 +173,7 @@ public: /// Get an advisor for the given context (i.e. machine function, etc) virtual std::unique_ptr<RegAllocEvictionAdvisor> - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) = 0; + getAdvisor(MachineFunction &MF, const RAGreedy &RA) = 0; AdvisorMode getAdvisorMode() const { return Mode; } protected: @@ -200,7 +200,7 @@ RegAllocEvictionAdvisorAnalysis *createDevelopmentModeAdvisor(); // out of RegAllocGreedy.cpp class DefaultEvictionAdvisor : public RegAllocEvictionAdvisor { public: - DefaultEvictionAdvisor(const MachineFunction &MF, const RAGreedy &RA) + DefaultEvictionAdvisor(MachineFunction &MF, const RAGreedy &RA) : RegAllocEvictionAdvisor(MF, RA) {} private: diff --git a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp index 6ea6dbcbbb74..7870574df5b2 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -440,143 +440,6 @@ Register RegAllocEvictionAdvisor::canReassign(LiveInterval &VirtReg, return PhysReg; } -/// shouldEvict - determine if A should evict the assigned live range B. The -/// eviction policy defined by this function together with the allocation order -/// defined by enqueue() decides which registers ultimately end up being split -/// and spilled. -/// -/// Cascade numbers are used to prevent infinite loops if this function is a -/// cyclic relation. -/// -/// @param A The live range to be assigned. -/// @param IsHint True when A is about to be assigned to its preferred -/// register. -/// @param B The live range to be evicted. -/// @param BreaksHint True when B is already assigned to its preferred register. -bool DefaultEvictionAdvisor::shouldEvict(LiveInterval &A, bool IsHint, - LiveInterval &B, - bool BreaksHint) const { - bool CanSplit = RA.getExtraInfo().getStage(B) < RS_Spill; - - // Be fairly aggressive about following hints as long as the evictee can be - // split. - if (CanSplit && IsHint && !BreaksHint) - return true; - - if (A.weight() > B.weight()) { - LLVM_DEBUG(dbgs() << "should evict: " << B << " w= " << B.weight() << '\n'); - return true; - } - return false; -} - -/// canEvictHintInterference - return true if the interference for VirtReg -/// on the PhysReg, which is VirtReg's hint, can be evicted in favor of VirtReg. -bool DefaultEvictionAdvisor::canEvictHintInterference( - LiveInterval &VirtReg, MCRegister PhysReg, - const SmallVirtRegSet &FixedRegisters) const { - EvictionCost MaxCost; - MaxCost.setBrokenHints(1); - return canEvictInterferenceBasedOnCost(VirtReg, PhysReg, true, MaxCost, - FixedRegisters); -} - -/// canEvictInterferenceBasedOnCost - Return true if all interferences between -/// VirtReg and PhysReg can be evicted. -/// -/// @param VirtReg Live range that is about to be assigned. -/// @param PhysReg Desired register for assignment. -/// @param IsHint True when PhysReg is VirtReg's preferred register. -/// @param MaxCost Only look for cheaper candidates and update with new cost -/// when returning true. -/// @returns True when interference can be evicted cheaper than MaxCost. -bool DefaultEvictionAdvisor::canEvictInterferenceBasedOnCost( - LiveInterval &VirtReg, MCRegister PhysReg, bool IsHint, - EvictionCost &MaxCost, const SmallVirtRegSet &FixedRegisters) const { - // It is only possible to evict virtual register interference. - if (Matrix->checkInterference(VirtReg, PhysReg) > LiveRegMatrix::IK_VirtReg) - return false; - - bool IsLocal = VirtReg.empty() || LIS->intervalIsInOneMBB(VirtReg); - - // Find VirtReg's cascade number. This will be unassigned if VirtReg was never - // involved in an eviction before. If a cascade number was assigned, deny - // evicting anything with the same or a newer cascade number. This prevents - // infinite eviction loops. - // - // This works out so a register without a cascade number is allowed to evict - // anything, and it can be evicted by anything. - unsigned Cascade = RA.getExtraInfo().getCascadeOrCurrentNext(VirtReg.reg()); - - EvictionCost Cost; - for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { - LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); - // If there is 10 or more interferences, chances are one is heavier. - const auto &Interferences = Q.interferingVRegs(10); - if (Interferences.size() >= 10) - return false; - - // Check if any interfering live range is heavier than MaxWeight. - for (LiveInterval *Intf : reverse(Interferences)) { - assert(Register::isVirtualRegister(Intf->reg()) && - "Only expecting virtual register interference from query"); - - // Do not allow eviction of a virtual register if we are in the middle - // of last-chance recoloring and this virtual register is one that we - // have scavenged a physical register for. - if (FixedRegisters.count(Intf->reg())) - return false; - - // Never evict spill products. They cannot split or spill. - if (RA.getExtraInfo().getStage(*Intf) == RS_Done) - return false; - // Once a live range becomes small enough, it is urgent that we find a - // register for it. This is indicated by an infinite spill weight. These - // urgent live ranges get to evict almost anything. - // - // Also allow urgent evictions of unspillable ranges from a strictly - // larger allocation order. - bool Urgent = - !VirtReg.isSpillable() && - (Intf->isSpillable() || - RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(VirtReg.reg())) < - RegClassInfo.getNumAllocatableRegs( - MRI->getRegClass(Intf->reg()))); - // Only evict older cascades or live ranges without a cascade. - unsigned IntfCascade = RA.getExtraInfo().getCascade(Intf->reg()); - if (Cascade <= IntfCascade) { - if (!Urgent) - return false; - // We permit breaking cascades for urgent evictions. It should be the - // last resort, though, so make it really expensive. - Cost.BrokenHints += 10; - } - // Would this break a satisfied hint? - bool BreaksHint = VRM->hasPreferredPhys(Intf->reg()); - // Update eviction cost. - Cost.BrokenHints += BreaksHint; - Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight()); - // Abort if this would be too expensive. - if (!(Cost < MaxCost)) - return false; - if (Urgent) - continue; - // Apply the eviction policy for non-urgent evictions. - if (!shouldEvict(VirtReg, IsHint, *Intf, BreaksHint)) - return false; - // If !MaxCost.isMax(), then we're just looking for a cheap register. - // Evicting another local live range in this case could lead to suboptimal - // coloring. - if (!MaxCost.isMax() && IsLocal && LIS->intervalIsInOneMBB(*Intf) && - (!EnableLocalReassign || !canReassign(*Intf, PhysReg))) { - return false; - } - } - } - MaxCost = Cost; - return true; -} - /// Return true if all interferences between VirtReg and PhysReg between /// Start and End can be evicted. /// @@ -757,44 +620,6 @@ bool RegAllocEvictionAdvisor::canAllocatePhysReg(unsigned CostPerUseLimit, return true; } -MCRegister DefaultEvictionAdvisor::tryFindEvictionCandidate( - LiveInterval &VirtReg, const AllocationOrder &Order, - uint8_t CostPerUseLimit, const SmallVirtRegSet &FixedRegisters) const { - // Keep track of the cheapest interference seen so far. - EvictionCost BestCost; - BestCost.setMax(); - MCRegister BestPhys; - auto MaybeOrderLimit = getOrderLimit(VirtReg, Order, CostPerUseLimit); - if (!MaybeOrderLimit) - return MCRegister::NoRegister; - unsigned OrderLimit = *MaybeOrderLimit; - - // When we are just looking for a reduced cost per use, don't break any - // hints, and only evict smaller spill weights. - if (CostPerUseLimit < uint8_t(~0u)) { - BestCost.BrokenHints = 0; - BestCost.MaxWeight = VirtReg.weight(); - } - - for (auto I = Order.begin(), E = Order.getOrderLimitEnd(OrderLimit); I != E; - ++I) { - MCRegister PhysReg = *I; - assert(PhysReg); - if (!canAllocatePhysReg(CostPerUseLimit, PhysReg) || - !canEvictInterferenceBasedOnCost(VirtReg, PhysReg, false, BestCost, - FixedRegisters)) - continue; - - // Best so far. - BestPhys = PhysReg; - - // Stop if the hint can be used. - if (I.isHint()) - break; - } - return BestPhys; -} - /// tryEvict - Try to evict all interferences for a physreg. /// @param VirtReg Currently unassigned virtual register. /// @param Order Physregs to try. @@ -2922,6 +2747,10 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) { RegCosts = TRI->getRegisterCosts(*MF); + ExtraInfo.emplace(); + EvictAdvisor = + getAnalysis<RegAllocEvictionAdvisorAnalysis>().getAdvisor(*MF, *this); + VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI); SpillerInstance.reset(createInlineSpiller(*this, *MF, *VRM, *VRAI)); @@ -2931,9 +2760,7 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) { SA.reset(new SplitAnalysis(*VRM, *LIS, *Loops)); SE.reset(new SplitEditor(*SA, *AA, *LIS, *VRM, *DomTree, *MBFI, *VRAI)); - ExtraInfo.emplace(); - EvictAdvisor = - getAnalysis<RegAllocEvictionAdvisorAnalysis>().getAdvisor(*MF, *this); + IntfCache.init(MF, Matrix->getLiveUnions(), Indexes, LIS, TRI); GlobalCand.resize(32); // This will grow as needed. SetOfBrokenHints.clear(); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 932f263d2558..041d7e5b4a4a 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -143,7 +143,7 @@ namespace { SelectionDAG &DAG; const TargetLowering &TLI; const SelectionDAGTargetInfo *STI; - CombineLevel Level; + CombineLevel Level = BeforeLegalizeTypes; CodeGenOpt::Level OptLevel; bool LegalDAG = false; bool LegalOperations = false; @@ -238,8 +238,7 @@ namespace { public: DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL) : DAG(D), TLI(D.getTargetLoweringInfo()), - STI(D.getSubtarget().getSelectionDAGInfo()), - Level(BeforeLegalizeTypes), OptLevel(OL), AA(AA) { + STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) { ForCodeSize = DAG.shouldOptForSize(); DisableGenericCombines = STI && STI->disableGenericCombines(OptLevel); @@ -441,6 +440,7 @@ namespace { SDValue visitSRA(SDNode *N); SDValue visitSRL(SDNode *N); SDValue visitFunnelShift(SDNode *N); + SDValue visitSHLSAT(SDNode *N); SDValue visitRotate(SDNode *N); SDValue visitABS(SDNode *N); SDValue visitBSWAP(SDNode *N); @@ -907,9 +907,8 @@ bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS, return true; } - if (N.getOpcode() != ISD::SELECT_CC || - !TLI.isConstTrueVal(N.getOperand(2).getNode()) || - !TLI.isConstFalseVal(N.getOperand(3).getNode())) + if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N.getOperand(2)) || + !TLI.isConstFalseVal(N.getOperand(3))) return false; if (TLI.getBooleanContents(N.getValueType()) == @@ -1654,6 +1653,8 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::ROTL: return visitRotate(N); case ISD::FSHL: case ISD::FSHR: return visitFunnelShift(N); + case ISD::SSHLSAT: + case ISD::USHLSAT: return visitSHLSAT(N); case ISD::ABS: return visitABS(N); case ISD::BSWAP: return visitBSWAP(N); case ISD::BITREVERSE: return visitBITREVERSE(N); @@ -5530,8 +5531,6 @@ bool DAGCombiner::SearchForAndLoads(SDNode *N, // Some constants may need fixing up later if they are too large. if (auto *C = dyn_cast<ConstantSDNode>(Op)) { - if (Mask->getValueType(0) != C->getValueType(0)) - return false; if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) && (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue()) NodesWithConsts.insert(N); @@ -5565,9 +5564,9 @@ bool DAGCombiner::SearchForAndLoads(SDNode *N, case ISD::AssertZext: { unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes(); EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits); - EVT VT = Op.getOpcode() == ISD::AssertZext - ? cast<VTSDNode>(Op.getOperand(1))->getVT() - : Op.getOperand(0).getValueType(); + EVT VT = Op.getOpcode() == ISD::AssertZext ? + cast<VTSDNode>(Op.getOperand(1))->getVT() : + Op.getOperand(0).getValueType(); // We can accept extending nodes if the mask is wider or an equal // width to the original type. @@ -5575,15 +5574,6 @@ bool DAGCombiner::SearchForAndLoads(SDNode *N, continue; break; } - case ISD::ANY_EXTEND: { - unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes(); - EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits); - EVT VT = Op.getOperand(0).getValueType(); - if (ExtVT.bitsGE(VT)) - break; - // Fallthrough to searching for nodes from the operands of the extend. - LLVM_FALLTHROUGH; - } case ISD::OR: case ISD::XOR: case ISD::AND: @@ -5643,14 +5633,12 @@ bool DAGCombiner::BackwardsPropagateMask(SDNode *N) { // masking. if (FixupNode) { LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump()); - SDValue MaskOpT = DAG.getZExtOrTrunc(MaskOp, SDLoc(FixupNode), - FixupNode->getValueType(0)); - SDValue And = - DAG.getNode(ISD::AND, SDLoc(FixupNode), FixupNode->getValueType(0), - SDValue(FixupNode, 0), MaskOpT); + SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode), + FixupNode->getValueType(0), + SDValue(FixupNode, 0), MaskOp); DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And); if (And.getOpcode() == ISD ::AND) - DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOpT); + DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp); } // Narrow any constants that need it. @@ -5659,12 +5647,10 @@ bool DAGCombiner::BackwardsPropagateMask(SDNode *N) { SDValue Op1 = LogicN->getOperand(1); if (isa<ConstantSDNode>(Op0)) - std::swap(Op0, Op1); + std::swap(Op0, Op1); - SDValue MaskOpT = - DAG.getZExtOrTrunc(MaskOp, SDLoc(Op1), Op1.getValueType()); - SDValue And = - DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), Op1, MaskOpT); + SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), + Op1, MaskOp); DAG.UpdateNodeOperands(LogicN, Op0, And); } @@ -5672,14 +5658,12 @@ bool DAGCombiner::BackwardsPropagateMask(SDNode *N) { // Create narrow loads. for (auto *Load : Loads) { LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump()); - SDValue MaskOpT = - DAG.getZExtOrTrunc(MaskOp, SDLoc(Load), Load->getValueType(0)); SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0), - SDValue(Load, 0), MaskOpT); + SDValue(Load, 0), MaskOp); DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And); if (And.getOpcode() == ISD ::AND) And = SDValue( - DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOpT), 0); + DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0); SDValue NewLoad = reduceLoadWidth(And.getNode()); assert(NewLoad && "Shouldn't be masking the load if it can't be narrowed"); @@ -8036,8 +8020,8 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { // fold !(x cc y) -> (x !cc y) unsigned N0Opcode = N0.getOpcode(); SDValue LHS, RHS, CC; - if (TLI.isConstTrueVal(N1.getNode()) && - isSetCCEquivalent(N0, LHS, RHS, CC, /*MatchStrict*/true)) { + if (TLI.isConstTrueVal(N1) && + isSetCCEquivalent(N0, LHS, RHS, CC, /*MatchStrict*/ true)) { ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), LHS.getValueType()); if (!LegalOperations || @@ -9348,6 +9332,22 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) { return SDValue(); } +SDValue DAGCombiner::visitSHLSAT(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + if (SDValue V = DAG.simplifyShift(N0, N1)) + return V; + + EVT VT = N0.getValueType(); + + // fold (*shlsat c1, c2) -> c1<<c2 + if (SDValue C = + DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, {N0, N1})) + return C; + + return SDValue(); +} + // Given a ABS node, detect the following pattern: // (ABS (SUB (EXTEND a), (EXTEND b))). // Generates UABD/SABD instruction. @@ -14580,7 +14580,7 @@ SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) { unsigned NumElts = 1; EVT VT = N->getValueType(0); if (VT.isVector() && DAG.isSplatValue(N1)) - NumElts = VT.getVectorNumElements(); + NumElts = VT.getVectorMinNumElements(); if (!MinUses || (N1->use_size() * NumElts) < MinUses) return SDValue(); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index bfde35935c7b..d8ef79fe9a7b 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1838,8 +1838,7 @@ FastISel::FastISel(FunctionLoweringInfo &FuncInfo, TII(*MF->getSubtarget().getInstrInfo()), TLI(*MF->getSubtarget().getTargetLowering()), TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo), - SkipTargetIndependentISel(SkipTargetIndependentISel), - LastLocalValue(nullptr), EmitStartPt(nullptr) {} + SkipTargetIndependentISel(SkipTargetIndependentISel) {} FastISel::~FastISel() = default; diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index 403f34573899..55f6f288f3e3 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -47,8 +47,7 @@ static cl::opt<int> HighLatencyCycles( "instructions take for targets with no itinerary")); ScheduleDAGSDNodes::ScheduleDAGSDNodes(MachineFunction &mf) - : ScheduleDAG(mf), BB(nullptr), DAG(nullptr), - InstrItins(mf.getSubtarget().getInstrItineraryData()) {} + : ScheduleDAG(mf), InstrItins(mf.getSubtarget().getInstrItineraryData()) {} /// Run - perform scheduling. /// @@ -577,7 +576,7 @@ void ScheduleDAGSDNodes::RegDefIter::InitNodeNumDefs() { // Construct a RegDefIter for this SUnit and find the first valid value. ScheduleDAGSDNodes::RegDefIter::RegDefIter(const SUnit *SU, const ScheduleDAGSDNodes *SD) - : SchedDAG(SD), Node(SU->getNode()), DefIdx(0), NodeNumDefs(0) { + : SchedDAG(SD), Node(SU->getNode()) { InitNodeNumDefs(); Advance(); } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index 8c28ce403c9b..99bbaeb19182 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -45,8 +45,8 @@ class InstrItineraryData; /// class ScheduleDAGSDNodes : public ScheduleDAG { public: - MachineBasicBlock *BB; - SelectionDAG *DAG; // DAG of the current basic block + MachineBasicBlock *BB = nullptr; + SelectionDAG *DAG = nullptr; // DAG of the current basic block const InstrItineraryData *InstrItins; /// The schedule. Null SUnit*'s represent noop instructions. @@ -138,8 +138,8 @@ class InstrItineraryData; class RegDefIter { const ScheduleDAGSDNodes *SchedDAG; const SDNode *Node; - unsigned DefIdx; - unsigned NodeNumDefs; + unsigned DefIdx = 0; + unsigned NodeNumDefs = 0; MVT ValueType; public: diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 45f3005e8f57..d5998d166d25 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2449,7 +2449,7 @@ SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits, switch (V.getOpcode()) { default: return TLI->SimplifyMultipleUseDemandedBits(V, DemandedBits, DemandedElts, - *this, 0); + *this); case ISD::Constant: { const APInt &CVal = cast<ConstantSDNode>(V)->getAPIntValue(); APInt NewVal = CVal & DemandedBits; @@ -3082,6 +3082,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1); + // TODO: SelfMultiply can be poison, but not undef. + SelfMultiply &= isGuaranteedNotToBeUndefOrPoison( + Op.getOperand(0), DemandedElts, false, Depth + 1); Known = KnownBits::mul(Known, Known2, SelfMultiply); break; } @@ -5240,6 +5243,8 @@ static llvm::Optional<APInt> FoldValue(unsigned Opcode, const APInt &C1, case ISD::UADDSAT: return C1.uadd_sat(C2); case ISD::SSUBSAT: return C1.ssub_sat(C2); case ISD::USUBSAT: return C1.usub_sat(C2); + case ISD::SSHLSAT: return C1.sshl_sat(C2); + case ISD::USHLSAT: return C1.ushl_sat(C2); case ISD::UDIV: if (!C2.getBoolValue()) break; diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 41460f78e1c2..01230a36e744 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4014,7 +4014,7 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { Type *Ty = I.getAllocatedType(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); auto &DL = DAG.getDataLayout(); - uint64_t TySize = DL.getTypeAllocSize(Ty); + TypeSize TySize = DL.getTypeAllocSize(Ty); MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign()); SDValue AllocSize = getValue(I.getArraySize()); @@ -4023,9 +4023,15 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { if (AllocSize.getValueType() != IntPtr) AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr); - AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, - AllocSize, - DAG.getConstant(TySize, dl, IntPtr)); + if (TySize.isScalable()) + AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, + DAG.getVScale(dl, IntPtr, + APInt(IntPtr.getScalarSizeInBits(), + TySize.getKnownMinValue()))); + else + AllocSize = + DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, + DAG.getConstant(TySize.getFixedValue(), dl, IntPtr)); // Handle alignment. If the requested alignment is less than or equal to // the stack alignment, ignore it. If the size is greater than or equal to @@ -6870,6 +6876,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::experimental_gc_relocate: visitGCRelocate(cast<GCRelocateInst>(I)); return; + case Intrinsic::instrprof_cover: + llvm_unreachable("instrprof failed to lower a cover"); case Intrinsic::instrprof_increment: llvm_unreachable("instrprof failed to lower an increment"); case Intrinsic::instrprof_value_profile: diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 77e11b364588..3c786904620a 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -319,7 +319,7 @@ SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL) CurDAG(new SelectionDAG(tm, OL)), SDB(std::make_unique<SelectionDAGBuilder>(*CurDAG, *FuncInfo, *SwiftError, OL)), - AA(), GFI(), OptLevel(OL), DAGSize(0) { + OptLevel(OL) { initializeGCModuleInfoPass(*PassRegistry::getPassRegistry()); initializeBranchProbabilityInfoWrapperPassPass( *PassRegistry::getPassRegistry()); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index e2db9633bfb9..dfda7d8b9f81 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -990,6 +990,24 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( return ReturnVal; } +/// Return two gc.results if present. First result is a block local +/// gc.result, second result is a non-block local gc.result. Corresponding +/// entry will be nullptr if not present. +static std::pair<const GCResultInst*, const GCResultInst*> +getGCResultLocality(const GCStatepointInst &S) { + std::pair<const GCResultInst *, const GCResultInst*> Res(nullptr, nullptr); + for (auto *U : S.users()) { + auto *GRI = dyn_cast<GCResultInst>(U); + if (!GRI) + continue; + if (GRI->getParent() == S.getParent()) + Res.first = GRI; + else + Res.second = GRI; + } + return Res; +} + void SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB /*= nullptr*/) { @@ -1075,12 +1093,11 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I, SDValue ReturnValue = LowerAsSTATEPOINT(SI); // Export the result value if needed - const std::pair<bool, bool> GCResultLocality = I.getGCResultLocality(); - Type *RetTy = I.getActualReturnType(); + const auto GCResultLocality = getGCResultLocality(I); - if (RetTy->isVoidTy() || - (!GCResultLocality.first && !GCResultLocality.second)) { - // The return value is not needed, just generate a poison value. + if (!GCResultLocality.first && !GCResultLocality.second) { + // The return value is not needed, just generate a poison value. + // Note: This covers the void return case. setValue(&I, DAG.getIntPtrConstant(-1, getCurSDLoc())); return; } @@ -1102,6 +1119,7 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I, // manually. // TODO: To eliminate this problem we can remove gc.result intrinsics // completely and make statepoint call to return a tuple. + Type *RetTy = GCResultLocality.second->getType(); unsigned Reg = FuncInfo.CreateRegs(RetTy); RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Reg, RetTy, @@ -1168,7 +1186,7 @@ void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) { // register because statepoint and actual call return types can be // different, and getValue() will use CopyFromReg of the wrong type, // which is always i32 in our case. - Type *RetTy = SI->getActualReturnType(); + Type *RetTy = CI.getType(); SDValue CopyFromReg = getCopyFromRegs(SI, RetTy); assert(CopyFromReg.getNode()); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index a98c21f16c71..f6d1fa87676f 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -63,7 +63,7 @@ bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs()); for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable, Attribute::DereferenceableOrNull, Attribute::NoAlias, - Attribute::NonNull}) + Attribute::NonNull, Attribute::NoUndef}) CallerAttrs.removeAttribute(Attr); if (CallerAttrs.hasAttributes()) @@ -606,6 +606,23 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, } bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, + const APInt &DemandedElts, + DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), + !DCI.isBeforeLegalizeOps()); + KnownBits Known; + + bool Simplified = + SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO); + if (Simplified) { + DCI.AddToWorklist(Op.getNode()); + DCI.CommitTargetLoweringOpt(TLO); + } + return Simplified; +} + +bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth, @@ -2247,8 +2264,12 @@ bool TargetLowering::SimplifyDemandedBits( } break; } - case ISD::ADD: case ISD::MUL: + // 'Quadratic Reciprocity': mul(x,x) -> 0 if we're only demanding bit[1] + if (DemandedBits == 2 && Op.getOperand(0) == Op.getOperand(1)) + return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT)); + LLVM_FALLTHROUGH; + case ISD::ADD: case ISD::SUB: { // Add, Sub, and Mul don't demand any bits in positions beyond that // of the highest bit demanded of them. @@ -3173,29 +3194,25 @@ bool TargetLowering::isSplatValueForTargetNode(SDValue Op, // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must // work with truncating build vectors and vectors with elements of less than // 8 bits. -bool TargetLowering::isConstTrueVal(const SDNode *N) const { +bool TargetLowering::isConstTrueVal(SDValue N) const { if (!N) return false; + unsigned EltWidth; APInt CVal; - if (auto *CN = dyn_cast<ConstantSDNode>(N)) { + if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false, + /*AllowTruncation=*/true)) { CVal = CN->getAPIntValue(); - } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) { - auto *CN = BV->getConstantSplatNode(); - if (!CN) - return false; - - // If this is a truncating build vector, truncate the splat value. - // Otherwise, we may fail to match the expected values below. - unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits(); - CVal = CN->getAPIntValue(); - if (BVEltWidth < CVal.getBitWidth()) - CVal = CVal.trunc(BVEltWidth); - } else { + EltWidth = N.getValueType().getScalarSizeInBits(); + } else return false; - } - switch (getBooleanContents(N->getValueType(0))) { + // If this is a truncating splat, truncate the splat value. + // Otherwise, we may fail to match the expected values below. + if (EltWidth < CVal.getBitWidth()) + CVal = CVal.trunc(EltWidth); + + switch (getBooleanContents(N.getValueType())) { case UndefinedBooleanContent: return CVal[0]; case ZeroOrOneBooleanContent: @@ -3207,7 +3224,7 @@ bool TargetLowering::isConstTrueVal(const SDNode *N) const { llvm_unreachable("Invalid boolean contents"); } -bool TargetLowering::isConstFalseVal(const SDNode *N) const { +bool TargetLowering::isConstFalseVal(SDValue N) const { if (!N) return false; @@ -3742,7 +3759,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 && TopSetCC.getOpcode() == ISD::SETCC && (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) && - (isConstFalseVal(N1C) || + (isConstFalseVal(N1) || isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) { bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) || diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SlotIndexes.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SlotIndexes.cpp index c933031ef37d..ffac68a223bf 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SlotIndexes.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SlotIndexes.cpp @@ -20,7 +20,7 @@ using namespace llvm; char SlotIndexes::ID = 0; -SlotIndexes::SlotIndexes() : MachineFunctionPass(ID), mf(nullptr) { +SlotIndexes::SlotIndexes() : MachineFunctionPass(ID) { initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); } |