diff options
Diffstat (limited to 'llvm')
21 files changed, 264 insertions, 90 deletions
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h index a78df16ca404..2901ab715810 100644 --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -605,6 +605,17 @@ unsigned LLVMGetEnumAttributeKind(LLVMAttributeRef A); uint64_t LLVMGetEnumAttributeValue(LLVMAttributeRef A); /** + * Create a type attribute + */ +LLVMAttributeRef LLVMCreateTypeAttribute(LLVMContextRef C, unsigned KindID, + LLVMTypeRef type_ref); + +/** + * Get the type attribute's value. + */ +LLVMTypeRef LLVMGetTypeAttributeValue(LLVMAttributeRef A); + +/** * Create a string attribute. */ LLVMAttributeRef LLVMCreateStringAttribute(LLVMContextRef C, @@ -626,6 +637,7 @@ const char *LLVMGetStringAttributeValue(LLVMAttributeRef A, unsigned *Length); */ LLVMBool LLVMIsEnumAttribute(LLVMAttributeRef A); LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A); +LLVMBool LLVMIsTypeAttribute(LLVMAttributeRef A); /** * Obtain a Type from a context by its registered name. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h index 39ae53c4e7fe..cf4c827b9267 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h @@ -111,6 +111,16 @@ public: return AttributeSpecs[idx].Attr; } + bool getAttrIsImplicitConstByIndex(uint32_t idx) const { + assert(idx < AttributeSpecs.size()); + return AttributeSpecs[idx].isImplicitConst(); + } + + int64_t getAttrImplicitConstValueByIndex(uint32_t idx) const { + assert(idx < AttributeSpecs.size()); + return AttributeSpecs[idx].getImplicitConstValue(); + } + /// Get the index of the specified attribute. /// /// Searches the this abbreviation declaration for the index of the specified diff --git a/llvm/include/llvm/Support/Host.h b/llvm/include/llvm/Support/Host.h index d4ef389450cc..b3c15f0683b9 100644 --- a/llvm/include/llvm/Support/Host.h +++ b/llvm/include/llvm/Support/Host.h @@ -65,6 +65,20 @@ namespace sys { StringRef getHostCPUNameForARM(StringRef ProcCpuinfoContent); StringRef getHostCPUNameForS390x(StringRef ProcCpuinfoContent); StringRef getHostCPUNameForBPF(); + + /// Helper functions to extract CPU details from CPUID on x86. + namespace x86 { + enum class VendorSignatures { + UNKNOWN, + GENUINE_INTEL, + AUTHENTIC_AMD, + }; + + /// Returns the host CPU's vendor. + /// MaxLeaf: if a non-nullptr pointer is specified, the EAX value will be + /// assigned to its pointee. + VendorSignatures getVendorSignature(unsigned *MaxLeaf = nullptr); + } // namespace x86 } } } diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index 2f80b4373b46..246db0fd2dd9 100644 --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -396,22 +396,17 @@ private: bool canVectorizeOuterLoop(); /// Return true if all of the instructions in the block can be speculatively - /// executed, and record the loads/stores that require masking. If's that - /// guard loads can be ignored under "assume safety" unless \p PreserveGuards - /// is true. This can happen when we introduces guards for which the original - /// "unguarded-loads are safe" assumption does not hold. For example, the - /// vectorizer's fold-tail transformation changes the loop to execute beyond - /// its original trip-count, under a proper guard, which should be preserved. + /// executed, and record the loads/stores that require masking. /// \p SafePtrs is a list of addresses that are known to be legal and we know /// that we can read from them without segfault. /// \p MaskedOp is a list of instructions that have to be transformed into /// calls to the appropriate masked intrinsic when the loop is vectorized. /// \p ConditionalAssumes is a list of assume instructions in predicated /// blocks that must be dropped if the CFG gets flattened. - bool blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs, - SmallPtrSetImpl<const Instruction *> &MaskedOp, - SmallPtrSetImpl<Instruction *> &ConditionalAssumes, - bool PreserveGuards = false) const; + bool blockCanBePredicated( + BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs, + SmallPtrSetImpl<const Instruction *> &MaskedOp, + SmallPtrSetImpl<Instruction *> &ConditionalAssumes) const; /// Updates the vectorization state by adding \p Phi to the inductions list. /// This can set \p Phi as the main induction of the loop if \p Phi is a diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp index 427f6f4942c3..5a55f3a04148 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp @@ -69,7 +69,7 @@ static void dumpRanges(const DWARFObject &Obj, raw_ostream &OS, } } -static void dumpLocation(raw_ostream &OS, DWARFFormValue &FormValue, +static void dumpLocation(raw_ostream &OS, const DWARFFormValue &FormValue, DWARFUnit *U, unsigned Indent, DIDumpOptions DumpOpts) { DWARFContext &Ctx = U->getContext(); @@ -230,21 +230,22 @@ static void dumpTypeName(raw_ostream &OS, const DWARFDie &D) { } static void dumpAttribute(raw_ostream &OS, const DWARFDie &Die, - uint64_t *OffsetPtr, dwarf::Attribute Attr, - dwarf::Form Form, unsigned Indent, + const DWARFAttribute &AttrValue, unsigned Indent, DIDumpOptions DumpOpts) { if (!Die.isValid()) return; const char BaseIndent[] = " "; OS << BaseIndent; OS.indent(Indent + 2); + dwarf::Attribute Attr = AttrValue.Attr; WithColor(OS, HighlightColor::Attribute) << formatv("{0}", Attr); + dwarf::Form Form = AttrValue.Value.getForm(); if (DumpOpts.Verbose || DumpOpts.ShowForm) OS << formatv(" [{0}]", Form); DWARFUnit *U = Die.getDwarfUnit(); - DWARFFormValue FormValue = DWARFFormValue::createFromUnit(Form, U, OffsetPtr); + const DWARFFormValue &FormValue = AttrValue.Value; OS << "\t("; @@ -631,16 +632,8 @@ void DWARFDie::dump(raw_ostream &OS, unsigned Indent, OS << '\n'; // Dump all data in the DIE for the attributes. - for (const auto &AttrSpec : AbbrevDecl->attributes()) { - if (AttrSpec.Form == DW_FORM_implicit_const) { - // We are dumping .debug_info section , - // implicit_const attribute values are not really stored here, - // but in .debug_abbrev section. So we just skip such attrs. - continue; - } - dumpAttribute(OS, *this, &offset, AttrSpec.Attr, AttrSpec.Form, - Indent, DumpOpts); - } + for (const DWARFAttribute &AttrValue : attributes()) + dumpAttribute(OS, *this, AttrValue, Indent, DumpOpts); DWARFDie child = getFirstChild(); if (DumpOpts.ShowChildren && DumpOpts.ChildRecurseDepth > 0 && child) { @@ -723,10 +716,16 @@ void DWARFDie::attribute_iterator::updateForIndex( // Add the previous byte size of any previous attribute value. AttrValue.Offset += AttrValue.ByteSize; uint64_t ParseOffset = AttrValue.Offset; - auto U = Die.getDwarfUnit(); - assert(U && "Die must have valid DWARF unit"); - AttrValue.Value = DWARFFormValue::createFromUnit( - AbbrDecl.getFormByIndex(Index), U, &ParseOffset); + if (AbbrDecl.getAttrIsImplicitConstByIndex(Index)) + AttrValue.Value = DWARFFormValue::createFromSValue( + AbbrDecl.getFormByIndex(Index), + AbbrDecl.getAttrImplicitConstValueByIndex(Index)); + else { + auto U = Die.getDwarfUnit(); + assert(U && "Die must have valid DWARF unit"); + AttrValue.Value = DWARFFormValue::createFromUnit( + AbbrDecl.getFormByIndex(Index), U, &ParseOffset); + } AttrValue.ByteSize = ParseOffset - AttrValue.Offset; } else { assert(Index == NumAttrs && "Indexes should be [0, NumAttrs) only"); diff --git a/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp b/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp index 7a84605211fb..2559765876d9 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFFormValue.cpp @@ -168,6 +168,7 @@ bool DWARFFormValue::skipValue(dwarf::Form Form, DataExtractor DebugInfoData, case DW_FORM_line_strp: case DW_FORM_GNU_ref_alt: case DW_FORM_GNU_strp_alt: + case DW_FORM_implicit_const: if (Optional<uint8_t> FixedSize = dwarf::getFixedFormByteSize(Form, Params)) { *OffsetPtr += *FixedSize; @@ -345,6 +346,9 @@ bool DWARFFormValue::extractValue(const DWARFDataExtractor &Data, case DW_FORM_ref_sig8: Value.uval = Data.getU64(OffsetPtr, &Err); break; + case DW_FORM_implicit_const: + // Value has been already set by DWARFFormValue::createFromSValue. + break; default: // DWARFFormValue::skipValue() will have caught this and caused all // DWARF DIEs to fail to be parsed, so this code is not be reachable. @@ -482,6 +486,7 @@ void DWARFFormValue::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const { break; case DW_FORM_sdata: + case DW_FORM_implicit_const: OS << Value.sval; break; case DW_FORM_udata: diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index 90ba69069bae..039b34ace6ab 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -164,6 +164,18 @@ uint64_t LLVMGetEnumAttributeValue(LLVMAttributeRef A) { return Attr.getValueAsInt(); } +LLVMAttributeRef LLVMCreateTypeAttribute(LLVMContextRef C, unsigned KindID, + LLVMTypeRef type_ref) { + auto &Ctx = *unwrap(C); + auto AttrKind = (Attribute::AttrKind)KindID; + return wrap(Attribute::get(Ctx, AttrKind, unwrap(type_ref))); +} + +LLVMTypeRef LLVMGetTypeAttributeValue(LLVMAttributeRef A) { + auto Attr = unwrap(A); + return wrap(Attr.getValueAsType()); +} + LLVMAttributeRef LLVMCreateStringAttribute(LLVMContextRef C, const char *K, unsigned KLength, const char *V, unsigned VLength) { @@ -194,6 +206,10 @@ LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A) { return unwrap(A).isStringAttribute(); } +LLVMBool LLVMIsTypeAttribute(LLVMAttributeRef A) { + return unwrap(A).isTypeAttribute(); +} + char *LLVMGetDiagInfoDescription(LLVMDiagnosticInfoRef DI) { std::string MsgStorage; raw_string_ostream Stream(MsgStorage); diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp index a1bd3cc12f1d..09146c47ff2c 100644 --- a/llvm/lib/Support/Host.cpp +++ b/llvm/lib/Support/Host.cpp @@ -417,11 +417,6 @@ StringRef sys::detail::getHostCPUNameForBPF() { #if defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64__) || defined(_M_X64) -enum VendorSignatures { - SIG_INTEL = 0x756e6547 /* Genu */, - SIG_AMD = 0x68747541 /* Auth */ -}; - // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max). // Check motivated by bug reports for OpenSSL crashing on CPUs without CPUID // support. Consequently, for i386, the presence of CPUID is checked first @@ -495,6 +490,42 @@ static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, #endif } +namespace llvm { +namespace sys { +namespace detail { +namespace x86 { + +VendorSignatures getVendorSignature(unsigned *MaxLeaf) { + unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0; + if (MaxLeaf == nullptr) + MaxLeaf = &EAX; + else + *MaxLeaf = 0; + + if (!isCpuIdSupported()) + return VendorSignatures::UNKNOWN; + + if (getX86CpuIDAndInfo(0, MaxLeaf, &EBX, &ECX, &EDX) || *MaxLeaf < 1) + return VendorSignatures::UNKNOWN; + + // "Genu ineI ntel" + if (EBX == 0x756e6547 && EDX == 0x49656e69 && ECX == 0x6c65746e) + return VendorSignatures::GENUINE_INTEL; + + // "Auth enti cAMD" + if (EBX == 0x68747541 && EDX == 0x69746e65 && ECX == 0x444d4163) + return VendorSignatures::AUTHENTIC_AMD; + + return VendorSignatures::UNKNOWN; +} + +} // namespace x86 +} // namespace detail +} // namespace sys +} // namespace llvm + +using namespace llvm::sys::detail::x86; + /// getX86CpuIDAndInfoEx - Execute the specified cpuid with subleaf and return /// the 4 values in the specified arguments. If we can't run cpuid on the host, /// return true. @@ -1092,14 +1123,12 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, } StringRef sys::getHostCPUName() { - unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0; - unsigned MaxLeaf, Vendor; - - if (!isCpuIdSupported()) + unsigned MaxLeaf = 0; + const VendorSignatures Vendor = getVendorSignature(&MaxLeaf); + if (Vendor == VendorSignatures::UNKNOWN) return "generic"; - if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) - return "generic"; + unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0; getX86CpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX); unsigned Family = 0, Model = 0; @@ -1114,10 +1143,10 @@ StringRef sys::getHostCPUName() { StringRef CPU; - if (Vendor == SIG_INTEL) { + if (Vendor == VendorSignatures::GENUINE_INTEL) { CPU = getIntelProcessorTypeAndSubtype(Family, Model, Features, &Type, &Subtype); - } else if (Vendor == SIG_AMD) { + } else if (Vendor == VendorSignatures::AUTHENTIC_AMD) { CPU = getAMDProcessorTypeAndSubtype(Family, Model, Features, &Type, &Subtype); } @@ -1219,6 +1248,19 @@ StringRef sys::getHostCPUName() { } #else StringRef sys::getHostCPUName() { return "generic"; } +namespace llvm { +namespace sys { +namespace detail { +namespace x86 { + +VendorSignatures getVendorSignature(unsigned *MaxLeaf) { + return VendorSignatures::UNKNOWN; +} + +} // namespace x86 +} // namespace detail +} // namespace sys +} // namespace llvm #endif #if defined(__linux__) && (defined(__i386__) || defined(__x86_64__)) diff --git a/llvm/lib/Target/ARM/ARMBlockPlacement.cpp b/llvm/lib/Target/ARM/ARMBlockPlacement.cpp index 581b4b9857af..9ba16003a97a 100644 --- a/llvm/lib/Target/ARM/ARMBlockPlacement.cpp +++ b/llvm/lib/Target/ARM/ARMBlockPlacement.cpp @@ -145,8 +145,7 @@ bool ARMBlockPlacement::runOnMachineFunction(MachineFunction &MF) { It++) { MachineBasicBlock *MBB = &*It; for (auto &Terminator : MBB->terminators()) { - if (Terminator.getOpcode() != ARM::t2LoopEnd && - Terminator.getOpcode() != ARM::t2LoopEndDec) + if (Terminator.getOpcode() != ARM::t2LoopEndDec) continue; MachineBasicBlock *LETarget = Terminator.getOperand(2).getMBB(); // The LE will become forwards branching if it branches to LoopExit @@ -204,10 +203,8 @@ void ARMBlockPlacement::moveBasicBlock(MachineBasicBlock *BB, if (!Terminator.isUnconditionalBranch()) { // The BB doesn't have an unconditional branch so it relied on // fall-through. Fix by adding an unconditional branch to the moved BB. - unsigned BrOpc = - BBUtils->isBBInRange(&Terminator, To, 254) ? ARM::tB : ARM::t2B; MachineInstrBuilder MIB = - BuildMI(From, Terminator.getDebugLoc(), TII->get(BrOpc)); + BuildMI(From, Terminator.getDebugLoc(), TII->get(ARM::t2B)); MIB.addMBB(To); MIB.addImm(ARMCC::CondCodes::AL); MIB.addReg(ARM::NoRegister); diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp index 61a924078f29..8dc532058492 100644 --- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp +++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp @@ -1467,14 +1467,15 @@ MachineInstr* ARMLowOverheadLoops::ExpandLoopStart(LowOverheadLoop &LoLoop) { void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) { auto RemovePredicate = [](MachineInstr *MI) { + if (MI->isDebugInstr()) + return; LLVM_DEBUG(dbgs() << "ARM Loops: Removing predicate from: " << *MI); - if (int PIdx = llvm::findFirstVPTPredOperandIdx(*MI)) { - assert(MI->getOperand(PIdx).getImm() == ARMVCC::Then && - "Expected Then predicate!"); - MI->getOperand(PIdx).setImm(ARMVCC::None); - MI->getOperand(PIdx+1).setReg(0); - } else - llvm_unreachable("trying to unpredicate a non-predicated instruction"); + int PIdx = llvm::findFirstVPTPredOperandIdx(*MI); + assert(PIdx >= 1 && "Trying to unpredicate a non-predicated instruction"); + assert(MI->getOperand(PIdx).getImm() == ARMVCC::Then && + "Expected Then predicate!"); + MI->getOperand(PIdx).setImm(ARMVCC::None); + MI->getOperand(PIdx + 1).setReg(0); }; for (auto &Block : LoLoop.getVPTBlocks()) { @@ -1518,8 +1519,13 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) { // - Insert a new vpst to predicate the instruction(s) that following // the divergent vpr def. MachineInstr *Divergent = VPTState::getDivergent(Block); + MachineBasicBlock *MBB = Divergent->getParent(); auto DivergentNext = ++MachineBasicBlock::iterator(Divergent); + while (DivergentNext != MBB->end() && DivergentNext->isDebugInstr()) + ++DivergentNext; + bool DivergentNextIsPredicated = + DivergentNext != MBB->end() && getVPTInstrPredicate(*DivergentNext) != ARMVCC::None; for (auto I = ++MachineBasicBlock::iterator(VPST), E = DivergentNext; diff --git a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp index 81f113b8302f..56823735e2d9 100644 --- a/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp +++ b/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp @@ -960,7 +960,8 @@ bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB, // Get the value that is added to/multiplied with the phi Value *OffsSecondOperand = Offs->getOperand(OffsSecondOp); - if (IncrementPerRound->getType() != OffsSecondOperand->getType()) + if (IncrementPerRound->getType() != OffsSecondOperand->getType() || + !L->isLoopInvariant(OffsSecondOperand)) // Something has gone wrong, abort return false; @@ -1165,6 +1166,8 @@ bool MVEGatherScatterLowering::runOnFunction(Function &F) { bool Changed = false; for (BasicBlock &BB : F) { + Changed |= SimplifyInstructionsInBlock(&BB); + for (Instruction &I : BB) { IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I); if (II && II->getIntrinsicID() == Intrinsic::masked_gather && diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp index b705208660df..cccac5595288 100644 --- a/llvm/lib/Target/ARM/MVETailPredication.cpp +++ b/llvm/lib/Target/ARM/MVETailPredication.cpp @@ -205,6 +205,10 @@ bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask, EnableTailPredication == TailPredication::ForceEnabled; Value *ElemCount = ActiveLaneMask->getOperand(1); + bool Changed = false; + if (!L->makeLoopInvariant(ElemCount, Changed)) + return false; + auto *EC= SE->getSCEV(ElemCount); auto *TC = SE->getSCEV(TripCount); int VectorWidth = diff --git a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp index 9a710b784fd1..c7f451cba14f 100644 --- a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp +++ b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp @@ -107,6 +107,12 @@ static bool StepOverPredicatedInstrs(MachineBasicBlock::instr_iterator &Iter, NumInstrsSteppedOver = 0; while (Iter != EndIter) { + if (Iter->isDebugInstr()) { + // Skip debug instructions + ++Iter; + continue; + } + NextPred = getVPTInstrPredicate(*Iter, PredReg); assert(NextPred != ARMVCC::Else && "VPT block pass does not expect Else preds"); @@ -170,6 +176,8 @@ CreateVPTBlock(MachineBasicBlock::instr_iterator &Iter, LLVM_DEBUG(for (MachineBasicBlock::instr_iterator AddedInstIter = std::next(BlockBeg); AddedInstIter != Iter; ++AddedInstIter) { + if (AddedInstIter->isDebugInstr()) + continue; dbgs() << " adding: "; AddedInstIter->dump(); }); @@ -197,7 +205,7 @@ CreateVPTBlock(MachineBasicBlock::instr_iterator &Iter, if (!IsVPRDefinedOrKilledByBlock(Iter, VPNOTBlockEndIter)) break; - LLVM_DEBUG(dbgs() << " removing VPNOT: "; Iter->dump();); + LLVM_DEBUG(dbgs() << " removing VPNOT: "; Iter->dump()); // Record the new size of the block BlockSize += ElseInstCnt; @@ -211,6 +219,9 @@ CreateVPTBlock(MachineBasicBlock::instr_iterator &Iter, // Note that we are using "Iter" to iterate over the block so we can update // it at the same time. for (; Iter != VPNOTBlockEndIter; ++Iter) { + if (Iter->isDebugInstr()) + continue; + // Find the register in which the predicate is int OpIdx = findFirstVPTPredOperandIdx(*Iter); assert(OpIdx != -1); diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 7833bfc1d1b6..26dc3afc899e 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -15154,17 +15154,38 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, return std::make_pair(0U, &PPC::LRRCRegClass); } - // If we name a VSX register, we can't defer to the base class because it - // will not recognize the correct register (their names will be VSL{0-31} - // and V{0-31} so they won't match). So we match them here. - if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') { - int VSNum = atoi(Constraint.data() + 3); - assert(VSNum >= 0 && VSNum <= 63 && - "Attempted to access a vsr out of range"); - if (VSNum < 32) - return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass); - return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass); + // Handle special cases of physical registers that are not properly handled + // by the base class. + if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') { + // If we name a VSX register, we can't defer to the base class because it + // will not recognize the correct register (their names will be VSL{0-31} + // and V{0-31} so they won't match). So we match them here. + if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') { + int VSNum = atoi(Constraint.data() + 3); + assert(VSNum >= 0 && VSNum <= 63 && + "Attempted to access a vsr out of range"); + if (VSNum < 32) + return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass); + return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass); + } + + // For float registers, we can't defer to the base class as it will match + // the SPILLTOVSRRC class. + if (Constraint.size() > 3 && Constraint[1] == 'f') { + int RegNum = atoi(Constraint.data() + 2); + if (RegNum > 31 || RegNum < 0) + report_fatal_error("Invalid floating point register number"); + if (VT == MVT::f32 || VT == MVT::i32) + return Subtarget.hasSPE() + ? std::make_pair(PPC::R0 + RegNum, &PPC::GPRCRegClass) + : std::make_pair(PPC::F0 + RegNum, &PPC::F4RCRegClass); + if (VT == MVT::f64 || VT == MVT::i64) + return Subtarget.hasSPE() + ? std::make_pair(PPC::S0 + RegNum, &PPC::SPERCRegClass) + : std::make_pair(PPC::F0 + RegNum, &PPC::F8RCRegClass); + } } + std::pair<unsigned, const TargetRegisterClass *> R = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index 9ace36f344a5..270134d84c61 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -1550,6 +1550,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, bool IsVarArg = CLI.IsVarArg; MachineFunction &MF = DAG.getMachineFunction(); EVT PtrVT = getPointerTy(MF.getDataLayout()); + LLVMContext &Ctx = *DAG.getContext(); // Detect unsupported vector argument and return types. if (Subtarget.hasVector()) { @@ -1559,7 +1560,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, // Analyze the operands of the call, assigning locations to each operand. SmallVector<CCValAssign, 16> ArgLocs; - SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); + SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx); ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); // We don't support GuaranteedTailCallOpt, only automatically-detected @@ -1584,14 +1585,25 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, if (VA.getLocInfo() == CCValAssign::Indirect) { // Store the argument in a stack slot and pass its address. - SDValue SpillSlot = DAG.CreateStackTemporary(Outs[I].ArgVT); + unsigned ArgIndex = Outs[I].OrigArgIndex; + EVT SlotVT; + if (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) { + // Allocate the full stack space for a promoted (and split) argument. + Type *OrigArgType = CLI.Args[Outs[I].OrigArgIndex].Ty; + EVT OrigArgVT = getValueType(MF.getDataLayout(), OrigArgType); + MVT PartVT = getRegisterTypeForCallingConv(Ctx, CLI.CallConv, OrigArgVT); + unsigned N = getNumRegistersForCallingConv(Ctx, CLI.CallConv, OrigArgVT); + SlotVT = EVT::getIntegerVT(Ctx, PartVT.getSizeInBits() * N); + } else { + SlotVT = Outs[I].ArgVT; + } + SDValue SpillSlot = DAG.CreateStackTemporary(SlotVT); int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); MemOpChains.push_back( DAG.getStore(Chain, DL, ArgValue, SpillSlot, MachinePointerInfo::getFixedStack(MF, FI))); // If the original argument was split (e.g. i128), we need // to store all parts of it here (and pass just one address). - unsigned ArgIndex = Outs[I].OrigArgIndex; assert (Outs[I].PartOffset == 0); while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) { SDValue PartValue = OutVals[I + 1]; @@ -1601,6 +1613,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, MemOpChains.push_back( DAG.getStore(Chain, DL, PartValue, Address, MachinePointerInfo::getFixedStack(MF, FI))); + assert((PartOffset + PartValue.getValueType().getStoreSize() <= + SlotVT.getStoreSize()) && "Not enough space for argument part!"); ++I; } ArgValue = SpillSlot; @@ -1694,7 +1708,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, // Assign locations to each value returned by this call. SmallVector<CCValAssign, 16> RetLocs; - CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); + CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx); RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); // Copy all of the result registers out of their specified physreg. diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp index 1660e41ba830..5e5e513cdfda 100644 --- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp @@ -149,6 +149,7 @@ bool Lowerer::lowerEarlyIntrinsics(Function &F) { bool Changed = false; CoroIdInst *CoroId = nullptr; SmallVector<CoroFreeInst *, 4> CoroFrees; + bool HasCoroSuspend = false; for (auto IB = inst_begin(F), IE = inst_end(F); IB != IE;) { Instruction &I = *IB++; if (auto *CB = dyn_cast<CallBase>(&I)) { @@ -163,6 +164,7 @@ bool Lowerer::lowerEarlyIntrinsics(Function &F) { // pass expects that there is at most one final suspend point. if (cast<CoroSuspendInst>(&I)->isFinal()) CB->setCannotDuplicate(); + HasCoroSuspend = true; break; case Intrinsic::coro_end_async: case Intrinsic::coro_end: @@ -213,6 +215,13 @@ bool Lowerer::lowerEarlyIntrinsics(Function &F) { if (CoroId) for (CoroFreeInst *CF : CoroFrees) CF->setArgOperand(0, CoroId); + // Coroutine suspention could potentially lead to any argument modified + // outside of the function, hence arguments should not have noalias + // attributes. + if (HasCoroSuspend) + for (Argument &A : F.args()) + if (A.hasNoAliasAttr()) + A.removeAttr(Attribute::NoAlias); return Changed; } diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp index e53e7605b254..e1e0d50979dc 100644 --- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp @@ -781,7 +781,13 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape, PromiseAlloca, DenseMap<Instruction *, llvm::Optional<APInt>>{}, false); // Create an entry for every spilled value. for (auto &S : FrameData.Spills) { - FieldIDType Id = B.addField(S.first->getType(), None); + Type *FieldType = S.first->getType(); + // For byval arguments, we need to store the pointed value in the frame, + // instead of the pointer itself. + if (const Argument *A = dyn_cast<Argument>(S.first)) + if (A->hasByValAttr()) + FieldType = FieldType->getPointerElementType(); + FieldIDType Id = B.addField(FieldType, None); FrameData.setFieldIndex(S.first, Id); } @@ -1149,6 +1155,7 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData, // Create a store instruction storing the value into the // coroutine frame. Instruction *InsertPt = nullptr; + bool NeedToCopyArgPtrValue = false; if (auto *Arg = dyn_cast<Argument>(Def)) { // For arguments, we will place the store instruction right after // the coroutine frame pointer instruction, i.e. bitcast of @@ -1159,6 +1166,9 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData, // from the coroutine function. Arg->getParent()->removeParamAttr(Arg->getArgNo(), Attribute::NoCapture); + if (Arg->hasByValAttr()) + NeedToCopyArgPtrValue = true; + } else if (auto *CSI = dyn_cast<AnyCoroSuspendInst>(Def)) { // Don't spill immediately after a suspend; splitting assumes // that the suspend will be followed by a branch. @@ -1193,7 +1203,15 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData, Builder.SetInsertPoint(InsertPt); auto *G = Builder.CreateConstInBoundsGEP2_32( FrameTy, FramePtr, 0, Index, Def->getName() + Twine(".spill.addr")); - Builder.CreateStore(Def, G); + if (NeedToCopyArgPtrValue) { + // For byval arguments, we need to store the pointed value in the frame, + // instead of the pointer itself. + auto *Value = + Builder.CreateLoad(Def->getType()->getPointerElementType(), Def); + Builder.CreateStore(Value, G); + } else { + Builder.CreateStore(Def, G); + } BasicBlock *CurrentBlock = nullptr; Value *CurrentReload = nullptr; @@ -1207,9 +1225,12 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData, auto *GEP = GetFramePointer(E.first); GEP->setName(E.first->getName() + Twine(".reload.addr")); - CurrentReload = Builder.CreateLoad( - FrameTy->getElementType(FrameData.getFieldIndex(E.first)), GEP, - E.first->getName() + Twine(".reload")); + if (NeedToCopyArgPtrValue) + CurrentReload = GEP; + else + CurrentReload = Builder.CreateLoad( + FrameTy->getElementType(FrameData.getFieldIndex(E.first)), GEP, + E.first->getName() + Twine(".reload")); TinyPtrVector<DbgDeclareInst *> DIs = FindDbgDeclareUses(Def); for (DbgDeclareInst *DDI : DIs) { diff --git a/llvm/lib/Transforms/IPO/ConstantMerge.cpp b/llvm/lib/Transforms/IPO/ConstantMerge.cpp index 67f1438b9b6a..8e81f4bad4af 100644 --- a/llvm/lib/Transforms/IPO/ConstantMerge.cpp +++ b/llvm/lib/Transforms/IPO/ConstantMerge.cpp @@ -95,6 +95,8 @@ isUnmergeableGlobal(GlobalVariable *GV, // Only process constants with initializers in the default address space. return !GV->isConstant() || !GV->hasDefinitiveInitializer() || GV->getType()->getAddressSpace() != 0 || GV->hasSection() || + // Don't touch thread-local variables. + GV->isThreadLocal() || // Don't touch values marked with attribute(used). UsedGlobals.count(GV); } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 2ab0848193f6..b8c21a0e1cd3 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -925,10 +925,7 @@ bool LoopVectorizationLegality::blockNeedsPredication(BasicBlock *BB) { bool LoopVectorizationLegality::blockCanBePredicated( BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs, SmallPtrSetImpl<const Instruction *> &MaskedOp, - SmallPtrSetImpl<Instruction *> &ConditionalAssumes, - bool PreserveGuards) const { - const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel(); - + SmallPtrSetImpl<Instruction *> &ConditionalAssumes) const { for (Instruction &I : *BB) { // Check that we don't have a constant expression that can trap as operand. for (Value *Operand : I.operands()) { @@ -956,11 +953,7 @@ bool LoopVectorizationLegality::blockCanBePredicated( if (!LI) return false; if (!SafePtrs.count(LI->getPointerOperand())) { - // !llvm.mem.parallel_loop_access implies if-conversion safety. - // Otherwise, record that the load needs (real or emulated) masking - // and let the cost model decide. - if (!IsAnnotatedParallel || PreserveGuards) - MaskedOp.insert(LI); + MaskedOp.insert(LI); continue; } } @@ -1276,8 +1269,7 @@ bool LoopVectorizationLegality::prepareToFoldTailByMasking() { // do not need predication such as the header block. for (BasicBlock *BB : TheLoop->blocks()) { if (!blockCanBePredicated(BB, SafePointers, TmpMaskedOp, - TmpConditionalAssumes, - /* MaskAllLoads= */ true)) { + TmpConditionalAssumes)) { LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking as requested.\n"); return false; } diff --git a/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp b/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp index 4a0a86168908..10e059adeb7d 100644 --- a/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp +++ b/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp @@ -100,11 +100,7 @@ FunctionCoverageSummary::get(const InstantiationGroup &Group, for (const auto &FCS : Summaries.drop_front()) { Summary.RegionCoverage.merge(FCS.RegionCoverage); Summary.LineCoverage.merge(FCS.LineCoverage); - - // Sum branch coverage across instantiation groups for the summary rather - // than "merge" the maximum count. This is a clearer view into whether all - // created branches are covered. - Summary.BranchCoverage += FCS.BranchCoverage; + Summary.BranchCoverage.merge(FCS.BranchCoverage); } return Summary; } diff --git a/llvm/tools/llvm-cov/CoverageSummaryInfo.h b/llvm/tools/llvm-cov/CoverageSummaryInfo.h index 4bc1c24a079f..62e7cad1012b 100644 --- a/llvm/tools/llvm-cov/CoverageSummaryInfo.h +++ b/llvm/tools/llvm-cov/CoverageSummaryInfo.h @@ -123,6 +123,11 @@ public: return *this; } + void merge(const BranchCoverageInfo &RHS) { + Covered = std::max(Covered, RHS.Covered); + NumBranches = std::max(NumBranches, RHS.NumBranches); + } + size_t getCovered() const { return Covered; } size_t getNumBranches() const { return NumBranches; } |