diff options
Diffstat (limited to 'llvm/lib/Transforms')
20 files changed, 377 insertions, 213 deletions
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp index 1fd8b88dd776..35adaa3bde65 100644 --- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp +++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp @@ -31,6 +31,7 @@ #include "llvm/IR/PatternMatch.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" +#include "llvm/Transforms/Utils/BuildLibCalls.h" #include "llvm/Transforms/Utils/Local.h" using namespace llvm; @@ -427,27 +428,73 @@ static bool tryToFPToSat(Instruction &I, TargetTransformInfo &TTI) { return true; } +/// Try to replace a mathlib call to sqrt with the LLVM intrinsic. This avoids +/// pessimistic codegen that has to account for setting errno and can enable +/// vectorization. +static bool +foldSqrt(Instruction &I, TargetTransformInfo &TTI, TargetLibraryInfo &TLI) { + // Match a call to sqrt mathlib function. + auto *Call = dyn_cast<CallInst>(&I); + if (!Call) + return false; + + Module *M = Call->getModule(); + LibFunc Func; + if (!TLI.getLibFunc(*Call, Func) || !isLibFuncEmittable(M, &TLI, Func)) + return false; + + if (Func != LibFunc_sqrt && Func != LibFunc_sqrtf && Func != LibFunc_sqrtl) + return false; + + // If (1) this is a sqrt libcall, (2) we can assume that NAN is not created, + // and (3) we would not end up lowering to a libcall anyway (which could + // change the value of errno), then: + // (1) the operand arg must not be less than -0.0. + // (2) errno won't be set. + // (3) it is safe to convert this to an intrinsic call. + // TODO: Check if the arg is known non-negative. + Type *Ty = Call->getType(); + if (TTI.haveFastSqrt(Ty) && Call->hasNoNaNs()) { + IRBuilder<> Builder(&I); + IRBuilderBase::FastMathFlagGuard Guard(Builder); + Builder.setFastMathFlags(Call->getFastMathFlags()); + + Function *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::sqrt, Ty); + Value *NewSqrt = Builder.CreateCall(Sqrt, Call->getArgOperand(0), "sqrt"); + I.replaceAllUsesWith(NewSqrt); + + // Explicitly erase the old call because a call with side effects is not + // trivially dead. + I.eraseFromParent(); + return true; + } + + return false; +} + /// This is the entry point for folds that could be implemented in regular /// InstCombine, but they are separated because they are not expected to /// occur frequently and/or have more than a constant-length pattern match. static bool foldUnusualPatterns(Function &F, DominatorTree &DT, - TargetTransformInfo &TTI) { + TargetTransformInfo &TTI, + TargetLibraryInfo &TLI) { bool MadeChange = false; for (BasicBlock &BB : F) { // Ignore unreachable basic blocks. if (!DT.isReachableFromEntry(&BB)) continue; - // Do not delete instructions under here and invalidate the iterator. + // Walk the block backwards for efficiency. We're matching a chain of // use->defs, so we're more likely to succeed by starting from the bottom. // Also, we want to avoid matching partial patterns. // TODO: It would be more efficient if we removed dead instructions // iteratively in this loop rather than waiting until the end. - for (Instruction &I : llvm::reverse(BB)) { + for (Instruction &I : make_early_inc_range(llvm::reverse(BB))) { MadeChange |= foldAnyOrAllBitsSet(I); MadeChange |= foldGuardedFunnelShift(I, DT); MadeChange |= tryToRecognizePopCount(I); MadeChange |= tryToFPToSat(I, TTI); + MadeChange |= foldSqrt(I, TTI, TLI); } } @@ -467,7 +514,7 @@ static bool runImpl(Function &F, AssumptionCache &AC, TargetTransformInfo &TTI, const DataLayout &DL = F.getParent()->getDataLayout(); TruncInstCombine TIC(AC, TLI, DL, DT); MadeChange |= TIC.run(F); - MadeChange |= foldUnusualPatterns(F, DT, TTI); + MadeChange |= foldUnusualPatterns(F, DT, TTI, TLI); return MadeChange; } diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp index 62cfc3294968..8c77b6937737 100644 --- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -249,7 +249,8 @@ doPromotion(Function *F, FunctionAnalysisManager &FAM, {LLVMContext::MD_range, LLVMContext::MD_nonnull, LLVMContext::MD_dereferenceable, LLVMContext::MD_dereferenceable_or_null, - LLVMContext::MD_align, LLVMContext::MD_noundef}); + LLVMContext::MD_align, LLVMContext::MD_noundef, + LLVMContext::MD_nontemporal}); } Args.push_back(LI); ArgAttrVec.push_back(AttributeSet()); @@ -631,8 +632,7 @@ static bool findArgParts(Argument *Arg, const DataLayout &DL, AAResults &AAR, // Sort parts by offset. append_range(ArgPartsVec, ArgParts); - sort(ArgPartsVec, - [](const auto &A, const auto &B) { return A.first < B.first; }); + sort(ArgPartsVec, llvm::less_first()); // Make sure the parts are non-overlapping. int64_t Offset = ArgPartsVec[0].first; diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 660ff3ee9563..83252fec3ea8 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -3328,7 +3328,7 @@ struct AANoAliasReturned final : AANoAliasImpl { } /// See AbstractAttribute::updateImpl(...). - virtual ChangeStatus updateImpl(Attributor &A) override { + ChangeStatus updateImpl(Attributor &A) override { auto CheckReturnValue = [&](Value &RV) -> bool { if (Constant *C = dyn_cast<Constant>(&RV)) @@ -3427,7 +3427,7 @@ struct AAIsDeadValueImpl : public AAIsDead { } /// See AbstractAttribute::getAsStr(). - virtual const std::string getAsStr() const override { + const std::string getAsStr() const override { return isAssumedDead() ? "assumed-dead" : "assumed-live"; } @@ -4500,9 +4500,8 @@ struct AAAlignImpl : AAAlign { // to avoid making the alignment explicit if it did not improve. /// See AbstractAttribute::getDeducedAttributes - virtual void - getDeducedAttributes(LLVMContext &Ctx, - SmallVectorImpl<Attribute> &Attrs) const override { + void getDeducedAttributes(LLVMContext &Ctx, + SmallVectorImpl<Attribute> &Attrs) const override { if (getAssumedAlign() > 1) Attrs.emplace_back( Attribute::getWithAlignment(Ctx, Align(getAssumedAlign()))); @@ -4709,7 +4708,7 @@ struct AANoReturnImpl : public AANoReturn { } /// See AbstractAttribute::updateImpl(Attributor &A). - virtual ChangeStatus updateImpl(Attributor &A) override { + ChangeStatus updateImpl(Attributor &A) override { auto CheckForNoReturn = [](Instruction &) { return false; }; bool UsedAssumedInformation = false; if (!A.checkForAllInstructions(CheckForNoReturn, *this, @@ -4972,9 +4971,8 @@ struct AANoCaptureImpl : public AANoCapture { ChangeStatus updateImpl(Attributor &A) override; /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...). - virtual void - getDeducedAttributes(LLVMContext &Ctx, - SmallVectorImpl<Attribute> &Attrs) const override { + void getDeducedAttributes(LLVMContext &Ctx, + SmallVectorImpl<Attribute> &Attrs) const override { if (!isAssumedNoCaptureMaybeReturned()) return; @@ -6848,7 +6846,7 @@ struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl { : AAPrivatizablePtrImpl(IRP, A) {} /// See AbstractAttribute::initialize(...). - virtual void initialize(Attributor &A) override { + void initialize(Attributor &A) override { // TODO: We can privatize more than arguments. indicatePessimisticFixpoint(); } @@ -7222,7 +7220,7 @@ struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl { : AAMemoryBehaviorImpl(IRP, A) {} /// See AbstractAttribute::updateImpl(Attributor &A). - virtual ChangeStatus updateImpl(Attributor &A) override; + ChangeStatus updateImpl(Attributor &A) override; /// See AbstractAttribute::manifest(...). ChangeStatus manifest(Attributor &A) override { @@ -7934,7 +7932,7 @@ struct AAMemoryLocationFunction final : public AAMemoryLocationImpl { : AAMemoryLocationImpl(IRP, A) {} /// See AbstractAttribute::updateImpl(Attributor &A). - virtual ChangeStatus updateImpl(Attributor &A) override { + ChangeStatus updateImpl(Attributor &A) override { const auto &MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE); @@ -9332,13 +9330,13 @@ struct AANoUndefCallSiteReturned final struct AACallEdgesImpl : public AACallEdges { AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {} - virtual const SetVector<Function *> &getOptimisticEdges() const override { + const SetVector<Function *> &getOptimisticEdges() const override { return CalledFunctions; } - virtual bool hasUnknownCallee() const override { return HasUnknownCallee; } + bool hasUnknownCallee() const override { return HasUnknownCallee; } - virtual bool hasNonAsmUnknownCallee() const override { + bool hasNonAsmUnknownCallee() const override { return HasUnknownCalleeNonAsm; } diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp index 49077f92884f..50710eaa1b57 100644 --- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp +++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp @@ -931,10 +931,9 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes, // a value can't capture arguments. Don't analyze them. if (F->onlyReadsMemory() && F->doesNotThrow() && F->getReturnType()->isVoidTy()) { - for (Function::arg_iterator A = F->arg_begin(), E = F->arg_end(); A != E; - ++A) { - if (A->getType()->isPointerTy() && !A->hasNoCaptureAttr()) { - A->addAttr(Attribute::NoCapture); + for (Argument &A : F->args()) { + if (A.getType()->isPointerTy() && !A.hasNoCaptureAttr()) { + A.addAttr(Attribute::NoCapture); ++NumNoCapture; Changed.insert(F); } @@ -942,44 +941,43 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes, continue; } - for (Function::arg_iterator A = F->arg_begin(), E = F->arg_end(); A != E; - ++A) { - if (!A->getType()->isPointerTy()) + for (Argument &A : F->args()) { + if (!A.getType()->isPointerTy()) continue; bool HasNonLocalUses = false; - if (!A->hasNoCaptureAttr()) { + if (!A.hasNoCaptureAttr()) { ArgumentUsesTracker Tracker(SCCNodes); - PointerMayBeCaptured(&*A, &Tracker); + PointerMayBeCaptured(&A, &Tracker); if (!Tracker.Captured) { if (Tracker.Uses.empty()) { // If it's trivially not captured, mark it nocapture now. - A->addAttr(Attribute::NoCapture); + A.addAttr(Attribute::NoCapture); ++NumNoCapture; Changed.insert(F); } else { // If it's not trivially captured and not trivially not captured, // then it must be calling into another function in our SCC. Save // its particulars for Argument-SCC analysis later. - ArgumentGraphNode *Node = AG[&*A]; + ArgumentGraphNode *Node = AG[&A]; for (Argument *Use : Tracker.Uses) { Node->Uses.push_back(AG[Use]); - if (Use != &*A) + if (Use != &A) HasNonLocalUses = true; } } } // Otherwise, it's captured. Don't bother doing SCC analysis on it. } - if (!HasNonLocalUses && !A->onlyReadsMemory()) { + if (!HasNonLocalUses && !A.onlyReadsMemory()) { // Can we determine that it's readonly/readnone/writeonly without doing // an SCC? Note that we don't allow any calls at all here, or else our // result will be dependent on the iteration order through the // functions in the SCC. SmallPtrSet<Argument *, 8> Self; - Self.insert(&*A); - Attribute::AttrKind R = determinePointerAccessAttrs(&*A, Self); + Self.insert(&A); + Attribute::AttrKind R = determinePointerAccessAttrs(&A, Self); if (R != Attribute::None) - if (addAccessAttr(A, R)) + if (addAccessAttr(&A, R)) Changed.insert(F); } } @@ -1017,12 +1015,10 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes, } bool SCCCaptured = false; - for (auto I = ArgumentSCC.begin(), E = ArgumentSCC.end(); - I != E && !SCCCaptured; ++I) { - ArgumentGraphNode *Node = *I; - if (Node->Uses.empty()) { - if (!Node->Definition->hasNoCaptureAttr()) - SCCCaptured = true; + for (ArgumentGraphNode *Node : ArgumentSCC) { + if (Node->Uses.empty() && !Node->Definition->hasNoCaptureAttr()) { + SCCCaptured = true; + break; } } if (SCCCaptured) @@ -1035,9 +1031,7 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes, ArgumentSCCNodes.insert(I->Definition); } - for (auto I = ArgumentSCC.begin(), E = ArgumentSCC.end(); - I != E && !SCCCaptured; ++I) { - ArgumentGraphNode *N = *I; + for (ArgumentGraphNode *N : ArgumentSCC) { for (ArgumentGraphNode *Use : N->Uses) { Argument *A = Use->Definition; if (A->hasNoCaptureAttr() || ArgumentSCCNodes.count(A)) @@ -1045,12 +1039,14 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes, SCCCaptured = true; break; } + if (SCCCaptured) + break; } if (SCCCaptured) continue; - for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) { - Argument *A = ArgumentSCC[i]->Definition; + for (ArgumentGraphNode *N : ArgumentSCC) { + Argument *A = N->Definition; A->addAttr(Attribute::NoCapture); ++NumNoCapture; Changed.insert(A->getParent()); @@ -1078,16 +1074,17 @@ static void addArgumentAttrs(const SCCNodeSet &SCCNodes, }; Attribute::AttrKind AccessAttr = Attribute::ReadNone; - for (unsigned i = 0, e = ArgumentSCC.size(); - i != e && AccessAttr != Attribute::None; ++i) { - Argument *A = ArgumentSCC[i]->Definition; + for (ArgumentGraphNode *N : ArgumentSCC) { + Argument *A = N->Definition; Attribute::AttrKind K = determinePointerAccessAttrs(A, ArgumentSCCNodes); AccessAttr = meetAccessAttr(AccessAttr, K); + if (AccessAttr == Attribute::None) + break; } if (AccessAttr != Attribute::None) { - for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) { - Argument *A = ArgumentSCC[i]->Definition; + for (ArgumentGraphNode *N : ArgumentSCC) { + Argument *A = N->Definition; if (addAccessAttr(A, AccessAttr)) Changed.insert(A->getParent()); } diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp index ec26db8bfc0b..6df0409256bb 100644 --- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp +++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp @@ -470,8 +470,7 @@ static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) { // Sort by offset. SmallVector<std::pair<uint64_t, Type *>, 16> TypesVector; append_range(TypesVector, Types); - sort(TypesVector, - [](const auto &A, const auto &B) { return A.first < B.first; }); + sort(TypesVector, llvm::less_first()); // Check that the types are non-overlapping. uint64_t Offset = 0; diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp index 6bf25df101fa..e3e4908f085b 100644 --- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp @@ -1778,35 +1778,48 @@ void LowerTypeTestsModule::replaceDirectCalls(Value *Old, Value *New) { Old->replaceUsesWithIf(New, isDirectCall); } +static void dropTypeTests(Module &M, Function &TypeTestFunc) { + for (Use &U : llvm::make_early_inc_range(TypeTestFunc.uses())) { + auto *CI = cast<CallInst>(U.getUser()); + // Find and erase llvm.assume intrinsics for this llvm.type.test call. + for (Use &CIU : llvm::make_early_inc_range(CI->uses())) + if (auto *Assume = dyn_cast<AssumeInst>(CIU.getUser())) + Assume->eraseFromParent(); + // If the assume was merged with another assume, we might have a use on a + // phi (which will feed the assume). Simply replace the use on the phi + // with "true" and leave the merged assume. + if (!CI->use_empty()) { + assert( + all_of(CI->users(), [](User *U) -> bool { return isa<PHINode>(U); })); + CI->replaceAllUsesWith(ConstantInt::getTrue(M.getContext())); + } + CI->eraseFromParent(); + } +} + bool LowerTypeTestsModule::lower() { Function *TypeTestFunc = M.getFunction(Intrinsic::getName(Intrinsic::type_test)); - if (DropTypeTests && TypeTestFunc) { - for (Use &U : llvm::make_early_inc_range(TypeTestFunc->uses())) { - auto *CI = cast<CallInst>(U.getUser()); - // Find and erase llvm.assume intrinsics for this llvm.type.test call. - for (Use &CIU : llvm::make_early_inc_range(CI->uses())) - if (auto *Assume = dyn_cast<AssumeInst>(CIU.getUser())) - Assume->eraseFromParent(); - // If the assume was merged with another assume, we might have a use on a - // phi (which will feed the assume). Simply replace the use on the phi - // with "true" and leave the merged assume. - if (!CI->use_empty()) { - assert(all_of(CI->users(), - [](User *U) -> bool { return isa<PHINode>(U); })); - CI->replaceAllUsesWith(ConstantInt::getTrue(M.getContext())); - } - CI->eraseFromParent(); + if (DropTypeTests) { + if (TypeTestFunc) + dropTypeTests(M, *TypeTestFunc); + // Normally we'd have already removed all @llvm.public.type.test calls, + // except for in the case where we originally were performing ThinLTO but + // decided not to in the backend. + Function *PublicTypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::public_type_test)); + if (PublicTypeTestFunc) + dropTypeTests(M, *PublicTypeTestFunc); + if (TypeTestFunc || PublicTypeTestFunc) { + // We have deleted the type intrinsics, so we no longer have enough + // information to reason about the liveness of virtual function pointers + // in GlobalDCE. + for (GlobalVariable &GV : M.globals()) + GV.eraseMetadata(LLVMContext::MD_vcall_visibility); + return true; } - - // We have deleted the type intrinsics, so we no longer have enough - // information to reason about the liveness of virtual function pointers - // in GlobalDCE. - for (GlobalVariable &GV : M.globals()) - GV.eraseMetadata(LLVMContext::MD_vcall_visibility); - - return true; + return false; } // If only some of the modules were split, we cannot correctly perform diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index 0b42fc151991..ef2384faa273 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -499,18 +499,6 @@ struct OMPInformationCache : public InformationCache { } #include "llvm/Frontend/OpenMP/OMPKinds.def" - // Remove the `noinline` attribute from `__kmpc`, `_OMP::` and `omp_` - // functions, except if `optnone` is present. - if (isOpenMPDevice(M)) { - for (Function &F : M) { - for (StringRef Prefix : {"__kmpc", "_ZN4_OMP", "omp_"}) - if (F.hasFnAttribute(Attribute::NoInline) && - F.getName().startswith(Prefix) && - !F.hasFnAttribute(Attribute::OptimizeNone)) - F.removeFnAttr(Attribute::NoInline); - } - } - // TODO: We should attach the attributes defined in OMPKinds.def. } diff --git a/llvm/lib/Transforms/IPO/SCCP.cpp b/llvm/lib/Transforms/IPO/SCCP.cpp index 26fb7d676429..0453af184a72 100644 --- a/llvm/lib/Transforms/IPO/SCCP.cpp +++ b/llvm/lib/Transforms/IPO/SCCP.cpp @@ -148,7 +148,7 @@ struct FunctionSpecializationLegacyPass : public ModulePass { AU.addRequired<TargetTransformInfoWrapperPass>(); } - virtual bool runOnModule(Module &M) override { + bool runOnModule(Module &M) override { if (skipModule(M)) return false; diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp index a360a768a2bc..ef7af551a328 100644 --- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -132,6 +132,14 @@ void promoteTypeIds(Module &M, StringRef ModuleId) { } } + if (Function *PublicTypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::public_type_test))) { + for (const Use &U : PublicTypeTestFunc->uses()) { + auto CI = cast<CallInst>(U.getUser()); + ExternalizeTypeId(CI, 1); + } + } + if (Function *TypeCheckedLoadFunc = M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load))) { for (const Use &U : TypeCheckedLoadFunc->uses()) { diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp index ad00c116ce0a..18efe99f7cb4 100644 --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -773,15 +773,14 @@ PreservedAnalyses WholeProgramDevirtPass::run(Module &M, return PreservedAnalyses::none(); } +namespace llvm { // Enable whole program visibility if enabled by client (e.g. linker) or // internal option, and not force disabled. -static bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO) { +bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO) { return (WholeProgramVisibilityEnabledInLTO || WholeProgramVisibility) && !DisableWholeProgramVisibility; } -namespace llvm { - /// If whole program visibility asserted, then upgrade all public vcall /// visibility metadata on vtable definitions to linkage unit visibility in /// Module IR (for regular or hybrid LTO). @@ -790,7 +789,7 @@ void updateVCallVisibilityInModule( const DenseSet<GlobalValue::GUID> &DynamicExportSymbols) { if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) return; - for (GlobalVariable &GV : M.globals()) + for (GlobalVariable &GV : M.globals()) { // Add linkage unit visibility to any variable with type metadata, which are // the vtable definitions. We won't have an existing vcall_visibility // metadata on vtable definitions with public visibility. @@ -800,6 +799,34 @@ void updateVCallVisibilityInModule( // linker, as we have no information on their eventual use. !DynamicExportSymbols.count(GV.getGUID())) GV.setVCallVisibilityMetadata(GlobalObject::VCallVisibilityLinkageUnit); + } +} + +void updatePublicTypeTestCalls(Module &M, + bool WholeProgramVisibilityEnabledInLTO) { + Function *PublicTypeTestFunc = + M.getFunction(Intrinsic::getName(Intrinsic::public_type_test)); + if (!PublicTypeTestFunc) + return; + if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) { + Function *TypeTestFunc = + Intrinsic::getDeclaration(&M, Intrinsic::type_test); + for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) { + auto *CI = cast<CallInst>(U.getUser()); + auto *NewCI = CallInst::Create( + TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)}, None, "", + CI); + CI->replaceAllUsesWith(NewCI); + CI->eraseFromParent(); + } + } else { + auto *True = ConstantInt::getTrue(M.getContext()); + for (Use &U : make_early_inc_range(PublicTypeTestFunc->uses())) { + auto *CI = cast<CallInst>(U.getUser()); + CI->replaceAllUsesWith(True); + CI->eraseFromParent(); + } + } } /// If whole program visibility asserted, then upgrade all public vcall diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index 827b25533513..664226ec187b 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -597,10 +597,9 @@ public: /// demanded bits. bool SimplifyDemandedInstructionBits(Instruction &Inst); - virtual Value * - SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &UndefElts, - unsigned Depth = 0, - bool AllowMultipleUsers = false) override; + Value *SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, + APInt &UndefElts, unsigned Depth = 0, + bool AllowMultipleUsers = false) override; /// Canonicalize the position of binops relative to shufflevector. Instruction *foldVectorBinop(BinaryOperator &Inst); diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index cf2754b1dd60..3274e36ab71a 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -1232,7 +1232,9 @@ bool AddressSanitizer::isInterestingAlloca(const AllocaInst &AI) { // dynamic alloca instrumentation for them as well. !AI.isUsedWithInAlloca() && // swifterror allocas are register promoted by ISel - !AI.isSwiftError()); + !AI.isSwiftError() && + // safe allocas are not interesting + !(SSGI && SSGI->isSafe(AI))); ProcessedAllocas[&AI] = IsInteresting; return IsInteresting; diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp index fd2eaee8b47d..013a119c5096 100644 --- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp +++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp @@ -213,10 +213,12 @@ bool LoopDataPrefetchLegacyPass::runOnFunction(Function &F) { bool LoopDataPrefetch::run() { // If PrefetchDistance is not set, don't run the pass. This gives an // opportunity for targets to run this pass for selected subtargets only - // (whose TTI sets PrefetchDistance). - if (getPrefetchDistance() == 0) + // (whose TTI sets PrefetchDistance and CacheLineSize). + if (getPrefetchDistance() == 0 || TTI->getCacheLineSize() == 0) { + LLVM_DEBUG(dbgs() << "Please set both PrefetchDistance and CacheLineSize " + "for loop data prefetch.\n"); return false; - assert(TTI->getCacheLineSize() && "Cache line size is not set for target"); + } bool MadeChange = false; diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp index c05906649f16..f1e1359255bd 100644 --- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp +++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp @@ -338,6 +338,9 @@ class LowerMatrixIntrinsics { Value *extractVector(unsigned I, unsigned J, unsigned NumElts, IRBuilder<> &Builder) const { Value *Vec = isColumnMajor() ? getColumn(J) : getRow(I); + assert(cast<FixedVectorType>(Vec->getType())->getNumElements() >= + NumElts && + "Extracted vector will contain poison values"); return Builder.CreateShuffleVector( Vec, createSequentialMask(isColumnMajor() ? I : J, NumElts, 0), "block"); @@ -1423,13 +1426,13 @@ public: FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize); MatrixTy TileResult; // Insert in the inner loop header. - Builder.SetInsertPoint(TI.InnerLoopHeader->getTerminator()); + Builder.SetInsertPoint(TI.KLoop.Header->getTerminator()); // Create PHI nodes for the result columns to accumulate across iterations. SmallVector<PHINode *, 4> ColumnPhis; for (unsigned I = 0; I < TileSize; I++) { auto *Phi = Builder.CreatePHI(TileVecTy, 2, "result.vec." + Twine(I)); Phi->addIncoming(ConstantAggregateZero::get(TileVecTy), - TI.RowLoopHeader->getSingleSuccessor()); + TI.RowLoop.Header->getSingleSuccessor()); TileResult.addVector(Phi); ColumnPhis.push_back(Phi); } @@ -1438,27 +1441,29 @@ public: // Res += Load(CurrentRow, K) * Load(K, CurrentColumn) Builder.SetInsertPoint(InnerBody->getTerminator()); // Load tiles of the operands. - MatrixTy A = loadMatrix(LPtr, {}, false, LShape, TI.CurrentRow, TI.CurrentK, - {TileSize, TileSize}, EltType, Builder); - MatrixTy B = loadMatrix(RPtr, {}, false, RShape, TI.CurrentK, TI.CurrentCol, - {TileSize, TileSize}, EltType, Builder); + MatrixTy A = + loadMatrix(LPtr, {}, false, LShape, TI.RowLoop.Index, TI.KLoop.Index, + {TileSize, TileSize}, EltType, Builder); + MatrixTy B = + loadMatrix(RPtr, {}, false, RShape, TI.KLoop.Index, TI.ColumnLoop.Index, + {TileSize, TileSize}, EltType, Builder); emitMatrixMultiply(TileResult, A, B, Builder, true, false, getFastMathFlags(MatMul)); // Store result after the inner loop is done. - Builder.SetInsertPoint(TI.RowLoopLatch->getTerminator()); + Builder.SetInsertPoint(TI.RowLoop.Latch->getTerminator()); storeMatrix(TileResult, Store->getPointerOperand(), Store->getAlign(), Store->isVolatile(), {LShape.NumRows, RShape.NumColumns}, - TI.CurrentRow, TI.CurrentCol, EltType, Builder); + TI.RowLoop.Index, TI.ColumnLoop.Index, EltType, Builder); for (unsigned I = 0; I < TileResult.getNumVectors(); I++) - ColumnPhis[I]->addIncoming(TileResult.getVector(I), TI.InnerLoopLatch); + ColumnPhis[I]->addIncoming(TileResult.getVector(I), TI.KLoop.Latch); // Force unrolling of a few iterations of the inner loop, to make sure there // is enough work per iteration. // FIXME: The unroller should make this decision directly instead, but // currently the cost-model is not up to the task. unsigned InnerLoopUnrollCount = std::min(10u, LShape.NumColumns / TileSize); - addStringMetadataToLoop(LI->getLoopFor(TI.InnerLoopHeader), + addStringMetadataToLoop(LI->getLoopFor(TI.KLoop.Header), "llvm.loop.unroll.count", InnerLoopUnrollCount); } diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp index 240fb5e60687..cd2ce8ce336e 100644 --- a/llvm/lib/Transforms/Scalar/Reassociate.cpp +++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp @@ -147,27 +147,27 @@ XorOpnd::XorOpnd(Value *V) { /// Instruction::isAssociative() because it includes operations like fsub. /// (This routine is only intended to be called for floating-point operations.) static bool hasFPAssociativeFlags(Instruction *I) { - assert(I && I->getType()->isFPOrFPVectorTy() && "Should only check FP ops"); + assert(I && isa<FPMathOperator>(I) && "Should only check FP ops"); return I->hasAllowReassoc() && I->hasNoSignedZeros(); } /// Return true if V is an instruction of the specified opcode and if it /// only has one use. static BinaryOperator *isReassociableOp(Value *V, unsigned Opcode) { - auto *I = dyn_cast<Instruction>(V); - if (I && I->hasOneUse() && I->getOpcode() == Opcode) - if (!isa<FPMathOperator>(I) || hasFPAssociativeFlags(I)) - return cast<BinaryOperator>(I); + auto *BO = dyn_cast<BinaryOperator>(V); + if (BO && BO->hasOneUse() && BO->getOpcode() == Opcode) + if (!isa<FPMathOperator>(BO) || hasFPAssociativeFlags(BO)) + return BO; return nullptr; } static BinaryOperator *isReassociableOp(Value *V, unsigned Opcode1, unsigned Opcode2) { - auto *I = dyn_cast<Instruction>(V); - if (I && I->hasOneUse() && - (I->getOpcode() == Opcode1 || I->getOpcode() == Opcode2)) - if (!isa<FPMathOperator>(I) || hasFPAssociativeFlags(I)) - return cast<BinaryOperator>(I); + auto *BO = dyn_cast<BinaryOperator>(V); + if (BO && BO->hasOneUse() && + (BO->getOpcode() == Opcode1 || BO->getOpcode() == Opcode2)) + if (!isa<FPMathOperator>(BO) || hasFPAssociativeFlags(BO)) + return BO; return nullptr; } @@ -778,7 +778,7 @@ void ReassociatePass::RewriteExprTree(BinaryOperator *I, Constant *Undef = UndefValue::get(I->getType()); NewOp = BinaryOperator::Create(Instruction::BinaryOps(Opcode), Undef, Undef, "", I); - if (NewOp->getType()->isFPOrFPVectorTy()) + if (isa<FPMathOperator>(NewOp)) NewOp->setFastMathFlags(I->getFastMathFlags()); } else { NewOp = NodesToRewrite.pop_back_val(); @@ -2227,7 +2227,7 @@ void ReassociatePass::OptimizeInst(Instruction *I) { // Don't optimize floating-point instructions unless they have the // appropriate FastMathFlags for reassociation enabled. - if (I->getType()->isFPOrFPVectorTy() && !hasFPAssociativeFlags(I)) + if (isa<FPMathOperator>(I) && !hasFPAssociativeFlags(I)) return; // Do not reassociate boolean (i1) expressions. We want to preserve the diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 00387ec426bf..878f9477a29d 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -825,6 +825,35 @@ static void PropagateCallSiteMetadata(CallBase &CB, Function::iterator FStart, } } +/// Bundle operands of the inlined function must be added to inlined call sites. +static void PropagateOperandBundles(Function::iterator InlinedBB, + Instruction *CallSiteEHPad) { + for (Instruction &II : llvm::make_early_inc_range(*InlinedBB)) { + CallBase *I = dyn_cast<CallBase>(&II); + if (!I) + continue; + // Skip call sites which already have a "funclet" bundle. + if (I->getOperandBundle(LLVMContext::OB_funclet)) + continue; + // Skip call sites which are nounwind intrinsics (as long as they don't + // lower into regular function calls in the course of IR transformations). + auto *CalledFn = + dyn_cast<Function>(I->getCalledOperand()->stripPointerCasts()); + if (CalledFn && CalledFn->isIntrinsic() && I->doesNotThrow() && + !IntrinsicInst::mayLowerToFunctionCall(CalledFn->getIntrinsicID())) + continue; + + SmallVector<OperandBundleDef, 1> OpBundles; + I->getOperandBundlesAsDefs(OpBundles); + OpBundles.emplace_back("funclet", CallSiteEHPad); + + Instruction *NewInst = CallBase::Create(I, OpBundles, I); + NewInst->takeName(I); + I->replaceAllUsesWith(NewInst); + I->eraseFromParent(); + } +} + namespace { /// Utility for cloning !noalias and !alias.scope metadata. When a code region /// using scoped alias metadata is inlined, the aliasing relationships may not @@ -2304,38 +2333,12 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // Update the lexical scopes of the new funclets and callsites. // Anything that had 'none' as its parent is now nested inside the callsite's // EHPad. - if (CallSiteEHPad) { for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end(); BB != E; ++BB) { - // Add bundle operands to any top-level call sites. - SmallVector<OperandBundleDef, 1> OpBundles; - for (Instruction &II : llvm::make_early_inc_range(*BB)) { - CallBase *I = dyn_cast<CallBase>(&II); - if (!I) - continue; - - // Skip call sites which are nounwind intrinsics. - auto *CalledFn = - dyn_cast<Function>(I->getCalledOperand()->stripPointerCasts()); - if (CalledFn && CalledFn->isIntrinsic() && I->doesNotThrow()) - continue; - - // Skip call sites which already have a "funclet" bundle. - if (I->getOperandBundle(LLVMContext::OB_funclet)) - continue; - - I->getOperandBundlesAsDefs(OpBundles); - OpBundles.emplace_back("funclet", CallSiteEHPad); - - Instruction *NewInst = CallBase::Create(I, OpBundles, I); - NewInst->takeName(I); - I->replaceAllUsesWith(NewInst); - I->eraseFromParent(); - - OpBundles.clear(); - } + // Add bundle operands to inlined call sites. + PropagateOperandBundles(BB, CallSiteEHPad); // It is problematic if the inlinee has a cleanupret which unwinds to // caller and we inline it into a call site which doesn't unwind but into diff --git a/llvm/lib/Transforms/Utils/MatrixUtils.cpp b/llvm/lib/Transforms/Utils/MatrixUtils.cpp index 6a137630deeb..e218773cf5da 100644 --- a/llvm/lib/Transforms/Utils/MatrixUtils.cpp +++ b/llvm/lib/Transforms/Utils/MatrixUtils.cpp @@ -70,35 +70,35 @@ BasicBlock *TileInfo::CreateLoop(BasicBlock *Preheader, BasicBlock *Exit, BasicBlock *TileInfo::CreateTiledLoops(BasicBlock *Start, BasicBlock *End, IRBuilderBase &B, DomTreeUpdater &DTU, LoopInfo &LI) { - Loop *ColLoop = LI.AllocateLoop(); - Loop *RowLoop = LI.AllocateLoop(); - Loop *InnerLoop = LI.AllocateLoop(); - RowLoop->addChildLoop(InnerLoop); - ColLoop->addChildLoop(RowLoop); + Loop *ColumnLoopInfo = LI.AllocateLoop(); + Loop *RowLoopInfo = LI.AllocateLoop(); + Loop *KLoopInfo = LI.AllocateLoop(); + RowLoopInfo->addChildLoop(KLoopInfo); + ColumnLoopInfo->addChildLoop(RowLoopInfo); if (Loop *ParentL = LI.getLoopFor(Start)) - ParentL->addChildLoop(ColLoop); + ParentL->addChildLoop(ColumnLoopInfo); else - LI.addTopLevelLoop(ColLoop); + LI.addTopLevelLoop(ColumnLoopInfo); BasicBlock *ColBody = CreateLoop(Start, End, B.getInt64(NumColumns), B.getInt64(TileSize), - "cols", B, DTU, ColLoop, LI); - BasicBlock *ColLatch = ColBody->getSingleSuccessor(); + "cols", B, DTU, ColumnLoopInfo, LI); + ColumnLoop.Latch = ColBody->getSingleSuccessor(); BasicBlock *RowBody = - CreateLoop(ColBody, ColLatch, B.getInt64(NumRows), B.getInt64(TileSize), - "rows", B, DTU, RowLoop, LI); - RowLoopLatch = RowBody->getSingleSuccessor(); + CreateLoop(ColBody, ColumnLoop.Latch, B.getInt64(NumRows), + B.getInt64(TileSize), "rows", B, DTU, RowLoopInfo, LI); + RowLoop.Latch = RowBody->getSingleSuccessor(); BasicBlock *InnerBody = - CreateLoop(RowBody, RowLoopLatch, B.getInt64(NumInner), - B.getInt64(TileSize), "inner", B, DTU, InnerLoop, LI); - InnerLoopLatch = InnerBody->getSingleSuccessor(); - ColumnLoopHeader = ColBody->getSinglePredecessor(); - RowLoopHeader = RowBody->getSinglePredecessor(); - InnerLoopHeader = InnerBody->getSinglePredecessor(); - CurrentRow = &*RowLoopHeader->begin(); - CurrentCol = &*ColumnLoopHeader->begin(); - CurrentK = &*InnerLoopHeader->begin(); + CreateLoop(RowBody, RowLoop.Latch, B.getInt64(NumInner), + B.getInt64(TileSize), "inner", B, DTU, KLoopInfo, LI); + KLoop.Latch = InnerBody->getSingleSuccessor(); + ColumnLoop.Header = ColBody->getSinglePredecessor(); + RowLoop.Header = RowBody->getSinglePredecessor(); + KLoop.Header = InnerBody->getSinglePredecessor(); + RowLoop.Index = &*RowLoop.Header->begin(); + ColumnLoop.Index = &*ColumnLoop.Header->begin(); + KLoop.Index = &*KLoop.Header->begin(); return InnerBody; } diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp index bca3b0538c5d..03087d8370d5 100644 --- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp @@ -75,39 +75,109 @@ static bool callHasFP128Argument(const CallInst *CI) { }); } -static Value *convertStrToNumber(CallInst *CI, StringRef &Str, Value *EndPtr, - int64_t Base, IRBuilderBase &B) { +// Convert the entire string Str representing an integer in Base, up to +// the terminating nul if present, to a constant according to the rules +// of strtoul[l] or, when AsSigned is set, of strtol[l]. On success +// return the result, otherwise null. +// The function assumes the string is encoded in ASCII and carefully +// avoids converting sequences (including "") that the corresponding +// library call might fail and set errno for. +static Value *convertStrToInt(CallInst *CI, StringRef &Str, Value *EndPtr, + uint64_t Base, bool AsSigned, IRBuilderBase &B) { if (Base < 2 || Base > 36) - // handle special zero base if (Base != 0) + // Fail for an invalid base (required by POSIX). return nullptr; - char *End; - std::string nptr = Str.str(); - errno = 0; - long long int Result = strtoll(nptr.c_str(), &End, Base); - if (errno) - return nullptr; + // Strip leading whitespace. + for (unsigned i = 0; i != Str.size(); ++i) + if (!isSpace((unsigned char)Str[i])) { + Str = Str.substr(i); + break; + } - // if we assume all possible target locales are ASCII supersets, - // then if strtoll successfully parses a number on the host, - // it will also successfully parse the same way on the target - if (*End != '\0') + if (Str.empty()) + // Fail for empty subject sequences (POSIX allows but doesn't require + // strtol[l]/strtoul[l] to fail with EINVAL). return nullptr; - if (!isIntN(CI->getType()->getPrimitiveSizeInBits(), Result)) - return nullptr; + // Strip but remember the sign. + bool Negate = Str[0] == '-'; + if (Str[0] == '-' || Str[0] == '+') { + Str = Str.drop_front(); + if (Str.empty()) + // Fail for a sign with nothing after it. + return nullptr; + } + + // Set Max to the absolute value of the minimum (for signed), or + // to the maximum (for unsigned) value representable in the type. + Type *RetTy = CI->getType(); + unsigned NBits = RetTy->getPrimitiveSizeInBits(); + uint64_t Max = AsSigned && Negate ? 1 : 0; + Max += AsSigned ? maxIntN(NBits) : maxUIntN(NBits); + + // Autodetect Base if it's zero and consume the "0x" prefix. + if (Str.size() > 1) { + if (Str[0] == '0') { + if (toUpper((unsigned char)Str[1]) == 'X') { + if (Str.size() == 2 || (Base && Base != 16)) + // Fail if Base doesn't allow the "0x" prefix or for the prefix + // alone that implementations like BSD set errno to EINVAL for. + return nullptr; + + Str = Str.drop_front(2); + Base = 16; + } + else if (Base == 0) + Base = 8; + } else if (Base == 0) + Base = 10; + } + else if (Base == 0) + Base = 10; + + // Convert the rest of the subject sequence, not including the sign, + // to its uint64_t representation (this assumes the source character + // set is ASCII). + uint64_t Result = 0; + for (unsigned i = 0; i != Str.size(); ++i) { + unsigned char DigVal = Str[i]; + if (isDigit(DigVal)) + DigVal = DigVal - '0'; + else { + DigVal = toUpper(DigVal); + if (isAlpha(DigVal)) + DigVal = DigVal - 'A' + 10; + else + return nullptr; + } + + if (DigVal >= Base) + // Fail if the digit is not valid in the Base. + return nullptr; + + // Add the digit and fail if the result is not representable in + // the (unsigned form of the) destination type. + bool VFlow; + Result = SaturatingMultiplyAdd(Result, Base, (uint64_t)DigVal, &VFlow); + if (VFlow || Result > Max) + return nullptr; + } if (EndPtr) { // Store the pointer to the end. - uint64_t ILen = End - nptr.c_str(); - Value *Off = B.getInt64(ILen); + Value *Off = B.getInt64(Str.size()); Value *StrBeg = CI->getArgOperand(0); Value *StrEnd = B.CreateInBoundsGEP(B.getInt8Ty(), StrBeg, Off, "endptr"); B.CreateStore(StrEnd, EndPtr); } - return ConstantInt::get(CI->getType(), Result); + if (Negate) + // Unsigned negation doesn't overflow. + Result = -Result; + + return ConstantInt::get(RetTy, Result); } static bool isOnlyUsedInComparisonWithZero(Value *V) { @@ -2531,27 +2601,35 @@ Value *LibCallSimplifier::optimizeToAscii(CallInst *CI, IRBuilderBase &B) { ConstantInt::get(CI->getType(), 0x7F)); } +// Fold calls to atoi, atol, and atoll. Value *LibCallSimplifier::optimizeAtoi(CallInst *CI, IRBuilderBase &B) { + CI->addParamAttr(0, Attribute::NoCapture); + StringRef Str; if (!getConstantStringInfo(CI->getArgOperand(0), Str)) return nullptr; - return convertStrToNumber(CI, Str, nullptr, 10, B); + return convertStrToInt(CI, Str, nullptr, 10, /*AsSigned=*/true, B); } -Value *LibCallSimplifier::optimizeStrtol(CallInst *CI, IRBuilderBase &B) { - StringRef Str; - if (!getConstantStringInfo(CI->getArgOperand(0), Str)) - return nullptr; - +// Fold calls to strtol, strtoll, strtoul, and strtoull. +Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B, + bool AsSigned) { Value *EndPtr = CI->getArgOperand(1); - if (isa<ConstantPointerNull>(EndPtr)) + if (isa<ConstantPointerNull>(EndPtr)) { + // With a null EndPtr, this function won't capture the main argument. + // It would be readonly too, except that it still may write to errno. + CI->addParamAttr(0, Attribute::NoCapture); EndPtr = nullptr; - else if (!isKnownNonZero(EndPtr, DL)) + } else if (!isKnownNonZero(EndPtr, DL)) + return nullptr; + + StringRef Str; + if (!getConstantStringInfo(CI->getArgOperand(0), Str)) return nullptr; if (ConstantInt *CInt = dyn_cast<ConstantInt>(CI->getArgOperand(2))) { - return convertStrToNumber(CI, Str, EndPtr, CInt->getSExtValue(), B); + return convertStrToInt(CI, Str, EndPtr, CInt->getSExtValue(), AsSigned, B); } return nullptr; @@ -3390,7 +3468,10 @@ Value *LibCallSimplifier::optimizeCall(CallInst *CI, IRBuilderBase &Builder) { return optimizeAtoi(CI, Builder); case LibFunc_strtol: case LibFunc_strtoll: - return optimizeStrtol(CI, Builder); + return optimizeStrToInt(CI, Builder, /*AsSigned=*/true); + case LibFunc_strtoul: + case LibFunc_strtoull: + return optimizeStrToInt(CI, Builder, /*AsSigned=*/false); case LibFunc_printf: return optimizePrintF(CI, Builder); case LibFunc_sprintf: diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index b887ea41676b..238b074089aa 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -798,8 +798,7 @@ public: // Override this function to handle the more complex control flow around the // three loops. - std::pair<BasicBlock *, Value *> - createVectorizedLoopSkeleton() final override { + std::pair<BasicBlock *, Value *> createVectorizedLoopSkeleton() final { return createEpilogueVectorizedLoopSkeleton(); } @@ -835,8 +834,7 @@ public: EPI, LVL, CM, BFI, PSI, Check) {} /// Implements the interface for creating a vectorized skeleton using the /// *main loop* strategy (ie the first pass of vplan execution). - std::pair<BasicBlock *, Value *> - createEpilogueVectorizedLoopSkeleton() final override; + std::pair<BasicBlock *, Value *> createEpilogueVectorizedLoopSkeleton() final; protected: /// Emits an iteration count bypass check once for the main loop (when \p @@ -866,8 +864,7 @@ public: } /// Implements the interface for creating a vectorized skeleton using the /// *epilogue loop* strategy (ie the second pass of vplan execution). - std::pair<BasicBlock *, Value *> - createEpilogueVectorizedLoopSkeleton() final override; + std::pair<BasicBlock *, Value *> createEpilogueVectorizedLoopSkeleton() final; protected: /// Emits an iteration count bypass check after the main vector loop has diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index cd044c78d900..d69d1e3d19f3 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -10972,9 +10972,7 @@ public: It != E; ++It) { PossibleRedValsVect.emplace_back(); auto RedValsVect = It->second.takeVector(); - stable_sort(RedValsVect, [](const auto &P1, const auto &P2) { - return P1.second < P2.second; - }); + stable_sort(RedValsVect, llvm::less_second()); for (const std::pair<Value *, unsigned> &Data : RedValsVect) PossibleRedValsVect.back().append(Data.second, Data.first); } |