aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp521
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp24
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp114
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp4
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp4
-rw-r--r--llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp2
-rw-r--r--llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp12
-rw-r--r--llvm/lib/Transforms/Scalar/LoopFlatten.cpp79
-rw-r--r--llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp23
-rw-r--r--llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp58
-rw-r--r--llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp55
-rw-r--r--llvm/lib/Transforms/Utils/Local.cpp3
-rw-r--r--llvm/lib/Transforms/Utils/SimplifyCFG.cpp13
-rw-r--r--llvm/lib/Transforms/Vectorize/LoopVectorize.cpp6
-rw-r--r--llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp69
15 files changed, 705 insertions, 282 deletions
diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 70a3f3067d9d..0a6f69bc73d5 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -77,6 +77,16 @@ STATISTIC(MaxAllocVersionsThinBackend,
"allocation during ThinLTO backend");
STATISTIC(UnclonableAllocsThinBackend,
"Number of unclonable ambigous allocations during ThinLTO backend");
+STATISTIC(RemovedEdgesWithMismatchedCallees,
+ "Number of edges removed due to mismatched callees (profiled vs IR)");
+STATISTIC(FoundProfiledCalleeCount,
+ "Number of profiled callees found via tail calls");
+STATISTIC(FoundProfiledCalleeDepth,
+ "Aggregate depth of profiled callees found via tail calls");
+STATISTIC(FoundProfiledCalleeMaxDepth,
+ "Maximum depth of profiled callees found via tail calls");
+STATISTIC(FoundProfiledCalleeNonUniquelyCount,
+ "Number of profiled callees found via multiple tail call chains");
static cl::opt<std::string> DotFilePathPrefix(
"memprof-dot-file-path-prefix", cl::init(""), cl::Hidden,
@@ -104,6 +114,12 @@ static cl::opt<std::string> MemProfImportSummary(
cl::desc("Import summary to use for testing the ThinLTO backend via opt"),
cl::Hidden);
+static cl::opt<unsigned>
+ TailCallSearchDepth("memprof-tail-call-search-depth", cl::init(5),
+ cl::Hidden,
+ cl::desc("Max depth to recursively search for missing "
+ "frames through tail calls."));
+
namespace llvm {
// Indicate we are linking with an allocator that supports hot/cold operator
// new interfaces.
@@ -365,8 +381,7 @@ protected:
/// Save lists of calls with MemProf metadata in each function, for faster
/// iteration.
- std::vector<std::pair<FuncTy *, std::vector<CallInfo>>>
- FuncToCallsWithMetadata;
+ MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
/// Map from callsite node to the enclosing caller function.
std::map<const ContextNode *, const FuncTy *> NodeToCallingFunc;
@@ -411,9 +426,25 @@ private:
return static_cast<const DerivedCCG *>(this)->getStackId(IdOrIndex);
}
- /// Returns true if the given call targets the given function.
- bool calleeMatchesFunc(CallTy Call, const FuncTy *Func) {
- return static_cast<DerivedCCG *>(this)->calleeMatchesFunc(Call, Func);
+ /// Returns true if the given call targets the callee of the given edge, or if
+ /// we were able to identify the call chain through intermediate tail calls.
+ /// In the latter case new context nodes are added to the graph for the
+ /// identified tail calls, and their synthesized nodes are added to
+ /// TailCallToContextNodeMap. The EdgeIter is updated in either case to the
+ /// next element after the input position (either incremented or updated after
+ /// removing the old edge).
+ bool
+ calleesMatch(CallTy Call, EdgeIter &EI,
+ MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap);
+
+ /// Returns true if the given call targets the given function, or if we were
+ /// able to identify the call chain through intermediate tail calls (in which
+ /// case FoundCalleeChain will be populated).
+ bool calleeMatchesFunc(
+ CallTy Call, const FuncTy *Func, const FuncTy *CallerFunc,
+ std::vector<std::pair<CallTy, FuncTy *>> &FoundCalleeChain) {
+ return static_cast<DerivedCCG *>(this)->calleeMatchesFunc(
+ Call, Func, CallerFunc, FoundCalleeChain);
}
/// Get a list of nodes corresponding to the stack ids in the given
@@ -553,7 +584,13 @@ private:
Instruction *>;
uint64_t getStackId(uint64_t IdOrIndex) const;
- bool calleeMatchesFunc(Instruction *Call, const Function *Func);
+ bool calleeMatchesFunc(
+ Instruction *Call, const Function *Func, const Function *CallerFunc,
+ std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain);
+ bool findProfiledCalleeThroughTailCalls(
+ const Function *ProfiledCallee, Value *CurCallee, unsigned Depth,
+ std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain,
+ bool &FoundMultipleCalleeChains);
uint64_t getLastStackId(Instruction *Call);
std::vector<uint64_t> getStackIdsWithContextNodesForCall(Instruction *Call);
void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
@@ -606,12 +643,31 @@ public:
function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
isPrevailing);
+ ~IndexCallsiteContextGraph() {
+ // Now that we are done with the graph it is safe to add the new
+ // CallsiteInfo structs to the function summary vectors. The graph nodes
+ // point into locations within these vectors, so we don't want to add them
+ // any earlier.
+ for (auto &I : FunctionCalleesToSynthesizedCallsiteInfos) {
+ auto *FS = I.first;
+ for (auto &Callsite : I.second)
+ FS->addCallsite(*Callsite.second);
+ }
+ }
+
private:
friend CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
IndexCall>;
uint64_t getStackId(uint64_t IdOrIndex) const;
- bool calleeMatchesFunc(IndexCall &Call, const FunctionSummary *Func);
+ bool calleeMatchesFunc(
+ IndexCall &Call, const FunctionSummary *Func,
+ const FunctionSummary *CallerFunc,
+ std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain);
+ bool findProfiledCalleeThroughTailCalls(
+ ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
+ std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
+ bool &FoundMultipleCalleeChains);
uint64_t getLastStackId(IndexCall &Call);
std::vector<uint64_t> getStackIdsWithContextNodesForCall(IndexCall &Call);
void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
@@ -630,6 +686,16 @@ private:
std::map<const FunctionSummary *, ValueInfo> FSToVIMap;
const ModuleSummaryIndex &Index;
+ function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
+ isPrevailing;
+
+ // Saves/owns the callsite info structures synthesized for missing tail call
+ // frames that we discover while building the graph.
+ // It maps from the summary of the function making the tail call, to a map
+ // of callee ValueInfo to corresponding synthesized callsite info.
+ std::unordered_map<FunctionSummary *,
+ std::map<ValueInfo, std::unique_ptr<CallsiteInfo>>>
+ FunctionCalleesToSynthesizedCallsiteInfos;
};
} // namespace
@@ -1493,7 +1559,7 @@ ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
}
}
if (!CallsWithMetadata.empty())
- FuncToCallsWithMetadata.push_back({&F, CallsWithMetadata});
+ FuncToCallsWithMetadata[&F] = CallsWithMetadata;
}
if (DumpCCG) {
@@ -1518,7 +1584,7 @@ IndexCallsiteContextGraph::IndexCallsiteContextGraph(
ModuleSummaryIndex &Index,
function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
isPrevailing)
- : Index(Index) {
+ : Index(Index), isPrevailing(isPrevailing) {
for (auto &I : Index) {
auto VI = Index.getValueInfo(I);
for (auto &S : VI.getSummaryList()) {
@@ -1572,7 +1638,7 @@ IndexCallsiteContextGraph::IndexCallsiteContextGraph(
CallsWithMetadata.push_back({&SN});
if (!CallsWithMetadata.empty())
- FuncToCallsWithMetadata.push_back({FS, CallsWithMetadata});
+ FuncToCallsWithMetadata[FS] = CallsWithMetadata;
if (!FS->allocs().empty() || !FS->callsites().empty())
FSToVIMap[FS] = VI;
@@ -1604,6 +1670,11 @@ void CallsiteContextGraph<DerivedCCG, FuncTy,
// this transformation for regular LTO, and for ThinLTO we can simulate that
// effect in the summary and perform the actual speculative devirtualization
// while cloning in the ThinLTO backend.
+
+ // Keep track of the new nodes synthesized for discovered tail calls missing
+ // from the profiled contexts.
+ MapVector<CallInfo, ContextNode *> TailCallToContextNodeMap;
+
for (auto Entry = NonAllocationCallToContextNodeMap.begin();
Entry != NonAllocationCallToContextNodeMap.end();) {
auto *Node = Entry->second;
@@ -1611,13 +1682,17 @@ void CallsiteContextGraph<DerivedCCG, FuncTy,
// Check all node callees and see if in the same function.
bool Removed = false;
auto Call = Node->Call.call();
- for (auto &Edge : Node->CalleeEdges) {
- if (!Edge->Callee->hasCall())
+ for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) {
+ auto Edge = *EI;
+ if (!Edge->Callee->hasCall()) {
+ ++EI;
continue;
+ }
assert(NodeToCallingFunc.count(Edge->Callee));
// Check if the called function matches that of the callee node.
- if (calleeMatchesFunc(Call, NodeToCallingFunc[Edge->Callee]))
+ if (calleesMatch(Call, EI, TailCallToContextNodeMap))
continue;
+ RemovedEdgesWithMismatchedCallees++;
// Work around by setting Node to have a null call, so it gets
// skipped during cloning. Otherwise assignFunctions will assert
// because its data structures are not designed to handle this case.
@@ -1629,6 +1704,11 @@ void CallsiteContextGraph<DerivedCCG, FuncTy,
if (!Removed)
Entry++;
}
+
+ // Add the new nodes after the above loop so that the iteration is not
+ // invalidated.
+ for (auto &[Call, Node] : TailCallToContextNodeMap)
+ NonAllocationCallToContextNodeMap[Call] = Node;
}
uint64_t ModuleCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
@@ -1642,8 +1722,173 @@ uint64_t IndexCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
return Index.getStackIdAtIndex(IdOrIndex);
}
-bool ModuleCallsiteContextGraph::calleeMatchesFunc(Instruction *Call,
- const Function *Func) {
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
+ CallTy Call, EdgeIter &EI,
+ MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap) {
+ auto Edge = *EI;
+ const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
+ const FuncTy *CallerFunc = NodeToCallingFunc[Edge->Caller];
+ // Will be populated in order of callee to caller if we find a chain of tail
+ // calls between the profiled caller and callee.
+ std::vector<std::pair<CallTy, FuncTy *>> FoundCalleeChain;
+ if (!calleeMatchesFunc(Call, ProfiledCalleeFunc, CallerFunc,
+ FoundCalleeChain)) {
+ ++EI;
+ return false;
+ }
+
+ // The usual case where the profiled callee matches that of the IR/summary.
+ if (FoundCalleeChain.empty()) {
+ ++EI;
+ return true;
+ }
+
+ auto AddEdge = [Edge, &EI](ContextNode *Caller, ContextNode *Callee) {
+ auto *CurEdge = Callee->findEdgeFromCaller(Caller);
+ // If there is already an edge between these nodes, simply update it and
+ // return.
+ if (CurEdge) {
+ CurEdge->ContextIds.insert(Edge->ContextIds.begin(),
+ Edge->ContextIds.end());
+ CurEdge->AllocTypes |= Edge->AllocTypes;
+ return;
+ }
+ // Otherwise, create a new edge and insert it into the caller and callee
+ // lists.
+ auto NewEdge = std::make_shared<ContextEdge>(
+ Callee, Caller, Edge->AllocTypes, Edge->ContextIds);
+ Callee->CallerEdges.push_back(NewEdge);
+ if (Caller == Edge->Caller) {
+ // If we are inserting the new edge into the current edge's caller, insert
+ // the new edge before the current iterator position, and then increment
+ // back to the current edge.
+ EI = Caller->CalleeEdges.insert(EI, NewEdge);
+ ++EI;
+ assert(*EI == Edge &&
+ "Iterator position not restored after insert and increment");
+ } else
+ Caller->CalleeEdges.push_back(NewEdge);
+ };
+
+ // Create new nodes for each found callee and connect in between the profiled
+ // caller and callee.
+ auto *CurCalleeNode = Edge->Callee;
+ for (auto &[NewCall, Func] : FoundCalleeChain) {
+ ContextNode *NewNode = nullptr;
+ // First check if we have already synthesized a node for this tail call.
+ if (TailCallToContextNodeMap.count(NewCall)) {
+ NewNode = TailCallToContextNodeMap[NewCall];
+ NewNode->ContextIds.insert(Edge->ContextIds.begin(),
+ Edge->ContextIds.end());
+ NewNode->AllocTypes |= Edge->AllocTypes;
+ } else {
+ FuncToCallsWithMetadata[Func].push_back({NewCall});
+ // Create Node and record node info.
+ NodeOwner.push_back(
+ std::make_unique<ContextNode>(/*IsAllocation=*/false, NewCall));
+ NewNode = NodeOwner.back().get();
+ NodeToCallingFunc[NewNode] = Func;
+ TailCallToContextNodeMap[NewCall] = NewNode;
+ NewNode->ContextIds = Edge->ContextIds;
+ NewNode->AllocTypes = Edge->AllocTypes;
+ }
+
+ // Hook up node to its callee node
+ AddEdge(NewNode, CurCalleeNode);
+
+ CurCalleeNode = NewNode;
+ }
+
+ // Hook up edge's original caller to new callee node.
+ AddEdge(Edge->Caller, CurCalleeNode);
+
+ // Remove old edge
+ Edge->Callee->eraseCallerEdge(Edge.get());
+ EI = Edge->Caller->CalleeEdges.erase(EI);
+
+ return true;
+}
+
+bool ModuleCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
+ const Function *ProfiledCallee, Value *CurCallee, unsigned Depth,
+ std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain,
+ bool &FoundMultipleCalleeChains) {
+ // Stop recursive search if we have already explored the maximum specified
+ // depth.
+ if (Depth > TailCallSearchDepth)
+ return false;
+
+ auto SaveCallsiteInfo = [&](Instruction *Callsite, Function *F) {
+ FoundCalleeChain.push_back({Callsite, F});
+ };
+
+ auto *CalleeFunc = dyn_cast<Function>(CurCallee);
+ if (!CalleeFunc) {
+ auto *Alias = dyn_cast<GlobalAlias>(CurCallee);
+ assert(Alias);
+ CalleeFunc = dyn_cast<Function>(Alias->getAliasee());
+ assert(CalleeFunc);
+ }
+
+ // Look for tail calls in this function, and check if they either call the
+ // profiled callee directly, or indirectly (via a recursive search).
+ // Only succeed if there is a single unique tail call chain found between the
+ // profiled caller and callee, otherwise we could perform incorrect cloning.
+ bool FoundSingleCalleeChain = false;
+ for (auto &BB : *CalleeFunc) {
+ for (auto &I : BB) {
+ auto *CB = dyn_cast<CallBase>(&I);
+ if (!CB || !CB->isTailCall())
+ continue;
+ auto *CalledValue = CB->getCalledOperand();
+ auto *CalledFunction = CB->getCalledFunction();
+ if (CalledValue && !CalledFunction) {
+ CalledValue = CalledValue->stripPointerCasts();
+ // Stripping pointer casts can reveal a called function.
+ CalledFunction = dyn_cast<Function>(CalledValue);
+ }
+ // Check if this is an alias to a function. If so, get the
+ // called aliasee for the checks below.
+ if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
+ assert(!CalledFunction &&
+ "Expected null called function in callsite for alias");
+ CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
+ }
+ if (!CalledFunction)
+ continue;
+ if (CalledFunction == ProfiledCallee) {
+ if (FoundSingleCalleeChain) {
+ FoundMultipleCalleeChains = true;
+ return false;
+ }
+ FoundSingleCalleeChain = true;
+ FoundProfiledCalleeCount++;
+ FoundProfiledCalleeDepth += Depth;
+ if (Depth > FoundProfiledCalleeMaxDepth)
+ FoundProfiledCalleeMaxDepth = Depth;
+ SaveCallsiteInfo(&I, CalleeFunc);
+ } else if (findProfiledCalleeThroughTailCalls(
+ ProfiledCallee, CalledFunction, Depth + 1,
+ FoundCalleeChain, FoundMultipleCalleeChains)) {
+ if (FoundMultipleCalleeChains)
+ return false;
+ if (FoundSingleCalleeChain) {
+ FoundMultipleCalleeChains = true;
+ return false;
+ }
+ FoundSingleCalleeChain = true;
+ SaveCallsiteInfo(&I, CalleeFunc);
+ }
+ }
+ }
+
+ return FoundSingleCalleeChain;
+}
+
+bool ModuleCallsiteContextGraph::calleeMatchesFunc(
+ Instruction *Call, const Function *Func, const Function *CallerFunc,
+ std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain) {
auto *CB = dyn_cast<CallBase>(Call);
if (!CB->getCalledOperand())
return false;
@@ -1652,11 +1897,117 @@ bool ModuleCallsiteContextGraph::calleeMatchesFunc(Instruction *Call,
if (CalleeFunc == Func)
return true;
auto *Alias = dyn_cast<GlobalAlias>(CalleeVal);
- return Alias && Alias->getAliasee() == Func;
+ if (Alias && Alias->getAliasee() == Func)
+ return true;
+
+ // Recursively search for the profiled callee through tail calls starting with
+ // the actual Callee. The discovered tail call chain is saved in
+ // FoundCalleeChain, and we will fixup the graph to include these callsites
+ // after returning.
+ // FIXME: We will currently redo the same recursive walk if we find the same
+ // mismatched callee from another callsite. We can improve this with more
+ // bookkeeping of the created chain of new nodes for each mismatch.
+ unsigned Depth = 1;
+ bool FoundMultipleCalleeChains = false;
+ if (!findProfiledCalleeThroughTailCalls(Func, CalleeVal, Depth,
+ FoundCalleeChain,
+ FoundMultipleCalleeChains)) {
+ LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: "
+ << Func->getName() << " from " << CallerFunc->getName()
+ << " that actually called " << CalleeVal->getName()
+ << (FoundMultipleCalleeChains
+ ? " (found multiple possible chains)"
+ : "")
+ << "\n");
+ if (FoundMultipleCalleeChains)
+ FoundProfiledCalleeNonUniquelyCount++;
+ return false;
+ }
+
+ return true;
}
-bool IndexCallsiteContextGraph::calleeMatchesFunc(IndexCall &Call,
- const FunctionSummary *Func) {
+bool IndexCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
+ ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
+ std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain,
+ bool &FoundMultipleCalleeChains) {
+ // Stop recursive search if we have already explored the maximum specified
+ // depth.
+ if (Depth > TailCallSearchDepth)
+ return false;
+
+ auto CreateAndSaveCallsiteInfo = [&](ValueInfo Callee, FunctionSummary *FS) {
+ // Make a CallsiteInfo for each discovered callee, if one hasn't already
+ // been synthesized.
+ if (!FunctionCalleesToSynthesizedCallsiteInfos.count(FS) ||
+ !FunctionCalleesToSynthesizedCallsiteInfos[FS].count(Callee))
+ // StackIds is empty (we don't have debug info available in the index for
+ // these callsites)
+ FunctionCalleesToSynthesizedCallsiteInfos[FS][Callee] =
+ std::make_unique<CallsiteInfo>(Callee, SmallVector<unsigned>());
+ CallsiteInfo *NewCallsiteInfo =
+ FunctionCalleesToSynthesizedCallsiteInfos[FS][Callee].get();
+ FoundCalleeChain.push_back({NewCallsiteInfo, FS});
+ };
+
+ // Look for tail calls in this function, and check if they either call the
+ // profiled callee directly, or indirectly (via a recursive search).
+ // Only succeed if there is a single unique tail call chain found between the
+ // profiled caller and callee, otherwise we could perform incorrect cloning.
+ bool FoundSingleCalleeChain = false;
+ for (auto &S : CurCallee.getSummaryList()) {
+ if (!GlobalValue::isLocalLinkage(S->linkage()) &&
+ !isPrevailing(CurCallee.getGUID(), S.get()))
+ continue;
+ auto *FS = dyn_cast<FunctionSummary>(S->getBaseObject());
+ if (!FS)
+ continue;
+ auto FSVI = CurCallee;
+ auto *AS = dyn_cast<AliasSummary>(S.get());
+ if (AS)
+ FSVI = AS->getAliaseeVI();
+ for (auto &CallEdge : FS->calls()) {
+ if (!CallEdge.second.hasTailCall())
+ continue;
+ if (CallEdge.first == ProfiledCallee) {
+ if (FoundSingleCalleeChain) {
+ FoundMultipleCalleeChains = true;
+ return false;
+ }
+ FoundSingleCalleeChain = true;
+ FoundProfiledCalleeCount++;
+ FoundProfiledCalleeDepth += Depth;
+ if (Depth > FoundProfiledCalleeMaxDepth)
+ FoundProfiledCalleeMaxDepth = Depth;
+ CreateAndSaveCallsiteInfo(CallEdge.first, FS);
+ // Add FS to FSToVIMap in case it isn't already there.
+ assert(!FSToVIMap.count(FS) || FSToVIMap[FS] == FSVI);
+ FSToVIMap[FS] = FSVI;
+ } else if (findProfiledCalleeThroughTailCalls(
+ ProfiledCallee, CallEdge.first, Depth + 1,
+ FoundCalleeChain, FoundMultipleCalleeChains)) {
+ if (FoundMultipleCalleeChains)
+ return false;
+ if (FoundSingleCalleeChain) {
+ FoundMultipleCalleeChains = true;
+ return false;
+ }
+ FoundSingleCalleeChain = true;
+ CreateAndSaveCallsiteInfo(CallEdge.first, FS);
+ // Add FS to FSToVIMap in case it isn't already there.
+ assert(!FSToVIMap.count(FS) || FSToVIMap[FS] == FSVI);
+ FSToVIMap[FS] = FSVI;
+ }
+ }
+ }
+
+ return FoundSingleCalleeChain;
+}
+
+bool IndexCallsiteContextGraph::calleeMatchesFunc(
+ IndexCall &Call, const FunctionSummary *Func,
+ const FunctionSummary *CallerFunc,
+ std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain) {
ValueInfo Callee =
dyn_cast_if_present<CallsiteInfo *>(Call.getBase())->Callee;
// If there is no summary list then this is a call to an externally defined
@@ -1666,11 +2017,38 @@ bool IndexCallsiteContextGraph::calleeMatchesFunc(IndexCall &Call,
? nullptr
: dyn_cast<AliasSummary>(Callee.getSummaryList()[0].get());
assert(FSToVIMap.count(Func));
- return Callee == FSToVIMap[Func] ||
- // If callee is an alias, check the aliasee, since only function
- // summary base objects will contain the stack node summaries and thus
- // get a context node.
- (Alias && Alias->getAliaseeVI() == FSToVIMap[Func]);
+ auto FuncVI = FSToVIMap[Func];
+ if (Callee == FuncVI ||
+ // If callee is an alias, check the aliasee, since only function
+ // summary base objects will contain the stack node summaries and thus
+ // get a context node.
+ (Alias && Alias->getAliaseeVI() == FuncVI))
+ return true;
+
+ // Recursively search for the profiled callee through tail calls starting with
+ // the actual Callee. The discovered tail call chain is saved in
+ // FoundCalleeChain, and we will fixup the graph to include these callsites
+ // after returning.
+ // FIXME: We will currently redo the same recursive walk if we find the same
+ // mismatched callee from another callsite. We can improve this with more
+ // bookkeeping of the created chain of new nodes for each mismatch.
+ unsigned Depth = 1;
+ bool FoundMultipleCalleeChains = false;
+ if (!findProfiledCalleeThroughTailCalls(
+ FuncVI, Callee, Depth, FoundCalleeChain, FoundMultipleCalleeChains)) {
+ LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: " << FuncVI
+ << " from " << FSToVIMap[CallerFunc]
+ << " that actually called " << Callee
+ << (FoundMultipleCalleeChains
+ ? " (found multiple possible chains)"
+ : "")
+ << "\n");
+ if (FoundMultipleCalleeChains)
+ FoundProfiledCalleeNonUniquelyCount++;
+ return false;
+ }
+
+ return true;
}
static std::string getAllocTypeString(uint8_t AllocTypes) {
@@ -2533,6 +2911,9 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
// that were previously assigned to call PreviousAssignedFuncClone,
// to record that they now call NewFuncClone.
for (auto CE : Clone->CallerEdges) {
+ // Skip any that have been removed on an earlier iteration.
+ if (!CE)
+ continue;
// Ignore any caller that does not have a recorded callsite Call.
if (!CE->Caller->hasCall())
continue;
@@ -2945,6 +3326,42 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
NumClonesCreated = NumClones;
};
+ auto CloneCallsite = [&](const CallsiteInfo &StackNode, CallBase *CB,
+ Function *CalledFunction) {
+ // Perform cloning if not yet done.
+ CloneFuncIfNeeded(/*NumClones=*/StackNode.Clones.size());
+
+ // Should have skipped indirect calls via mayHaveMemprofSummary.
+ assert(CalledFunction);
+ assert(!IsMemProfClone(*CalledFunction));
+
+ // Update the calls per the summary info.
+ // Save orig name since it gets updated in the first iteration
+ // below.
+ auto CalleeOrigName = CalledFunction->getName();
+ for (unsigned J = 0; J < StackNode.Clones.size(); J++) {
+ // Do nothing if this version calls the original version of its
+ // callee.
+ if (!StackNode.Clones[J])
+ continue;
+ auto NewF = M.getOrInsertFunction(
+ getMemProfFuncName(CalleeOrigName, StackNode.Clones[J]),
+ CalledFunction->getFunctionType());
+ CallBase *CBClone;
+ // Copy 0 is the original function.
+ if (!J)
+ CBClone = CB;
+ else
+ CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
+ CBClone->setCalledFunction(NewF);
+ ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofCall", CBClone)
+ << ore::NV("Call", CBClone) << " in clone "
+ << ore::NV("Caller", CBClone->getFunction())
+ << " assigned to call function clone "
+ << ore::NV("Callee", NewF.getCallee()));
+ }
+ };
+
// Locate the summary for F.
ValueInfo TheFnVI = findValueInfoForFunc(F, M, ImportSummary);
// If not found, this could be an imported local (see comment in
@@ -2974,6 +3391,23 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
auto SI = FS->callsites().begin();
auto AI = FS->allocs().begin();
+ // To handle callsite infos synthesized for tail calls which have missing
+ // frames in the profiled context, map callee VI to the synthesized callsite
+ // info.
+ DenseMap<ValueInfo, CallsiteInfo> MapTailCallCalleeVIToCallsite;
+ // Iterate the callsites for this function in reverse, since we place all
+ // those synthesized for tail calls at the end.
+ for (auto CallsiteIt = FS->callsites().rbegin();
+ CallsiteIt != FS->callsites().rend(); CallsiteIt++) {
+ auto &Callsite = *CallsiteIt;
+ // Stop as soon as we see a non-synthesized callsite info (see comment
+ // above loop). All the entries added for discovered tail calls have empty
+ // stack ids.
+ if (!Callsite.StackIdIndices.empty())
+ break;
+ MapTailCallCalleeVIToCallsite.insert({Callsite.Callee, Callsite});
+ }
+
// Assume for now that the instructions are in the exact same order
// as when the summary was created, but confirm this is correct by
// matching the stack ids.
@@ -3126,37 +3560,16 @@ bool MemProfContextDisambiguation::applyImport(Module &M) {
}
#endif
- // Perform cloning if not yet done.
- CloneFuncIfNeeded(/*NumClones=*/StackNode.Clones.size());
-
- // Should have skipped indirect calls via mayHaveMemprofSummary.
- assert(CalledFunction);
- assert(!IsMemProfClone(*CalledFunction));
-
- // Update the calls per the summary info.
- // Save orig name since it gets updated in the first iteration
- // below.
- auto CalleeOrigName = CalledFunction->getName();
- for (unsigned J = 0; J < StackNode.Clones.size(); J++) {
- // Do nothing if this version calls the original version of its
- // callee.
- if (!StackNode.Clones[J])
- continue;
- auto NewF = M.getOrInsertFunction(
- getMemProfFuncName(CalleeOrigName, StackNode.Clones[J]),
- CalledFunction->getFunctionType());
- CallBase *CBClone;
- // Copy 0 is the original function.
- if (!J)
- CBClone = CB;
- else
- CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
- CBClone->setCalledFunction(NewF);
- ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofCall", CBClone)
- << ore::NV("Call", CBClone) << " in clone "
- << ore::NV("Caller", CBClone->getFunction())
- << " assigned to call function clone "
- << ore::NV("Callee", NewF.getCallee()));
+ CloneCallsite(StackNode, CB, CalledFunction);
+ } else if (CB->isTailCall()) {
+ // Locate the synthesized callsite info for the callee VI, if any was
+ // created, and use that for cloning.
+ ValueInfo CalleeVI =
+ findValueInfoForFunc(*CalledFunction, M, ImportSummary);
+ if (CalleeVI && MapTailCallCalleeVIToCallsite.count(CalleeVI)) {
+ auto Callsite = MapTailCallCalleeVIToCallsite.find(CalleeVI);
+ assert(Callsite != MapTailCallCalleeVIToCallsite.end());
+ CloneCallsite(Callsite->second, CB, CalledFunction);
}
}
// Memprof and callsite metadata on memory allocations no longer needed.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 96b612254ca5..c7e6f32c5406 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1723,6 +1723,30 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
I, Builder.CreateIntrinsic(Intrinsic::ctpop, {I.getType()},
{Builder.CreateOr(A, B)}));
+ // Fold the log2_ceil idiom:
+ // zext(ctpop(A) >u/!= 1) + (ctlz(A, true) ^ (BW - 1))
+ // -->
+ // BW - ctlz(A - 1, false)
+ const APInt *XorC;
+ if (match(&I,
+ m_c_Add(
+ m_ZExt(m_ICmp(Pred, m_Intrinsic<Intrinsic::ctpop>(m_Value(A)),
+ m_One())),
+ m_OneUse(m_ZExtOrSelf(m_OneUse(m_Xor(
+ m_OneUse(m_TruncOrSelf(m_OneUse(
+ m_Intrinsic<Intrinsic::ctlz>(m_Deferred(A), m_One())))),
+ m_APInt(XorC))))))) &&
+ (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_NE) &&
+ *XorC == A->getType()->getScalarSizeInBits() - 1) {
+ Value *Sub = Builder.CreateAdd(A, Constant::getAllOnesValue(A->getType()));
+ Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {A->getType()},
+ {Sub, Builder.getFalse()});
+ Value *Ret = Builder.CreateSub(
+ ConstantInt::get(A->getType(), A->getType()->getScalarSizeInBits()),
+ Ctlz, "", /*HasNUW*/ true, /*HasNSW*/ true);
+ return replaceInstUsesWith(I, Builder.CreateZExtOrTrunc(Ret, I.getType()));
+ }
+
if (Instruction *Res = foldSquareSumInt(I))
return Res;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index c03f50d75814..0620752e3213 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -46,44 +46,6 @@ static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS,
return Builder.CreateFCmp(NewPred, LHS, RHS);
}
-/// Transform BITWISE_OP(BSWAP(A),BSWAP(B)) or
-/// BITWISE_OP(BSWAP(A), Constant) to BSWAP(BITWISE_OP(A, B))
-/// \param I Binary operator to transform.
-/// \return Pointer to node that must replace the original binary operator, or
-/// null pointer if no transformation was made.
-static Value *SimplifyBSwap(BinaryOperator &I,
- InstCombiner::BuilderTy &Builder) {
- assert(I.isBitwiseLogicOp() && "Unexpected opcode for bswap simplifying");
-
- Value *OldLHS = I.getOperand(0);
- Value *OldRHS = I.getOperand(1);
-
- Value *NewLHS;
- if (!match(OldLHS, m_BSwap(m_Value(NewLHS))))
- return nullptr;
-
- Value *NewRHS;
- const APInt *C;
-
- if (match(OldRHS, m_BSwap(m_Value(NewRHS)))) {
- // OP( BSWAP(x), BSWAP(y) ) -> BSWAP( OP(x, y) )
- if (!OldLHS->hasOneUse() && !OldRHS->hasOneUse())
- return nullptr;
- // NewRHS initialized by the matcher.
- } else if (match(OldRHS, m_APInt(C))) {
- // OP( BSWAP(x), CONSTANT ) -> BSWAP( OP(x, BSWAP(CONSTANT) ) )
- if (!OldLHS->hasOneUse())
- return nullptr;
- NewRHS = ConstantInt::get(I.getType(), C->byteSwap());
- } else
- return nullptr;
-
- Value *BinOp = Builder.CreateBinOp(I.getOpcode(), NewLHS, NewRHS);
- Function *F = Intrinsic::getDeclaration(I.getModule(), Intrinsic::bswap,
- I.getType());
- return Builder.CreateCall(F, BinOp);
-}
-
/// Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise
/// (V < Lo || V >= Hi). This method expects that Lo < Hi. IsSigned indicates
/// whether to treat V, Lo, and Hi as signed or not.
@@ -2159,6 +2121,64 @@ Instruction *InstCombinerImpl::foldBinOpOfDisplacedShifts(BinaryOperator &I) {
return BinaryOperator::Create(ShiftOp, NewC, ShAmt);
}
+// Fold and/or/xor with two equal intrinsic IDs:
+// bitwise(fshl (A, B, ShAmt), fshl(C, D, ShAmt))
+// -> fshl(bitwise(A, C), bitwise(B, D), ShAmt)
+// bitwise(fshr (A, B, ShAmt), fshr(C, D, ShAmt))
+// -> fshr(bitwise(A, C), bitwise(B, D), ShAmt)
+// bitwise(bswap(A), bswap(B)) -> bswap(bitwise(A, B))
+// bitwise(bswap(A), C) -> bswap(bitwise(A, bswap(C)))
+// bitwise(bitreverse(A), bitreverse(B)) -> bitreverse(bitwise(A, B))
+// bitwise(bitreverse(A), C) -> bitreverse(bitwise(A, bitreverse(C)))
+static Instruction *
+foldBitwiseLogicWithIntrinsics(BinaryOperator &I,
+ InstCombiner::BuilderTy &Builder) {
+ assert(I.isBitwiseLogicOp() && "Should and/or/xor");
+ if (!I.getOperand(0)->hasOneUse())
+ return nullptr;
+ IntrinsicInst *X = dyn_cast<IntrinsicInst>(I.getOperand(0));
+ if (!X)
+ return nullptr;
+
+ IntrinsicInst *Y = dyn_cast<IntrinsicInst>(I.getOperand(1));
+ if (Y && (!Y->hasOneUse() || X->getIntrinsicID() != Y->getIntrinsicID()))
+ return nullptr;
+
+ Intrinsic::ID IID = X->getIntrinsicID();
+ const APInt *RHSC;
+ // Try to match constant RHS.
+ if (!Y && (!(IID == Intrinsic::bswap || IID == Intrinsic::bitreverse) ||
+ !match(I.getOperand(1), m_APInt(RHSC))))
+ return nullptr;
+
+ switch (IID) {
+ case Intrinsic::fshl:
+ case Intrinsic::fshr: {
+ if (X->getOperand(2) != Y->getOperand(2))
+ return nullptr;
+ Value *NewOp0 =
+ Builder.CreateBinOp(I.getOpcode(), X->getOperand(0), Y->getOperand(0));
+ Value *NewOp1 =
+ Builder.CreateBinOp(I.getOpcode(), X->getOperand(1), Y->getOperand(1));
+ Function *F = Intrinsic::getDeclaration(I.getModule(), IID, I.getType());
+ return CallInst::Create(F, {NewOp0, NewOp1, X->getOperand(2)});
+ }
+ case Intrinsic::bswap:
+ case Intrinsic::bitreverse: {
+ Value *NewOp0 = Builder.CreateBinOp(
+ I.getOpcode(), X->getOperand(0),
+ Y ? Y->getOperand(0)
+ : ConstantInt::get(I.getType(), IID == Intrinsic::bswap
+ ? RHSC->byteSwap()
+ : RHSC->reverseBits()));
+ Function *F = Intrinsic::getDeclaration(I.getModule(), IID, I.getType());
+ return CallInst::Create(F, {NewOp0});
+ }
+ default:
+ return nullptr;
+ }
+}
+
// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
// here. We should standardize that construct where it is needed or choose some
// other way to ensure that commutated variants of patterns are not missed.
@@ -2194,9 +2214,6 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
if (Value *V = foldUsingDistributiveLaws(I))
return replaceInstUsesWith(I, V);
- if (Value *V = SimplifyBSwap(I, Builder))
- return replaceInstUsesWith(I, V);
-
if (Instruction *R = foldBinOpShiftWithShift(I))
return R;
@@ -2688,6 +2705,9 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
return Res;
+ if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
+ return Res;
+
return nullptr;
}
@@ -3347,9 +3367,6 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
if (Value *V = foldUsingDistributiveLaws(I))
return replaceInstUsesWith(I, V);
- if (Value *V = SimplifyBSwap(I, Builder))
- return replaceInstUsesWith(I, V);
-
Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
Type *Ty = I.getType();
if (Ty->isIntOrIntVectorTy(1)) {
@@ -3884,6 +3901,9 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
return BinaryOperator::CreateAnd(X, ConstantInt::get(Ty, *C1 | *C2));
}
+ if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
+ return Res;
+
return nullptr;
}
@@ -4507,9 +4527,6 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
if (SimplifyDemandedInstructionBits(I))
return &I;
- if (Value *V = SimplifyBSwap(I, Builder))
- return replaceInstUsesWith(I, V);
-
if (Instruction *R = foldNot(I))
return R;
@@ -4799,5 +4816,8 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
if (Instruction *Res = foldBinOpOfDisplacedShifts(I))
return Res;
+ if (Instruction *Res = foldBitwiseLogicWithIntrinsics(I, Builder))
+ return Res;
+
return nullptr;
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 40b48699f758..64fbd5543a9e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1884,6 +1884,10 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
return crossLogicOpFold;
}
+ // Try to fold into bitreverse if bswap is the root of the expression tree.
+ if (Instruction *BitOp = matchBSwapOrBitReverse(*II, /*MatchBSwaps*/ false,
+ /*MatchBitReversals*/ true))
+ return BitOp;
break;
}
case Intrinsic::masked_load:
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index ab55f235920a..21bfc91148bf 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1704,11 +1704,11 @@ Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
if (CmpRHS != CmpLHS && isa<Constant>(CmpRHS) && !isa<Constant>(CmpLHS)) {
if (CmpLHS == TrueVal && Pred == ICmpInst::ICMP_EQ) {
// Transform (X == C) ? X : Y -> (X == C) ? C : Y
- SI.setOperand(1, CmpRHS);
+ replaceOperand(SI, 1, CmpRHS);
Changed = true;
} else if (CmpLHS == FalseVal && Pred == ICmpInst::ICMP_NE) {
// Transform (X != C) ? Y : X -> (X != C) ? Y : C
- SI.setOperand(2, CmpRHS);
+ replaceOperand(SI, 2, CmpRHS);
Changed = true;
}
}
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index e3deafa49bd9..5e7e08eaa997 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -216,7 +216,7 @@ static cl::opt<bool> ClInstrumentWrites(
cl::Hidden, cl::init(true));
static cl::opt<bool>
- ClUseStackSafety("asan-use-stack-safety", cl::Hidden, cl::init(false),
+ ClUseStackSafety("asan-use-stack-safety", cl::Hidden, cl::init(true),
cl::Hidden, cl::desc("Use Stack Safety analysis results"),
cl::Optional);
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 6b95c7028d93..c20fc942eaf0 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -617,9 +617,7 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
std::vector<uint8_t> Indexes;
JamCRC JC;
for (auto &BB : F) {
- const Instruction *TI = BB.getTerminator();
- for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
- BasicBlock *Succ = TI->getSuccessor(I);
+ for (BasicBlock *Succ : successors(&BB)) {
auto BI = findBBInfo(Succ);
if (BI == nullptr)
continue;
@@ -658,10 +656,10 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
<< " CRC = " << JC.getCRC()
<< ", Selects = " << SIVisitor.getNumOfSelectInsts()
<< ", Edges = " << MST.numEdges() << ", ICSites = "
- << ValueSites[IPVK_IndirectCallTarget].size());
- LLVM_DEBUG(dbgs() << ", Memops = " << ValueSites[IPVK_MemOPSize].size()
- << ", High32 CRC = " << JCH.getCRC());
- LLVM_DEBUG(dbgs() << ", Hash = " << FunctionHash << "\n";);
+ << ValueSites[IPVK_IndirectCallTarget].size()
+ << ", Memops = " << ValueSites[IPVK_MemOPSize].size()
+ << ", High32 CRC = " << JCH.getCRC()
+ << ", Hash = " << FunctionHash << "\n";);
if (PGOTraceFuncHash != "-" && F.getName().contains(PGOTraceFuncHash))
dbgs() << "Funcname=" << F.getName() << ", Hash=" << FunctionHash
diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index eef94636578d..533cefaf1061 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -207,6 +207,12 @@ struct FlattenInfo {
match(MatchedMul, m_c_Mul(m_Trunc(m_Specific(OuterInductionPHI)),
m_Value(MatchedItCount)));
+ // Matches the pattern ptr+i*M+j, with the two additions being done via GEP.
+ bool IsGEP = match(U, m_GEP(m_GEP(m_Value(), m_Value(MatchedMul)),
+ m_Specific(InnerInductionPHI))) &&
+ match(MatchedMul, m_c_Mul(m_Specific(OuterInductionPHI),
+ m_Value(MatchedItCount)));
+
if (!MatchedItCount)
return false;
@@ -224,7 +230,7 @@ struct FlattenInfo {
// Look through extends if the IV has been widened. Don't look through
// extends if we already looked through a trunc.
- if (Widened && IsAdd &&
+ if (Widened && (IsAdd || IsGEP) &&
(isa<SExtInst>(MatchedItCount) || isa<ZExtInst>(MatchedItCount))) {
assert(MatchedItCount->getType() == InnerInductionPHI->getType() &&
"Unexpected type mismatch in types after widening");
@@ -236,7 +242,7 @@ struct FlattenInfo {
LLVM_DEBUG(dbgs() << "Looking for inner trip count: ";
InnerTripCount->dump());
- if ((IsAdd || IsAddTrunc) && MatchedItCount == InnerTripCount) {
+ if ((IsAdd || IsAddTrunc || IsGEP) && MatchedItCount == InnerTripCount) {
LLVM_DEBUG(dbgs() << "Found. This sse is optimisable\n");
ValidOuterPHIUses.insert(MatchedMul);
LinearIVUses.insert(U);
@@ -646,33 +652,40 @@ static OverflowResult checkOverflow(FlattenInfo &FI, DominatorTree *DT,
if (OR != OverflowResult::MayOverflow)
return OR;
- for (Value *V : FI.LinearIVUses) {
- for (Value *U : V->users()) {
- if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {
- for (Value *GEPUser : U->users()) {
- auto *GEPUserInst = cast<Instruction>(GEPUser);
- if (!isa<LoadInst>(GEPUserInst) &&
- !(isa<StoreInst>(GEPUserInst) &&
- GEP == GEPUserInst->getOperand(1)))
- continue;
- if (!isGuaranteedToExecuteForEveryIteration(GEPUserInst,
- FI.InnerLoop))
- continue;
- // The IV is used as the operand of a GEP which dominates the loop
- // latch, and the IV is at least as wide as the address space of the
- // GEP. In this case, the GEP would wrap around the address space
- // before the IV increment wraps, which would be UB.
- if (GEP->isInBounds() &&
- V->getType()->getIntegerBitWidth() >=
- DL.getPointerTypeSizeInBits(GEP->getType())) {
- LLVM_DEBUG(
- dbgs() << "use of linear IV would be UB if overflow occurred: ";
- GEP->dump());
- return OverflowResult::NeverOverflows;
- }
- }
+ auto CheckGEP = [&](GetElementPtrInst *GEP, Value *GEPOperand) {
+ for (Value *GEPUser : GEP->users()) {
+ auto *GEPUserInst = cast<Instruction>(GEPUser);
+ if (!isa<LoadInst>(GEPUserInst) &&
+ !(isa<StoreInst>(GEPUserInst) && GEP == GEPUserInst->getOperand(1)))
+ continue;
+ if (!isGuaranteedToExecuteForEveryIteration(GEPUserInst, FI.InnerLoop))
+ continue;
+ // The IV is used as the operand of a GEP which dominates the loop
+ // latch, and the IV is at least as wide as the address space of the
+ // GEP. In this case, the GEP would wrap around the address space
+ // before the IV increment wraps, which would be UB.
+ if (GEP->isInBounds() &&
+ GEPOperand->getType()->getIntegerBitWidth() >=
+ DL.getPointerTypeSizeInBits(GEP->getType())) {
+ LLVM_DEBUG(
+ dbgs() << "use of linear IV would be UB if overflow occurred: ";
+ GEP->dump());
+ return true;
}
}
+ return false;
+ };
+
+ // Check if any IV user is, or is used by, a GEP that would cause UB if the
+ // multiply overflows.
+ for (Value *V : FI.LinearIVUses) {
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(V))
+ if (GEP->getNumIndices() == 1 && CheckGEP(GEP, GEP->getOperand(1)))
+ return OverflowResult::NeverOverflows;
+ for (Value *U : V->users())
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(U))
+ if (CheckGEP(GEP, V))
+ return OverflowResult::NeverOverflows;
}
return OverflowResult::MayOverflow;
@@ -778,6 +791,18 @@ static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
OuterValue = Builder.CreateTrunc(FI.OuterInductionPHI, V->getType(),
"flatten.trunciv");
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(V)) {
+ // Replace the GEP with one that uses OuterValue as the offset.
+ auto *InnerGEP = cast<GetElementPtrInst>(GEP->getOperand(0));
+ Value *Base = InnerGEP->getOperand(0);
+ // When the base of the GEP doesn't dominate the outer induction phi then
+ // we need to insert the new GEP where the old GEP was.
+ if (!DT->dominates(Base, &*Builder.GetInsertPoint()))
+ Builder.SetInsertPoint(cast<Instruction>(V));
+ OuterValue = Builder.CreateGEP(GEP->getSourceElementType(), Base,
+ OuterValue, "flatten." + V->getName());
+ }
+
LLVM_DEBUG(dbgs() << "Replacing: "; V->dump(); dbgs() << "with: ";
OuterValue->dump());
V->replaceAllUsesWith(OuterValue);
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 3f02441b74ba..b98f823ab00b 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1975,19 +1975,10 @@ insertRelocationStores(iterator_range<Value::user_iterator> GCRelocs,
assert(AllocaMap.count(OriginalValue));
Value *Alloca = AllocaMap[OriginalValue];
- // Emit store into the related alloca
- // All gc_relocates are i8 addrspace(1)* typed, and it must be bitcasted to
- // the correct type according to alloca.
+ // Emit store into the related alloca.
assert(Relocate->getNextNode() &&
"Should always have one since it's not a terminator");
- IRBuilder<> Builder(Relocate->getNextNode());
- Value *CastedRelocatedValue =
- Builder.CreateBitCast(Relocate,
- cast<AllocaInst>(Alloca)->getAllocatedType(),
- suffixed_name_or(Relocate, ".casted", ""));
-
- new StoreInst(CastedRelocatedValue, Alloca,
- cast<Instruction>(CastedRelocatedValue)->getNextNode());
+ new StoreInst(Relocate, Alloca, Relocate->getNextNode());
#ifndef NDEBUG
VisitedLiveValues.insert(OriginalValue);
@@ -2620,13 +2611,9 @@ static bool inlineGetBaseAndOffset(Function &F,
Value *Base =
findBasePointer(Callsite->getOperand(0), DVCache, KnownBases);
assert(!DVCache.count(Callsite));
- auto *BaseBC = IRBuilder<>(Callsite).CreateBitCast(
- Base, Callsite->getType(), suffixed_name_or(Base, ".cast", ""));
- if (BaseBC != Base)
- DVCache[BaseBC] = Base;
- Callsite->replaceAllUsesWith(BaseBC);
- if (!BaseBC->hasName())
- BaseBC->takeName(Callsite);
+ Callsite->replaceAllUsesWith(Base);
+ if (!Base->hasName())
+ Base->takeName(Callsite);
Callsite->eraseFromParent();
break;
}
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 225dd454068c..d2fed11445e4 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -1093,67 +1093,25 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
// => add the offset
//
// %gep2 ; clone of %gep
- // %new.gep = gep %gep2, <offset / sizeof(*%gep)>
+ // %new.gep = gep i8, %gep2, %offset
// %gep ; will be removed
// ... %gep ...
//
// => replace all uses of %gep with %new.gep and remove %gep
//
// %gep2 ; clone of %gep
- // %new.gep = gep %gep2, <offset / sizeof(*%gep)>
- // ... %new.gep ...
- //
- // If AccumulativeByteOffset is not a multiple of sizeof(*%gep), we emit an
- // uglygep (http://llvm.org/docs/GetElementPtr.html#what-s-an-uglygep):
- // bitcast %gep2 to i8*, add the offset, and bitcast the result back to the
- // type of %gep.
- //
- // %gep2 ; clone of %gep
- // %0 = bitcast %gep2 to i8*
- // %uglygep = gep %0, <offset>
- // %new.gep = bitcast %uglygep to <type of %gep>
+ // %new.gep = gep i8, %gep2, %offset
// ... %new.gep ...
Instruction *NewGEP = GEP->clone();
NewGEP->insertBefore(GEP);
- // Per ANSI C standard, signed / unsigned = unsigned and signed % unsigned =
- // unsigned.. Therefore, we cast ElementTypeSizeOfGEP to signed because it is
- // used with unsigned integers later.
- int64_t ElementTypeSizeOfGEP = static_cast<int64_t>(
- DL->getTypeAllocSize(GEP->getResultElementType()));
Type *PtrIdxTy = DL->getIndexType(GEP->getType());
- if (AccumulativeByteOffset % ElementTypeSizeOfGEP == 0) {
- // Very likely. As long as %gep is naturally aligned, the byte offset we
- // extracted should be a multiple of sizeof(*%gep).
- int64_t Index = AccumulativeByteOffset / ElementTypeSizeOfGEP;
- NewGEP = GetElementPtrInst::Create(GEP->getResultElementType(), NewGEP,
- ConstantInt::get(PtrIdxTy, Index, true),
- GEP->getName(), GEP);
- NewGEP->copyMetadata(*GEP);
- // Inherit the inbounds attribute of the original GEP.
- cast<GetElementPtrInst>(NewGEP)->setIsInBounds(GEPWasInBounds);
- } else {
- // Unlikely but possible. For example,
- // #pragma pack(1)
- // struct S {
- // int a[3];
- // int64 b[8];
- // };
- // #pragma pack()
- //
- // Suppose the gep before extraction is &s[i + 1].b[j + 3]. After
- // extraction, it becomes &s[i].b[j] and AccumulativeByteOffset is
- // sizeof(S) + 3 * sizeof(int64) = 100, which is not a multiple of
- // sizeof(int64).
- //
- // Emit an uglygep in this case.
- IRBuilder<> Builder(GEP);
- NewGEP = cast<Instruction>(Builder.CreateGEP(
- Builder.getInt8Ty(), NewGEP,
- {ConstantInt::get(PtrIdxTy, AccumulativeByteOffset, true)}, "uglygep",
- GEPWasInBounds));
- NewGEP->copyMetadata(*GEP);
- }
+ IRBuilder<> Builder(GEP);
+ NewGEP = cast<Instruction>(Builder.CreateGEP(
+ Builder.getInt8Ty(), NewGEP,
+ {ConstantInt::get(PtrIdxTy, AccumulativeByteOffset, true)},
+ GEP->getName(), GEPWasInBounds));
+ NewGEP->copyMetadata(*GEP);
GEP->replaceAllUsesWith(NewGEP);
GEP->eraseFromParent();
diff --git a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
index ca1f3a0c0ae3..2cce6eb22341 100644
--- a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
@@ -233,13 +233,9 @@ private:
void factorArrayIndex(Value *ArrayIdx, const SCEV *Base, uint64_t ElementSize,
GetElementPtrInst *GEP);
- // Emit code that computes the "bump" from Basis to C. If the candidate is a
- // GEP and the bump is not divisible by the element size of the GEP, this
- // function sets the BumpWithUglyGEP flag to notify its caller to bump the
- // basis using an ugly GEP.
+ // Emit code that computes the "bump" from Basis to C.
static Value *emitBump(const Candidate &Basis, const Candidate &C,
- IRBuilder<> &Builder, const DataLayout *DL,
- bool &BumpWithUglyGEP);
+ IRBuilder<> &Builder, const DataLayout *DL);
const DataLayout *DL = nullptr;
DominatorTree *DT = nullptr;
@@ -581,26 +577,11 @@ static void unifyBitWidth(APInt &A, APInt &B) {
Value *StraightLineStrengthReduce::emitBump(const Candidate &Basis,
const Candidate &C,
IRBuilder<> &Builder,
- const DataLayout *DL,
- bool &BumpWithUglyGEP) {
+ const DataLayout *DL) {
APInt Idx = C.Index->getValue(), BasisIdx = Basis.Index->getValue();
unifyBitWidth(Idx, BasisIdx);
APInt IndexOffset = Idx - BasisIdx;
- BumpWithUglyGEP = false;
- if (Basis.CandidateKind == Candidate::GEP) {
- APInt ElementSize(
- IndexOffset.getBitWidth(),
- DL->getTypeAllocSize(
- cast<GetElementPtrInst>(Basis.Ins)->getResultElementType()));
- APInt Q, R;
- APInt::sdivrem(IndexOffset, ElementSize, Q, R);
- if (R == 0)
- IndexOffset = Q;
- else
- BumpWithUglyGEP = true;
- }
-
// Compute Bump = C - Basis = (i' - i) * S.
// Common case 1: if (i' - i) is 1, Bump = S.
if (IndexOffset == 1)
@@ -645,8 +626,7 @@ void StraightLineStrengthReduce::rewriteCandidateWithBasis(
return;
IRBuilder<> Builder(C.Ins);
- bool BumpWithUglyGEP;
- Value *Bump = emitBump(Basis, C, Builder, DL, BumpWithUglyGEP);
+ Value *Bump = emitBump(Basis, C, Builder, DL);
Value *Reduced = nullptr; // equivalent to but weaker than C.Ins
switch (C.CandidateKind) {
case Candidate::Add:
@@ -673,28 +653,13 @@ void StraightLineStrengthReduce::rewriteCandidateWithBasis(
}
break;
}
- case Candidate::GEP:
- {
- Type *OffsetTy = DL->getIndexType(C.Ins->getType());
+ case Candidate::GEP: {
bool InBounds = cast<GetElementPtrInst>(C.Ins)->isInBounds();
- if (BumpWithUglyGEP) {
- // C = (char *)Basis + Bump
- unsigned AS = Basis.Ins->getType()->getPointerAddressSpace();
- Type *CharTy = PointerType::get(Basis.Ins->getContext(), AS);
- Reduced = Builder.CreateBitCast(Basis.Ins, CharTy);
- Reduced =
- Builder.CreateGEP(Builder.getInt8Ty(), Reduced, Bump, "", InBounds);
- Reduced = Builder.CreateBitCast(Reduced, C.Ins->getType());
- } else {
- // C = gep Basis, Bump
- // Canonicalize bump to pointer size.
- Bump = Builder.CreateSExtOrTrunc(Bump, OffsetTy);
- Reduced = Builder.CreateGEP(
- cast<GetElementPtrInst>(Basis.Ins)->getResultElementType(), Basis.Ins,
- Bump, "", InBounds);
- }
- break;
- }
+ // C = (char *)Basis + Bump
+ Reduced =
+ Builder.CreateGEP(Builder.getInt8Ty(), Basis.Ins, Bump, "", InBounds);
+ break;
+ }
default:
llvm_unreachable("C.CandidateKind is invalid");
};
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index c76cc9db16d7..b9cad764aaef 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -3905,7 +3905,8 @@ bool llvm::recognizeBSwapOrBitReverseIdiom(
SmallVectorImpl<Instruction *> &InsertedInsts) {
if (!match(I, m_Or(m_Value(), m_Value())) &&
!match(I, m_FShl(m_Value(), m_Value(), m_Value())) &&
- !match(I, m_FShr(m_Value(), m_Value(), m_Value())))
+ !match(I, m_FShr(m_Value(), m_Value(), m_Value())) &&
+ !match(I, m_BSwap(m_Value())))
return false;
if (!MatchBSwaps && !MatchBitReversals)
return false;
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 61d891d65346..7515e539e7fb 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -6919,18 +6919,17 @@ static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder,
auto *Ty = cast<IntegerType>(SI->getCondition()->getType());
Builder.SetInsertPoint(SI);
- auto *ShiftC = ConstantInt::get(Ty, Shift);
- auto *Sub = Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base));
- auto *LShr = Builder.CreateLShr(Sub, ShiftC);
- auto *Shl = Builder.CreateShl(Sub, Ty->getBitWidth() - Shift);
- auto *Rot = Builder.CreateOr(LShr, Shl);
+ Value *Sub =
+ Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base));
+ Value *Rot = Builder.CreateIntrinsic(
+ Ty, Intrinsic::fshl,
+ {Sub, Sub, ConstantInt::get(Ty, Ty->getBitWidth() - Shift)});
SI->replaceUsesOfWith(SI->getCondition(), Rot);
for (auto Case : SI->cases()) {
auto *Orig = Case.getCaseValue();
auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base);
- Case.setValue(
- cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(ShiftC->getValue()))));
+ Case.setValue(cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(Shift))));
}
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 51ce88480c08..9743fa0e7402 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5004,9 +5004,8 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor(
VectorizationFactor Candidate(i, C.first, ScalarCost.ScalarCost);
#ifndef NDEBUG
- unsigned AssumedMinimumVscale = 1;
- if (std::optional<unsigned> VScale = getVScaleForTuning(OrigLoop, TTI))
- AssumedMinimumVscale = *VScale;
+ unsigned AssumedMinimumVscale =
+ getVScaleForTuning(OrigLoop, TTI).value_or(1);
unsigned Width =
Candidate.Width.isScalable()
? Candidate.Width.getKnownMinValue() * AssumedMinimumVscale
@@ -8031,6 +8030,7 @@ void VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlan &Plan) {
VPValue *EdgeMask = createEdgeMask(Predecessor, BB, Plan);
if (!EdgeMask) { // Mask of predecessor is all-one so mask of block is too.
BlockMaskCache[BB] = EdgeMask;
+ return;
}
if (!BlockMask) { // BlockMask has its initialized nullptr value.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8e22b54f002d..055fbb00871f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -6894,6 +6894,31 @@ protected:
};
} // namespace
+/// Returns the cost of the shuffle instructions with the given \p Kind, vector
+/// type \p Tp and optional \p Mask. Adds SLP-specifc cost estimation for insert
+/// subvector pattern.
+static InstructionCost
+getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
+ VectorType *Tp, ArrayRef<int> Mask = std::nullopt,
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
+ int Index = 0, VectorType *SubTp = nullptr,
+ ArrayRef<const Value *> Args = std::nullopt) {
+ if (Kind != TTI::SK_PermuteTwoSrc)
+ return TTI.getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args);
+ int NumSrcElts = Tp->getElementCount().getKnownMinValue();
+ int NumSubElts;
+ if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
+ Mask, NumSrcElts, NumSubElts, Index)) {
+ if (Index + NumSubElts > NumSrcElts &&
+ Index + NumSrcElts <= static_cast<int>(Mask.size()))
+ return TTI.getShuffleCost(
+ TTI::SK_InsertSubvector,
+ FixedVectorType::get(Tp->getElementType(), Mask.size()), std::nullopt,
+ TTI::TCK_RecipThroughput, Index, Tp);
+ }
+ return TTI.getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args);
+}
+
/// Merges shuffle masks and emits final shuffle instruction, if required. It
/// supports shuffling of 2 input vectors. It implements lazy shuffles emission,
/// when the actual shuffle instruction is generated only if this is actually
@@ -7141,15 +7166,15 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
std::optional<TTI::ShuffleKind> RegShuffleKind =
CheckPerRegistersShuffle(SubMask);
if (!RegShuffleKind) {
- Cost += TTI.getShuffleCost(
- *ShuffleKinds[Part],
+ Cost += ::getShuffleCost(
+ TTI, *ShuffleKinds[Part],
FixedVectorType::get(VL.front()->getType(), NumElts), MaskSlice);
continue;
}
if (*RegShuffleKind != TTI::SK_PermuteSingleSrc ||
!ShuffleVectorInst::isIdentityMask(SubMask, EltsPerVector)) {
- Cost += TTI.getShuffleCost(
- *RegShuffleKind,
+ Cost += ::getShuffleCost(
+ TTI, *RegShuffleKind,
FixedVectorType::get(VL.front()->getType(), EltsPerVector),
SubMask);
}
@@ -7222,8 +7247,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
- return TTI.getShuffleCost(TTI::SK_PermuteTwoSrc,
- cast<VectorType>(V1->getType()), Mask);
+ return ::getShuffleCost(TTI, TTI::SK_PermuteTwoSrc,
+ cast<VectorType>(V1->getType()), Mask);
}
InstructionCost createShuffleVector(Value *V1, ArrayRef<int> Mask) const {
// Empty mask or identity mask are free.
@@ -8101,7 +8126,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
for (unsigned I = OffsetEnd + 1 - Offset; I < VecSz; ++I)
Mask[I] =
((I >= InMask.size()) || InMask.test(I)) ? PoisonMaskElem : I;
- Cost += TTI->getShuffleCost(TTI::SK_PermuteTwoSrc, InsertVecTy, Mask);
+ Cost +=
+ ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, InsertVecTy, Mask);
}
}
return Cost;
@@ -8428,8 +8454,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return I->getOpcode() == E->getAltOpcode();
},
Mask);
- VecCost += TTIRef.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc,
- FinalVecTy, Mask);
+ VecCost += ::getShuffleCost(TTIRef, TargetTransformInfo::SK_PermuteTwoSrc,
+ FinalVecTy, Mask);
// Patterns like [fadd,fsub] can be combined into a single instruction
// in x86. Reordering them into [fsub,fadd] blocks this pattern. So we
// need to take into account their order when looking for the most used
@@ -9133,7 +9159,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
auto *FTy =
FixedVectorType::get(TEs.back()->Scalars.front()->getType(), VF);
InstructionCost C =
- TTI->getShuffleCost(TTI::SK_PermuteTwoSrc, FTy, Mask);
+ ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, FTy, Mask);
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of vector node and external "
"insertelement users.\n";
@@ -11991,8 +12017,12 @@ Value *BoUpSLP::vectorizeTree(
IRBuilder<>::InsertPointGuard Guard(Builder);
if (auto *IVec = dyn_cast<Instruction>(Vec))
Builder.SetInsertPoint(IVec->getNextNonDebugInstruction());
- Vec = Builder.CreateIntCast(Vec, VU->getType(),
- BWIt->second.second);
+ Vec = Builder.CreateIntCast(
+ Vec,
+ FixedVectorType::get(
+ cast<VectorType>(VU->getType())->getElementType(),
+ cast<FixedVectorType>(Vec->getType())->getNumElements()),
+ BWIt->second.second);
VectorCasts.try_emplace(Scalar, Vec);
} else {
Vec = VecIt->second;
@@ -13070,10 +13100,14 @@ bool BoUpSLP::collectValuesToDemote(
if (isa<Constant>(V))
return true;
- // If the value is not a vectorized instruction in the expression with only
- // one use, it cannot be demoted.
+ // If the value is not a vectorized instruction in the expression and not used
+ // by the insertelement instruction and not used in multiple vector nodes, it
+ // cannot be demoted.
auto *I = dyn_cast<Instruction>(V);
- if (!I || !I->hasOneUse() || !getTreeEntry(I) || !Visited.insert(I).second)
+ if (!I || !getTreeEntry(I) || MultiNodeScalars.contains(I) ||
+ !Visited.insert(I).second || all_of(I->users(), [&](User *U) {
+ return isa<InsertElementInst>(U) && !getTreeEntry(U);
+ }))
return false;
unsigned Start = 0;
@@ -13144,11 +13178,6 @@ bool BoUpSLP::collectValuesToDemote(
}
void BoUpSLP::computeMinimumValueSizes() {
- // If there are no external uses, the expression tree must be rooted by a
- // store. We can't demote in-memory values, so there is nothing to do here.
- if (ExternalUses.empty())
- return;
-
// We only attempt to truncate integer expressions.
auto &TreeRoot = VectorizableTree[0]->Scalars;
auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
generated by cgit v1.3 (git 2.53.0) at 2026-08-12 05:32:20 +0000