aboutsummaryrefslogtreecommitdiff
path: root/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/SelectionDAG/StatepointLowering.cpp')
-rw-r--r--lib/CodeGen/SelectionDAG/StatepointLowering.cpp109
1 files changed, 67 insertions, 42 deletions
diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 90a1b350fc94..395e9a8a4fc5 100644
--- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -1,9 +1,8 @@
//===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
@@ -348,16 +347,28 @@ static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
}
+static MachineMemOperand* getMachineMemOperand(MachineFunction &MF,
+ FrameIndexSDNode &FI) {
+ auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
+ auto MMOFlags = MachineMemOperand::MOStore |
+ MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
+ auto &MFI = MF.getFrameInfo();
+ return MF.getMachineMemOperand(PtrInfo, MMOFlags,
+ MFI.getObjectSize(FI.getIndex()),
+ MFI.getObjectAlignment(FI.getIndex()));
+}
+
/// Spill a value incoming to the statepoint. It might be either part of
/// vmstate
/// or gcstate. In both cases unconditionally spill it on the stack unless it
/// is a null constant. Return pair with first element being frame index
/// containing saved value and second element with outgoing chain from the
/// emitted store
-static std::pair<SDValue, SDValue>
+static std::tuple<SDValue, SDValue, MachineMemOperand*>
spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
SelectionDAGBuilder &Builder) {
SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
+ MachineMemOperand* MMO = nullptr;
// Emit new store if we didn't do it for this ptr before
if (!Loc.getNode()) {
@@ -367,10 +378,6 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
// We use TargetFrameIndex so that isel will not select it into LEA
Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
- // TODO: We can create TokenFactor node instead of
- // chaining stores one after another, this may allow
- // a bit more optimal scheduling for them
-
#ifndef NDEBUG
// Right now we always allocate spill slots that are of the same
// size as the value we're about to spill (the size of spillee can
@@ -382,15 +389,18 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
"Bad spill: stack slot does not match!");
#endif
+ auto &MF = Builder.DAG.getMachineFunction();
+ auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
- MachinePointerInfo::getFixedStack(
- Builder.DAG.getMachineFunction(), Index));
+ PtrInfo);
+ MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
+
Builder.StatepointLowering.setLocation(Incoming, Loc);
}
assert(Loc.getNode());
- return std::make_pair(Loc, Chain);
+ return std::make_tuple(Loc, Chain, MMO);
}
/// Lower a single value incoming to a statepoint node. This value can be
@@ -398,7 +408,11 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
/// case constants and allocas, then fall back to spilling if required.
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
SmallVectorImpl<SDValue> &Ops,
+ SmallVectorImpl<MachineMemOperand*> &MemRefs,
SelectionDAGBuilder &Builder) {
+ // Note: We know all of these spills are independent, but don't bother to
+ // exploit that chain wise. DAGCombine will happily do so as needed, so
+ // doing it here would be a small compile time win at most.
SDValue Chain = Builder.getRoot();
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
@@ -417,6 +431,11 @@ static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
"Incoming value is a frame index!");
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Builder.getFrameIndexTy()));
+
+ auto &MF = Builder.DAG.getMachineFunction();
+ auto *MMO = getMachineMemOperand(MF, *FI);
+ MemRefs.push_back(MMO);
+
} else if (LiveInOnly) {
// If this value is live in (not live-on-return, or live-through), we can
// treat it the same way patchpoint treats it's "live in" values. We'll
@@ -433,8 +452,10 @@ static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
// need to be optional since it requires a lot of complexity on the
// runtime side which not all would support.
auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
- Ops.push_back(Res.first);
- Chain = Res.second;
+ Ops.push_back(std::get<0>(Res));
+ if (auto *MMO = std::get<2>(Res))
+ MemRefs.push_back(MMO);
+ Chain = std::get<1>(Res);;
}
Builder.DAG.setRoot(Chain);
@@ -449,7 +470,7 @@ static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
/// will be set to the last value spilled (if any were).
static void
lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
- SelectionDAGBuilder::StatepointLoweringInfo &SI,
+ SmallVectorImpl<MachineMemOperand*> &MemRefs, SelectionDAGBuilder::StatepointLoweringInfo &SI,
SelectionDAGBuilder &Builder) {
// Lower the deopt and gc arguments for this statepoint. Layout will be:
// deopt argument length, deopt arguments.., gc arguments...
@@ -533,7 +554,7 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
if (!Incoming.getNode())
Incoming = Builder.getValue(V);
const bool LiveInValue = LiveInDeopt && !isGCValue(V);
- lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
+ lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, MemRefs, Builder);
}
// Finally, go ahead and lower all the gc arguments. There's no prefixed
@@ -544,11 +565,11 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
for (unsigned i = 0; i < SI.Bases.size(); ++i) {
const Value *Base = SI.Bases[i];
lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
- Ops, Builder);
+ Ops, MemRefs, Builder);
const Value *Ptr = SI.Ptrs[i];
lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
- Ops, Builder);
+ Ops, MemRefs, Builder);
}
// If there are any explicit spill slots passed to the statepoint, record
@@ -564,6 +585,10 @@ lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
"Incoming value is a frame index!");
Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
Builder.getFrameIndexTy()));
+
+ auto &MF = Builder.DAG.getMachineFunction();
+ auto *MMO = getMachineMemOperand(MF, *FI);
+ MemRefs.push_back(MMO);
}
}
@@ -630,7 +655,8 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
// Lower statepoint vmstate and gcstate arguments
SmallVector<SDValue, 10> LoweredMetaArgs;
- lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
+ SmallVector<MachineMemOperand*, 16> MemRefs;
+ lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, SI, *this);
// Now that we've emitted the spills, we need to update the root so that the
// call sequence is ordered correctly.
@@ -746,8 +772,9 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
// input. This allows someone else to chain off us as needed.
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
- SDNode *StatepointMCNode =
- DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
+ MachineSDNode *StatepointMCNode =
+ DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
+ DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
SDNode *SinkNode = StatepointMCNode;
@@ -799,7 +826,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
void
SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
const BasicBlock *EHPadBB /*= nullptr*/) {
- assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg &&
+ assert(ISP.getCall()->getCallingConv() != CallingConv::AnyReg &&
"anyregcc is not supported on statepoints!");
#ifndef NDEBUG
@@ -832,7 +859,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
}
StatepointLoweringInfo SI(DAG);
- populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
+ populateCallLoweringInfo(SI.CLI, ISP.getCall(),
ImmutableStatepoint::CallArgsBeginPos,
ISP.getNumCallArgs(), ActualCallee,
ISP.getActualReturnType(), false /* IsPatchPoint */);
@@ -859,7 +886,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
const GCResultInst *GCResult = ISP.getGCResult();
Type *RetTy = ISP.getActualReturnType();
if (!RetTy->isVoidTy() && GCResult) {
- if (GCResult->getParent() != ISP.getCallSite().getParent()) {
+ if (GCResult->getParent() != ISP.getCall()->getParent()) {
// Result value will be used in a different basic block so we need to
// export it now. Default exporting mechanism will not work here because
// statepoint call has a different type than the actual call. It means
@@ -871,7 +898,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
unsigned Reg = FuncInfo.CreateRegs(RetTy);
RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
DAG.getDataLayout(), Reg, RetTy,
- ISP.getCallSite().getCallingConv());
+ ISP.getCall()->getCallingConv());
SDValue Chain = DAG.getEntryNode();
RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
@@ -891,22 +918,22 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
}
void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
- ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
+ const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
bool VarArgDisallowed, bool ForceVoidReturnTy) {
StatepointLoweringInfo SI(DAG);
- unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
+ unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
populateCallLoweringInfo(
- SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
- ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
+ SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
+ ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
false);
if (!VarArgDisallowed)
- SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
+ SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
- auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
+ auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
- auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes());
+ auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
SI.ID = SD.StatepointID.getValueOr(DefaultID);
SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
@@ -918,15 +945,14 @@ void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
// NB! The GC arguments are deliberately left empty.
if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
- const Instruction *Inst = CS.getInstruction();
- ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
- setValue(Inst, ReturnVal);
+ ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
+ setValue(Call, ReturnVal);
}
}
void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
- ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
- LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
+ const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
+ LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
/* VarArgDisallowed = */ false,
/* ForceVoidReturnTy = */ false);
}
@@ -986,11 +1012,11 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
}
SDValue SpillSlot =
- DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
+ DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
- // Be conservative: flush all pending loads
- // TODO: Probably we can be less restrictive on this,
- // it may allow more scheduling opportunities.
+ // Note: We know all of these reloads are independent, but don't bother to
+ // exploit that chain wise. DAGCombine will happily do so as needed, so
+ // doing it here would be a small compile time win at most.
SDValue Chain = getRoot();
SDValue SpillLoad =
@@ -1000,7 +1026,6 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
*DerivedPtrLocation));
- // Again, be conservative, don't emit pending loads
DAG.setRoot(SpillLoad.getValue(1));
assert(SpillLoad.getNode());
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-07 14:46:36 +0000