aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp')
-rw-r--r--llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp77
1 files changed, 55 insertions, 22 deletions
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 0f2836e1e7fb..3168bcd53eda 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -13,7 +13,7 @@
/// All control flow is handled using predicated instructions and
/// a predicate stack. Each Scalar ALU controls the operations of 64 Vector
/// ALUs. The Scalar ALU can update the predicate for any of the Vector ALUs
-/// by writting to the 64-bit EXEC register (each bit corresponds to a
+/// by writing to the 64-bit EXEC register (each bit corresponds to a
/// single vector ALU). Typically, for predicates, a vector ALU will write
/// to its bit of the VCC register (like EXEC VCC is 64-bits, one for each
/// Vector ALU) and then the ScalarALU will AND the VCC register with the
@@ -38,7 +38,8 @@
/// %vgpr0 = V_ADD_F32 %vgpr0, %vgpr0 // Do the IF block of the branch
///
/// label0:
-/// %sgpr0 = S_OR_SAVEEXEC_B64 %sgpr0 // Restore the exec mask for the Then block
+/// %sgpr0 = S_OR_SAVEEXEC_B64 %sgpr0 // Restore the exec mask for the Then
+/// // block
/// %exec = S_XOR_B64 %sgpr0, %exec // Update the exec mask
/// S_BRANCH_EXECZ label1 // Use our branch optimization
/// // instruction again.
@@ -52,6 +53,8 @@
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveVariables.h"
+#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
using namespace llvm;
@@ -69,6 +72,8 @@ private:
const SIRegisterInfo *TRI = nullptr;
const SIInstrInfo *TII = nullptr;
LiveIntervals *LIS = nullptr;
+ LiveVariables *LV = nullptr;
+ MachineDominatorTree *MDT = nullptr;
MachineRegisterInfo *MRI = nullptr;
SetVector<MachineInstr*> LoweredEndCf;
DenseSet<Register> LoweredIf;
@@ -141,6 +146,7 @@ public:
void getAnalysisUsage(AnalysisUsage &AU) const override {
// Should preserve the same set that TwoAddressInstructions does.
+ AU.addPreserved<MachineDominatorTree>();
AU.addPreserved<SlotIndexes>();
AU.addPreserved<LiveIntervals>();
AU.addPreservedID(LiveVariablesID);
@@ -234,6 +240,8 @@ void SILowerControlFlow::emitIf(MachineInstr &MI) {
BuildMI(MBB, I, DL, TII->get(AndOpc), Tmp)
.addReg(CopyReg)
.add(Cond);
+ if (LV)
+ LV->replaceKillInstruction(Cond.getReg(), MI, *And);
setImpSCCDefDead(*And, true);
@@ -251,6 +259,8 @@ void SILowerControlFlow::emitIf(MachineInstr &MI) {
MachineInstr *SetExec =
BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec)
.addReg(Tmp, RegState::Kill);
+ if (LV)
+ LV->getVarInfo(Tmp).Kills.push_back(SetExec);
// Skip ahead to the unconditional branch in case there are other terminators
// present.
@@ -304,6 +314,8 @@ void SILowerControlFlow::emitElse(MachineInstr &MI) {
MachineInstr *OrSaveExec =
BuildMI(MBB, Start, DL, TII->get(OrSaveExecOpc), SaveReg)
.add(MI.getOperand(1)); // Saved EXEC
+ if (LV)
+ LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *OrSaveExec);
MachineBasicBlock *DestBB = MI.getOperand(2).getMBB();
@@ -377,15 +389,22 @@ void SILowerControlFlow::emitIfBreak(MachineInstr &MI) {
And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), AndReg)
.addReg(Exec)
.add(MI.getOperand(1));
+ if (LV)
+ LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *And);
Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst)
.addReg(AndReg)
.add(MI.getOperand(2));
if (LIS)
LIS->createAndComputeVirtRegInterval(AndReg);
- } else
+ } else {
Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst)
.add(MI.getOperand(1))
.add(MI.getOperand(2));
+ if (LV)
+ LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *Or);
+ }
+ if (LV)
+ LV->replaceKillInstruction(MI.getOperand(2).getReg(), MI, *Or);
if (LIS) {
if (And)
@@ -471,6 +490,14 @@ MachineBasicBlock *SILowerControlFlow::emitEndCf(MachineInstr &MI) {
MachineBasicBlock *SplitBB = &MBB;
if (NeedBlockSplit) {
SplitBB = MBB.splitAt(MI, /*UpdateLiveIns*/true, LIS);
+ if (MDT && SplitBB != &MBB) {
+ MachineDomTreeNode *MBBNode = (*MDT)[&MBB];
+ SmallVector<MachineDomTreeNode *> Children(MBBNode->begin(),
+ MBBNode->end());
+ MachineDomTreeNode *SplitBBNode = MDT->addNewBlock(SplitBB, &MBB);
+ for (MachineDomTreeNode *Child : Children)
+ MDT->changeImmediateDominator(Child, SplitBBNode);
+ }
Opcode = OrTermrOpc;
InsPt = MI;
}
@@ -479,6 +506,8 @@ MachineBasicBlock *SILowerControlFlow::emitEndCf(MachineInstr &MI) {
BuildMI(MBB, InsPt, DL, TII->get(Opcode), Exec)
.addReg(Exec)
.add(MI.getOperand(0));
+ if (LV)
+ LV->replaceKillInstruction(MI.getOperand(0).getReg(), MI, *NewMI);
LoweredEndCf.insert(NewMI);
@@ -570,7 +599,12 @@ void SILowerControlFlow::optimizeEndCf() {
LLVM_DEBUG(dbgs() << "Skip redundant "; MI->dump());
if (LIS)
LIS->RemoveMachineInstrFromMaps(*MI);
+ Register Reg;
+ if (LV)
+ Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg();
MI->eraseFromParent();
+ if (LV)
+ LV->recomputeForSingleDefVirtReg(Reg);
removeMBBifRedundant(MBB);
}
}
@@ -686,6 +720,8 @@ void SILowerControlFlow::lowerInitExec(MachineBasicBlock *MBB,
auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg)
.addReg(InputReg)
.addImm((MI.getOperand(1).getImm() & Mask) | 0x70000);
+ if (LV)
+ LV->recomputeForSingleDefVirtReg(InputReg);
auto BfmMI =
BuildMI(*MBB, FirstMI, DL,
TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec)
@@ -694,6 +730,8 @@ void SILowerControlFlow::lowerInitExec(MachineBasicBlock *MBB,
auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32))
.addReg(CountReg, RegState::Kill)
.addImm(WavefrontSize);
+ if (LV)
+ LV->getVarInfo(CountReg).Kills.push_back(CmpMI);
auto CmovMI =
BuildMI(*MBB, FirstMI, DL,
TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
@@ -719,23 +757,6 @@ void SILowerControlFlow::lowerInitExec(MachineBasicBlock *MBB,
}
bool SILowerControlFlow::removeMBBifRedundant(MachineBasicBlock &MBB) {
- auto GetFallThroughSucc = [=](MachineBasicBlock *B) -> MachineBasicBlock * {
- auto *S = B->getNextNode();
- if (!S)
- return nullptr;
- if (B->isSuccessor(S)) {
- // The only fallthrough candidate
- MachineBasicBlock::iterator I(B->getFirstInstrTerminator());
- MachineBasicBlock::iterator E = B->end();
- for (; I != E; I++) {
- if (I->isBranch() && TII->getBranchDestBlock(*I) == S)
- // We have unoptimized branch to layout successor
- return nullptr;
- }
- }
- return S;
- };
-
for (auto &I : MBB.instrs()) {
if (!I.isDebugInstr() && !I.isUnconditionalBranch())
return false;
@@ -748,7 +769,7 @@ bool SILowerControlFlow::removeMBBifRedundant(MachineBasicBlock &MBB) {
while (!MBB.predecessors().empty()) {
MachineBasicBlock *P = *MBB.pred_begin();
- if (GetFallThroughSucc(P) == &MBB)
+ if (P->getFallThrough() == &MBB)
FallThrough = P;
P->ReplaceUsesOfBlockWith(&MBB, Succ);
}
@@ -757,10 +778,19 @@ bool SILowerControlFlow::removeMBBifRedundant(MachineBasicBlock &MBB) {
for (auto &I : MBB.instrs())
LIS->RemoveMachineInstrFromMaps(I);
}
+ if (MDT) {
+ // If Succ, the single successor of MBB, is dominated by MBB, MDT needs
+ // updating by changing Succ's idom to the one of MBB; otherwise, MBB must
+ // be a leaf node in MDT and could be erased directly.
+ if (MDT->dominates(&MBB, Succ))
+ MDT->changeImmediateDominator(MDT->getNode(Succ),
+ MDT->getNode(&MBB)->getIDom());
+ MDT->eraseNode(&MBB);
+ }
MBB.clear();
MBB.eraseFromParent();
if (FallThrough && !FallThrough->isLayoutSuccessor(Succ)) {
- if (!GetFallThroughSucc(Succ)) {
+ if (!Succ->canFallThrough()) {
MachineFunction *MF = FallThrough->getParent();
MachineFunction::iterator FallThroughPos(FallThrough);
MF->splice(std::next(FallThroughPos), Succ);
@@ -780,6 +810,9 @@ bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
// This doesn't actually need LiveIntervals, but we can preserve them.
LIS = getAnalysisIfAvailable<LiveIntervals>();
+ // This doesn't actually need LiveVariables, but we can preserve them.
+ LV = getAnalysisIfAvailable<LiveVariables>();
+ MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MRI = &MF.getRegInfo();
BoolRC = TRI->getBoolRC();
generated by cgit v1.3 (git 2.53.0) at 2026-08-10 04:50:15 +0000