diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp | 174 |
1 files changed, 110 insertions, 64 deletions
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp index 08e1d6b87b0d..df0c2080e079 100644 --- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp +++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "AMDGPU.h" +#include "AMDGPUTargetMachine.h" #include "GCNSubtarget.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/UniformityAnalysis.h" @@ -21,7 +22,6 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicsAMDGPU.h" -#include "llvm/InitializePasses.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" @@ -36,7 +36,9 @@ namespace { using StackEntry = std::pair<BasicBlock *, Value *>; using StackVector = SmallVector<StackEntry, 16>; -class SIAnnotateControlFlow : public FunctionPass { +class SIAnnotateControlFlow { +private: + Function *F; UniformityInfo *UA; Type *Boolean; @@ -49,18 +51,18 @@ class SIAnnotateControlFlow : public FunctionPass { UndefValue *BoolUndef; Constant *IntMaskZero; - Function *If; - Function *Else; - Function *IfBreak; - Function *Loop; - Function *EndCf; + Function *If = nullptr; + Function *Else = nullptr; + Function *IfBreak = nullptr; + Function *Loop = nullptr; + Function *EndCf = nullptr; DominatorTree *DT; StackVector Stack; LoopInfo *LI; - void initialize(Module &M, const GCNSubtarget &ST); + void initialize(const GCNSubtarget &ST); bool isUniform(BranchInst *T); @@ -88,41 +90,27 @@ class SIAnnotateControlFlow : public FunctionPass { bool closeControlFlow(BasicBlock *BB); -public: - static char ID; - - SIAnnotateControlFlow() : FunctionPass(ID) {} - - bool runOnFunction(Function &F) override; - - StringRef getPassName() const override { return "SI annotate control flow"; } + Function *getDecl(Function *&Cache, Intrinsic::ID ID, ArrayRef<Type *> Tys) { + if (!Cache) + Cache = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys); + return Cache; + } - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired<LoopInfoWrapperPass>(); - AU.addRequired<DominatorTreeWrapperPass>(); - AU.addRequired<UniformityInfoWrapperPass>(); - AU.addPreserved<LoopInfoWrapperPass>(); - AU.addPreserved<DominatorTreeWrapperPass>(); - AU.addRequired<TargetPassConfig>(); - FunctionPass::getAnalysisUsage(AU); +public: + SIAnnotateControlFlow(Function &F, const GCNSubtarget &ST, DominatorTree &DT, + LoopInfo &LI, UniformityInfo &UA) + : F(&F), UA(&UA), DT(&DT), LI(&LI) { + initialize(ST); } + + bool run(); }; } // end anonymous namespace -INITIALIZE_PASS_BEGIN(SIAnnotateControlFlow, DEBUG_TYPE, - "Annotate SI Control Flow", false, false) -INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) -INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass) -INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) -INITIALIZE_PASS_END(SIAnnotateControlFlow, DEBUG_TYPE, - "Annotate SI Control Flow", false, false) - -char SIAnnotateControlFlow::ID = 0; - /// Initialize all the types and constants used in the pass -void SIAnnotateControlFlow::initialize(Module &M, const GCNSubtarget &ST) { - LLVMContext &Context = M.getContext(); +void SIAnnotateControlFlow::initialize(const GCNSubtarget &ST) { + LLVMContext &Context = F->getContext(); Void = Type::getVoidTy(Context); Boolean = Type::getInt1Ty(Context); @@ -134,14 +122,6 @@ void SIAnnotateControlFlow::initialize(Module &M, const GCNSubtarget &ST) { BoolFalse = ConstantInt::getFalse(Context); BoolUndef = PoisonValue::get(Boolean); IntMaskZero = ConstantInt::get(IntMask, 0); - - If = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_if, { IntMask }); - Else = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_else, - { IntMask, IntMask }); - IfBreak = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_if_break, - { IntMask }); - Loop = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_loop, { IntMask }); - EndCf = Intrinsic::getDeclaration(&M, Intrinsic::amdgcn_end_cf, { IntMask }); } /// Is the branch condition uniform or did the StructurizeCFG pass @@ -207,7 +187,8 @@ bool SIAnnotateControlFlow::openIf(BranchInst *Term) { return false; IRBuilder<> IRB(Term); - Value *IfCall = IRB.CreateCall(If, {Term->getCondition()}); + Value *IfCall = IRB.CreateCall(getDecl(If, Intrinsic::amdgcn_if, IntMask), + {Term->getCondition()}); Value *Cond = IRB.CreateExtractValue(IfCall, {0}); Value *Mask = IRB.CreateExtractValue(IfCall, {1}); Term->setCondition(Cond); @@ -222,7 +203,8 @@ bool SIAnnotateControlFlow::insertElse(BranchInst *Term) { } IRBuilder<> IRB(Term); - Value *ElseCall = IRB.CreateCall(Else, {popSaved()}); + Value *ElseCall = IRB.CreateCall( + getDecl(Else, Intrinsic::amdgcn_else, {IntMask, IntMask}), {popSaved()}); Value *Cond = IRB.CreateExtractValue(ElseCall, {0}); Value *Mask = IRB.CreateExtractValue(ElseCall, {1}); Term->setCondition(Cond); @@ -235,16 +217,22 @@ Value *SIAnnotateControlFlow::handleLoopCondition( Value *Cond, PHINode *Broken, llvm::Loop *L, BranchInst *Term) { auto CreateBreak = [this, Cond, Broken](Instruction *I) -> CallInst * { - return IRBuilder<>(I).CreateCall(IfBreak, {Cond, Broken}); + return IRBuilder<>(I).CreateCall( + getDecl(IfBreak, Intrinsic::amdgcn_if_break, IntMask), {Cond, Broken}); }; if (Instruction *Inst = dyn_cast<Instruction>(Cond)) { BasicBlock *Parent = Inst->getParent(); Instruction *Insert; - if (L->contains(Inst)) { + if (LI->getLoopFor(Parent) == L) { + // Insert IfBreak in the same BB as Cond, which can help + // SILowerControlFlow to know that it does not have to insert an + // AND with EXEC. Insert = Parent->getTerminator(); + } else if (L->contains(Inst)) { + Insert = Term; } else { - Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime(); + Insert = &*L->getHeader()->getFirstNonPHIOrDbgOrLifetime(); } return CreateBreak(Insert); @@ -259,7 +247,7 @@ Value *SIAnnotateControlFlow::handleLoopCondition( } if (isa<Argument>(Cond)) { - Instruction *Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime(); + Instruction *Insert = &*L->getHeader()->getFirstNonPHIOrDbgOrLifetime(); return CreateBreak(Insert); } @@ -296,7 +284,8 @@ bool SIAnnotateControlFlow::handleLoop(BranchInst *Term) { Broken->addIncoming(PHIValue, Pred); } - CallInst *LoopCall = IRBuilder<>(Term).CreateCall(Loop, {Arg}); + CallInst *LoopCall = IRBuilder<>(Term).CreateCall( + getDecl(Loop, Intrinsic::amdgcn_loop, IntMask), {Arg}); Term->setCondition(LoopCall); push(Term->getSuccessor(0), Arg); @@ -341,7 +330,7 @@ bool SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) { // condition, for now just avoid copying these DebugLocs so that stepping // out of the then/else block in a debugger doesn't step to the condition. IRB.SetCurrentDebugLocation(DebugLoc()); - IRB.CreateCall(EndCf, {Exec}); + IRB.CreateCall(getDecl(EndCf, Intrinsic::amdgcn_end_cf, IntMask), {Exec}); } return true; @@ -349,17 +338,12 @@ bool SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) { /// Annotate the control flow with intrinsics so the backend can /// recognize if/then/else and loops. -bool SIAnnotateControlFlow::runOnFunction(Function &F) { - DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); - LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); - UA = &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo(); - TargetPassConfig &TPC = getAnalysis<TargetPassConfig>(); - const TargetMachine &TM = TPC.getTM<TargetMachine>(); - +bool SIAnnotateControlFlow::run() { bool Changed = false; - initialize(*F.getParent(), TM.getSubtarget<GCNSubtarget>(F)); - for (df_iterator<BasicBlock *> I = df_begin(&F.getEntryBlock()), - E = df_end(&F.getEntryBlock()); I != E; ++I) { + + for (df_iterator<BasicBlock *> I = df_begin(&F->getEntryBlock()), + E = df_end(&F->getEntryBlock()); + I != E; ++I) { BasicBlock *BB = *I; BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator()); @@ -401,7 +385,69 @@ bool SIAnnotateControlFlow::runOnFunction(Function &F) { return Changed; } +PreservedAnalyses SIAnnotateControlFlowPass::run(Function &F, + FunctionAnalysisManager &FAM) { + const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F); + + DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F); + UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(F); + LoopInfo &LI = FAM.getResult<LoopAnalysis>(F); + + SIAnnotateControlFlow Impl(F, ST, DT, LI, UI); + + bool Changed = Impl.run(); + if (!Changed) + return PreservedAnalyses::all(); + + // TODO: Is LoopInfo preserved? + PreservedAnalyses PA = PreservedAnalyses::none(); + PA.preserve<DominatorTreeAnalysis>(); + return PA; +} + +class SIAnnotateControlFlowLegacy : public FunctionPass { +public: + static char ID; + + SIAnnotateControlFlowLegacy() : FunctionPass(ID) {} + + StringRef getPassName() const override { return "SI annotate control flow"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired<LoopInfoWrapperPass>(); + AU.addRequired<DominatorTreeWrapperPass>(); + AU.addRequired<UniformityInfoWrapperPass>(); + AU.addPreserved<LoopInfoWrapperPass>(); + AU.addPreserved<DominatorTreeWrapperPass>(); + AU.addRequired<TargetPassConfig>(); + FunctionPass::getAnalysisUsage(AU); + } + + bool runOnFunction(Function &F) override { + DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); + LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); + UniformityInfo &UI = + getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo(); + TargetPassConfig &TPC = getAnalysis<TargetPassConfig>(); + const TargetMachine &TM = TPC.getTM<TargetMachine>(); + const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F); + + SIAnnotateControlFlow Impl(F, ST, DT, LI, UI); + return Impl.run(); + } +}; + +INITIALIZE_PASS_BEGIN(SIAnnotateControlFlowLegacy, DEBUG_TYPE, + "Annotate SI Control Flow", false, false) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_END(SIAnnotateControlFlowLegacy, DEBUG_TYPE, + "Annotate SI Control Flow", false, false) + +char SIAnnotateControlFlowLegacy::ID = 0; + /// Create the annotation pass -FunctionPass *llvm::createSIAnnotateControlFlowPass() { - return new SIAnnotateControlFlow(); +FunctionPass *llvm::createSIAnnotateControlFlowLegacyPass() { + return new SIAnnotateControlFlowLegacy(); } |