aboutsummaryrefslogtreecommitdiff
path: root/lib/CodeGen/CGStmt.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/CGStmt.cpp')
-rw-r--r--lib/CodeGen/CGStmt.cpp524
1 files changed, 423 insertions, 101 deletions
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index 0bc51ddb5178..c2b64a7662b6 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -16,14 +16,15 @@
#include "CodeGenModule.h"
#include "TargetInfo.h"
#include "clang/AST/StmtVisitor.h"
-#include "clang/Sema/SemaDiagnostic.h"
#include "clang/Basic/PrettyStackTrace.h"
#include "clang/Basic/TargetInfo.h"
+#include "clang/Sema/LoopHint.h"
+#include "clang/Sema/SemaDiagnostic.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/IR/CallSite.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Intrinsics.h"
-#include "llvm/Support/CallSite.h"
using namespace clang;
using namespace CodeGen;
@@ -43,6 +44,7 @@ void CodeGenFunction::EmitStopPoint(const Stmt *S) {
void CodeGenFunction::EmitStmt(const Stmt *S) {
assert(S && "Null statement?");
+ PGO.setCurrentStmt(S);
// These statements have their own debug info handling.
if (EmitSimpleStmt(S))
@@ -75,7 +77,6 @@ void CodeGenFunction::EmitStmt(const Stmt *S) {
case Stmt::SEHExceptStmtClass:
case Stmt::SEHFinallyStmtClass:
case Stmt::MSDependentExistsStmtClass:
- case Stmt::OMPParallelDirectiveClass:
llvm_unreachable("invalid statement class to emit generically");
case Stmt::NullStmtClass:
case Stmt::CompoundStmtClass:
@@ -172,6 +173,54 @@ void CodeGenFunction::EmitStmt(const Stmt *S) {
case Stmt::SEHTryStmtClass:
EmitSEHTryStmt(cast<SEHTryStmt>(*S));
break;
+ case Stmt::SEHLeaveStmtClass:
+ EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S));
+ break;
+ case Stmt::OMPParallelDirectiveClass:
+ EmitOMPParallelDirective(cast<OMPParallelDirective>(*S));
+ break;
+ case Stmt::OMPSimdDirectiveClass:
+ EmitOMPSimdDirective(cast<OMPSimdDirective>(*S));
+ break;
+ case Stmt::OMPForDirectiveClass:
+ EmitOMPForDirective(cast<OMPForDirective>(*S));
+ break;
+ case Stmt::OMPSectionsDirectiveClass:
+ EmitOMPSectionsDirective(cast<OMPSectionsDirective>(*S));
+ break;
+ case Stmt::OMPSectionDirectiveClass:
+ EmitOMPSectionDirective(cast<OMPSectionDirective>(*S));
+ break;
+ case Stmt::OMPSingleDirectiveClass:
+ EmitOMPSingleDirective(cast<OMPSingleDirective>(*S));
+ break;
+ case Stmt::OMPMasterDirectiveClass:
+ EmitOMPMasterDirective(cast<OMPMasterDirective>(*S));
+ break;
+ case Stmt::OMPCriticalDirectiveClass:
+ EmitOMPCriticalDirective(cast<OMPCriticalDirective>(*S));
+ break;
+ case Stmt::OMPParallelForDirectiveClass:
+ EmitOMPParallelForDirective(cast<OMPParallelForDirective>(*S));
+ break;
+ case Stmt::OMPParallelSectionsDirectiveClass:
+ EmitOMPParallelSectionsDirective(cast<OMPParallelSectionsDirective>(*S));
+ break;
+ case Stmt::OMPTaskDirectiveClass:
+ EmitOMPTaskDirective(cast<OMPTaskDirective>(*S));
+ break;
+ case Stmt::OMPTaskyieldDirectiveClass:
+ EmitOMPTaskyieldDirective(cast<OMPTaskyieldDirective>(*S));
+ break;
+ case Stmt::OMPBarrierDirectiveClass:
+ EmitOMPBarrierDirective(cast<OMPBarrierDirective>(*S));
+ break;
+ case Stmt::OMPTaskwaitDirectiveClass:
+ EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
+ break;
+ case Stmt::OMPFlushDirectiveClass:
+ EmitOMPFlushDirective(cast<OMPFlushDirective>(*S));
+ break;
}
}
@@ -217,7 +266,7 @@ CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
E = S.body_end()-GetLast; I != E; ++I)
EmitStmt(*I);
- llvm::Value *RetAlloca = 0;
+ llvm::Value *RetAlloca = nullptr;
if (GetLast) {
// We have to special case labels here. They are statements, but when put
// at the end of a statement expression, they yield the value of their
@@ -242,7 +291,7 @@ CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
EmitAnyExprToMem(cast<Expr>(LastStmt), RetAlloca, Qualifiers(),
/*IsInit*/false);
}
-
+
}
return RetAlloca;
@@ -309,9 +358,8 @@ void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
bool inserted = false;
- for (llvm::BasicBlock::use_iterator
- i = block->use_begin(), e = block->use_end(); i != e; ++i) {
- if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(*i)) {
+ for (llvm::User *u : block->users()) {
+ if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) {
CurFn->getBasicBlockList().insertAfter(insn->getParent(), block);
inserted = true;
break;
@@ -358,7 +406,9 @@ void CodeGenFunction::EmitLabel(const LabelDecl *D) {
ResolveBranchFixups(Dest.getBlock());
}
+ RegionCounter Cnt = getPGORegionCounter(D->getStmt());
EmitBlock(Dest.getBlock());
+ Cnt.beginRegion(Builder);
}
/// Change the cleanup scope of the labels in this lexical scope to
@@ -391,7 +441,23 @@ void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
}
void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
- EmitStmt(S.getSubStmt());
+ const Stmt *SubStmt = S.getSubStmt();
+ switch (SubStmt->getStmtClass()) {
+ case Stmt::DoStmtClass:
+ EmitDoStmt(cast<DoStmt>(*SubStmt), S.getAttrs());
+ break;
+ case Stmt::ForStmtClass:
+ EmitForStmt(cast<ForStmt>(*SubStmt), S.getAttrs());
+ break;
+ case Stmt::WhileStmtClass:
+ EmitWhileStmt(cast<WhileStmt>(*SubStmt), S.getAttrs());
+ break;
+ case Stmt::CXXForRangeStmtClass:
+ EmitCXXForRangeStmt(cast<CXXForRangeStmt>(*SubStmt), S.getAttrs());
+ break;
+ default:
+ EmitStmt(SubStmt);
+ }
}
void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
@@ -429,7 +495,8 @@ void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
// C99 6.8.4.1: The first substatement is executed if the expression compares
// unequal to 0. The condition must be a scalar type.
- LexicalScope ConditionScope(*this, S.getSourceRange());
+ LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
+ RegionCounter Cnt = getPGORegionCounter(&S);
if (S.getConditionVariable())
EmitAutoVarDecl(*S.getConditionVariable());
@@ -447,6 +514,8 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
// If the skipped block has no labels in it, just emit the executed block.
// This avoids emitting dead code and simplifies the CFG substantially.
if (!ContainsLabel(Skipped)) {
+ if (CondConstant)
+ Cnt.beginRegion(Builder);
if (Executed) {
RunCleanupsScope ExecutedScope(*this);
EmitStmt(Executed);
@@ -462,10 +531,12 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
llvm::BasicBlock *ElseBlock = ContBlock;
if (S.getElse())
ElseBlock = createBasicBlock("if.else");
- EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock);
+
+ EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, Cnt.getCount());
// Emit the 'then' code.
- EmitBlock(ThenBlock);
+ EmitBlock(ThenBlock);
+ Cnt.beginRegion(Builder);
{
RunCleanupsScope ThenScope(*this);
EmitStmt(S.getThen());
@@ -474,30 +545,121 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
// Emit the 'else' code if present.
if (const Stmt *Else = S.getElse()) {
- // There is no need to emit line number for unconditional branch.
- if (getDebugInfo())
- Builder.SetCurrentDebugLocation(llvm::DebugLoc());
- EmitBlock(ElseBlock);
+ {
+ // There is no need to emit line number for unconditional branch.
+ SuppressDebugLocation S(Builder);
+ EmitBlock(ElseBlock);
+ }
{
RunCleanupsScope ElseScope(*this);
EmitStmt(Else);
}
- // There is no need to emit line number for unconditional branch.
- if (getDebugInfo())
- Builder.SetCurrentDebugLocation(llvm::DebugLoc());
- EmitBranch(ContBlock);
+ {
+ // There is no need to emit line number for unconditional branch.
+ SuppressDebugLocation S(Builder);
+ EmitBranch(ContBlock);
+ }
}
// Emit the continuation block for code after the if.
EmitBlock(ContBlock, true);
}
-void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
+void CodeGenFunction::EmitCondBrHints(llvm::LLVMContext &Context,
+ llvm::BranchInst *CondBr,
+ const ArrayRef<const Attr *> &Attrs) {
+ // Return if there are no hints.
+ if (Attrs.empty())
+ return;
+
+ // Add vectorize and unroll hints to the metadata on the conditional branch.
+ SmallVector<llvm::Value *, 2> Metadata(1);
+ for (const auto *Attr : Attrs) {
+ const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
+
+ // Skip non loop hint attributes
+ if (!LH)
+ continue;
+
+ LoopHintAttr::OptionType Option = LH->getOption();
+ int ValueInt = LH->getValue();
+
+ const char *MetadataName;
+ switch (Option) {
+ case LoopHintAttr::Vectorize:
+ case LoopHintAttr::VectorizeWidth:
+ MetadataName = "llvm.loop.vectorize.width";
+ break;
+ case LoopHintAttr::Interleave:
+ case LoopHintAttr::InterleaveCount:
+ MetadataName = "llvm.loop.interleave.count";
+ break;
+ case LoopHintAttr::Unroll:
+ MetadataName = "llvm.loop.unroll.enable";
+ break;
+ case LoopHintAttr::UnrollCount:
+ MetadataName = "llvm.loop.unroll.count";
+ break;
+ }
+
+ llvm::Value *Value;
+ llvm::MDString *Name;
+ switch (Option) {
+ case LoopHintAttr::Vectorize:
+ case LoopHintAttr::Interleave:
+ if (ValueInt == 1) {
+ // FIXME: In the future I will modifiy the behavior of the metadata
+ // so we can enable/disable vectorization and interleaving separately.
+ Name = llvm::MDString::get(Context, "llvm.loop.vectorize.enable");
+ Value = Builder.getTrue();
+ break;
+ }
+ // Vectorization/interleaving is disabled, set width/count to 1.
+ ValueInt = 1;
+ // Fallthrough.
+ case LoopHintAttr::VectorizeWidth:
+ case LoopHintAttr::InterleaveCount:
+ Name = llvm::MDString::get(Context, MetadataName);
+ Value = llvm::ConstantInt::get(Int32Ty, ValueInt);
+ break;
+ case LoopHintAttr::Unroll:
+ Name = llvm::MDString::get(Context, MetadataName);
+ Value = (ValueInt == 0) ? Builder.getFalse() : Builder.getTrue();
+ break;
+ case LoopHintAttr::UnrollCount:
+ Name = llvm::MDString::get(Context, MetadataName);
+ Value = llvm::ConstantInt::get(Int32Ty, ValueInt);
+ break;
+ }
+
+ SmallVector<llvm::Value *, 2> OpValues;
+ OpValues.push_back(Name);
+ OpValues.push_back(Value);
+
+ // Set or overwrite metadata indicated by Name.
+ Metadata.push_back(llvm::MDNode::get(Context, OpValues));
+ }
+
+ if (!Metadata.empty()) {
+ // Add llvm.loop MDNode to CondBr.
+ llvm::MDNode *LoopID = llvm::MDNode::get(Context, Metadata);
+ LoopID->replaceOperandWith(0, LoopID); // First op points to itself.
+
+ CondBr->setMetadata("llvm.loop", LoopID);
+ }
+}
+
+void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
+ const ArrayRef<const Attr *> &WhileAttrs) {
+ RegionCounter Cnt = getPGORegionCounter(&S);
+
// Emit the header for the loop, which will also become
// the continue target.
JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
EmitBlock(LoopHeader.getBlock());
+ LoopStack.push(LoopHeader.getBlock());
+
// Create an exit block for when the condition fails, which will
// also become the break target.
JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
@@ -535,13 +697,17 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
if (ConditionScope.requiresCleanups())
ExitBlock = createBasicBlock("while.exit");
-
- Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
+ llvm::BranchInst *CondBr =
+ Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), Cnt));
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
EmitBranchThroughCleanup(LoopExit);
}
+
+ // Attach metadata to loop body conditional branch.
+ EmitCondBrHints(LoopBody->getContext(), CondBr, WhileAttrs);
}
// Emit the loop body. We have to emit this in a cleanup scope
@@ -549,6 +715,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
{
RunCleanupsScope BodyScope(*this);
EmitBlock(LoopBody);
+ Cnt.beginRegion(Builder);
EmitStmt(S.getBody());
}
@@ -560,6 +727,8 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
// Branch to the loop header again.
EmitBranch(LoopHeader.getBlock());
+ LoopStack.pop();
+
// Emit the exit block.
EmitBlock(LoopExit.getBlock(), true);
@@ -569,23 +738,27 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
SimplifyForwardingBlocks(LoopHeader.getBlock());
}
-void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
+void CodeGenFunction::EmitDoStmt(const DoStmt &S,
+ const ArrayRef<const Attr *> &DoAttrs) {
JumpDest LoopExit = getJumpDestInCurrentScope("do.end");
JumpDest LoopCond = getJumpDestInCurrentScope("do.cond");
+ RegionCounter Cnt = getPGORegionCounter(&S);
+
// Store the blocks to use for break and continue.
BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond));
// Emit the body of the loop.
llvm::BasicBlock *LoopBody = createBasicBlock("do.body");
- EmitBlock(LoopBody);
+
+ LoopStack.push(LoopBody);
+
+ EmitBlockWithFallThrough(LoopBody, Cnt);
{
RunCleanupsScope BodyScope(*this);
EmitStmt(S.getBody());
}
- BreakContinueStack.pop_back();
-
EmitBlock(LoopCond.getBlock());
// C99 6.8.5.2: "The evaluation of the controlling expression takes place
@@ -596,6 +769,8 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
// compares unequal to 0. The condition must be a scalar type.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
+ BreakContinueStack.pop_back();
+
// "do {} while (0)" is common in macros, avoid extra blocks. Be sure
// to correctly handle break/continue though.
bool EmitBoolCondBranch = true;
@@ -604,8 +779,16 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
EmitBoolCondBranch = false;
// As long as the condition is true, iterate the loop.
- if (EmitBoolCondBranch)
- Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock());
+ if (EmitBoolCondBranch) {
+ llvm::BranchInst *CondBr =
+ Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(),
+ PGO.createLoopWeights(S.getCond(), Cnt));
+
+ // Attach metadata to loop body conditional branch.
+ EmitCondBrHints(LoopBody->getContext(), CondBr, DoAttrs);
+ }
+
+ LoopStack.pop();
// Emit the exit block.
EmitBlock(LoopExit.getBlock());
@@ -616,7 +799,8 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
SimplifyForwardingBlocks(LoopCond.getBlock());
}
-void CodeGenFunction::EmitForStmt(const ForStmt &S) {
+void CodeGenFunction::EmitForStmt(const ForStmt &S,
+ const ArrayRef<const Attr *> &ForAttrs) {
JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
RunCleanupsScope ForScope(*this);
@@ -629,6 +813,8 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
if (S.getInit())
EmitStmt(S.getInit());
+ RegionCounter Cnt = getPGORegionCounter(&S);
+
// Start the loop with a block that tests the condition.
// If there's an increment, the continue scope will be overwritten
// later.
@@ -636,6 +822,18 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
llvm::BasicBlock *CondBlock = Continue.getBlock();
EmitBlock(CondBlock);
+ LoopStack.push(CondBlock);
+
+ // If the for loop doesn't have an increment we can just use the
+ // condition as the continue block. Otherwise we'll need to create
+ // a block for it (in the current scope, i.e. in the scope of the
+ // condition), and that we will become our continue block.
+ if (S.getInc())
+ Continue = getJumpDestInCurrentScope("for.inc");
+
+ // Store the blocks to use for break and continue.
+ BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
+
// Create a cleanup scope for the condition variable cleanups.
RunCleanupsScope ConditionScope(*this);
@@ -657,7 +855,13 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
// C99 6.8.5p2/p4: The first substatement is executed if the expression
// compares unequal to 0. The condition must be a scalar type.
- EmitBranchOnBoolExpr(S.getCond(), ForBody, ExitBlock);
+ llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
+ llvm::BranchInst *CondBr =
+ Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), Cnt));
+
+ // Attach metadata to loop body conditional branch.
+ EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs);
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
@@ -669,16 +873,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
// Treat it as a non-zero constant. Don't even create a new block for the
// body, just fall into it.
}
-
- // If the for loop doesn't have an increment we can just use the
- // condition as the continue block. Otherwise we'll need to create
- // a block for it (in the current scope, i.e. in the scope of the
- // condition), and that we will become our continue block.
- if (S.getInc())
- Continue = getJumpDestInCurrentScope("for.inc");
-
- // Store the blocks to use for break and continue.
- BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
+ Cnt.beginRegion(Builder);
{
// Create a separate cleanup scope for the body, in case it is not
@@ -703,11 +898,15 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
if (DI)
DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd());
+ LoopStack.pop();
+
// Emit the fall-through block.
EmitBlock(LoopExit.getBlock(), true);
}
-void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
+void
+CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
+ const ArrayRef<const Attr *> &ForAttrs) {
JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
RunCleanupsScope ForScope(*this);
@@ -720,12 +919,16 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
EmitStmt(S.getRangeStmt());
EmitStmt(S.getBeginEndStmt());
+ RegionCounter Cnt = getPGORegionCounter(&S);
+
// Start the loop with a block that tests the condition.
// If there's an increment, the continue scope will be overwritten
// later.
llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
EmitBlock(CondBlock);
+ LoopStack.push(CondBlock);
+
// If there are any cleanups between here and the loop-exit scope,
// create a block to stage a loop exit along.
llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
@@ -737,7 +940,12 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
// The body is executed if the expression, contextually converted
// to bool, is true.
- EmitBranchOnBoolExpr(S.getCond(), ForBody, ExitBlock);
+ llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
+ llvm::BranchInst *CondBr = Builder.CreateCondBr(
+ BoolCondVal, ForBody, ExitBlock, PGO.createLoopWeights(S.getCond(), Cnt));
+
+ // Attach metadata to loop body conditional branch.
+ EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs);
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
@@ -745,6 +953,7 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
}
EmitBlock(ForBody);
+ Cnt.beginRegion(Builder);
// Create a block for the increment. In case of a 'continue', we jump there.
JumpDest Continue = getJumpDestInCurrentScope("for.inc");
@@ -772,6 +981,8 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
if (DI)
DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd());
+ LoopStack.pop();
+
// Emit the fall-through block.
EmitBlock(LoopExit.getBlock(), true);
}
@@ -809,7 +1020,8 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
// FIXME: Clean this up by using an LValue for ReturnTemp,
// EmitStoreThroughLValue, and EmitAnyExpr.
- if (S.getNRVOCandidate() && S.getNRVOCandidate()->isNRVOVariable()) {
+ if (getLangOpts().ElideConstructors &&
+ S.getNRVOCandidate() && S.getNRVOCandidate()->isNRVOVariable()) {
// Apply the named return value optimization for this return statement,
// which means doing nothing: the appropriate result has already been
// constructed into the NRVO variable.
@@ -818,12 +1030,12 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
// that the cleanup code should not destroy the variable.
if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
Builder.CreateStore(Builder.getTrue(), NRVOFlag);
- } else if (!ReturnValue) {
+ } else if (!ReturnValue || (RV && RV->getType()->isVoidType())) {
// Make sure not to return anything, but evaluate the expression
// for side effects.
if (RV)
EmitAnyExpr(RV);
- } else if (RV == 0) {
+ } else if (!RV) {
// Do nothing (return value is left uninitialized)
} else if (FnRetTy->isReferenceType()) {
// If this function returns a reference, take the address of the expression
@@ -853,7 +1065,7 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
}
++NumReturnExprs;
- if (RV == 0 || RV->isEvaluatable(getContext()))
+ if (!RV || RV->isEvaluatable(getContext()))
++NumSimpleReturnExprs;
cleanupScope.ForceCleanup();
@@ -866,9 +1078,8 @@ void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
if (HaveInsertPoint())
EmitStopPoint(&S);
- for (DeclStmt::const_decl_iterator I = S.decl_begin(), E = S.decl_end();
- I != E; ++I)
- EmitDecl(**I);
+ for (const auto *I : S.decls())
+ EmitDecl(*I);
}
void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
@@ -880,8 +1091,7 @@ void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
if (HaveInsertPoint())
EmitStopPoint(&S);
- JumpDest Block = BreakContinueStack.back().BreakBlock;
- EmitBranchThroughCleanup(Block);
+ EmitBranchThroughCleanup(BreakContinueStack.back().BreakBlock);
}
void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
@@ -893,8 +1103,7 @@ void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
if (HaveInsertPoint())
EmitStopPoint(&S);
- JumpDest Block = BreakContinueStack.back().ContinueBlock;
- EmitBranchThroughCleanup(Block);
+ EmitBranchThroughCleanup(BreakContinueStack.back().ContinueBlock);
}
/// EmitCaseStmtRange - If case statement range is not too big then
@@ -906,11 +1115,13 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());
llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());
+ RegionCounter CaseCnt = getPGORegionCounter(&S);
+
// Emit the code for this case. We do this first to make sure it is
// properly chained from our predecessor before generating the
// switch machinery to enter this block.
- EmitBlock(createBasicBlock("sw.bb"));
- llvm::BasicBlock *CaseDest = Builder.GetInsertBlock();
+ llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
+ EmitBlockWithFallThrough(CaseDest, CaseCnt);
EmitStmt(S.getSubStmt());
// If range is empty, do nothing.
@@ -921,7 +1132,18 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
// FIXME: parameters such as this should not be hardcoded.
if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
// Range is small enough to add multiple switch instruction cases.
- for (unsigned i = 0, e = Range.getZExtValue() + 1; i != e; ++i) {
+ uint64_t Total = CaseCnt.getCount();
+ unsigned NCases = Range.getZExtValue() + 1;
+ // We only have one region counter for the entire set of cases here, so we
+ // need to divide the weights evenly between the generated cases, ensuring
+ // that the total weight is preserved. E.g., a weight of 5 over three cases
+ // will be distributed as weights of 2, 2, and 1.
+ uint64_t Weight = Total / NCases, Rem = Total % NCases;
+ for (unsigned I = 0; I != NCases; ++I) {
+ if (SwitchWeights)
+ SwitchWeights->push_back(Weight + (Rem ? 1 : 0));
+ if (Rem)
+ Rem--;
SwitchInsn->addCase(Builder.getInt(LHS), CaseDest);
LHS++;
}
@@ -946,7 +1168,19 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS));
llvm::Value *Cond =
Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds");
- Builder.CreateCondBr(Cond, CaseDest, FalseDest);
+
+ llvm::MDNode *Weights = nullptr;
+ if (SwitchWeights) {
+ uint64_t ThisCount = CaseCnt.getCount();
+ uint64_t DefaultCount = (*SwitchWeights)[0];
+ Weights = PGO.createBranchWeights(ThisCount, DefaultCount);
+
+ // Since we're chaining the switch default through each large case range, we
+ // need to update the weight for the default, ie, the first case, to include
+ // this case.
+ (*SwitchWeights)[0] += ThisCount;
+ }
+ Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
// Restore the appropriate insertion point.
if (RestoreBB)
@@ -959,7 +1193,7 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
// If there is no enclosing switch instance that we're aware of, then this
// case statement and its block can be elided. This situation only happens
// when we've constant-folded the switch, are emitting the constant case,
- // and part of the constant case includes another case statement. For
+ // and part of the constant case includes another case statement. For
// instance: switch (4) { case 4: do { case 5: } while (1); }
if (!SwitchInsn) {
EmitStmt(S.getSubStmt());
@@ -972,17 +1206,22 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
return;
}
+ RegionCounter CaseCnt = getPGORegionCounter(&S);
llvm::ConstantInt *CaseVal =
Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));
- // If the body of the case is just a 'break', and if there was no fallthrough,
- // try to not emit an empty block.
- if ((CGM.getCodeGenOpts().OptimizationLevel > 0) &&
+ // If the body of the case is just a 'break', try to not emit an empty block.
+ // If we're profiling or we're not optimizing, leave the block in for better
+ // debug and coverage analysis.
+ if (!CGM.getCodeGenOpts().ProfileInstrGenerate &&
+ CGM.getCodeGenOpts().OptimizationLevel > 0 &&
isa<BreakStmt>(S.getSubStmt())) {
JumpDest Block = BreakContinueStack.back().BreakBlock;
// Only do this optimization if there are no cleanups that need emitting.
if (isObviouslyBranchWithoutCleanups(Block)) {
+ if (SwitchWeights)
+ SwitchWeights->push_back(CaseCnt.getCount());
SwitchInsn->addCase(CaseVal, Block.getBlock());
// If there was a fallthrough into this case, make sure to redirect it to
@@ -995,8 +1234,10 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
}
}
- EmitBlock(createBasicBlock("sw.bb"));
- llvm::BasicBlock *CaseDest = Builder.GetInsertBlock();
+ llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
+ EmitBlockWithFallThrough(CaseDest, CaseCnt);
+ if (SwitchWeights)
+ SwitchWeights->push_back(CaseCnt.getCount());
SwitchInsn->addCase(CaseVal, CaseDest);
// Recursively emitting the statement is acceptable, but is not wonderful for
@@ -1012,10 +1253,19 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
const CaseStmt *NextCase = dyn_cast<CaseStmt>(S.getSubStmt());
// Otherwise, iteratively add consecutive cases to this switch stmt.
- while (NextCase && NextCase->getRHS() == 0) {
+ while (NextCase && NextCase->getRHS() == nullptr) {
CurCase = NextCase;
- llvm::ConstantInt *CaseVal =
+ llvm::ConstantInt *CaseVal =
Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));
+
+ CaseCnt = getPGORegionCounter(NextCase);
+ if (SwitchWeights)
+ SwitchWeights->push_back(CaseCnt.getCount());
+ if (CGM.getCodeGenOpts().ProfileInstrGenerate) {
+ CaseDest = createBasicBlock("sw.bb");
+ EmitBlockWithFallThrough(CaseDest, CaseCnt);
+ }
+
SwitchInsn->addCase(CaseVal, CaseDest);
NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
}
@@ -1028,7 +1278,10 @@ void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S) {
llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
assert(DefaultBlock->empty() &&
"EmitDefaultStmt: Default block already defined?");
- EmitBlock(DefaultBlock);
+
+ RegionCounter Cnt = getPGORegionCounter(&S);
+ EmitBlockWithFallThrough(DefaultBlock, Cnt);
+
EmitStmt(S.getSubStmt());
}
@@ -1061,7 +1314,7 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S,
bool &FoundCase,
SmallVectorImpl<const Stmt*> &ResultStmts) {
// If this is a null statement, just succeed.
- if (S == 0)
+ if (!S)
return Case ? CSFC_Success : CSFC_FallThrough;
// If this is the switchcase (case 4: or default) that we're looking for, then
@@ -1069,7 +1322,7 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S,
if (const SwitchCase *SC = dyn_cast<SwitchCase>(S)) {
if (S == Case) {
FoundCase = true;
- return CollectStatementsForCase(SC->getSubStmt(), 0, FoundCase,
+ return CollectStatementsForCase(SC->getSubStmt(), nullptr, FoundCase,
ResultStmts);
}
@@ -1080,7 +1333,7 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S,
// If we are in the live part of the code and we found our break statement,
// return a success!
- if (Case == 0 && isa<BreakStmt>(S))
+ if (!Case && isa<BreakStmt>(S))
return CSFC_Success;
// If this is a switch statement, then it might contain the SwitchCase, the
@@ -1125,7 +1378,7 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S,
// statements in the compound statement as candidates for inclusion.
assert(FoundCase && "Didn't find case but returned fallthrough?");
// We recursively found Case, so we're not looking for it anymore.
- Case = 0;
+ Case = nullptr;
// If we found the case and skipped declarations, we can't do the
// optimization.
@@ -1139,7 +1392,7 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S,
// If we have statements in our range, then we know that the statements are
// live and need to be added to the set of statements we're tracking.
for (; I != E; ++I) {
- switch (CollectStatementsForCase(*I, 0, FoundCase, ResultStmts)) {
+ switch (CollectStatementsForCase(*I, nullptr, FoundCase, ResultStmts)) {
case CSFC_Failure: return CSFC_Failure;
case CSFC_FallThrough:
// A fallthrough result means that the statement was simple and just
@@ -1185,11 +1438,12 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S,
static bool FindCaseStatementsForValue(const SwitchStmt &S,
const llvm::APSInt &ConstantCondValue,
SmallVectorImpl<const Stmt*> &ResultStmts,
- ASTContext &C) {
+ ASTContext &C,
+ const SwitchCase *&ResultCase) {
// First step, find the switch case that is being branched to. We can do this
// efficiently by scanning the SwitchCase list.
const SwitchCase *Case = S.getSwitchCaseList();
- const DefaultStmt *DefaultCase = 0;
+ const DefaultStmt *DefaultCase = nullptr;
for (; Case; Case = Case->getNextSwitchCase()) {
// It's either a default or case. Just remember the default statement in
@@ -1211,10 +1465,10 @@ static bool FindCaseStatementsForValue(const SwitchStmt &S,
// If we didn't find a matching case, we use a default if it exists, or we
// elide the whole switch body!
- if (Case == 0) {
+ if (!Case) {
// It is safe to elide the body of the switch if it doesn't contain labels
// etc. If it is safe, return successfully with an empty ResultStmts list.
- if (DefaultCase == 0)
+ if (!DefaultCase)
return !CodeGenFunction::ContainsLabel(&S);
Case = DefaultCase;
}
@@ -1228,21 +1482,16 @@ static bool FindCaseStatementsForValue(const SwitchStmt &S,
// while (1) {
// case 4: ...
bool FoundCase = false;
+ ResultCase = Case;
return CollectStatementsForCase(S.getBody(), Case, FoundCase,
ResultStmts) != CSFC_Failure &&
FoundCase;
}
void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
- JumpDest SwitchExit = getJumpDestInCurrentScope("sw.epilog");
-
- RunCleanupsScope ConditionScope(*this);
-
- if (S.getConditionVariable())
- EmitAutoVarDecl(*S.getConditionVariable());
-
// Handle nested switch statements.
llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
+ SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
// See if we can constant fold the condition of the switch and therefore only
@@ -1250,19 +1499,31 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
llvm::APSInt ConstantCondValue;
if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) {
SmallVector<const Stmt*, 4> CaseStmts;
+ const SwitchCase *Case = nullptr;
if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,
- getContext())) {
+ getContext(), Case)) {
+ if (Case) {
+ RegionCounter CaseCnt = getPGORegionCounter(Case);
+ CaseCnt.beginRegion(Builder);
+ }
RunCleanupsScope ExecutedScope(*this);
+ // Emit the condition variable if needed inside the entire cleanup scope
+ // used by this special case for constant folded switches.
+ if (S.getConditionVariable())
+ EmitAutoVarDecl(*S.getConditionVariable());
+
// At this point, we are no longer "within" a switch instance, so
// we can temporarily enforce this to ensure that any embedded case
// statements are not emitted.
- SwitchInsn = 0;
+ SwitchInsn = nullptr;
// Okay, we can dead code eliminate everything except this case. Emit the
// specified series of statements and we're good.
for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i)
EmitStmt(CaseStmts[i]);
+ RegionCounter ExitCnt = getPGORegionCounter(&S);
+ ExitCnt.beginRegion(Builder);
// Now we want to restore the saved switch instance so that nested
// switches continue to function properly
@@ -1272,6 +1533,11 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
}
}
+ JumpDest SwitchExit = getJumpDestInCurrentScope("sw.epilog");
+
+ RunCleanupsScope ConditionScope(*this);
+ if (S.getConditionVariable())
+ EmitAutoVarDecl(*S.getConditionVariable());
llvm::Value *CondV = EmitScalarExpr(S.getCond());
// Create basic block to hold stuff that comes after switch
@@ -1280,12 +1546,29 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
// failure.
llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default");
SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock);
+ if (PGO.haveRegionCounts()) {
+ // Walk the SwitchCase list to find how many there are.
+ uint64_t DefaultCount = 0;
+ unsigned NumCases = 0;
+ for (const SwitchCase *Case = S.getSwitchCaseList();
+ Case;
+ Case = Case->getNextSwitchCase()) {
+ if (isa<DefaultStmt>(Case))
+ DefaultCount = getPGORegionCounter(Case).getCount();
+ NumCases += 1;
+ }
+ SwitchWeights = new SmallVector<uint64_t, 16>();
+ SwitchWeights->reserve(NumCases);
+ // The default needs to be first. We store the edge count, so we already
+ // know the right weight.
+ SwitchWeights->push_back(DefaultCount);
+ }
CaseRangeBlock = DefaultBlock;
// Clear the insertion point to indicate we are in unreachable code.
Builder.ClearInsertionPoint();
- // All break statements jump to NextBlock. If BreakContinueStack is non empty
+ // All break statements jump to NextBlock. If BreakContinueStack is non-empty
// then reuse last ContinueBlock.
JumpDest OuterContinue;
if (!BreakContinueStack.empty())
@@ -1320,14 +1603,26 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
// Emit continuation.
EmitBlock(SwitchExit.getBlock(), true);
-
+ RegionCounter ExitCnt = getPGORegionCounter(&S);
+ ExitCnt.beginRegion(Builder);
+
+ if (SwitchWeights) {
+ assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
+ "switch weights do not match switch cases");
+ // If there's only one jump destination there's no sense weighting it.
+ if (SwitchWeights->size() > 1)
+ SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
+ PGO.createBranchWeights(*SwitchWeights));
+ delete SwitchWeights;
+ }
SwitchInsn = SavedSwitchInsn;
+ SwitchWeights = SavedSwitchWeights;
CaseRangeBlock = SavedCRBlock;
}
static std::string
SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
- SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=0) {
+ SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=nullptr) {
std::string Result;
while (*Constraint) {
@@ -1493,7 +1788,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
Name = GAS->getOutputName(i);
TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
- assert(IsValid && "Failed to parse output constraint");
+ assert(IsValid && "Failed to parse output constraint");
OutputConstraintInfos.push_back(Info);
}
@@ -1791,20 +2086,32 @@ static LValue InitCapturedStruct(CodeGenFunction &CGF, const CapturedStmt &S) {
return SlotLV;
}
+static void InitVLACaptures(CodeGenFunction &CGF, const CapturedStmt &S) {
+ for (auto &C : S.captures()) {
+ if (C.capturesVariable()) {
+ QualType QTy;
+ auto VD = C.getCapturedVar();
+ if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
+ QTy = PVD->getOriginalType();
+ else
+ QTy = VD->getType();
+ if (QTy->isVariablyModifiedType()) {
+ CGF.EmitVariablyModifiedType(QTy);
+ }
+ }
+ }
+}
+
/// Generate an outlined function for the body of a CapturedStmt, store any
/// captured variables into the captured struct, and call the outlined function.
llvm::Function *
CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
- const CapturedDecl *CD = S.getCapturedDecl();
- const RecordDecl *RD = S.getCapturedRecordDecl();
- assert(CD->hasBody() && "missing CapturedDecl body");
-
LValue CapStruct = InitCapturedStruct(*this, S);
// Emit the CapturedDecl
CodeGenFunction CGF(CGM, true);
CGF.CapturedStmtInfo = new CGCapturedStmtInfo(S, K);
- llvm::Function *F = CGF.GenerateCapturedStmtFunction(CD, RD, S.getLocStart());
+ llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
delete CGF.CapturedStmtInfo;
// Emit call to the helper function.
@@ -1813,13 +2120,21 @@ CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
return F;
}
+llvm::Value *
+CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
+ LValue CapStruct = InitCapturedStruct(*this, S);
+ return CapStruct.getAddress();
+}
+
/// Creates the outlined function for a CapturedStmt.
llvm::Function *
-CodeGenFunction::GenerateCapturedStmtFunction(const CapturedDecl *CD,
- const RecordDecl *RD,
- SourceLocation Loc) {
+CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
assert(CapturedStmtInfo &&
"CapturedStmtInfo should be set when generating the captured function");
+ const CapturedDecl *CD = S.getCapturedDecl();
+ const RecordDecl *RD = S.getCapturedRecordDecl();
+ SourceLocation Loc = S.getLocStart();
+ assert(CD->hasBody() && "missing CapturedDecl body");
// Build the argument list.
ASTContext &Ctx = CGM.getContext();
@@ -1829,8 +2144,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedDecl *CD,
// Create the function declaration.
FunctionType::ExtInfo ExtInfo;
const CGFunctionInfo &FuncInfo =
- CGM.getTypes().arrangeFunctionDeclaration(Ctx.VoidTy, Args, ExtInfo,
- /*IsVariadic=*/false);
+ CGM.getTypes().arrangeFreeFunctionDeclaration(Ctx.VoidTy, Args, ExtInfo,
+ /*IsVariadic=*/false);
llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
llvm::Function *F =
@@ -1839,13 +2154,17 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedDecl *CD,
CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
// Generate the function.
- StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getBody()->getLocStart());
-
+ StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args,
+ CD->getLocation(),
+ CD->getBody()->getLocStart());
// Set the context parameter in CapturedStmtInfo.
llvm::Value *DeclPtr = LocalDeclMap[CD->getContextParam()];
assert(DeclPtr && "missing context parameter for CapturedStmt");
CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
+ // Initialize variable-length arrays.
+ InitVLACaptures(*this, S);
+
// If 'this' is captured, load it into CXXThisValue.
if (CapturedStmtInfo->isCXXThisExprCaptured()) {
FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
@@ -1855,8 +2174,11 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedDecl *CD,
CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();
}
+ PGO.assignRegionCounters(CD, F);
CapturedStmtInfo->EmitBody(*this, CD->getBody());
FinishFunction(CD->getBodyRBrace());
+ PGO.emitInstrumentationData();
+ PGO.destroyRegionCounters();
return F;
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-05 23:56:35 +0000