aboutsummaryrefslogtreecommitdiff
path: root/lib/CodeGen/CodeGenFunction.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/CodeGen/CodeGenFunction.cpp')
-rw-r--r--lib/CodeGen/CodeGenFunction.cpp374
1 files changed, 280 insertions, 94 deletions
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index ce1b44559dcc..5ca3a78bb4fa 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -15,13 +15,14 @@
#include "CGCUDARuntime.h"
#include "CGCXXABI.h"
#include "CGDebugInfo.h"
+#include "CGOpenMPRuntime.h"
#include "CodeGenModule.h"
+#include "CodeGenPGO.h"
#include "TargetInfo.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/StmtCXX.h"
-#include "clang/Basic/OpenCL.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "clang/Frontend/CodeGenOptions.h"
@@ -34,22 +35,23 @@ using namespace CodeGen;
CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
: CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
- Builder(cgm.getModule().getContext()), CapturedStmtInfo(0),
- SanitizePerformTypeCheck(CGM.getSanOpts().Null |
- CGM.getSanOpts().Alignment |
- CGM.getSanOpts().ObjectSize |
- CGM.getSanOpts().Vptr),
- SanOpts(&CGM.getSanOpts()), AutoreleaseResult(false), BlockInfo(0),
- BlockPointer(0), LambdaThisCaptureField(0), NormalCleanupDest(0),
- NextCleanupDestIndex(1), FirstBlockInfo(0), EHResumeBlock(0),
- ExceptionSlot(0), EHSelectorSlot(0), DebugInfo(CGM.getModuleDebugInfo()),
- DisableDebugInfo(false), DidCallStackSave(false), IndirectBranch(0),
- SwitchInsn(0), CaseRangeBlock(0), UnreachableBlock(0), NumReturnExprs(0),
- NumSimpleReturnExprs(0), CXXABIThisDecl(0), CXXABIThisValue(0),
- CXXThisValue(0), CXXDefaultInitExprThis(0),
- CXXStructorImplicitParamDecl(0), CXXStructorImplicitParamValue(0),
- OutermostConditional(0), CurLexicalScope(0), TerminateLandingPad(0),
- TerminateHandler(0), TrapBB(0) {
+ Builder(cgm.getModule().getContext(), llvm::ConstantFolder(),
+ CGBuilderInserterTy(this)),
+ CapturedStmtInfo(nullptr), SanOpts(&CGM.getLangOpts().Sanitize),
+ IsSanitizerScope(false), AutoreleaseResult(false), BlockInfo(nullptr),
+ BlockPointer(nullptr), LambdaThisCaptureField(nullptr),
+ NormalCleanupDest(nullptr), NextCleanupDestIndex(1),
+ FirstBlockInfo(nullptr), EHResumeBlock(nullptr), ExceptionSlot(nullptr),
+ EHSelectorSlot(nullptr), DebugInfo(CGM.getModuleDebugInfo()),
+ DisableDebugInfo(false), DidCallStackSave(false), IndirectBranch(nullptr),
+ PGO(cgm), SwitchInsn(nullptr), SwitchWeights(nullptr),
+ CaseRangeBlock(nullptr), UnreachableBlock(nullptr), NumReturnExprs(0),
+ NumSimpleReturnExprs(0), CXXABIThisDecl(nullptr),
+ CXXABIThisValue(nullptr), CXXThisValue(nullptr),
+ CXXDefaultInitExprThis(nullptr), CXXStructorImplicitParamDecl(nullptr),
+ CXXStructorImplicitParamValue(nullptr), OutermostConditional(nullptr),
+ CurLexicalScope(nullptr), TerminateLandingPad(nullptr),
+ TerminateHandler(nullptr), TrapBB(nullptr) {
if (!suppressNewContext)
CGM.getCXXABI().getMangleContext().startNewFunction();
@@ -71,6 +73,10 @@ CodeGenFunction::~CodeGenFunction() {
// something.
if (FirstBlockInfo)
destroyBlockInfos(FirstBlockInfo);
+
+ if (getLangOpts().OpenMP) {
+ CGM.getOpenMPRuntime().FunctionFinished(*this);
+ }
}
@@ -157,7 +163,7 @@ void CodeGenFunction::EmitReturnBlock() {
// cleans up functions which started with a unified return block.
if (ReturnBlock.getBlock()->hasOneUse()) {
llvm::BranchInst *BI =
- dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->use_begin());
+ dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
if (BI && BI->isUnconditional() &&
BI->getSuccessor(0) == ReturnBlock.getBlock()) {
// Reset insertion point, including debug location, and delete the
@@ -255,7 +261,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
// Remove the AllocaInsertPt instruction, which is just a convenience for us.
llvm::Instruction *Ptr = AllocaInsertPt;
- AllocaInsertPt = 0;
+ AllocaInsertPt = nullptr;
Ptr->eraseFromParent();
// If someone took the address of a label but never did an indirect goto, we
@@ -275,6 +281,14 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
if (CGM.getCodeGenOpts().EmitDeclMetadata)
EmitDeclMetadata();
+
+ for (SmallVectorImpl<std::pair<llvm::Instruction *, llvm::Value *> >::iterator
+ I = DeferredReplacements.begin(),
+ E = DeferredReplacements.end();
+ I != E; ++I) {
+ I->first->replaceAllUsesWith(I->second);
+ I->first->eraseFromParent();
+ }
}
/// ShouldInstrumentFunction - Return true if the current function should be
@@ -330,6 +344,8 @@ static void GenOpenCLArgMetadata(const FunctionDecl *FD, llvm::Function *Fn,
// Each MDNode is a list in the form of "key", N number of values which is
// the same number of values as their are kernel arguments.
+ const PrintingPolicy &Policy = ASTCtx.getPrintingPolicy();
+
// MDNode for the kernel argument address space qualifiers.
SmallVector<llvm::Value*, 8> addressQuals;
addressQuals.push_back(llvm::MDString::get(Context, "kernel_arg_addr_space"));
@@ -363,7 +379,8 @@ static void GenOpenCLArgMetadata(const FunctionDecl *FD, llvm::Function *Fn,
pointeeTy.getAddressSpace())));
// Get argument type name.
- std::string typeName = pointeeTy.getUnqualifiedType().getAsString() + "*";
+ std::string typeName =
+ pointeeTy.getUnqualifiedType().getAsString(Policy) + "*";
// Turn "unsigned type" to "utype"
std::string::size_type pos = typeName.find("unsigned");
@@ -381,10 +398,15 @@ static void GenOpenCLArgMetadata(const FunctionDecl *FD, llvm::Function *Fn,
if (pointeeTy.isVolatileQualified())
typeQuals += typeQuals.empty() ? "volatile" : " volatile";
} else {
- addressQuals.push_back(Builder.getInt32(0));
+ uint32_t AddrSpc = 0;
+ if (ty->isImageType())
+ AddrSpc =
+ CGM.getContext().getTargetAddressSpace(LangAS::opencl_global);
+
+ addressQuals.push_back(Builder.getInt32(AddrSpc));
// Get argument type name.
- std::string typeName = ty.getUnqualifiedType().getAsString();
+ std::string typeName = ty.getUnqualifiedType().getAsString(Policy);
// Turn "unsigned type" to "utype"
std::string::size_type pos = typeName.find("unsigned");
@@ -399,16 +421,17 @@ static void GenOpenCLArgMetadata(const FunctionDecl *FD, llvm::Function *Fn,
if (ty.isVolatileQualified())
typeQuals += typeQuals.empty() ? "volatile" : " volatile";
}
-
+
argTypeQuals.push_back(llvm::MDString::get(Context, typeQuals));
// Get image access qualifier:
if (ty->isImageType()) {
- if (parm->hasAttr<OpenCLImageAccessAttr>() &&
- parm->getAttr<OpenCLImageAccessAttr>()->getAccess() == CLIA_write_only)
+ const OpenCLImageAccessAttr *A = parm->getAttr<OpenCLImageAccessAttr>();
+ if (A && A->isWriteOnly())
accessQuals.push_back(llvm::MDString::get(Context, "write_only"));
else
accessQuals.push_back(llvm::MDString::get(Context, "read_only"));
+ // FIXME: what about read_write?
} else
accessQuals.push_back(llvm::MDString::get(Context, "none"));
@@ -438,16 +461,15 @@ void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
GenOpenCLArgMetadata(FD, Fn, CGM, Context, kernelMDArgs,
Builder, getContext());
- if (FD->hasAttr<VecTypeHintAttr>()) {
- VecTypeHintAttr *attr = FD->getAttr<VecTypeHintAttr>();
- QualType hintQTy = attr->getTypeHint();
+ if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
+ QualType hintQTy = A->getTypeHint();
const ExtVectorType *hintEltQTy = hintQTy->getAs<ExtVectorType>();
bool isSignedInteger =
hintQTy->isSignedIntegerType() ||
(hintEltQTy && hintEltQTy->getElementType()->isSignedIntegerType());
llvm::Value *attrMDArgs[] = {
llvm::MDString::get(Context, "vec_type_hint"),
- llvm::UndefValue::get(CGM.getTypes().ConvertType(attr->getTypeHint())),
+ llvm::UndefValue::get(CGM.getTypes().ConvertType(A->getTypeHint())),
llvm::ConstantInt::get(
llvm::IntegerType::get(Context, 32),
llvm::APInt(32, (uint64_t)(isSignedInteger ? 1 : 0)))
@@ -455,24 +477,22 @@ void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
kernelMDArgs.push_back(llvm::MDNode::get(Context, attrMDArgs));
}
- if (FD->hasAttr<WorkGroupSizeHintAttr>()) {
- WorkGroupSizeHintAttr *attr = FD->getAttr<WorkGroupSizeHintAttr>();
+ if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
llvm::Value *attrMDArgs[] = {
llvm::MDString::get(Context, "work_group_size_hint"),
- Builder.getInt32(attr->getXDim()),
- Builder.getInt32(attr->getYDim()),
- Builder.getInt32(attr->getZDim())
+ Builder.getInt32(A->getXDim()),
+ Builder.getInt32(A->getYDim()),
+ Builder.getInt32(A->getZDim())
};
kernelMDArgs.push_back(llvm::MDNode::get(Context, attrMDArgs));
}
- if (FD->hasAttr<ReqdWorkGroupSizeAttr>()) {
- ReqdWorkGroupSizeAttr *attr = FD->getAttr<ReqdWorkGroupSizeAttr>();
+ if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
llvm::Value *attrMDArgs[] = {
llvm::MDString::get(Context, "reqd_work_group_size"),
- Builder.getInt32(attr->getXDim()),
- Builder.getInt32(attr->getYDim()),
- Builder.getInt32(attr->getZDim())
+ Builder.getInt32(A->getXDim()),
+ Builder.getInt32(A->getYDim()),
+ Builder.getInt32(A->getZDim())
};
kernelMDArgs.push_back(llvm::MDNode::get(Context, attrMDArgs));
}
@@ -483,37 +503,55 @@ void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
OpenCLKernelMetadata->addOperand(kernelMDNode);
}
+/// Determine whether the function F ends with a return stmt.
+static bool endsWithReturn(const Decl* F) {
+ const Stmt *Body = nullptr;
+ if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
+ Body = FD->getBody();
+ else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
+ Body = OMD->getBody();
+
+ if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
+ auto LastStmt = CS->body_rbegin();
+ if (LastStmt != CS->body_rend())
+ return isa<ReturnStmt>(*LastStmt);
+ }
+ return false;
+}
+
void CodeGenFunction::StartFunction(GlobalDecl GD,
QualType RetTy,
llvm::Function *Fn,
const CGFunctionInfo &FnInfo,
const FunctionArgList &Args,
+ SourceLocation Loc,
SourceLocation StartLoc) {
const Decl *D = GD.getDecl();
DidCallStackSave = false;
CurCodeDecl = D;
- CurFuncDecl = (D ? D->getNonClosureContext() : 0);
+ CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
FnRetTy = RetTy;
CurFn = Fn;
CurFnInfo = &FnInfo;
assert(CurFn->isDeclaration() && "Function already has body?");
- if (CGM.getSanitizerBlacklist().isIn(*Fn)) {
+ if (CGM.getSanitizerBlacklist().isIn(*Fn))
SanOpts = &SanitizerOptions::Disabled;
- SanitizePerformTypeCheck = false;
- }
// Pass inline keyword to optimizer if it appears explicitly on any
- // declaration.
- if (!CGM.getCodeGenOpts().NoInline)
- if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
- for (FunctionDecl::redecl_iterator RI = FD->redecls_begin(),
- RE = FD->redecls_end(); RI != RE; ++RI)
+ // declaration. Also, in the case of -fno-inline attach NoInline
+ // attribute to all function that are not marked AlwaysInline.
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
+ if (!CGM.getCodeGenOpts().NoInline) {
+ for (auto RI : FD->redecls())
if (RI->isInlineSpecified()) {
Fn->addFnAttr(llvm::Attribute::InlineHint);
break;
}
+ } else if (!FD->hasAttr<AlwaysInlineAttr>())
+ Fn->addFnAttr(llvm::Attribute::NoInline);
+ }
if (getLangOpts().OpenCL) {
// Add metadata for a kernel function.
@@ -562,9 +600,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD,
QualType FnType =
getContext().getFunctionType(RetTy, ArgTypes,
FunctionProtoType::ExtProtoInfo());
-
- DI->setLocation(StartLoc);
- DI->EmitFunctionStart(GD, FnType, CurFn, Builder);
+ DI->EmitFunctionStart(GD, Loc, StartLoc, FnType, CurFn, Builder);
}
if (ShouldInstrumentFunction())
@@ -575,12 +611,27 @@ void CodeGenFunction::StartFunction(GlobalDecl GD,
if (RetTy->isVoidType()) {
// Void type; nothing to return.
- ReturnValue = 0;
+ ReturnValue = nullptr;
+
+ // Count the implicit return.
+ if (!endsWithReturn(D))
+ ++NumReturnExprs;
} else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect &&
!hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
// Indirect aggregate return; emit returned value directly into sret slot.
// This reduces code size, and affects correctness in C++.
- ReturnValue = CurFn->arg_begin();
+ auto AI = CurFn->arg_begin();
+ if (CurFnInfo->getReturnInfo().isSRetAfterThis())
+ ++AI;
+ ReturnValue = AI;
+ } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
+ !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
+ // Load the sret pointer from the argument struct and return into that.
+ unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
+ llvm::Function::arg_iterator EI = CurFn->arg_end();
+ --EI;
+ llvm::Value *Addr = Builder.CreateStructGEP(EI, Idx);
+ ReturnValue = Builder.CreateLoad(Addr, "agg.result");
} else {
ReturnValue = CreateIRTemp(RetTy, "retval");
@@ -645,12 +696,34 @@ void CodeGenFunction::StartFunction(GlobalDecl GD,
void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args,
const Stmt *Body) {
+ RegionCounter Cnt = getPGORegionCounter(Body);
+ Cnt.beginRegion(Builder);
if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
EmitCompoundStmtWithoutScope(*S);
else
EmitStmt(Body);
}
+/// When instrumenting to collect profile data, the counts for some blocks
+/// such as switch cases need to not include the fall-through counts, so
+/// emit a branch around the instrumentation code. When not instrumenting,
+/// this just calls EmitBlock().
+void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
+ RegionCounter &Cnt) {
+ llvm::BasicBlock *SkipCountBB = nullptr;
+ if (HaveInsertPoint() && CGM.getCodeGenOpts().ProfileInstrGenerate) {
+ // When instrumenting for profiling, the fallthrough to certain
+ // statements needs to skip over the instrumentation code so that we
+ // get an accurate count.
+ SkipCountBB = createBasicBlock("skipcount");
+ EmitBranch(SkipCountBB);
+ }
+ EmitBlock(BB);
+ Cnt.beginRegion(Builder, /*AddIncomingFallThrough=*/true);
+ if (SkipCountBB)
+ EmitBlock(SkipCountBB);
+}
+
/// Tries to mark the given function nounwind based on the
/// non-existence of any throwing calls within it. We believe this is
/// lightweight enough to do at -O0.
@@ -688,30 +761,47 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
// Check if we should generate debug info for this function.
if (FD->hasAttr<NoDebugAttr>())
- DebugInfo = NULL; // disable debug info indefinitely for this function
+ DebugInfo = nullptr; // disable debug info indefinitely for this function
FunctionArgList Args;
- QualType ResTy = FD->getResultType();
+ QualType ResTy = FD->getReturnType();
CurGD = GD;
- const CXXMethodDecl *MD;
- if ((MD = dyn_cast<CXXMethodDecl>(FD)) && MD->isInstance()) {
+ const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
+ if (MD && MD->isInstance()) {
if (CGM.getCXXABI().HasThisReturn(GD))
ResTy = MD->getThisType(getContext());
- CGM.getCXXABI().BuildInstanceFunctionParams(*this, ResTy, Args);
+ CGM.getCXXABI().buildThisParam(*this, Args);
}
for (unsigned i = 0, e = FD->getNumParams(); i != e; ++i)
Args.push_back(FD->getParamDecl(i));
+ if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
+ CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
+
SourceRange BodyRange;
if (Stmt *Body = FD->getBody()) BodyRange = Body->getSourceRange();
CurEHLocation = BodyRange.getEnd();
+ // Use the location of the start of the function to determine where
+ // the function definition is located. By default use the location
+ // of the declaration as the location for the subprogram. A function
+ // may lack a declaration in the source code if it is created by code
+ // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
+ SourceLocation Loc = FD->getLocation();
+
+ // If this is a function specialization then use the pattern body
+ // as the location for the function.
+ if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
+ if (SpecDecl->hasBody(SpecDecl))
+ Loc = SpecDecl->getLocation();
+
// Emit the standard function prologue.
- StartFunction(GD, ResTy, Fn, FnInfo, Args, BodyRange.getBegin());
+ StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
// Generate the body of the function.
+ PGO.assignRegionCounters(GD.getDecl(), CurFn);
if (isa<CXXDestructorDecl>(FD))
EmitDestructorBody(Args);
else if (isa<CXXConstructorDecl>(FD))
@@ -753,12 +843,13 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
// If the '}' that terminates a function is reached, and the value of the
// function call is used by the caller, the behavior is undefined.
if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() &&
- !FD->getResultType()->isVoidType() && Builder.GetInsertBlock()) {
- if (SanOpts->Return)
+ !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
+ if (SanOpts->Return) {
+ SanitizerScope SanScope(this);
EmitCheck(Builder.getFalse(), "missing_return",
EmitCheckSourceLocation(FD->getLocation()),
ArrayRef<llvm::Value *>(), CRK_Unrecoverable);
- else if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+ } else if (CGM.getCodeGenOpts().OptimizationLevel == 0)
Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap));
Builder.CreateUnreachable();
Builder.ClearInsertionPoint();
@@ -771,6 +862,9 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
// a quick pass now to see if we can.
if (!CurFn->doesNotThrow())
TryMarkNoThrow(CurFn);
+
+ PGO.emitInstrumentationData();
+ PGO.destroyRegionCounters();
}
/// ContainsLabel - Return true if the statement contains a label in it. If
@@ -778,7 +872,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
/// that we can just remove the code.
bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
// Null statement, not a label!
- if (S == 0) return false;
+ if (!S) return false;
// If this is a label, we have to emit the code, consider something like:
// if (0) { ... foo: bar(); } goto foo;
@@ -810,7 +904,7 @@ bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
/// inside of it, this is fine.
bool CodeGenFunction::containsBreak(const Stmt *S) {
// Null statement, not a label!
- if (S == 0) return false;
+ if (!S) return false;
// If this is a switch or loop that defines its own break scope, then we can
// include it and anything inside of it.
@@ -869,19 +963,25 @@ ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &ResultInt) {
///
void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
llvm::BasicBlock *TrueBlock,
- llvm::BasicBlock *FalseBlock) {
+ llvm::BasicBlock *FalseBlock,
+ uint64_t TrueCount) {
Cond = Cond->IgnoreParens();
if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
+
// Handle X && Y in a condition.
if (CondBOp->getOpcode() == BO_LAnd) {
+ RegionCounter Cnt = getPGORegionCounter(CondBOp);
+
// If we have "1 && X", simplify the code. "0 && X" would have constant
// folded if the case was simple enough.
bool ConstantBool = false;
if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
ConstantBool) {
// br(1 && X) -> br(X).
- return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
+ Cnt.beginRegion(Builder);
+ return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
+ TrueCount);
}
// If we have "X && 1", simplify the code to use an uncond branch.
@@ -889,33 +989,42 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
ConstantBool) {
// br(X && 1) -> br(X).
- return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock);
+ return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock,
+ TrueCount);
}
// Emit the LHS as a conditional. If the LHS conditional is false, we
// want to jump to the FalseBlock.
llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
+ // The counter tells us how often we evaluate RHS, and all of TrueCount
+ // can be propagated to that branch.
+ uint64_t RHSCount = Cnt.getCount();
ConditionalEvaluation eval(*this);
- EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock);
+ EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount);
EmitBlock(LHSTrue);
// Any temporaries created here are conditional.
+ Cnt.beginRegion(Builder);
eval.begin(*this);
- EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
+ EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount);
eval.end(*this);
return;
}
if (CondBOp->getOpcode() == BO_LOr) {
+ RegionCounter Cnt = getPGORegionCounter(CondBOp);
+
// If we have "0 || X", simplify the code. "1 || X" would have constant
// folded if the case was simple enough.
bool ConstantBool = false;
if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
!ConstantBool) {
// br(0 || X) -> br(X).
- return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
+ Cnt.beginRegion(Builder);
+ return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
+ TrueCount);
}
// If we have "X || 0", simplify the code to use an uncond branch.
@@ -923,20 +1032,28 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
!ConstantBool) {
// br(X || 0) -> br(X).
- return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock);
+ return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock,
+ TrueCount);
}
// Emit the LHS as a conditional. If the LHS conditional is true, we
// want to jump to the TrueBlock.
llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
+ // We have the count for entry to the RHS and for the whole expression
+ // being true, so we can divy up True count between the short circuit and
+ // the RHS.
+ uint64_t LHSCount = Cnt.getParentCount() - Cnt.getCount();
+ uint64_t RHSCount = TrueCount - LHSCount;
ConditionalEvaluation eval(*this);
- EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse);
+ EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount);
EmitBlock(LHSFalse);
// Any temporaries created here are conditional.
+ Cnt.beginRegion(Builder);
eval.begin(*this);
- EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock);
+ EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, RHSCount);
+
eval.end(*this);
return;
@@ -945,8 +1062,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
// br(!x, t, f) -> br(x, f, t)
- if (CondUOp->getOpcode() == UO_LNot)
- return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock);
+ if (CondUOp->getOpcode() == UO_LNot) {
+ // Negate the count.
+ uint64_t FalseCount = PGO.getCurrentRegionCount() - TrueCount;
+ // Negate the condition and swap the destination blocks.
+ return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
+ FalseCount);
+ }
}
if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
@@ -954,17 +1076,32 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
+ RegionCounter Cnt = getPGORegionCounter(CondOp);
ConditionalEvaluation cond(*this);
- EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock);
+ EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock, Cnt.getCount());
+
+ // When computing PGO branch weights, we only know the overall count for
+ // the true block. This code is essentially doing tail duplication of the
+ // naive code-gen, introducing new edges for which counts are not
+ // available. Divide the counts proportionally between the LHS and RHS of
+ // the conditional operator.
+ uint64_t LHSScaledTrueCount = 0;
+ if (TrueCount) {
+ double LHSRatio = Cnt.getCount() / (double) Cnt.getParentCount();
+ LHSScaledTrueCount = TrueCount * LHSRatio;
+ }
cond.begin(*this);
EmitBlock(LHSBlock);
- EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock);
+ Cnt.beginRegion(Builder);
+ EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
+ LHSScaledTrueCount);
cond.end(*this);
cond.begin(*this);
EmitBlock(RHSBlock);
- EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock);
+ EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
+ TrueCount - LHSScaledTrueCount);
cond.end(*this);
return;
@@ -980,9 +1117,15 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
return;
}
+ // Create branch weights based on the number of times we get here and the
+ // number of times the condition should be true.
+ uint64_t CurrentCount = std::max(PGO.getCurrentRegionCount(), TrueCount);
+ llvm::MDNode *Weights = PGO.createBranchWeights(TrueCount,
+ CurrentCount - TrueCount);
+
// Emit the code with the fully general case.
llvm::Value *CondV = EvaluateExprAsBool(Cond);
- Builder.CreateCondBr(CondV, TrueBlock, FalseBlock);
+ Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights);
}
/// ErrorUnsupported - Print out an error that codegen doesn't support the
@@ -1075,7 +1218,7 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
getContext().getAsArrayType(Ty))) {
QualType eltType;
llvm::Value *numElts;
- llvm::tie(numElts, eltType) = getVLASize(vlaType);
+ std::tie(numElts, eltType) = getVLASize(vlaType);
SizeVal = numElts;
CharUnits eltSize = getContext().getTypeSizeInChars(eltType);
@@ -1087,7 +1230,7 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
}
} else {
SizeVal = CGM.getSize(Size);
- vla = 0;
+ vla = nullptr;
}
// If the type contains a pointer to data member we can't memset it to zero.
@@ -1124,7 +1267,7 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
// Make sure that there is a block for the indirect goto.
- if (IndirectBranch == 0)
+ if (!IndirectBranch)
GetIndirectGotoBlock();
llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
@@ -1158,7 +1301,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
// If it's a VLA, we have to load the stored size. Note that
// this is the size of the VLA in bytes, not its size in elements.
- llvm::Value *numVLAElements = 0;
+ llvm::Value *numVLAElements = nullptr;
if (isa<VariableArrayType>(arrayType)) {
numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).first;
@@ -1251,7 +1394,7 @@ CodeGenFunction::getVLASize(QualType type) {
std::pair<llvm::Value*, QualType>
CodeGenFunction::getVLASize(const VariableArrayType *type) {
// The number of elements so far; always size_t.
- llvm::Value *numElements = 0;
+ llvm::Value *numElements = nullptr;
QualType elementType;
do {
@@ -1264,7 +1407,7 @@ CodeGenFunction::getVLASize(const VariableArrayType *type) {
numElements = vlaSize;
} else {
// It's undefined behavior if this wraps around, so mark it that way.
- // FIXME: Teach -fcatch-undefined-behavior to trap this.
+ // FIXME: Teach -fsanitize=undefined to trap this.
numElements = Builder.CreateNUWMul(numElements, vlaSize);
}
} while ((type = getContext().getAsVariableArrayType(elementType)));
@@ -1308,6 +1451,10 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::ObjCObjectPointer:
llvm_unreachable("type class is never variably-modified!");
+ case Type::Adjusted:
+ type = cast<AdjustedType>(ty)->getAdjustedType();
+ break;
+
case Type::Decayed:
type = cast<DecayedType>(ty)->getPointeeType();
break;
@@ -1354,6 +1501,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
// greater than zero.
if (SanOpts->VLABound &&
size->getType()->isSignedIntegerType()) {
+ SanitizerScope SanScope(this);
llvm::Value *Zero = llvm::Constant::getNullValue(Size->getType());
llvm::Constant *StaticArgs[] = {
EmitCheckSourceLocation(size->getLocStart()),
@@ -1375,7 +1523,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
case Type::FunctionProto:
case Type::FunctionNoProto:
- type = cast<FunctionType>(ty)->getResultType();
+ type = cast<FunctionType>(ty)->getReturnType();
break;
case Type::Paren:
@@ -1464,12 +1612,10 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
// FIXME We create a new bitcast for every annotation because that's what
// llvm-gcc was doing.
- for (specific_attr_iterator<AnnotateAttr>
- ai = D->specific_attr_begin<AnnotateAttr>(),
- ae = D->specific_attr_end<AnnotateAttr>(); ai != ae; ++ai)
+ for (const auto *I : D->specific_attrs<AnnotateAttr>())
EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation),
Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()),
- (*ai)->getAnnotation(), D->getLocation());
+ I->getAnnotation(), D->getLocation());
}
llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
@@ -1479,15 +1625,13 @@ llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
CGM.Int8PtrTy);
- for (specific_attr_iterator<AnnotateAttr>
- ai = D->specific_attr_begin<AnnotateAttr>(),
- ae = D->specific_attr_end<AnnotateAttr>(); ai != ae; ++ai) {
+ for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
// FIXME Always emit the cast inst so we can differentiate between
// annotation on the first field of a struct and annotation on the struct
// itself.
if (VTy != CGM.Int8PtrTy)
V = Builder.Insert(new llvm::BitCastInst(V, CGM.Int8PtrTy));
- V = EmitAnnotationCall(F, V, (*ai)->getAnnotation(), D->getLocation());
+ V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation());
V = Builder.CreateBitCast(V, VTy);
}
@@ -1495,3 +1639,45 @@ llvm::Value *CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
}
CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
+
+CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
+ : CGF(CGF) {
+ assert(!CGF->IsSanitizerScope);
+ CGF->IsSanitizerScope = true;
+}
+
+CodeGenFunction::SanitizerScope::~SanitizerScope() {
+ CGF->IsSanitizerScope = false;
+}
+
+void CodeGenFunction::InsertHelper(llvm::Instruction *I,
+ const llvm::Twine &Name,
+ llvm::BasicBlock *BB,
+ llvm::BasicBlock::iterator InsertPt) const {
+ LoopStack.InsertHelper(I);
+ if (IsSanitizerScope) {
+ I->setMetadata(
+ CGM.getModule().getMDKindID("nosanitize"),
+ llvm::MDNode::get(CGM.getLLVMContext(), ArrayRef<llvm::Value *>()));
+ }
+}
+
+template <bool PreserveNames>
+void CGBuilderInserter<PreserveNames>::InsertHelper(
+ llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
+ llvm::BasicBlock::iterator InsertPt) const {
+ llvm::IRBuilderDefaultInserter<PreserveNames>::InsertHelper(I, Name, BB,
+ InsertPt);
+ if (CGF)
+ CGF->InsertHelper(I, Name, BB, InsertPt);
+}
+
+#ifdef NDEBUG
+#define PreserveNames false
+#else
+#define PreserveNames true
+#endif
+template void CGBuilderInserter<PreserveNames>::InsertHelper(
+ llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
+ llvm::BasicBlock::iterator InsertPt) const;
+#undef PreserveNames
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-18 00:50:54 +0000