diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp')
| -rw-r--r-- | contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp | 1749 |
1 files changed, 1227 insertions, 522 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp index a3f90449bb4c..f8f997909977 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExpr.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "CGCUDARuntime.h" #include "CGCXXABI.h" #include "CGCall.h" #include "CGCleanup.h" @@ -25,41 +26,43 @@ #include "clang/AST/Attr.h" #include "clang/AST/DeclObjC.h" #include "clang/AST/NSAPI.h" +#include "clang/AST/StmtVisitor.h" #include "clang/Basic/Builtins.h" #include "clang/Basic/CodeGenOptions.h" #include "clang/Basic/SourceManager.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsWebAssembly.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/IR/MatrixBuilder.h" +#include "llvm/Passes/OptimizationLevel.h" #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Path.h" +#include "llvm/Support/SaveAndRestore.h" +#include "llvm/Support/xxhash.h" #include "llvm/Transforms/Utils/SanitizerStats.h" +#include <optional> #include <string> using namespace clang; using namespace CodeGen; +// Experiment to make sanitizers easier to debug +static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization( + "ubsan-unique-traps", llvm::cl::Optional, + llvm::cl::desc("Deoptimize traps for UBSAN so there is 1 trap per check"), + llvm::cl::init(false)); + //===--------------------------------------------------------------------===// // Miscellaneous Helper Methods //===--------------------------------------------------------------------===// -llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) { - unsigned addressSpace = - cast<llvm::PointerType>(value->getType())->getAddressSpace(); - - llvm::PointerType *destType = Int8PtrTy; - if (addressSpace) - destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace); - - if (value->getType() == destType) return value; - return Builder.CreateBitCast(value, destType); -} - /// CreateTempAlloca - This creates a alloca and inserts it into the entry /// block. Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, @@ -68,7 +71,7 @@ Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, llvm::Value *ArraySize) { auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); Alloca->setAlignment(Align.getAsAlign()); - return Address(Alloca, Align); + return Address(Alloca, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates a alloca and inserts it into the entry @@ -92,13 +95,13 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, // otherwise alloca is inserted at the current insertion point of the // builder. if (!ArraySize) - Builder.SetInsertPoint(AllocaInsertPt); + Builder.SetInsertPoint(getPostAllocaInsertPoint()); V = getTargetHooks().performAddrSpaceCast( *this, V, getASTAllocaAddressSpace(), LangAS::Default, Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); } - return Address(V, Align); + return Address(V, Ty, Align, KnownNonNull); } /// CreateTempAlloca - This creates an alloca and inserts it into the entry @@ -120,23 +123,10 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, const Twine &Name) { CharUnits Align = - CharUnits::fromQuantity(CGM.getDataLayout().getABITypeAlignment(Ty)); + CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty)); return CreateTempAlloca(Ty, Align, Name); } -void CodeGenFunction::InitTempAlloca(Address Var, llvm::Value *Init) { - auto *Alloca = Var.getPointer(); - assert(isa<llvm::AllocaInst>(Alloca) || - (isa<llvm::AddrSpaceCastInst>(Alloca) && - isa<llvm::AllocaInst>( - cast<llvm::AddrSpaceCastInst>(Alloca)->getPointerOperand()))); - - auto *Store = new llvm::StoreInst(Init, Alloca, /*volatile*/ false, - Var.getAlignment().getAsAlign()); - llvm::BasicBlock *Block = AllocaInsertPt->getParent(); - Block->getInstList().insertAfter(AllocaInsertPt->getIterator(), Store); -} - Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { CharUnits Align = getContext().getTypeAlignInChars(Ty); return CreateTempAlloca(ConvertType(Ty), Align, Name); @@ -154,13 +144,12 @@ Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, /*ArraySize=*/nullptr, Alloca); if (Ty->isConstantMatrixType()) { - auto *ArrayTy = cast<llvm::ArrayType>(Result.getType()->getElementType()); + auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType()); auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), ArrayTy->getNumElements()); - Result = Address( - Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()), - Result.getAlignment()); + Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(), + KnownNonNull); } return Result; } @@ -198,8 +187,18 @@ llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) { /// EmitIgnoredExpr - Emit code to compute the specified expression, /// ignoring the result. void CodeGenFunction::EmitIgnoredExpr(const Expr *E) { - if (E->isRValue()) - return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true); + if (E->isPRValue()) + return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true); + + // if this is a bitfield-resulting conditional operator, we can special case + // emit this. The normal 'EmitLValue' version of this is particularly + // difficult to codegen for, since creating a single "LValue" for two + // different sized arguments here is not particularly doable. + if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>( + E->IgnoreParenNoopCasts(getContext()))) { + if (CondOp->getObjectKind() == OK_BitField) + return EmitIgnoredConditionalOperator(CondOp); + } // Just emit it as an l-value and drop the result. EmitLValue(E); @@ -400,26 +399,24 @@ static Address createReferenceTemporary(CodeGenFunction &CGF, QualType Ty = Inner->getType(); if (CGF.CGM.getCodeGenOpts().MergeAllConstants && (Ty->isArrayType() || Ty->isRecordType()) && - CGF.CGM.isTypeConstant(Ty, true)) + Ty.isConstantStorage(CGF.getContext(), true, false)) if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) { - if (auto AddrSpace = CGF.getTarget().getConstantAddressSpace()) { - auto AS = AddrSpace.getValue(); - auto *GV = new llvm::GlobalVariable( - CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, - llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr, - llvm::GlobalValue::NotThreadLocal, - CGF.getContext().getTargetAddressSpace(AS)); - CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); - GV->setAlignment(alignment.getAsAlign()); - llvm::Constant *C = GV; - if (AS != LangAS::Default) - C = TCG.performAddrSpaceCast( - CGF.CGM, GV, AS, LangAS::Default, - GV->getValueType()->getPointerTo( - CGF.getContext().getTargetAddressSpace(LangAS::Default))); - // FIXME: Should we put the new global into a COMDAT? - return Address(C, alignment); - } + auto AS = CGF.CGM.GetGlobalConstantAddressSpace(); + auto *GV = new llvm::GlobalVariable( + CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr, + llvm::GlobalValue::NotThreadLocal, + CGF.getContext().getTargetAddressSpace(AS)); + CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); + GV->setAlignment(alignment.getAsAlign()); + llvm::Constant *C = GV; + if (AS != LangAS::Default) + C = TCG.performAddrSpaceCast( + CGF.CGM, GV, AS, LangAS::Default, + GV->getValueType()->getPointerTo( + CGF.getContext().getTargetAddressSpace(LangAS::Default))); + // FIXME: Should we put the new global into a COMDAT? + return Address(C, GV->getValueType(), alignment); } return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca); } @@ -435,7 +432,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF, /// Helper method to check if the underlying ABI is AAPCS static bool isAAPCS(const TargetInfo &TargetInfo) { - return TargetInfo.getABI().startswith("aapcs"); + return TargetInfo.getABI().starts_with("aapcs"); } LValue CodeGenFunction:: @@ -453,10 +450,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { ownership != Qualifiers::OCL_ExplicitNone) { Address Object = createReferenceTemporary(*this, M, E); if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) { - Object = Address(llvm::ConstantExpr::getBitCast(Var, - ConvertTypeForMem(E->getType()) - ->getPointerTo(Object.getAddressSpace())), - Object.getAlignment()); + llvm::Type *Ty = ConvertTypeForMem(E->getType()); + Object = Object.withElementType(Ty); // createReferenceTemporary will promote the temporary to a global with a // constant initializer if it can. It can only do this to a value of @@ -511,10 +506,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { Address Object = createReferenceTemporary(*this, M, E, &Alloca); if (auto *Var = dyn_cast<llvm::GlobalVariable>( Object.getPointer()->stripPointerCasts())) { - Object = Address(llvm::ConstantExpr::getBitCast( - cast<llvm::Constant>(Object.getPointer()), - ConvertTypeForMem(E->getType())->getPointerTo()), - Object.getAlignment()); + llvm::Type *TemporaryType = ConvertTypeForMem(E->getType()); + Object = Object.withElementType(TemporaryType); // If the temporary is a global and has a constant initializer or is a // constant temporary that we promoted to a global, we may have already // initialized it. @@ -540,13 +533,17 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end // marker. Instead, start the lifetime of a conditional temporary earlier // so that it's unconditional. Don't do this with sanitizers which need - // more precise lifetime marks. + // more precise lifetime marks. However when inside an "await.suspend" + // block, we should always avoid conditional cleanup because it creates + // boolean marker that lives across await_suspend, which can destroy coro + // frame. ConditionalEvaluation *OldConditional = nullptr; CGBuilderTy::InsertPoint OldIP; if (isInConditionalBranch() && !E->getType().isDestructedType() && - !SanOpts.has(SanitizerKind::HWAddress) && - !SanOpts.has(SanitizerKind::Memory) && - !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) { + ((!SanOpts.has(SanitizerKind::HWAddress) && + !SanOpts.has(SanitizerKind::Memory) && + !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) || + inSuspendBlock())) { OldConditional = OutermostConditional; OutermostConditional = nullptr; @@ -580,8 +577,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { // Perform derived-to-base casts and/or field accesses, to get from the // temporary object we created (and, potentially, for which we extended // the lifetime) to the subobject we're binding the reference to. - for (unsigned I = Adjustments.size(); I != 0; --I) { - SubobjectAdjustment &Adjustment = Adjustments[I-1]; + for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) { switch (Adjustment.Kind) { case SubobjectAdjustment::DerivedToBaseAdjustment: Object = @@ -667,9 +663,9 @@ bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) { } bool CodeGenFunction::sanitizePerformTypeCheck() const { - return SanOpts.has(SanitizerKind::Null) | - SanOpts.has(SanitizerKind::Alignment) | - SanOpts.has(SanitizerKind::ObjectSize) | + return SanOpts.has(SanitizerKind::Null) || + SanOpts.has(SanitizerKind::Alignment) || + SanOpts.has(SanitizerKind::ObjectSize) || SanOpts.has(SanitizerKind::Vptr); } @@ -751,30 +747,29 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *Min = Builder.getFalse(); llvm::Value *NullIsUnknown = Builder.getFalse(); llvm::Value *Dynamic = Builder.getFalse(); - llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy); llvm::Value *LargeEnough = Builder.CreateICmpUGE( - Builder.CreateCall(F, {CastAddr, Min, NullIsUnknown, Dynamic}), Size); + Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size); Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize)); } } - uint64_t AlignVal = 0; + llvm::MaybeAlign AlignVal; llvm::Value *PtrAsInt = nullptr; if (SanOpts.has(SanitizerKind::Alignment) && !SkippedChecks.has(SanitizerKind::Alignment)) { - AlignVal = Alignment.getQuantity(); + AlignVal = Alignment.getAsMaybeAlign(); if (!Ty->isIncompleteType() && !AlignVal) AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr, /*ForPointeeType=*/true) - .getQuantity(); + .getAsMaybeAlign(); // The glvalue must be suitably aligned. - if (AlignVal > 1 && - (!PtrToAlloca || PtrToAlloca->getAlignment() < AlignVal)) { + if (AlignVal && *AlignVal > llvm::Align(1) && + (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) { PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy); llvm::Value *Align = Builder.CreateAnd( - PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal - 1)); + PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1)); llvm::Value *Aligned = Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0)); if (Aligned != True) @@ -783,12 +778,9 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, } if (Checks.size() > 0) { - // Make sure we're not losing information. Alignment needs to be a power of - // 2 - assert(!AlignVal || (uint64_t)1 << llvm::Log2_64(AlignVal) == AlignVal); llvm::Constant *StaticData[] = { EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty), - llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2_64(AlignVal) : 1), + llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1), llvm::ConstantInt::get(Int8Ty, TCK)}; EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData, PtrAsInt ? PtrAsInt : Ptr); @@ -826,15 +818,14 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(), Out); - // Blacklist based on the mangled type. - if (!CGM.getContext().getSanitizerBlacklist().isBlacklistedType( - SanitizerKind::Vptr, Out.str())) { + // Contained in NoSanitizeList based on the mangled type. + if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr, + Out.str())) { llvm::hash_code TypeHash = hash_value(Out.str()); // Load the vptr, and compute hash_16_bytes(TypeHash, vptr). llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash); - llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0); - Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), getPointerAlign()); + Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign()); llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr); llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty); @@ -850,9 +841,9 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::ConstantInt::get(IntPtrTy, CacheSize-1)); llvm::Value *Indices[] = { Builder.getInt32(0), Slot }; - llvm::Value *CacheVal = - Builder.CreateAlignedLoad(Builder.CreateInBoundsGEP(Cache, Indices), - getPointerAlign()); + llvm::Value *CacheVal = Builder.CreateAlignedLoad( + IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices), + getPointerAlign()); // If the hash isn't in the cache, call a runtime handler to perform the // hard work of checking whether the vptr is for an object of the right @@ -878,44 +869,6 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, } } -/// Determine whether this expression refers to a flexible array member in a -/// struct. We disable array bounds checks for such members. -static bool isFlexibleArrayMemberExpr(const Expr *E) { - // For compatibility with existing code, we treat arrays of length 0 or - // 1 as flexible array members. - // FIXME: This is inconsistent with the warning code in SemaChecking. Unify - // the two mechanisms. - const ArrayType *AT = E->getType()->castAsArrayTypeUnsafe(); - if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) { - // FIXME: Sema doesn't treat [1] as a flexible array member if the bound - // was produced by macro expansion. - if (CAT->getSize().ugt(1)) - return false; - } else if (!isa<IncompleteArrayType>(AT)) - return false; - - E = E->IgnoreParens(); - - // A flexible array member must be the last member in the class. - if (const auto *ME = dyn_cast<MemberExpr>(E)) { - // FIXME: If the base type of the member expr is not FD->getParent(), - // this should not be treated as a flexible array member access. - if (const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl())) { - // FIXME: Sema doesn't treat a T[1] union member as a flexible array - // member, only a T[0] or T[] member gets that treatment. - if (FD->getParent()->isUnion()) - return true; - RecordDecl::field_iterator FI( - DeclContext::decl_iterator(const_cast<FieldDecl *>(FD))); - return ++FI == FD->getParent()->field_end(); - } - } else if (const auto *IRE = dyn_cast<ObjCIvarRefExpr>(E)) { - return IRE->getDecl()->getNextIvar() == nullptr; - } - - return false; -} - llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E, QualType EltTy) { ASTContext &C = getContext(); @@ -957,8 +910,11 @@ llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E, /// If Base is known to point to the start of an array, return the length of /// that array. Return 0 if the length cannot be determined. -static llvm::Value *getArrayIndexingBound( - CodeGenFunction &CGF, const Expr *Base, QualType &IndexedType) { +static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF, + const Expr *Base, + QualType &IndexedType, + LangOptions::StrictFlexArraysLevelKind + StrictFlexArraysLevel) { // For the vector indexing extension, the bound is the number of elements. if (const VectorType *VT = Base->getType()->getAs<VectorType>()) { IndexedType = Base->getType(); @@ -969,17 +925,23 @@ static llvm::Value *getArrayIndexingBound( if (const auto *CE = dyn_cast<CastExpr>(Base)) { if (CE->getCastKind() == CK_ArrayToPointerDecay && - !isFlexibleArrayMemberExpr(CE->getSubExpr())) { + !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(), + StrictFlexArraysLevel)) { + CodeGenFunction::SanitizerScope SanScope(&CGF); + IndexedType = CE->getSubExpr()->getType(); const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe(); if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) return CGF.Builder.getInt(CAT->getSize()); - else if (const auto *VAT = dyn_cast<VariableArrayType>(AT)) + + if (const auto *VAT = dyn_cast<VariableArrayType>(AT)) return CGF.getVLASize(VAT).NumElts; // Ignore pass_object_size here. It's not applicable on decayed pointers. } } + CodeGenFunction::SanitizerScope SanScope(&CGF); + QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0}; if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) { IndexedType = Base->getType(); @@ -989,18 +951,248 @@ static llvm::Value *getArrayIndexingBound( return nullptr; } +namespace { + +/// \p StructAccessBase returns the base \p Expr of a field access. It returns +/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.: +/// +/// p in p-> a.b.c +/// +/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're +/// looking for: +/// +/// struct s { +/// struct s *ptr; +/// int count; +/// char array[] __attribute__((counted_by(count))); +/// }; +/// +/// If we have an expression like \p p->ptr->array[index], we want the +/// \p MemberExpr for \p p->ptr instead of \p p. +class StructAccessBase + : public ConstStmtVisitor<StructAccessBase, const Expr *> { + const RecordDecl *ExpectedRD; + + bool IsExpectedRecordDecl(const Expr *E) const { + QualType Ty = E->getType(); + if (Ty->isPointerType()) + Ty = Ty->getPointeeType(); + return ExpectedRD == Ty->getAsRecordDecl(); + } + +public: + StructAccessBase(const RecordDecl *ExpectedRD) : ExpectedRD(ExpectedRD) {} + + //===--------------------------------------------------------------------===// + // Visitor Methods + //===--------------------------------------------------------------------===// + + // NOTE: If we build C++ support for counted_by, then we'll have to handle + // horrors like this: + // + // struct S { + // int x, y; + // int blah[] __attribute__((counted_by(x))); + // } s; + // + // int foo(int index, int val) { + // int (S::*IHatePMDs)[] = &S::blah; + // (s.*IHatePMDs)[index] = val; + // } + + const Expr *Visit(const Expr *E) { + return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(E); + } + + const Expr *VisitStmt(const Stmt *S) { return nullptr; } + + // These are the types we expect to return (in order of most to least + // likely): + // + // 1. DeclRefExpr - This is the expression for the base of the structure. + // It's exactly what we want to build an access to the \p counted_by + // field. + // 2. MemberExpr - This is the expression that has the same \p RecordDecl + // as the flexble array member's lexical enclosing \p RecordDecl. This + // allows us to catch things like: "p->p->array" + // 3. CompoundLiteralExpr - This is for people who create something + // heretical like (struct foo has a flexible array member): + // + // (struct foo){ 1, 2 }.blah[idx]; + const Expr *VisitDeclRefExpr(const DeclRefExpr *E) { + return IsExpectedRecordDecl(E) ? E : nullptr; + } + const Expr *VisitMemberExpr(const MemberExpr *E) { + if (IsExpectedRecordDecl(E) && E->isArrow()) + return E; + const Expr *Res = Visit(E->getBase()); + return !Res && IsExpectedRecordDecl(E) ? E : Res; + } + const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) { + return IsExpectedRecordDecl(E) ? E : nullptr; + } + const Expr *VisitCallExpr(const CallExpr *E) { + return IsExpectedRecordDecl(E) ? E : nullptr; + } + + const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) { + if (IsExpectedRecordDecl(E)) + return E; + return Visit(E->getBase()); + } + const Expr *VisitCastExpr(const CastExpr *E) { + return Visit(E->getSubExpr()); + } + const Expr *VisitParenExpr(const ParenExpr *E) { + return Visit(E->getSubExpr()); + } + const Expr *VisitUnaryAddrOf(const UnaryOperator *E) { + return Visit(E->getSubExpr()); + } + const Expr *VisitUnaryDeref(const UnaryOperator *E) { + return Visit(E->getSubExpr()); + } +}; + +} // end anonymous namespace + +using RecIndicesTy = + SmallVector<std::pair<const RecordDecl *, llvm::Value *>, 8>; + +static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD, + const FieldDecl *FD, RecIndicesTy &Indices) { + const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD); + int64_t FieldNo = -1; + for (const Decl *D : RD->decls()) { + if (const auto *Field = dyn_cast<FieldDecl>(D)) { + FieldNo = Layout.getLLVMFieldNo(Field); + if (FD == Field) { + Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo))); + return true; + } + } + + if (const auto *Record = dyn_cast<RecordDecl>(D)) { + ++FieldNo; + if (getGEPIndicesToField(CGF, Record, FD, Indices)) { + if (RD->isUnion()) + FieldNo = 0; + Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo))); + return true; + } + } + } + + return false; +} + +/// This method is typically called in contexts where we can't generate +/// side-effects, like in __builtin_dynamic_object_size. When finding +/// expressions, only choose those that have either already been emitted or can +/// be loaded without side-effects. +/// +/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be +/// within the top-level struct. +/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl. +llvm::Value *CodeGenFunction::EmitCountedByFieldExpr( + const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) { + const RecordDecl *RD = CountDecl->getParent()->getOuterLexicalRecordContext(); + + // Find the base struct expr (i.e. p in p->a.b.c.d). + const Expr *StructBase = StructAccessBase(RD).Visit(Base); + if (!StructBase || StructBase->HasSideEffects(getContext())) + return nullptr; + + llvm::Value *Res = nullptr; + if (const auto *DRE = dyn_cast<DeclRefExpr>(StructBase)) { + Res = EmitDeclRefLValue(DRE).getPointer(*this); + Res = Builder.CreateAlignedLoad(ConvertType(DRE->getType()), Res, + getPointerAlign(), "dre.load"); + } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(StructBase)) { + LValue LV = EmitMemberExpr(ME); + Address Addr = LV.getAddress(*this); + Res = Addr.getPointer(); + } else if (StructBase->getType()->isPointerType()) { + LValueBaseInfo BaseInfo; + TBAAAccessInfo TBAAInfo; + Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo); + Res = Addr.getPointer(); + } else { + return nullptr; + } + + llvm::Value *Zero = Builder.getInt32(0); + RecIndicesTy Indices; + + getGEPIndicesToField(*this, RD, CountDecl, Indices); + + for (auto I = Indices.rbegin(), E = Indices.rend(); I != E; ++I) + Res = Builder.CreateInBoundsGEP( + ConvertType(QualType(I->first->getTypeForDecl(), 0)), Res, + {Zero, I->second}, "..counted_by.gep"); + + return Builder.CreateAlignedLoad(ConvertType(CountDecl->getType()), Res, + getIntAlign(), "..counted_by.load"); +} + +const FieldDecl *CodeGenFunction::FindCountedByField(const FieldDecl *FD) { + if (!FD || !FD->hasAttr<CountedByAttr>()) + return nullptr; + + const auto *CBA = FD->getAttr<CountedByAttr>(); + if (!CBA) + return nullptr; + + auto GetNonAnonStructOrUnion = + [](const RecordDecl *RD) -> const RecordDecl * { + while (RD && RD->isAnonymousStructOrUnion()) { + const auto *R = dyn_cast<RecordDecl>(RD->getDeclContext()); + if (!R) + return nullptr; + RD = R; + } + return RD; + }; + const RecordDecl *EnclosingRD = GetNonAnonStructOrUnion(FD->getParent()); + if (!EnclosingRD) + return nullptr; + + DeclarationName DName(CBA->getCountedByField()); + DeclContext::lookup_result Lookup = EnclosingRD->lookup(DName); + + if (Lookup.empty()) + return nullptr; + + const NamedDecl *ND = Lookup.front(); + if (const auto *IFD = dyn_cast<IndirectFieldDecl>(ND)) + ND = IFD->getAnonField(); + + return dyn_cast<FieldDecl>(ND); +} + void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index, QualType IndexType, bool Accessed) { assert(SanOpts.has(SanitizerKind::ArrayBounds) && "should not be called unless adding bounds checks"); - SanitizerScope SanScope(this); - + const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = + getLangOpts().getStrictFlexArraysLevel(); QualType IndexedType; - llvm::Value *Bound = getArrayIndexingBound(*this, Base, IndexedType); + llvm::Value *Bound = + getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel); + + EmitBoundsCheckImpl(E, Bound, Index, IndexType, IndexedType, Accessed); +} + +void CodeGenFunction::EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound, + llvm::Value *Index, + QualType IndexType, + QualType IndexedType, bool Accessed) { if (!Bound) return; + SanitizerScope SanScope(this); + bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType(); llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned); llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false); @@ -1016,7 +1208,6 @@ void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, SanitizerHandler::OutOfBounds, StaticData, Index); } - CodeGenFunction::ComplexPairTy CodeGenFunction:: EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV, bool isInc, bool isPre) { @@ -1067,11 +1258,10 @@ void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E, // LValue Expression Emission //===----------------------------------------------------------------------===// -/// EmitPointerWithAlignment - Given an expression of pointer type, try to -/// derive a more accurate bound on the alignment of the pointer. -Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, - LValueBaseInfo *BaseInfo, - TBAAAccessInfo *TBAAInfo) { +static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo, + TBAAAccessInfo *TBAAInfo, + KnownNonNull_t IsKnownNonNull, + CodeGenFunction &CGF) { // We allow this with ObjC object pointers because of fragile ABIs. assert(E->getType()->isPointerType() || E->getType()->isObjCObjectPointerType()); @@ -1080,7 +1270,7 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, // Casts: if (const CastExpr *CE = dyn_cast<CastExpr>(E)) { if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE)) - CGM.EmitExplicitCastExprType(ECE, this); + CGF.CGM.EmitExplicitCastExprType(ECE, &CGF); switch (CE->getCastKind()) { // Non-converting casts (but not C's implicit conversion from void*). @@ -1093,47 +1283,51 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, LValueBaseInfo InnerBaseInfo; TBAAAccessInfo InnerTBAAInfo; - Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), - &InnerBaseInfo, - &InnerTBAAInfo); + Address Addr = CGF.EmitPointerWithAlignment( + CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull); if (BaseInfo) *BaseInfo = InnerBaseInfo; if (TBAAInfo) *TBAAInfo = InnerTBAAInfo; if (isa<ExplicitCastExpr>(CE)) { LValueBaseInfo TargetTypeBaseInfo; TBAAAccessInfo TargetTypeTBAAInfo; - CharUnits Align = CGM.getNaturalPointeeTypeAlignment( + CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment( E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo); if (TBAAInfo) - *TBAAInfo = CGM.mergeTBAAInfoForCast(*TBAAInfo, - TargetTypeTBAAInfo); + *TBAAInfo = + CGF.CGM.mergeTBAAInfoForCast(*TBAAInfo, TargetTypeTBAAInfo); // If the source l-value is opaque, honor the alignment of the // casted-to type. if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { if (BaseInfo) BaseInfo->mergeForCast(TargetTypeBaseInfo); - Addr = Address(Addr.getPointer(), Align); + Addr = Address(Addr.getPointer(), Addr.getElementType(), Align, + IsKnownNonNull); } } - if (SanOpts.has(SanitizerKind::CFIUnrelatedCast) && + if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast) && CE->getCastKind() == CK_BitCast) { if (auto PT = E->getType()->getAs<PointerType>()) - EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr.getPointer(), - /*MayBeNull=*/true, - CodeGenFunction::CFITCK_UnrelatedCast, - CE->getBeginLoc()); + CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr, + /*MayBeNull=*/true, + CodeGenFunction::CFITCK_UnrelatedCast, + CE->getBeginLoc()); } - return CE->getCastKind() != CK_AddressSpaceConversion - ? Builder.CreateBitCast(Addr, ConvertType(E->getType())) - : Builder.CreateAddrSpaceCast(Addr, - ConvertType(E->getType())); + + llvm::Type *ElemTy = + CGF.ConvertTypeForMem(E->getType()->getPointeeType()); + Addr = Addr.withElementType(ElemTy); + if (CE->getCastKind() == CK_AddressSpaceConversion) + Addr = CGF.Builder.CreateAddrSpaceCast(Addr, + CGF.ConvertType(E->getType())); + return Addr; } break; // Array-to-pointer decay. case CK_ArrayToPointerDecay: - return EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo); + return CGF.EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo); // Derived-to-base conversions. case CK_UncheckedDerivedToBase: @@ -1142,13 +1336,15 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, // conservatively pretend that the complete object is of the base class // type. if (TBAAInfo) - *TBAAInfo = CGM.getTBAAAccessInfo(E->getType()); - Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), BaseInfo); + *TBAAInfo = CGF.CGM.getTBAAAccessInfo(E->getType()); + Address Addr = CGF.EmitPointerWithAlignment( + CE->getSubExpr(), BaseInfo, nullptr, + (KnownNonNull_t)(IsKnownNonNull || + CE->getCastKind() == CK_UncheckedDerivedToBase)); auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl(); - return GetAddressOfBaseClass(Addr, Derived, - CE->path_begin(), CE->path_end(), - ShouldNullCheckClassCastValue(CE), - CE->getExprLoc()); + return CGF.GetAddressOfBaseClass( + Addr, Derived, CE->path_begin(), CE->path_end(), + CGF.ShouldNullCheckClassCastValue(CE), CE->getExprLoc()); } // TODO: Is there any reason to treat base-to-derived conversions @@ -1161,10 +1357,26 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, // Unary &. if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) { if (UO->getOpcode() == UO_AddrOf) { - LValue LV = EmitLValue(UO->getSubExpr()); + LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull); + if (BaseInfo) *BaseInfo = LV.getBaseInfo(); + if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); + return LV.getAddress(CGF); + } + } + + // std::addressof and variants. + if (auto *Call = dyn_cast<CallExpr>(E)) { + switch (Call->getBuiltinCallee()) { + default: + break; + case Builtin::BIaddressof: + case Builtin::BI__addressof: + case Builtin::BI__builtin_addressof: { + LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull); if (BaseInfo) *BaseInfo = LV.getBaseInfo(); if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); - return LV.getAddress(*this); + return LV.getAddress(CGF); + } } } @@ -1172,8 +1384,21 @@ Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, // Otherwise, use the alignment of the type. CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); - return Address(EmitScalarExpr(E), Align); + CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); + llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType()); + return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull); +} + +/// EmitPointerWithAlignment - Given an expression of pointer type, try to +/// derive a more accurate bound on the alignment of the pointer. +Address CodeGenFunction::EmitPointerWithAlignment( + const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo, + KnownNonNull_t IsKnownNonNull) { + Address Addr = + ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, *this); + if (IsKnownNonNull && !Addr.isKnownNonNull()) + Addr.setKnownNonNull(); + return Addr; } llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) { @@ -1218,9 +1443,10 @@ RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E, LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E, const char *Name) { ErrorUnsupported(E, Name); - llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType())); - return MakeAddrLValue(Address(llvm::UndefValue::get(Ty), CharUnits::One()), - E->getType()); + llvm::Type *ElTy = ConvertType(E->getType()); + llvm::Type *Ty = UnqualPtrTy; + return MakeAddrLValue( + Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType()); } bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) { @@ -1282,7 +1508,24 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { /// type of the same size of the lvalue's type. If the lvalue has a variable /// length type, this is not possible. /// -LValue CodeGenFunction::EmitLValue(const Expr *E) { +LValue CodeGenFunction::EmitLValue(const Expr *E, + KnownNonNull_t IsKnownNonNull) { + LValue LV = EmitLValueHelper(E, IsKnownNonNull); + if (IsKnownNonNull && !LV.isKnownNonNull()) + LV.setKnownNonNull(); + return LV; +} + +static QualType getConstantExprReferredType(const FullExpr *E, + const ASTContext &Ctx) { + const Expr *SE = E->getSubExpr()->IgnoreImplicit(); + if (isa<OpaqueValueExpr>(SE)) + return SE->getType(); + return cast<CallExpr>(SE)->getCallReturnType(Ctx)->getPointeeType(); +} + +LValue CodeGenFunction::EmitLValueHelper(const Expr *E, + KnownNonNull_t IsKnownNonNull) { ApplyDebugLocation DL(*this, E); switch (E->getStmtClass()) { default: return EmitUnsupportedLValue(E, "l-value expression"); @@ -1310,7 +1553,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { case Expr::UserDefinedLiteralClass: return EmitCallExprLValue(cast<CallExpr>(E)); case Expr::CXXRewrittenBinaryOperatorClass: - return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm()); + return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm(), + IsKnownNonNull); case Expr::VAArgExprClass: return EmitVAArgExprLValue(cast<VAArgExpr>(E)); case Expr::DeclRefExprClass: @@ -1318,16 +1562,16 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { case Expr::ConstantExprClass: { const ConstantExpr *CE = cast<ConstantExpr>(E); if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) { - QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit()) - ->getCallReturnType(getContext()); + QualType RetType = getConstantExprReferredType(CE, getContext()); return MakeNaturalAlignAddrLValue(Result, RetType); } - return EmitLValue(cast<ConstantExpr>(E)->getSubExpr()); + return EmitLValue(cast<ConstantExpr>(E)->getSubExpr(), IsKnownNonNull); } case Expr::ParenExprClass: - return EmitLValue(cast<ParenExpr>(E)->getSubExpr()); + return EmitLValue(cast<ParenExpr>(E)->getSubExpr(), IsKnownNonNull); case Expr::GenericSelectionExprClass: - return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr()); + return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr(), + IsKnownNonNull); case Expr::PredefinedExprClass: return EmitPredefinedLValue(cast<PredefinedExpr>(E)); case Expr::StringLiteralClass: @@ -1351,14 +1595,16 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { case Expr::ExprWithCleanupsClass: { const auto *cleanups = cast<ExprWithCleanups>(E); RunCleanupsScope Scope(*this); - LValue LV = EmitLValue(cleanups->getSubExpr()); + LValue LV = EmitLValue(cleanups->getSubExpr(), IsKnownNonNull); if (LV.isSimple()) { // Defend against branches out of gnu statement expressions surrounded by // cleanups. - llvm::Value *V = LV.getPointer(*this); + Address Addr = LV.getAddress(*this); + llvm::Value *V = Addr.getPointer(); Scope.ForceCleanup({&V}); - return LValue::MakeAddr(Address(V, LV.getAlignment()), LV.getType(), - getContext(), LV.getBaseInfo(), LV.getTBAAInfo()); + return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()), + LV.getType(), getContext(), LV.getBaseInfo(), + LV.getTBAAInfo()); } // FIXME: Is it possible to create an ExprWithCleanups that produces a // bitfield lvalue or some other non-simple lvalue? @@ -1368,12 +1614,12 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { case Expr::CXXDefaultArgExprClass: { auto *DAE = cast<CXXDefaultArgExpr>(E); CXXDefaultArgExprScope Scope(*this, DAE); - return EmitLValue(DAE->getExpr()); + return EmitLValue(DAE->getExpr(), IsKnownNonNull); } case Expr::CXXDefaultInitExprClass: { auto *DIE = cast<CXXDefaultInitExpr>(E); CXXDefaultInitExprScope Scope(*this, DIE); - return EmitLValue(DIE->getExpr()); + return EmitLValue(DIE->getExpr(), IsKnownNonNull); } case Expr::CXXTypeidExprClass: return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E)); @@ -1394,6 +1640,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E)); case Expr::ExtVectorElementExprClass: return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E)); + case Expr::CXXThisExprClass: + return MakeAddrLValue(LoadCXXThisAddress(), E->getType()); case Expr::MemberExprClass: return EmitMemberExpr(cast<MemberExpr>(E)); case Expr::CompoundLiteralExprClass: @@ -1403,11 +1651,12 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { case Expr::BinaryConditionalOperatorClass: return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E)); case Expr::ChooseExprClass: - return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr()); + return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(), IsKnownNonNull); case Expr::OpaqueValueExprClass: return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E)); case Expr::SubstNonTypeTemplateParmExprClass: - return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement()); + return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(), + IsKnownNonNull); case Expr::ImplicitCastExprClass: case Expr::CStyleCastExprClass: case Expr::CXXFunctionalCastExprClass: @@ -1629,21 +1878,7 @@ static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2); } else { const EnumDecl *ED = ET->getDecl(); - llvm::Type *LTy = CGF.ConvertTypeForMem(ED->getIntegerType()); - unsigned Bitwidth = LTy->getScalarSizeInBits(); - unsigned NumNegativeBits = ED->getNumNegativeBits(); - unsigned NumPositiveBits = ED->getNumPositiveBits(); - - if (NumNegativeBits) { - unsigned NumBits = std::max(NumNegativeBits, NumPositiveBits + 1); - assert(NumBits <= Bitwidth); - End = llvm::APInt(Bitwidth, 1) << (NumBits - 1); - Min = -End; - } else { - assert(NumPositiveBits <= Bitwidth); - End = llvm::APInt(Bitwidth, 1) << NumPositiveBits; - Min = llvm::APInt(Bitwidth, 0); - } + ED->getValueRange(End, Min); } return true; } @@ -1711,27 +1946,46 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isNontemporal) { - if (!CGM.getCodeGenOpts().PreserveVec3Type) { - // For better performance, handle vector loads differently. - if (Ty->isVectorType()) { - const llvm::Type *EltTy = Addr.getElementType(); - - const auto *VTy = cast<llvm::FixedVectorType>(EltTy); - - // Handle vectors of size 3 like size 4 for better performance. - if (VTy->getNumElements() == 3) { - - // Bitcast to vec4 type. - auto *vec4Ty = llvm::FixedVectorType::get(VTy->getElementType(), 4); - Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4"); - // Now load value. - llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4"); - - // Shuffle vector to get vec3. - V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, - "extractVec"); - return EmitFromMemory(V, Ty); - } + if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) + if (GV->isThreadLocal()) + Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), + NotKnownNonNull); + + if (const auto *ClangVecTy = Ty->getAs<VectorType>()) { + // Boolean vectors use `iN` as storage type. + if (ClangVecTy->isExtVectorBoolType()) { + llvm::Type *ValTy = ConvertType(Ty); + unsigned ValNumElems = + cast<llvm::FixedVectorType>(ValTy)->getNumElements(); + // Load the `iP` storage object (P is the padded vector size). + auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits"); + const auto *RawIntTy = RawIntV->getType(); + assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors"); + // Bitcast iP --> <P x i1>. + auto *PaddedVecTy = llvm::FixedVectorType::get( + Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits()); + llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy); + // Shuffle <P x i1> --> <N x i1> (N is the actual bit size). + V = emitBoolVecConversion(V, ValNumElems, "extractvec"); + + return EmitFromMemory(V, Ty); + } + + // Handle vectors of size 3 like size 4 for better performance. + const llvm::Type *EltTy = Addr.getElementType(); + const auto *VTy = cast<llvm::FixedVectorType>(EltTy); + + if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) { + + llvm::VectorType *vec4Ty = + llvm::FixedVectorType::get(VTy->getElementType(), 4); + Address Cast = Addr.withElementType(vec4Ty); + // Now load value. + llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4"); + + // Shuffle vector to get vec3. + V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec"); + return EmitFromMemory(V, Ty); } } @@ -1746,7 +2000,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, if (isNontemporal) { llvm::MDNode *Node = llvm::MDNode::get( Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); - Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); + Load->setMetadata(llvm::LLVMContext::MD_nontemporal, Node); } CGM.DecorateInstructionWithTBAA(Load, TBAAInfo); @@ -1755,8 +2009,11 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, // In order to prevent the optimizer from throwing away the check, don't // attach range metadata to the load. } else if (CGM.getCodeGenOpts().OptimizationLevel > 0) - if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) + if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) { Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo); + Load->setMetadata(llvm::LLVMContext::MD_noundef, + llvm::MDNode::get(getLLVMContext(), std::nullopt)); + } return EmitFromMemory(Load, Ty); } @@ -1782,6 +2039,17 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { "wrong value rep of bool"); return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool"); } + if (Ty->isExtVectorBoolType()) { + const auto *RawIntTy = Value->getType(); + // Bitcast iP --> <P x i1>. + auto *PaddedVecTy = llvm::FixedVectorType::get( + Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits()); + auto *V = Builder.CreateBitCast(Value, PaddedVecTy); + // Shuffle <P x i1> --> <N x i1> (N is the actual bit size). + llvm::Type *ValTy = ConvertType(Ty); + unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements(); + return emitBoolVecConversion(V, ValNumElems, "extractvec"); + } return Value; } @@ -1790,22 +2058,20 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { // MatrixType), if it points to a array (the memory type of MatrixType). static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, bool IsVector = true) { - auto *ArrayTy = dyn_cast<llvm::ArrayType>( - cast<llvm::PointerType>(Addr.getPointer()->getType())->getElementType()); + auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType()); if (ArrayTy && IsVector) { auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), ArrayTy->getNumElements()); - return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy)); + return Addr.withElementType(VectorTy); } - auto *VectorTy = dyn_cast<llvm::VectorType>( - cast<llvm::PointerType>(Addr.getPointer()->getType())->getElementType()); + auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType()); if (VectorTy && !IsVector) { auto *ArrayTy = llvm::ArrayType::get( VectorTy->getElementType(), cast<llvm::FixedVectorType>(VectorTy)->getNumElements()); - return Address(CGF.Builder.CreateElementBitCast(Addr, ArrayTy)); + return Addr.withElementType(ArrayTy); } return Addr; @@ -1828,11 +2094,23 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo, bool isInit, bool isNontemporal) { - if (!CGM.getCodeGenOpts().PreserveVec3Type) { - // Handle vectors differently to get better performance. - if (Ty->isVectorType()) { - llvm::Type *SrcTy = Value->getType(); - auto *VecTy = dyn_cast<llvm::VectorType>(SrcTy); + if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) + if (GV->isThreadLocal()) + Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV), + NotKnownNonNull); + + llvm::Type *SrcTy = Value->getType(); + if (const auto *ClangVecTy = Ty->getAs<VectorType>()) { + auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy); + if (VecTy && ClangVecTy->isExtVectorBoolType()) { + auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType()); + // Expand to the memory bit width. + unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits(); + // <N x i1> --> <P x i1>. + Value = emitBoolVecConversion(Value, MemNumElems, "insertvec"); + // <P x i1> --> iP. + Value = Builder.CreateBitCast(Value, MemIntTy); + } else if (!CGM.getCodeGenOpts().PreserveVec3Type) { // Handle vec3 special. if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) { // Our source is a vec3, do a shuffle vector to make it a vec4. @@ -1841,7 +2119,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4); } if (Addr.getElementType() != SrcTy) { - Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp"); + Addr = Addr.withElementType(SrcTy); } } } @@ -1861,7 +2139,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, llvm::MDNode *Node = llvm::MDNode::get(Store->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); - Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); + Store->setMetadata(llvm::LLVMContext::MD_nontemporal, Node); } CGM.DecorateInstructionWithTBAA(Store, TBAAInfo); @@ -1939,10 +2217,15 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { return EmitLoadOfGlobalRegLValue(LV); if (LV.isMatrixElt()) { + llvm::Value *Idx = LV.getMatrixIdx(); + if (CGM.getCodeGenOpts().OptimizationLevel > 0) { + const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>(); + llvm::MatrixBuilder MB(Builder); + MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened()); + } llvm::LoadInst *Load = Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified()); - return RValue::get( - Builder.CreateExtractElement(Load, LV.getMatrixIdx(), "matrixext")); + return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext")); } assert(LV.isBitField() && "Unknown LValue type!"); @@ -1990,6 +2273,14 @@ RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) { llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(), LV.isVolatileQualified()); + // HLSL allows treating scalars as one-element vectors. Converting the scalar + // IR value to a vector here allows the rest of codegen to behave as normal. + if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) { + llvm::Type *DstTy = llvm::FixedVectorType::get(Vec->getType(), 1); + llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty); + Vec = Builder.CreateInsertElement(DstTy, Vec, Zero, "cast.splat"); + } + const llvm::Constant *Elts = LV.getExtVectorElts(); // If the result of the expression is a non-vector type, we must be extracting @@ -2018,9 +2309,7 @@ Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) { QualType EQT = LV.getType()->castAs<VectorType>()->getElementType(); llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT); - Address CastToPointerElement = - Builder.CreateElementBitCast(VectorAddress, VectorElementTy, - "conv.ptr.element"); + Address CastToPointerElement = VectorAddress.withElementType(VectorElementTy); const llvm::Constant *Elts = LV.getExtVectorElts(); unsigned ix = getAccessedFieldNo(0, Elts); @@ -2064,8 +2353,19 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, // Read/modify/write the vector, inserting the new element. llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(), Dst.isVolatileQualified()); + auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType()); + if (IRStoreTy) { + auto *IRVecTy = llvm::FixedVectorType::get( + Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits()); + Vec = Builder.CreateBitCast(Vec, IRVecTy); + // iN --> <N x i1>. + } Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(), Dst.getVectorIdx(), "vecins"); + if (IRStoreTy) { + // <N x i1> --> <iN>. + Vec = Builder.CreateBitCast(Vec, IRStoreTy); + } Builder.CreateStore(Vec, Dst.getVectorAddress(), Dst.isVolatileQualified()); return; @@ -2080,9 +2380,15 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, return EmitStoreThroughGlobalRegLValue(Src, Dst); if (Dst.isMatrixElt()) { - llvm::Value *Vec = Builder.CreateLoad(Dst.getMatrixAddress()); - Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(), - Dst.getMatrixIdx(), "matins"); + llvm::Value *Idx = Dst.getMatrixIdx(); + if (CGM.getCodeGenOpts().OptimizationLevel > 0) { + const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>(); + llvm::MatrixBuilder MB(Builder); + MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened()); + } + llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress()); + llvm::Value *Vec = + Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins"); Builder.CreateStore(Vec, Dst.getMatrixAddress(), Dst.isVolatileQualified()); return; @@ -2244,10 +2550,20 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst) { + // HLSL allows storing to scalar values through ExtVector component LValues. + // To support this we need to handle the case where the destination address is + // a scalar. + Address DstAddr = Dst.getExtVectorAddress(); + if (!DstAddr.getElementType()->isVectorTy()) { + assert(!Dst.getType()->isVectorType() && + "this should only occur for non-vector l-values"); + Builder.CreateStore(Src.getScalarVal(), DstAddr, Dst.isVolatileQualified()); + return; + } + // This access turns into a read/modify/write of the vector. Load the input // value now. - llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(), - Dst.isVolatileQualified()); + llvm::Value *Vec = Builder.CreateLoad(DstAddr, Dst.isVolatileQualified()); const llvm::Constant *Elts = Dst.getExtVectorElts(); llvm::Value *SrcVal = Src.getScalarVal(); @@ -2295,7 +2611,8 @@ void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src, llvm_unreachable("unexpected shorten vector length"); } } else { - // If the Src is a scalar (not a vector) it must be updating one element. + // If the Src is a scalar (not a vector), and the target is a vector it must + // be updating one element. unsigned InIdx = getAccessedFieldNo(0, Elts); llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx); Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt); @@ -2428,14 +2745,6 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E, } } -static llvm::Value * -EmitBitCastOfLValueToProperType(CodeGenFunction &CGF, - llvm::Value *V, llvm::Type *IRType, - StringRef Name = StringRef()) { - unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace(); - return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name); -} - static LValue EmitThreadPrivateVarDeclLValue( CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr, llvm::Type *RealVarTy, SourceLocation Loc) { @@ -2446,22 +2755,24 @@ static LValue EmitThreadPrivateVarDeclLValue( Addr = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc); - Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy); + Addr = Addr.withElementType(RealVarTy); return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); } static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD, QualType T) { - llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = + std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); - // Return an invalid address if variable is MT_To and unified - // memory is not enabled. For all other cases: MT_Link and - // MT_To with unified memory, return a valid address. - if (!Res || (*Res == OMPDeclareTargetDeclAttr::MT_To && + // Return an invalid address if variable is MT_To (or MT_Enter starting with + // OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link + // and MT_To (or MT_Enter) with unified memory, return a valid address. + if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To || + *Res == OMPDeclareTargetDeclAttr::MT_Enter) && !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) return Address::invalid(); assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) || - (*Res == OMPDeclareTargetDeclAttr::MT_To && + ((*Res == OMPDeclareTargetDeclAttr::MT_To || + *Res == OMPDeclareTargetDeclAttr::MT_Enter) && CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) && "Expected link clause OR to clause with unified memory enabled."); QualType PtrTy = CGF.getContext().getPointerType(VD->getType()); @@ -2477,10 +2788,11 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal, Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); + QualType PointeeType = RefLVal.getType()->getPointeeType(); CharUnits Align = CGM.getNaturalTypeAlignment( - RefLVal.getType()->getPointeeType(), PointeeBaseInfo, PointeeTBAAInfo, + PointeeType, PointeeBaseInfo, PointeeTBAAInfo, /* forPointeeType= */ true); - return Address(Load, Align); + return Address(Load, ConvertTypeForMem(PointeeType), Align); } LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { @@ -2497,9 +2809,10 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) { llvm::Value *Addr = Builder.CreateLoad(Ptr); - return Address(Addr, CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), - BaseInfo, TBAAInfo, - /*forPointeeType=*/true)); + return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()), + CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo, + TBAAInfo, + /*forPointeeType=*/true)); } LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr, @@ -2520,17 +2833,20 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T); // Check if the variable is marked as declare target with link clause in // device codegen. - if (CGF.getLangOpts().OpenMPIsDevice) { + if (CGF.getLangOpts().OpenMPIsTargetDevice) { Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T); if (Addr.isValid()) return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); } llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD); + + if (VD->getTLSKind() != VarDecl::TLS_None) + V = CGF.Builder.CreateThreadLocalAddress(V); + llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType()); - V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); CharUnits Alignment = CGF.getContext().getDeclAlign(VD); - Address Addr(V, Alignment); + Address Addr(V, RealVarTy, Alignment); // Emit reference to the private copy of the variable if it is an OpenMP // threadprivate variable. if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd && @@ -2555,19 +2871,6 @@ static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM, } llvm::Constant *V = CGM.GetAddrOfFunction(GD); - if (!FD->hasPrototype()) { - if (const FunctionProtoType *Proto = - FD->getType()->getAs<FunctionProtoType>()) { - // Ugly case: for a K&R-style definition, the type of the definition - // isn't the same as the type of a use. Correct for this with a - // bitcast. - QualType NoProtoType = - CGM.getContext().getFunctionNoProtoType(Proto->getReturnType()); - NoProtoType = CGM.getContext().getPointerType(NoProtoType); - V = llvm::ConstantExpr::getBitCast(V, - CGM.getTypes().ConvertType(NoProtoType)); - } - } return V; } @@ -2582,9 +2885,8 @@ static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E, static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD, llvm::Value *ThisValue) { - QualType TagType = CGF.getContext().getTagDeclType(FD->getParent()); - LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType); - return CGF.EmitLValueForField(LV, FD); + + return CGF.EmitLValueForLambdaField(FD, ThisValue); } /// Named Registers are named metadata pointing to the register name @@ -2612,7 +2914,7 @@ static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) { llvm::Value *Ptr = llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0)); - return LValue::MakeGlobalReg(Address(Ptr, Alignment), VD->getType()); + return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType()); } /// Determine whether we can emit a reference to \p VD from the current @@ -2620,8 +2922,7 @@ static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) { /// this context. static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF, const DeclRefExpr *E, - const VarDecl *VD, - bool IsConstant) { + const VarDecl *VD) { // For a variable declared in an enclosing scope, do not emit a spurious // reference even if we have a capture, as that will emit an unwarranted // reference to our capture state, and will likely generate worse code than @@ -2654,7 +2955,7 @@ static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF, // We can emit a spurious reference only if the linkage implies that we'll // be emitting a non-interposable symbol that will be retained until link // time. - switch (CGF.CGM.getLLVMLinkageVarDefinition(VD, IsConstant)) { + switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) { case llvm::GlobalValue::ExternalLinkage: case llvm::GlobalValue::LinkOnceODRLinkage: case llvm::GlobalValue::WeakODRLinkage: @@ -2685,7 +2986,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { // constant value directly instead. if (E->isNonOdrUse() == NOUR_Constant && (VD->getType()->isReferenceType() || - !canEmitSpuriousReferenceToVariable(*this, E, VD, true))) { + !canEmitSpuriousReferenceToVariable(*this, E, VD))) { VD->getAnyInitializer(VD); llvm::Constant *Val = ConstantEmitter(*this).emitAbstract( E->getLocation(), *VD->evaluateValue(), VD->getType()); @@ -2698,9 +2999,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { getContext().getDeclAlign(VD)); llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType()); auto *PTy = llvm::PointerType::get( - VarTy, getContext().getTargetAddressSpace(VD->getType())); - if (PTy != Addr.getType()) - Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy); + VarTy, getTypes().getTargetAddressSpace(VD->getType())); + Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy); } else { // Should we be using the alignment of the constant pointer we emitted? CharUnits Alignment = @@ -2708,7 +3008,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { /* BaseInfo= */ nullptr, /* TBAAInfo= */ nullptr, /* forPointeeType= */ true); - Addr = Address(Val, Alignment); + Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment); } return MakeAddrLValue(Addr, T, AlignmentSource::Decl); } @@ -2739,8 +3039,10 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { LValue CapLVal = EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), CapturedStmtInfo->getContextValue()); + Address LValueAddress = CapLVal.getAddress(*this); CapLVal = MakeAddrLValue( - Address(CapLVal.getPointer(*this), getContext().getDeclAlign(VD)), + Address(LValueAddress.getPointer(), LValueAddress.getElementType(), + getContext().getDeclAlign(VD)), CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl), CapLVal.getTBAAInfo()); // Mark lvalue as nontemporal if the variable is marked as nontemporal @@ -2785,15 +3087,20 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { // Otherwise, it might be static local we haven't emitted yet for // some reason; most likely, because it's in an outer function. } else if (VD->isStaticLocal()) { - addr = Address(CGM.getOrCreateStaticVarDecl( - *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false)), - getContext().getDeclAlign(VD)); + llvm::Constant *var = CGM.getOrCreateStaticVarDecl( + *VD, CGM.getLLVMLinkageVarDefinition(VD)); + addr = Address( + var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD)); // No other cases for now. } else { llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?"); } + // Handle threadlocal function locals. + if (VD->getTLSKind() != VarDecl::TLS_None) + addr = addr.withPointer( + Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull); // Check for OpenMP threadprivate variables. if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && @@ -2832,14 +3139,32 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { return LV; } - if (const auto *FD = dyn_cast<FunctionDecl>(ND)) - return EmitFunctionDeclLValue(*this, E, FD); + if (const auto *FD = dyn_cast<FunctionDecl>(ND)) { + LValue LV = EmitFunctionDeclLValue(*this, E, FD); + + // Emit debuginfo for the function declaration if the target wants to. + if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) { + if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) { + auto *Fn = + cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts()); + if (!Fn->getSubprogram()) + DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn); + } + } + + return LV; + } // FIXME: While we're emitting a binding from an enclosing scope, all other // DeclRefExprs we see should be implicitly treated as if they also refer to // an enclosing scope. - if (const auto *BD = dyn_cast<BindingDecl>(ND)) + if (const auto *BD = dyn_cast<BindingDecl>(ND)) { + if (E->refersToEnclosingVariableOrCapture()) { + auto *FD = LambdaCaptureFields.lookup(BD); + return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue); + } return EmitLValue(BD->getBinding()); + } // We can form DeclRefExprs naming GUID declarations when reconstituting // non-type template parameters into expressions. @@ -2847,9 +3172,20 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD), T, AlignmentSource::Decl); - if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) - return MakeAddrLValue(CGM.GetAddrOfTemplateParamObject(TPO), T, - AlignmentSource::Decl); + if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) { + auto ATPO = CGM.GetAddrOfTemplateParamObject(TPO); + auto AS = getLangASFromTargetAS(ATPO.getAddressSpace()); + + if (AS != T.getAddressSpace()) { + auto TargetAS = getContext().getTargetAddressSpace(T.getAddressSpace()); + auto PtrTy = ATPO.getElementType()->getPointerTo(TargetAS); + auto ASC = getTargetHooks().performAddrSpaceCast( + CGM, ATPO.getPointer(), AS, T.getAddressSpace(), PtrTy); + ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment()); + } + + return MakeAddrLValue(ATPO, T, AlignmentSource::Decl); + } llvm_unreachable("Unhandled DeclRefExpr"); } @@ -2935,7 +3271,7 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) { auto SL = E->getFunctionName(); assert(SL != nullptr && "No StringLiteral name in PredefinedExpr"); StringRef FnName = CurFn->getName(); - if (FnName.startswith("\01")) + if (FnName.starts_with("\01")) FnName = FnName.substr(1); StringRef NameItems[] = { PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName}; @@ -2988,10 +3324,9 @@ llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) { // Format the type name as if for a diagnostic, including quotes and // optionally an 'aka'. SmallString<32> Buffer; - CGM.getDiags().ConvertArgToString(DiagnosticsEngine::ak_qualtype, - (intptr_t)T.getAsOpaquePtr(), - StringRef(), StringRef(), None, Buffer, - None); + CGM.getDiags().ConvertArgToString( + DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(), + StringRef(), std::nullopt, Buffer, std::nullopt); llvm::Constant *Components[] = { Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo), @@ -3020,7 +3355,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) { // Floating-point types which fit into intptr_t are bitcast to integers // and then passed directly (after zero-extension, if necessary). if (V->getType()->isFloatingPointTy()) { - unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedSize(); + unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue(); if (Bits <= TargetTy->getIntegerBitWidth()) V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(), Bits)); @@ -3084,7 +3419,8 @@ llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) { auto FilenameGV = CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src"); CGM.getSanitizerMetadata()->disableSanitizerForGlobal( - cast<llvm::GlobalVariable>(FilenameGV.getPointer())); + cast<llvm::GlobalVariable>( + FilenameGV.getPointer()->stripPointerCasts())); Filename = FilenameGV.getPointer(); Line = PLoc.getLine(); Column = PLoc.getColumn(); @@ -3114,7 +3450,7 @@ enum class CheckRecoverableKind { static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) { assert(Kind.countPopulation() == 1); - if (Kind == SanitizerKind::Function || Kind == SanitizerKind::Vptr) + if (Kind == SanitizerKind::Vptr) return CheckRecoverableKind::AlwaysRecoverable; else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable) return CheckRecoverableKind::Unrecoverable; @@ -3142,7 +3478,7 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, CheckRecoverableKind RecoverKind, bool IsFatal, llvm::BasicBlock *ContBB) { assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable); - Optional<ApplyDebugLocation> DL; + std::optional<ApplyDebugLocation> DL; if (!CGF.Builder.getCurrentDebugLocation()) { // Ensure that the call has at least an artificial debug location. DL.emplace(CGF, SourceLocation()); @@ -3162,12 +3498,12 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, bool MayReturn = !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable; - llvm::AttrBuilder B; + llvm::AttrBuilder B(CGF.getLLVMContext()); if (!MayReturn) { B.addAttribute(llvm::Attribute::NoReturn) .addAttribute(llvm::Attribute::NoUnwind); } - B.addAttribute(llvm::Attribute::UWTable); + B.addUWTableAttr(llvm::UWTableKind::Default); llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction( FnType, FnName, @@ -3190,7 +3526,7 @@ void CodeGenFunction::EmitCheck( assert(IsSanitizerScope); assert(Checked.size() > 0); assert(CheckHandler >= 0 && - size_t(CheckHandler) < llvm::array_lengthof(SanitizerHandlers)); + size_t(CheckHandler) < std::size(SanitizerHandlers)); const StringRef CheckName = SanitizerHandlers[CheckHandler].Name; llvm::Value *FatalCond = nullptr; @@ -3252,13 +3588,15 @@ void CodeGenFunction::EmitCheck( // Emit handler arguments and create handler function type. if (!StaticArgs.empty()) { llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs); - auto *InfoPtr = - new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false, - llvm::GlobalVariable::PrivateLinkage, Info); + auto *InfoPtr = new llvm::GlobalVariable( + CGM.getModule(), Info->getType(), false, + llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr, + llvm::GlobalVariable::NotThreadLocal, + CGM.getDataLayout().getDefaultGlobalsAddressSpace()); InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr); - Args.push_back(Builder.CreateBitCast(InfoPtr, Int8PtrTy)); - ArgTypes.push_back(Int8PtrTy); + Args.push_back(InfoPtr); + ArgTypes.push_back(Args.back()->getType()); } for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) { @@ -3323,8 +3661,7 @@ void CodeGenFunction::EmitCfiSlowPathCheck( "__cfi_slowpath_diag", llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false)); - CheckCall = Builder.CreateCall( - SlowPathFn, {TypeId, Ptr, Builder.CreateBitCast(InfoPtr, Int8PtrTy)}); + CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr, InfoPtr}); } else { SlowPathFn = CGM.getModule().getOrInsertFunction( "__cfi_slowpath", @@ -3347,14 +3684,12 @@ void CodeGenFunction::EmitCfiCheckStub() { llvm::Function *F = llvm::Function::Create( llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false), llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M); + F->setAlignment(llvm::Align(4096)); CGM.setDSOLocal(F); llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F); - // FIXME: consider emitting an intrinsic call like - // call void @llvm.cfi_check(i64 %0, i8* %1, i8* %2) - // which can be lowered in CrossDSOCFI pass to the actual contents of - // __cfi_check. This would allow inlining of __cfi_check calls. - llvm::CallInst::Create( - llvm::Intrinsic::getDeclaration(M, llvm::Intrinsic::trap), "", BB); + // CrossDSOCFI pass is not executed if there is no executable code. + SmallVector<llvm::Value*> Args{F->getArg(2), F->getArg(1)}; + llvm::CallInst::Create(M->getFunction("__cfi_check_fail"), Args, "", BB); llvm::ReturnInst::Create(Ctx, nullptr, BB); } @@ -3369,9 +3704,9 @@ void CodeGenFunction::EmitCfiCheckFail() { SanitizerScope SanScope(this); FunctionArgList Args; ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy, - ImplicitParamDecl::Other); + ImplicitParamKind::Other); ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy, - ImplicitParamDecl::Other); + ImplicitParamKind::Other); Args.push_back(&ArgData); Args.push_back(&ArgAddr); @@ -3382,14 +3717,14 @@ void CodeGenFunction::EmitCfiCheckFail() { llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false), llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule()); - CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F); + CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false); CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F); F->setVisibility(llvm::GlobalValue::HiddenVisibility); StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args, SourceLocation()); - // This function should not be affected by blacklist. This function does + // This function is not affected by NoSanitizeList. This function does // not have a source location, but "src:*" would still apply. Revert any // changes to SanOpts made in StartFunction. SanOpts = CGM.getLangOpts().Sanitize; @@ -3415,7 +3750,8 @@ void CodeGenFunction::EmitCfiCheckFail() { CfiCheckFailDataTy, Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0, 0); - Address CheckKindAddr(V, getIntAlign()); + + Address CheckKindAddr(V, Int8Ty, getIntAlign()); llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr); llvm::Value *AllVtables = llvm::MetadataAsValue::get( @@ -3458,7 +3794,7 @@ void CodeGenFunction::EmitUnreachable(SourceLocation Loc) { EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), SanitizerKind::Unreachable), SanitizerHandler::BuiltinUnreachable, - EmitCheckSourceLocation(Loc), None); + EmitCheckSourceLocation(Loc), std::nullopt); } Builder.CreateUnreachable(); } @@ -3471,32 +3807,37 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked, // check-type per function to save on code size. if (TrapBBs.size() <= CheckHandlerID) TrapBBs.resize(CheckHandlerID + 1); + llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID]; - if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB) { + if (!ClSanitizeDebugDeoptimization && + CGM.getCodeGenOpts().OptimizationLevel && TrapBB && + (!CurCodeDecl || !CurCodeDecl->hasAttr<OptimizeNoneAttr>())) { + auto Call = TrapBB->begin(); + assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB"); + + Call->applyMergedLocation(Call->getDebugLoc(), + Builder.getCurrentDebugLocation()); + Builder.CreateCondBr(Checked, Cont, TrapBB); + } else { TrapBB = createBasicBlock("trap"); Builder.CreateCondBr(Checked, Cont, TrapBB); EmitBlock(TrapBB); - llvm::CallInst *TrapCall = - Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap), - llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID)); + llvm::CallInst *TrapCall = Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::ubsantrap), + llvm::ConstantInt::get(CGM.Int8Ty, ClSanitizeDebugDeoptimization + ? TrapBB->getParent()->size() + : CheckHandlerID)); if (!CGM.getCodeGenOpts().TrapFuncName.empty()) { auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name", CGM.getCodeGenOpts().TrapFuncName); - TrapCall->addAttribute(llvm::AttributeList::FunctionIndex, A); + TrapCall->addFnAttr(A); } TrapCall->setDoesNotReturn(); TrapCall->setDoesNotThrow(); Builder.CreateUnreachable(); - } else { - auto Call = TrapBB->begin(); - assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB"); - - Call->applyMergedLocation(Call->getDebugLoc(), - Builder.getCurrentDebugLocation()); - Builder.CreateCondBr(Checked, Cont, TrapBB); } EmitBlock(Cont); @@ -3509,7 +3850,7 @@ llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) { if (!CGM.getCodeGenOpts().TrapFuncName.empty()) { auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name", CGM.getCodeGenOpts().TrapFuncName); - TrapCall->addAttribute(llvm::AttributeList::FunctionIndex, A); + TrapCall->addFnAttr(A); } return TrapCall; @@ -3528,7 +3869,7 @@ Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E, // If the array type was an incomplete type, we need to make sure // the decay ends up being the right type. llvm::Type *NewTy = ConvertType(E->getType()); - Addr = Builder.CreateElementBitCast(Addr, NewTy); + Addr = Addr.withElementType(NewTy); // Note that VLA pointers are always decayed, so we don't need to do // anything here. @@ -3547,7 +3888,7 @@ Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E, if (BaseInfo) *BaseInfo = LV.getBaseInfo(); if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType); - return Builder.CreateElementBitCast(Addr, ConvertTypeForMem(EltType)); + return Addr.withElementType(ConvertTypeForMem(EltType)); } /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an @@ -3567,6 +3908,7 @@ static const Expr *isSimpleArrayDecayOperand(const Expr *E) { } static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, + llvm::Type *elemType, llvm::Value *ptr, ArrayRef<llvm::Value*> indices, bool inbounds, @@ -3574,11 +3916,11 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, SourceLocation loc, const llvm::Twine &name = "arrayidx") { if (inbounds) { - return CGF.EmitCheckedInBoundsGEP(ptr, indices, signedIndices, + return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices, CodeGenFunction::NotSubtraction, loc, name); } else { - return CGF.Builder.CreateGEP(ptr, indices, name); + return CGF.Builder.CreateGEP(elemType, ptr, indices, name); } } @@ -3606,6 +3948,33 @@ static QualType getFixedSizeElementType(const ASTContext &ctx, return eltType; } +static bool hasBPFPreserveStaticOffset(const RecordDecl *D) { + return D && D->hasAttr<BPFPreserveStaticOffsetAttr>(); +} + +static bool hasBPFPreserveStaticOffset(const Expr *E) { + if (!E) + return false; + QualType PointeeType = E->getType()->getPointeeType(); + if (PointeeType.isNull()) + return false; + if (const auto *BaseDecl = PointeeType->getAsRecordDecl()) + return hasBPFPreserveStaticOffset(BaseDecl); + return false; +} + +// Wraps Addr with a call to llvm.preserve.static.offset intrinsic. +static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF, + Address &Addr) { + if (!CGF.getTarget().getTriple().isBPF()) + return Addr; + + llvm::Function *Fn = + CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset); + llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()}); + return Address(Call, Addr.getElementType(), Addr.getAlignment()); +} + /// Given an array base, check whether its member access belongs to a record /// with preserve_access_index attribute or not. static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) { @@ -3651,7 +4020,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, const llvm::Twine &name = "arrayidx") { // All the indices except that last must be zero. #ifndef NDEBUG - for (auto idx : indices.drop_back()) + for (auto *idx : indices.drop_back()) assert(isa<llvm::ConstantInt>(idx) && cast<llvm::ConstantInt>(idx)->isZero()); #endif @@ -3667,13 +4036,16 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, CharUnits eltAlign = getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); + if (hasBPFPreserveStaticOffset(Base)) + addr = wrapWithBPFPreserveStaticOffset(CGF, addr); + llvm::Value *eltPtr; auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back()); if (!LastIndex || (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) { eltPtr = emitArraySubscriptGEP( - CGF, addr.getPointer(), indices, inbounds, signedIndices, - loc, name); + CGF, addr.getElementType(), addr.getPointer(), indices, inbounds, + signedIndices, loc, name); } else { // Remember the original array subscript for bpf target unsigned idx = LastIndex->getZExtValue(); @@ -3686,7 +4058,62 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, idx, DbgInfo); } - return Address(eltPtr, eltAlign); + return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign); +} + +/// The offset of a field from the beginning of the record. +static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD, + const FieldDecl *FD, int64_t &Offset) { + ASTContext &Ctx = CGF.getContext(); + const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD); + unsigned FieldNo = 0; + + for (const Decl *D : RD->decls()) { + if (const auto *Record = dyn_cast<RecordDecl>(D)) + if (getFieldOffsetInBits(CGF, Record, FD, Offset)) { + Offset += Layout.getFieldOffset(FieldNo); + return true; + } + + if (const auto *Field = dyn_cast<FieldDecl>(D)) + if (FD == Field) { + Offset += Layout.getFieldOffset(FieldNo); + return true; + } + + if (isa<FieldDecl>(D)) + ++FieldNo; + } + + return false; +} + +/// Returns the relative offset difference between \p FD1 and \p FD2. +/// \code +/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2) +/// \endcode +/// Both fields must be within the same struct. +static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF, + const FieldDecl *FD1, + const FieldDecl *FD2) { + const RecordDecl *FD1OuterRec = + FD1->getParent()->getOuterLexicalRecordContext(); + const RecordDecl *FD2OuterRec = + FD2->getParent()->getOuterLexicalRecordContext(); + + if (FD1OuterRec != FD2OuterRec) + // Fields must be within the same RecordDecl. + return std::optional<int64_t>(); + + int64_t FD1Offset = 0; + if (!getFieldOffsetInBits(CGF, FD1OuterRec, FD1, FD1Offset)) + return std::optional<int64_t>(); + + int64_t FD2Offset = 0; + if (!getFieldOffsetInBits(CGF, FD2OuterRec, FD2, FD2Offset)) + return std::optional<int64_t>(); + + return std::make_optional<int64_t>(FD1Offset - FD2Offset); } LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, @@ -3791,19 +4218,15 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, // interfaces, so we can't rely on GEP to do this scaling // correctly, so we need to cast to i8*. FIXME: is this actually // true? A lot of other things in the fragile ABI would break... - llvm::Type *OrigBaseTy = Addr.getType(); - Addr = Builder.CreateElementBitCast(Addr, Int8Ty); + llvm::Type *OrigBaseElemTy = Addr.getElementType(); // Do the GEP. CharUnits EltAlign = getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize); llvm::Value *EltPtr = - emitArraySubscriptGEP(*this, Addr.getPointer(), ScaledIdx, false, - SignedIndices, E->getExprLoc()); - Addr = Address(EltPtr, EltAlign); - - // Cast back. - Addr = Builder.CreateBitCast(Addr, OrigBaseTy); + emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx, + false, SignedIndices, E->getExprLoc()); + Addr = Address(EltPtr, OrigBaseElemTy, EltAlign); } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { // If this is A[i] where A is an array, the frontend will have decayed the // base to be a ArrayToPointerDecay implicit cast. While correct, it is @@ -3820,6 +4243,47 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, ArrayLV = EmitLValue(Array); auto *Idx = EmitIdxAfterBase(/*Promote*/true); + if (SanOpts.has(SanitizerKind::ArrayBounds)) { + // If the array being accessed has a "counted_by" attribute, generate + // bounds checking code. The "count" field is at the top level of the + // struct or in an anonymous struct, that's also at the top level. Future + // expansions may allow the "count" to reside at any place in the struct, + // but the value of "counted_by" will be a "simple" path to the count, + // i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or + // similar to emit the correct GEP. + const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = + getLangOpts().getStrictFlexArraysLevel(); + + if (const auto *ME = dyn_cast<MemberExpr>(Array); + ME && + ME->isFlexibleArrayMemberLike(getContext(), StrictFlexArraysLevel) && + ME->getMemberDecl()->hasAttr<CountedByAttr>()) { + const FieldDecl *FAMDecl = dyn_cast<FieldDecl>(ME->getMemberDecl()); + if (const FieldDecl *CountFD = FindCountedByField(FAMDecl)) { + if (std::optional<int64_t> Diff = + getOffsetDifferenceInBits(*this, CountFD, FAMDecl)) { + CharUnits OffsetDiff = CGM.getContext().toCharUnitsFromBits(*Diff); + + // Create a GEP with a byte offset between the FAM and count and + // use that to load the count value. + Addr = Builder.CreatePointerBitCastOrAddrSpaceCast( + ArrayLV.getAddress(*this), Int8PtrTy, Int8Ty); + + llvm::Type *CountTy = ConvertType(CountFD->getType()); + llvm::Value *Res = Builder.CreateInBoundsGEP( + Int8Ty, Addr.getPointer(), + Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep"); + Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(), + ".counted_by.load"); + + // Now emit the bounds checking. + EmitBoundsCheckImpl(E, Res, Idx, E->getIdx()->getType(), + Array->getType(), Accessed); + } + } + } + } + // Propagate the alignment from the array itself to the result. QualType arrayType = Array->getType(); Addr = emitArraySubscriptGEP( @@ -3881,7 +4345,7 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, // If the array type was an incomplete type, we need to make sure // the decay ends up being the right type. llvm::Type *NewTy = CGF.ConvertType(BaseTy); - Addr = CGF.Builder.CreateElementBitCast(Addr, NewTy); + Addr = Addr.withElementType(NewTy); // Note that VLA pointers are always decayed, so we don't need to do // anything here. @@ -3891,8 +4355,7 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay"); } - return CGF.Builder.CreateElementBitCast(Addr, - CGF.ConvertTypeForMem(ElTy)); + return Addr.withElementType(CGF.ConvertTypeForMem(ElTy)); } LValueBaseInfo TypeBaseInfo; TBAAAccessInfo TypeTBAAInfo; @@ -3900,7 +4363,8 @@ static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo); BaseInfo.mergeForCast(TypeBaseInfo); TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo); - return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)), Align); + return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)), + CGF.ConvertTypeForMem(ElTy), Align); } return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); } @@ -3933,14 +4397,15 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, llvm::APSInt ConstLength; if (Length) { // Idx = LowerBound + Length - 1; - if (Optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) { + if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) { ConstLength = CL->zextOrTrunc(PointerWidthInBits); Length = nullptr; } auto *LowerBound = E->getLowerBound(); llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false); if (LowerBound) { - if (Optional<llvm::APSInt> LB = LowerBound->getIntegerConstantExpr(C)) { + if (std::optional<llvm::APSInt> LB = + LowerBound->getIntegerConstantExpr(C)) { ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits); LowerBound = nullptr; } @@ -3980,12 +4445,13 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, : BaseTy; if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) { Length = VAT->getSizeExpr(); - if (Optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) { + if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) { ConstLength = *L; Length = nullptr; } } else { auto *CAT = C.getAsConstantArrayType(ArrayTy); + assert(CAT && "unexpected type for array initializer"); ConstLength = CAT->getSize(); } if (Length) { @@ -4170,22 +4636,45 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { /// Given that we are currently emitting a lambda, emit an l-value for /// one of its members. -LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) { - assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent()->isLambda()); - assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent() == Field->getParent()); - QualType LambdaTagType = - getContext().getTagDeclType(Field->getParent()); - LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue, LambdaTagType); +/// +LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field, + llvm::Value *ThisValue) { + bool HasExplicitObjectParameter = false; + if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(CurCodeDecl)) { + HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction(); + assert(MD->getParent()->isLambda()); + assert(MD->getParent() == Field->getParent()); + } + LValue LambdaLV; + if (HasExplicitObjectParameter) { + const VarDecl *D = cast<CXXMethodDecl>(CurCodeDecl)->getParamDecl(0); + auto It = LocalDeclMap.find(D); + assert(It != LocalDeclMap.end() && "explicit parameter not loaded?"); + Address AddrOfExplicitObject = It->getSecond(); + if (D->getType()->isReferenceType()) + LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(), + AlignmentSource::Decl); + else + LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(), + D->getType().getNonReferenceType()); + } else { + QualType LambdaTagType = getContext().getTagDeclType(Field->getParent()); + LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType); + } return EmitLValueForField(LambdaLV, Field); } +LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) { + return EmitLValueForLambdaField(Field, CXXABIThisValue); +} + /// Get the field index in the debug info. The debug info structure/union /// will ignore the unnamed bitfields. unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex) { unsigned I = 0, Skipped = 0; - for (auto F : Rec->getDefinition()->fields()) { + for (auto *F : Rec->getDefinition()->fields()) { if (I == FieldIndex) break; if (F->isUnnamedBitfield()) @@ -4204,7 +4693,7 @@ static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base, CGF.getContext().getFieldOffset(Field)); if (Offset.isZero()) return Base; - Base = CGF.Builder.CreateElementBitCast(Base, CGF.Int8Ty); + Base = Base.withElementType(CGF.Int8Ty); return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset); } @@ -4274,6 +4763,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, Address Addr = base.getAddress(*this); unsigned Idx = RL.getLLVMFieldNo(field); const RecordDecl *rec = field->getParent(); + if (hasBPFPreserveStaticOffset(rec)) + Addr = wrapWithBPFPreserveStaticOffset(*this, Addr); if (!UseVolatile) { if (!IsInPreservedAIRegion && (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) { @@ -4292,8 +4783,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, UseVolatile ? Info.VolatileStorageSize : Info.StorageSize; // Get the access type. llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS); - if (Addr.getElementType() != FieldIntTy) - Addr = Builder.CreateElementBitCast(Addr, FieldIntTy); + Addr = Addr.withElementType(FieldIntTy); if (UseVolatile) { const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity(); if (VolatileOffset) @@ -4347,6 +4837,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, } Address addr = base.getAddress(*this); + if (hasBPFPreserveStaticOffset(rec)) + addr = wrapWithBPFPreserveStaticOffset(*this, addr); if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) { if (CGM.getCodeGenOpts().StrictVTablePointers && ClassDef->isDynamicClass()) { @@ -4355,7 +4847,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, // fields may leak the real address of dynamic object, which could result // in miscompilation when leaked pointer would be compared. auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer()); - addr = Address(stripped, addr.getAlignment()); + addr = Address(stripped, addr.getElementType(), addr.getAlignment()); } } @@ -4366,8 +4858,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, hasAnyVptr(FieldType, getContext())) // Because unions can easily skip invariant.barriers, we need to add // a barrier every time CXXRecord field with vptr is referenced. - addr = Address(Builder.CreateLaunderInvariantGroup(addr.getPointer()), - addr.getAlignment()); + addr = Builder.CreateLaunderInvariantGroup(addr); if (IsInPreservedAIRegion || (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) { @@ -4377,12 +4868,11 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, addr = Address( Builder.CreatePreserveUnionAccessIndex( addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), - addr.getAlignment()); + addr.getElementType(), addr.getAlignment()); } if (FieldType->isReferenceType()) - addr = Builder.CreateElementBitCast( - addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName()); + addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType)); } else { if (!IsInPreservedAIRegion && (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) @@ -4407,11 +4897,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, } // Make sure that the address is pointing to the right type. This is critical - // for both unions and structs. A union needs a bitcast, a struct element - // will need a bitcast if the LLVM type laid out doesn't match the desired - // type. - addr = Builder.CreateElementBitCast( - addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName()); + // for both unions and structs. + addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType)); if (field->hasAttr<AnnotateAttr>()) addr = EmitFieldAnnotations(field, addr); @@ -4438,7 +4925,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(LValue Base, // Make sure that the address is pointing to the right type. llvm::Type *llvmType = ConvertTypeForMem(FieldType); - V = Builder.CreateElementBitCast(V, llvmType, Field->getName()); + V = V.withElementType(llvmType); // TODO: Generate TBAA information that describes this access as a structure // member access and not just an access to an object of the field's type. This @@ -4490,102 +4977,150 @@ LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) { /// Emit the operand of a glvalue conditional operator. This is either a glvalue /// or a (possibly-parenthesized) throw-expression. If this is a throw, no /// LValue is returned and the current block has been terminated. -static Optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF, - const Expr *Operand) { +static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF, + const Expr *Operand) { if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Operand->IgnoreParens())) { CGF.EmitCXXThrowExpr(ThrowExpr, /*KeepInsertionPoint*/false); - return None; + return std::nullopt; } return CGF.EmitLValue(Operand); } -LValue CodeGenFunction:: -EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { - if (!expr->isGLValue()) { - // ?: here should be an aggregate. - assert(hasAggregateEvaluationKind(expr->getType()) && - "Unexpected conditional operator!"); - return EmitAggExprToLValue(expr); - } - - OpaqueValueMapping binding(*this, expr); - - const Expr *condExpr = expr->getCond(); +namespace { +// Handle the case where the condition is a constant evaluatable simple integer, +// which means we don't have to separately handle the true/false blocks. +std::optional<LValue> HandleConditionalOperatorLValueSimpleCase( + CodeGenFunction &CGF, const AbstractConditionalOperator *E) { + const Expr *condExpr = E->getCond(); bool CondExprBool; - if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) { - const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr(); - if (!CondExprBool) std::swap(live, dead); + if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) { + const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr(); + if (!CondExprBool) + std::swap(Live, Dead); - if (!ContainsLabel(dead)) { + if (!CGF.ContainsLabel(Dead)) { // If the true case is live, we need to track its region. if (CondExprBool) - incrementProfileCounter(expr); + CGF.incrementProfileCounter(E); // If a throw expression we emit it and return an undefined lvalue // because it can't be used. - if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(live->IgnoreParens())) { - EmitCXXThrowExpr(ThrowExpr); - llvm::Type *Ty = - llvm::PointerType::getUnqual(ConvertType(dead->getType())); - return MakeAddrLValue( - Address(llvm::UndefValue::get(Ty), CharUnits::One()), - dead->getType()); + if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) { + CGF.EmitCXXThrowExpr(ThrowExpr); + llvm::Type *ElemTy = CGF.ConvertType(Dead->getType()); + llvm::Type *Ty = CGF.UnqualPtrTy; + return CGF.MakeAddrLValue( + Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()), + Dead->getType()); } - return EmitLValue(live); + return CGF.EmitLValue(Live); } } + return std::nullopt; +} +struct ConditionalInfo { + llvm::BasicBlock *lhsBlock, *rhsBlock; + std::optional<LValue> LHS, RHS; +}; - llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true"); - llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false"); - llvm::BasicBlock *contBlock = createBasicBlock("cond.end"); - - ConditionalEvaluation eval(*this); - EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, getProfileCount(expr)); +// Create and generate the 3 blocks for a conditional operator. +// Leaves the 'current block' in the continuation basic block. +template<typename FuncTy> +ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF, + const AbstractConditionalOperator *E, + const FuncTy &BranchGenFunc) { + ConditionalInfo Info{CGF.createBasicBlock("cond.true"), + CGF.createBasicBlock("cond.false"), std::nullopt, + std::nullopt}; + llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end"); + + CodeGenFunction::ConditionalEvaluation eval(CGF); + CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock, + CGF.getProfileCount(E)); // Any temporaries created here are conditional. - EmitBlock(lhsBlock); - incrementProfileCounter(expr); - eval.begin(*this); - Optional<LValue> lhs = - EmitLValueOrThrowExpression(*this, expr->getTrueExpr()); - eval.end(*this); - - if (lhs && !lhs->isSimple()) - return EmitUnsupportedLValue(expr, "conditional operator"); + CGF.EmitBlock(Info.lhsBlock); + CGF.incrementProfileCounter(E); + eval.begin(CGF); + Info.LHS = BranchGenFunc(CGF, E->getTrueExpr()); + eval.end(CGF); + Info.lhsBlock = CGF.Builder.GetInsertBlock(); - lhsBlock = Builder.GetInsertBlock(); - if (lhs) - Builder.CreateBr(contBlock); + if (Info.LHS) + CGF.Builder.CreateBr(endBlock); // Any temporaries created here are conditional. - EmitBlock(rhsBlock); - eval.begin(*this); - Optional<LValue> rhs = - EmitLValueOrThrowExpression(*this, expr->getFalseExpr()); - eval.end(*this); - if (rhs && !rhs->isSimple()) - return EmitUnsupportedLValue(expr, "conditional operator"); - rhsBlock = Builder.GetInsertBlock(); + CGF.EmitBlock(Info.rhsBlock); + eval.begin(CGF); + Info.RHS = BranchGenFunc(CGF, E->getFalseExpr()); + eval.end(CGF); + Info.rhsBlock = CGF.Builder.GetInsertBlock(); + CGF.EmitBlock(endBlock); + + return Info; +} +} // namespace + +void CodeGenFunction::EmitIgnoredConditionalOperator( + const AbstractConditionalOperator *E) { + if (!E->isGLValue()) { + // ?: here should be an aggregate. + assert(hasAggregateEvaluationKind(E->getType()) && + "Unexpected conditional operator!"); + return (void)EmitAggExprToLValue(E); + } + + OpaqueValueMapping binding(*this, E); + if (HandleConditionalOperatorLValueSimpleCase(*this, E)) + return; + + EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) { + CGF.EmitIgnoredExpr(E); + return LValue{}; + }); +} +LValue CodeGenFunction::EmitConditionalOperatorLValue( + const AbstractConditionalOperator *expr) { + if (!expr->isGLValue()) { + // ?: here should be an aggregate. + assert(hasAggregateEvaluationKind(expr->getType()) && + "Unexpected conditional operator!"); + return EmitAggExprToLValue(expr); + } + + OpaqueValueMapping binding(*this, expr); + if (std::optional<LValue> Res = + HandleConditionalOperatorLValueSimpleCase(*this, expr)) + return *Res; + + ConditionalInfo Info = EmitConditionalBlocks( + *this, expr, [](CodeGenFunction &CGF, const Expr *E) { + return EmitLValueOrThrowExpression(CGF, E); + }); - EmitBlock(contBlock); + if ((Info.LHS && !Info.LHS->isSimple()) || + (Info.RHS && !Info.RHS->isSimple())) + return EmitUnsupportedLValue(expr, "conditional operator"); - if (lhs && rhs) { - llvm::PHINode *phi = - Builder.CreatePHI(lhs->getPointer(*this)->getType(), 2, "cond-lvalue"); - phi->addIncoming(lhs->getPointer(*this), lhsBlock); - phi->addIncoming(rhs->getPointer(*this), rhsBlock); - Address result(phi, std::min(lhs->getAlignment(), rhs->getAlignment())); + if (Info.LHS && Info.RHS) { + Address lhsAddr = Info.LHS->getAddress(*this); + Address rhsAddr = Info.RHS->getAddress(*this); + llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue"); + phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock); + phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock); + Address result(phi, lhsAddr.getElementType(), + std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment())); AlignmentSource alignSource = - std::max(lhs->getBaseInfo().getAlignmentSource(), - rhs->getBaseInfo().getAlignmentSource()); + std::max(Info.LHS->getBaseInfo().getAlignmentSource(), + Info.RHS->getBaseInfo().getAlignmentSource()); TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator( - lhs->getTBAAInfo(), rhs->getTBAAInfo()); + Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo()); return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource), TBAAInfo); } else { - assert((lhs || rhs) && + assert((Info.LHS || Info.RHS) && "both operands of glvalue conditional are throw-expressions?"); - return lhs ? *lhs : *rhs; + return Info.LHS ? *Info.LHS : *Info.RHS; } } @@ -4608,7 +5143,6 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_IntegralToPointer: case CK_PointerToIntegral: case CK_PointerToBoolean: - case CK_VectorSplat: case CK_IntegralCast: case CK_BooleanToSignedIntegral: case CK_IntegralToBoolean: @@ -4643,6 +5177,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_FixedPointToBoolean: case CK_FixedPointToIntegral: case CK_IntegralToFixedPoint: + case CK_MatrixCast: return EmitUnsupportedLValue(E, "unexpected cast lvalue"); case CK_Dependent: @@ -4667,10 +5202,28 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { case CK_UserDefinedConversion: case CK_CPointerToObjCPointerCast: case CK_BlockPointerToObjCPointerCast: - case CK_NoOp: case CK_LValueToRValue: return EmitLValue(E->getSubExpr()); + case CK_NoOp: { + // CK_NoOp can model a qualification conversion, which can remove an array + // bound and change the IR type. + // FIXME: Once pointee types are removed from IR, remove this. + LValue LV = EmitLValue(E->getSubExpr()); + // Propagate the volatile qualifer to LValue, if exist in E. + if (E->changesVolatileQualification()) + LV.getQuals() = E->getType().getQualifiers(); + if (LV.isSimple()) { + Address V = LV.getAddress(*this); + if (V.isValid()) { + llvm::Type *T = ConvertTypeForMem(E->getType()); + if (V.getElementType() != T) + LV.setAddress(V.withElementType(T)); + } + } + return LV; + } + case CK_UncheckedDerivedToBase: case CK_DerivedToBase: { const auto *DerivedClassTy = @@ -4711,7 +5264,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { Derived.getPointer(), E->getType()); if (SanOpts.has(SanitizerKind::CFIDerivedCast)) - EmitVTablePtrCheckForCast(E->getType(), Derived.getPointer(), + EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false, CFITCK_DerivedCast, E->getBeginLoc()); @@ -4724,11 +5277,11 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { CGM.EmitExplicitCastExprType(CE, this); LValue LV = EmitLValue(E->getSubExpr()); - Address V = Builder.CreateBitCast(LV.getAddress(*this), - ConvertType(CE->getTypeAsWritten())); + Address V = LV.getAddress(*this).withElementType( + ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType())); if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) - EmitVTablePtrCheckForCast(E->getType(), V.getPointer(), + EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false, CFITCK_UnrelatedCast, E->getBeginLoc()); @@ -4742,18 +5295,25 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { *this, LV.getPointer(*this), E->getSubExpr()->getType().getAddressSpace(), E->getType().getAddressSpace(), ConvertType(DestTy)); - return MakeAddrLValue(Address(V, LV.getAddress(*this).getAlignment()), + return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()), + LV.getAddress(*this).getAlignment()), E->getType(), LV.getBaseInfo(), LV.getTBAAInfo()); } case CK_ObjCObjectLValueCast: { LValue LV = EmitLValue(E->getSubExpr()); - Address V = Builder.CreateElementBitCast(LV.getAddress(*this), - ConvertType(E->getType())); + Address V = LV.getAddress(*this).withElementType(ConvertType(E->getType())); return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(), CGM.getTBAAInfoForSubobject(LV, E->getType())); } case CK_ZeroToOCLOpaqueType: llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid"); + + case CK_VectorSplat: { + // LValue results of vector splats are only supported in HLSL. + if (!getLangOpts().HLSL) + return EmitUnsupportedLValue(E, "unexpected cast lvalue"); + return EmitLValue(E->getSubExpr()); + } } llvm_unreachable("Unhandled lvalue cast kind?"); @@ -4832,9 +5392,12 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E)) return EmitCUDAKernelCallExpr(CE, ReturnValue); + // A CXXOperatorCallExpr is created even for explicit object methods, but + // these should be treated like static function call. if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E)) - if (const CXXMethodDecl *MD = - dyn_cast_or_null<CXXMethodDecl>(CE->getCalleeDecl())) + if (const auto *MD = + dyn_cast_if_present<CXXMethodDecl>(CE->getCalleeDecl()); + MD && MD->isImplicitObjectMemberFunction()) return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue); CGCallee callee = EmitCallee(E->getCallee()); @@ -4858,21 +5421,66 @@ RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E, return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue); } +// Detect the unusual situation where an inline version is shadowed by a +// non-inline version. In that case we should pick the external one +// everywhere. That's GCC behavior too. +static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) { + for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl()) + if (!PD->isInlineBuiltinDeclaration()) + return false; + return true; +} + static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); if (auto builtinID = FD->getBuiltinID()) { - // Replaceable builtin provide their own implementation of a builtin. Unless - // we are in the builtin implementation itself, don't call the actual - // builtin. If we are in the builtin implementation, avoid trivial infinite - // recursion. - if (!FD->isInlineBuiltinDeclaration() || - CGF.CurFn->getName() == FD->getName()) + std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str(); + std::string NoBuiltins = "no-builtins"; + + StringRef Ident = CGF.CGM.getMangledName(GD); + std::string FDInlineName = (Ident + ".inline").str(); + + bool IsPredefinedLibFunction = + CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID); + bool HasAttributeNoBuiltin = + CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) || + CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins); + + // When directing calling an inline builtin, call it through it's mangled + // name to make it clear it's not the actual builtin. + if (CGF.CurFn->getName() != FDInlineName && + OnlyHasInlineBuiltinDeclaration(FD)) { + llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD); + llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr); + llvm::Module *M = Fn->getParent(); + llvm::Function *Clone = M->getFunction(FDInlineName); + if (!Clone) { + Clone = llvm::Function::Create(Fn->getFunctionType(), + llvm::GlobalValue::InternalLinkage, + Fn->getAddressSpace(), FDInlineName, M); + Clone->addFnAttr(llvm::Attribute::AlwaysInline); + } + return CGCallee::forDirect(Clone, GD); + } + + // Replaceable builtins provide their own implementation of a builtin. If we + // are in an inline builtin implementation, avoid trivial infinite + // recursion. Honor __attribute__((no_builtin("foo"))) or + // __attribute__((no_builtin)) on the current function unless foo is + // not a predefined library function which means we must generate the + // builtin no matter what. + else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin) return CGCallee::forBuiltin(builtinID, FD); } - llvm::Constant *calleePtr = EmitFunctionDeclPointer(CGF.CGM, GD); - return CGCallee::forDirect(calleePtr, GD); + llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD); + if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice && + FD->hasAttr<CUDAGlobalAttr>()) + CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub( + cast<llvm::GlobalValue>(CalleePtr->stripPointerCasts())); + + return CGCallee::forDirect(CalleePtr, GD); } CGCallee CodeGenFunction::EmitCallee(const Expr *E) { @@ -4913,7 +5521,7 @@ CGCallee CodeGenFunction::EmitCallee(const Expr *E) { functionType = ptrType->getPointeeType(); } else { functionType = E->getType(); - calleePtr = EmitLValue(E).getPointer(*this); + calleePtr = EmitLValue(E, KnownNonNull).getPointer(*this); } assert(functionType->isFunctionType()); @@ -5013,8 +5621,8 @@ CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) { } Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { - return Builder.CreateElementBitCast(CGM.GetAddrOfMSGuidDecl(E->getGuidDecl()), - ConvertType(E->getType())); + return CGM.GetAddrOfMSGuidDecl(E->getGuidDecl()) + .withElementType(ConvertType(E->getType())); } LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) { @@ -5056,6 +5664,15 @@ llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface, return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar); } +llvm::Value * +CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface, + const ObjCIvarDecl *Ivar) { + llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar); + QualType PointerDiffType = getContext().getPointerDiffType(); + return Builder.CreateZExtOrTrunc(OffsetValue, + getTypes().ConvertType(PointerDiffType)); +} + LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy, llvm::Value *BaseValue, const ObjCIvarDecl *Ivar, @@ -5106,35 +5723,57 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee const Decl *TargetDecl = OrigCallee.getAbstractInfo().getCalleeDecl().getDecl(); + assert((!isa_and_present<FunctionDecl>(TargetDecl) || + !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) && + "trying to emit a call to an immediate function"); + CalleeType = getContext().getCanonicalType(CalleeType); auto PointeeType = cast<PointerType>(CalleeType)->getPointeeType(); CGCallee Callee = OrigCallee; - if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) && - (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { + if (SanOpts.has(SanitizerKind::Function) && + (!TargetDecl || !isa<FunctionDecl>(TargetDecl)) && + !isa<FunctionNoProtoType>(PointeeType)) { if (llvm::Constant *PrefixSig = CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) { SanitizerScope SanScope(this); - // Remove any (C++17) exception specifications, to allow calling e.g. a - // noexcept function through a non-noexcept pointer. - auto ProtoTy = - getContext().getFunctionTypeWithExceptionSpec(PointeeType, EST_None); - llvm::Constant *FTRTTIConst = - CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true); - llvm::Type *PrefixStructTyElems[] = {PrefixSig->getType(), Int32Ty}; + auto *TypeHash = getUBSanFunctionTypeHash(PointeeType); + + llvm::Type *PrefixSigType = PrefixSig->getType(); llvm::StructType *PrefixStructTy = llvm::StructType::get( - CGM.getLLVMContext(), PrefixStructTyElems, /*isPacked=*/true); + CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true); llvm::Value *CalleePtr = Callee.getFunctionPointer(); - llvm::Value *CalleePrefixStruct = Builder.CreateBitCast( - CalleePtr, llvm::PointerType::getUnqual(PrefixStructTy)); + // On 32-bit Arm, the low bit of a function pointer indicates whether + // it's using the Arm or Thumb instruction set. The actual first + // instruction lives at the same address either way, so we must clear + // that low bit before using the function address to find the prefix + // structure. + // + // This applies to both Arm and Thumb target triples, because + // either one could be used in an interworking context where it + // might be passed function pointers of both types. + llvm::Value *AlignedCalleePtr; + if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) { + llvm::Value *CalleeAddress = + Builder.CreatePtrToInt(CalleePtr, IntPtrTy); + llvm::Value *Mask = llvm::ConstantInt::get(IntPtrTy, ~1); + llvm::Value *AlignedCalleeAddress = + Builder.CreateAnd(CalleeAddress, Mask); + AlignedCalleePtr = + Builder.CreateIntToPtr(AlignedCalleeAddress, CalleePtr->getType()); + } else { + AlignedCalleePtr = CalleePtr; + } + + llvm::Value *CalleePrefixStruct = AlignedCalleePtr; llvm::Value *CalleeSigPtr = - Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0); + Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 0); llvm::Value *CalleeSig = - Builder.CreateAlignedLoad(CalleeSigPtr, getIntAlign()); + Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign()); llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig); llvm::BasicBlock *Cont = createBasicBlock("cont"); @@ -5142,19 +5781,17 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee Builder.CreateCondBr(CalleeSigMatch, TypeCheck, Cont); EmitBlock(TypeCheck); - llvm::Value *CalleeRTTIPtr = - Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1); - llvm::Value *CalleeRTTIEncoded = - Builder.CreateAlignedLoad(CalleeRTTIPtr, getPointerAlign()); - llvm::Value *CalleeRTTI = - DecodeAddrUsedInPrologue(CalleePtr, CalleeRTTIEncoded); - llvm::Value *CalleeRTTIMatch = - Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst); + llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad( + Int32Ty, + Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 1), + getPointerAlign()); + llvm::Value *CalleeTypeHashMatch = + Builder.CreateICmpEQ(CalleeTypeHash, TypeHash); llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()), EmitCheckTypeDescriptor(CalleeType)}; - EmitCheck(std::make_pair(CalleeRTTIMatch, SanitizerKind::Function), + EmitCheck(std::make_pair(CalleeTypeHashMatch, SanitizerKind::Function), SanitizerHandler::FunctionTypeMismatch, StaticData, - {CalleePtr, CalleeRTTI, FTRTTIConst}); + {CalleePtr}); Builder.CreateBr(Cont); EmitBlock(Cont); @@ -5179,9 +5816,8 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD); llvm::Value *CalleePtr = Callee.getFunctionPointer(); - llvm::Value *CastedCallee = Builder.CreateBitCast(CalleePtr, Int8PtrTy); llvm::Value *TypeTest = Builder.CreateCall( - CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedCallee, TypeId}); + CGM.getIntrinsic(llvm::Intrinsic::type_test), {CalleePtr, TypeId}); auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD); llvm::Constant *StaticData[] = { @@ -5191,18 +5827,17 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee }; if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) { EmitCfiSlowPathCheck(SanitizerKind::CFIICall, TypeTest, CrossDsoTypeId, - CastedCallee, StaticData); + CalleePtr, StaticData); } else { EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIICall), SanitizerHandler::CFICheckFail, StaticData, - {CastedCallee, llvm::UndefValue::get(IntPtrTy)}); + {CalleePtr, llvm::UndefValue::get(IntPtrTy)}); } } CallArgList Args; if (Chain) - Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), - CGM.getContext().VoidPtrTy); + Args.add(RValue::get(Chain), CGM.getContext().VoidPtrTy); // C++17 requires that we evaluate arguments to a call using assignment syntax // right-to-left, and that we evaluate arguments to certain other operators @@ -5211,6 +5846,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee // destruction order is not necessarily reverse construction order. // FIXME: Revisit this based on C++ committee response to unimplementability. EvaluationOrder Order = EvaluationOrder::Default; + bool StaticOperator = false; if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) { if (OCE->isAssignmentOp()) Order = EvaluationOrder::ForceRightToLeft; @@ -5228,10 +5864,22 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee break; } } + + if (const auto *MD = + dyn_cast_if_present<CXXMethodDecl>(OCE->getCalleeDecl()); + MD && MD->isStatic()) + StaticOperator = true; } - EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(), - E->getDirectCallee(), /*ParamsToSkip*/ 0, Order); + auto Arguments = E->arguments(); + if (StaticOperator) { + // If we're calling a static operator, we need to emit the object argument + // and ignore it. + EmitIgnoredExpr(E->getArg(0)); + Arguments = drop_begin(Arguments, 1); + } + EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), Arguments, + E->getDirectCallee(), /*ParamsToSkip=*/0, Order); const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall( Args, FnType, /*ChainCall=*/Chain); @@ -5266,16 +5914,31 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee Callee.setFunctionPointer(CalleePtr); } + // HIP function pointer contains kernel handle when it is used in triple + // chevron. The kernel stub needs to be loaded from kernel handle and used + // as callee. + if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice && + isa<CUDAKernelCallExpr>(E) && + (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { + llvm::Value *Handle = Callee.getFunctionPointer(); + auto *Stub = Builder.CreateLoad( + Address(Handle, Handle->getType(), CGM.getPointerAlign())); + Callee.setFunctionPointer(Stub); + } llvm::CallBase *CallOrInvoke = nullptr; RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke, - E->getExprLoc()); + E == MustTailCall, E->getExprLoc()); // Generate function declaration DISuprogram in order to be used // in debug info about call sites. if (CGDebugInfo *DI = getDebugInfo()) { - if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) - DI->EmitFuncDeclForCallSite(CallOrInvoke, QualType(FnType, 0), + if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) { + FunctionArgList Args; + QualType ResTy = BuildFunctionArgList(CalleeDecl, Args); + DI->EmitFuncDeclForCallSite(CallOrInvoke, + DI->getFunctionType(CalleeDecl, ResTy, Args), CalleeDecl); + } } return Call; @@ -5330,6 +5993,48 @@ void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) { cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node); } +void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) { + llvm::Type *EltTy = Val->getType()->getScalarType(); + if (!EltTy->isFloatTy()) + return; + + if ((getLangOpts().OpenCL && + !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) || + (getLangOpts().HIP && getLangOpts().CUDAIsDevice && + !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) { + // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 3ulp + // + // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt + // build option allows an application to specify that single precision + // floating-point divide (x/y and 1/x) and sqrt used in the program + // source are correctly rounded. + // + // TODO: CUDA has a prec-sqrt flag + SetFPAccuracy(Val, 3.0f); + } +} + +void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) { + llvm::Type *EltTy = Val->getType()->getScalarType(); + if (!EltTy->isFloatTy()) + return; + + if ((getLangOpts().OpenCL && + !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) || + (getLangOpts().HIP && getLangOpts().CUDAIsDevice && + !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) { + // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5ulp + // + // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt + // build option allows an application to specify that single precision + // floating-point divide (x/y and 1/x) and sqrt used in the program + // source are correctly rounded. + // + // TODO: CUDA has a prec-div flag + SetFPAccuracy(Val, 2.5f); + } +} + namespace { struct LValueOrRValue { LValue LV; @@ -5365,7 +6070,7 @@ static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF, // directly into the slot. typedef CodeGenFunction::OpaqueValueMappingData OVMA; OVMA opaqueData; - if (ov == resultExpr && ov->isRValue() && !forLValue && + if (ov == resultExpr && ov->isPRValue() && !forLValue && CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) { CGF.EmitAggExpr(ov->getSourceExpr(), slot); LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(), |