diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp')
| -rw-r--r-- | contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp | 715 |
1 files changed, 510 insertions, 205 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp index 47e41261e095..f22321f0e738 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprConstant.cpp @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// +#include "ABIInfoImpl.h" #include "CGCXXABI.h" #include "CGObjCRuntime.h" #include "CGRecordLayout.h" @@ -25,10 +26,12 @@ #include "clang/Basic/Builtins.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Sequence.h" +#include "llvm/Analysis/ConstantFolding.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" +#include <optional> using namespace clang; using namespace CodeGen; @@ -46,7 +49,7 @@ struct ConstantAggregateBuilderUtils { CharUnits getAlignment(const llvm::Constant *C) const { return CharUnits::fromQuantity( - CGM.getDataLayout().getABITypeAlignment(C->getType())); + CGM.getDataLayout().getABITypeAlign(C->getType())); } CharUnits getSize(llvm::Type *Ty) const { @@ -94,7 +97,7 @@ class ConstantAggregateBuilder : private ConstantAggregateBuilderUtils { bool NaturalLayout = true; bool split(size_t Index, CharUnits Hint); - Optional<size_t> splitAt(CharUnits Pos); + std::optional<size_t> splitAt(CharUnits Pos); static llvm::Constant *buildFrom(CodeGenModule &CGM, ArrayRef<llvm::Constant *> Elems, @@ -158,12 +161,12 @@ bool ConstantAggregateBuilder::add(llvm::Constant *C, CharUnits Offset, } // Uncommon case: constant overlaps what we've already created. - llvm::Optional<size_t> FirstElemToReplace = splitAt(Offset); + std::optional<size_t> FirstElemToReplace = splitAt(Offset); if (!FirstElemToReplace) return false; CharUnits CSize = getSize(C); - llvm::Optional<size_t> LastElemToReplace = splitAt(Offset + CSize); + std::optional<size_t> LastElemToReplace = splitAt(Offset + CSize); if (!LastElemToReplace) return false; @@ -222,10 +225,10 @@ bool ConstantAggregateBuilder::addBits(llvm::APInt Bits, uint64_t OffsetInBits, // Partial byte: update the existing integer if there is one. If we // can't split out a 1-CharUnit range to update, then we can't add // these bits and fail the entire constant emission. - llvm::Optional<size_t> FirstElemToUpdate = splitAt(OffsetInChars); + std::optional<size_t> FirstElemToUpdate = splitAt(OffsetInChars); if (!FirstElemToUpdate) return false; - llvm::Optional<size_t> LastElemToUpdate = + std::optional<size_t> LastElemToUpdate = splitAt(OffsetInChars + CharUnits::One()); if (!LastElemToUpdate) return false; @@ -283,8 +286,8 @@ bool ConstantAggregateBuilder::addBits(llvm::APInt Bits, uint64_t OffsetInBits, /// Returns a position within Elems and Offsets such that all elements /// before the returned index end before Pos and all elements at or after /// the returned index begin at or after Pos. Splits elements as necessary -/// to ensure this. Returns None if we find something we can't split. -Optional<size_t> ConstantAggregateBuilder::splitAt(CharUnits Pos) { +/// to ensure this. Returns std::nullopt if we find something we can't split. +std::optional<size_t> ConstantAggregateBuilder::splitAt(CharUnits Pos) { if (Pos >= Size) return Offsets.size(); @@ -305,7 +308,7 @@ Optional<size_t> ConstantAggregateBuilder::splitAt(CharUnits Pos) { // Try to decompose it into smaller constants. if (!split(LastAtOrBeforePosIndex, Pos)) - return None; + return std::nullopt; } } @@ -391,7 +394,7 @@ bool ConstantAggregateBuilder::split(size_t Index, CharUnits Hint) { static llvm::Constant * EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, - llvm::Type *CommonElementType, unsigned ArrayBound, + llvm::Type *CommonElementType, uint64_t ArrayBound, SmallVectorImpl<llvm::Constant *> &Elements, llvm::Constant *Filler); @@ -439,22 +442,33 @@ llvm::Constant *ConstantAggregateBuilder::buildFrom( // Can't emit as an array, carry on to emit as a struct. } + // The size of the constant we plan to generate. This is usually just + // the size of the initialized type, but in AllowOversized mode (i.e. + // flexible array init), it can be larger. CharUnits DesiredSize = Utils.getSize(DesiredTy); + if (Size > DesiredSize) { + assert(AllowOversized && "Elems are oversized"); + DesiredSize = Size; + } + + // The natural alignment of an unpacked LLVM struct with the given elements. CharUnits Align = CharUnits::One(); for (llvm::Constant *C : Elems) Align = std::max(Align, Utils.getAlignment(C)); + + // The natural size of an unpacked LLVM struct with the given elements. CharUnits AlignedSize = Size.alignTo(Align); bool Packed = false; ArrayRef<llvm::Constant*> UnpackedElems = Elems; llvm::SmallVector<llvm::Constant*, 32> UnpackedElemStorage; - if ((DesiredSize < AlignedSize && !AllowOversized) || - DesiredSize.alignTo(Align) != DesiredSize) { - // The natural layout would be the wrong size; force use of a packed layout. + if (DesiredSize < AlignedSize || DesiredSize.alignTo(Align) != DesiredSize) { + // The natural layout would be too big; force use of a packed layout. NaturalLayout = false; Packed = true; } else if (DesiredSize > AlignedSize) { - // The constant would be too small. Add padding to fix it. + // The natural layout would be too small. Add padding to fix it. (This + // is ignored if we choose a packed layout.) UnpackedElemStorage.assign(Elems.begin(), Elems.end()); UnpackedElemStorage.push_back(Utils.getPadding(DesiredSize - Size)); UnpackedElems = UnpackedElemStorage; @@ -482,7 +496,7 @@ llvm::Constant *ConstantAggregateBuilder::buildFrom( // If we're using the packed layout, pad it out to the desired size if // necessary. if (Packed) { - assert((SizeSoFar <= DesiredSize || AllowOversized) && + assert(SizeSoFar <= DesiredSize && "requested size is too small for contents"); if (SizeSoFar < DesiredSize) PackedElems.push_back(Utils.getPadding(DesiredSize - SizeSoFar)); @@ -506,12 +520,12 @@ void ConstantAggregateBuilder::condense(CharUnits Offset, llvm::Type *DesiredTy) { CharUnits Size = getSize(DesiredTy); - llvm::Optional<size_t> FirstElemToReplace = splitAt(Offset); + std::optional<size_t> FirstElemToReplace = splitAt(Offset); if (!FirstElemToReplace) return; size_t First = *FirstElemToReplace; - llvm::Optional<size_t> LastElemToReplace = splitAt(Offset + Size); + std::optional<size_t> LastElemToReplace = splitAt(Offset + Size); if (!LastElemToReplace) return; size_t Last = *LastElemToReplace; @@ -532,8 +546,8 @@ void ConstantAggregateBuilder::condense(CharUnits Offset, } llvm::Constant *Replacement = buildFrom( - CGM, makeArrayRef(Elems).slice(First, Length), - makeArrayRef(Offsets).slice(First, Length), Offset, getSize(DesiredTy), + CGM, ArrayRef(Elems).slice(First, Length), + ArrayRef(Offsets).slice(First, Length), Offset, getSize(DesiredTy), /*known to have natural layout=*/false, DesiredTy, false); replace(Elems, First, Last, {Replacement}); replace(Offsets, First, Last, {Offset}); @@ -551,12 +565,13 @@ class ConstStructBuilder { public: static llvm::Constant *BuildStruct(ConstantEmitter &Emitter, - InitListExpr *ILE, QualType StructTy); + const InitListExpr *ILE, + QualType StructTy); static llvm::Constant *BuildStruct(ConstantEmitter &Emitter, const APValue &Value, QualType ValTy); static bool UpdateStruct(ConstantEmitter &Emitter, ConstantAggregateBuilder &Const, CharUnits Offset, - InitListExpr *Updater); + const InitListExpr *Updater); private: ConstStructBuilder(ConstantEmitter &Emitter, @@ -571,9 +586,9 @@ private: bool AllowOverwrite = false); bool AppendBitField(const FieldDecl *Field, uint64_t FieldOffset, - llvm::ConstantInt *InitExpr, bool AllowOverwrite = false); + llvm::Constant *InitExpr, bool AllowOverwrite = false); - bool Build(InitListExpr *ILE, bool AllowOverwrite); + bool Build(const InitListExpr *ILE, bool AllowOverwrite); bool Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase, const CXXRecordDecl *VTableClass, CharUnits BaseOffset); llvm::Constant *Finalize(QualType Ty); @@ -595,9 +610,25 @@ bool ConstStructBuilder::AppendBytes(CharUnits FieldOffsetInChars, return Builder.add(InitCst, StartOffset + FieldOffsetInChars, AllowOverwrite); } -bool ConstStructBuilder::AppendBitField( - const FieldDecl *Field, uint64_t FieldOffset, llvm::ConstantInt *CI, - bool AllowOverwrite) { +bool ConstStructBuilder::AppendBitField(const FieldDecl *Field, + uint64_t FieldOffset, llvm::Constant *C, + bool AllowOverwrite) { + + llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(C); + if (!CI) { + // Constants for long _BitInt types are sometimes split into individual + // bytes. Try to fold these back into an integer constant. If that doesn't + // work out, then we are trying to initialize a bitfield with a non-trivial + // constant, this must require run-time code. + llvm::Type *LoadType = + CGM.getTypes().convertTypeForLoadStore(Field->getType(), C->getType()); + llvm::Constant *FoldedConstant = llvm::ConstantFoldLoadFromConst( + C, LoadType, llvm::APInt::getZero(32), CGM.getDataLayout()); + CI = dyn_cast_if_present<llvm::ConstantInt>(FoldedConstant); + if (!CI) + return false; + } + const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(Field->getParent()); const CGBitFieldInfo &Info = RL.getBitFieldInfo(Field); @@ -622,7 +653,7 @@ bool ConstStructBuilder::AppendBitField( static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, ConstantAggregateBuilder &Const, CharUnits Offset, QualType Type, - InitListExpr *Updater) { + const InitListExpr *Updater) { if (Type->isRecordType()) return ConstStructBuilder::UpdateStruct(Emitter, Const, Offset, Updater); @@ -634,7 +665,7 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, llvm::Type *ElemTy = Emitter.CGM.getTypes().ConvertTypeForMem(ElemType); llvm::Constant *FillC = nullptr; - if (Expr *Filler = Updater->getArrayFiller()) { + if (const Expr *Filler = Updater->getArrayFiller()) { if (!isa<NoInitExpr>(Filler)) { FillC = Emitter.tryEmitAbstractForMemory(Filler, ElemType); if (!FillC) @@ -643,9 +674,9 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, } unsigned NumElementsToUpdate = - FillC ? CAT->getSize().getZExtValue() : Updater->getNumInits(); + FillC ? CAT->getZExtSize() : Updater->getNumInits(); for (unsigned I = 0; I != NumElementsToUpdate; ++I, Offset += ElemSize) { - Expr *Init = nullptr; + const Expr *Init = nullptr; if (I < Updater->getNumInits()) Init = Updater->getInit(I); @@ -654,7 +685,7 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, return false; } else if (!Init || isa<NoInitExpr>(Init)) { continue; - } else if (InitListExpr *ChildILE = dyn_cast<InitListExpr>(Init)) { + } else if (const auto *ChildILE = dyn_cast<InitListExpr>(Init)) { if (!EmitDesignatedInitUpdater(Emitter, Const, Offset, ElemType, ChildILE)) return false; @@ -670,7 +701,7 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter, return true; } -bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) { +bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) { RecordDecl *RD = ILE->getType()->castAs<RecordType>()->getDecl(); const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD); @@ -692,18 +723,26 @@ bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) { !declaresSameEntity(ILE->getInitializedFieldInUnion(), Field)) continue; - // Don't emit anonymous bitfields or zero-sized fields. - if (Field->isUnnamedBitfield() || Field->isZeroSize(CGM.getContext())) + // Don't emit anonymous bitfields. + if (Field->isUnnamedBitField()) continue; // Get the initializer. A struct can include fields without initializers, // we just use explicit null values for them. - Expr *Init = nullptr; + const Expr *Init = nullptr; if (ElementNo < ILE->getNumInits()) Init = ILE->getInit(ElementNo++); - if (Init && isa<NoInitExpr>(Init)) + if (isa_and_nonnull<NoInitExpr>(Init)) continue; + // Zero-sized fields are not emitted, but their initializers may still + // prevent emission of this struct as a constant. + if (isEmptyFieldForLayout(CGM.getContext(), Field)) { + if (Init->HasSideEffects(CGM.getContext())) + return false; + continue; + } + // When emitting a DesignatedInitUpdateExpr, a nested InitListExpr // represents additional overwriting of our current constant value, and not // a new constant to emit independently. @@ -740,15 +779,9 @@ bool ConstStructBuilder::Build(InitListExpr *ILE, bool AllowOverwrite) { AllowOverwrite = true; } else { // Otherwise we have a bitfield. - if (auto *CI = dyn_cast<llvm::ConstantInt>(EltInit)) { - if (!AppendBitField(Field, Layout.getFieldOffset(FieldNo), CI, - AllowOverwrite)) - return false; - } else { - // We are trying to initialize a bitfield with a non-trivial constant, - // this must require run-time code. + if (!AppendBitField(Field, Layout.getFieldOffset(FieldNo), EltInit, + AllowOverwrite)) return false; - } } } @@ -779,8 +812,14 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, // Add a vtable pointer, if we need one and it hasn't already been added. if (Layout.hasOwnVFPtr()) { llvm::Constant *VTableAddressPoint = - CGM.getCXXABI().getVTableAddressPointForConstExpr( - BaseSubobject(CD, Offset), VTableClass); + CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset), + VTableClass); + if (auto Authentication = CGM.getVTablePointerAuthentication(CD)) { + VTableAddressPoint = Emitter.tryEmitConstantSignedPointer( + VTableAddressPoint, *Authentication); + if (!VTableAddressPoint) + return false; + } if (!AppendBytes(Offset, VTableAddressPoint)) return false; } @@ -819,7 +858,8 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, continue; // Don't emit anonymous bitfields or zero-sized fields. - if (Field->isUnnamedBitfield() || Field->isZeroSize(CGM.getContext())) + if (Field->isUnnamedBitField() || + isEmptyFieldForLayout(CGM.getContext(), *Field)) continue; // Emit the value of the initializer. @@ -842,7 +882,7 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, } else { // Otherwise we have a bitfield. if (!AppendBitField(*Field, Layout.getFieldOffset(FieldNo) + OffsetBits, - cast<llvm::ConstantInt>(EltInit), AllowOverwrite)) + EltInit, AllowOverwrite)) return false; } } @@ -851,13 +891,14 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, } llvm::Constant *ConstStructBuilder::Finalize(QualType Type) { + Type = Type.getNonReferenceType(); RecordDecl *RD = Type->castAs<RecordType>()->getDecl(); llvm::Type *ValTy = CGM.getTypes().ConvertType(Type); return Builder.build(ValTy, RD->hasFlexibleArrayMember()); } llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter, - InitListExpr *ILE, + const InitListExpr *ILE, QualType ValTy) { ConstantAggregateBuilder Const(Emitter.CGM); ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero()); @@ -884,7 +925,8 @@ llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter, bool ConstStructBuilder::UpdateStruct(ConstantEmitter &Emitter, ConstantAggregateBuilder &Const, - CharUnits Offset, InitListExpr *Updater) { + CharUnits Offset, + const InitListExpr *Updater) { return ConstStructBuilder(Emitter, Const, Offset) .Build(Updater, /*AllowOverwrite*/ true); } @@ -893,17 +935,16 @@ bool ConstStructBuilder::UpdateStruct(ConstantEmitter &Emitter, // ConstExprEmitter //===----------------------------------------------------------------------===// -static ConstantAddress tryEmitGlobalCompoundLiteral(CodeGenModule &CGM, - CodeGenFunction *CGF, - const CompoundLiteralExpr *E) { +static ConstantAddress +tryEmitGlobalCompoundLiteral(ConstantEmitter &emitter, + const CompoundLiteralExpr *E) { + CodeGenModule &CGM = emitter.CGM; CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType()); if (llvm::GlobalVariable *Addr = CGM.getAddrOfConstantCompoundLiteralIfEmitted(E)) - return ConstantAddress(Addr, Align); + return ConstantAddress(Addr, Addr->getValueType(), Align); LangAS addressSpace = E->getType().getAddressSpace(); - - ConstantEmitter emitter(CGM, CGF); llvm::Constant *C = emitter.tryEmitForInitializer(E->getInitializer(), addressSpace, E->getType()); if (!C) { @@ -912,25 +953,25 @@ static ConstantAddress tryEmitGlobalCompoundLiteral(CodeGenModule &CGM, return ConstantAddress::invalid(); } - auto GV = new llvm::GlobalVariable(CGM.getModule(), C->getType(), - CGM.isTypeConstant(E->getType(), true), - llvm::GlobalValue::InternalLinkage, - C, ".compoundliteral", nullptr, - llvm::GlobalVariable::NotThreadLocal, - CGM.getContext().getTargetAddressSpace(addressSpace)); + auto GV = new llvm::GlobalVariable( + CGM.getModule(), C->getType(), + E->getType().isConstantStorage(CGM.getContext(), true, false), + llvm::GlobalValue::InternalLinkage, C, ".compoundliteral", nullptr, + llvm::GlobalVariable::NotThreadLocal, + CGM.getContext().getTargetAddressSpace(addressSpace)); emitter.finalize(GV); GV->setAlignment(Align.getAsAlign()); CGM.setAddrOfConstantCompoundLiteral(E, GV); - return ConstantAddress(GV, Align); + return ConstantAddress(GV, GV->getValueType(), Align); } static llvm::Constant * EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, - llvm::Type *CommonElementType, unsigned ArrayBound, + llvm::Type *CommonElementType, uint64_t ArrayBound, SmallVectorImpl<llvm::Constant *> &Elements, llvm::Constant *Filler) { // Figure out how long the initial prefix of non-zero elements is. - unsigned NonzeroLength = ArrayBound; + uint64_t NonzeroLength = ArrayBound; if (Elements.size() < NonzeroLength && Filler->isNullValue()) NonzeroLength = Elements.size(); if (NonzeroLength == Elements.size()) { @@ -942,7 +983,7 @@ EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, return llvm::ConstantAggregateZero::get(DesiredType); // Add a zeroinitializer array filler if we have lots of trailing zeroes. - unsigned TrailingZeroes = ArrayBound - NonzeroLength; + uint64_t TrailingZeroes = ArrayBound - NonzeroLength; if (TrailingZeroes >= 8) { assert(Elements.size() >= NonzeroLength && "missing initializer for non-zero element"); @@ -952,7 +993,7 @@ EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, if (CommonElementType && NonzeroLength >= 8) { llvm::Constant *Initial = llvm::ConstantArray::get( llvm::ArrayType::get(CommonElementType, NonzeroLength), - makeArrayRef(Elements).take_front(NonzeroLength)); + ArrayRef(Elements).take_front(NonzeroLength)); Elements.resize(2); Elements[0] = Initial; } else { @@ -992,8 +1033,8 @@ EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType, // // Constant folding is currently missing support for a few features supported // here: CK_ToUnion, CK_ReinterpretMemberPointer, and DesignatedInitUpdateExpr. -class ConstExprEmitter : - public StmtVisitor<ConstExprEmitter, llvm::Constant*, QualType> { +class ConstExprEmitter + : public ConstStmtVisitor<ConstExprEmitter, llvm::Constant *, QualType> { CodeGenModule &CGM; ConstantEmitter &Emitter; llvm::LLVMContext &VMContext; @@ -1006,43 +1047,60 @@ public: // Visitor Methods //===--------------------------------------------------------------------===// - llvm::Constant *VisitStmt(Stmt *S, QualType T) { - return nullptr; - } + llvm::Constant *VisitStmt(const Stmt *S, QualType T) { return nullptr; } - llvm::Constant *VisitConstantExpr(ConstantExpr *CE, QualType T) { + llvm::Constant *VisitConstantExpr(const ConstantExpr *CE, QualType T) { if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE)) return Result; return Visit(CE->getSubExpr(), T); } - llvm::Constant *VisitParenExpr(ParenExpr *PE, QualType T) { + llvm::Constant *VisitParenExpr(const ParenExpr *PE, QualType T) { return Visit(PE->getSubExpr(), T); } llvm::Constant * - VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *PE, + VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *PE, QualType T) { return Visit(PE->getReplacement(), T); } - llvm::Constant *VisitGenericSelectionExpr(GenericSelectionExpr *GE, + llvm::Constant *VisitGenericSelectionExpr(const GenericSelectionExpr *GE, QualType T) { return Visit(GE->getResultExpr(), T); } - llvm::Constant *VisitChooseExpr(ChooseExpr *CE, QualType T) { + llvm::Constant *VisitChooseExpr(const ChooseExpr *CE, QualType T) { return Visit(CE->getChosenSubExpr(), T); } - llvm::Constant *VisitCompoundLiteralExpr(CompoundLiteralExpr *E, QualType T) { + llvm::Constant *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E, + QualType T) { return Visit(E->getInitializer(), T); } - llvm::Constant *VisitCastExpr(CastExpr *E, QualType destType) { + llvm::Constant *ProduceIntToIntCast(const Expr *E, QualType DestType) { + QualType FromType = E->getType(); + // See also HandleIntToIntCast in ExprConstant.cpp + if (FromType->isIntegerType()) + if (llvm::Constant *C = Visit(E, FromType)) + if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) { + unsigned SrcWidth = CGM.getContext().getIntWidth(FromType); + unsigned DstWidth = CGM.getContext().getIntWidth(DestType); + if (DstWidth == SrcWidth) + return CI; + llvm::APInt A = FromType->isSignedIntegerType() + ? CI->getValue().sextOrTrunc(DstWidth) + : CI->getValue().zextOrTrunc(DstWidth); + return llvm::ConstantInt::get(CGM.getLLVMContext(), A); + } + return nullptr; + } + + llvm::Constant *VisitCastExpr(const CastExpr *E, QualType destType) { if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E)) CGM.EmitExplicitCastExprType(ECE, Emitter.CGF); - Expr *subExpr = E->getSubExpr(); + const Expr *subExpr = E->getSubExpr(); switch (E->getCastKind()) { case CK_ToUnion: { @@ -1091,16 +1149,38 @@ public: destAS, destTy); } - case CK_LValueToRValue: + case CK_LValueToRValue: { + // We don't really support doing lvalue-to-rvalue conversions here; any + // interesting conversions should be done in Evaluate(). But as a + // special case, allow compound literals to support the gcc extension + // allowing "struct x {int x;} x = (struct x) {};". + if (const auto *E = + dyn_cast<CompoundLiteralExpr>(subExpr->IgnoreParens())) + return Visit(E->getInitializer(), destType); + return nullptr; + } + case CK_AtomicToNonAtomic: case CK_NonAtomicToAtomic: case CK_NoOp: case CK_ConstructorConversion: return Visit(subExpr, destType); + case CK_ArrayToPointerDecay: + if (const auto *S = dyn_cast<StringLiteral>(subExpr)) + return CGM.GetAddrOfConstantStringFromLiteral(S).getPointer(); + return nullptr; + case CK_NullToPointer: + if (Visit(subExpr, destType)) + return CGM.EmitNullConstant(destType); + return nullptr; + case CK_IntToOCLSampler: llvm_unreachable("global sampler variables are not generated"); + case CK_IntegralCast: + return ProduceIntToIntCast(subExpr, destType); + case CK_Dependent: llvm_unreachable("saw dependent cast!"); case CK_BuiltinFnToFnPtr: @@ -1135,7 +1215,6 @@ public: case CK_CPointerToObjCPointerCast: case CK_BlockPointerToObjCPointerCast: case CK_AnyPointerToBlockPointerCast: - case CK_ArrayToPointerDecay: case CK_FunctionToPointerDecay: case CK_BaseToDerived: case CK_DerivedToBase: @@ -1154,8 +1233,6 @@ public: case CK_IntegralComplexToFloatingComplex: case CK_PointerToIntegral: case CK_PointerToBoolean: - case CK_NullToPointer: - case CK_IntegralCast: case CK_BooleanToSignedIntegral: case CK_IntegralToPointer: case CK_IntegralToBoolean: @@ -1171,64 +1248,131 @@ public: case CK_IntegralToFixedPoint: case CK_ZeroToOCLOpaqueType: case CK_MatrixCast: + case CK_HLSLVectorTruncation: + case CK_HLSLArrayRValue: return nullptr; } llvm_unreachable("Invalid CastKind"); } - llvm::Constant *VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE, QualType T) { + llvm::Constant *VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *DIE, + QualType T) { // No need for a DefaultInitExprScope: we don't handle 'this' in a // constant expression. return Visit(DIE->getExpr(), T); } - llvm::Constant *VisitExprWithCleanups(ExprWithCleanups *E, QualType T) { + llvm::Constant *VisitExprWithCleanups(const ExprWithCleanups *E, QualType T) { return Visit(E->getSubExpr(), T); } - llvm::Constant *VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E, - QualType T) { - return Visit(E->getSubExpr(), T); + llvm::Constant *VisitIntegerLiteral(const IntegerLiteral *I, QualType T) { + return llvm::ConstantInt::get(CGM.getLLVMContext(), I->getValue()); + } + + static APValue withDestType(ASTContext &Ctx, const Expr *E, QualType SrcType, + QualType DestType, const llvm::APSInt &Value) { + if (!Ctx.hasSameType(SrcType, DestType)) { + if (DestType->isFloatingType()) { + llvm::APFloat Result = + llvm::APFloat(Ctx.getFloatTypeSemantics(DestType), 1); + llvm::RoundingMode RM = + E->getFPFeaturesInEffect(Ctx.getLangOpts()).getRoundingMode(); + if (RM == llvm::RoundingMode::Dynamic) + RM = llvm::RoundingMode::NearestTiesToEven; + Result.convertFromAPInt(Value, Value.isSigned(), RM); + return APValue(Result); + } + } + return APValue(Value); } - llvm::Constant *EmitArrayInitialization(InitListExpr *ILE, QualType T) { + llvm::Constant *EmitArrayInitialization(const InitListExpr *ILE, QualType T) { auto *CAT = CGM.getContext().getAsConstantArrayType(ILE->getType()); assert(CAT && "can't emit array init for non-constant-bound array"); - unsigned NumInitElements = ILE->getNumInits(); - unsigned NumElements = CAT->getSize().getZExtValue(); + uint64_t NumInitElements = ILE->getNumInits(); + const uint64_t NumElements = CAT->getZExtSize(); + for (const auto *Init : ILE->inits()) { + if (const auto *Embed = + dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) { + NumInitElements += Embed->getDataElementCount() - 1; + if (NumInitElements > NumElements) { + NumInitElements = NumElements; + break; + } + } + } // Initialising an array requires us to automatically // initialise any elements that have not been initialised explicitly - unsigned NumInitableElts = std::min(NumInitElements, NumElements); + uint64_t NumInitableElts = std::min<uint64_t>(NumInitElements, NumElements); QualType EltType = CAT->getElementType(); // Initialize remaining array elements. llvm::Constant *fillC = nullptr; - if (Expr *filler = ILE->getArrayFiller()) { + if (const Expr *filler = ILE->getArrayFiller()) { fillC = Emitter.tryEmitAbstractForMemory(filler, EltType); if (!fillC) return nullptr; } // Copy initializer elements. - SmallVector<llvm::Constant*, 16> Elts; + SmallVector<llvm::Constant *, 16> Elts; if (fillC && fillC->isNullValue()) Elts.reserve(NumInitableElts + 1); else Elts.reserve(NumElements); llvm::Type *CommonElementType = nullptr; - for (unsigned i = 0; i < NumInitableElts; ++i) { - Expr *Init = ILE->getInit(i); - llvm::Constant *C = Emitter.tryEmitPrivateForMemory(Init, EltType); + auto Emit = [&](const Expr *Init, unsigned ArrayIndex) { + llvm::Constant *C = nullptr; + C = Emitter.tryEmitPrivateForMemory(Init, EltType); if (!C) - return nullptr; - if (i == 0) + return false; + if (ArrayIndex == 0) CommonElementType = C->getType(); else if (C->getType() != CommonElementType) CommonElementType = nullptr; Elts.push_back(C); + return true; + }; + + unsigned ArrayIndex = 0; + QualType DestTy = CAT->getElementType(); + for (unsigned i = 0; i < ILE->getNumInits(); ++i) { + const Expr *Init = ILE->getInit(i); + if (auto *EmbedS = dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) { + StringLiteral *SL = EmbedS->getDataStringLiteral(); + llvm::APSInt Value(CGM.getContext().getTypeSize(DestTy), + DestTy->isUnsignedIntegerType()); + llvm::Constant *C; + for (unsigned I = EmbedS->getStartingElementPos(), + N = EmbedS->getDataElementCount(); + I != EmbedS->getStartingElementPos() + N; ++I) { + Value = SL->getCodeUnit(I); + if (DestTy->isIntegerType()) { + C = llvm::ConstantInt::get(CGM.getLLVMContext(), Value); + } else { + C = Emitter.tryEmitPrivateForMemory( + withDestType(CGM.getContext(), Init, EmbedS->getType(), DestTy, + Value), + EltType); + } + if (!C) + return nullptr; + Elts.push_back(C); + ArrayIndex++; + } + if ((ArrayIndex - EmbedS->getDataElementCount()) == 0) + CommonElementType = C->getType(); + else if (C->getType() != CommonElementType) + CommonElementType = nullptr; + } else { + if (!Emit(Init, ArrayIndex)) + return nullptr; + ArrayIndex++; + } } llvm::ArrayType *Desired = @@ -1237,16 +1381,17 @@ public: fillC); } - llvm::Constant *EmitRecordInitialization(InitListExpr *ILE, QualType T) { + llvm::Constant *EmitRecordInitialization(const InitListExpr *ILE, + QualType T) { return ConstStructBuilder::BuildStruct(Emitter, ILE, T); } - llvm::Constant *VisitImplicitValueInitExpr(ImplicitValueInitExpr* E, + llvm::Constant *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E, QualType T) { return CGM.EmitNullConstant(T); } - llvm::Constant *VisitInitListExpr(InitListExpr *ILE, QualType T) { + llvm::Constant *VisitInitListExpr(const InitListExpr *ILE, QualType T) { if (ILE->isTransparent()) return Visit(ILE->getInit(0), T); @@ -1259,8 +1404,9 @@ public: return nullptr; } - llvm::Constant *VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E, - QualType destType) { + llvm::Constant * + VisitDesignatedInitUpdateExpr(const DesignatedInitUpdateExpr *E, + QualType destType) { auto C = Visit(E->getBase(), destType); if (!C) return nullptr; @@ -1274,12 +1420,13 @@ public: llvm::Type *ValTy = CGM.getTypes().ConvertType(destType); bool HasFlexibleArray = false; - if (auto *RT = destType->getAs<RecordType>()) + if (const auto *RT = destType->getAs<RecordType>()) HasFlexibleArray = RT->getDecl()->hasFlexibleArrayMember(); return Const.build(ValTy, HasFlexibleArray); } - llvm::Constant *VisitCXXConstructExpr(CXXConstructExpr *E, QualType Ty) { + llvm::Constant *VisitCXXConstructExpr(const CXXConstructExpr *E, + QualType Ty) { if (!E->getConstructor()->isTrivial()) return nullptr; @@ -1289,32 +1436,38 @@ public: assert(E->getConstructor()->isCopyOrMoveConstructor() && "trivial ctor has argument but isn't a copy/move ctor"); - Expr *Arg = E->getArg(0); + const Expr *Arg = E->getArg(0); assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) && "argument to copy ctor is of wrong type"); - return Visit(Arg, Ty); + // Look through the temporary; it's just converting the value to an + // lvalue to pass it to the constructor. + if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Arg)) + return Visit(MTE->getSubExpr(), Ty); + // Don't try to support arbitrary lvalue-to-rvalue conversions for now. + return nullptr; } return CGM.EmitNullConstant(Ty); } - llvm::Constant *VisitStringLiteral(StringLiteral *E, QualType T) { + llvm::Constant *VisitStringLiteral(const StringLiteral *E, QualType T) { // This is a string literal initializing an array in an initializer. return CGM.GetConstantArrayFromStringLiteral(E); } - llvm::Constant *VisitObjCEncodeExpr(ObjCEncodeExpr *E, QualType T) { + llvm::Constant *VisitObjCEncodeExpr(const ObjCEncodeExpr *E, QualType T) { // This must be an @encode initializing an array in a static initializer. // Don't emit it as the address of the string, emit the string data itself // as an inline array. std::string Str; CGM.getContext().getObjCEncodingForType(E->getEncodedType(), Str); const ConstantArrayType *CAT = CGM.getContext().getAsConstantArrayType(T); + assert(CAT && "String data not of constant array type!"); // Resize the string to the right size, adding zeros at the end, or // truncating as needed. - Str.resize(CAT->getSize().getZExtValue(), '\0'); + Str.resize(CAT->getZExtSize(), '\0'); return llvm::ConstantDataArray::getString(VMContext, Str, false); } @@ -1322,6 +1475,17 @@ public: return Visit(E->getSubExpr(), T); } + llvm::Constant *VisitUnaryMinus(const UnaryOperator *U, QualType T) { + if (llvm::Constant *C = Visit(U->getSubExpr(), T)) + if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) + return llvm::ConstantInt::get(CGM.getLLVMContext(), -CI->getValue()); + return nullptr; + } + + llvm::Constant *VisitPackIndexingExpr(const PackIndexingExpr *E, QualType T) { + return Visit(E->getSelectedExpr(), T); + } + // Utility methods llvm::Type *ConvertType(QualType T) { return CGM.getTypes().ConvertType(T); @@ -1366,15 +1530,12 @@ ConstantEmitter::tryEmitAbstract(const APValue &value, QualType destType) { llvm::Constant *ConstantEmitter::tryEmitConstantExpr(const ConstantExpr *CE) { if (!CE->hasAPValueResult()) return nullptr; - const Expr *Inner = CE->getSubExpr()->IgnoreImplicit(); - QualType RetType; - if (auto *Call = dyn_cast<CallExpr>(Inner)) - RetType = Call->getCallReturnType(CGF->getContext()); - else if (auto *Ctor = dyn_cast<CXXConstructExpr>(Inner)) - RetType = Ctor->getType(); - llvm::Constant *Res = - emitAbstract(CE->getBeginLoc(), CE->getAPValueResult(), RetType); - return Res; + + QualType RetType = CE->getType(); + if (CE->isGLValue()) + RetType = CGM.getContext().getLValueReferenceType(RetType); + + return emitAbstract(CE->getBeginLoc(), CE->getAPValueResult(), RetType); } llvm::Constant * @@ -1392,9 +1553,11 @@ ConstantEmitter::emitAbstract(const Expr *E, QualType destType) { llvm::Constant * ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value, - QualType destType) { + QualType destType, + bool EnablePtrAuthFunctionTypeDiscrimination) { auto state = pushAbstract(); - auto C = tryEmitPrivate(value, destType); + auto C = + tryEmitPrivate(value, destType, EnablePtrAuthFunctionTypeDiscrimination); C = validateAndPopAbstract(C, state); if (!C) { CGM.Error(loc, @@ -1504,7 +1667,7 @@ namespace { // messing around with llvm::Constant structures, which never itself // does anything that should be visible in compiler output. for (auto &entry : Locations) { - assert(entry.first->getParent() == nullptr && "not a placeholder!"); + assert(entry.first->getName() == "" && "not a placeholder!"); entry.first->replaceAllUsesWith(entry.second); entry.first->eraseFromParent(); } @@ -1544,7 +1707,7 @@ namespace { } void setLocation(llvm::GlobalVariable *placeholder) { - assert(Locations.find(placeholder) == Locations.end() && + assert(!Locations.contains(placeholder) && "already found location for placeholder!"); // Lazily fill in IndexValues with the values from Indices. @@ -1567,13 +1730,8 @@ namespace { IndexValues[i] = llvm::ConstantInt::get(CGM.Int32Ty, Indices[i]); } - // Form a GEP and then bitcast to the placeholder type so that the - // replacement will succeed. - llvm::Constant *location = - llvm::ConstantExpr::getInBoundsGetElementPtr(BaseValueTy, - Base, IndexValues); - location = llvm::ConstantExpr::getBitCast(location, - placeholder->getType()); + llvm::Constant *location = llvm::ConstantExpr::getInBoundsGetElementPtr( + BaseValueTy, Base, IndexValues); Locations.insert({placeholder, location}); } @@ -1623,33 +1781,25 @@ llvm::Constant *ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &D) { if (CD->isTrivial() && CD->isDefaultConstructor()) return CGM.EmitNullConstant(D.getType()); } - InConstantContext = true; } + InConstantContext = D.hasConstantInitialization(); QualType destType = D.getType(); + const Expr *E = D.getInit(); + assert(E && "No initializer to emit"); + + if (!destType->isReferenceType()) { + QualType nonMemoryDestType = getNonMemoryType(CGM, destType); + if (llvm::Constant *C = ConstExprEmitter(*this).Visit(E, nonMemoryDestType)) + return emitForMemory(C, destType); + } // Try to emit the initializer. Note that this can allow some things that // are not allowed by tryEmitPrivateForMemory alone. - if (auto value = D.evaluateValue()) { + if (APValue *value = D.evaluateValue()) return tryEmitPrivateForMemory(*value, destType); - } - - // FIXME: Implement C++11 [basic.start.init]p2: if the initializer of a - // reference is a constant expression, and the reference binds to a temporary, - // then constant initialization is performed. ConstExprEmitter will - // incorrectly emit a prvalue constant in this case, and the calling code - // interprets that as the (pointer) value of the reference, rather than the - // desired value of the referee. - if (destType->isReferenceType()) - return nullptr; - const Expr *E = D.getInit(); - assert(E && "No initializer to emit"); - - auto nonMemoryDestType = getNonMemoryType(CGM, destType); - auto C = - ConstExprEmitter(*this).Visit(const_cast<Expr*>(E), nonMemoryDestType); - return (C ? emitForMemory(C, destType) : nullptr); + return nullptr; } llvm::Constant * @@ -1681,6 +1831,43 @@ llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const APValue &value, return (C ? emitForMemory(C, destType) : nullptr); } +/// Try to emit a constant signed pointer, given a raw pointer and the +/// destination ptrauth qualifier. +/// +/// This can fail if the qualifier needs address discrimination and the +/// emitter is in an abstract mode. +llvm::Constant * +ConstantEmitter::tryEmitConstantSignedPointer(llvm::Constant *UnsignedPointer, + PointerAuthQualifier Schema) { + assert(Schema && "applying trivial ptrauth schema"); + + if (Schema.hasKeyNone()) + return UnsignedPointer; + + unsigned Key = Schema.getKey(); + + // Create an address placeholder if we're using address discrimination. + llvm::GlobalValue *StorageAddress = nullptr; + if (Schema.isAddressDiscriminated()) { + // We can't do this if the emitter is in an abstract state. + if (isAbstract()) + return nullptr; + + StorageAddress = getCurrentAddrPrivate(); + } + + llvm::ConstantInt *Discriminator = + llvm::ConstantInt::get(CGM.IntPtrTy, Schema.getExtraDiscriminator()); + + llvm::Constant *SignedPointer = CGM.getConstantSignedPointer( + UnsignedPointer, Key, StorageAddress, Discriminator); + + if (Schema.isAddressDiscriminated()) + registerCurrentAddrPrivate(SignedPointer, StorageAddress); + + return SignedPointer; +} + llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM, llvm::Constant *C, QualType destType) { @@ -1704,9 +1891,33 @@ llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM, } // Zero-extend bool. - if (C->getType()->isIntegerTy(1)) { + if (C->getType()->isIntegerTy(1) && !destType->isBitIntType()) { llvm::Type *boolTy = CGM.getTypes().ConvertTypeForMem(destType); - return llvm::ConstantExpr::getZExt(C, boolTy); + llvm::Constant *Res = llvm::ConstantFoldCastOperand( + llvm::Instruction::ZExt, C, boolTy, CGM.getDataLayout()); + assert(Res && "Constant folding must succeed"); + return Res; + } + + if (destType->isBitIntType()) { + ConstantAggregateBuilder Builder(CGM); + llvm::Type *LoadStoreTy = CGM.getTypes().convertTypeForLoadStore(destType); + // ptrtoint/inttoptr should not involve _BitInt in constant expressions, so + // casting to ConstantInt is safe here. + auto *CI = cast<llvm::ConstantInt>(C); + llvm::Constant *Res = llvm::ConstantFoldCastOperand( + destType->isSignedIntegerOrEnumerationType() ? llvm::Instruction::SExt + : llvm::Instruction::ZExt, + CI, LoadStoreTy, CGM.getDataLayout()); + if (CGM.getTypes().typeRequiresSplitIntoByteArray(destType, C->getType())) { + // Long _BitInt has array of bytes as in-memory type. + // So, split constant into individual bytes. + llvm::Type *DesiredTy = CGM.getTypes().ConvertTypeForMem(destType); + llvm::APInt Value = cast<llvm::ConstantInt>(Res)->getValue(); + Builder.addBits(Value, /*OffsetInBits=*/0, /*AllowOverwrite=*/false); + return Builder.build(DesiredTy, /*AllowOversized*/ false); + } + return Res; } return C; @@ -1714,6 +1925,12 @@ llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM, llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E, QualType destType) { + assert(!destType->isVoidType() && "can't emit a void constant"); + + if (!destType->isReferenceType()) + if (llvm::Constant *C = ConstExprEmitter(*this).Visit(E, destType)) + return C; + Expr::EvalResult Result; bool Success = false; @@ -1723,13 +1940,10 @@ llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E, else Success = E->EvaluateAsRValue(Result, CGM.getContext(), InConstantContext); - llvm::Constant *C; if (Success && !Result.HasSideEffects) - C = tryEmitPrivate(Result.Val, destType); - else - C = ConstExprEmitter(*this).Visit(const_cast<Expr*>(E), destType); + return tryEmitPrivate(Result.Val, destType); - return C; + return nullptr; } llvm::Constant *CodeGenModule::getNullPointer(llvm::PointerType *T, QualType QT) { @@ -1758,14 +1972,18 @@ class ConstantLValueEmitter : public ConstStmtVisitor<ConstantLValueEmitter, ConstantEmitter &Emitter; const APValue &Value; QualType DestType; + bool EnablePtrAuthFunctionTypeDiscrimination; // Befriend StmtVisitorBase so that we don't have to expose Visit*. friend StmtVisitorBase; public: ConstantLValueEmitter(ConstantEmitter &emitter, const APValue &value, - QualType destType) - : CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType) {} + QualType destType, + bool EnablePtrAuthFunctionTypeDiscrimination = true) + : CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType), + EnablePtrAuthFunctionTypeDiscrimination( + EnablePtrAuthFunctionTypeDiscrimination) {} llvm::Constant *tryEmit(); @@ -1788,6 +2006,12 @@ private: ConstantLValue VisitMaterializeTemporaryExpr( const MaterializeTemporaryExpr *E); + ConstantLValue emitPointerAuthSignConstant(const CallExpr *E); + llvm::Constant *emitPointerAuthPointer(const Expr *E); + unsigned emitPointerAuthKey(const Expr *E); + std::pair<llvm::Constant *, llvm::ConstantInt *> + emitPointerAuthDiscriminator(const Expr *E); + bool hasNonZeroOffset() const { return !Value.getLValueOffset().isZero(); } @@ -1803,13 +2027,7 @@ private: if (!hasNonZeroOffset()) return C; - llvm::Type *origPtrTy = C->getType(); - unsigned AS = origPtrTy->getPointerAddressSpace(); - llvm::Type *charPtrTy = CGM.Int8Ty->getPointerTo(AS); - C = llvm::ConstantExpr::getBitCast(C, charPtrTy); - C = llvm::ConstantExpr::getGetElementPtr(CGM.Int8Ty, C, getOffset()); - C = llvm::ConstantExpr::getPointerCast(C, origPtrTy); - return C; + return llvm::ConstantExpr::getGetElementPtr(CGM.Int8Ty, C, getOffset()); } }; @@ -1870,8 +2088,9 @@ ConstantLValueEmitter::tryEmitAbsolute(llvm::Type *destTy) { // FIXME: signedness depends on the original integer type. auto intptrTy = CGM.getDataLayout().getIntPtrType(destPtrTy); llvm::Constant *C; - C = llvm::ConstantExpr::getIntegerCast(getOffset(), intptrTy, - /*isSigned*/ false); + C = llvm::ConstantFoldIntegerCast(getOffset(), intptrTy, /*isSigned*/ false, + CGM.getDataLayout()); + assert(C && "Must have folded, as Offset is a ConstantInt"); C = llvm::ConstantExpr::getIntToPtr(C, destPtrTy); return C; } @@ -1887,10 +2106,30 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) { if (D->hasAttr<WeakRefAttr>()) return CGM.GetWeakRefReference(D).getPointer(); - if (auto FD = dyn_cast<FunctionDecl>(D)) - return CGM.GetAddrOfFunction(FD); + auto PtrAuthSign = [&](llvm::Constant *C) { + CGPointerAuthInfo AuthInfo; + + if (EnablePtrAuthFunctionTypeDiscrimination) + AuthInfo = CGM.getFunctionPointerAuthInfo(DestType); - if (auto VD = dyn_cast<VarDecl>(D)) { + if (AuthInfo) { + if (hasNonZeroOffset()) + return ConstantLValue(nullptr); + + C = applyOffset(C); + C = CGM.getConstantSignedPointer( + C, AuthInfo.getKey(), nullptr, + cast_or_null<llvm::ConstantInt>(AuthInfo.getDiscriminator())); + return ConstantLValue(C, /*applied offset*/ true); + } + + return ConstantLValue(C); + }; + + if (const auto *FD = dyn_cast<FunctionDecl>(D)) + return PtrAuthSign(CGM.getRawFunctionPointer(FD)); + + if (const auto *VD = dyn_cast<VarDecl>(D)) { // We can never refer to a variable with local storage. if (!VD->hasLocalStorage()) { if (VD->isFileVarDecl() || VD->hasExternalStorage()) @@ -1898,30 +2137,26 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) { if (VD->isLocalVarDecl()) { return CGM.getOrCreateStaticVarDecl( - *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false)); + *VD, CGM.getLLVMLinkageVarDefinition(VD)); } } } - if (auto *GD = dyn_cast<MSGuidDecl>(D)) + if (const auto *GD = dyn_cast<MSGuidDecl>(D)) return CGM.GetAddrOfMSGuidDecl(GD); - if (auto *TPO = dyn_cast<TemplateParamObjectDecl>(D)) + if (const auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(D)) + return CGM.GetAddrOfUnnamedGlobalConstantDecl(GCD); + + if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(D)) return CGM.GetAddrOfTemplateParamObject(TPO); return nullptr; } // Handle typeid(T). - if (TypeInfoLValue TI = base.dyn_cast<TypeInfoLValue>()) { - llvm::Type *StdTypeInfoPtrTy = - CGM.getTypes().ConvertType(base.getTypeInfoType())->getPointerTo(); - llvm::Constant *TypeInfo = - CGM.GetAddrOfRTTIDescriptor(QualType(TI.getType(), 0)); - if (TypeInfo->getType() != StdTypeInfoPtrTy) - TypeInfo = llvm::ConstantExpr::getBitCast(TypeInfo, StdTypeInfoPtrTy); - return TypeInfo; - } + if (TypeInfoLValue TI = base.dyn_cast<TypeInfoLValue>()) + return CGM.GetAddrOfRTTIDescriptor(QualType(TI.getType(), 0)); // Otherwise, it must be an expression. return Visit(base.get<const Expr*>()); @@ -1936,7 +2171,9 @@ ConstantLValueEmitter::VisitConstantExpr(const ConstantExpr *E) { ConstantLValue ConstantLValueEmitter::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) { - return tryEmitGlobalCompoundLiteral(CGM, Emitter.CGF, E); + ConstantEmitter CompoundLiteralEmitter(CGM, Emitter.CGF); + CompoundLiteralEmitter.setInConstantContext(Emitter.isInConstantContext()); + return tryEmitGlobalCompoundLiteral(CompoundLiteralEmitter, E); } ConstantLValue @@ -1953,7 +2190,7 @@ static ConstantLValue emitConstantObjCStringLiteral(const StringLiteral *S, QualType T, CodeGenModule &CGM) { auto C = CGM.getObjCRuntime().GenerateConstantString(S); - return C.getElementBitCast(CGM.getTypes().ConvertTypeForMem(T)); + return C.withElementType(CGM.getTypes().ConvertTypeForMem(T)); } ConstantLValue @@ -1965,7 +2202,7 @@ ConstantLValue ConstantLValueEmitter::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) { assert(E->isExpressibleAsConstantInitializer() && "this boxed expression can't be emitted as a compile-time constant"); - auto *SL = cast<StringLiteral>(E->getSubExpr()->IgnoreParenCasts()); + const auto *SL = cast<StringLiteral>(E->getSubExpr()->IgnoreParenCasts()); return emitConstantObjCStringLiteral(SL, E->getType(), CGM); } @@ -1978,28 +2215,84 @@ ConstantLValue ConstantLValueEmitter::VisitAddrLabelExpr(const AddrLabelExpr *E) { assert(Emitter.CGF && "Invalid address of label expression outside function"); llvm::Constant *Ptr = Emitter.CGF->GetAddrOfLabel(E->getLabel()); - Ptr = llvm::ConstantExpr::getBitCast(Ptr, - CGM.getTypes().ConvertType(E->getType())); return Ptr; } ConstantLValue ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) { unsigned builtin = E->getBuiltinCallee(); + if (builtin == Builtin::BI__builtin_function_start) + return CGM.GetFunctionStart( + E->getArg(0)->getAsBuiltinConstantDeclRef(CGM.getContext())); + + if (builtin == Builtin::BI__builtin_ptrauth_sign_constant) + return emitPointerAuthSignConstant(E); + if (builtin != Builtin::BI__builtin___CFStringMakeConstantString && builtin != Builtin::BI__builtin___NSStringMakeConstantString) return nullptr; - auto literal = cast<StringLiteral>(E->getArg(0)->IgnoreParenCasts()); + const auto *Literal = cast<StringLiteral>(E->getArg(0)->IgnoreParenCasts()); if (builtin == Builtin::BI__builtin___NSStringMakeConstantString) { - return CGM.getObjCRuntime().GenerateConstantString(literal); + return CGM.getObjCRuntime().GenerateConstantString(Literal); } else { // FIXME: need to deal with UCN conversion issues. - return CGM.GetAddrOfConstantCFString(literal); + return CGM.GetAddrOfConstantCFString(Literal); } } ConstantLValue +ConstantLValueEmitter::emitPointerAuthSignConstant(const CallExpr *E) { + llvm::Constant *UnsignedPointer = emitPointerAuthPointer(E->getArg(0)); + unsigned Key = emitPointerAuthKey(E->getArg(1)); + auto [StorageAddress, OtherDiscriminator] = + emitPointerAuthDiscriminator(E->getArg(2)); + + llvm::Constant *SignedPointer = CGM.getConstantSignedPointer( + UnsignedPointer, Key, StorageAddress, OtherDiscriminator); + return SignedPointer; +} + +llvm::Constant *ConstantLValueEmitter::emitPointerAuthPointer(const Expr *E) { + Expr::EvalResult Result; + bool Succeeded = E->EvaluateAsRValue(Result, CGM.getContext()); + assert(Succeeded); + (void)Succeeded; + + // The assertions here are all checked by Sema. + assert(Result.Val.isLValue()); + if (isa<FunctionDecl>(Result.Val.getLValueBase().get<const ValueDecl *>())) + assert(Result.Val.getLValueOffset().isZero()); + return ConstantEmitter(CGM, Emitter.CGF) + .emitAbstract(E->getExprLoc(), Result.Val, E->getType(), false); +} + +unsigned ConstantLValueEmitter::emitPointerAuthKey(const Expr *E) { + return E->EvaluateKnownConstInt(CGM.getContext()).getZExtValue(); +} + +std::pair<llvm::Constant *, llvm::ConstantInt *> +ConstantLValueEmitter::emitPointerAuthDiscriminator(const Expr *E) { + E = E->IgnoreParens(); + + if (const auto *Call = dyn_cast<CallExpr>(E)) { + if (Call->getBuiltinCallee() == + Builtin::BI__builtin_ptrauth_blend_discriminator) { + llvm::Constant *Pointer = ConstantEmitter(CGM).emitAbstract( + Call->getArg(0), Call->getArg(0)->getType()); + auto *Extra = cast<llvm::ConstantInt>(ConstantEmitter(CGM).emitAbstract( + Call->getArg(1), Call->getArg(1)->getType())); + return {Pointer, Extra}; + } + } + + llvm::Constant *Result = ConstantEmitter(CGM).emitAbstract(E, E->getType()); + if (Result->getType()->isPointerTy()) + return {Result, nullptr}; + return {nullptr, cast<llvm::ConstantInt>(Result)}; +} + +ConstantLValue ConstantLValueEmitter::VisitBlockExpr(const BlockExpr *E) { StringRef functionName; if (auto CGF = Emitter.CGF) @@ -2024,22 +2317,22 @@ ConstantLValue ConstantLValueEmitter::VisitMaterializeTemporaryExpr( const MaterializeTemporaryExpr *E) { assert(E->getStorageDuration() == SD_Static); - SmallVector<const Expr *, 2> CommaLHSs; - SmallVector<SubobjectAdjustment, 2> Adjustments; - const Expr *Inner = - E->getSubExpr()->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments); + const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments(); return CGM.GetAddrOfGlobalTemporary(E, Inner); } -llvm::Constant *ConstantEmitter::tryEmitPrivate(const APValue &Value, - QualType DestType) { +llvm::Constant * +ConstantEmitter::tryEmitPrivate(const APValue &Value, QualType DestType, + bool EnablePtrAuthFunctionTypeDiscrimination) { switch (Value.getKind()) { case APValue::None: case APValue::Indeterminate: // Out-of-lifetime and indeterminate values can be modeled as 'undef'. return llvm::UndefValue::get(CGM.getTypes().ConvertType(DestType)); case APValue::LValue: - return ConstantLValueEmitter(*this, Value, DestType).tryEmit(); + return ConstantLValueEmitter(*this, Value, DestType, + EnablePtrAuthFunctionTypeDiscrimination) + .tryEmit(); case APValue::Int: return llvm::ConstantInt::get(CGM.getLLVMContext(), Value.getInt()); case APValue::FixedPoint: @@ -2091,6 +2384,9 @@ llvm::Constant *ConstantEmitter::tryEmitPrivate(const APValue &Value, Inits[I] = llvm::ConstantInt::get(CGM.getLLVMContext(), Elt.getInt()); else if (Elt.isFloat()) Inits[I] = llvm::ConstantFP::get(CGM.getLLVMContext(), Elt.getFloat()); + else if (Elt.isIndeterminate()) + Inits[I] = llvm::UndefValue::get(CGM.getTypes().ConvertType( + DestType->castAs<VectorType>()->getElementType())); else llvm_unreachable("unsupported vector element type"); } @@ -2153,6 +2449,11 @@ llvm::Constant *ConstantEmitter::tryEmitPrivate(const APValue &Value, llvm::ArrayType *Desired = cast<llvm::ArrayType>(CGM.getTypes().ConvertType(DestType)); + + // Fix the type of incomplete arrays if the initializer isn't empty. + if (DestType->isIncompleteArrayType() && !Elts.empty()) + Desired = llvm::ArrayType::get(Desired->getElementType(), Elts.size()); + return EmitArrayConstant(CGM, Desired, CommonElementType, NumElements, Elts, Filler); } @@ -2177,7 +2478,8 @@ void CodeGenModule::setAddrOfConstantCompoundLiteral( ConstantAddress CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) { assert(E->isFileScope() && "not a file-scope compound literal expr"); - return tryEmitGlobalCompoundLiteral(*this, nullptr, E); + ConstantEmitter emitter(*this); + return tryEmitGlobalCompoundLiteral(emitter, E); } llvm::Constant * @@ -2225,8 +2527,10 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); // Ignore empty bases. - if (base->isEmpty() || - CGM.getContext().getASTRecordLayout(base).getNonVirtualSize() + if (isEmptyRecordForLayout(CGM.getContext(), I.getType()) || + CGM.getContext() + .getASTRecordLayout(base) + .getNonVirtualSize() .isZero()) continue; @@ -2240,7 +2544,8 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, for (const auto *Field : record->fields()) { // Fill in non-bitfields. (Bitfields always use a zero pattern, which we // will fill in later.) - if (!Field->isBitField() && !Field->isZeroSize(CGM.getContext())) { + if (!Field->isBitField() && + !isEmptyFieldForLayout(CGM.getContext(), Field)) { unsigned fieldIndex = layout.getLLVMFieldNo(Field); elements[fieldIndex] = CGM.EmitNullConstant(Field->getType()); } @@ -2262,7 +2567,7 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM, cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); // Ignore empty bases. - if (base->isEmpty()) + if (isEmptyRecordForLayout(CGM.getContext(), I.getType())) continue; unsigned fieldIndex = layout.getVirtualBaseIndex(base); @@ -2319,7 +2624,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { llvm::Constant *Element = ConstantEmitter::emitNullForMemory(*this, ElementTy); - unsigned NumElements = CAT->getSize().getZExtValue(); + unsigned NumElements = CAT->getZExtSize(); SmallVector<llvm::Constant *, 8> Array(NumElements, Element); return llvm::ConstantArray::get(ATy, Array); } |