diff options
Diffstat (limited to 'lib/CodeGen/TargetInfo.cpp')
| -rw-r--r-- | lib/CodeGen/TargetInfo.cpp | 1110 |
1 files changed, 720 insertions, 390 deletions
diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp index f75e59d6f2d8..6ad349515769 100644 --- a/lib/CodeGen/TargetInfo.cpp +++ b/lib/CodeGen/TargetInfo.cpp @@ -15,15 +15,16 @@ #include "TargetInfo.h" #include "ABIInfo.h" #include "CGCXXABI.h" +#include "CGValue.h" #include "CodeGenFunction.h" #include "clang/AST/RecordLayout.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Type.h" #include "llvm/Support/raw_ostream.h" - #include <algorithm> // std::sort using namespace clang; @@ -64,6 +65,19 @@ static CGCXXABI::RecordArgABI getRecordArgABI(QualType T, return getRecordArgABI(RT, CXXABI); } +/// Pass transparent unions as if they were the type of the first element. Sema +/// should ensure that all elements of the union have the same "machine type". +static QualType useFirstFieldIfTransparentUnion(QualType Ty) { + if (const RecordType *UT = Ty->getAsUnionType()) { + const RecordDecl *UD = UT->getDecl(); + if (UD->hasAttr<TransparentUnionAttr>()) { + assert(!UD->field_empty() && "sema created an empty transparent union"); + return UD->field_begin()->getType(); + } + } + return Ty; +} + CGCXXABI &ABIInfo::getCXXABI() const { return CGT.getCXXABI(); } @@ -84,6 +98,15 @@ const TargetInfo &ABIInfo::getTarget() const { return CGT.getTarget(); } +bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { + return false; +} + +bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, + uint64_t Members) const { + return false; +} + void ABIArgInfo::dump() const { raw_ostream &OS = llvm::errs(); OS << "(ABIArgInfo Kind="; @@ -498,18 +521,39 @@ static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF, return Ty; } +/// Returns true if this type can be passed in SSE registers with the +/// X86_VectorCall calling convention. Shared between x86_32 and x86_64. +static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) { + if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { + if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) + return true; + } else if (const VectorType *VT = Ty->getAs<VectorType>()) { + // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX + // registers specially. + unsigned VecSize = Context.getTypeSize(VT); + if (VecSize == 128 || VecSize == 256 || VecSize == 512) + return true; + } + return false; +} + +/// Returns true if this aggregate is small enough to be passed in SSE registers +/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64. +static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) { + return NumMembers <= 4; +} + //===----------------------------------------------------------------------===// // X86-32 ABI Implementation //===----------------------------------------------------------------------===// /// \brief Similar to llvm::CCState, but for Clang. struct CCState { - CCState(unsigned CC) : CC(CC), FreeRegs(0) {} + CCState(unsigned CC) : CC(CC), FreeRegs(0), FreeSSERegs(0) {} unsigned CC; unsigned FreeRegs; - unsigned StackOffset; - bool UseInAlloca; + unsigned FreeSSERegs; }; /// X86_32ABIInfo - The X86-32 ABI information. @@ -530,6 +574,17 @@ class X86_32ABIInfo : public ABIInfo { return (Size == 8 || Size == 16 || Size == 32 || Size == 64); } + bool isHomogeneousAggregateBaseType(QualType Ty) const override { + // FIXME: Assumes vectorcall is in use. + return isX86VectorTypeForVectorCall(getContext(), Ty); + } + + bool isHomogeneousAggregateSmallEnough(const Type *Ty, + uint64_t NumMembers) const override { + // FIXME: Assumes vectorcall is in use. + return isX86VectorCallAggregateSmallEnough(NumMembers); + } + bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const; /// getIndirectResult - Give a source type \arg Ty, return a suitable result @@ -593,6 +648,14 @@ public: return X86AdjustInlineAsmType(CGF, Constraint, Ty); } + void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue, + std::string &Constraints, + std::vector<llvm::Type *> &ResultRegTypes, + std::vector<llvm::Type *> &ResultTruncRegTypes, + std::vector<LValue> &ResultRegDests, + std::string &AsmString, + unsigned NumOutputs) const override; + llvm::Constant * getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override { unsigned Sig = (0xeb << 0) | // jmp rel8 @@ -606,6 +669,85 @@ public: } +/// Rewrite input constraint references after adding some output constraints. +/// In the case where there is one output and one input and we add one output, +/// we need to replace all operand references greater than or equal to 1: +/// mov $0, $1 +/// mov eax, $1 +/// The result will be: +/// mov $0, $2 +/// mov eax, $2 +static void rewriteInputConstraintReferences(unsigned FirstIn, + unsigned NumNewOuts, + std::string &AsmString) { + std::string Buf; + llvm::raw_string_ostream OS(Buf); + size_t Pos = 0; + while (Pos < AsmString.size()) { + size_t DollarStart = AsmString.find('$', Pos); + if (DollarStart == std::string::npos) + DollarStart = AsmString.size(); + size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart); + if (DollarEnd == std::string::npos) + DollarEnd = AsmString.size(); + OS << StringRef(&AsmString[Pos], DollarEnd - Pos); + Pos = DollarEnd; + size_t NumDollars = DollarEnd - DollarStart; + if (NumDollars % 2 != 0 && Pos < AsmString.size()) { + // We have an operand reference. + size_t DigitStart = Pos; + size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart); + if (DigitEnd == std::string::npos) + DigitEnd = AsmString.size(); + StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart); + unsigned OperandIndex; + if (!OperandStr.getAsInteger(10, OperandIndex)) { + if (OperandIndex >= FirstIn) + OperandIndex += NumNewOuts; + OS << OperandIndex; + } else { + OS << OperandStr; + } + Pos = DigitEnd; + } + } + AsmString = std::move(OS.str()); +} + +/// Add output constraints for EAX:EDX because they are return registers. +void X86_32TargetCodeGenInfo::addReturnRegisterOutputs( + CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints, + std::vector<llvm::Type *> &ResultRegTypes, + std::vector<llvm::Type *> &ResultTruncRegTypes, + std::vector<LValue> &ResultRegDests, std::string &AsmString, + unsigned NumOutputs) const { + uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType()); + + // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is + // larger. + if (!Constraints.empty()) + Constraints += ','; + if (RetWidth <= 32) { + Constraints += "={eax}"; + ResultRegTypes.push_back(CGF.Int32Ty); + } else { + // Use the 'A' constraint for EAX:EDX. + Constraints += "=A"; + ResultRegTypes.push_back(CGF.Int64Ty); + } + + // Truncate EAX or EAX:EDX to an integer of the appropriate size. + llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth); + ResultTruncRegTypes.push_back(CoerceTy); + + // Coerce the integer by bitcasting the return slot pointer. + ReturnSlot.setAddress(CGF.Builder.CreateBitCast(ReturnSlot.getAddress(), + CoerceTy->getPointerTo())); + ResultRegDests.push_back(ReturnSlot); + + rewriteInputConstraintReferences(NumOutputs, 1, AsmString); +} + /// shouldReturnTypeInRegister - Determine if the given type should be /// passed in a register (for the Darwin ABI). bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty, @@ -670,6 +812,14 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy, CCState &State) con if (RetTy->isVoidType()) return ABIArgInfo::getIgnore(); + const Type *Base = nullptr; + uint64_t NumElts = 0; + if (State.CC == llvm::CallingConv::X86_VectorCall && + isHomogeneousAggregate(RetTy, Base, NumElts)) { + // The LLVM struct type for such an aggregate should lower properly. + return ABIArgInfo::getDirect(); + } + if (const VectorType *VT = RetTy->getAs<VectorType>()) { // On Darwin, some vectors are returned in registers. if (IsDarwinVectorABI) { @@ -842,7 +992,8 @@ bool X86_32ABIInfo::shouldUseInReg(QualType Ty, CCState &State, State.FreeRegs -= SizeInRegs; - if (State.CC == llvm::CallingConv::X86_FastCall) { + if (State.CC == llvm::CallingConv::X86_FastCall || + State.CC == llvm::CallingConv::X86_VectorCall) { if (Size > 32) return false; @@ -867,17 +1018,38 @@ bool X86_32ABIInfo::shouldUseInReg(QualType Ty, CCState &State, ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State) const { // FIXME: Set alignment on indirect arguments. - if (isAggregateTypeForABI(Ty)) { - if (const RecordType *RT = Ty->getAs<RecordType>()) { - // Check with the C++ ABI first. - CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()); - if (RAA == CGCXXABI::RAA_Indirect) { - return getIndirectResult(Ty, false, State); - } else if (RAA == CGCXXABI::RAA_DirectInMemory) { - // The field index doesn't matter, we'll fix it up later. - return ABIArgInfo::getInAlloca(/*FieldIndex=*/0); - } + Ty = useFirstFieldIfTransparentUnion(Ty); + + // Check with the C++ ABI first. + const RecordType *RT = Ty->getAs<RecordType>(); + if (RT) { + CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()); + if (RAA == CGCXXABI::RAA_Indirect) { + return getIndirectResult(Ty, false, State); + } else if (RAA == CGCXXABI::RAA_DirectInMemory) { + // The field index doesn't matter, we'll fix it up later. + return ABIArgInfo::getInAlloca(/*FieldIndex=*/0); + } + } + + // vectorcall adds the concept of a homogenous vector aggregate, similar + // to other targets. + const Type *Base = nullptr; + uint64_t NumElts = 0; + if (State.CC == llvm::CallingConv::X86_VectorCall && + isHomogeneousAggregate(Ty, Base, NumElts)) { + if (State.FreeSSERegs >= NumElts) { + State.FreeSSERegs -= NumElts; + if (Ty->isBuiltinType() || Ty->isVectorType()) + return ABIArgInfo::getDirect(); + return ABIArgInfo::getExpand(); + } + return getIndirectResult(Ty, /*ByVal=*/false, State); + } + + if (isAggregateTypeForABI(Ty)) { + if (RT) { // Structs are always byval on win32, regardless of what they contain. if (IsWin32StructABI) return getIndirectResult(Ty, true, State); @@ -909,7 +1081,9 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, if (getContext().getTypeSize(Ty) <= 4*32 && canExpandIndirectArgument(Ty, getContext())) return ABIArgInfo::getExpandWithPadding( - State.CC == llvm::CallingConv::X86_FastCall, PaddingType); + State.CC == llvm::CallingConv::X86_FastCall || + State.CC == llvm::CallingConv::X86_VectorCall, + PaddingType); return getIndirectResult(Ty, true, State); } @@ -952,7 +1126,10 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { CCState State(FI.getCallingConvention()); if (State.CC == llvm::CallingConv::X86_FastCall) State.FreeRegs = 2; - else if (FI.getHasRegParm()) + else if (State.CC == llvm::CallingConv::X86_VectorCall) { + State.FreeRegs = 2; + State.FreeSSERegs = 6; + } else if (FI.getHasRegParm()) State.FreeRegs = FI.getRegParm(); else State.FreeRegs = DefaultNumRegisterParameters; @@ -968,6 +1145,10 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const { } } + // The chain argument effectively gives us another free register. + if (FI.isChainCall()) + ++State.FreeRegs; + bool UsedInAlloca = false; for (auto &I : FI.arguments()) { I.info = classifyArgumentType(I.type, State); @@ -1002,6 +1183,26 @@ X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields, } } +static bool isArgInAlloca(const ABIArgInfo &Info) { + // Leave ignored and inreg arguments alone. + switch (Info.getKind()) { + case ABIArgInfo::InAlloca: + return true; + case ABIArgInfo::Indirect: + assert(Info.getIndirectByVal()); + return true; + case ABIArgInfo::Ignore: + return false; + case ABIArgInfo::Direct: + case ABIArgInfo::Extend: + case ABIArgInfo::Expand: + if (Info.getInReg()) + return false; + return true; + } + llvm_unreachable("invalid enum"); +} + void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const { assert(IsWin32StructABI && "inalloca only supported on win32"); @@ -1009,9 +1210,19 @@ void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const { SmallVector<llvm::Type *, 6> FrameFields; unsigned StackOffset = 0; + CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end(); - // Put the sret parameter into the inalloca struct if it's in memory. + // Put 'this' into the struct before 'sret', if necessary. + bool IsThisCall = + FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall; ABIArgInfo &Ret = FI.getReturnInfo(); + if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall && + isArgInAlloca(I->info)) { + addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type); + ++I; + } + + // Put the sret parameter into the inalloca struct if it's in memory. if (Ret.isIndirect() && !Ret.getInReg()) { CanQualType PtrTy = getContext().getPointerType(FI.getReturnType()); addFieldToArgStruct(FrameFields, StackOffset, Ret, PtrTy); @@ -1020,30 +1231,13 @@ void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const { } // Skip the 'this' parameter in ecx. - CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end(); - if (FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall) + if (IsThisCall) ++I; // Put arguments passed in memory into the struct. for (; I != E; ++I) { - - // Leave ignored and inreg arguments alone. - switch (I->info.getKind()) { - case ABIArgInfo::Indirect: - assert(I->info.getIndirectByVal()); - break; - case ABIArgInfo::Ignore: - continue; - case ABIArgInfo::Direct: - case ABIArgInfo::Extend: - if (I->info.getInReg()) - continue; - break; - default: - break; - } - - addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type); + if (isArgInAlloca(I->info)) + addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type); } FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields, @@ -1107,22 +1301,12 @@ bool X86_32TargetCodeGenInfo::isStructReturnInRegABI( return true; switch (Triple.getOS()) { - case llvm::Triple::AuroraUX: case llvm::Triple::DragonFly: case llvm::Triple::FreeBSD: case llvm::Triple::OpenBSD: case llvm::Triple::Bitrig: - return true; case llvm::Triple::Win32: - switch (Triple.getEnvironment()) { - case llvm::Triple::UnknownEnvironment: - case llvm::Triple::Cygnus: - case llvm::Triple::GNU: - case llvm::Triple::MSVC: - return true; - default: - return false; - } + return true; default: return false; } @@ -1325,7 +1509,8 @@ public: /// WinX86_64ABIInfo - The Windows X86_64 ABI information. class WinX86_64ABIInfo : public ABIInfo { - ABIArgInfo classify(QualType Ty, bool IsReturnType) const; + ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, + bool IsReturnType) const; public: WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {} @@ -1334,12 +1519,24 @@ public: llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const override; + + bool isHomogeneousAggregateBaseType(QualType Ty) const override { + // FIXME: Assumes vectorcall is in use. + return isX86VectorTypeForVectorCall(getContext(), Ty); + } + + bool isHomogeneousAggregateSmallEnough(const Type *Ty, + uint64_t NumMembers) const override { + // FIXME: Assumes vectorcall is in use. + return isX86VectorCallAggregateSmallEnough(NumMembers); + } }; class X86_64TargetCodeGenInfo : public TargetCodeGenInfo { + bool HasAVX; public: X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new X86_64ABIInfo(CGT, HasAVX)) {} + : TargetCodeGenInfo(new X86_64ABIInfo(CGT, HasAVX)), HasAVX(HasAVX) {} const X86_64ABIInfo &getABIInfo() const { return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo()); @@ -1399,6 +1596,9 @@ public: return llvm::ConstantInt::get(CGM.Int32Ty, Sig); } + unsigned getOpenMPSimdDefaultAlignment(QualType) const override { + return HasAVX ? 32 : 16; + } }; static std::string qualifyWindowsLibrary(llvm::StringRef Lib) { @@ -1430,9 +1630,10 @@ public: }; class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo { + bool HasAVX; public: - WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) - : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)) {} + WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) + : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)), HasAVX(HasAVX) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override { return 7; @@ -1459,6 +1660,10 @@ public: llvm::SmallString<32> &Opt) const override { Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\""; } + + unsigned getOpenMPSimdDefaultAlignment(QualType) const override { + return HasAVX ? 32 : 16; + } }; } @@ -1586,10 +1791,25 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, } if (Ty->isMemberPointerType()) { - if (Ty->isMemberFunctionPointerType() && Has64BitPointers) - Lo = Hi = Integer; - else + if (Ty->isMemberFunctionPointerType()) { + if (Has64BitPointers) { + // If Has64BitPointers, this is an {i64, i64}, so classify both + // Lo and Hi now. + Lo = Hi = Integer; + } else { + // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that + // straddles an eightbyte boundary, Hi should be classified as well. + uint64_t EB_FuncPtr = (OffsetBase) / 64; + uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64; + if (EB_FuncPtr != EB_ThisAdj) { + Lo = Hi = Integer; + } else { + Current = Integer; + } + } + } else { Current = Integer; + } return; } @@ -2171,7 +2391,7 @@ GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi, // the second element at offset 8. Check for this: unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo); unsigned HiAlign = TD.getABITypeAlignment(Hi); - unsigned HiStart = llvm::DataLayout::RoundUpAlignment(LoSize, HiAlign); + unsigned HiStart = llvm::RoundUpToAlignment(LoSize, HiAlign); assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!"); // To handle this, we have to increase the size of the low part so that the @@ -2190,7 +2410,7 @@ GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi, } } - llvm::StructType *Result = llvm::StructType::get(Lo, Hi, NULL); + llvm::StructType *Result = llvm::StructType::get(Lo, Hi, nullptr); // Verify that the second element is at an 8-byte offset. @@ -2267,7 +2487,7 @@ classifyReturnType(QualType RetTy) const { assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification."); ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()), llvm::Type::getX86_FP80Ty(getVMContext()), - NULL); + nullptr); break; } @@ -2333,6 +2553,8 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType( bool isNamedArg) const { + Ty = useFirstFieldIfTransparentUnion(Ty); + X86_64ABIInfo::Class Lo, Hi; classify(Ty, 0, Lo, Hi, isNamedArg); @@ -2468,23 +2690,21 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { if (FI.getReturnInfo().isIndirect()) --freeIntRegs; - bool isVariadic = FI.isVariadic(); - unsigned numRequiredArgs = 0; - if (isVariadic) - numRequiredArgs = FI.getRequiredArgs().getNumRequiredArgs(); + // The chain argument effectively gives us another free register. + if (FI.isChainCall()) + ++freeIntRegs; + unsigned NumRequiredArgs = FI.getNumRequiredArgs(); // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers // get assigned (in left-to-right order) for passing as follows... + unsigned ArgNo = 0; for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end(); - it != ie; ++it) { - bool isNamedArg = true; - if (isVariadic) - isNamedArg = (it - FI.arg_begin()) < - static_cast<signed>(numRequiredArgs); + it != ie; ++it, ++ArgNo) { + bool IsNamedArg = ArgNo < NumRequiredArgs; unsigned neededInt, neededSSE; it->info = classifyArgumentType(it->type, freeIntRegs, neededInt, - neededSSE, isNamedArg); + neededSSE, IsNamedArg); // AMD64-ABI 3.2.3p3: If there are no registers available for any // eightbyte of an argument, the whole argument is passed on the @@ -2674,7 +2894,7 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Type *DoubleTy = CGF.DoubleTy; llvm::Type *DblPtrTy = llvm::PointerType::getUnqual(DoubleTy); - llvm::StructType *ST = llvm::StructType::get(DoubleTy, DoubleTy, NULL); + llvm::StructType *ST = llvm::StructType::get(DoubleTy, DoubleTy, nullptr); llvm::Value *V, *Tmp = CGF.CreateMemTemp(Ty); Tmp = CGF.Builder.CreateBitCast(Tmp, ST->getPointerTo()); V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrLo, @@ -2717,7 +2937,8 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, return ResAddr; } -ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, bool IsReturnType) const { +ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, + bool IsReturnType) const { if (Ty->isVoidType()) return ABIArgInfo::getIgnore(); @@ -2725,7 +2946,9 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, bool IsReturnType) const { if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - uint64_t Size = getContext().getTypeSize(Ty); + TypeInfo Info = getContext().getTypeInfo(Ty); + uint64_t Width = Info.Width; + unsigned Align = getContext().toCharUnitsFromBits(Info.Align).getQuantity(); const RecordType *RT = Ty->getAs<RecordType>(); if (RT) { @@ -2738,11 +2961,26 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, bool IsReturnType) const { return ABIArgInfo::getIndirect(0, /*ByVal=*/false); // FIXME: mingw-w64-gcc emits 128-bit struct as i128 - if (Size == 128 && getTarget().getTriple().isWindowsGNUEnvironment()) + if (Width == 128 && getTarget().getTriple().isWindowsGNUEnvironment()) return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), - Size)); + Width)); } + // vectorcall adds the concept of a homogenous vector aggregate, similar to + // other targets. + const Type *Base = nullptr; + uint64_t NumElts = 0; + if (FreeSSERegs && isHomogeneousAggregate(Ty, Base, NumElts)) { + if (FreeSSERegs >= NumElts) { + FreeSSERegs -= NumElts; + if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType()) + return ABIArgInfo::getDirect(); + return ABIArgInfo::getExpand(); + } + return ABIArgInfo::getIndirect(Align, /*ByVal=*/false); + } + + if (Ty->isMemberPointerType()) { // If the member pointer is represented by an LLVM int or ptr, pass it // directly. @@ -2754,25 +2992,35 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, bool IsReturnType) const { if (RT || Ty->isMemberPointerType()) { // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is // not 1, 2, 4, or 8 bytes, must be passed by reference." - if (Size > 64 || !llvm::isPowerOf2_64(Size)) + if (Width > 64 || !llvm::isPowerOf2_64(Width)) return ABIArgInfo::getIndirect(0, /*ByVal=*/false); // Otherwise, coerce it to a small integer. - return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); + return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width)); } - if (Ty->isPromotableIntegerType()) + // Bool type is always extended to the ABI, other builtin types are not + // extended. + const BuiltinType *BT = Ty->getAs<BuiltinType>(); + if (BT && BT->getKind() == BuiltinType::Bool) return ABIArgInfo::getExtend(); return ABIArgInfo::getDirect(); } void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { + bool IsVectorCall = + FI.getCallingConvention() == llvm::CallingConv::X86_VectorCall; + + // We can use up to 4 SSE return registers with vectorcall. + unsigned FreeSSERegs = IsVectorCall ? 4 : 0; if (!getCXXABI().classifyReturnType(FI)) - FI.getReturnInfo() = classify(FI.getReturnType(), true); + FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true); + // We can use up to 6 SSE register parameters with vectorcall. + FreeSSERegs = IsVectorCall ? 6 : 0; for (auto &I : FI.arguments()) - I.info = classify(I.type, false); + I.info = classify(I.type, FreeSSERegs, false); } llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, @@ -2812,9 +3060,14 @@ class NaClX86_64ABIInfo : public ABIInfo { }; class NaClX86_64TargetCodeGenInfo : public TargetCodeGenInfo { + bool HasAVX; public: - NaClX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new NaClX86_64ABIInfo(CGT, HasAVX)) {} + NaClX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) + : TargetCodeGenInfo(new NaClX86_64ABIInfo(CGT, HasAVX)), HasAVX(HasAVX) { + } + unsigned getOpenMPSimdDefaultAlignment(QualType) const override { + return HasAVX ? 32 : 16; + } }; } @@ -2835,11 +3088,19 @@ llvm::Value *NaClX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // PowerPC-32 - namespace { -class PPC32TargetCodeGenInfo : public DefaultTargetCodeGenInfo { +/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information. +class PPC32_SVR4_ABIInfo : public DefaultABIInfo { public: - PPC32TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {} + PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {} + + llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, + CodeGenFunction &CGF) const override; +}; + +class PPC32TargetCodeGenInfo : public TargetCodeGenInfo { +public: + PPC32TargetCodeGenInfo(CodeGenTypes &CGT) : TargetCodeGenInfo(new PPC32_SVR4_ABIInfo(CGT)) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { // This is recovered from gcc output. @@ -2848,10 +3109,104 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; + + unsigned getOpenMPSimdDefaultAlignment(QualType) const override { + return 16; // Natural alignment for Altivec vectors. + } }; } +llvm::Value *PPC32_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, + QualType Ty, + CodeGenFunction &CGF) const { + if (const ComplexType *CTy = Ty->getAs<ComplexType>()) { + // TODO: Implement this. For now ignore. + (void)CTy; + return nullptr; + } + + bool isI64 = Ty->isIntegerType() && getContext().getTypeSize(Ty) == 64; + bool isInt = Ty->isIntegerType() || Ty->isPointerType() || Ty->isAggregateType(); + llvm::Type *CharPtr = CGF.Int8PtrTy; + llvm::Type *CharPtrPtr = CGF.Int8PtrPtrTy; + + CGBuilderTy &Builder = CGF.Builder; + llvm::Value *GPRPtr = Builder.CreateBitCast(VAListAddr, CharPtr, "gprptr"); + llvm::Value *GPRPtrAsInt = Builder.CreatePtrToInt(GPRPtr, CGF.Int32Ty); + llvm::Value *FPRPtrAsInt = Builder.CreateAdd(GPRPtrAsInt, Builder.getInt32(1)); + llvm::Value *FPRPtr = Builder.CreateIntToPtr(FPRPtrAsInt, CharPtr); + llvm::Value *OverflowAreaPtrAsInt = Builder.CreateAdd(FPRPtrAsInt, Builder.getInt32(3)); + llvm::Value *OverflowAreaPtr = Builder.CreateIntToPtr(OverflowAreaPtrAsInt, CharPtrPtr); + llvm::Value *RegsaveAreaPtrAsInt = Builder.CreateAdd(OverflowAreaPtrAsInt, Builder.getInt32(4)); + llvm::Value *RegsaveAreaPtr = Builder.CreateIntToPtr(RegsaveAreaPtrAsInt, CharPtrPtr); + llvm::Value *GPR = Builder.CreateLoad(GPRPtr, false, "gpr"); + // Align GPR when TY is i64. + if (isI64) { + llvm::Value *GPRAnd = Builder.CreateAnd(GPR, Builder.getInt8(1)); + llvm::Value *CC64 = Builder.CreateICmpEQ(GPRAnd, Builder.getInt8(1)); + llvm::Value *GPRPlusOne = Builder.CreateAdd(GPR, Builder.getInt8(1)); + GPR = Builder.CreateSelect(CC64, GPRPlusOne, GPR); + } + llvm::Value *FPR = Builder.CreateLoad(FPRPtr, false, "fpr"); + llvm::Value *OverflowArea = Builder.CreateLoad(OverflowAreaPtr, false, "overflow_area"); + llvm::Value *OverflowAreaAsInt = Builder.CreatePtrToInt(OverflowArea, CGF.Int32Ty); + llvm::Value *RegsaveArea = Builder.CreateLoad(RegsaveAreaPtr, false, "regsave_area"); + llvm::Value *RegsaveAreaAsInt = Builder.CreatePtrToInt(RegsaveArea, CGF.Int32Ty); + + llvm::Value *CC = Builder.CreateICmpULT(isInt ? GPR : FPR, + Builder.getInt8(8), "cond"); + + llvm::Value *RegConstant = Builder.CreateMul(isInt ? GPR : FPR, + Builder.getInt8(isInt ? 4 : 8)); + + llvm::Value *OurReg = Builder.CreateAdd(RegsaveAreaAsInt, Builder.CreateSExt(RegConstant, CGF.Int32Ty)); + + if (Ty->isFloatingType()) + OurReg = Builder.CreateAdd(OurReg, Builder.getInt32(32)); + + llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs"); + llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow"); + llvm::BasicBlock *Cont = CGF.createBasicBlock("cont"); + + Builder.CreateCondBr(CC, UsingRegs, UsingOverflow); + + CGF.EmitBlock(UsingRegs); + + llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); + llvm::Value *Result1 = Builder.CreateIntToPtr(OurReg, PTy); + // Increase the GPR/FPR indexes. + if (isInt) { + GPR = Builder.CreateAdd(GPR, Builder.getInt8(isI64 ? 2 : 1)); + Builder.CreateStore(GPR, GPRPtr); + } else { + FPR = Builder.CreateAdd(FPR, Builder.getInt8(1)); + Builder.CreateStore(FPR, FPRPtr); + } + CGF.EmitBranch(Cont); + + CGF.EmitBlock(UsingOverflow); + + // Increase the overflow area. + llvm::Value *Result2 = Builder.CreateIntToPtr(OverflowAreaAsInt, PTy); + OverflowAreaAsInt = Builder.CreateAdd(OverflowAreaAsInt, Builder.getInt32(isInt ? 4 : 8)); + Builder.CreateStore(Builder.CreateIntToPtr(OverflowAreaAsInt, CharPtr), OverflowAreaPtr); + CGF.EmitBranch(Cont); + + CGF.EmitBlock(Cont); + + llvm::PHINode *Result = CGF.Builder.CreatePHI(PTy, 2, "vaarg.addr"); + Result->addIncoming(Result1, UsingRegs); + Result->addIncoming(Result2, UsingOverflow); + + if (Ty->isAggregateType()) { + llvm::Value *AGGPtr = Builder.CreateBitCast(Result, CharPtrPtr, "aggrptr") ; + return Builder.CreateLoad(AGGPtr, false, "aggr"); + } + + return Result; +} + bool PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const { @@ -2914,12 +3269,14 @@ public: bool isPromotableTypeForABI(QualType Ty) const; bool isAlignedParamType(QualType Ty) const; - bool isHomogeneousAggregate(QualType Ty, const Type *&Base, - uint64_t &Members) const; ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType Ty) const; + bool isHomogeneousAggregateBaseType(QualType Ty) const override; + bool isHomogeneousAggregateSmallEnough(const Type *Ty, + uint64_t Members) const override; + // TODO: We can add more logic to computeInfo to improve performance. // Example: For aggregate arguments that fit in a register, we could // use getDirectInReg (as is done below for structs containing a single @@ -2964,6 +3321,10 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; + + unsigned getOpenMPSimdDefaultAlignment(QualType) const override { + return 16; // Natural alignment for Altivec and VSX vectors. + } }; class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo { @@ -2977,6 +3338,10 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; + + unsigned getOpenMPSimdDefaultAlignment(QualType) const override { + return 16; // Natural alignment for Altivec vectors. + } }; } @@ -3054,9 +3419,8 @@ PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty) const { /// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous /// aggregate. Base is set to the base element type, and Members is set /// to the number of base elements. -bool -PPC64_SVR4_ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base, - uint64_t &Members) const { +bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base, + uint64_t &Members) const { if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) { uint64_t NElements = AT->getSize().getZExtValue(); if (NElements == 0) @@ -3070,6 +3434,22 @@ PPC64_SVR4_ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base, return false; Members = 0; + + // If this is a C++ record, check the bases first. + if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { + for (const auto &I : CXXRD->bases()) { + // Ignore empty records. + if (isEmptyRecord(getContext(), I.getType(), true)) + continue; + + uint64_t FldMembers; + if (!isHomogeneousAggregate(I.getType(), Base, FldMembers)) + return false; + + Members += FldMembers; + } + } + for (const auto *FD : RD->fields()) { // Ignore (non-zero arrays of) empty records. QualType FT = FD->getType(); @@ -3109,19 +3489,9 @@ PPC64_SVR4_ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base, Ty = CT->getElementType(); } - // Homogeneous aggregates for ELFv2 must have base types of float, - // double, long double, or 128-bit vectors. - if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { - if (BT->getKind() != BuiltinType::Float && - BT->getKind() != BuiltinType::Double && - BT->getKind() != BuiltinType::LongDouble) - return false; - } else if (const VectorType *VT = Ty->getAs<VectorType>()) { - if (getContext().getTypeSize(VT) != 128) - return false; - } else { + // Most ABIs only support float, double, and some vector type widths. + if (!isHomogeneousAggregateBaseType(Ty)) return false; - } // The base type must be the same for all members. Types that // agree in both total size and mode (float vs. vector) are @@ -3134,18 +3504,40 @@ PPC64_SVR4_ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base, getContext().getTypeSize(Base) != getContext().getTypeSize(TyPtr)) return false; } + return Members > 0 && isHomogeneousAggregateSmallEnough(Base, Members); +} + +bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { + // Homogeneous aggregates for ELFv2 must have base types of float, + // double, long double, or 128-bit vectors. + if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { + if (BT->getKind() == BuiltinType::Float || + BT->getKind() == BuiltinType::Double || + BT->getKind() == BuiltinType::LongDouble) + return true; + } + if (const VectorType *VT = Ty->getAs<VectorType>()) { + if (getContext().getTypeSize(VT) == 128) + return true; + } + return false; +} +bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateSmallEnough( + const Type *Base, uint64_t Members) const { // Vector types require one register, floating point types require one // or two registers depending on their size. - uint32_t NumRegs = Base->isVectorType() ? 1 : - (getContext().getTypeSize(Base) + 63) / 64; + uint32_t NumRegs = + Base->isVectorType() ? 1 : (getContext().getTypeSize(Base) + 63) / 64; // Homogeneous Aggregates may occupy at most 8 registers. - return (Members > 0 && Members * NumRegs <= 8); + return Members * NumRegs <= 8; } ABIArgInfo PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { + Ty = useFirstFieldIfTransparentUnion(Ty); + if (Ty->isAnyComplexType()) return ABIArgInfo::getDirect(); @@ -3252,7 +3644,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { llvm::Type *CoerceTy; if (Bits > GPRBits) { CoerceTy = llvm::IntegerType::get(getVMContext(), GPRBits); - CoerceTy = llvm::StructType::get(CoerceTy, CoerceTy, NULL); + CoerceTy = llvm::StructType::get(CoerceTy, CoerceTy, nullptr); } else CoerceTy = llvm::IntegerType::get(getVMContext(), llvm::RoundUpToAlignment(Bits, 8)); @@ -3429,76 +3821,19 @@ private: bool isDarwinPCS() const { return Kind == DarwinPCS; } ABIArgInfo classifyReturnType(QualType RetTy) const; - ABIArgInfo classifyArgumentType(QualType RetTy, unsigned &AllocatedVFP, - bool &IsHA, unsigned &AllocatedGPR, - bool &IsSmallAggr, bool IsNamedArg) const; - bool isIllegalVectorType(QualType Ty) const; + ABIArgInfo classifyArgumentType(QualType RetTy) const; + bool isHomogeneousAggregateBaseType(QualType Ty) const override; + bool isHomogeneousAggregateSmallEnough(const Type *Ty, + uint64_t Members) const override; - virtual void computeInfo(CGFunctionInfo &FI) const { - // To correctly handle Homogeneous Aggregate, we need to keep track of the - // number of SIMD and Floating-point registers allocated so far. - // If the argument is an HFA or an HVA and there are sufficient unallocated - // SIMD and Floating-point registers, then the argument is allocated to SIMD - // and Floating-point Registers (with one register per member of the HFA or - // HVA). Otherwise, the NSRN is set to 8. - unsigned AllocatedVFP = 0; - - // To correctly handle small aggregates, we need to keep track of the number - // of GPRs allocated so far. If the small aggregate can't all fit into - // registers, it will be on stack. We don't allow the aggregate to be - // partially in registers. - unsigned AllocatedGPR = 0; - - // Find the number of named arguments. Variadic arguments get special - // treatment with the Darwin ABI. - unsigned NumRequiredArgs = (FI.isVariadic() ? - FI.getRequiredArgs().getNumRequiredArgs() : - FI.arg_size()); + bool isIllegalVectorType(QualType Ty) const; + void computeInfo(CGFunctionInfo &FI) const override { if (!getCXXABI().classifyReturnType(FI)) FI.getReturnInfo() = classifyReturnType(FI.getReturnType()); - for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end(); - it != ie; ++it) { - unsigned PreAllocation = AllocatedVFP, PreGPR = AllocatedGPR; - bool IsHA = false, IsSmallAggr = false; - const unsigned NumVFPs = 8; - const unsigned NumGPRs = 8; - bool IsNamedArg = ((it - FI.arg_begin()) < - static_cast<signed>(NumRequiredArgs)); - it->info = classifyArgumentType(it->type, AllocatedVFP, IsHA, - AllocatedGPR, IsSmallAggr, IsNamedArg); - - // Under AAPCS the 64-bit stack slot alignment means we can't pass HAs - // as sequences of floats since they'll get "holes" inserted as - // padding by the back end. - if (IsHA && AllocatedVFP > NumVFPs && !isDarwinPCS() && - getContext().getTypeAlign(it->type) < 64) { - uint32_t NumStackSlots = getContext().getTypeSize(it->type); - NumStackSlots = llvm::RoundUpToAlignment(NumStackSlots, 64) / 64; - - llvm::Type *CoerceTy = llvm::ArrayType::get( - llvm::Type::getDoubleTy(getVMContext()), NumStackSlots); - it->info = ABIArgInfo::getDirect(CoerceTy); - } - - // If we do not have enough VFP registers for the HA, any VFP registers - // that are unallocated are marked as unavailable. To achieve this, we add - // padding of (NumVFPs - PreAllocation) floats. - if (IsHA && AllocatedVFP > NumVFPs && PreAllocation < NumVFPs) { - llvm::Type *PaddingTy = llvm::ArrayType::get( - llvm::Type::getFloatTy(getVMContext()), NumVFPs - PreAllocation); - it->info.setPaddingType(PaddingTy); - } - // If we do not have enough GPRs for the small aggregate, any GPR regs - // that are unallocated are marked as unavailable. - if (IsSmallAggr && AllocatedGPR > NumGPRs && PreGPR < NumGPRs) { - llvm::Type *PaddingTy = llvm::ArrayType::get( - llvm::Type::getInt32Ty(getVMContext()), NumGPRs - PreGPR); - it->info = - ABIArgInfo::getDirect(it->info.getCoerceToType(), 0, PaddingTy); - } - } + for (auto &it : FI.arguments()) + it.info = classifyArgumentType(it.type); } llvm::Value *EmitDarwinVAArg(llvm::Value *VAListAddr, QualType Ty, @@ -3508,7 +3843,7 @@ private: CodeGenFunction &CGF) const; virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { + CodeGenFunction &CGF) const override { return isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF) : EmitAAPCSVAArg(VAListAddr, Ty, CGF); } @@ -3529,63 +3864,34 @@ public: }; } -static bool isHomogeneousAggregate(QualType Ty, const Type *&Base, - ASTContext &Context, - uint64_t *HAMembers = nullptr); +ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty) const { + Ty = useFirstFieldIfTransparentUnion(Ty); -ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, - unsigned &AllocatedVFP, - bool &IsHA, - unsigned &AllocatedGPR, - bool &IsSmallAggr, - bool IsNamedArg) const { // Handle illegal vector types here. if (isIllegalVectorType(Ty)) { uint64_t Size = getContext().getTypeSize(Ty); if (Size <= 32) { llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext()); - AllocatedGPR++; return ABIArgInfo::getDirect(ResType); } if (Size == 64) { llvm::Type *ResType = llvm::VectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2); - AllocatedVFP++; return ABIArgInfo::getDirect(ResType); } if (Size == 128) { llvm::Type *ResType = llvm::VectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4); - AllocatedVFP++; return ABIArgInfo::getDirect(ResType); } - AllocatedGPR++; return ABIArgInfo::getIndirect(0, /*ByVal=*/false); } - if (Ty->isVectorType()) - // Size of a legal vector should be either 64 or 128. - AllocatedVFP++; - if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { - if (BT->getKind() == BuiltinType::Half || - BT->getKind() == BuiltinType::Float || - BT->getKind() == BuiltinType::Double || - BT->getKind() == BuiltinType::LongDouble) - AllocatedVFP++; - } if (!isAggregateTypeForABI(Ty)) { // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); - if (!Ty->isFloatingType() && !Ty->isVectorType()) { - unsigned Alignment = getContext().getTypeAlign(Ty); - if (!isDarwinPCS() && Alignment > 64) - AllocatedGPR = llvm::RoundUpToAlignment(AllocatedGPR, Alignment / 64); - - int RegsNeeded = getContext().getTypeSize(Ty) > 64 ? 2 : 1; - AllocatedGPR += RegsNeeded; - } return (Ty->isPromotableIntegerType() && isDarwinPCS() ? ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); @@ -3594,9 +3900,8 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, // Structures with either a non-trivial destructor or a non-trivial // copy constructor are always indirect. if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { - AllocatedGPR++; return ABIArgInfo::getIndirect(0, /*ByVal=*/RAA == - CGCXXABI::RAA_DirectInMemory); + CGCXXABI::RAA_DirectInMemory); } // Empty records are always ignored on Darwin, but actually passed in C++ mode @@ -3605,36 +3910,23 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS()) return ABIArgInfo::getIgnore(); - ++AllocatedGPR; return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext())); } // Homogeneous Floating-point Aggregates (HFAs) need to be expanded. const Type *Base = nullptr; uint64_t Members = 0; - if (isHomogeneousAggregate(Ty, Base, getContext(), &Members)) { - IsHA = true; - if (!IsNamedArg && isDarwinPCS()) { - // With the Darwin ABI, variadic arguments are always passed on the stack - // and should not be expanded. Treat variadic HFAs as arrays of doubles. - uint64_t Size = getContext().getTypeSize(Ty); - llvm::Type *BaseTy = llvm::Type::getDoubleTy(getVMContext()); - return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64)); - } - AllocatedVFP += Members; - return ABIArgInfo::getExpand(); + if (isHomogeneousAggregate(Ty, Base, Members)) { + return ABIArgInfo::getDirect( + llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members)); } // Aggregates <= 16 bytes are passed directly in registers or on the stack. uint64_t Size = getContext().getTypeSize(Ty); if (Size <= 128) { unsigned Alignment = getContext().getTypeAlign(Ty); - if (!isDarwinPCS() && Alignment > 64) - AllocatedGPR = llvm::RoundUpToAlignment(AllocatedGPR, Alignment / 64); - Size = 64 * ((Size + 63) / 64); // round up to multiple of 8 bytes - AllocatedGPR += Size / 64; - IsSmallAggr = true; + // We use a pair of i64 for 16-byte aggregate with 8-byte alignment. // For aggregates with 16-byte alignment, we use i128. if (Alignment < 128 && Size == 128) { @@ -3644,7 +3936,6 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); } - AllocatedGPR++; return ABIArgInfo::getIndirect(0, /*ByVal=*/false); } @@ -3670,7 +3961,8 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const { return ABIArgInfo::getIgnore(); const Type *Base = nullptr; - if (isHomogeneousAggregate(RetTy, Base, getContext())) + uint64_t Members = 0; + if (isHomogeneousAggregate(RetTy, Base, Members)) // Homogeneous Floating-point Aggregates (HFAs) are returned directly. return ABIArgInfo::getDirect(); @@ -3698,9 +3990,46 @@ bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const { return false; } -static llvm::Value *EmitAArch64VAArg(llvm::Value *VAListAddr, QualType Ty, - int AllocatedGPR, int AllocatedVFP, - bool IsIndirect, CodeGenFunction &CGF) { +bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { + // Homogeneous aggregates for AAPCS64 must have base types of a floating + // point type or a short-vector type. This is the same as the 32-bit ABI, + // but with the difference that any floating-point type is allowed, + // including __fp16. + if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { + if (BT->isFloatingPoint()) + return true; + } else if (const VectorType *VT = Ty->getAs<VectorType>()) { + unsigned VecSize = getContext().getTypeSize(VT); + if (VecSize == 64 || VecSize == 128) + return true; + } + return false; +} + +bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, + uint64_t Members) const { + return Members <= 4; +} + +llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, + QualType Ty, + CodeGenFunction &CGF) const { + ABIArgInfo AI = classifyArgumentType(Ty); + bool IsIndirect = AI.isIndirect(); + + llvm::Type *BaseTy = CGF.ConvertType(Ty); + if (IsIndirect) + BaseTy = llvm::PointerType::getUnqual(BaseTy); + else if (AI.getCoerceToType()) + BaseTy = AI.getCoerceToType(); + + unsigned NumRegs = 1; + if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) { + BaseTy = ArrTy->getElementType(); + NumRegs = ArrTy->getNumElements(); + } + bool IsFPR = BaseTy->isFloatingPointTy() || BaseTy->isVectorTy(); + // The AArch64 va_list type and handling is specified in the Procedure Call // Standard, section B.4: // @@ -3720,21 +4049,19 @@ static llvm::Value *EmitAArch64VAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *reg_offs_p = nullptr, *reg_offs = nullptr; int reg_top_index; - int RegSize; - if (AllocatedGPR) { - assert(!AllocatedVFP && "Arguments never split between int & VFP regs"); + int RegSize = IsIndirect ? 8 : getContext().getTypeSize(Ty) / 8; + if (!IsFPR) { // 3 is the field number of __gr_offs reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs"); reg_top_index = 1; // field number for __gr_top - RegSize = 8 * AllocatedGPR; + RegSize = llvm::RoundUpToAlignment(RegSize, 8); } else { - assert(!AllocatedGPR && "Argument must go in VFP or int regs"); // 4 is the field number of __vr_offs. reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs"); reg_top_index = 2; // field number for __vr_top - RegSize = 16 * AllocatedVFP; + RegSize = 16 * NumRegs; } //======================================= @@ -3758,7 +4085,7 @@ static llvm::Value *EmitAArch64VAArg(llvm::Value *VAListAddr, QualType Ty, // Integer arguments may need to correct register alignment (for example a // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we // align __gr_offs to calculate the potential address. - if (AllocatedGPR && !IsIndirect && Ctx.getTypeAlign(Ty) > 64) { + if (!IsFPR && !IsIndirect && Ctx.getTypeAlign(Ty) > 64) { int Align = Ctx.getTypeAlign(Ty) / 8; reg_offs = CGF.Builder.CreateAdd( @@ -3806,8 +4133,8 @@ static llvm::Value *EmitAArch64VAArg(llvm::Value *VAListAddr, QualType Ty, } const Type *Base = nullptr; - uint64_t NumMembers; - bool IsHFA = isHomogeneousAggregate(Ty, Base, Ctx, &NumMembers); + uint64_t NumMembers = 0; + bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers); if (IsHFA && NumMembers > 1) { // Homogeneous aggregates passed in registers will have their elements split // and stored 16-bytes apart regardless of size (they're notionally in qN, @@ -3926,18 +4253,6 @@ static llvm::Value *EmitAArch64VAArg(llvm::Value *VAListAddr, QualType Ty, return ResAddr; } -llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - - unsigned AllocatedGPR = 0, AllocatedVFP = 0; - bool IsHA = false, IsSmallAggr = false; - ABIArgInfo AI = classifyArgumentType(Ty, AllocatedVFP, IsHA, AllocatedGPR, - IsSmallAggr, false /*IsNamedArg*/); - - return EmitAArch64VAArg(VAListAddr, Ty, AllocatedGPR, AllocatedVFP, - AI.isIndirect(), CGF); -} - llvm::Value *AArch64ABIInfo::EmitDarwinVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const { // We do not support va_arg for aggregates or illegal vector types. @@ -3950,7 +4265,8 @@ llvm::Value *AArch64ABIInfo::EmitDarwinVAArg(llvm::Value *VAListAddr, QualType T uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8; const Type *Base = nullptr; - bool isHA = isHomogeneousAggregate(Ty, Base, getContext()); + uint64_t Members = 0; + bool isHA = isHomogeneousAggregate(Ty, Base, Members); bool isIndirect = false; // Arguments bigger than 16 bytes which aren't homogeneous aggregates should @@ -4022,7 +4338,7 @@ private: public: ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : ABIInfo(CGT), Kind(_Kind), NumVFPs(16), NumGPRs(4) { - setRuntimeCC(); + setCCs(); resetAllocatedRegs(); } @@ -4057,6 +4373,10 @@ private: bool &IsCPRC) const; bool isIllegalVectorType(QualType Ty) const; + bool isHomogeneousAggregateBaseType(QualType Ty) const override; + bool isHomogeneousAggregateSmallEnough(const Type *Ty, + uint64_t Members) const override; + void computeInfo(CGFunctionInfo &FI) const override; llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, @@ -4064,7 +4384,7 @@ private: llvm::CallingConv::ID getLLVMDefaultCC() const; llvm::CallingConv::ID getABIDefaultCC() const; - void setRuntimeCC(); + void setCCs(); void markAllocatedGPRs(unsigned Alignment, unsigned NumRequired) const; void markAllocatedVFPs(unsigned Alignment, unsigned NumRequired) const; @@ -4183,11 +4503,11 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const { llvm::Type *PaddingTy = llvm::ArrayType::get( llvm::Type::getInt32Ty(getVMContext()), NumGPRs - PreAllocationGPRs); if (I.info.canHaveCoerceToType()) { - I.info = ABIArgInfo::getDirect(I.info.getCoerceToType() /* type */, 0 /* offset */, - PaddingTy); + I.info = ABIArgInfo::getDirect(I.info.getCoerceToType() /* type */, + 0 /* offset */, PaddingTy, true); } else { I.info = ABIArgInfo::getDirect(nullptr /* type */, 0 /* offset */, - PaddingTy); + PaddingTy, true); } } } @@ -4223,7 +4543,7 @@ llvm::CallingConv::ID ARMABIInfo::getABIDefaultCC() const { llvm_unreachable("bad ABI kind"); } -void ARMABIInfo::setRuntimeCC() { +void ARMABIInfo::setCCs() { assert(getRuntimeCC() == llvm::CallingConv::C); // Don't muddy up the IR with a ton of explicit annotations if @@ -4231,90 +4551,9 @@ void ARMABIInfo::setRuntimeCC() { llvm::CallingConv::ID abiCC = getABIDefaultCC(); if (abiCC != getLLVMDefaultCC()) RuntimeCC = abiCC; -} - -/// isHomogeneousAggregate - Return true if a type is an AAPCS-VFP homogeneous -/// aggregate. If HAMembers is non-null, the number of base elements -/// contained in the type is returned through it; this is used for the -/// recursive calls that check aggregate component types. -static bool isHomogeneousAggregate(QualType Ty, const Type *&Base, - ASTContext &Context, uint64_t *HAMembers) { - uint64_t Members = 0; - if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) { - if (!isHomogeneousAggregate(AT->getElementType(), Base, Context, &Members)) - return false; - Members *= AT->getSize().getZExtValue(); - } else if (const RecordType *RT = Ty->getAs<RecordType>()) { - const RecordDecl *RD = RT->getDecl(); - if (RD->hasFlexibleArrayMember()) - return false; - - Members = 0; - for (const auto *FD : RD->fields()) { - uint64_t FldMembers; - if (!isHomogeneousAggregate(FD->getType(), Base, Context, &FldMembers)) - return false; - - Members = (RD->isUnion() ? - std::max(Members, FldMembers) : Members + FldMembers); - } - } else { - Members = 1; - if (const ComplexType *CT = Ty->getAs<ComplexType>()) { - Members = 2; - Ty = CT->getElementType(); - } - - // Homogeneous aggregates for AAPCS-VFP must have base types of float, - // double, or 64-bit or 128-bit vectors. - if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { - if (BT->getKind() != BuiltinType::Float && - BT->getKind() != BuiltinType::Double && - BT->getKind() != BuiltinType::LongDouble) - return false; - } else if (const VectorType *VT = Ty->getAs<VectorType>()) { - unsigned VecSize = Context.getTypeSize(VT); - if (VecSize != 64 && VecSize != 128) - return false; - } else { - return false; - } - - // The base type must be the same for all members. Vector types of the - // same total size are treated as being equivalent here. - const Type *TyPtr = Ty.getTypePtr(); - if (!Base) - Base = TyPtr; - - if (Base != TyPtr) { - // Homogeneous aggregates are defined as containing members with the - // same machine type. There are two cases in which two members have - // different TypePtrs but the same machine type: - - // 1) Vectors of the same length, regardless of the type and number - // of their members. - const bool SameLengthVectors = Base->isVectorType() && TyPtr->isVectorType() - && (Context.getTypeSize(Base) == Context.getTypeSize(TyPtr)); - - // 2) In the 32-bit AAPCS, `double' and `long double' have the same - // machine type. This is not the case for the 64-bit AAPCS. - const bool SameSizeDoubles = - ( ( Base->isSpecificBuiltinType(BuiltinType::Double) - && TyPtr->isSpecificBuiltinType(BuiltinType::LongDouble)) - || ( Base->isSpecificBuiltinType(BuiltinType::LongDouble) - && TyPtr->isSpecificBuiltinType(BuiltinType::Double))) - && (Context.getTypeSize(Base) == Context.getTypeSize(TyPtr)); - - if (!SameLengthVectors && !SameSizeDoubles) - return false; - } - } - - // Homogeneous Aggregates can have at most 4 members of the base type. - if (HAMembers) - *HAMembers = Members; - return (Members > 0 && Members <= 4); + BuiltinCC = (getABIKind() == APCS ? + llvm::CallingConv::ARM_APCS : llvm::CallingConv::ARM_AAPCS); } /// markAllocatedVFPs - update VFPRegs according to the alignment and @@ -4382,6 +4621,9 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, // with a Base Type of a single- or double-precision floating-point type, // 64-bit containerized vectors or 128-bit containerized vectors with one // to four Elements. + bool IsEffectivelyAAPCS_VFP = getABIKind() == AAPCS_VFP && !isVariadic; + + Ty = useFirstFieldIfTransparentUnion(Ty); // Handle illegal vector types here. if (isIllegalVectorType(Ty)) { @@ -4451,8 +4693,8 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, unsigned Size = getContext().getTypeSize(Ty); if (!IsCPRC) markAllocatedGPRs(Size > 32 ? 2 : 1, (Size + 31) / 32); - return (Ty->isPromotableIntegerType() ? - ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); + return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend() + : ABIArgInfo::getDirect()); } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { @@ -4464,12 +4706,12 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (isEmptyRecord(getContext(), Ty, true)) return ABIArgInfo::getIgnore(); - if (getABIKind() == ARMABIInfo::AAPCS_VFP && !isVariadic) { + if (IsEffectivelyAAPCS_VFP) { // Homogeneous Aggregates need to be expanded when we can fit the aggregate // into VFP registers. const Type *Base = nullptr; uint64_t Members = 0; - if (isHomogeneousAggregate(Ty, Base, getContext(), &Members)) { + if (isHomogeneousAggregate(Ty, Base, Members)) { assert(Base && "Base class should be set for homogeneous aggregate"); // Base can be a floating-point or a vector. if (Base->isVectorType()) { @@ -4485,7 +4727,7 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, markAllocatedVFPs(2, Members * 2); } IsCPRC = true; - return ABIArgInfo::getDirect(); + return ABIArgInfo::getDirect(nullptr, 0, nullptr, false); } } @@ -4523,9 +4765,7 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, markAllocatedGPRs(2, SizeRegs * 2); } - llvm::Type *STy = - llvm::StructType::get(llvm::ArrayType::get(ElemTy, SizeRegs), NULL); - return ABIArgInfo::getDirect(STy); + return ABIArgInfo::getDirect(llvm::ArrayType::get(ElemTy, SizeRegs)); } static bool isIntegerLikeType(QualType Ty, ASTContext &Context, @@ -4615,6 +4855,8 @@ static bool isIntegerLikeType(QualType Ty, ASTContext &Context, ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic) const { + bool IsEffectivelyAAPCS_VFP = getABIKind() == AAPCS_VFP && !isVariadic; + if (RetTy->isVoidType()) return ABIArgInfo::getIgnore(); @@ -4629,8 +4871,8 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, if (const EnumType *EnumTy = RetTy->getAs<EnumType>()) RetTy = EnumTy->getDecl()->getIntegerType(); - return (RetTy->isPromotableIntegerType() ? - ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); + return RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend() + : ABIArgInfo::getDirect(); } // Are we following APCS? @@ -4643,8 +4885,8 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, // FIXME: Consider using 2 x vector types if the back end handles them // correctly. if (RetTy->isAnyComplexType()) - return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), - getContext().getTypeSize(RetTy))); + return ABIArgInfo::getDirect(llvm::IntegerType::get( + getVMContext(), getContext().getTypeSize(RetTy))); // Integer like structures are returned in r0. if (isIntegerLikeType(RetTy, getContext(), getVMContext())) { @@ -4668,12 +4910,13 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, return ABIArgInfo::getIgnore(); // Check for homogeneous aggregates with AAPCS-VFP. - if (getABIKind() == AAPCS_VFP && !isVariadic) { + if (IsEffectivelyAAPCS_VFP) { const Type *Base = nullptr; - if (isHomogeneousAggregate(RetTy, Base, getContext())) { + uint64_t Members; + if (isHomogeneousAggregate(RetTy, Base, Members)) { assert(Base && "Base class should be set for homogeneous aggregate"); // Homogeneous Aggregates are returned directly. - return ABIArgInfo::getDirect(); + return ABIArgInfo::getDirect(nullptr, 0, nullptr, false); } } @@ -4712,6 +4955,27 @@ bool ARMABIInfo::isIllegalVectorType(QualType Ty) const { return false; } +bool ARMABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { + // Homogeneous aggregates for AAPCS-VFP must have base types of float, + // double, or 64-bit or 128-bit vectors. + if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { + if (BT->getKind() == BuiltinType::Float || + BT->getKind() == BuiltinType::Double || + BT->getKind() == BuiltinType::LongDouble) + return true; + } else if (const VectorType *VT = Ty->getAs<VectorType>()) { + unsigned VecSize = getContext().getTypeSize(VT); + if (VecSize == 64 || VecSize == 128) + return true; + } + return false; +} + +bool ARMABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, + uint64_t Members) const { + return Members <= 4; +} + llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const { llvm::Type *BP = CGF.Int8PtrTy; @@ -4876,6 +5140,10 @@ ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const { if (const EnumType *EnumTy = Ty->getAs<EnumType>()) Ty = EnumTy->getDecl()->getIntegerType(); + // Return aggregates type as indirect by value + if (isAggregateTypeForABI(Ty)) + return ABIArgInfo::getIndirect(0, /* byval */ true); + return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); } @@ -4953,9 +5221,10 @@ void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::Function *F, StringRef Name, // Get "nvvm.annotations" metadata node llvm::NamedMDNode *MD = M->getOrInsertNamedMetadata("nvvm.annotations"); - llvm::Value *MDVals[] = { - F, llvm::MDString::get(Ctx, Name), - llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), Operand)}; + llvm::Metadata *MDVals[] = { + llvm::ConstantAsMetadata::get(F), llvm::MDString::get(Ctx, Name), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), Operand))}; // Append metadata to nvvm.annotations MD->addOperand(llvm::MDNode::get(Ctx, MDVals)); } @@ -5424,6 +5693,8 @@ llvm::Type *MipsABIInfo::getPaddingType(uint64_t OrigOffset, ABIArgInfo MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const { + Ty = useFirstFieldIfTransparentUnion(Ty); + uint64_t OrigOffset = Offset; uint64_t TySize = getContext().getTypeSize(Ty); uint64_t Align = getContext().getTypeAlign(Ty) / 8; @@ -5524,7 +5795,7 @@ ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const { return ABIArgInfo::getDirect(); // O32 returns integer vectors in registers and N32/N64 returns all small - // aggregates in registers.. + // aggregates in registers. if (!IsO32 || (RetTy->isVectorType() && !RetTy->hasFloatingRepresentation())) { ABIArgInfo ArgInfo = @@ -5562,10 +5833,13 @@ llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Type *BP = CGF.Int8PtrTy; llvm::Type *BPP = CGF.Int8PtrPtrTy; - // Integer arguments are promoted 32-bit on O32 and 64-bit on N32/N64. + // Integer arguments are promoted to 32-bit on O32 and 64-bit on N32/N64. + // Pointers are also promoted in the same way but this only matters for N32. unsigned SlotSizeInBits = IsO32 ? 32 : 64; - if (Ty->isIntegerType() && - CGF.getContext().getIntWidth(Ty) < SlotSizeInBits) { + unsigned PtrWidth = getTarget().getPointerWidth(0); + if ((Ty->isIntegerType() && + CGF.getContext().getIntWidth(Ty) < SlotSizeInBits) || + (Ty->isPointerType() && PtrWidth < SlotSizeInBits)) { Ty = CGF.getContext().getIntTypeForBitwidth(SlotSizeInBits, Ty->isSignedIntegerType()); } @@ -5577,7 +5851,6 @@ llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, std::min(getContext().getTypeAlign(Ty) / 8, StackAlignInBytes); llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty)); llvm::Value *AddrTyped; - unsigned PtrWidth = getTarget().getPointerWidth(0); llvm::IntegerType *IntTy = (PtrWidth == 32) ? CGF.Int32Ty : CGF.Int64Ty; if (TypeAlign > MinABIStackAlignInBytes) { @@ -5667,20 +5940,24 @@ void TCETargetCodeGenInfo::SetTargetAttributes(const Decl *D, llvm::NamedMDNode *OpenCLMetadata = M.getModule().getOrInsertNamedMetadata("opencl.kernel_wg_size_info"); - SmallVector<llvm::Value*, 5> Operands; - Operands.push_back(F); + SmallVector<llvm::Metadata *, 5> Operands; + Operands.push_back(llvm::ConstantAsMetadata::get(F)); - Operands.push_back(llvm::Constant::getIntegerValue(M.Int32Ty, - llvm::APInt(32, Attr->getXDim()))); - Operands.push_back(llvm::Constant::getIntegerValue(M.Int32Ty, - llvm::APInt(32, Attr->getYDim()))); - Operands.push_back(llvm::Constant::getIntegerValue(M.Int32Ty, - llvm::APInt(32, Attr->getZDim()))); + Operands.push_back( + llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue( + M.Int32Ty, llvm::APInt(32, Attr->getXDim())))); + Operands.push_back( + llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue( + M.Int32Ty, llvm::APInt(32, Attr->getYDim())))); + Operands.push_back( + llvm::ConstantAsMetadata::get(llvm::Constant::getIntegerValue( + M.Int32Ty, llvm::APInt(32, Attr->getZDim())))); // Add a boolean constant operand for "required" (true) or "hint" (false) // for implementing the work_group_size_hint attr later. Currently // always true as the hint is not yet implemented. - Operands.push_back(llvm::ConstantInt::getTrue(Context)); + Operands.push_back( + llvm::ConstantAsMetadata::get(llvm::ConstantInt::getTrue(Context))); OpenCLMetadata->addOperand(llvm::MDNode::get(Context, Operands)); } } @@ -5822,6 +6099,45 @@ llvm::Value *HexagonABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, return AddrTyped; } +//===----------------------------------------------------------------------===// +// AMDGPU ABI Implementation +//===----------------------------------------------------------------------===// + +namespace { + +class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo { +public: + AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT) + : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {} + void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &M) const override; +}; + +} + +void AMDGPUTargetCodeGenInfo::SetTargetAttributes( + const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &M) const { + const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); + if (!FD) + return; + + if (const auto Attr = FD->getAttr<AMDGPUNumVGPRAttr>()) { + llvm::Function *F = cast<llvm::Function>(GV); + uint32_t NumVGPR = Attr->getNumVGPR(); + if (NumVGPR != 0) + F->addFnAttr("amdgpu_num_vgpr", llvm::utostr(NumVGPR)); + } + + if (const auto Attr = FD->getAttr<AMDGPUNumSGPRAttr>()) { + llvm::Function *F = cast<llvm::Function>(GV); + unsigned NumSGPR = Attr->getNumSGPR(); + if (NumSGPR != 0) + F->addFnAttr("amdgpu_num_sgpr", llvm::utostr(NumSGPR)); + } +} + //===----------------------------------------------------------------------===// // SPARC v9 ABI Implementation. @@ -6405,8 +6721,8 @@ void XCoreTargetCodeGenInfo::emitTargetMD(const Decl *D, llvm::GlobalValue *GV, SmallStringEnc Enc; if (getTypeString(Enc, D, CGM, TSC)) { llvm::LLVMContext &Ctx = CGM.getModule().getContext(); - llvm::SmallVector<llvm::Value *, 2> MDVals; - MDVals.push_back(GV); + llvm::SmallVector<llvm::Metadata *, 2> MDVals; + MDVals.push_back(llvm::ConstantAsMetadata::get(GV)); MDVals.push_back(llvm::MDString::get(Ctx, Enc.str())); llvm::NamedMDNode *MD = CGM.getModule().getOrInsertNamedMetadata("xcore.typestrings"); @@ -6424,26 +6740,25 @@ static bool extractFieldType(SmallVectorImpl<FieldEncoding> &FE, const RecordDecl *RD, const CodeGen::CodeGenModule &CGM, TypeStringCache &TSC) { - for (RecordDecl::field_iterator I = RD->field_begin(), E = RD->field_end(); - I != E; ++I) { + for (const auto *Field : RD->fields()) { SmallStringEnc Enc; Enc += "m("; - Enc += I->getName(); + Enc += Field->getName(); Enc += "){"; - if (I->isBitField()) { + if (Field->isBitField()) { Enc += "b("; llvm::raw_svector_ostream OS(Enc); OS.resync(); - OS << I->getBitWidthValue(CGM.getContext()); + OS << Field->getBitWidthValue(CGM.getContext()); OS.flush(); Enc += ':'; } - if (!appendType(Enc, I->getType(), CGM, TSC)) + if (!appendType(Enc, Field->getType(), CGM, TSC)) return false; - if (I->isBitField()) + if (Field->isBitField()) Enc += ')'; Enc += '}'; - FE.push_back(FieldEncoding(!I->getName().empty(), Enc)); + FE.push_back(FieldEncoding(!Field->getName().empty(), Enc)); } return true; } @@ -6752,6 +7067,14 @@ static bool getTypeString(SmallStringEnc &Enc, const Decl *D, // Driver code //===----------------------------------------------------------------------===// +const llvm::Triple &CodeGenModule::getTriple() const { + return getTarget().getTriple(); +} + +bool CodeGenModule::supportsCOMDAT() const { + return !getTriple().isOSBinFormatMachO(); +} + const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { if (TheTargetCodeGenInfo) return *TheTargetCodeGenInfo; @@ -6772,9 +7095,7 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types, false)); case llvm::Triple::aarch64: - case llvm::Triple::aarch64_be: - case llvm::Triple::arm64: - case llvm::Triple::arm64_be: { + case llvm::Triple::aarch64_be: { AArch64ABIInfo::ABIKind Kind = AArch64ABIInfo::AAPCS; if (getTarget().getABI() == "darwinpcs") Kind = AArch64ABIInfo::DarwinPCS; @@ -6809,16 +7130,20 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { return *(TheTargetCodeGenInfo = new PPC32TargetCodeGenInfo(Types)); case llvm::Triple::ppc64: if (Triple.isOSBinFormatELF()) { - // FIXME: Should be switchable via command-line option. PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1; + if (getTarget().getABI() == "elfv2") + Kind = PPC64_SVR4_ABIInfo::ELFv2; + return *(TheTargetCodeGenInfo = new PPC64_SVR4_TargetCodeGenInfo(Types, Kind)); } else return *(TheTargetCodeGenInfo = new PPC64TargetCodeGenInfo(Types)); case llvm::Triple::ppc64le: { assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!"); - // FIXME: Should be switchable via command-line option. PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv2; + if (getTarget().getABI() == "elfv1") + Kind = PPC64_SVR4_ABIInfo::ELFv1; + return *(TheTargetCodeGenInfo = new PPC64_SVR4_TargetCodeGenInfo(Types, Kind)); } @@ -6840,7 +7165,7 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { bool IsDarwinVectorABI = Triple.isOSDarwin(); bool IsSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(Triple, CodeGenOpts); - bool IsWin32FloatStructABI = Triple.isWindowsMSVCEnvironment(); + bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing(); if (Triple.getOS() == llvm::Triple::Win32) { return *(TheTargetCodeGenInfo = @@ -6862,17 +7187,22 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { switch (Triple.getOS()) { case llvm::Triple::Win32: - return *(TheTargetCodeGenInfo = new WinX86_64TargetCodeGenInfo(Types)); + return *(TheTargetCodeGenInfo = + new WinX86_64TargetCodeGenInfo(Types, HasAVX)); case llvm::Triple::NaCl: - return *(TheTargetCodeGenInfo = new NaClX86_64TargetCodeGenInfo(Types, - HasAVX)); + return *(TheTargetCodeGenInfo = + new NaClX86_64TargetCodeGenInfo(Types, HasAVX)); default: - return *(TheTargetCodeGenInfo = new X86_64TargetCodeGenInfo(Types, - HasAVX)); + return *(TheTargetCodeGenInfo = + new X86_64TargetCodeGenInfo(Types, HasAVX)); } } case llvm::Triple::hexagon: return *(TheTargetCodeGenInfo = new HexagonTargetCodeGenInfo(Types)); + case llvm::Triple::r600: + return *(TheTargetCodeGenInfo = new AMDGPUTargetCodeGenInfo(Types)); + case llvm::Triple::amdgcn: + return *(TheTargetCodeGenInfo = new AMDGPUTargetCodeGenInfo(Types)); case llvm::Triple::sparcv9: return *(TheTargetCodeGenInfo = new SparcV9TargetCodeGenInfo(Types)); case llvm::Triple::xcore: |