diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp')
| -rw-r--r-- | contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp | 825 |
1 files changed, 551 insertions, 274 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp index a2384456ea94..2673e4a5cee7 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp @@ -16,6 +16,7 @@ #include "CGCXXABI.h" #include "CGCleanup.h" #include "CGDebugInfo.h" +#include "CGHLSLRuntime.h" #include "CGOpenMPRuntime.h" #include "CodeGenModule.h" #include "CodeGenPGO.h" @@ -43,8 +44,11 @@ #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Operator.h" #include "llvm/Support/CRC.h" +#include "llvm/Support/xxhash.h" #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" +#include <optional> + using namespace clang; using namespace CodeGen; @@ -78,7 +82,6 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) EHStack.setCGF(this); SetFastMathFlags(CurFPFeatures); - SetFPModel(); } CodeGenFunction::~CodeGenFunction() { @@ -105,19 +108,9 @@ clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) { case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore; case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap; case LangOptions::FPE_Strict: return llvm::fp::ebStrict; + default: + llvm_unreachable("Unsupported FP Exception Behavior"); } - llvm_unreachable("Unsupported FP Exception Behavior"); -} - -void CodeGenFunction::SetFPModel() { - llvm::RoundingMode RM = getLangOpts().getFPRoundingMode(); - auto fpExceptionBehavior = ToConstrainedExceptMD( - getLangOpts().getFPExceptionMode()); - - Builder.setDefaultConstrainedRounding(RM); - Builder.setDefaultConstrainedExcept(fpExceptionBehavior); - Builder.setIsFPConstrained(fpExceptionBehavior != llvm::fp::ebIgnore || - RM != llvm::RoundingMode::NearestTiesToEven); } void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) { @@ -156,12 +149,11 @@ void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) { FMFGuard.emplace(CGF.Builder); - llvm::RoundingMode NewRoundingBehavior = - static_cast<llvm::RoundingMode>(FPFeatures.getRoundingMode()); + llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode(); CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior); auto NewExceptionBehavior = ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>( - FPFeatures.getFPExceptionMode())); + FPFeatures.getExceptionMode())); CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior); CGF.SetFastMathFlags(FPFeatures); @@ -183,10 +175,11 @@ void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) { mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs()); mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs()); mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero()); - mergeFnAttrValue("unsafe-fp-math", FPFeatures.getAllowFPReassociate() && - FPFeatures.getAllowReciprocal() && - FPFeatures.getAllowApproxFunc() && - FPFeatures.getNoSignedZero()); + mergeFnAttrValue( + "unsafe-fp-math", + FPFeatures.getAllowFPReassociate() && FPFeatures.getAllowReciprocal() && + FPFeatures.getAllowApproxFunc() && FPFeatures.getNoSignedZero() && + FPFeatures.allowFPContractAcrossStatement()); } CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() { @@ -199,8 +192,8 @@ LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) { LValueBaseInfo BaseInfo; TBAAAccessInfo TBAAInfo; CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo); - return LValue::MakeAddr(Address(V, Alignment), T, getContext(), BaseInfo, - TBAAInfo); + Address Addr(V, ConvertTypeForMem(T), Alignment); + return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo); } /// Given a value of type T* that may not be to a complete object, @@ -211,7 +204,8 @@ CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) { TBAAAccessInfo TBAAInfo; CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, /* forPointeeType= */ true); - return MakeAddrLValue(Address(V, Align), T, BaseInfo, TBAAInfo); + Address Addr(V, ConvertTypeForMem(T), Align); + return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); } @@ -254,7 +248,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) { case Type::Enum: case Type::ObjCObjectPointer: case Type::Pipe: - case Type::ExtInt: + case Type::BitInt: return TEK_Scalar; // Complexes. @@ -327,8 +321,10 @@ llvm::DebugLoc CodeGenFunction::EmitReturnBlock() { static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) { if (!BB) return; - if (!BB->use_empty()) - return CGF.CurFn->getBasicBlockList().push_back(BB); + if (!BB->use_empty()) { + CGF.CurFn->insert(CGF.CurFn->end(), BB); + return; + } delete BB; } @@ -366,17 +362,18 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { bool HasOnlyLifetimeMarkers = HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth); bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers; + + std::optional<ApplyDebugLocation> OAL; if (HasCleanups) { // Make sure the line table doesn't jump back into the body for // the ret after it's been at EndLoc. - Optional<ApplyDebugLocation> AL; if (CGDebugInfo *DI = getDebugInfo()) { if (OnlySimpleReturnStmts) DI->EmitLocation(Builder, EndLoc); else // We may not have a valid end location. Try to apply it anyway, and // fall back to an artificial location if needed. - AL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc); + OAL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc); } PopCleanupBlocks(PrologueCleanupDepth); @@ -432,6 +429,14 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { AllocaInsertPt = nullptr; Ptr->eraseFromParent(); + // PostAllocaInsertPt, if created, was lazily created when it was required, + // remove it now since it was just created for our own convenience. + if (PostAllocaInsertPt) { + llvm::Instruction *PostPtr = PostAllocaInsertPt; + PostAllocaInsertPt = nullptr; + PostPtr->eraseFromParent(); + } + // If someone took the address of a label but never did an indirect goto, we // made a zero entry PHI node, which is illegal, zap it now. if (IndirectBranch) { @@ -479,28 +484,34 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { if (auto *VT = dyn_cast<llvm::VectorType>(A.getType())) LargestVectorWidth = std::max((uint64_t)LargestVectorWidth, - VT->getPrimitiveSizeInBits().getKnownMinSize()); + VT->getPrimitiveSizeInBits().getKnownMinValue()); // Update vector width based on return type. if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType())) LargestVectorWidth = std::max((uint64_t)LargestVectorWidth, - VT->getPrimitiveSizeInBits().getKnownMinSize()); + VT->getPrimitiveSizeInBits().getKnownMinValue()); + + if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth) + LargestVectorWidth = CurFnInfo->getMaxVectorWidth(); - // Add the required-vector-width attribute. This contains the max width from: + // Add the min-legal-vector-width attribute. This contains the max width from: // 1. min-vector-width attribute used in the source program. // 2. Any builtins used that have a vector width specified. // 3. Values passed in and out of inline assembly. // 4. Width of vector arguments and return types for this function. - // 5. Width of vector aguments and return types for functions called by this + // 5. Width of vector arguments and return types for functions called by this // function. - CurFn->addFnAttr("min-legal-vector-width", llvm::utostr(LargestVectorWidth)); + if (getContext().getTargetInfo().getTriple().isX86()) + CurFn->addFnAttr("min-legal-vector-width", + llvm::utostr(LargestVectorWidth)); - // Add vscale attribute if appropriate. - if (getLangOpts().ArmSveVectorBits) { - unsigned VScale = getLangOpts().ArmSveVectorBits / 128; - CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(getLLVMContext(), - VScale, VScale)); + // Add vscale_range attribute if appropriate. + std::optional<std::pair<unsigned, unsigned>> VScaleRange = + getContext().getTargetInfo().getVScaleRange(getLangOpts()); + if (VScaleRange) { + CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs( + getLLVMContext(), VScaleRange->first, VScaleRange->second)); } // If we generated an unreachable return block, delete it now. @@ -529,6 +540,12 @@ bool CodeGenFunction::ShouldInstrumentFunction() { return true; } +bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() { + if (!CurFuncDecl) + return false; + return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>(); +} + /// ShouldXRayInstrument - Return true if the current function should be /// instrumented with XRay nop sleds. bool CodeGenFunction::ShouldXRayInstrumentFunction() const { @@ -551,52 +568,30 @@ bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const { XRayInstrKind::Typed); } -llvm::Constant * -CodeGenFunction::EncodeAddrForUseInPrologue(llvm::Function *F, - llvm::Constant *Addr) { - // Addresses stored in prologue data can't require run-time fixups and must - // be PC-relative. Run-time fixups are undesirable because they necessitate - // writable text segments, which are unsafe. And absolute addresses are - // undesirable because they break PIE mode. - - // Add a layer of indirection through a private global. Taking its address - // won't result in a run-time fixup, even if Addr has linkonce_odr linkage. - auto *GV = new llvm::GlobalVariable(CGM.getModule(), Addr->getType(), - /*isConstant=*/true, - llvm::GlobalValue::PrivateLinkage, Addr); - - // Create a PC-relative address. - auto *GOTAsInt = llvm::ConstantExpr::getPtrToInt(GV, IntPtrTy); - auto *FuncAsInt = llvm::ConstantExpr::getPtrToInt(F, IntPtrTy); - auto *PCRelAsInt = llvm::ConstantExpr::getSub(GOTAsInt, FuncAsInt); - return (IntPtrTy == Int32Ty) - ? PCRelAsInt - : llvm::ConstantExpr::getTrunc(PCRelAsInt, Int32Ty); +llvm::ConstantInt * +CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const { + // Remove any (C++17) exception specifications, to allow calling e.g. a + // noexcept function through a non-noexcept pointer. + if (!Ty->isFunctionNoProtoType()) + Ty = getContext().getFunctionTypeWithExceptionSpec(Ty, EST_None); + std::string Mangled; + llvm::raw_string_ostream Out(Mangled); + CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out, false); + return llvm::ConstantInt::get( + CGM.Int32Ty, static_cast<uint32_t>(llvm::xxh3_64bits(Mangled))); } -llvm::Value * -CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F, - llvm::Value *EncodedAddr) { - // Reconstruct the address of the global. - auto *PCRelAsInt = Builder.CreateSExt(EncodedAddr, IntPtrTy); - auto *FuncAsInt = Builder.CreatePtrToInt(F, IntPtrTy, "func_addr.int"); - auto *GOTAsInt = Builder.CreateAdd(PCRelAsInt, FuncAsInt, "global_addr.int"); - auto *GOTAddr = Builder.CreateIntToPtr(GOTAsInt, Int8PtrPtrTy, "global_addr"); - - // Load the original pointer through the global. - return Builder.CreateLoad(Address(GOTAddr, getPointerAlign()), - "decoded_addr"); -} - -void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD, - llvm::Function *Fn) -{ - if (!FD->hasAttr<OpenCLKernelAttr>()) +void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD, + llvm::Function *Fn) { + if (!FD->hasAttr<OpenCLKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>()) return; llvm::LLVMContext &Context = getLLVMContext(); - CGM.GenOpenCLArgMetadata(Fn, FD, this); + CGM.GenKernelArgMetadata(Fn, FD, this); + + if (!getLangOpts().OpenCL) + return; if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) { QualType HintQTy = A->getTypeHint(); @@ -688,6 +683,19 @@ static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) { return true; } +bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) { + const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl(); + return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory; +} + +bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) { + return getTarget().getTriple().getArch() == llvm::Triple::x86 && + getTarget().getCXXABI().isMicrosoft() && + llvm::any_of(MD->parameters(), [&](ParmVarDecl *P) { + return isInAllocaArgument(CGM.getCXXABI(), P->getType()); + }); +} + /// Return the UBSan prologue signature for \p FD if one is available. static llvm::Constant *getPrologueSignature(CodeGenModule &CGM, const FunctionDecl *FD) { @@ -712,7 +720,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, CurCodeDecl = D; const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D); if (FD && FD->usesSEHTry()) - CurSEHParent = FD; + CurSEHParent = GD; CurFuncDecl = (D ? D->getNonClosureContext() : nullptr); FnRetTy = RetTy; CurFn = Fn; @@ -731,44 +739,60 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, #include "clang/Basic/Sanitizers.def" #undef SANITIZER - } while (0); + } while (false); if (D) { + const bool SanitizeBounds = SanOpts.hasOneOf(SanitizerKind::Bounds); + SanitizerMask no_sanitize_mask; bool NoSanitizeCoverage = false; - for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) { - // Apply the no_sanitize* attributes to SanOpts. - SanitizerMask mask = Attr->getMask(); - SanOpts.Mask &= ~mask; - if (mask & SanitizerKind::Address) - SanOpts.set(SanitizerKind::KernelAddress, false); - if (mask & SanitizerKind::KernelAddress) - SanOpts.set(SanitizerKind::Address, false); - if (mask & SanitizerKind::HWAddress) - SanOpts.set(SanitizerKind::KernelHWAddress, false); - if (mask & SanitizerKind::KernelHWAddress) - SanOpts.set(SanitizerKind::HWAddress, false); - + for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) { + no_sanitize_mask |= Attr->getMask(); // SanitizeCoverage is not handled by SanOpts. if (Attr->hasCoverage()) NoSanitizeCoverage = true; } + // Apply the no_sanitize* attributes to SanOpts. + SanOpts.Mask &= ~no_sanitize_mask; + if (no_sanitize_mask & SanitizerKind::Address) + SanOpts.set(SanitizerKind::KernelAddress, false); + if (no_sanitize_mask & SanitizerKind::KernelAddress) + SanOpts.set(SanitizerKind::Address, false); + if (no_sanitize_mask & SanitizerKind::HWAddress) + SanOpts.set(SanitizerKind::KernelHWAddress, false); + if (no_sanitize_mask & SanitizerKind::KernelHWAddress) + SanOpts.set(SanitizerKind::HWAddress, false); + + if (SanitizeBounds && !SanOpts.hasOneOf(SanitizerKind::Bounds)) + Fn->addFnAttr(llvm::Attribute::NoSanitizeBounds); + if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage()) Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage); + + // Some passes need the non-negated no_sanitize attribute. Pass them on. + if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) { + if (no_sanitize_mask & SanitizerKind::Thread) + Fn->addFnAttr("no_sanitize_thread"); + } } - // Apply sanitizer attributes to the function. - if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress)) - Fn->addFnAttr(llvm::Attribute::SanitizeAddress); - if (SanOpts.hasOneOf(SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress)) - Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress); - if (SanOpts.has(SanitizerKind::MemTag)) - Fn->addFnAttr(llvm::Attribute::SanitizeMemTag); - if (SanOpts.has(SanitizerKind::Thread)) - Fn->addFnAttr(llvm::Attribute::SanitizeThread); - if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory)) - Fn->addFnAttr(llvm::Attribute::SanitizeMemory); + if (ShouldSkipSanitizerInstrumentation()) { + CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation); + } else { + // Apply sanitizer attributes to the function. + if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress)) + Fn->addFnAttr(llvm::Attribute::SanitizeAddress); + if (SanOpts.hasOneOf(SanitizerKind::HWAddress | + SanitizerKind::KernelHWAddress)) + Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress); + if (SanOpts.has(SanitizerKind::MemtagStack)) + Fn->addFnAttr(llvm::Attribute::SanitizeMemTag); + if (SanOpts.has(SanitizerKind::Thread)) + Fn->addFnAttr(llvm::Attribute::SanitizeThread); + if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory)) + Fn->addFnAttr(llvm::Attribute::SanitizeMemory); + } if (SanOpts.has(SanitizerKind::SafeStack)) Fn->addFnAttr(llvm::Attribute::SafeStack); if (SanOpts.has(SanitizerKind::ShadowCallStack)) @@ -846,8 +870,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups; if (FuncGroups > 1) { - auto FuncName = llvm::makeArrayRef<uint8_t>( - CurFn->getName().bytes_begin(), CurFn->getName().bytes_end()); + auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(), + CurFn->getName().bytes_end()); auto Group = crc32(FuncName) % FuncGroups; if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup && !AlwaysXRayAttr) @@ -855,9 +879,18 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, } } - if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone) - if (CGM.isProfileInstrExcluded(Fn, Loc)) + if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone) { + switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) { + case ProfileList::Skip: + Fn->addFnAttr(llvm::Attribute::SkipProfile); + break; + case ProfileList::Forbid: Fn->addFnAttr(llvm::Attribute::NoProfile); + break; + case ProfileList::Allow: + break; + } + } unsigned Count, Offset; if (const auto *Attr = @@ -873,6 +906,15 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, if (Offset) Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset)); } + // Instruct that functions for COFF/CodeView targets should start with a + // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64 + // backends as they don't need it -- instructions on these architectures are + // always atomically patchable at runtime. + if (CGM.getCodeGenOpts().HotPatch && + getContext().getTargetInfo().getTriple().isX86() && + getContext().getTargetInfo().getTriple().getEnvironment() != + llvm::Triple::CODE16) + Fn->addFnAttr("patchable-function", "prologue-short-redirect"); // Add no-jump-tables value. if (CGM.getCodeGenOpts().NoUseJumpTables) @@ -895,34 +937,43 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, if (D && D->hasAttr<NoProfileFunctionAttr>()) Fn->addFnAttr(llvm::Attribute::NoProfile); - if (FD && getLangOpts().OpenCL) { + if (D) { + // Function attributes take precedence over command line flags. + if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) { + switch (A->getThunkType()) { + case FunctionReturnThunksAttr::Kind::Keep: + break; + case FunctionReturnThunksAttr::Kind::Extern: + Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern); + break; + } + } else if (CGM.getCodeGenOpts().FunctionReturnThunks) + Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern); + } + + if (FD && (getLangOpts().OpenCL || + (getLangOpts().HIP && getLangOpts().CUDAIsDevice))) { // Add metadata for a kernel function. - EmitOpenCLKernelMetadata(FD, Fn); + EmitKernelMetadata(FD, Fn); } // If we are checking function types, emit a function type signature as // prologue data. - if (FD && getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function)) { + if (FD && SanOpts.has(SanitizerKind::Function)) { if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) { - // Remove any (C++17) exception specifications, to allow calling e.g. a - // noexcept function through a non-noexcept pointer. - auto ProtoTy = getContext().getFunctionTypeWithExceptionSpec( - FD->getType(), EST_None); - llvm::Constant *FTRTTIConst = - CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true); - llvm::Constant *FTRTTIConstEncoded = - EncodeAddrForUseInPrologue(Fn, FTRTTIConst); - llvm::Constant *PrologueStructElems[] = {PrologueSig, FTRTTIConstEncoded}; - llvm::Constant *PrologueStructConst = - llvm::ConstantStruct::getAnon(PrologueStructElems, /*Packed=*/true); - Fn->setPrologueData(PrologueStructConst); + llvm::LLVMContext &Ctx = Fn->getContext(); + llvm::MDBuilder MDB(Ctx); + Fn->setMetadata( + llvm::LLVMContext::MD_func_sanitize, + MDB.createRTTIPointerPrologue( + PrologueSig, getUBSanFunctionTypeHash(FD->getType()))); } } // If we're checking nullability, we need to know whether we can check the // return value. Initialize the flag to 'true' and refine it in EmitParmDecl. if (SanOpts.has(SanitizerKind::NullabilityReturn)) { - auto Nullability = FnRetTy->getNullability(getContext()); + auto Nullability = FnRetTy->getNullability(); if (Nullability && *Nullability == NullabilityKind::NonNull) { if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>())) @@ -947,10 +998,16 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>()))) Fn->addFnAttr(llvm::Attribute::NoRecurse); - if (FD) { - Builder.setIsFPConstrained(FD->hasAttr<StrictFPAttr>()); - if (FD->hasAttr<StrictFPAttr>()) - Fn->addFnAttr(llvm::Attribute::StrictFP); + llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode(); + llvm::fp::ExceptionBehavior FPExceptionBehavior = + ToConstrainedExceptMD(getLangOpts().getDefaultExceptionMode()); + Builder.setDefaultConstrainedRounding(RM); + Builder.setDefaultConstrainedExcept(FPExceptionBehavior); + if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) || + (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore || + RM != llvm::RoundingMode::NearestTiesToEven))) { + Builder.setIsFPConstrained(true); + Fn->addFnAttr(llvm::Attribute::StrictFP); } // If a custom alignment is used, force realigning to this alignment on @@ -959,6 +1016,10 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, CGM.getCodeGenOpts().StackAlignment)) Fn->addFnAttr("stackrealign"); + // "main" doesn't need to zero out call-used registers. + if (FD && FD->isMain()) + Fn->removeFnAttr("zero-call-used-regs"); + llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn); // Create a marker to make it easy to insert allocas into the entryblock @@ -975,7 +1036,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // precise source location of the checked return statement. if (requiresReturnValueCheck()) { ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr"); - InitTempAlloca(ReturnLocation, llvm::ConstantPointerNull::get(Int8PtrTy)); + Builder.CreateStore(llvm::ConstantPointerNull::get(Int8PtrTy), + ReturnLocation); } // Emit subprogram debug descriptor. @@ -983,16 +1045,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, // Reconstruct the type from the argument list so that implicit parameters, // such as 'this' and 'vtt', show up in the debug info. Preserve the calling // convention. - CallingConv CC = CallingConv::CC_C; - if (FD) - if (const auto *SrcFnTy = FD->getType()->getAs<FunctionType>()) - CC = SrcFnTy->getCallConv(); - SmallVector<QualType, 16> ArgTypes; - for (const VarDecl *VD : Args) - ArgTypes.push_back(VD->getType()); - QualType FnType = getContext().getFunctionType( - RetTy, ArgTypes, FunctionProtoType::ExtProtoInfo(CC)); - DI->emitFunctionStart(GD, Loc, StartLoc, FnType, CurFn, CurFuncIsThunk); + DI->emitFunctionStart(GD, Loc, StartLoc, + DI->getFunctionType(FD, RetTy, Args), CurFn, + CurFuncIsThunk); } if (ShouldInstrumentFunction()) { @@ -1044,7 +1099,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, Fn->addFnAttr("packed-stack"); } - if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX) + if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX && + !CGM.getDiags().isIgnored(diag::warn_fe_backend_frame_larger_than, Loc)) Fn->addFnAttr("warn-stack-size", std::to_string(CGM.getCodeGenOpts().WarnStackSize)); @@ -1061,13 +1117,13 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, auto AI = CurFn->arg_begin(); if (CurFnInfo->getReturnInfo().isSRetAfterThis()) ++AI; - ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign()); + ReturnValue = + Address(&*AI, ConvertType(RetTy), + CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull); if (!CurFnInfo->getReturnInfo().getIndirectByVal()) { - ReturnValuePointer = - CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr"); - Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast( - ReturnValue.getPointer(), Int8PtrTy), - ReturnValuePointer); + ReturnValuePointer = CreateDefaultAlignTempAlloca( + ReturnValue.getPointer()->getType(), "result.ptr"); + Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer); } } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca && !hasScalarEvaluationKind(CurFnInfo->getReturnType())) { @@ -1076,12 +1132,13 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; llvm::Value *Addr = Builder.CreateStructGEP( - EI->getType()->getPointerElementType(), &*EI, Idx); + CurFnInfo->getArgStruct(), &*EI, Idx); llvm::Type *Ty = cast<llvm::GetElementPtrInst>(Addr)->getResultElementType(); - ReturnValuePointer = Address(Addr, getPointerAlign()); + ReturnValuePointer = Address(Addr, Ty, getPointerAlign()); Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result"); - ReturnValue = Address(Addr, CGM.getNaturalTypeAlignment(RetTy)); + ReturnValue = Address(Addr, ConvertType(RetTy), + CGM.getNaturalTypeAlignment(RetTy), KnownNonNull); } else { ReturnValue = CreateIRTemp(RetTy, "retval"); @@ -1102,13 +1159,19 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, if (getLangOpts().OpenMP && CurCodeDecl) CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl); + // Handle emitting HLSL entry functions. + if (D && D->hasAttr<HLSLShaderAttr>()) + CGM.getHLSLRuntime().emitEntryFunction(FD, Fn); + EmitFunctionProlog(*CurFnInfo, CurFn, Args); - if (D && isa<CXXMethodDecl>(D) && cast<CXXMethodDecl>(D)->isInstance()) { - CGM.getCXXABI().EmitInstanceFunctionProlog(*this); - const CXXMethodDecl *MD = cast<CXXMethodDecl>(D); - if (MD->getParent()->isLambda() && - MD->getOverloadedOperator() == OO_Call) { + if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(D); + MD && !MD->isStatic()) { + bool IsInLambda = + MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call; + if (MD->isImplicitObjectMemberFunction()) + CGM.getCXXABI().EmitInstanceFunctionProlog(*this); + if (IsInLambda) { // We're in a lambda; figure out the captures. MD->getParent()->getCaptureFields(LambdaCaptureFields, LambdaThisCaptureField); @@ -1138,7 +1201,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, VLASizeMap[VAT->getSizeExpr()] = ExprArg; } } - } else { + } else if (MD->isImplicitObjectMemberFunction()) { // Not in a lambda; just use 'this' from the method. // FIXME: Should we generate a new load for each use of 'this'? The // fast register allocator would be happier... @@ -1151,11 +1214,10 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, SkippedChecks.set(SanitizerKind::ObjectSize, true); QualType ThisTy = MD->getThisType(); - // If this is the call operator of a lambda with no capture-default, it + // If this is the call operator of a lambda with no captures, it // may have a static invoker function, which may call this operator with // a null 'this' pointer. - if (isLambdaCallOperator(MD) && - MD->getParent()->getLambdaCaptureDefault() == LCD_None) + if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda()) SkippedChecks.set(SanitizerKind::Null, true); EmitTypeCheck( @@ -1165,27 +1227,26 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, } // If any of the arguments have a variably modified type, make sure to - // emit the type size. - for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); - i != e; ++i) { - const VarDecl *VD = *i; - - // Dig out the type as written from ParmVarDecls; it's unclear whether - // the standard (C99 6.9.1p10) requires this, but we're following the - // precedent set by gcc. - QualType Ty; - if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD)) - Ty = PVD->getOriginalType(); - else - Ty = VD->getType(); + // emit the type size, but only if the function is not naked. Naked functions + // have no prolog to run this evaluation. + if (!FD || !FD->hasAttr<NakedAttr>()) { + for (const VarDecl *VD : Args) { + // Dig out the type as written from ParmVarDecls; it's unclear whether + // the standard (C99 6.9.1p10) requires this, but we're following the + // precedent set by gcc. + QualType Ty; + if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD)) + Ty = PVD->getOriginalType(); + else + Ty = VD->getType(); - if (Ty->isVariablyModifiedType()) - EmitVariablyModifiedType(Ty); + if (Ty->isVariablyModifiedType()) + EmitVariablyModifiedType(Ty); + } } // Emit a location at the end of the prologue. if (CGDebugInfo *DI = getDebugInfo()) DI->EmitLocation(Builder, StartLoc); - // TODO: Do we need to handle this in two places like we do with // target-features/target-cpu? if (CurFuncDecl) @@ -1195,15 +1256,11 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy, void CodeGenFunction::EmitFunctionBody(const Stmt *Body) { incrementProfileCounter(Body); + maybeCreateMCDCCondBitmap(); if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body)) EmitCompoundStmtWithoutScope(*S); else EmitStmt(Body); - - // This is checked after emitting the function body so we know if there - // are any permitted infinite loops. - if (checkIfFunctionMustProgress()) - CurFn->addFnAttr(llvm::Attribute::MustProgress); } /// When instrumenting to collect profile data, the counts for some blocks @@ -1250,7 +1307,7 @@ QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD, QualType ResTy = FD->getReturnType(); const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD); - if (MD && MD->isInstance()) { + if (MD && MD->isImplicitObjectMemberFunction()) { if (CGM.getCXXABI().HasThisReturn(GD)) ResTy = MD->getThisType(); else if (CGM.getCXXABI().hasMostDerivedReturn(GD)) @@ -1275,7 +1332,7 @@ QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD, auto *Implicit = ImplicitParamDecl::Create( getContext(), Param->getDeclContext(), Param->getLocation(), - /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamDecl::Other); + /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamKind::Other); SizeArguments[Param] = Implicit; Args.push_back(Implicit); } @@ -1289,18 +1346,53 @@ QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD, void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, const CGFunctionInfo &FnInfo) { + assert(Fn && "generating code for null Function"); const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); CurGD = GD; FunctionArgList Args; QualType ResTy = BuildFunctionArgList(GD, Args); + if (FD->isInlineBuiltinDeclaration()) { + // When generating code for a builtin with an inline declaration, use a + // mangled name to hold the actual body, while keeping an external + // definition in case the function pointer is referenced somewhere. + std::string FDInlineName = (Fn->getName() + ".inline").str(); + llvm::Module *M = Fn->getParent(); + llvm::Function *Clone = M->getFunction(FDInlineName); + if (!Clone) { + Clone = llvm::Function::Create(Fn->getFunctionType(), + llvm::GlobalValue::InternalLinkage, + Fn->getAddressSpace(), FDInlineName, M); + Clone->addFnAttr(llvm::Attribute::AlwaysInline); + } + Fn->setLinkage(llvm::GlobalValue::ExternalLinkage); + Fn = Clone; + } else { + // Detect the unusual situation where an inline version is shadowed by a + // non-inline version. In that case we should pick the external one + // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way + // to detect that situation before we reach codegen, so do some late + // replacement. + for (const FunctionDecl *PD = FD->getPreviousDecl(); PD; + PD = PD->getPreviousDecl()) { + if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) { + std::string FDInlineName = (Fn->getName() + ".inline").str(); + llvm::Module *M = Fn->getParent(); + if (llvm::Function *Clone = M->getFunction(FDInlineName)) { + Clone->replaceAllUsesWith(Fn); + Clone->eraseFromParent(); + } + break; + } + } + } + // Check if we should generate debug info for this function. if (FD->hasAttr<NoDebugAttr>()) { // Clear non-distinct debug info that was possibly attached to the function // due to an earlier declaration without the nodebug attribute - if (Fn) - Fn->setSubprogram(nullptr); + Fn->setSubprogram(nullptr); // Disable debug info indefinitely for this function DebugInfo = nullptr; } @@ -1345,8 +1437,12 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, // Save parameters for coroutine function. if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body)) - for (const auto *ParamDecl : FD->parameters()) - FnArgs.push_back(ParamDecl); + llvm::append_range(FnArgs, FD->parameters()); + + // Ensure that the function adheres to the forward progress guarantee, which + // is required by certain optimizations. + if (checkIfFunctionMustProgress()) + CurFn->addFnAttr(llvm::Attribute::MustProgress); // Generate the body of the function. PGO.assignRegionCounters(GD, CurFn); @@ -1363,6 +1459,17 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, // The lambda static invoker function is special, because it forwards or // clones the body of the function call operator (but is actually static). EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD)); + } else if (isa<CXXMethodDecl>(FD) && + isLambdaCallOperator(cast<CXXMethodDecl>(FD)) && + !FnInfo.isDelegateCall() && + cast<CXXMethodDecl>(FD)->getParent()->getLambdaStaticInvoker() && + hasInAllocaArg(cast<CXXMethodDecl>(FD))) { + // If emitting a lambda with static invoker on X86 Windows, change + // the call operator body. + // Make sure that this is a call operator with an inalloca arg and check + // for delegate call to make sure this is the original call op and not the + // new forwarding function for the static invoker. + EmitLambdaInAllocaCallOpBody(cast<CXXMethodDecl>(FD)); } else if (FD->isDefaulted() && isa<CXXMethodDecl>(FD) && (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() || cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) { @@ -1390,7 +1497,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, llvm::Value *IsFalse = Builder.getFalse(); EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return), SanitizerHandler::MissingReturn, - EmitCheckSourceLocation(FD->getLocation()), None); + EmitCheckSourceLocation(FD->getLocation()), std::nullopt); } else if (ShouldEmitUnreachable) { if (CGM.getCodeGenOpts().OptimizationLevel == 0) EmitTrapCall(llvm::Intrinsic::trap); @@ -1495,6 +1602,13 @@ bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) { bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond, bool &ResultBool, bool AllowLabels) { + // If MC/DC is enabled, disable folding so that we can instrument all + // conditions to yield complete test vectors. We still keep track of + // folded conditions during region mapping and visualization. + if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() && + CGM.getCodeGenOpts().MCDCCoverage) + return false; + llvm::APSInt ResultInt; if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels)) return false; @@ -1523,16 +1637,20 @@ bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond, return true; } +/// Strip parentheses and simplistic logical-NOT operators. +const Expr *CodeGenFunction::stripCond(const Expr *C) { + while (const UnaryOperator *Op = dyn_cast<UnaryOperator>(C->IgnoreParens())) { + if (Op->getOpcode() != UO_LNot) + break; + C = Op->getSubExpr(); + } + return C->IgnoreParens(); +} + /// Determine whether the given condition is an instrumentable condition /// (i.e. no "&&" or "||"). bool CodeGenFunction::isInstrumentedCondition(const Expr *C) { - // Bypass simplistic logical-NOT operator before determining whether the - // condition contains any other logical operator. - if (const UnaryOperator *UnOp = dyn_cast<UnaryOperator>(C->IgnoreParens())) - if (UnOp->getOpcode() == UO_LNot) - C = UnOp->getSubExpr(); - - const BinaryOperator *BOp = dyn_cast<BinaryOperator>(C->IgnoreParens()); + const BinaryOperator *BOp = dyn_cast<BinaryOperator>(stripCond(C)); return (!BOp || !BOp->isLogicalOp()); } @@ -1549,9 +1667,9 @@ void CodeGenFunction::EmitBranchToCounterBlock( if (!InstrumentRegions || !isInstrumentedCondition(Cond)) return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH); - llvm::BasicBlock *ThenBlock = NULL; - llvm::BasicBlock *ElseBlock = NULL; - llvm::BasicBlock *NextBlock = NULL; + llvm::BasicBlock *ThenBlock = nullptr; + llvm::BasicBlock *ElseBlock = nullptr; + llvm::BasicBlock *NextBlock = nullptr; // Create the block we'll use to increment the appropriate counter. llvm::BasicBlock *CounterIncrBlock = createBasicBlock("lop.rhscnt"); @@ -1611,17 +1729,19 @@ void CodeGenFunction::EmitBranchToCounterBlock( /// statement) to the specified blocks. Based on the condition, this might try /// to simplify the codegen of the conditional based on the branch. /// \param LH The value of the likelihood attribute on the True branch. -void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, - llvm::BasicBlock *TrueBlock, - llvm::BasicBlock *FalseBlock, - uint64_t TrueCount, - Stmt::Likelihood LH) { +/// \param ConditionalOp Used by MC/DC code coverage to track the result of the +/// ConditionalOperator (ternary) through a recursive call for the operator's +/// LHS and RHS nodes. +void CodeGenFunction::EmitBranchOnBoolExpr( + const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, + uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp) { Cond = Cond->IgnoreParens(); if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) { - // Handle X && Y in a condition. if (CondBOp->getOpcode() == BO_LAnd) { + MCDCLogOpStack.push_back(CondBOp); + // If we have "1 && X", simplify the code. "0 && X" would have constant // folded if the case was simple enough. bool ConstantBool = false; @@ -1629,8 +1749,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, ConstantBool) { // br(1 && X) -> br(X). incrementProfileCounter(CondBOp); - return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock, - FalseBlock, TrueCount, LH); + EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock, + FalseBlock, TrueCount, LH); + MCDCLogOpStack.pop_back(); + return; } // If we have "X && 1", simplify the code to use an uncond branch. @@ -1638,8 +1760,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) && ConstantBool) { // br(X && 1) -> br(X). - return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock, - FalseBlock, TrueCount, LH, CondBOp); + EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock, + FalseBlock, TrueCount, LH, CondBOp); + MCDCLogOpStack.pop_back(); + return; } // Emit the LHS as a conditional. If the LHS conditional is false, we @@ -1668,11 +1792,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock, FalseBlock, TrueCount, LH); eval.end(*this); - + MCDCLogOpStack.pop_back(); return; } if (CondBOp->getOpcode() == BO_LOr) { + MCDCLogOpStack.push_back(CondBOp); + // If we have "0 || X", simplify the code. "1 || X" would have constant // folded if the case was simple enough. bool ConstantBool = false; @@ -1680,8 +1806,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, !ConstantBool) { // br(0 || X) -> br(X). incrementProfileCounter(CondBOp); - return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, - FalseBlock, TrueCount, LH); + EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, + FalseBlock, TrueCount, LH); + MCDCLogOpStack.pop_back(); + return; } // If we have "X || 0", simplify the code to use an uncond branch. @@ -1689,10 +1817,11 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) && !ConstantBool) { // br(X || 0) -> br(X). - return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock, - FalseBlock, TrueCount, LH, CondBOp); + EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock, + FalseBlock, TrueCount, LH, CondBOp); + MCDCLogOpStack.pop_back(); + return; } - // Emit the LHS as a conditional. If the LHS conditional is true, we // want to jump to the TrueBlock. llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false"); @@ -1723,14 +1852,20 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, RHSCount, LH); eval.end(*this); - + MCDCLogOpStack.pop_back(); return; } } if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) { // br(!x, t, f) -> br(x, f, t) - if (CondUOp->getOpcode() == UO_LNot) { + // Avoid doing this optimization when instrumenting a condition for MC/DC. + // LNot is taken as part of the condition for simplicity, and changing its + // sense negatively impacts test vector tracking. + bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() && + CGM.getCodeGenOpts().MCDCCoverage && + isInstrumentedCondition(Cond); + if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) { // Negate the count. uint64_t FalseCount = getCurrentProfileCount() - TrueCount; // The values of the enum are chosen to make this negation possible. @@ -1770,14 +1905,14 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, { ApplyDebugLocation DL(*this, Cond); EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock, - LHSScaledTrueCount, LH); + LHSScaledTrueCount, LH, CondOp); } cond.end(*this); cond.begin(*this); EmitBlock(RHSBlock); EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock, - TrueCount - LHSScaledTrueCount, LH); + TrueCount - LHSScaledTrueCount, LH, CondOp); cond.end(*this); return; @@ -1800,6 +1935,21 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, CondV = EvaluateExprAsBool(Cond); } + // If not at the top of the logical operator nest, update MCDC temp with the + // boolean result of the evaluated condition. + if (!MCDCLogOpStack.empty()) { + const Expr *MCDCBaseExpr = Cond; + // When a nested ConditionalOperator (ternary) is encountered in a boolean + // expression, MC/DC tracks the result of the ternary, and this is tied to + // the ConditionalOperator expression and not the ternary's LHS or RHS. If + // this is the case, the ConditionalOperator expression is passed through + // the ConditionalOp parameter and then used as the MCDC base expression. + if (ConditionalOp) + MCDCBaseExpr = ConditionalOp; + + maybeUpdateMCDCCondBitmap(MCDCBaseExpr, CondV); + } + llvm::MDNode *Weights = nullptr; llvm::MDNode *Unpredictable = nullptr; @@ -1851,8 +2001,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, llvm::Value *baseSizeInChars = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity()); - Address begin = - Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin"); + Address begin = dest.withElementType(CGF.Int8Ty); llvm::Value *end = Builder.CreateInBoundsGEP( begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end"); @@ -1871,7 +2020,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, dest.getAlignment().alignmentOfArrayElement(baseSize); // memcpy the individual element bit-pattern. - Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars, + Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src, baseSizeInChars, /*volatile*/ false); // Go to the next element. @@ -1896,9 +2045,8 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { } } - // Cast the dest ptr to the appropriate i8 pointer type. if (DestPtr.getElementType() != Int8Ty) - DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty); + DestPtr = DestPtr.withElementType(Int8Ty); // Get size and alignment info for this aggregate. CharUnits size = getContext().getTypeSizeInChars(Ty); @@ -1943,8 +2091,7 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { NullConstant, Twine()); CharUnits NullAlign = DestPtr.getAlignment(); NullVariable->setAlignment(NullAlign.getAsAlign()); - Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()), - NullAlign); + Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign); if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal); @@ -2058,11 +2205,12 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType, } llvm::Type *baseType = ConvertType(eltType); - addr = Builder.CreateElementBitCast(addr, baseType, "array.begin"); + addr = addr.withElementType(baseType); } else { // Create the actual GEP. addr = Address(Builder.CreateInBoundsGEP( addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"), + ConvertTypeForMem(eltType), addr.getAlignment()); } @@ -2153,15 +2301,19 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { case Type::ConstantMatrix: case Type::Record: case Type::Enum: - case Type::Elaborated: + case Type::Using: case Type::TemplateSpecialization: case Type::ObjCTypeParam: case Type::ObjCObject: case Type::ObjCInterface: case Type::ObjCObjectPointer: - case Type::ExtInt: + case Type::BitInt: llvm_unreachable("type class is never variably-modified!"); + case Type::Elaborated: + type = cast<ElaboratedType>(ty)->getNamedType(); + break; + case Type::Adjusted: type = cast<AdjustedType>(ty)->getAdjustedType(); break; @@ -2199,32 +2351,36 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { // Unknown size indication requires no size computation. // Otherwise, evaluate and record it. - if (const Expr *size = vat->getSizeExpr()) { + if (const Expr *sizeExpr = vat->getSizeExpr()) { // It's possible that we might have emitted this already, // e.g. with a typedef and a pointer to it. - llvm::Value *&entry = VLASizeMap[size]; + llvm::Value *&entry = VLASizeMap[sizeExpr]; if (!entry) { - llvm::Value *Size = EmitScalarExpr(size); + llvm::Value *size = EmitScalarExpr(sizeExpr); // C11 6.7.6.2p5: // If the size is an expression that is not an integer constant // expression [...] each time it is evaluated it shall have a value // greater than zero. - if (SanOpts.has(SanitizerKind::VLABound) && - size->getType()->isSignedIntegerType()) { + if (SanOpts.has(SanitizerKind::VLABound)) { SanitizerScope SanScope(this); - llvm::Value *Zero = llvm::Constant::getNullValue(Size->getType()); + llvm::Value *Zero = llvm::Constant::getNullValue(size->getType()); + clang::QualType SEType = sizeExpr->getType(); + llvm::Value *CheckCondition = + SEType->isSignedIntegerType() + ? Builder.CreateICmpSGT(size, Zero) + : Builder.CreateICmpUGT(size, Zero); llvm::Constant *StaticArgs[] = { - EmitCheckSourceLocation(size->getBeginLoc()), - EmitCheckTypeDescriptor(size->getType())}; - EmitCheck(std::make_pair(Builder.CreateICmpSGT(Size, Zero), - SanitizerKind::VLABound), - SanitizerHandler::VLABoundNotPositive, StaticArgs, Size); + EmitCheckSourceLocation(sizeExpr->getBeginLoc()), + EmitCheckTypeDescriptor(SEType)}; + EmitCheck(std::make_pair(CheckCondition, SanitizerKind::VLABound), + SanitizerHandler::VLABoundNotPositive, StaticArgs, size); } // Always zexting here would be wrong if it weren't // undefined behavior to have a negative bound. - entry = Builder.CreateIntCast(Size, SizeTy, /*signed*/ false); + // FIXME: What about when size's type is larger than size_t? + entry = Builder.CreateIntCast(size, SizeTy, /*signed*/ false); } } type = vat->getElementType(); @@ -2240,6 +2396,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) { case Type::TypeOf: case Type::UnaryTransform: case Type::Attributed: + case Type::BTFTagAttributed: case Type::SubstTemplateTypeParm: case Type::MacroQualified: // Keep walking after single level desugaring. @@ -2359,8 +2516,6 @@ void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue) { - if (auto *CE = dyn_cast<CastExpr>(E)) - E = CE->getSubExprAsWritten(); QualType Ty = E->getType(); SourceLocation Loc = E->getExprLoc(); @@ -2375,8 +2530,8 @@ llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn, const AnnotateAttr *Attr) { SmallVector<llvm::Value *, 5> Args = { AnnotatedVal, - Builder.CreateBitCast(CGM.EmitAnnotationString(AnnotationStr), Int8PtrTy), - Builder.CreateBitCast(CGM.EmitAnnotationUnit(Location), Int8PtrTy), + CGM.EmitAnnotationString(AnnotationStr), + CGM.EmitAnnotationUnit(Location), CGM.EmitAnnotationLineNo(Location), }; if (Attr) @@ -2386,12 +2541,10 @@ llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn, void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) { assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute"); - // FIXME We create a new bitcast for every annotation because that's what - // llvm-gcc was doing. for (const auto *I : D->specific_attrs<AnnotateAttr>()) - EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation), - Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()), - I->getAnnotation(), D->getLocation(), I); + EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation, + {V->getType(), CGM.ConstGlobalsPtrTy}), + V, I->getAnnotation(), D->getLocation(), I); } Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, @@ -2399,20 +2552,24 @@ Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D, assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute"); llvm::Value *V = Addr.getPointer(); llvm::Type *VTy = V->getType(); + auto *PTy = dyn_cast<llvm::PointerType>(VTy); + unsigned AS = PTy ? PTy->getAddressSpace() : 0; + llvm::PointerType *IntrinTy = + llvm::PointerType::get(CGM.getLLVMContext(), AS); llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation, - CGM.Int8PtrTy); + {IntrinTy, CGM.ConstGlobalsPtrTy}); for (const auto *I : D->specific_attrs<AnnotateAttr>()) { // FIXME Always emit the cast inst so we can differentiate between // annotation on the first field of a struct and annotation on the struct // itself. - if (VTy != CGM.Int8PtrTy) - V = Builder.CreateBitCast(V, CGM.Int8PtrTy); + if (VTy != IntrinTy) + V = Builder.CreateBitCast(V, IntrinTy); V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I); V = Builder.CreateBitCast(V, VTy); } - return Address(V, Addr.getAlignment()); + return Address(V, Addr.getElementType(), Addr.getAlignment()); } CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { } @@ -2433,7 +2590,7 @@ void CodeGenFunction::InsertHelper(llvm::Instruction *I, llvm::BasicBlock::iterator InsertPt) const { LoopStack.InsertHelper(I); if (IsSanitizerScope) - CGM.getSanitizerMetadata()->disableSanitizerForInstruction(I); + I->setNoSanitizeMetadata(); } void CGBuilderInserter::InsertHelper( @@ -2472,18 +2629,19 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc, std::string MissingFeature; llvm::StringMap<bool> CallerFeatureMap; CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD); + // When compiling in HipStdPar mode we have to be conservative in rejecting + // target specific features in the FE, and defer the possible error to the + // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is + // referenced by an accelerator executable function, we emit an error. + bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice; if (BuiltinID) { - StringRef FeatureList( - CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID)); - // Return if the builtin doesn't have any required features. - if (FeatureList.empty()) - return; - assert(FeatureList.find(' ') == StringRef::npos && - "Space in feature list"); - TargetFeatures TF(CallerFeatureMap); - if (!TF.hasRequiredFeatures(FeatureList)) + StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID)); + if (!Builtin::evaluateRequiredTargetFeatures( + FeatureList, CallerFeatureMap) && !IsHipStdPar) { CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature) - << TargetDecl->getDeclName() << FeatureList; + << TargetDecl->getDeclName() + << FeatureList; + } } else if (!TargetDecl->isMultiVersion() && TargetDecl->hasAttr<TargetAttr>()) { // Get the required features for the callee. @@ -2512,9 +2670,20 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc, return false; } return true; - })) + }) && !IsHipStdPar) CGM.getDiags().Report(Loc, diag::err_function_needs_feature) << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature; + } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) { + llvm::StringMap<bool> CalleeFeatureMap; + CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl); + + for (const auto &F : CalleeFeatureMap) { + if (F.getValue() && (!CallerFeatureMap.lookup(F.getKey()) || + !CallerFeatureMap.find(F.getKey())->getValue()) && + !IsHipStdPar) + CGM.getDiags().Report(Loc, diag::err_function_needs_feature) + << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey(); + } } } @@ -2527,12 +2696,41 @@ void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) { CGM.getSanStats().create(IRB, SSK); } -llvm::Value * -CodeGenFunction::FormResolverCondition(const MultiVersionResolverOption &RO) { +void CodeGenFunction::EmitKCFIOperandBundle( + const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) { + const FunctionProtoType *FP = + Callee.getAbstractInfo().getCalleeFunctionProtoType(); + if (FP) + Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar())); +} + +llvm::Value *CodeGenFunction::FormAArch64ResolverCondition( + const MultiVersionResolverOption &RO) { + llvm::SmallVector<StringRef, 8> CondFeatures; + for (const StringRef &Feature : RO.Conditions.Features) { + // Form condition for features which are not yet enabled in target + if (!getContext().getTargetInfo().hasFeature(Feature)) + CondFeatures.push_back(Feature); + } + if (!CondFeatures.empty()) { + return EmitAArch64CpuSupports(CondFeatures); + } + return nullptr; +} + +llvm::Value *CodeGenFunction::FormX86ResolverCondition( + const MultiVersionResolverOption &RO) { llvm::Value *Condition = nullptr; - if (!RO.Conditions.Architecture.empty()) - Condition = EmitX86CpuIs(RO.Conditions.Architecture); + if (!RO.Conditions.Architecture.empty()) { + StringRef Arch = RO.Conditions.Architecture; + // If arch= specifies an x86-64 micro-architecture level, test the feature + // with __builtin_cpu_supports, otherwise use __builtin_cpu_is. + if (Arch.starts_with("x86-64")) + Condition = EmitX86CpuSupports({Arch}); + else + Condition = EmitX86CpuIs(Arch); + } if (!RO.Conditions.Features.empty()) { llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features); @@ -2552,9 +2750,8 @@ static void CreateMultiVersionResolverReturn(CodeGenModule &CGM, return; } - llvm::SmallVector<llvm::Value *, 10> Args; - llvm::for_each(Resolver->args(), - [&](llvm::Argument &Arg) { Args.push_back(&Arg); }); + llvm::SmallVector<llvm::Value *, 10> Args( + llvm::make_pointer_range(Resolver->args())); llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args); Result->setTailCallKind(llvm::CallInst::TCK_MustTail); @@ -2567,8 +2764,72 @@ static void CreateMultiVersionResolverReturn(CodeGenModule &CGM, void CodeGenFunction::EmitMultiVersionResolver( llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) { - assert(getContext().getTargetInfo().getTriple().isX86() && - "Only implemented for x86 targets"); + + llvm::Triple::ArchType ArchType = + getContext().getTargetInfo().getTriple().getArch(); + + switch (ArchType) { + case llvm::Triple::x86: + case llvm::Triple::x86_64: + EmitX86MultiVersionResolver(Resolver, Options); + return; + case llvm::Triple::aarch64: + EmitAArch64MultiVersionResolver(Resolver, Options); + return; + + default: + assert(false && "Only implemented for x86 and AArch64 targets"); + } +} + +void CodeGenFunction::EmitAArch64MultiVersionResolver( + llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) { + assert(!Options.empty() && "No multiversion resolver options found"); + assert(Options.back().Conditions.Features.size() == 0 && + "Default case must be last"); + bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc(); + assert(SupportsIFunc && + "Multiversion resolver requires target IFUNC support"); + bool AArch64CpuInitialized = false; + llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver); + + for (const MultiVersionResolverOption &RO : Options) { + Builder.SetInsertPoint(CurBlock); + llvm::Value *Condition = FormAArch64ResolverCondition(RO); + + // The 'default' or 'all features enabled' case. + if (!Condition) { + CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function, + SupportsIFunc); + return; + } + + if (!AArch64CpuInitialized) { + Builder.SetInsertPoint(CurBlock, CurBlock->begin()); + EmitAArch64CpuInit(); + AArch64CpuInitialized = true; + Builder.SetInsertPoint(CurBlock); + } + + llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver); + CGBuilderTy RetBuilder(*this, RetBlock); + CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function, + SupportsIFunc); + CurBlock = createBasicBlock("resolver_else", Resolver); + Builder.CreateCondBr(Condition, RetBlock, CurBlock); + } + + // If no default, emit an unreachable. + Builder.SetInsertPoint(CurBlock); + llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap); + TrapCall->setDoesNotReturn(); + TrapCall->setDoesNotThrow(); + Builder.CreateUnreachable(); + Builder.ClearInsertionPoint(); +} + +void CodeGenFunction::EmitX86MultiVersionResolver( + llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) { bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc(); @@ -2579,7 +2840,7 @@ void CodeGenFunction::EmitMultiVersionResolver( for (const MultiVersionResolverOption &RO : Options) { Builder.SetInsertPoint(CurBlock); - llvm::Value *Condition = FormResolverCondition(RO); + llvm::Value *Condition = FormX86ResolverCondition(RO); // The 'default' or 'generic' case. if (!Condition) { @@ -2644,7 +2905,7 @@ void CodeGenFunction::emitAlignmentAssumptionCheck( SanitizerScope SanScope(this); if (!OffsetValue) - OffsetValue = Builder.getInt1(0); // no offset. + OffsetValue = Builder.getInt1(false); // no offset. llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc), EmitCheckSourceLocation(SecondaryLoc), @@ -2692,3 +2953,19 @@ CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond, } llvm_unreachable("Unknown Likelihood"); } + +llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec, + unsigned NumElementsDst, + const llvm::Twine &Name) { + auto *SrcTy = cast<llvm::FixedVectorType>(SrcVec->getType()); + unsigned NumElementsSrc = SrcTy->getNumElements(); + if (NumElementsSrc == NumElementsDst) + return SrcVec; + + std::vector<int> ShuffleMask(NumElementsDst, -1); + for (unsigned MaskIdx = 0; + MaskIdx < std::min<>(NumElementsDst, NumElementsSrc); ++MaskIdx) + ShuffleMask[MaskIdx] = MaskIdx; + + return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name); +} |