diff options
Diffstat (limited to 'contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp')
| -rw-r--r-- | contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp | 2498 |
1 files changed, 1840 insertions, 658 deletions
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp index 49a1396b58e3..1280bcd36de9 100644 --- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp +++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp @@ -11,28 +11,29 @@ //===----------------------------------------------------------------------===// #include "CodeGenModule.h" +#include "ABIInfo.h" #include "CGBlocks.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" #include "CGCall.h" #include "CGDebugInfo.h" +#include "CGHLSLRuntime.h" #include "CGObjCRuntime.h" #include "CGOpenCLRuntime.h" #include "CGOpenMPRuntime.h" -#include "CGOpenMPRuntimeAMDGCN.h" -#include "CGOpenMPRuntimeNVPTX.h" +#include "CGOpenMPRuntimeGPU.h" #include "CodeGenFunction.h" #include "CodeGenPGO.h" #include "ConstantEmitter.h" #include "CoverageMappingGen.h" #include "TargetInfo.h" #include "clang/AST/ASTContext.h" +#include "clang/AST/ASTLambda.h" #include "clang/AST/CharUnits.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclObjC.h" #include "clang/AST/DeclTemplate.h" #include "clang/AST/Mangle.h" -#include "clang/AST/RecordLayout.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/AST/StmtVisitor.h" #include "clang/Basic/Builtins.h" @@ -44,12 +45,15 @@ #include "clang/Basic/SourceManager.h" #include "clang/Basic/TargetInfo.h" #include "clang/Basic/Version.h" +#include "clang/CodeGen/BackendUtil.h" #include "clang/CodeGen/ConstantInitBuilder.h" #include "clang/Frontend/FrontendDiagnostic.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" +#include "llvm/IR/AttributeMask.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Intrinsics.h" @@ -57,20 +61,24 @@ #include "llvm/IR/Module.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProfReader.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/CRC.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MD5.h" #include "llvm/Support/TimeProfiler.h" +#include "llvm/Support/xxhash.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/TargetParser/X86TargetParser.h" +#include <optional> using namespace clang; using namespace CodeGen; static llvm::cl::opt<bool> LimitedCoverage( - "limited-coverage-experimental", llvm::cl::ZeroOrMore, llvm::cl::Hidden, - llvm::cl::desc("Emit limited coverage mapping information (experimental)"), - llvm::cl::init(false)); + "limited-coverage-experimental", llvm::cl::Hidden, + llvm::cl::desc("Emit limited coverage mapping information (experimental)")); static const char AnnotationSection[] = "llvm.metadata"; @@ -94,12 +102,237 @@ static CGCXXABI *createCXXABI(CodeGenModule &CGM) { llvm_unreachable("invalid C++ ABI kind"); } -CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, +static std::unique_ptr<TargetCodeGenInfo> +createTargetCodeGenInfo(CodeGenModule &CGM) { + const TargetInfo &Target = CGM.getTarget(); + const llvm::Triple &Triple = Target.getTriple(); + const CodeGenOptions &CodeGenOpts = CGM.getCodeGenOpts(); + + switch (Triple.getArch()) { + default: + return createDefaultTargetCodeGenInfo(CGM); + + case llvm::Triple::le32: + return createPNaClTargetCodeGenInfo(CGM); + case llvm::Triple::m68k: + return createM68kTargetCodeGenInfo(CGM); + case llvm::Triple::mips: + case llvm::Triple::mipsel: + if (Triple.getOS() == llvm::Triple::NaCl) + return createPNaClTargetCodeGenInfo(CGM); + return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/true); + + case llvm::Triple::mips64: + case llvm::Triple::mips64el: + return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/false); + + case llvm::Triple::avr: { + // For passing parameters, R8~R25 are used on avr, and R18~R25 are used + // on avrtiny. For passing return value, R18~R25 are used on avr, and + // R22~R25 are used on avrtiny. + unsigned NPR = Target.getABI() == "avrtiny" ? 6 : 18; + unsigned NRR = Target.getABI() == "avrtiny" ? 4 : 8; + return createAVRTargetCodeGenInfo(CGM, NPR, NRR); + } + + case llvm::Triple::aarch64: + case llvm::Triple::aarch64_32: + case llvm::Triple::aarch64_be: { + AArch64ABIKind Kind = AArch64ABIKind::AAPCS; + if (Target.getABI() == "darwinpcs") + Kind = AArch64ABIKind::DarwinPCS; + else if (Triple.isOSWindows()) + return createWindowsAArch64TargetCodeGenInfo(CGM, AArch64ABIKind::Win64); + + return createAArch64TargetCodeGenInfo(CGM, Kind); + } + + case llvm::Triple::wasm32: + case llvm::Triple::wasm64: { + WebAssemblyABIKind Kind = WebAssemblyABIKind::MVP; + if (Target.getABI() == "experimental-mv") + Kind = WebAssemblyABIKind::ExperimentalMV; + return createWebAssemblyTargetCodeGenInfo(CGM, Kind); + } + + case llvm::Triple::arm: + case llvm::Triple::armeb: + case llvm::Triple::thumb: + case llvm::Triple::thumbeb: { + if (Triple.getOS() == llvm::Triple::Win32) + return createWindowsARMTargetCodeGenInfo(CGM, ARMABIKind::AAPCS_VFP); + + ARMABIKind Kind = ARMABIKind::AAPCS; + StringRef ABIStr = Target.getABI(); + if (ABIStr == "apcs-gnu") + Kind = ARMABIKind::APCS; + else if (ABIStr == "aapcs16") + Kind = ARMABIKind::AAPCS16_VFP; + else if (CodeGenOpts.FloatABI == "hard" || + (CodeGenOpts.FloatABI != "soft" && + (Triple.getEnvironment() == llvm::Triple::GNUEABIHF || + Triple.getEnvironment() == llvm::Triple::MuslEABIHF || + Triple.getEnvironment() == llvm::Triple::EABIHF))) + Kind = ARMABIKind::AAPCS_VFP; + + return createARMTargetCodeGenInfo(CGM, Kind); + } + + case llvm::Triple::ppc: { + if (Triple.isOSAIX()) + return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/false); + + bool IsSoftFloat = + CodeGenOpts.FloatABI == "soft" || Target.hasFeature("spe"); + return createPPC32TargetCodeGenInfo(CGM, IsSoftFloat); + } + case llvm::Triple::ppcle: { + bool IsSoftFloat = CodeGenOpts.FloatABI == "soft"; + return createPPC32TargetCodeGenInfo(CGM, IsSoftFloat); + } + case llvm::Triple::ppc64: + if (Triple.isOSAIX()) + return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/true); + + if (Triple.isOSBinFormatELF()) { + PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv1; + if (Target.getABI() == "elfv2") + Kind = PPC64_SVR4_ABIKind::ELFv2; + bool IsSoftFloat = CodeGenOpts.FloatABI == "soft"; + + return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, IsSoftFloat); + } + return createPPC64TargetCodeGenInfo(CGM); + case llvm::Triple::ppc64le: { + assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!"); + PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv2; + if (Target.getABI() == "elfv1") + Kind = PPC64_SVR4_ABIKind::ELFv1; + bool IsSoftFloat = CodeGenOpts.FloatABI == "soft"; + + return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, IsSoftFloat); + } + + case llvm::Triple::nvptx: + case llvm::Triple::nvptx64: + return createNVPTXTargetCodeGenInfo(CGM); + + case llvm::Triple::msp430: + return createMSP430TargetCodeGenInfo(CGM); + + case llvm::Triple::riscv32: + case llvm::Triple::riscv64: { + StringRef ABIStr = Target.getABI(); + unsigned XLen = Target.getPointerWidth(LangAS::Default); + unsigned ABIFLen = 0; + if (ABIStr.ends_with("f")) + ABIFLen = 32; + else if (ABIStr.ends_with("d")) + ABIFLen = 64; + bool EABI = ABIStr.ends_with("e"); + return createRISCVTargetCodeGenInfo(CGM, XLen, ABIFLen, EABI); + } + + case llvm::Triple::systemz: { + bool SoftFloat = CodeGenOpts.FloatABI == "soft"; + bool HasVector = !SoftFloat && Target.getABI() == "vector"; + return createSystemZTargetCodeGenInfo(CGM, HasVector, SoftFloat); + } + + case llvm::Triple::tce: + case llvm::Triple::tcele: + return createTCETargetCodeGenInfo(CGM); + + case llvm::Triple::x86: { + bool IsDarwinVectorABI = Triple.isOSDarwin(); + bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing(); + + if (Triple.getOS() == llvm::Triple::Win32) { + return createWinX86_32TargetCodeGenInfo( + CGM, IsDarwinVectorABI, IsWin32FloatStructABI, + CodeGenOpts.NumRegisterParameters); + } + return createX86_32TargetCodeGenInfo( + CGM, IsDarwinVectorABI, IsWin32FloatStructABI, + CodeGenOpts.NumRegisterParameters, CodeGenOpts.FloatABI == "soft"); + } + + case llvm::Triple::x86_64: { + StringRef ABI = Target.getABI(); + X86AVXABILevel AVXLevel = (ABI == "avx512" ? X86AVXABILevel::AVX512 + : ABI == "avx" ? X86AVXABILevel::AVX + : X86AVXABILevel::None); + + switch (Triple.getOS()) { + case llvm::Triple::Win32: + return createWinX86_64TargetCodeGenInfo(CGM, AVXLevel); + default: + return createX86_64TargetCodeGenInfo(CGM, AVXLevel); + } + } + case llvm::Triple::hexagon: + return createHexagonTargetCodeGenInfo(CGM); + case llvm::Triple::lanai: + return createLanaiTargetCodeGenInfo(CGM); + case llvm::Triple::r600: + return createAMDGPUTargetCodeGenInfo(CGM); + case llvm::Triple::amdgcn: + return createAMDGPUTargetCodeGenInfo(CGM); + case llvm::Triple::sparc: + return createSparcV8TargetCodeGenInfo(CGM); + case llvm::Triple::sparcv9: + return createSparcV9TargetCodeGenInfo(CGM); + case llvm::Triple::xcore: + return createXCoreTargetCodeGenInfo(CGM); + case llvm::Triple::arc: + return createARCTargetCodeGenInfo(CGM); + case llvm::Triple::spir: + case llvm::Triple::spir64: + return createCommonSPIRTargetCodeGenInfo(CGM); + case llvm::Triple::spirv32: + case llvm::Triple::spirv64: + return createSPIRVTargetCodeGenInfo(CGM); + case llvm::Triple::ve: + return createVETargetCodeGenInfo(CGM); + case llvm::Triple::csky: { + bool IsSoftFloat = !Target.hasFeature("hard-float-abi"); + bool hasFP64 = + Target.hasFeature("fpuv2_df") || Target.hasFeature("fpuv3_df"); + return createCSKYTargetCodeGenInfo(CGM, IsSoftFloat ? 0 + : hasFP64 ? 64 + : 32); + } + case llvm::Triple::bpfeb: + case llvm::Triple::bpfel: + return createBPFTargetCodeGenInfo(CGM); + case llvm::Triple::loongarch32: + case llvm::Triple::loongarch64: { + StringRef ABIStr = Target.getABI(); + unsigned ABIFRLen = 0; + if (ABIStr.ends_with("f")) + ABIFRLen = 32; + else if (ABIStr.ends_with("d")) + ABIFRLen = 64; + return createLoongArchTargetCodeGenInfo( + CGM, Target.getPointerWidth(LangAS::Default), ABIFRLen); + } + } +} + +const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { + if (!TheTargetCodeGenInfo) + TheTargetCodeGenInfo = createTargetCodeGenInfo(*this); + return *TheTargetCodeGenInfo; +} + +CodeGenModule::CodeGenModule(ASTContext &C, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + const HeaderSearchOptions &HSO, const PreprocessorOptions &PPO, const CodeGenOptions &CGO, llvm::Module &M, DiagnosticsEngine &diags, CoverageSourceInfo *CoverageInfo) - : Context(C), LangOpts(C.getLangOpts()), HeaderSearchOpts(HSO), + : Context(C), LangOpts(C.getLangOpts()), FS(FS), HeaderSearchOpts(HSO), PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags), Target(C.getTargetInfo()), ABI(createCXXABI(*this)), VMContext(M.getContext()), Types(*this), VTables(*this), @@ -116,9 +349,10 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, BFloatTy = llvm::Type::getBFloatTy(LLVMContext); FloatTy = llvm::Type::getFloatTy(LLVMContext); DoubleTy = llvm::Type::getDoubleTy(LLVMContext); - PointerWidthInBits = C.getTargetInfo().getPointerWidth(0); + PointerWidthInBits = C.getTargetInfo().getPointerWidth(LangAS::Default); PointerAlignInBytes = - C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity(); + C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(LangAS::Default)) + .getQuantity(); SizeSizeInBytes = C.toCharUnitsFromBits(C.getTargetInfo().getMaxPointerWidth()).getQuantity(); IntAlignInBytes = @@ -128,12 +362,23 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth()); IntPtrTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getMaxPointerWidth()); - Int8PtrTy = Int8Ty->getPointerTo(0); - Int8PtrPtrTy = Int8PtrTy->getPointerTo(0); - AllocaInt8PtrTy = Int8Ty->getPointerTo( - M.getDataLayout().getAllocaAddrSpace()); + Int8PtrTy = llvm::PointerType::get(LLVMContext, 0); + const llvm::DataLayout &DL = M.getDataLayout(); + AllocaInt8PtrTy = + llvm::PointerType::get(LLVMContext, DL.getAllocaAddrSpace()); + GlobalsInt8PtrTy = + llvm::PointerType::get(LLVMContext, DL.getDefaultGlobalsAddressSpace()); + ConstGlobalsPtrTy = llvm::PointerType::get( + LLVMContext, C.getTargetAddressSpace(GetGlobalConstantAddressSpace())); ASTAllocaAddressSpace = getTargetCodeGenInfo().getASTAllocaAddressSpace(); + // Build C++20 Module initializers. + // TODO: Add Microsoft here once we know the mangling required for the + // initializers. + CXX20ModuleInits = + LangOpts.CPlusPlusModules && getCXXABI().getMangleContext().getKind() == + ItaniumMangleContext::MK_Itanium; + RuntimeCC = getTargetCodeGenInfo().getABIInfo().getRuntimeCC(); if (LangOpts.ObjC) @@ -144,6 +389,8 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, createOpenMPRuntime(); if (LangOpts.CUDA) createCUDARuntime(); + if (LangOpts.HLSL) + createHLSLRuntime(); // Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0. if (LangOpts.Sanitize.has(SanitizerKind::Thread) || @@ -153,8 +400,9 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, // If debug info or coverage generation is enabled, create the CGDebugInfo // object. - if (CodeGenOpts.getDebugInfo() != codegenoptions::NoDebugInfo || - CodeGenOpts.EmitGcovArcs || CodeGenOpts.EmitGcovNotes) + if (CodeGenOpts.getDebugInfo() != llvm::codegenoptions::NoDebugInfo || + CodeGenOpts.CoverageNotesFile.size() || + CodeGenOpts.CoverageDataFile.size()) DebugInfo.reset(new CGDebugInfo(*this)); Block.GlobalUniqueCount = 0; @@ -164,16 +412,13 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, if (CodeGenOpts.hasProfileClangUse()) { auto ReaderOrErr = llvm::IndexedInstrProfReader::create( - CodeGenOpts.ProfileInstrumentUsePath, CodeGenOpts.ProfileRemappingFile); - if (auto E = ReaderOrErr.takeError()) { - unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, - "Could not read profile %0: %1"); - llvm::handleAllErrors(std::move(E), [&](const llvm::ErrorInfoBase &EI) { - getDiags().Report(DiagID) << CodeGenOpts.ProfileInstrumentUsePath - << EI.message(); - }); - } else - PGOReader = std::move(ReaderOrErr.get()); + CodeGenOpts.ProfileInstrumentUsePath, *FS, + CodeGenOpts.ProfileRemappingFile); + // We're checking for profile read errors in CompilerInvocation, so if + // there was an error it should've already been caught. If it hasn't been + // somehow, trip an assertion. + assert(ReaderOrErr); + PGOReader = std::move(ReaderOrErr.get()); } // If coverage mapping generation is enabled, create the @@ -191,22 +436,7 @@ CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO, Path = Entry.second + Path.substr(Entry.first.size()); break; } - llvm::MD5 Md5; - Md5.update(Path); - llvm::MD5::MD5Result R; - Md5.final(R); - SmallString<32> Str; - llvm::MD5::stringifyResult(R, Str); - // Convert MD5hash to Decimal. Demangler suffixes can either contain - // numbers or characters but not both. - llvm::APInt IntHash(128, Str.str(), 16); - // Prepend "__uniq" before the hash for tools like profilers to understand - // that this symbol is of internal linkage type. The "__uniq" is the - // pre-determined prefix that is used to tell tools that this symbol was - // created with -funique-internal-linakge-symbols and the tools can strip or - // keep the prefix as needed. - ModuleNameHash = (Twine(".__uniq.") + - Twine(toString(IntHash, /* Radix = */ 10, /* Signed = */false))).str(); + ModuleNameHash = llvm::getUniqueInternalLinkagePostfix(Path); } } @@ -242,14 +472,10 @@ void CodeGenModule::createOpenMPRuntime() { switch (getTriple().getArch()) { case llvm::Triple::nvptx: case llvm::Triple::nvptx64: - assert(getLangOpts().OpenMPIsDevice && - "OpenMP NVPTX is only prepared to deal with device code."); - OpenMPRuntime.reset(new CGOpenMPRuntimeNVPTX(*this)); - break; case llvm::Triple::amdgcn: - assert(getLangOpts().OpenMPIsDevice && - "OpenMP AMDGCN is only prepared to deal with device code."); - OpenMPRuntime.reset(new CGOpenMPRuntimeAMDGCN(*this)); + assert(getLangOpts().OpenMPIsTargetDevice && + "OpenMP AMDGPU/NVPTX is only prepared to deal with device code."); + OpenMPRuntime.reset(new CGOpenMPRuntimeGPU(*this)); break; default: if (LangOpts.OpenMPSimd) @@ -264,13 +490,17 @@ void CodeGenModule::createCUDARuntime() { CUDARuntime.reset(CreateNVCUDARuntime(*this)); } +void CodeGenModule::createHLSLRuntime() { + HLSLRuntime.reset(new CGHLSLRuntime(*this)); +} + void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) { Replacements[Name] = C; } void CodeGenModule::applyReplacements() { for (auto &I : Replacements) { - StringRef MangledName = I.first(); + StringRef MangledName = I.first; llvm::Constant *Replacement = I.second; llvm::GlobalValue *Entry = GetGlobalValue(MangledName); if (!Entry) @@ -315,22 +545,83 @@ void CodeGenModule::applyGlobalValReplacements() { // This is only used in aliases that we created and we know they have a // linear structure. -static const llvm::GlobalObject *getAliasedGlobal( - const llvm::GlobalIndirectSymbol &GIS) { - llvm::SmallPtrSet<const llvm::GlobalIndirectSymbol*, 4> Visited; - const llvm::Constant *C = &GIS; - for (;;) { - C = C->stripPointerCasts(); - if (auto *GO = dyn_cast<llvm::GlobalObject>(C)) - return GO; - // stripPointerCasts will not walk over weak aliases. - auto *GIS2 = dyn_cast<llvm::GlobalIndirectSymbol>(C); - if (!GIS2) - return nullptr; - if (!Visited.insert(GIS2).second) - return nullptr; - C = GIS2->getIndirectSymbol(); +static const llvm::GlobalValue *getAliasedGlobal(const llvm::GlobalValue *GV) { + const llvm::Constant *C; + if (auto *GA = dyn_cast<llvm::GlobalAlias>(GV)) + C = GA->getAliasee(); + else if (auto *GI = dyn_cast<llvm::GlobalIFunc>(GV)) + C = GI->getResolver(); + else + return GV; + + const auto *AliaseeGV = dyn_cast<llvm::GlobalValue>(C->stripPointerCasts()); + if (!AliaseeGV) + return nullptr; + + const llvm::GlobalValue *FinalGV = AliaseeGV->getAliaseeObject(); + if (FinalGV == GV) + return nullptr; + + return FinalGV; +} + +static bool checkAliasedGlobal( + const ASTContext &Context, DiagnosticsEngine &Diags, SourceLocation Location, + bool IsIFunc, const llvm::GlobalValue *Alias, const llvm::GlobalValue *&GV, + const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames, + SourceRange AliasRange) { + GV = getAliasedGlobal(Alias); + if (!GV) { + Diags.Report(Location, diag::err_cyclic_alias) << IsIFunc; + return false; + } + + if (GV->hasCommonLinkage()) { + const llvm::Triple &Triple = Context.getTargetInfo().getTriple(); + if (Triple.getObjectFormat() == llvm::Triple::XCOFF) { + Diags.Report(Location, diag::err_alias_to_common); + return false; + } + } + + if (GV->isDeclaration()) { + Diags.Report(Location, diag::err_alias_to_undefined) << IsIFunc << IsIFunc; + Diags.Report(Location, diag::note_alias_requires_mangled_name) + << IsIFunc << IsIFunc; + // Provide a note if the given function is not found and exists as a + // mangled name. + for (const auto &[Decl, Name] : MangledDeclNames) { + if (const auto *ND = dyn_cast<NamedDecl>(Decl.getDecl())) { + if (ND->getName() == GV->getName()) { + Diags.Report(Location, diag::note_alias_mangled_name_alternative) + << Name + << FixItHint::CreateReplacement( + AliasRange, + (Twine(IsIFunc ? "ifunc" : "alias") + "(\"" + Name + "\")") + .str()); + } + } + } + return false; } + + if (IsIFunc) { + // Check resolver function type. + const auto *F = dyn_cast<llvm::Function>(GV); + if (!F) { + Diags.Report(Location, diag::err_alias_to_undefined) + << IsIFunc << IsIFunc; + return false; + } + + llvm::FunctionType *FTy = F->getFunctionType(); + if (!FTy->getReturnType()->isPointerTy()) { + Diags.Report(Location, diag::err_ifunc_resolver_return); + return false; + } + } + + return true; } void CodeGenModule::checkAliases() { @@ -342,32 +633,27 @@ void CodeGenModule::checkAliases() { for (const GlobalDecl &GD : Aliases) { const auto *D = cast<ValueDecl>(GD.getDecl()); SourceLocation Location; + SourceRange Range; bool IsIFunc = D->hasAttr<IFuncAttr>(); - if (const Attr *A = D->getDefiningAttr()) + if (const Attr *A = D->getDefiningAttr()) { Location = A->getLocation(); - else + Range = A->getRange(); + } else llvm_unreachable("Not an alias or ifunc?"); + StringRef MangledName = getMangledName(GD); - llvm::GlobalValue *Entry = GetGlobalValue(MangledName); - auto *Alias = cast<llvm::GlobalIndirectSymbol>(Entry); - const llvm::GlobalValue *GV = getAliasedGlobal(*Alias); - if (!GV) { - Error = true; - Diags.Report(Location, diag::err_cyclic_alias) << IsIFunc; - } else if (GV->isDeclaration()) { + llvm::GlobalValue *Alias = GetGlobalValue(MangledName); + const llvm::GlobalValue *GV = nullptr; + if (!checkAliasedGlobal(getContext(), Diags, Location, IsIFunc, Alias, GV, + MangledDeclNames, Range)) { Error = true; - Diags.Report(Location, diag::err_alias_to_undefined) - << IsIFunc << IsIFunc; - } else if (IsIFunc) { - // Check resolver function type. - llvm::FunctionType *FTy = dyn_cast<llvm::FunctionType>( - GV->getType()->getPointerElementType()); - assert(FTy); - if (!FTy->getReturnType()->isPointerTy()) - Diags.Report(Location, diag::err_ifunc_resolver_return); + continue; } - llvm::Constant *Aliasee = Alias->getIndirectSymbol(); + llvm::Constant *Aliasee = + IsIFunc ? cast<llvm::GlobalIFunc>(Alias)->getResolver() + : cast<llvm::GlobalAlias>(Alias)->getAliasee(); + llvm::GlobalValue *AliaseeGV; if (auto CE = dyn_cast<llvm::ConstantExpr>(Aliasee)) AliaseeGV = cast<llvm::GlobalValue>(CE->getOperand(0)); @@ -386,13 +672,17 @@ void CodeGenModule::checkAliases() { // compatibility with gcc we implement it by just pointing the alias // to its aliasee's aliasee. We also warn, since the user is probably // expecting the link to be weak. - if (auto GA = dyn_cast<llvm::GlobalIndirectSymbol>(AliaseeGV)) { + if (auto *GA = dyn_cast<llvm::GlobalAlias>(AliaseeGV)) { if (GA->isInterposable()) { Diags.Report(Location, diag::warn_alias_to_weak_alias) << GV->getName() << GA->getName() << IsIFunc; Aliasee = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast( - GA->getIndirectSymbol(), Alias->getType()); - Alias->setIndirectSymbol(Aliasee); + GA->getAliasee(), Alias->getType()); + + if (IsIFunc) + cast<llvm::GlobalIFunc>(Alias)->setResolver(Aliasee); + else + cast<llvm::GlobalAlias>(Alias)->setAliasee(Aliasee); } } } @@ -401,8 +691,7 @@ void CodeGenModule::checkAliases() { for (const GlobalDecl &GD : Aliases) { StringRef MangledName = getMangledName(GD); - llvm::GlobalValue *Entry = GetGlobalValue(MangledName); - auto *Alias = cast<llvm::GlobalIndirectSymbol>(Entry); + llvm::GlobalValue *Alias = GetGlobalValue(MangledName); Alias->replaceAllUsesWith(llvm::UndefValue::get(Alias->getType())); Alias->eraseFromParent(); } @@ -410,6 +699,8 @@ void CodeGenModule::checkAliases() { void CodeGenModule::clear() { DeferredDeclsToEmit.clear(); + EmittedDeferredDecls.clear(); + DeferredAnnotations.clear(); if (OpenMPRuntime) OpenMPRuntime->clear(); } @@ -431,56 +722,109 @@ void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags, } } +static std::optional<llvm::GlobalValue::VisibilityTypes> +getLLVMVisibility(clang::LangOptions::VisibilityFromDLLStorageClassKinds K) { + // Map to LLVM visibility. + switch (K) { + case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Keep: + return std::nullopt; + case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Default: + return llvm::GlobalValue::DefaultVisibility; + case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Hidden: + return llvm::GlobalValue::HiddenVisibility; + case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Protected: + return llvm::GlobalValue::ProtectedVisibility; + } + llvm_unreachable("unknown option value!"); +} + +void setLLVMVisibility(llvm::GlobalValue &GV, + std::optional<llvm::GlobalValue::VisibilityTypes> V) { + if (!V) + return; + + // Reset DSO locality before setting the visibility. This removes + // any effects that visibility options and annotations may have + // had on the DSO locality. Setting the visibility will implicitly set + // appropriate globals to DSO Local; however, this will be pessimistic + // w.r.t. to the normal compiler IRGen. + GV.setDSOLocal(false); + GV.setVisibility(*V); +} + static void setVisibilityFromDLLStorageClass(const clang::LangOptions &LO, llvm::Module &M) { if (!LO.VisibilityFromDLLStorageClass) return; - llvm::GlobalValue::VisibilityTypes DLLExportVisibility = - CodeGenModule::GetLLVMVisibility(LO.getDLLExportVisibility()); - llvm::GlobalValue::VisibilityTypes NoDLLStorageClassVisibility = - CodeGenModule::GetLLVMVisibility(LO.getNoDLLStorageClassVisibility()); - llvm::GlobalValue::VisibilityTypes ExternDeclDLLImportVisibility = - CodeGenModule::GetLLVMVisibility(LO.getExternDeclDLLImportVisibility()); - llvm::GlobalValue::VisibilityTypes ExternDeclNoDLLStorageClassVisibility = - CodeGenModule::GetLLVMVisibility( - LO.getExternDeclNoDLLStorageClassVisibility()); + std::optional<llvm::GlobalValue::VisibilityTypes> DLLExportVisibility = + getLLVMVisibility(LO.getDLLExportVisibility()); + + std::optional<llvm::GlobalValue::VisibilityTypes> + NoDLLStorageClassVisibility = + getLLVMVisibility(LO.getNoDLLStorageClassVisibility()); + + std::optional<llvm::GlobalValue::VisibilityTypes> + ExternDeclDLLImportVisibility = + getLLVMVisibility(LO.getExternDeclDLLImportVisibility()); + + std::optional<llvm::GlobalValue::VisibilityTypes> + ExternDeclNoDLLStorageClassVisibility = + getLLVMVisibility(LO.getExternDeclNoDLLStorageClassVisibility()); for (llvm::GlobalValue &GV : M.global_values()) { if (GV.hasAppendingLinkage() || GV.hasLocalLinkage()) continue; - // Reset DSO locality before setting the visibility. This removes - // any effects that visibility options and annotations may have - // had on the DSO locality. Setting the visibility will implicitly set - // appropriate globals to DSO Local; however, this will be pessimistic - // w.r.t. to the normal compiler IRGen. - GV.setDSOLocal(false); - - if (GV.isDeclarationForLinker()) { - GV.setVisibility(GV.getDLLStorageClass() == - llvm::GlobalValue::DLLImportStorageClass - ? ExternDeclDLLImportVisibility - : ExternDeclNoDLLStorageClassVisibility); - } else { - GV.setVisibility(GV.getDLLStorageClass() == - llvm::GlobalValue::DLLExportStorageClass - ? DLLExportVisibility - : NoDLLStorageClassVisibility); - } + if (GV.isDeclarationForLinker()) + setLLVMVisibility(GV, GV.getDLLStorageClass() == + llvm::GlobalValue::DLLImportStorageClass + ? ExternDeclDLLImportVisibility + : ExternDeclNoDLLStorageClassVisibility); + else + setLLVMVisibility(GV, GV.getDLLStorageClass() == + llvm::GlobalValue::DLLExportStorageClass + ? DLLExportVisibility + : NoDLLStorageClassVisibility); GV.setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); } } +static bool isStackProtectorOn(const LangOptions &LangOpts, + const llvm::Triple &Triple, + clang::LangOptions::StackProtectorMode Mode) { + if (Triple.isAMDGPU() || Triple.isNVPTX()) + return false; + return LangOpts.getStackProtector() == Mode; +} + void CodeGenModule::Release() { + Module *Primary = getContext().getCurrentNamedModule(); + if (CXX20ModuleInits && Primary && !Primary->isHeaderLikeModule()) + EmitModuleInitializers(Primary); EmitDeferred(); + DeferredDecls.insert(EmittedDeferredDecls.begin(), + EmittedDeferredDecls.end()); + EmittedDeferredDecls.clear(); EmitVTablesOpportunistically(); applyGlobalValReplacements(); applyReplacements(); - checkAliases(); emitMultiVersionFunctions(); - EmitCXXGlobalInitFunc(); + + if (Context.getLangOpts().IncrementalExtensions && + GlobalTopLevelStmtBlockInFlight.first) { + const TopLevelStmtDecl *TLSD = GlobalTopLevelStmtBlockInFlight.second; + GlobalTopLevelStmtBlockInFlight.first->FinishFunction(TLSD->getEndLoc()); + GlobalTopLevelStmtBlockInFlight = {nullptr, nullptr}; + } + + // Module implementations are initialized the same way as a regular TU that + // imports one or more modules. + if (CXX20ModuleInits && Primary && Primary->isInterfaceOrPartition()) + EmitCXXModuleInitFunc(Primary); + else + EmitCXXGlobalInitFunc(); EmitCXXGlobalCleanUpFunc(); registerGlobalDtorsWithAtExit(); EmitCXXThreadLocalInitFunc(); @@ -506,10 +850,14 @@ void CodeGenModule::Release() { if (PGOStats.hasDiagnostics()) PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName); } + llvm::stable_sort(GlobalCtors, [](const Structor &L, const Structor &R) { + return L.LexOrder < R.LexOrder; + }); EmitCtorList(GlobalCtors, "llvm.global_ctors"); EmitCtorList(GlobalDtors, "llvm.global_dtors"); EmitGlobalAnnotations(); EmitStaticExternCAliases(); + checkAliases(); EmitDeferredUnusedCoverageMappings(); CodeGenPGO(*this).setValueProfilingFlag(getModule()); if (CoverageMapping) @@ -518,25 +866,56 @@ void CodeGenModule::Release() { CodeGenFunction(*this).EmitCfiCheckFail(); CodeGenFunction(*this).EmitCfiCheckStub(); } + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) + finalizeKCFITypes(); emitAtAvailableLinkGuard(); - if (Context.getTargetInfo().getTriple().isWasm() && - !Context.getTargetInfo().getTriple().isOSEmscripten()) { + if (Context.getTargetInfo().getTriple().isWasm()) EmitMainVoidAlias(); + + if (getTriple().isAMDGPU()) { + // Emit amdgpu_code_object_version module flag, which is code object version + // times 100. + if (getTarget().getTargetOpts().CodeObjectVersion != + llvm::CodeObjectVersionKind::COV_None) { + getModule().addModuleFlag(llvm::Module::Error, + "amdgpu_code_object_version", + getTarget().getTargetOpts().CodeObjectVersion); + } + + // Currently, "-mprintf-kind" option is only supported for HIP + if (LangOpts.HIP) { + auto *MDStr = llvm::MDString::get( + getLLVMContext(), (getTarget().getTargetOpts().AMDGPUPrintfKindVal == + TargetOptions::AMDGPUPrintfKind::Hostcall) + ? "hostcall" + : "buffered"); + getModule().addModuleFlag(llvm::Module::Error, "amdgpu_printf_kind", + MDStr); + } } - // Emit reference of __amdgpu_device_library_preserve_asan_functions to - // preserve ASAN functions in bitcode libraries. - if (LangOpts.Sanitize.has(SanitizerKind::Address) && getTriple().isAMDGPU()) { - auto *FT = llvm::FunctionType::get(VoidTy, {}); - auto *F = llvm::Function::Create( - FT, llvm::GlobalValue::ExternalLinkage, - "__amdgpu_device_library_preserve_asan_functions", &getModule()); - auto *Var = new llvm::GlobalVariable( - getModule(), FT->getPointerTo(), - /*isConstant=*/true, llvm::GlobalValue::WeakAnyLinkage, F, - "__amdgpu_device_library_preserve_asan_functions_ptr", nullptr, - llvm::GlobalVariable::NotThreadLocal); - addCompilerUsedGlobal(Var); + // Emit a global array containing all external kernels or device variables + // used by host functions and mark it as used for CUDA/HIP. This is necessary + // to get kernels or device variables in archives linked in even if these + // kernels or device variables are only used in host functions. + if (!Context.CUDAExternalDeviceDeclODRUsedByHost.empty()) { + SmallVector<llvm::Constant *, 8> UsedArray; + for (auto D : Context.CUDAExternalDeviceDeclODRUsedByHost) { + GlobalDecl GD; + if (auto *FD = dyn_cast<FunctionDecl>(D)) + GD = GlobalDecl(FD, KernelReferenceKind::Kernel); + else + GD = GlobalDecl(D); + UsedArray.push_back(llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast( + GetAddrOfGlobal(GD), Int8PtrTy)); + } + + llvm::ArrayType *ATy = llvm::ArrayType::get(Int8PtrTy, UsedArray.size()); + + auto *GV = new llvm::GlobalVariable( + getModule(), ATy, false, llvm::GlobalValue::InternalLinkage, + llvm::ConstantArray::get(ATy, UsedArray), "__clang_gpu_used_external"); + addCompilerUsedGlobal(GV); } emitLLVMUsed(); @@ -581,7 +960,7 @@ void CodeGenModule::Release() { if (Context.getLangOpts().SemanticInterposition) // Require various optimization to respect semantic interposition. - getModule().setSemanticInterposition(1); + getModule().setSemanticInterposition(true); if (CodeGenOpts.EmitCodeView) { // Indicate that we want CodeView in the metadata. @@ -601,6 +980,10 @@ void CodeGenModule::Release() { // Function ID tables for EH Continuation Guard. getModule().addModuleFlag(llvm::Module::Warning, "ehcontguard", 1); } + if (Context.getLangOpts().Kernel) { + // Note if we are compiling with /kernel. + getModule().addModuleFlag(llvm::Module::Warning, "ms-kernel", 1); + } if (CodeGenOpts.OptimizationLevel > 0 && CodeGenOpts.StrictVTablePointers) { // We don't support LTO with 2 with different StrictVTablePointers // FIXME: we could support it by stripping all the information introduced @@ -631,6 +1014,36 @@ void CodeGenModule::Release() { Context.getTypeSizeInChars(Context.getWideCharType()).getQuantity(); getModule().addModuleFlag(llvm::Module::Error, "wchar_size", WCharWidth); + if (getTriple().isOSzOS()) { + getModule().addModuleFlag(llvm::Module::Warning, + "zos_product_major_version", + uint32_t(CLANG_VERSION_MAJOR)); + getModule().addModuleFlag(llvm::Module::Warning, + "zos_product_minor_version", + uint32_t(CLANG_VERSION_MINOR)); + getModule().addModuleFlag(llvm::Module::Warning, "zos_product_patchlevel", + uint32_t(CLANG_VERSION_PATCHLEVEL)); + std::string ProductId = getClangVendor() + "clang"; + getModule().addModuleFlag(llvm::Module::Error, "zos_product_id", + llvm::MDString::get(VMContext, ProductId)); + + // Record the language because we need it for the PPA2. + StringRef lang_str = languageToString( + LangStandard::getLangStandardForKind(LangOpts.LangStd).Language); + getModule().addModuleFlag(llvm::Module::Error, "zos_cu_language", + llvm::MDString::get(VMContext, lang_str)); + + time_t TT = PreprocessorOpts.SourceDateEpoch + ? *PreprocessorOpts.SourceDateEpoch + : std::time(nullptr); + getModule().addModuleFlag(llvm::Module::Max, "zos_translation_time", + static_cast<uint64_t>(TT)); + + // Multiple modes will be supported here. + getModule().addModuleFlag(llvm::Module::Error, "zos_le_char_mode", + llvm::MDString::get(VMContext, "ascii")); + } + llvm::Triple::ArchType Arch = Context.getTargetInfo().getTriple().getArch(); if ( Arch == llvm::Triple::arm || Arch == llvm::Triple::armeb @@ -667,36 +1080,83 @@ void CodeGenModule::Release() { CodeGenOpts.SanitizeCfiCanonicalJumpTables); } + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) { + getModule().addModuleFlag(llvm::Module::Override, "kcfi", 1); + // KCFI assumes patchable-function-prefix is the same for all indirectly + // called functions. Store the expected offset for code generation. + if (CodeGenOpts.PatchableFunctionEntryOffset) + getModule().addModuleFlag(llvm::Module::Override, "kcfi-offset", + CodeGenOpts.PatchableFunctionEntryOffset); + } + if (CodeGenOpts.CFProtectionReturn && Target.checkCFProtectionReturnSupported(getDiags())) { // Indicate that we want to instrument return control flow protection. - getModule().addModuleFlag(llvm::Module::Override, "cf-protection-return", + getModule().addModuleFlag(llvm::Module::Min, "cf-protection-return", 1); } if (CodeGenOpts.CFProtectionBranch && Target.checkCFProtectionBranchSupported(getDiags())) { // Indicate that we want to instrument branch control flow protection. - getModule().addModuleFlag(llvm::Module::Override, "cf-protection-branch", + getModule().addModuleFlag(llvm::Module::Min, "cf-protection-branch", 1); } - if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_32 || + if (CodeGenOpts.FunctionReturnThunks) + getModule().addModuleFlag(llvm::Module::Override, "function_return_thunk_extern", 1); + + if (CodeGenOpts.IndirectBranchCSPrefix) + getModule().addModuleFlag(llvm::Module::Override, "indirect_branch_cs_prefix", 1); + + // Add module metadata for return address signing (ignoring + // non-leaf/all) and stack tagging. These are actually turned on by function + // attributes, but we use module metadata to emit build attributes. This is + // needed for LTO, where the function attributes are inside bitcode + // serialised into a global variable by the time build attributes are + // emitted, so we can't access them. LTO objects could be compiled with + // different flags therefore module flags are set to "Min" behavior to achieve + // the same end result of the normal build where e.g BTI is off if any object + // doesn't support it. + if (Context.getTargetInfo().hasFeature("ptrauth") && + LangOpts.getSignReturnAddressScope() != + LangOptions::SignReturnAddressScopeKind::None) + getModule().addModuleFlag(llvm::Module::Override, + "sign-return-address-buildattr", 1); + if (LangOpts.Sanitize.has(SanitizerKind::MemtagStack)) + getModule().addModuleFlag(llvm::Module::Override, + "tag-stack-memory-buildattr", 1); + + if (Arch == llvm::Triple::thumb || Arch == llvm::Triple::thumbeb || + Arch == llvm::Triple::arm || Arch == llvm::Triple::armeb || + Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_32 || Arch == llvm::Triple::aarch64_be) { - getModule().addModuleFlag(llvm::Module::Error, - "branch-target-enforcement", - LangOpts.BranchTargetEnforcement); + if (LangOpts.BranchTargetEnforcement) + getModule().addModuleFlag(llvm::Module::Min, "branch-target-enforcement", + 1); + if (LangOpts.BranchProtectionPAuthLR) + getModule().addModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", + 1); + if (LangOpts.GuardedControlStack) + getModule().addModuleFlag(llvm::Module::Min, "guarded-control-stack", 1); + if (LangOpts.hasSignReturnAddress()) + getModule().addModuleFlag(llvm::Module::Min, "sign-return-address", 1); + if (LangOpts.isSignReturnAddressScopeAll()) + getModule().addModuleFlag(llvm::Module::Min, "sign-return-address-all", + 1); + if (!LangOpts.isSignReturnAddressWithAKey()) + getModule().addModuleFlag(llvm::Module::Min, + "sign-return-address-with-bkey", 1); + } - getModule().addModuleFlag(llvm::Module::Error, "sign-return-address", - LangOpts.hasSignReturnAddress()); + if (CodeGenOpts.StackClashProtector) + getModule().addModuleFlag( + llvm::Module::Override, "probe-stack", + llvm::MDString::get(TheModule.getContext(), "inline-asm")); - getModule().addModuleFlag(llvm::Module::Error, "sign-return-address-all", - LangOpts.isSignReturnAddressScopeAll()); - - getModule().addModuleFlag(llvm::Module::Error, - "sign-return-address-with-bkey", - !LangOpts.isSignReturnAddressWithAKey()); - } + if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096) + getModule().addModuleFlag(llvm::Module::Min, "stack-probe-size", + CodeGenOpts.StackProbeSize); if (!CodeGenOpts.MemoryProfileOutput.empty()) { llvm::LLVMContext &Ctx = TheModule.getContext(); @@ -720,19 +1180,20 @@ void CodeGenModule::Release() { // Indicate whether this Module was compiled with -fopenmp if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd) getModule().addModuleFlag(llvm::Module::Max, "openmp", LangOpts.OpenMP); - if (getLangOpts().OpenMPIsDevice) + if (getLangOpts().OpenMPIsTargetDevice) getModule().addModuleFlag(llvm::Module::Max, "openmp-device", LangOpts.OpenMP); // Emit OpenCL specific module metadata: OpenCL/SPIR version. - if (LangOpts.OpenCL) { + if (LangOpts.OpenCL || (LangOpts.CUDAIsDevice && getTriple().isSPIRV())) { EmitOpenCLMetadata(); // Emit SPIR version. if (getTriple().isSPIR()) { // SPIR v2.0 s2.12 - The SPIR version used by the module is stored in the // opencl.spir.version named metadata. - // C++ is backwards compatible with OpenCL v2.0. - auto Version = LangOpts.OpenCLCPlusPlus ? 200 : LangOpts.OpenCLVersion; + // C++ for OpenCL has a distinct mapping for version compatibility with + // OpenCL. + auto Version = LangOpts.getOpenCLCompatibleVersion(); llvm::Metadata *SPIRVerElts[] = { llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( Int32Ty, Version / 100)), @@ -745,6 +1206,10 @@ void CodeGenModule::Release() { } } + // HLSL related end of code gen work items. + if (LangOpts.HLSL) + getHLSLRuntime().finishCodeGen(); + if (uint32_t PLevel = Context.getLangOpts().PICLevel) { assert(PLevel < 3 && "Invalid PIC Level"); getModule().setPICLevel(static_cast<llvm::PICLevel::Level>(PLevel)); @@ -763,13 +1228,25 @@ void CodeGenModule::Release() { if (CM != ~0u) { llvm::CodeModel::Model codeModel = static_cast<llvm::CodeModel::Model>(CM); getModule().setCodeModel(codeModel); + + if ((CM == llvm::CodeModel::Medium || CM == llvm::CodeModel::Large) && + Context.getTargetInfo().getTriple().getArch() == + llvm::Triple::x86_64) { + getModule().setLargeDataThreshold(getCodeGenOpts().LargeDataThreshold); + } } } if (CodeGenOpts.NoPLT) getModule().setRtLibUseGOT(); + if (getTriple().isOSBinFormatELF() && + CodeGenOpts.DirectAccessExternalData != + getModule().getDirectAccessExternalData()) { + getModule().setDirectAccessExternalData( + CodeGenOpts.DirectAccessExternalData); + } if (CodeGenOpts.UnwindTables) - getModule().setUwtable(); + getModule().setUwtable(llvm::UWTableKind(CodeGenOpts.UnwindTables)); switch (CodeGenOpts.getFramePointer()) { case CodeGenOptions::FramePointerKind::None: @@ -788,7 +1265,8 @@ void CodeGenModule::Release() { if (getCodeGenOpts().EmitDeclMetadata) EmitDeclMetadata(); - if (getCodeGenOpts().EmitGcovArcs || getCodeGenOpts().EmitGcovNotes) + if (getCodeGenOpts().CoverageNotesFile.size() || + getCodeGenOpts().CoverageDataFile.size()) EmitCoverageFile(); if (CGDebugInfo *DI = getModuleDebugInfo()) @@ -805,16 +1283,32 @@ void CodeGenModule::Release() { if (!getCodeGenOpts().StackProtectorGuardReg.empty()) getModule().setStackProtectorGuardReg( getCodeGenOpts().StackProtectorGuardReg); + if (!getCodeGenOpts().StackProtectorGuardSymbol.empty()) + getModule().setStackProtectorGuardSymbol( + getCodeGenOpts().StackProtectorGuardSymbol); if (getCodeGenOpts().StackProtectorGuardOffset != INT_MAX) getModule().setStackProtectorGuardOffset( getCodeGenOpts().StackProtectorGuardOffset); if (getCodeGenOpts().StackAlignment) getModule().setOverrideStackAlignment(getCodeGenOpts().StackAlignment); + if (getCodeGenOpts().SkipRaxSetup) + getModule().addModuleFlag(llvm::Module::Override, "SkipRaxSetup", 1); + if (getLangOpts().RegCall4) + getModule().addModuleFlag(llvm::Module::Override, "RegCallv4", 1); + + if (getContext().getTargetInfo().getMaxTLSAlign()) + getModule().addModuleFlag(llvm::Module::Error, "MaxTLSAlign", + getContext().getTargetInfo().getMaxTLSAlign()); + + getTargetCodeGenInfo().emitTargetGlobals(*this); getTargetCodeGenInfo().emitTargetMetadata(*this, MangledDeclNames); EmitBackendOptionsMetadata(getCodeGenOpts()); + // If there is device offloading code embed it in the host now. + EmbedObject(&getModule(), CodeGenOpts, getDiags()); + // Set visibility from DLL storage class // We do this at the end of LLVM IR generation; after any operation // that might affect the DLL storage class or the visibility, and @@ -825,9 +1319,8 @@ void CodeGenModule::Release() { void CodeGenModule::EmitOpenCLMetadata() { // SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the // opencl.ocl.version named metadata node. - // C++ is backwards compatible with OpenCL v2.0. - // FIXME: We might need to add CXX version at some point too? - auto Version = LangOpts.OpenCLCPlusPlus ? 200 : LangOpts.OpenCLVersion; + // C++ for OpenCL has a distinct mapping for versions compatibile with OpenCL. + auto Version = LangOpts.getOpenCLCompatibleVersion(); llvm::Metadata *OCLVerElts[] = { llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( Int32Ty, Version / 100)), @@ -840,15 +1333,10 @@ void CodeGenModule::EmitOpenCLMetadata() { } void CodeGenModule::EmitBackendOptionsMetadata( - const CodeGenOptions CodeGenOpts) { - switch (getTriple().getArch()) { - default: - break; - case llvm::Triple::riscv32: - case llvm::Triple::riscv64: - getModule().addModuleFlag(llvm::Module::Error, "SmallDataLimit", + const CodeGenOptions &CodeGenOpts) { + if (getTriple().isRISCV()) { + getModule().addModuleFlag(llvm::Module::Min, "SmallDataLimit", CodeGenOpts.SmallDataLimit); - break; } } @@ -977,8 +1465,6 @@ llvm::ConstantInt *CodeGenModule::getSize(CharUnits size) { void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV, const NamedDecl *D) const { - if (GV->hasDLLImportStorageClass()) - return; // Internal definitions always have default visibility. if (GV->hasLocalLinkage()) { GV->setVisibility(llvm::GlobalValue::DefaultVisibility); @@ -986,9 +1472,39 @@ void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV, } if (!D) return; + // Set visibility for definitions, and for declarations if requested globally // or set explicitly. LinkageInfo LV = D->getLinkageAndVisibility(); + + // OpenMP declare target variables must be visible to the host so they can + // be registered. We require protected visibility unless the variable has + // the DT_nohost modifier and does not need to be registered. + if (Context.getLangOpts().OpenMP && + Context.getLangOpts().OpenMPIsTargetDevice && isa<VarDecl>(D) && + D->hasAttr<OMPDeclareTargetDeclAttr>() && + D->getAttr<OMPDeclareTargetDeclAttr>()->getDevType() != + OMPDeclareTargetDeclAttr::DT_NoHost && + LV.getVisibility() == HiddenVisibility) { + GV->setVisibility(llvm::GlobalValue::ProtectedVisibility); + return; + } + + if (GV->hasDLLExportStorageClass() || GV->hasDLLImportStorageClass()) { + // Reject incompatible dlllstorage and visibility annotations. + if (!LV.isVisibilityExplicit()) + return; + if (GV->hasDLLExportStorageClass()) { + if (LV.getVisibility() == HiddenVisibility) + getDiags().Report(D->getLocation(), + diag::err_hidden_visibility_dllexport); + } else if (LV.getVisibility() != DefaultVisibility) { + getDiags().Report(D->getLocation(), + diag::err_non_default_visibility_dllimport); + } + return; + } + if (LV.isVisibilityExplicit() || getLangOpts().SetVisibilityForExternDecls || !GV->isDeclarationForLinker()) GV->setVisibility(GetLLVMVisibility(LV.getVisibility())); @@ -1007,6 +1523,7 @@ static bool shouldAssumeDSOLocal(const CodeGenModule &CGM, return false; const llvm::Triple &TT = CGM.getTriple(); + const auto &CGOpts = CGM.getCodeGenOpts(); if (TT.isWindowsGNUEnvironment()) { // In MinGW, variables without DLLImport can still be automatically // imported from a DLL by the linker; don't mark variables that @@ -1017,7 +1534,8 @@ static bool shouldAssumeDSOLocal(const CodeGenModule &CGM, // such variables can't be marked as DSO local. (Native TLS variables // can't be dllimported at all, though.) if (GV->isDeclarationForLinker() && isa<llvm::GlobalVariable>(GV) && - (!GV->isThreadLocal() || CGM.getCodeGenOpts().EmulatedTLS)) + (!GV->isThreadLocal() || CGM.getCodeGenOpts().EmulatedTLS) && + CGOpts.AutoImport) return false; } @@ -1040,7 +1558,6 @@ static bool shouldAssumeDSOLocal(const CodeGenModule &CGM, return false; // If this is not an executable, don't assume anything is local. - const auto &CGOpts = CGM.getCodeGenOpts(); llvm::Reloc::Model RM = CGOpts.RelocationModel; const auto &LOpts = CGM.getLangOpts(); if (RM != llvm::Reloc::Static && !LOpts.PIE) { @@ -1115,7 +1632,9 @@ void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV, if (D && D->isExternallyVisible()) { if (D->hasAttr<DLLImportAttr>()) GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass); - else if (D->hasAttr<DLLExportAttr>() && !GV->isDeclarationForLinker()) + else if ((D->hasAttr<DLLExportAttr>() || + shouldMapVisibilityToDLLExport(D)) && + !GV->isDeclarationForLinker()) GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass); } } @@ -1194,6 +1713,27 @@ static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM, Out << ".resolver"; } +static void AppendTargetVersionMangling(const CodeGenModule &CGM, + const TargetVersionAttr *Attr, + raw_ostream &Out) { + if (Attr->isDefaultVersion()) { + Out << ".default"; + return; + } + Out << "._"; + const TargetInfo &TI = CGM.getTarget(); + llvm::SmallVector<StringRef, 8> Feats; + Attr->getFeatures(Feats); + llvm::stable_sort(Feats, [&TI](const StringRef FeatL, const StringRef FeatR) { + return TI.multiVersionSortPriority(FeatL) < + TI.multiVersionSortPriority(FeatR); + }); + for (const auto &Feat : Feats) { + Out << 'M'; + Out << Feat; + } +} + static void AppendTargetMangling(const CodeGenModule &CGM, const TargetAttr *Attr, raw_ostream &Out) { if (Attr->isDefaultVersion()) @@ -1201,21 +1741,21 @@ static void AppendTargetMangling(const CodeGenModule &CGM, Out << '.'; const TargetInfo &Target = CGM.getTarget(); - ParsedTargetAttr Info = - Attr->parse([&Target](StringRef LHS, StringRef RHS) { - // Multiversioning doesn't allow "no-${feature}", so we can - // only have "+" prefixes here. - assert(LHS.startswith("+") && RHS.startswith("+") && - "Features should always have a prefix."); - return Target.multiVersionSortPriority(LHS.substr(1)) > - Target.multiVersionSortPriority(RHS.substr(1)); - }); + ParsedTargetAttr Info = Target.parseTargetAttr(Attr->getFeaturesStr()); + llvm::sort(Info.Features, [&Target](StringRef LHS, StringRef RHS) { + // Multiversioning doesn't allow "no-${feature}", so we can + // only have "+" prefixes here. + assert(LHS.starts_with("+") && RHS.starts_with("+") && + "Features should always have a prefix."); + return Target.multiVersionSortPriority(LHS.substr(1)) > + Target.multiVersionSortPriority(RHS.substr(1)); + }); bool IsFirst = true; - if (!Info.Architecture.empty()) { + if (!Info.CPU.empty()) { IsFirst = false; - Out << "arch_" << Info.Architecture; + Out << "arch_" << Info.CPU; } for (StringRef Feat : Info.Features) { @@ -1235,6 +1775,41 @@ static bool isUniqueInternalLinkageDecl(GlobalDecl GD, (CGM.getFunctionLinkage(GD) == llvm::GlobalValue::InternalLinkage); } +static void AppendTargetClonesMangling(const CodeGenModule &CGM, + const TargetClonesAttr *Attr, + unsigned VersionIndex, + raw_ostream &Out) { + const TargetInfo &TI = CGM.getTarget(); + if (TI.getTriple().isAArch64()) { + StringRef FeatureStr = Attr->getFeatureStr(VersionIndex); + if (FeatureStr == "default") { + Out << ".default"; + return; + } + Out << "._"; + SmallVector<StringRef, 8> Features; + FeatureStr.split(Features, "+"); + llvm::stable_sort(Features, + [&TI](const StringRef FeatL, const StringRef FeatR) { + return TI.multiVersionSortPriority(FeatL) < + TI.multiVersionSortPriority(FeatR); + }); + for (auto &Feat : Features) { + Out << 'M'; + Out << Feat; + } + } else { + Out << '.'; + StringRef FeatureStr = Attr->getFeatureStr(VersionIndex); + if (FeatureStr.starts_with("arch=")) + Out << "arch_" << FeatureStr.substr(sizeof("arch=") - 1); + else + Out << FeatureStr; + + Out << '.' << Attr->getMangledIndex(VersionIndex); + } +} + static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD, const NamedDecl *ND, bool OmitMultiVersionMangling = false) { @@ -1253,7 +1828,10 @@ static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD, if (FD && FD->getType()->castAs<FunctionType>()->getCallConv() == CC_X86RegCall) { - Out << "__regcall3__" << II->getName(); + if (CGM.getLangOpts().RegCall4) + Out << "__regcall4__" << II->getName(); + else + Out << "__regcall3__" << II->getName(); } else if (FD && FD->hasAttr<CUDAGlobalAttr>() && GD.getKernelReferenceKind() == KernelReferenceKind::Stub) { Out << "__device_stub__" << II->getName(); @@ -1288,21 +1866,30 @@ static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD, case MultiVersionKind::Target: AppendTargetMangling(CGM, FD->getAttr<TargetAttr>(), Out); break; + case MultiVersionKind::TargetVersion: + AppendTargetVersionMangling(CGM, FD->getAttr<TargetVersionAttr>(), Out); + break; + case MultiVersionKind::TargetClones: + AppendTargetClonesMangling(CGM, FD->getAttr<TargetClonesAttr>(), + GD.getMultiVersionIndex(), Out); + break; case MultiVersionKind::None: llvm_unreachable("None multiversion type isn't valid here"); } } // Make unique name for device side static file-scope variable for HIP. - if (CGM.getContext().shouldExternalizeStaticVar(ND) && + if (CGM.getContext().shouldExternalize(ND) && CGM.getLangOpts().GPURelocatableDeviceCode && - CGM.getLangOpts().CUDAIsDevice && !CGM.getLangOpts().CUID.empty()) - CGM.printPostfixForExternalizedStaticVar(Out); + CGM.getLangOpts().CUDAIsDevice) + CGM.printPostfixForExternalizedDecl(Out, ND); + return std::string(Out.str()); } void CodeGenModule::UpdateMultiVersionNames(GlobalDecl GD, - const FunctionDecl *FD) { + const FunctionDecl *FD, + StringRef &CurName) { if (!FD->isMultiVersion()) return; @@ -1334,7 +1921,11 @@ void CodeGenModule::UpdateMultiVersionNames(GlobalDecl GD, if (ExistingRecord != std::end(Manglings)) Manglings.remove(&(*ExistingRecord)); auto Result = Manglings.insert(std::make_pair(OtherName, OtherGD)); - MangledDeclNames[OtherGD.getCanonicalDecl()] = Result.first->first(); + StringRef OtherNameRef = MangledDeclNames[OtherGD.getCanonicalDecl()] = + Result.first->first(); + // If this is the current decl is being created, make sure we update the name. + if (GD.getCanonicalDecl() == OtherGD.getCanonicalDecl()) + CurName = OtherNameRef; if (llvm::GlobalValue *Entry = GetGlobalValue(NonTargetName)) Entry->setName(OtherName); } @@ -1359,8 +1950,7 @@ StringRef CodeGenModule::getMangledName(GlobalDecl GD) { // static device variable depends on whether the variable is referenced by // a host or device host function. Therefore the mangled name cannot be // cached. - if (!LangOpts.CUDAIsDevice || - !getContext().mayExternalizeStaticVar(GD.getDecl())) { + if (!LangOpts.CUDAIsDevice || !getContext().mayExternalize(GD.getDecl())) { auto FoundName = MangledDeclNames.find(CanonicalGD); if (FoundName != MangledDeclNames.end()) return FoundName->second; @@ -1380,7 +1970,7 @@ StringRef CodeGenModule::getMangledName(GlobalDecl GD) { // directly between host- and device-compilations, the host- and // device-mangling in host compilation could help catching certain ones. assert(!isa<FunctionDecl>(ND) || !ND->hasAttr<CUDAGlobalAttr>() || - getLangOpts().CUDAIsDevice || + getContext().shouldExternalize(ND) || getLangOpts().CUDAIsDevice || (getContext().getAuxTargetInfo() && (getContext().getAuxTargetInfo()->getCXXABI() != getContext().getTargetInfo().getCXXABI())) || @@ -1415,6 +2005,16 @@ StringRef CodeGenModule::getBlockMangledName(GlobalDecl GD, return Result.first->first(); } +const GlobalDecl CodeGenModule::getMangledNameDecl(StringRef Name) { + auto it = MangledDeclNames.begin(); + while (it != MangledDeclNames.end()) { + if (it->second == Name) + return it->first; + it++; + } + return GlobalDecl(); +} + llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) { return getModule().getNamedValue(Name); } @@ -1422,9 +2022,10 @@ llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) { /// AddGlobalCtor - Add a function to the list that will be called before /// main() runs. void CodeGenModule::AddGlobalCtor(llvm::Function *Ctor, int Priority, + unsigned LexOrder, llvm::Constant *AssociatedData) { // FIXME: Type coercion of void()* types. - GlobalCtors.push_back(Structor(Priority, Ctor, AssociatedData)); + GlobalCtors.push_back(Structor(Priority, LexOrder, Ctor, AssociatedData)); } /// AddGlobalDtor - Add a function to the list that will be called @@ -1438,7 +2039,7 @@ void CodeGenModule::AddGlobalDtor(llvm::Function *Dtor, int Priority, } // FIXME: Type coercion of void()* types. - GlobalDtors.push_back(Structor(Priority, Dtor, nullptr)); + GlobalDtors.push_back(Structor(Priority, ~0U, Dtor, nullptr)); } void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) { @@ -1459,9 +2060,9 @@ void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) { for (const auto &I : Fns) { auto ctor = ctors.beginStruct(CtorStructTy); ctor.addInt(Int32Ty, I.Priority); - ctor.add(llvm::ConstantExpr::getBitCast(I.Initializer, CtorPFTy)); + ctor.add(I.Initializer); if (I.AssociatedData) - ctor.add(llvm::ConstantExpr::getBitCast(I.AssociatedData, VoidPtrTy)); + ctor.add(I.AssociatedData); else ctor.addNullPointer(VoidPtrTy); ctor.finishAndAddTo(ctors); @@ -1474,7 +2075,7 @@ void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) { // The LTO linker doesn't seem to like it when we set an alignment // on appending variables. Take it off as a workaround. - list->setAlignment(llvm::None); + list->setAlignment(std::nullopt); Fns.clear(); } @@ -1488,16 +2089,7 @@ CodeGenModule::getFunctionLinkage(GlobalDecl GD) { if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(D)) return getCXXABI().getCXXDestructorLinkage(Linkage, Dtor, GD.getDtorType()); - if (isa<CXXConstructorDecl>(D) && - cast<CXXConstructorDecl>(D)->isInheritingConstructor() && - Context.getTargetInfo().getCXXABI().isMicrosoft()) { - // Our approach to inheriting constructors is fundamentally different from - // that used by the MS ABI, so keep our inheriting constructor thunks - // internal rather than trying to pick an unambiguous mangling for them. - return llvm::GlobalValue::InternalLinkage; - } - - return getLLVMLinkageForDeclarator(D, Linkage, /*IsConstantVariable=*/false); + return getLLVMLinkageForDeclarator(D, Linkage); } llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) { @@ -1507,6 +2099,24 @@ llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) { return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString())); } +llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) { + if (auto *FnType = T->getAs<FunctionProtoType>()) + T = getContext().getFunctionType( + FnType->getReturnType(), FnType->getParamTypes(), + FnType->getExtProtoInfo().withExceptionSpec(EST_None)); + + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleCanonicalTypeName( + T, Out, getCodeGenOpts().SanitizeCfiICallNormalizeIntegers); + + if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers) + Out << ".normalized"; + + return llvm::ConstantInt::get(Int32Ty, + static_cast<uint32_t>(llvm::xxHash64(OutName))); +} + void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk) { @@ -1563,7 +2173,7 @@ static unsigned ArgInfoAddressSpace(LangAS AS) { } } -void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn, +void CodeGenModule::GenKernelArgMetadata(llvm::Function *Fn, const FunctionDecl *FD, CodeGenFunction *CGF) { assert(((FD && CGF) || (!FD && !CGF)) && @@ -1595,13 +2205,18 @@ void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn, if (FD && CGF) for (unsigned i = 0, e = FD->getNumParams(); i != e; ++i) { const ParmVarDecl *parm = FD->getParamDecl(i); + // Get argument name. + argNames.push_back(llvm::MDString::get(VMContext, parm->getName())); + + if (!getLangOpts().OpenCL) + continue; QualType ty = parm->getType(); std::string typeQuals; // Get image and pipe access qualifier: if (ty->isImageType() || ty->isPipeType()) { const Decl *PDecl = parm; - if (auto *TD = dyn_cast<TypedefType>(ty)) + if (const auto *TD = ty->getAs<TypedefType>()) PDecl = TD->getDecl(); const OpenCLAccessAttr *A = PDecl->getAttr<OpenCLAccessAttr>(); if (A && A->isWriteOnly()) @@ -1613,9 +2228,6 @@ void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn, } else accessQuals.push_back(llvm::MDString::get(VMContext, "none")); - // Get argument name. - argNames.push_back(llvm::MDString::get(VMContext, parm->getName())); - auto getTypeSpelling = [&](QualType Ty) { auto typeName = Ty.getUnqualifiedType().getAsString(Policy); @@ -1688,17 +2300,20 @@ void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn, argTypeQuals.push_back(llvm::MDString::get(VMContext, typeQuals)); } - Fn->setMetadata("kernel_arg_addr_space", - llvm::MDNode::get(VMContext, addressQuals)); - Fn->setMetadata("kernel_arg_access_qual", - llvm::MDNode::get(VMContext, accessQuals)); - Fn->setMetadata("kernel_arg_type", - llvm::MDNode::get(VMContext, argTypeNames)); - Fn->setMetadata("kernel_arg_base_type", - llvm::MDNode::get(VMContext, argBaseTypeNames)); - Fn->setMetadata("kernel_arg_type_qual", - llvm::MDNode::get(VMContext, argTypeQuals)); - if (getCodeGenOpts().EmitOpenCLArgMetadata) + if (getLangOpts().OpenCL) { + Fn->setMetadata("kernel_arg_addr_space", + llvm::MDNode::get(VMContext, addressQuals)); + Fn->setMetadata("kernel_arg_access_qual", + llvm::MDNode::get(VMContext, accessQuals)); + Fn->setMetadata("kernel_arg_type", + llvm::MDNode::get(VMContext, argTypeNames)); + Fn->setMetadata("kernel_arg_base_type", + llvm::MDNode::get(VMContext, argBaseTypeNames)); + Fn->setMetadata("kernel_arg_type_qual", + llvm::MDNode::get(VMContext, argTypeQuals)); + } + if (getCodeGenOpts().EmitOpenCLArgMetadata || + getCodeGenOpts().HIPSaveKernelArgName) Fn->setMetadata("kernel_arg_name", llvm::MDNode::get(VMContext, argNames)); } @@ -1732,11 +2347,11 @@ static bool requiresMemberFunctionPointerTypeMetadata(CodeGenModule &CGM, // Only functions whose address can be taken with a member function pointer // need this sort of type metadata. - return !MD->isStatic() && !MD->isVirtual() && !isa<CXXConstructorDecl>(MD) && - !isa<CXXDestructorDecl>(MD); + return MD->isImplicitObjectMemberFunction() && !MD->isVirtual() && + !isa<CXXConstructorDecl, CXXDestructorDecl>(MD); } -std::vector<const CXXRecordDecl *> +SmallVector<const CXXRecordDecl *, 0> CodeGenModule::getMostBaseClasses(const CXXRecordDecl *RD) { llvm::SetVector<const CXXRecordDecl *> MostBases; @@ -1753,25 +2368,32 @@ CodeGenModule::getMostBaseClasses(const CXXRecordDecl *RD) { void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F) { - llvm::AttrBuilder B; + llvm::AttrBuilder B(F->getContext()); - if (CodeGenOpts.UnwindTables) - B.addAttribute(llvm::Attribute::UWTable); + if ((!D || !D->hasAttr<NoUwtableAttr>()) && CodeGenOpts.UnwindTables) + B.addUWTableAttr(llvm::UWTableKind(CodeGenOpts.UnwindTables)); if (CodeGenOpts.StackClashProtector) B.addAttribute("probe-stack", "inline-asm"); + if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096) + B.addAttribute("stack-probe-size", + std::to_string(CodeGenOpts.StackProbeSize)); + if (!hasUnwindExceptions(LangOpts)) B.addAttribute(llvm::Attribute::NoUnwind); - if (!D || !D->hasAttr<NoStackProtectorAttr>()) { - if (LangOpts.getStackProtector() == LangOptions::SSPOn) - B.addAttribute(llvm::Attribute::StackProtect); - else if (LangOpts.getStackProtector() == LangOptions::SSPStrong) - B.addAttribute(llvm::Attribute::StackProtectStrong); - else if (LangOpts.getStackProtector() == LangOptions::SSPReq) - B.addAttribute(llvm::Attribute::StackProtectReq); - } + if (D && D->hasAttr<NoStackProtectorAttr>()) + ; // Do nothing. + else if (D && D->hasAttr<StrictGuardStackCheckAttr>() && + isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPOn)) + B.addAttribute(llvm::Attribute::StackProtectStrong); + else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPOn)) + B.addAttribute(llvm::Attribute::StackProtect); + else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPStrong)) + B.addAttribute(llvm::Attribute::StackProtectStrong); + else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPReq)) + B.addAttribute(llvm::Attribute::StackProtectReq); if (!D) { // If we don't have a declaration to control inlining, the function isn't @@ -1781,10 +2403,22 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) B.addAttribute(llvm::Attribute::NoInline); - F->addAttributes(llvm::AttributeList::FunctionIndex, B); + F->addFnAttrs(B); return; } + // Handle SME attributes that apply to function definitions, + // rather than to function prototypes. + if (D->hasAttr<ArmLocallyStreamingAttr>()) + B.addAttribute("aarch64_pstate_sm_body"); + + if (auto *Attr = D->getAttr<ArmNewAttr>()) { + if (Attr->isNewZA()) + B.addAttribute("aarch64_pstate_za_new"); + if (Attr->isNewZT0()) + B.addAttribute("aarch64_new_zt0"); + } + // Track whether we need to add the optnone LLVM attribute, // starting with the default for this optimization level. bool ShouldAddOptNone = @@ -1868,7 +2502,7 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, B.addAttribute(llvm::Attribute::MinSize); } - F->addAttributes(llvm::AttributeList::FunctionIndex, B); + F->addFnAttrs(B); unsigned alignment = D->getMaxAlignment() / Context.getCharWidth(); if (alignment) @@ -1883,8 +2517,8 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, // functions. If the current target's C++ ABI requires this and this is a // member function, set its alignment accordingly. if (getTarget().getCXXABI().areMemberFunctionsAligned()) { - if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D)) - F->setAlignment(llvm::Align(2)); + if (isa<CXXMethodDecl>(D) && F->getPointerAlignment(getDataLayout()) < 2) + F->setAlignment(std::max(llvm::Align(2), F->getAlign().valueOrOne())); } // In the cross-dso CFI mode with canonical jump tables, we want !type @@ -1913,18 +2547,9 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, } } -void CodeGenModule::setLLVMFunctionFEnvAttributes(const FunctionDecl *D, - llvm::Function *F) { - if (D->hasAttr<StrictFPAttr>()) { - llvm::AttrBuilder FuncAttrs; - FuncAttrs.addAttribute("strictfp"); - F->addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs); - } -} - void CodeGenModule::SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV) { const Decl *D = GD.getDecl(); - if (dyn_cast_or_null<NamedDecl>(D)) + if (isa_and_nonnull<NamedDecl>(D)) setGVProperties(GV, GD); else GV->setVisibility(llvm::GlobalValue::DefaultVisibility); @@ -1932,16 +2557,19 @@ void CodeGenModule::SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV) { if (D && D->hasAttr<UsedAttr>()) addUsedOrCompilerUsedGlobal(GV); - if (CodeGenOpts.KeepStaticConsts && D && isa<VarDecl>(D)) { - const auto *VD = cast<VarDecl>(D); - if (VD->getType().isConstQualified() && - VD->getStorageDuration() == SD_Static) - addUsedOrCompilerUsedGlobal(GV); - } + if (const auto *VD = dyn_cast_if_present<VarDecl>(D); + VD && + ((CodeGenOpts.KeepPersistentStorageVariables && + (VD->getStorageDuration() == SD_Static || + VD->getStorageDuration() == SD_Thread)) || + (CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static && + VD->getType().isConstQualified()))) + addUsedOrCompilerUsedGlobal(GV); } bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD, - llvm::AttrBuilder &Attrs) { + llvm::AttrBuilder &Attrs, + bool SetTargetFeatures) { // Add target-cpu and target-features attributes to functions. If // we have a decl for the function and it has a target attribute then // parse that and add it to the feature set. @@ -1951,9 +2579,12 @@ bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD, const auto *FD = dyn_cast_or_null<FunctionDecl>(GD.getDecl()); FD = FD ? FD->getMostRecentDecl() : FD; const auto *TD = FD ? FD->getAttr<TargetAttr>() : nullptr; + const auto *TV = FD ? FD->getAttr<TargetVersionAttr>() : nullptr; + assert((!TD || !TV) && "both target_version and target specified"); const auto *SD = FD ? FD->getAttr<CPUSpecificAttr>() : nullptr; + const auto *TC = FD ? FD->getAttr<TargetClonesAttr>() : nullptr; bool AddedAttr = false; - if (TD || SD) { + if (TD || TV || SD || TC) { llvm::StringMap<bool> FeatureMap; getContext().getFunctionFeatureMap(FeatureMap, GD); @@ -1966,16 +2597,23 @@ bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD, // get and parse the target attribute so we can get the cpu for // the function. if (TD) { - ParsedTargetAttr ParsedAttr = TD->parse(); - if (!ParsedAttr.Architecture.empty() && - getTarget().isValidCPUName(ParsedAttr.Architecture)) { - TargetCPU = ParsedAttr.Architecture; + ParsedTargetAttr ParsedAttr = + Target.parseTargetAttr(TD->getFeaturesStr()); + if (!ParsedAttr.CPU.empty() && + getTarget().isValidCPUName(ParsedAttr.CPU)) { + TargetCPU = ParsedAttr.CPU; TuneCPU = ""; // Clear the tune CPU. } if (!ParsedAttr.Tune.empty() && getTarget().isValidCPUName(ParsedAttr.Tune)) TuneCPU = ParsedAttr.Tune; } + + if (SD) { + // Apply the given CPU name as the 'tune-cpu' so that the optimizer can + // favor this processor. + TuneCPU = SD->getCPUName(GD.getMultiVersionIndex())->getName(); + } } else { // Otherwise just add the existing target cpu and target features to the // function. @@ -1990,7 +2628,10 @@ bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD, Attrs.addAttribute("tune-cpu", TuneCPU); AddedAttr = true; } - if (!Features.empty()) { + if (!Features.empty() && SetTargetFeatures) { + llvm::erase_if(Features, [&](const std::string& F) { + return getTarget().isReadOnlyFeature(F.substr(1)); + }); llvm::sort(Features); Attrs.addAttribute("target-features", llvm::join(Features, ",")); AddedAttr = true; @@ -2025,17 +2666,17 @@ void CodeGenModule::setNonAliasAttributes(GlobalDecl GD, if (!D->getAttr<SectionAttr>()) F->addFnAttr("implicit-section-name", SA->getName()); - llvm::AttrBuilder Attrs; + llvm::AttrBuilder Attrs(F->getContext()); if (GetCPUAndFeaturesAttributes(GD, Attrs)) { // We know that GetCPUAndFeaturesAttributes will always have the // newest set, since it has the newest possible FunctionDecl, so the // new ones should replace the old. - llvm::AttrBuilder RemoveAttrs; + llvm::AttributeMask RemoveAttrs; RemoveAttrs.addAttribute("target-cpu"); RemoveAttrs.addAttribute("target-features"); RemoveAttrs.addAttribute("tune-cpu"); - F->removeAttributes(llvm::AttributeList::FunctionIndex, RemoveAttrs); - F->addAttributes(llvm::AttributeList::FunctionIndex, Attrs); + F->removeFnAttrs(RemoveAttrs); + F->addFnAttrs(Attrs); } } @@ -2092,6 +2733,54 @@ void CodeGenModule::CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD, F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId)); } +void CodeGenModule::setKCFIType(const FunctionDecl *FD, llvm::Function *F) { + llvm::LLVMContext &Ctx = F->getContext(); + llvm::MDBuilder MDB(Ctx); + F->setMetadata(llvm::LLVMContext::MD_kcfi_type, + llvm::MDNode::get( + Ctx, MDB.createConstant(CreateKCFITypeId(FD->getType())))); +} + +static bool allowKCFIIdentifier(StringRef Name) { + // KCFI type identifier constants are only necessary for external assembly + // functions, which means it's safe to skip unusual names. Subset of + // MCAsmInfo::isAcceptableChar() and MCAsmInfoXCOFF::isAcceptableChar(). + return llvm::all_of(Name, [](const char &C) { + return llvm::isAlnum(C) || C == '_' || C == '.'; + }); +} + +void CodeGenModule::finalizeKCFITypes() { + llvm::Module &M = getModule(); + for (auto &F : M.functions()) { + // Remove KCFI type metadata from non-address-taken local functions. + bool AddressTaken = F.hasAddressTaken(); + if (!AddressTaken && F.hasLocalLinkage()) + F.eraseMetadata(llvm::LLVMContext::MD_kcfi_type); + + // Generate a constant with the expected KCFI type identifier for all + // address-taken function declarations to support annotating indirectly + // called assembly functions. + if (!AddressTaken || !F.isDeclaration()) + continue; + + const llvm::ConstantInt *Type; + if (const llvm::MDNode *MD = F.getMetadata(llvm::LLVMContext::MD_kcfi_type)) + Type = llvm::mdconst::extract<llvm::ConstantInt>(MD->getOperand(0)); + else + continue; + + StringRef Name = F.getName(); + if (!allowKCFIIdentifier(Name)) + continue; + + std::string Asm = (".weak __kcfi_typeid_" + Name + "\n.set __kcfi_typeid_" + + Name + ", " + Twine(Type->getZExtValue()) + "\n") + .str(); + M.appendModuleInlineAsm(Asm); + } +} + void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, bool IsIncompleteFunction, bool IsThunk) { @@ -2118,7 +2807,7 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, F->arg_begin()->getType() ->canLosslesslyBitCastTo(F->getReturnType()) && "unexpected this return"); - F->addAttribute(1, llvm::Attribute::Returned); + F->addParamAttr(0, llvm::Attribute::Returned); } // Only a few attributes are set on declarations; these may later be @@ -2136,6 +2825,13 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, else if (const auto *SA = FD->getAttr<SectionAttr>()) F->setSection(SA->getName()); + if (const auto *EA = FD->getAttr<ErrorAttr>()) { + if (EA->isError()) + F->addFnAttr("dontcall-error", EA->getUserDiagnostic()); + else if (EA->isWarning()) + F->addFnAttr("dontcall-warn", EA->getUserDiagnostic()); + } + // If we plan on emitting this inline builtin, we can't treat it as a builtin. if (FD->isInlineBuiltinDeclaration()) { const FunctionDecl *FDBody; @@ -2144,15 +2840,13 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, assert(HasBody && "Inline builtin declarations should always have an " "available body!"); if (shouldEmitFunction(FDBody)) - F->addAttribute(llvm::AttributeList::FunctionIndex, - llvm::Attribute::NoBuiltin); + F->addFnAttr(llvm::Attribute::NoBuiltin); } if (FD->isReplaceableGlobalAllocationFunction()) { // A replaceable global allocation function does not act like a builtin by // default, only if it is invoked by a new-expression or delete-expression. - F->addAttribute(llvm::AttributeList::FunctionIndex, - llvm::Attribute::NoBuiltin); + F->addFnAttr(llvm::Attribute::NoBuiltin); } if (isa<CXXConstructorDecl>(FD) || isa<CXXDestructorDecl>(FD)) @@ -2169,9 +2863,15 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, !CodeGenOpts.SanitizeCfiCanonicalJumpTables) CreateFunctionTypeMetadataForIcall(FD, F); + if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) + setKCFIType(FD, F); + if (getLangOpts().OpenMP && FD->hasAttr<OMPDeclareSimdDeclAttr>()) getOpenMPRuntime().emitDeclareSimdFunction(FD, F); + if (CodeGenOpts.InlineMaxStackSize != UINT_MAX) + F->addFnAttr("inline-max-stacksize", llvm::utostr(CodeGenOpts.InlineMaxStackSize)); + if (const auto *CB = FD->getAttr<CallbackAttr>()) { // Annotate the callback behavior as metadata: // - The callback callee (as argument number). @@ -2281,9 +2981,9 @@ static void addLinkOptionsPostorder(CodeGenModule &CGM, Module *Mod, } // Import this module's dependencies. - for (unsigned I = Mod->Imports.size(); I > 0; --I) { - if (Visited.insert(Mod->Imports[I - 1]).second) - addLinkOptionsPostorder(CGM, Mod->Imports[I-1], Metadata, Visited); + for (Module *Import : llvm::reverse(Mod->Imports)) { + if (Visited.insert(Import).second) + addLinkOptionsPostorder(CGM, Import, Metadata, Visited); } // Add linker options to link against the libraries/frameworks @@ -2296,13 +2996,12 @@ static void addLinkOptionsPostorder(CodeGenModule &CGM, Module *Mod, if (Mod->UseExportAsModuleLinkName) return; - for (unsigned I = Mod->LinkLibraries.size(); I > 0; --I) { + for (const Module::LinkLibrary &LL : llvm::reverse(Mod->LinkLibraries)) { // Link against a framework. Frameworks are currently Darwin only, so we // don't to ask TargetCodeGenInfo for the spelling of the linker option. - if (Mod->LinkLibraries[I-1].IsFramework) { - llvm::Metadata *Args[2] = { - llvm::MDString::get(Context, "-framework"), - llvm::MDString::get(Context, Mod->LinkLibraries[I - 1].Library)}; + if (LL.IsFramework) { + llvm::Metadata *Args[2] = {llvm::MDString::get(Context, "-framework"), + llvm::MDString::get(Context, LL.Library)}; Metadata.push_back(llvm::MDNode::get(Context, Args)); continue; @@ -2312,19 +3011,51 @@ static void addLinkOptionsPostorder(CodeGenModule &CGM, Module *Mod, if (IsELF) { llvm::Metadata *Args[2] = { llvm::MDString::get(Context, "lib"), - llvm::MDString::get(Context, Mod->LinkLibraries[I - 1].Library), + llvm::MDString::get(Context, LL.Library), }; Metadata.push_back(llvm::MDNode::get(Context, Args)); } else { llvm::SmallString<24> Opt; - CGM.getTargetCodeGenInfo().getDependentLibraryOption( - Mod->LinkLibraries[I - 1].Library, Opt); + CGM.getTargetCodeGenInfo().getDependentLibraryOption(LL.Library, Opt); auto *OptString = llvm::MDString::get(Context, Opt); Metadata.push_back(llvm::MDNode::get(Context, OptString)); } } } +void CodeGenModule::EmitModuleInitializers(clang::Module *Primary) { + assert(Primary->isNamedModuleUnit() && + "We should only emit module initializers for named modules."); + + // Emit the initializers in the order that sub-modules appear in the + // source, first Global Module Fragments, if present. + if (auto GMF = Primary->getGlobalModuleFragment()) { + for (Decl *D : getContext().getModuleInitializers(GMF)) { + if (isa<ImportDecl>(D)) + continue; + assert(isa<VarDecl>(D) && "GMF initializer decl is not a var?"); + EmitTopLevelDecl(D); + } + } + // Second any associated with the module, itself. + for (Decl *D : getContext().getModuleInitializers(Primary)) { + // Skip import decls, the inits for those are called explicitly. + if (isa<ImportDecl>(D)) + continue; + EmitTopLevelDecl(D); + } + // Third any associated with the Privat eMOdule Fragment, if present. + if (auto PMF = Primary->getPrivateModuleFragment()) { + for (Decl *D : getContext().getModuleInitializers(PMF)) { + // Skip import decls, the inits for those are called explicitly. + if (isa<ImportDecl>(D)) + continue; + assert(isa<VarDecl>(D) && "PMF initializer decl is not a var?"); + EmitTopLevelDecl(D); + } + } +} + void CodeGenModule::EmitModuleLinkOptions() { // Collect the set of all of the modules we want to visit to emit link // options, which is essentially the imported modules and all of their @@ -2410,8 +3141,8 @@ void CodeGenModule::EmitDeferred() { // Note we should not clear CUDADeviceVarODRUsedByHost since it is still // needed for further handling. if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) - for (const auto *V : getContext().CUDADeviceVarODRUsedByHost) - DeferredDeclsToEmit.push_back(V); + llvm::append_range(DeferredDeclsToEmit, + getContext().CUDADeviceVarODRUsedByHost); // Stop if we're out of both deferred vtables and deferred declarations. if (DeferredDeclsToEmit.empty()) @@ -2484,6 +3215,13 @@ void CodeGenModule::EmitVTablesOpportunistically() { } void CodeGenModule::EmitGlobalAnnotations() { + for (const auto& [MangledName, VD] : DeferredAnnotations) { + llvm::GlobalValue *GV = GetGlobalValue(MangledName); + if (GV) + AddGlobalAnnotations(VD, GV); + } + DeferredAnnotations.clear(); + if (Annotations.empty()) return; @@ -2503,9 +3241,10 @@ llvm::Constant *CodeGenModule::EmitAnnotationString(StringRef Str) { // Not found yet, create a new global. llvm::Constant *s = llvm::ConstantDataArray::getString(getLLVMContext(), Str); - auto *gv = - new llvm::GlobalVariable(getModule(), s->getType(), true, - llvm::GlobalValue::PrivateLinkage, s, ".str"); + auto *gv = new llvm::GlobalVariable( + getModule(), s->getType(), true, llvm::GlobalValue::PrivateLinkage, s, + ".str", nullptr, llvm::GlobalValue::NotThreadLocal, + ConstGlobalsPtrTy->getAddressSpace()); gv->setSection(AnnotationSection); gv->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); AStr = gv; @@ -2531,7 +3270,7 @@ llvm::Constant *CodeGenModule::EmitAnnotationLineNo(SourceLocation L) { llvm::Constant *CodeGenModule::EmitAnnotationArgs(const AnnotateAttr *Attr) { ArrayRef<Expr *> Exprs = {Attr->args_begin(), Attr->args_size()}; if (Exprs.empty()) - return llvm::ConstantPointerNull::get(Int8PtrTy); + return llvm::ConstantPointerNull::get(ConstGlobalsPtrTy); llvm::FoldingSetNodeID ID; for (Expr *E : Exprs) { @@ -2555,10 +3294,9 @@ llvm::Constant *CodeGenModule::EmitAnnotationArgs(const AnnotateAttr *Attr) { ".args"); GV->setSection(AnnotationSection); GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); - auto *Bitcasted = llvm::ConstantExpr::getBitCast(GV, Int8PtrTy); - Lookup = Bitcasted; - return Bitcasted; + Lookup = GV; + return GV; } llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV, @@ -2570,19 +3308,18 @@ llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV, *LineNoCst = EmitAnnotationLineNo(L), *Args = EmitAnnotationArgs(AA); - llvm::Constant *ASZeroGV = GV; - if (GV->getAddressSpace() != 0) { - ASZeroGV = llvm::ConstantExpr::getAddrSpaceCast( - GV, GV->getValueType()->getPointerTo(0)); + llvm::Constant *GVInGlobalsAS = GV; + if (GV->getAddressSpace() != + getDataLayout().getDefaultGlobalsAddressSpace()) { + GVInGlobalsAS = llvm::ConstantExpr::getAddrSpaceCast( + GV, + llvm::PointerType::get( + GV->getContext(), getDataLayout().getDefaultGlobalsAddressSpace())); } // Create the ConstantStruct for the global annotation. llvm::Constant *Fields[] = { - llvm::ConstantExpr::getBitCast(ASZeroGV, Int8PtrTy), - llvm::ConstantExpr::getBitCast(AnnoGV, Int8PtrTy), - llvm::ConstantExpr::getBitCast(UnitGV, Int8PtrTy), - LineNoCst, - Args, + GVInGlobalsAS, AnnoGV, UnitGV, LineNoCst, Args, }; return llvm::ConstantStruct::getAnon(Fields); } @@ -2601,34 +3338,35 @@ bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn, // NoSanitize by function name. if (NoSanitizeL.containsFunction(Kind, Fn->getName())) return true; - // NoSanitize by location. + // NoSanitize by location. Check "mainfile" prefix. + auto &SM = Context.getSourceManager(); + FileEntryRef MainFile = *SM.getFileEntryRefForID(SM.getMainFileID()); + if (NoSanitizeL.containsMainFile(Kind, MainFile.getName())) + return true; + + // Check "src" prefix. if (Loc.isValid()) return NoSanitizeL.containsLocation(Kind, Loc); // If location is unknown, this may be a compiler-generated function. Assume // it's located in the main file. - auto &SM = Context.getSourceManager(); - if (const auto *MainFile = SM.getFileEntryForID(SM.getMainFileID())) { - return NoSanitizeL.containsFile(Kind, MainFile->getName()); - } - return false; + return NoSanitizeL.containsFile(Kind, MainFile.getName()); } -bool CodeGenModule::isInNoSanitizeList(llvm::GlobalVariable *GV, +bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind, + llvm::GlobalVariable *GV, SourceLocation Loc, QualType Ty, StringRef Category) const { - // For now globals can be ignored only in ASan and KASan. - const SanitizerMask EnabledAsanMask = - LangOpts.Sanitize.Mask & - (SanitizerKind::Address | SanitizerKind::KernelAddress | - SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress | - SanitizerKind::MemTag); - if (!EnabledAsanMask) - return false; const auto &NoSanitizeL = getContext().getNoSanitizeList(); - if (NoSanitizeL.containsGlobal(EnabledAsanMask, GV->getName(), Category)) + if (NoSanitizeL.containsGlobal(Kind, GV->getName(), Category)) return true; - if (NoSanitizeL.containsLocation(EnabledAsanMask, Loc, Category)) + auto &SM = Context.getSourceManager(); + if (NoSanitizeL.containsMainFile( + Kind, SM.getFileEntryRefForID(SM.getMainFileID())->getName(), + Category)) + return true; + if (NoSanitizeL.containsLocation(Kind, Loc, Category)) return true; + // Check global type. if (!Ty.isNull()) { // Drill down the array types: if global variable of a fixed type is @@ -2639,7 +3377,7 @@ bool CodeGenModule::isInNoSanitizeList(llvm::GlobalVariable *GV, // Only record types (classes, structs etc.) are ignored. if (Ty->isRecordType()) { std::string TypeStr = Ty.getAsString(getContext().getPrintingPolicy()); - if (NoSanitizeL.containsType(EnabledAsanMask, TypeStr, Category)) + if (NoSanitizeL.containsType(Kind, TypeStr, Category)) return true; } } @@ -2672,32 +3410,44 @@ bool CodeGenModule::imbueXRayAttrs(llvm::Function *Fn, SourceLocation Loc, return true; } -bool CodeGenModule::isProfileInstrExcluded(llvm::Function *Fn, - SourceLocation Loc) const { +ProfileList::ExclusionType +CodeGenModule::isFunctionBlockedByProfileList(llvm::Function *Fn, + SourceLocation Loc) const { const auto &ProfileList = getContext().getProfileList(); // If the profile list is empty, then instrument everything. if (ProfileList.isEmpty()) - return false; + return ProfileList::Allow; CodeGenOptions::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr(); // First, check the function name. - Optional<bool> V = ProfileList.isFunctionExcluded(Fn->getName(), Kind); - if (V.hasValue()) + if (auto V = ProfileList.isFunctionExcluded(Fn->getName(), Kind)) return *V; // Next, check the source location. - if (Loc.isValid()) { - Optional<bool> V = ProfileList.isLocationExcluded(Loc, Kind); - if (V.hasValue()) + if (Loc.isValid()) + if (auto V = ProfileList.isLocationExcluded(Loc, Kind)) return *V; - } // If location is unknown, this may be a compiler-generated function. Assume // it's located in the main file. auto &SM = Context.getSourceManager(); - if (const auto *MainFile = SM.getFileEntryForID(SM.getMainFileID())) { - Optional<bool> V = ProfileList.isFileExcluded(MainFile->getName(), Kind); - if (V.hasValue()) + if (auto MainFile = SM.getFileEntryRefForID(SM.getMainFileID())) + if (auto V = ProfileList.isFileExcluded(MainFile->getName(), Kind)) return *V; + return ProfileList.getDefault(Kind); +} + +ProfileList::ExclusionType +CodeGenModule::isFunctionBlockedFromProfileInstr(llvm::Function *Fn, + SourceLocation Loc) const { + auto V = isFunctionBlockedByProfileList(Fn, Loc); + if (V != ProfileList::Allow) + return V; + + auto NumGroups = getCodeGenOpts().ProfileTotalFunctionGroups; + if (NumGroups > 1) { + auto Group = llvm::crc32(arrayRefFromStringRef(Fn->getName())) % NumGroups; + if (Group != getCodeGenOpts().ProfileSelectedFunctionGroup) + return ProfileList::Skip; } - return ProfileList.getDefault(); + return ProfileList::Allow; } bool CodeGenModule::MustBeEmitted(const ValueDecl *Global) { @@ -2705,12 +3455,14 @@ bool CodeGenModule::MustBeEmitted(const ValueDecl *Global) { if (LangOpts.EmitAllDecls) return true; - if (CodeGenOpts.KeepStaticConsts) { - const auto *VD = dyn_cast<VarDecl>(Global); - if (VD && VD->getType().isConstQualified() && - VD->getStorageDuration() == SD_Static) - return true; - } + const auto *VD = dyn_cast<VarDecl>(Global); + if (VD && + ((CodeGenOpts.KeepPersistentStorageVariables && + (VD->getStorageDuration() == SD_Static || + VD->getStorageDuration() == SD_Thread)) || + (CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static && + VD->getType().isConstQualified()))) + return true; return getContext().DeclMustBeEmitted(Global); } @@ -2723,7 +3475,7 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) { // we have if the level of the declare target attribute is -1. Note that we // check somewhere else if we should emit this at all. if (LangOpts.OpenMP >= 50 && !LangOpts.OpenMPSimd) { - llvm::Optional<OMPDeclareTargetDeclAttr *> ActiveAttr = + std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr = OMPDeclareTargetDeclAttr::getActiveAttr(Global); if (!ActiveAttr || (*ActiveAttr)->getLevel() != (unsigned)-1) return false; @@ -2735,17 +3487,25 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) { // explicitly instantiated, so they should not be emitted eagerly. return false; } - if (const auto *VD = dyn_cast<VarDecl>(Global)) + if (const auto *VD = dyn_cast<VarDecl>(Global)) { if (Context.getInlineVariableDefinitionKind(VD) == ASTContext::InlineVariableDefinitionKind::WeakUnknown) // A definition of an inline constexpr static data member may change // linkage later if it's redeclared outside the class. return false; + if (CXX20ModuleInits && VD->getOwningModule() && + !VD->getOwningModule()->isModuleMapModule()) { + // For CXX20, module-owned initializers need to be deferred, since it is + // not known at this point if they will be run for the current module or + // as part of the initializer for an imported one. + return false; + } + } // If OpenMP is enabled and threadprivates must be generated like TLS, delay // codegen for global variables, because they may be marked as threadprivate. if (LangOpts.OpenMP && LangOpts.OpenMPUseTLS && getContext().getTargetInfo().isTLSSupported() && isa<VarDecl>(Global) && - !isTypeConstant(Global->getType(), false) && + !Global->getType().isConstantStorage(getContext(), false, false) && !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(Global)) return false; @@ -2760,7 +3520,7 @@ ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) { // Look for an existing global. if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name)) - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); ConstantEmitter Emitter(*this); llvm::Constant *Init; @@ -2794,15 +3554,44 @@ ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) { GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); setDSOLocal(GV); - llvm::Constant *Addr = GV; if (!V.isAbsent()) { Emitter.finalize(GV); - } else { - llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType()); - Addr = llvm::ConstantExpr::getBitCast( - GV, Ty->getPointerTo(GV->getAddressSpace())); + return ConstantAddress(GV, GV->getValueType(), Alignment); } - return ConstantAddress(Addr, Alignment); + + llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType()); + return ConstantAddress(GV, Ty, Alignment); +} + +ConstantAddress CodeGenModule::GetAddrOfUnnamedGlobalConstantDecl( + const UnnamedGlobalConstantDecl *GCD) { + CharUnits Alignment = getContext().getTypeAlignInChars(GCD->getType()); + + llvm::GlobalVariable **Entry = nullptr; + Entry = &UnnamedGlobalConstantDeclMap[GCD]; + if (*Entry) + return ConstantAddress(*Entry, (*Entry)->getValueType(), Alignment); + + ConstantEmitter Emitter(*this); + llvm::Constant *Init; + + const APValue &V = GCD->getValue(); + + assert(!V.isAbsent()); + Init = Emitter.emitForInitializer(V, GCD->getType().getAddressSpace(), + GCD->getType()); + + auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(), + /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, Init, + ".constant"); + GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); + GV->setAlignment(Alignment.getAsAlign()); + + Emitter.finalize(GV); + + *Entry = GV; + return ConstantAddress(GV, GV->getValueType(), Alignment); } ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject( @@ -2811,7 +3600,7 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject( CharUnits Alignment = getNaturalTypeAlignment(TPO->getType()); if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name)) - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); ConstantEmitter Emitter(*this); llvm::Constant *Init = Emitter.emitForInitializer( @@ -2822,14 +3611,18 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject( return ConstantAddress::invalid(); } - auto *GV = new llvm::GlobalVariable( - getModule(), Init->getType(), - /*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name); + llvm::GlobalValue::LinkageTypes Linkage = + isExternallyVisible(TPO->getLinkageAndVisibility().getLinkage()) + ? llvm::GlobalValue::LinkOnceODRLinkage + : llvm::GlobalValue::InternalLinkage; + auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(), + /*isConstant=*/true, Linkage, Init, Name); + setGVProperties(GV, TPO); if (supportsCOMDAT()) GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); Emitter.finalize(GV); - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); } ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { @@ -2841,11 +3634,8 @@ ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { // See if there is already something with the target's name in the module. llvm::GlobalValue *Entry = GetGlobalValue(AA->getAliasee()); - if (Entry) { - unsigned AS = getContext().getTargetAddressSpace(VD->getType()); - auto Ptr = llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS)); - return ConstantAddress(Ptr, Alignment); - } + if (Entry) + return ConstantAddress(Entry, DeclTy, Alignment); llvm::Constant *Aliasee; if (isa<llvm::FunctionType>(DeclTy)) @@ -2853,13 +3643,22 @@ ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) { GlobalDecl(cast<FunctionDecl>(VD)), /*ForVTable=*/false); else - Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, 0, nullptr); + Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, LangAS::Default, + nullptr); auto *F = cast<llvm::GlobalValue>(Aliasee); F->setLinkage(llvm::Function::ExternalWeakLinkage); WeakRefReferences.insert(F); - return ConstantAddress(Aliasee, Alignment); + return ConstantAddress(Aliasee, DeclTy, Alignment); +} + +template <typename AttrT> static bool hasImplicitAttr(const ValueDecl *D) { + if (!D) + return false; + if (auto *A = D->getAttr<AttrT>()) + return A->isImplicit(); + return D->isImplicit(); } void CodeGenModule::EmitGlobal(GlobalDecl GD) { @@ -2883,14 +3682,24 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { return emitCPUDispatchDefinition(GD); // If this is CUDA, be selective about which declarations we emit. + // Non-constexpr non-lambda implicit host device functions are not emitted + // unless they are used on device side. if (LangOpts.CUDA) { if (LangOpts.CUDAIsDevice) { - if (!Global->hasAttr<CUDADeviceAttr>() && + const auto *FD = dyn_cast<FunctionDecl>(Global); + if ((!Global->hasAttr<CUDADeviceAttr>() || + (LangOpts.OffloadImplicitHostDeviceTemplates && FD && + hasImplicitAttr<CUDAHostAttr>(FD) && + hasImplicitAttr<CUDADeviceAttr>(FD) && !FD->isConstexpr() && + !isLambdaCallOperator(FD) && + !getContext().CUDAImplicitHostDeviceFunUsedByDevice.count(FD))) && !Global->hasAttr<CUDAGlobalAttr>() && !Global->hasAttr<CUDAConstantAttr>() && !Global->hasAttr<CUDASharedAttr>() && !Global->getType()->isCUDADeviceBuiltinSurfaceType() && - !Global->getType()->isCUDADeviceBuiltinTextureType()) + !Global->getType()->isCUDADeviceBuiltinTextureType() && + !(LangOpts.HIPStdPar && isa<FunctionDecl>(Global) && + !Global->hasAttr<CUDAHostAttr>())) return; } else { // We need to emit host-side 'shadows' for all global @@ -2916,7 +3725,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { if (MustBeEmitted(Global)) EmitOMPDeclareReduction(DRD); return; - } else if (auto *DMD = dyn_cast<OMPDeclareMapperDecl>(Global)) { + } + if (auto *DMD = dyn_cast<OMPDeclareMapperDecl>(Global)) { if (MustBeEmitted(Global)) EmitOMPDeclareMapper(DMD); return; @@ -2925,6 +3735,14 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // Ignore declarations, they will be emitted on their first use. if (const auto *FD = dyn_cast<FunctionDecl>(Global)) { + // Update deferred annotations with the latest declaration if the function + // function was already used or defined. + if (FD->hasAttr<AnnotateAttr>()) { + StringRef MangledName = getMangledName(GD); + if (GetGlobalValue(MangledName)) + DeferredAnnotations[MangledName] = FD; + } + // Forward declarations are emitted lazily on first use. if (!FD->doesThisDeclarationHaveABody()) { if (!FD->doesDeclarationForceExternallyVisibleDefinition()) @@ -2947,16 +3765,25 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { !Context.isMSStaticDataMemberInlineDefinition(VD)) { if (LangOpts.OpenMP) { // Emit declaration of the must-be-emitted declare target variable. - if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = + if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) { + + // If this variable has external storage and doesn't require special + // link handling we defer to its canonical definition. + if (VD->hasExternalStorage() && + Res != OMPDeclareTargetDeclAttr::MT_Link) + return; + bool UnifiedMemoryEnabled = getOpenMPRuntime().hasRequiresUnifiedSharedMemory(); - if (*Res == OMPDeclareTargetDeclAttr::MT_To && + if ((*Res == OMPDeclareTargetDeclAttr::MT_To || + *Res == OMPDeclareTargetDeclAttr::MT_Enter) && !UnifiedMemoryEnabled) { (void)GetAddrOfGlobalVar(VD); } else { assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) || - (*Res == OMPDeclareTargetDeclAttr::MT_To && + ((*Res == OMPDeclareTargetDeclAttr::MT_To || + *Res == OMPDeclareTargetDeclAttr::MT_Enter) && UnifiedMemoryEnabled)) && "Link clause or to clause with unified memory expected."); (void)getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); @@ -2980,6 +3807,7 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { if (MustBeEmitted(Global) && MayBeEmittedEagerly(Global)) { // Emit the definition if it can't be deferred. EmitGlobalDefinition(GD); + addEmittedDeferredDecl(GD); return; } @@ -3036,7 +3864,7 @@ namespace { if (!BuiltinID || !BI.isLibFunction(BuiltinID)) return false; StringRef BuiltinName = BI.getName(BuiltinID); - if (BuiltinName.startswith("__builtin_") && + if (BuiltinName.starts_with("__builtin_") && Name == BuiltinName.slice(strlen("__builtin_"), StringRef::npos)) { return true; } @@ -3139,10 +3967,22 @@ CodeGenModule::isTriviallyRecursive(const FunctionDecl *FD) { bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) { if (getFunctionLinkage(GD) != llvm::Function::AvailableExternallyLinkage) return true; + const auto *F = cast<FunctionDecl>(GD.getDecl()); if (CodeGenOpts.OptimizationLevel == 0 && !F->hasAttr<AlwaysInlineAttr>()) return false; + // We don't import function bodies from other named module units since that + // behavior may break ABI compatibility of the current unit. + if (const Module *M = F->getOwningModule(); + M && M->getTopLevelModule()->isNamedModule() && + getContext().getCurrentNamedModule() != M->getTopLevelModule() && + !F->hasAttr<AlwaysInlineAttr>()) + return false; + + if (F->hasAttr<NoInlineAttr>()) + return false; + if (F->hasAttr<DLLImportAttr>() && !F->hasAttr<AlwaysInlineAttr>()) { // Check whether it would be safe to inline this dllimport function. DLLImportFunctionVisitor Visitor; @@ -3163,6 +4003,11 @@ bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) { } } + // Inline builtins declaration must be emitted. They often are fortified + // functions. + if (F->isInlineBuiltinDeclaration()) + return true; + // PR9614. Avoid cases where the source code is lying to us. An available // externally function should have an equivalent function somewhere else, // but a function that calls itself through asm label/`__builtin_` trickery is @@ -3183,7 +4028,15 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD, auto *Spec = FD->getAttr<CPUSpecificAttr>(); for (unsigned I = 0; I < Spec->cpus_size(); ++I) EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr); - // Requires multiple emits. + } else if (FD->isTargetClonesMultiVersion()) { + auto *Clone = FD->getAttr<TargetClonesAttr>(); + for (unsigned I = 0; I < Clone->featuresStrs_size(); ++I) + if (Clone->isFirstOfVersion(I)) + EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr); + // Ensure that the resolver function is also emitted. + GetOrCreateMultiVersionResolver(GD); + } else if (FD->hasAttr<TargetVersionAttr>()) { + GetOrCreateMultiVersionResolver(GD); } else EmitGlobalFunctionDefinition(GD, GV); } @@ -3243,64 +4096,151 @@ static unsigned TargetMVPriority(const TargetInfo &TI, const CodeGenFunction::MultiVersionResolverOption &RO) { unsigned Priority = 0; - for (StringRef Feat : RO.Conditions.Features) + unsigned NumFeatures = 0; + for (StringRef Feat : RO.Conditions.Features) { Priority = std::max(Priority, TI.multiVersionSortPriority(Feat)); + NumFeatures++; + } if (!RO.Conditions.Architecture.empty()) Priority = std::max( Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture)); + + Priority += TI.multiVersionFeatureCost() * NumFeatures; + return Priority; } +// Multiversion functions should be at most 'WeakODRLinkage' so that a different +// TU can forward declare the function without causing problems. Particularly +// in the cases of CPUDispatch, this causes issues. This also makes sure we +// work with internal linkage functions, so that the same function name can be +// used with internal linkage in multiple TUs. +llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM, + GlobalDecl GD) { + const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); + if (FD->getFormalLinkage() == Linkage::Internal) + return llvm::GlobalValue::InternalLinkage; + return llvm::GlobalValue::WeakODRLinkage; +} + void CodeGenModule::emitMultiVersionFunctions() { std::vector<GlobalDecl> MVFuncsToEmit; MultiVersionFuncs.swap(MVFuncsToEmit); for (GlobalDecl GD : MVFuncsToEmit) { + const auto *FD = cast<FunctionDecl>(GD.getDecl()); + assert(FD && "Expected a FunctionDecl"); + SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options; - const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); - getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options](const FunctionDecl *CurFD) { - GlobalDecl CurGD{ - (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; - StringRef MangledName = getMangledName(CurGD); - llvm::Constant *Func = GetGlobalValue(MangledName); - if (!Func) { - if (CurFD->isDefined()) { - EmitGlobalFunctionDefinition(CurGD, nullptr); - Func = GetGlobalValue(MangledName); + if (FD->isTargetMultiVersion()) { + getContext().forEachMultiversionedFunctionVersion( + FD, [this, &GD, &Options](const FunctionDecl *CurFD) { + GlobalDecl CurGD{ + (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (CurFD->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = + getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); + } + assert(Func && "This should have just been created"); + } + if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) { + const auto *TA = CurFD->getAttr<TargetAttr>(); + llvm::SmallVector<StringRef, 8> Feats; + TA->getAddedFeatures(Feats); + Options.emplace_back(cast<llvm::Function>(Func), + TA->getArchitecture(), Feats); } else { - const CGFunctionInfo &FI = - getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); - Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, - /*DontDefer=*/false, ForDefinition); + const auto *TVA = CurFD->getAttr<TargetVersionAttr>(); + llvm::SmallVector<StringRef, 8> Feats; + TVA->getFeatures(Feats); + Options.emplace_back(cast<llvm::Function>(Func), + /*Architecture*/ "", Feats); } - assert(Func && "This should have just been created"); + }); + } else if (FD->isTargetClonesMultiVersion()) { + const auto *TC = FD->getAttr<TargetClonesAttr>(); + for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size(); + ++VersionIndex) { + if (!TC->isFirstOfVersion(VersionIndex)) + continue; + GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD), + VersionIndex}; + StringRef Version = TC->getFeatureStr(VersionIndex); + StringRef MangledName = getMangledName(CurGD); + llvm::Constant *Func = GetGlobalValue(MangledName); + if (!Func) { + if (FD->isDefined()) { + EmitGlobalFunctionDefinition(CurGD, nullptr); + Func = GetGlobalValue(MangledName); + } else { + const CGFunctionInfo &FI = + getTypes().arrangeGlobalDeclaration(CurGD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); } + assert(Func && "This should have just been created"); + } - const auto *TA = CurFD->getAttr<TargetAttr>(); - llvm::SmallVector<StringRef, 8> Feats; - TA->getAddedFeatures(Feats); - - Options.emplace_back(cast<llvm::Function>(Func), - TA->getArchitecture(), Feats); - }); + StringRef Architecture; + llvm::SmallVector<StringRef, 1> Feature; - llvm::Function *ResolverFunc; - const TargetInfo &TI = getTarget(); + if (getTarget().getTriple().isAArch64()) { + if (Version != "default") { + llvm::SmallVector<StringRef, 8> VerFeats; + Version.split(VerFeats, "+"); + for (auto &CurFeat : VerFeats) + Feature.push_back(CurFeat.trim()); + } + } else { + if (Version.starts_with("arch=")) + Architecture = Version.drop_front(sizeof("arch=") - 1); + else if (Version != "default") + Feature.push_back(Version); + } - if (TI.supportsIFunc() || FD->isTargetMultiVersion()) { - ResolverFunc = cast<llvm::Function>( - GetGlobalValue((getMangledName(GD) + ".resolver").str())); - ResolverFunc->setLinkage(llvm::Function::WeakODRLinkage); + Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature); + } } else { - ResolverFunc = cast<llvm::Function>(GetGlobalValue(getMangledName(GD))); + assert(0 && "Expected a target or target_clones multiversion function"); + continue; } - if (supportsCOMDAT()) + llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); + if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant)) { + ResolverConstant = IFunc->getResolver(); + if (FD->isTargetClonesMultiVersion()) { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); + std::string MangledName = getMangledNameImpl( + *this, GD, FD, /*OmitMultiVersionMangling=*/true); + // In prior versions of Clang, the mangling for ifuncs incorrectly + // included an .ifunc suffix. This alias is generated for backward + // compatibility. It is deprecated, and may be removed in the future. + auto *Alias = llvm::GlobalAlias::create( + DeclTy, 0, getMultiversionLinkage(*this, GD), + MangledName + ".ifunc", IFunc, &getModule()); + SetCommonAttributes(FD, Alias); + } + } + llvm::Function *ResolverFunc = cast<llvm::Function>(ResolverConstant); + + ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD)); + + if (!ResolverFunc->hasLocalLinkage() && supportsCOMDAT()) ResolverFunc->setComdat( getModule().getOrInsertComdat(ResolverFunc->getName())); + const TargetInfo &TI = getTarget(); llvm::stable_sort( Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS, const CodeGenFunction::MultiVersionResolverOption &RHS) { @@ -3325,24 +4265,24 @@ void CodeGenModule::emitMultiVersionFunctions() { void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); assert(FD && "Not a FunctionDecl?"); + assert(FD->isCPUDispatchMultiVersion() && "Not a multiversion function?"); const auto *DD = FD->getAttr<CPUDispatchAttr>(); assert(DD && "Not a cpu_dispatch Function?"); - llvm::Type *DeclTy = getTypes().ConvertType(FD->getType()); - if (const auto *CXXFD = dyn_cast<CXXMethodDecl>(FD)) { - const CGFunctionInfo &FInfo = getTypes().arrangeCXXMethodDeclaration(CXXFD); - DeclTy = getTypes().GetFunctionType(FInfo); - } + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); StringRef ResolverName = getMangledName(GD); + UpdateMultiVersionNames(GD, FD, ResolverName); llvm::Type *ResolverType; GlobalDecl ResolverGD; - if (getTarget().supportsIFunc()) + if (getTarget().supportsIFunc()) { ResolverType = llvm::FunctionType::get( llvm::PointerType::get(DeclTy, - Context.getTargetAddressSpace(FD->getType())), + getTypes().getTargetAddressSpace(FD->getType())), false); + } else { ResolverType = DeclTy; ResolverGD = GD; @@ -3350,7 +4290,7 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { auto *ResolverFunc = cast<llvm::Function>(GetOrCreateLLVMFunction( ResolverName, ResolverType, ResolverGD, /*ForVTable=*/false)); - ResolverFunc->setLinkage(llvm::Function::WeakODRLinkage); + ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD)); if (supportsCOMDAT()) ResolverFunc->setComdat( getModule().getOrInsertComdat(ResolverFunc->getName())); @@ -3386,10 +4326,9 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { Target.getCPUSpecificCPUDispatchFeatures(II->getName(), Features); llvm::transform(Features, Features.begin(), [](StringRef Str) { return Str.substr(1); }); - Features.erase(std::remove_if( - Features.begin(), Features.end(), [&Target](StringRef Feat) { - return !Target.validateCpuSupports(Feat); - }), Features.end()); + llvm::erase_if(Features, [&Target](StringRef Feat) { + return !Target.validateCpuSupports(Feat); + }); Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features); ++Index; } @@ -3397,8 +4336,8 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { llvm::stable_sort( Options, [](const CodeGenFunction::MultiVersionResolverOption &LHS, const CodeGenFunction::MultiVersionResolverOption &RHS) { - return CodeGenFunction::GetX86CpuSupportsMask(LHS.Conditions.Features) > - CodeGenFunction::GetX86CpuSupportsMask(RHS.Conditions.Features); + return llvm::X86::getCpuSupportsMask(LHS.Conditions.Features) > + llvm::X86::getCpuSupportsMask(RHS.Conditions.Features); }); // If the list contains multiple 'default' versions, such as when it contains @@ -3406,8 +4345,9 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { // always run on at least a 'pentium'). We do this by deleting the 'least // advanced' (read, lowest mangling letter). while (Options.size() > 1 && - CodeGenFunction::GetX86CpuSupportsMask( - (Options.end() - 2)->Conditions.Features) == 0) { + llvm::all_of(llvm::X86::getCpuSupportsMask( + (Options.end() - 2)->Conditions.Features), + [](auto X) { return X == 0; })) { StringRef LHSName = (Options.end() - 2)->Function->getName(); StringRef RHSName = (Options.end() - 1)->Function->getName(); if (LHSName.compare(RHSName) < 0) @@ -3420,16 +4360,27 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { CGF.EmitMultiVersionResolver(ResolverFunc, Options); if (getTarget().supportsIFunc()) { + llvm::GlobalValue::LinkageTypes Linkage = getMultiversionLinkage(*this, GD); + auto *IFunc = cast<llvm::GlobalValue>(GetOrCreateMultiVersionResolver(GD)); + + // Fix up function declarations that were created for cpu_specific before + // cpu_dispatch was known + if (!isa<llvm::GlobalIFunc>(IFunc)) { + assert(cast<llvm::Function>(IFunc)->isDeclaration()); + auto *GI = llvm::GlobalIFunc::create(DeclTy, 0, Linkage, "", ResolverFunc, + &getModule()); + GI->takeName(IFunc); + IFunc->replaceAllUsesWith(GI); + IFunc->eraseFromParent(); + IFunc = GI; + } + std::string AliasName = getMangledNameImpl( *this, GD, FD, /*OmitMultiVersionMangling=*/true); llvm::Constant *AliasFunc = GetGlobalValue(AliasName); if (!AliasFunc) { - auto *IFunc = cast<llvm::GlobalIFunc>(GetOrCreateLLVMFunction( - AliasName, DeclTy, GD, /*ForVTable=*/false, /*DontDefer=*/true, - /*IsThunk=*/false, llvm::AttributeList(), NotForDefinition)); - auto *GA = llvm::GlobalAlias::create( - DeclTy, 0, getFunctionLinkage(GD), AliasName, IFunc, &getModule()); - GA->setLinkage(llvm::Function::WeakODRLinkage); + auto *GA = llvm::GlobalAlias::create(DeclTy, 0, Linkage, AliasName, IFunc, + &getModule()); SetCommonAttributes(GD, GA); } } @@ -3437,39 +4388,48 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { /// If a dispatcher for the specified mangled name is not in the module, create /// and return an llvm Function with the specified type. -llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver( - GlobalDecl GD, llvm::Type *DeclTy, const FunctionDecl *FD) { +llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { + const auto *FD = cast<FunctionDecl>(GD.getDecl()); + assert(FD && "Not a FunctionDecl?"); + std::string MangledName = getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true); // Holds the name of the resolver, in ifunc mode this is the ifunc (which has // a separate resolver). std::string ResolverName = MangledName; - if (getTarget().supportsIFunc()) - ResolverName += ".ifunc"; - else if (FD->isTargetMultiVersion()) + if (getTarget().supportsIFunc()) { + if (!FD->isTargetClonesMultiVersion()) + ResolverName += ".ifunc"; + } else if (FD->isTargetMultiVersion()) { ResolverName += ".resolver"; + } - // If this already exists, just return that one. + // If the resolver has already been created, just return it. if (llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName)) return ResolverGV; - // Since this is the first time we've created this IFunc, make sure - // that we put this multiversioned function into the list to be - // replaced later if necessary (target multiversioning only). - if (!FD->isCPUDispatchMultiVersion() && !FD->isCPUSpecificMultiVersion()) + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); + + // The resolver needs to be created. For target and target_clones, defer + // creation until the end of the TU. + if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion()) MultiVersionFuncs.push_back(GD); - if (getTarget().supportsIFunc()) { + // For cpu_specific, don't create an ifunc yet because we don't know if the + // cpu_dispatch will be emitted in this translation unit. + if (getTarget().supportsIFunc() && !FD->isCPUSpecificMultiVersion()) { llvm::Type *ResolverType = llvm::FunctionType::get( - llvm::PointerType::get( - DeclTy, getContext().getTargetAddressSpace(FD->getType())), + llvm::PointerType::get(DeclTy, + getTypes().getTargetAddressSpace(FD->getType())), false); llvm::Constant *Resolver = GetOrCreateLLVMFunction( MangledName + ".resolver", ResolverType, GlobalDecl{}, /*ForVTable=*/false); - llvm::GlobalIFunc *GIF = llvm::GlobalIFunc::create( - DeclTy, 0, llvm::Function::WeakODRLinkage, "", Resolver, &getModule()); + llvm::GlobalIFunc *GIF = + llvm::GlobalIFunc::create(DeclTy, 0, getMultiversionLinkage(*this, GD), + "", Resolver, &getModule()); GIF->setName(ResolverName); SetCommonAttributes(FD, GIF); @@ -3501,7 +4461,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( // the iFunc instead. Name Mangling will handle the rest of the changes. if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(D)) { // For the device mark the function as one that should be emitted. - if (getLangOpts().OpenMPIsDevice && OpenMPRuntime && + if (getLangOpts().OpenMPIsTargetDevice && OpenMPRuntime && !OpenMPRuntime->markAsGlobalTarget(GD) && FD->isDefined() && !DontDefer && !IsForDefinition) { if (const FunctionDecl *FDDef = FD->getDefinition()) { @@ -3517,10 +4477,9 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( } if (FD->isMultiVersion()) { - if (FD->hasAttr<TargetAttr>()) - UpdateMultiVersionNames(GD, FD); + UpdateMultiVersionNames(GD, FD, MangledName); if (!IsForDefinition) - return GetOrCreateMultiVersionResolver(GD, Ty, FD); + return GetOrCreateMultiVersionResolver(GD); } } @@ -3534,7 +4493,8 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( } // Handle dropped DLL attributes. - if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>()) { + if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() && + !shouldMapVisibilityToDLLExport(cast_or_null<NamedDecl>(D))) { Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); setDSOLocal(Entry); } @@ -3565,7 +4525,7 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( // (If function is requested for a definition, we always need to create a new // function, not just return a bitcast.) if (!IsForDefinition) - return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo()); + return Entry; } // This function doesn't have a complete type (for example, the return @@ -3585,6 +4545,11 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( llvm::Function::Create(FTy, llvm::Function::ExternalLinkage, Entry ? StringRef() : MangledName, &getModule()); + // Store the declaration associated with this function so it is potentially + // updated by further declarations or definitions and emitted at the end. + if (D && D->hasAttr<AnnotateAttr>()) + DeferredAnnotations[MangledName] = cast<ValueDecl>(D); + // If we already created a function with the same mangled name (but different // type) before, take its name and add it to the list of functions to be // replaced with F at the end of CodeGen. @@ -3605,24 +4570,22 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( Entry->removeDeadConstantUsers(); } - llvm::Constant *BC = llvm::ConstantExpr::getBitCast( - F, Entry->getValueType()->getPointerTo()); - addGlobalValReplacement(Entry, BC); + addGlobalValReplacement(Entry, F); } assert(F->getName() == MangledName && "name was uniqued!"); if (D) SetFunctionAttributes(GD, F, IsIncompleteFunction, IsThunk); - if (ExtraAttrs.hasAttributes(llvm::AttributeList::FunctionIndex)) { - llvm::AttrBuilder B(ExtraAttrs, llvm::AttributeList::FunctionIndex); - F->addAttributes(llvm::AttributeList::FunctionIndex, B); + if (ExtraAttrs.hasFnAttrs()) { + llvm::AttrBuilder B(F->getContext(), ExtraAttrs.getFnAttrs()); + F->addFnAttrs(B); } if (!DontDefer) { // All MSVC dtors other than the base dtor are linkonce_odr and delegate to // each other bottoming out with the base dtor. Therefore we emit non-base // dtors on usage, even if there is no dtor definition in the TU. - if (D && isa<CXXDestructorDecl>(D) && + if (isa_and_nonnull<CXXDestructorDecl>(D) && getCXXABI().useThunkForDtorVariant(cast<CXXDestructorDecl>(D), GD.getDtorType())) addDeferredDeclToEmit(GD); @@ -3669,20 +4632,16 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction( return F; } - llvm::Type *PTy = llvm::PointerType::getUnqual(Ty); - return llvm::ConstantExpr::getBitCast(F, PTy); + return F; } /// GetAddrOfFunction - Return the address of the given function. If Ty is /// non-null, then this function will use the specified type if it has to /// create it (this occurs when we see a definition of the function). -llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD, - llvm::Type *Ty, - bool ForVTable, - bool DontDefer, - ForDefinition_t IsForDefinition) { - assert(!cast<FunctionDecl>(GD.getDecl())->isConsteval() && - "consteval function should never be emitted"); +llvm::Constant * +CodeGenModule::GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty, bool ForVTable, + bool DontDefer, + ForDefinition_t IsForDefinition) { // If there was no specific requested type, just convert it now. if (!Ty) { const auto *FD = cast<FunctionDecl>(GD.getDecl()); @@ -3710,11 +4669,18 @@ llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD, cast<llvm::Function>(F->stripPointerCasts()), GD); if (IsForDefinition) return F; - return llvm::ConstantExpr::getBitCast(Handle, Ty->getPointerTo()); + return Handle; } return F; } +llvm::Constant *CodeGenModule::GetFunctionStart(const ValueDecl *Decl) { + llvm::GlobalValue *F = + cast<llvm::GlobalValue>(GetAddrOfFunction(Decl)->stripPointerCasts()); + + return llvm::NoCFIValue::get(F); +} + static const FunctionDecl * GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) { TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl(); @@ -3761,8 +4727,7 @@ CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name, bool AssumeConvergent) { if (AssumeConvergent) { ExtraAttrs = - ExtraAttrs.addAttribute(VMContext, llvm::AttributeList::FunctionIndex, - llvm::Attribute::Convergent); + ExtraAttrs.addFnAttribute(VMContext, llvm::Attribute::Convergent); } llvm::Constant *C = @@ -3794,26 +4759,6 @@ CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name, return {FTy, C}; } -/// isTypeConstant - Determine whether an object of this type can be emitted -/// as a constant. -/// -/// If ExcludeCtor is true, the duration when the object's constructor runs -/// will not be considered. The caller will need to verify that the object is -/// not written to during its construction. -bool CodeGenModule::isTypeConstant(QualType Ty, bool ExcludeCtor) { - if (!Ty.isConstant(Context) && !Ty->isReferenceType()) - return false; - - if (Context.getLangOpts().CPlusPlus) { - if (const CXXRecordDecl *Record - = Context.getBaseElementType(Ty)->getAsCXXRecordDecl()) - return ExcludeCtor && !Record->hasMutableFields() && - Record->hasTrivialDestructor(); - } - - return true; -} - /// GetOrCreateLLVMGlobal - If the specified mangled name is not in the module, /// create and return an llvm GlobalVariable with the specified type and address /// space. If there is something in the module with the specified name, return @@ -3827,10 +4772,11 @@ bool CodeGenModule::isTypeConstant(QualType Ty, bool ExcludeCtor) { /// mangled name but some other type. llvm::Constant * CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, - unsigned AddrSpace, const VarDecl *D, + LangAS AddrSpace, const VarDecl *D, ForDefinition_t IsForDefinition) { // Lookup the entry, lazily creating it if necessary. llvm::GlobalValue *Entry = GetGlobalValue(MangledName); + unsigned TargetAS = getContext().getTargetAddressSpace(AddrSpace); if (Entry) { if (WeakRefReferences.erase(Entry)) { if (D && !D->hasAttr<WeakAttr>()) @@ -3838,13 +4784,14 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, } // Handle dropped DLL attributes. - if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>()) + if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>() && + !shouldMapVisibilityToDLLExport(D)) Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass); if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D) getOpenMPRuntime().registerTargetGlobalVariable(D, Entry); - if (Entry->getValueType() == Ty && Entry->getAddressSpace() == AddrSpace) + if (Entry->getValueType() == Ty && Entry->getAddressSpace() == TargetAS) return Entry; // If there are two attempts to define the same mangled name, issue an @@ -3868,24 +4815,22 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, } // Make sure the result is of the correct type. - if (Entry->getType()->getAddressSpace() != AddrSpace) { - return llvm::ConstantExpr::getAddrSpaceCast(Entry, - Ty->getPointerTo(AddrSpace)); - } + if (Entry->getType()->getAddressSpace() != TargetAS) + return llvm::ConstantExpr::getAddrSpaceCast( + Entry, llvm::PointerType::get(Ty->getContext(), TargetAS)); // (If global is requested for a definition, we always need to create a new // global, not just return a bitcast.) if (!IsForDefinition) - return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo(AddrSpace)); + return Entry; } auto DAddrSpace = GetGlobalVarAddressSpace(D); - auto TargetAddrSpace = getContext().getTargetAddressSpace(DAddrSpace); auto *GV = new llvm::GlobalVariable( getModule(), Ty, false, llvm::GlobalValue::ExternalLinkage, nullptr, MangledName, nullptr, llvm::GlobalVariable::NotThreadLocal, - TargetAddrSpace); + getContext().getTargetAddressSpace(DAddrSpace)); // If we already created a global with the same mangled name (but different // type) before, take its name and remove it from its parent. @@ -3893,9 +4838,7 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, GV->takeName(Entry); if (!Entry->use_empty()) { - llvm::Constant *NewPtrForOldDecl = - llvm::ConstantExpr::getBitCast(GV, Entry->getType()); - Entry->replaceAllUsesWith(NewPtrForOldDecl); + Entry->replaceAllUsesWith(GV); } Entry->eraseFromParent(); @@ -3919,7 +4862,7 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, // FIXME: This code is overly simple and should be merged with other global // handling. - GV->setConstant(isTypeConstant(D->getType(), false)); + GV->setConstant(D->getType().isConstantStorage(getContext(), false, false)); GV->setAlignment(getContext().getDeclAlign(D).getAsAlign()); @@ -3952,6 +4895,10 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, isExternallyVisible(D->getLinkageAndVisibility().getLinkage())) GV->setSection(".cp.rodata"); + // Handle code model attribute + if (const auto *CMA = D->getAttr<CodeModelAttr>()) + GV->setCodeModel(CMA->getModel()); + // Check if we a have a const declaration with an initializer, we may be // able to emit it as available_externally to expose it's value to the // optimizer. @@ -3996,7 +4943,8 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, } } - if (GV->isDeclaration()) { + if (D && + D->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly) { getTargetCodeGenInfo().setTargetAttributes(D, GV, *this); // External HIP managed variables needed to be recorded for transformation // in both device and host compilations. @@ -4005,13 +4953,17 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, getCUDARuntime().handleVarRegistration(D, *GV); } + if (D) + SanitizerMD->reportGlobal(GV, *D); + LangAS ExpectedAS = D ? D->getType().getAddressSpace() : (LangOpts.OpenCL ? LangAS::opencl_global : LangAS::Default); - assert(getContext().getTargetAddressSpace(ExpectedAS) == AddrSpace); + assert(getContext().getTargetAddressSpace(ExpectedAS) == TargetAS); if (DAddrSpace != ExpectedAS) { return getTargetCodeGenInfo().performAddrSpaceCast( - *this, GV, DAddrSpace, ExpectedAS, Ty->getPointerTo(AddrSpace)); + *this, GV, DAddrSpace, ExpectedAS, + llvm::PointerType::get(getLLVMContext(), TargetAS)); } return GV; @@ -4045,7 +4997,7 @@ CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition) { llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable( StringRef Name, llvm::Type *Ty, llvm::GlobalValue::LinkageTypes Linkage, - unsigned Alignment) { + llvm::Align Alignment) { llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name); llvm::GlobalVariable *OldGV = nullptr; @@ -4069,9 +5021,7 @@ llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable( GV->takeName(OldGV); if (!OldGV->use_empty()) { - llvm::Constant *NewPtrForOldDecl = - llvm::ConstantExpr::getBitCast(GV, OldGV->getType()); - OldGV->replaceAllUsesWith(NewPtrForOldDecl); + OldGV->replaceAllUsesWith(GV); } OldGV->eraseFromParent(); @@ -4081,7 +5031,7 @@ llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable( !GV->hasAvailableExternallyLinkage()) GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); - GV->setAlignment(llvm::MaybeAlign(Alignment)); + GV->setAlignment(Alignment); return GV; } @@ -4101,8 +5051,7 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D, Ty = getTypes().ConvertTypeForMem(ASTTy); StringRef MangledName = getMangledName(D); - return GetOrCreateLLVMGlobal(MangledName, Ty, - getContext().getTargetAddressSpace(ASTTy), D, + return GetOrCreateLLVMGlobal(MangledName, Ty, ASTTy.getAddressSpace(), D, IsForDefinition); } @@ -4111,10 +5060,8 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D, llvm::Constant * CodeGenModule::CreateRuntimeVariable(llvm::Type *Ty, StringRef Name) { - auto AddrSpace = - getContext().getLangOpts().OpenCL - ? getContext().getTargetAddressSpace(LangAS::opencl_global) - : 0; + LangAS AddrSpace = getContext().getLangOpts().OpenCL ? LangAS::opencl_global + : LangAS::Default; auto *Ret = GetOrCreateLLVMGlobal(Name, Ty, AddrSpace, nullptr); setDSOLocal(cast<llvm::GlobalValue>(Ret->stripPointerCasts())); return Ret; @@ -4153,16 +5100,15 @@ CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const { } LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) { - LangAS AddrSpace = LangAS::Default; if (LangOpts.OpenCL) { - AddrSpace = D ? D->getType().getAddressSpace() : LangAS::opencl_global; - assert(AddrSpace == LangAS::opencl_global || - AddrSpace == LangAS::opencl_global_device || - AddrSpace == LangAS::opencl_global_host || - AddrSpace == LangAS::opencl_constant || - AddrSpace == LangAS::opencl_local || - AddrSpace >= LangAS::FirstTargetAddressSpace); - return AddrSpace; + LangAS AS = D ? D->getType().getAddressSpace() : LangAS::opencl_global; + assert(AS == LangAS::opencl_global || + AS == LangAS::opencl_global_device || + AS == LangAS::opencl_global_host || + AS == LangAS::opencl_constant || + AS == LangAS::opencl_local || + AS >= LangAS::FirstTargetAddressSpace); + return AS; } if (LangOpts.SYCLIsDevice && @@ -4170,16 +5116,17 @@ LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) { return LangAS::sycl_global; if (LangOpts.CUDA && LangOpts.CUDAIsDevice) { - if (D && D->hasAttr<CUDAConstantAttr>()) - return LangAS::cuda_constant; - else if (D && D->hasAttr<CUDASharedAttr>()) - return LangAS::cuda_shared; - else if (D && D->hasAttr<CUDADeviceAttr>()) - return LangAS::cuda_device; - else if (D && D->getType().isConstQualified()) - return LangAS::cuda_constant; - else - return LangAS::cuda_device; + if (D) { + if (D->hasAttr<CUDAConstantAttr>()) + return LangAS::cuda_constant; + if (D->hasAttr<CUDASharedAttr>()) + return LangAS::cuda_shared; + if (D->hasAttr<CUDADeviceAttr>()) + return LangAS::cuda_device; + if (D->getType().isConstQualified()) + return LangAS::cuda_constant; + } + return LangAS::cuda_device; } if (LangOpts.OpenMP) { @@ -4196,8 +5143,16 @@ LangAS CodeGenModule::GetGlobalConstantAddressSpace() const { return LangAS::opencl_constant; if (LangOpts.SYCLIsDevice) return LangAS::sycl_global; + if (LangOpts.HIP && LangOpts.CUDAIsDevice && getTriple().isSPIRV()) + // For HIPSPV map literals to cuda_device (maps to CrossWorkGroup in SPIR-V) + // instead of default AS (maps to Generic in SPIR-V). Otherwise, we end up + // with OpVariable instructions with Generic storage class which is not + // allowed (SPIR-V V1.6 s3.42.8). Also, mapping literals to SPIR-V + // UniformConstant storage class is not viable as pointers to it may not be + // casted to Generic pointers which are used to model HIP's "flat" pointers. + return LangAS::cuda_device; if (auto AS = getTarget().getConstantAddressSpace()) - return AS.getValue(); + return *AS; return LangAS::Default; } @@ -4218,7 +5173,8 @@ castStringLiteralToDefaultAddressSpace(CodeGenModule &CGM, if (AS != LangAS::Default) Cast = CGM.getTargetCodeGenInfo().performAddrSpaceCast( CGM, GV, AS, LangAS::Default, - GV->getValueType()->getPointerTo( + llvm::PointerType::get( + CGM.getLLVMContext(), CGM.getContext().getTargetAddressSpace(LangAS::Default))); } return Cast; @@ -4236,7 +5192,7 @@ void CodeGenModule::MaybeHandleStaticInExternC(const SomeDecl *D, return; // Must have internal linkage and an ordinary name. - if (!D->getIdentifier() || D->getFormalLinkage() != InternalLinkage) + if (!D->getIdentifier() || D->getFormalLinkage() != Linkage::Internal) return; // Must be in an extern "C" context. Entities declared directly within @@ -4261,11 +5217,6 @@ static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) { if (!CGM.supportsCOMDAT()) return false; - // Do not set COMDAT attribute for CUDA/HIP stub functions to prevent - // them being "merged" by the COMDAT Folding linker optimization. - if (D.hasAttr<CUDAGlobalAttr>()) - return false; - if (D.hasAttr<SelectAnyAttr>()) return true; @@ -4287,6 +5238,10 @@ static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) { llvm_unreachable("No such linkage"); } +bool CodeGenModule::supportsCOMDAT() const { + return getTriple().supportsCOMDAT(); +} + void CodeGenModule::maybeSetTrivialComdat(const Decl &D, llvm::GlobalObject &GO) { if (!shouldBeInCOMDAT(*this, D)) @@ -4305,19 +5260,25 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, // If this is OpenMP device, check if it is legal to emit this global // normally. - if (LangOpts.OpenMPIsDevice && OpenMPRuntime && + if (LangOpts.OpenMPIsTargetDevice && OpenMPRuntime && OpenMPRuntime->emitTargetGlobalVariable(D)) return; llvm::TrackingVH<llvm::Constant> Init; bool NeedsGlobalCtor = false; + // Whether the definition of the variable is available externally. + // If yes, we shouldn't emit the GloablCtor and GlobalDtor for the variable + // since this is the job for its original source. + bool IsDefinitionAvailableExternally = + getContext().GetGVALinkageForVariable(D) == GVA_AvailableExternally; bool NeedsGlobalDtor = + !IsDefinitionAvailableExternally && D->needsDestruction(getContext()) == QualType::DK_cxx_destructor; const VarDecl *InitDecl; const Expr *InitExpr = D->getAnyInitializer(InitDecl); - Optional<ConstantEmitter> emitter; + std::optional<ConstantEmitter> emitter; // CUDA E.2.4.1 "__shared__ variables cannot have an initialization // as part of their declaration." Sema has already checked for @@ -4362,8 +5323,12 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, T = D->getType(); if (getLangOpts().CPlusPlus) { + if (InitDecl->hasFlexibleArrayInit(getContext())) + ErrorUnsupported(D, "flexible array initializer"); Init = EmitNullConstant(T); - NeedsGlobalCtor = true; + + if (!IsDefinitionAvailableExternally) + NeedsGlobalCtor = true; } else { ErrorUnsupported(D, "static initializer"); Init = llvm::UndefValue::get(getTypes().ConvertType(T)); @@ -4375,6 +5340,14 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, // also don't need to register a destructor. if (getLangOpts().CPlusPlus && !NeedsGlobalDtor) DelayedCXXInitPosition.erase(D); + +#ifndef NDEBUG + CharUnits VarSize = getContext().getTypeSizeInChars(ASTTy) + + InitDecl->getFlexibleArrayInitChars(getContext()); + CharUnits CstSize = CharUnits::fromQuantity( + getDataLayout().getTypeAllocSize(Init->getType())); + assert(VarSize == CstSize && "Emitted constant has unexpected size"); +#endif } } @@ -4425,8 +5398,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, AddGlobalAnnotations(D, GV); // Set the llvm linkage type as appropriate. - llvm::GlobalValue::LinkageTypes Linkage = - getLLVMLinkageVarDefinition(D, GV->isConstant()); + llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(D); // CUDA B.2.1 "The __device__ qualifier declares a variable that resides on // the device. [...]" @@ -4435,10 +5407,12 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, // Is accessible from all the threads within the grid and from the host // through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize() // / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())." - if (GV && LangOpts.CUDA) { + if (LangOpts.CUDA) { if (LangOpts.CUDAIsDevice) { if (Linkage != llvm::GlobalValue::InternalLinkage && - (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>())) + (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() || + D->getType()->isCUDADeviceBuiltinSurfaceType() || + D->getType()->isCUDADeviceBuiltinTextureType())) GV->setExternallyInitialized(true); } else { getCUDARuntime().internalizeDeviceSideVar(D, Linkage); @@ -4452,7 +5426,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, // If it is safe to mark the global 'constant', do so now. GV->setConstant(!NeedsGlobalCtor && !NeedsGlobalDtor && - isTypeConstant(D->getType(), true)); + D->getType().isConstantStorage(getContext(), true, true)); // If it is in a read-only section, mark it 'constant'. if (const SectionAttr *SA = D->getAttr<SectionAttr>()) { @@ -4461,7 +5435,12 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, GV->setConstant(true); } - GV->setAlignment(getContext().getDeclAlign(D).getAsAlign()); + CharUnits AlignVal = getContext().getDeclAlign(D); + // Check for alignment specifed in an 'omp allocate' directive. + if (std::optional<CharUnits> AlignValFromAllocate = + getOMPAllocateAlignment(D)) + AlignVal = *AlignValFromAllocate; + GV->setAlignment(AlignVal.getAsAlign()); // On Darwin, unlike other Itanium C++ ABI platforms, the thread-wrapper // function is only defined alongside the variable, not also alongside @@ -4516,7 +5495,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D, if (NeedsGlobalCtor || NeedsGlobalDtor) EmitCXXGlobalVarDeclInitFunc(D, GV, NeedsGlobalCtor); - SanitizerMD->reportGlobalToASan(GV, *D, NeedsGlobalCtor); + SanitizerMD->reportGlobal(GV, *D, NeedsGlobalCtor); // Emit global variable debug information. if (CGDebugInfo *DI = getModuleDebugInfo()) @@ -4529,8 +5508,8 @@ void CodeGenModule::EmitExternalVarDeclaration(const VarDecl *D) { if (getCodeGenOpts().hasReducedDebugInfo()) { QualType ASTTy = D->getType(); llvm::Type *Ty = getTypes().ConvertTypeForMem(D->getType()); - llvm::Constant *GV = GetOrCreateLLVMGlobal( - D->getName(), Ty, getContext().getTargetAddressSpace(ASTTy), D); + llvm::Constant *GV = + GetOrCreateLLVMGlobal(D->getName(), Ty, ASTTy.getAddressSpace(), D); DI->EmitExternalVariable( cast<llvm::GlobalVariable>(GV->stripPointerCasts()), D); } @@ -4612,17 +5591,14 @@ static bool isVarDeclStrongDefinition(const ASTContext &Context, return false; } -llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageForDeclarator( - const DeclaratorDecl *D, GVALinkage Linkage, bool IsConstantVariable) { +llvm::GlobalValue::LinkageTypes +CodeGenModule::getLLVMLinkageForDeclarator(const DeclaratorDecl *D, + GVALinkage Linkage) { if (Linkage == GVA_Internal) return llvm::Function::InternalLinkage; - if (D->hasAttr<WeakAttr>()) { - if (IsConstantVariable) - return llvm::GlobalVariable::WeakODRLinkage; - else - return llvm::GlobalVariable::WeakAnyLinkage; - } + if (D->hasAttr<WeakAttr>()) + return llvm::GlobalVariable::WeakAnyLinkage; if (const auto *FD = D->getAsFunction()) if (FD->isMultiVersion() && Linkage == GVA_AvailableExternally) @@ -4687,10 +5663,10 @@ llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageForDeclarator( return llvm::GlobalVariable::ExternalLinkage; } -llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageVarDefinition( - const VarDecl *VD, bool IsConstant) { +llvm::GlobalValue::LinkageTypes +CodeGenModule::getLLVMLinkageVarDefinition(const VarDecl *VD) { GVALinkage Linkage = getContext().GetGVALinkageForVariable(VD); - return getLLVMLinkageForDeclarator(VD, Linkage, IsConstant); + return getLLVMLinkageForDeclarator(VD, Linkage); } /// Replace the uses of a function that was declared with a non-proto type. @@ -4747,7 +5723,7 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, } // Add any parameter attributes. - newArgAttrs.push_back(oldAttrs.getParamAttributes(argNo)); + newArgAttrs.push_back(oldAttrs.getParamAttrs(argNo)); argNo++; } if (dontTransform) @@ -4762,7 +5738,7 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, callSite->getOperandBundlesAsDefs(newBundles); llvm::CallBase *newCall; - if (dyn_cast<llvm::CallInst>(callSite)) { + if (isa<llvm::CallInst>(callSite)) { newCall = llvm::CallInst::Create(newFn, newArgs, newBundles, "", callSite); } else { @@ -4775,9 +5751,9 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, if (!newCall->getType()->isVoidTy()) newCall->takeName(callSite); - newCall->setAttributes(llvm::AttributeList::get( - newFn->getContext(), oldAttrs.getFnAttributes(), - oldAttrs.getRetAttributes(), newArgAttrs)); + newCall->setAttributes( + llvm::AttributeList::get(newFn->getContext(), oldAttrs.getFnAttrs(), + oldAttrs.getRetAttrs(), newArgAttrs)); newCall->setCallingConv(callSite->getCallingConv()); // Finally, remove the old call, replacing any uses with the new one. @@ -4855,9 +5831,6 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD, maybeSetTrivialComdat(*D, *Fn); - // Set CodeGen attributes that represent floating point environment. - setLLVMFunctionFEnvAttributes(D, Fn); - CodeGenFunction(*this).GenerateCode(GD, Fn, FI); setNonAliasAttributes(GD, Fn); @@ -4867,8 +5840,8 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD, AddGlobalCtor(Fn, CA->getPriority()); if (const DestructorAttr *DA = D->getAttr<DestructorAttr>()) AddGlobalDtor(Fn, DA->getPriority(), true); - if (D->hasAttr<AnnotateAttr>()) - AddGlobalAnnotations(D, Fn); + if (getLangOpts().OpenMP && D->hasAttr<OMPDeclareTargetDeclAttr>()) + getOpenMPRuntime().emitDeclareTargetFunction(D, GV); } void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) { @@ -4902,10 +5875,10 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) { /*ForVTable=*/false); LT = getFunctionLinkage(GD); } else { - Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, 0, + Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, LangAS::Default, /*D=*/nullptr); if (const auto *VD = dyn_cast<VarDecl>(GD.getDecl())) - LT = getLLVMLinkageVarDefinition(VD, D->getType().isConstQualified()); + LT = getLLVMLinkageVarDefinition(VD); else LT = getFunctionLinkage(GD); } @@ -4932,8 +5905,7 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) { // Remove it and replace uses of it with the alias. GA->takeName(Entry); - Entry->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(GA, - Entry->getType())); + Entry->replaceAllUsesWith(GA); Entry->eraseFromParent(); } else { GA->setName(MangledName); @@ -4952,6 +5924,11 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) { setTLSMode(GA, *VD); SetCommonAttributes(GD, GA); + + // Emit global alias debug information. + if (isa<VarDecl>(D)) + if (CGDebugInfo *DI = getModuleDebugInfo()) + DI->EmitGlobalAlias(cast<llvm::GlobalValue>(GA->getAliasee()->stripPointerCasts()), GD); } void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) { @@ -4983,8 +5960,9 @@ void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) { Aliases.push_back(GD); llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType()); + llvm::Type *ResolverTy = llvm::GlobalIFunc::getResolverFunctionType(DeclTy); llvm::Constant *Resolver = - GetOrCreateLLVMFunction(IFA->getResolver(), DeclTy, GD, + GetOrCreateLLVMFunction(IFA->getResolver(), ResolverTy, {}, /*ForVTable=*/false); llvm::GlobalIFunc *GIF = llvm::GlobalIFunc::create(DeclTy, 0, llvm::Function::ExternalLinkage, @@ -5005,12 +5983,13 @@ void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) { // Remove it and replace uses of it with the ifunc. GIF->takeName(Entry); - Entry->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(GIF, - Entry->getType())); + Entry->replaceAllUsesWith(GIF); Entry->eraseFromParent(); } else GIF->setName(MangledName); - + if (auto *F = dyn_cast<llvm::Function>(Resolver)) { + F->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation); + } SetCommonAttributes(GD, GIF); } @@ -5064,7 +6043,8 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { StringLength); if (auto *C = Entry.second) - return ConstantAddress(C, CharUnits::fromQuantity(C->getAlignment())); + return ConstantAddress( + C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment())); llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty); llvm::Constant *Zeros[] = { Zero, Zero }; @@ -5086,7 +6066,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { switch (CFRuntime) { default: break; - case LangOptions::CoreFoundationABI::Swift: LLVM_FALLTHROUGH; + case LangOptions::CoreFoundationABI::Swift: [[fallthrough]]; case LangOptions::CoreFoundationABI::Swift5_0: CFConstantStringClassName = Triple.isOSDarwin() ? "$s15SwiftFoundation19_NSCFConstantStringCN" @@ -5151,7 +6131,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { auto Fields = Builder.beginStruct(STy); // Class pointer. - Fields.add(cast<llvm::ConstantExpr>(CFConstantStringClassRef)); + Fields.add(cast<llvm::Constant>(CFConstantStringClassRef)); // Flags. if (IsSwiftABI) { @@ -5164,7 +6144,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { // String pointer. llvm::Constant *C = nullptr; if (isUTF16) { - auto Arr = llvm::makeArrayRef( + auto Arr = llvm::ArrayRef( reinterpret_cast<uint16_t *>(const_cast<char *>(Entry.first().data())), Entry.first().size() / 2); C = llvm::ConstantDataArray::get(VMContext, Arr); @@ -5173,7 +6153,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { } // Note: -fwritable-strings doesn't make the backing store strings of - // CFStrings writable. (See <rdar://problem/10657500>) + // CFStrings writable. auto *GV = new llvm::GlobalVariable(getModule(), C->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, C, ".str"); @@ -5199,9 +6179,6 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { llvm::Constant *Str = llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); - if (isUTF16) - // Cast the UTF16 string to the correct type. - Str = llvm::ConstantExpr::getBitCast(Str, Int8PtrTy); Fields.add(Str); // String length. @@ -5230,10 +6207,11 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { switch (Triple.getObjectFormat()) { case llvm::Triple::UnknownObjectFormat: llvm_unreachable("unknown file format"); + case llvm::Triple::DXContainer: case llvm::Triple::GOFF: - llvm_unreachable("GOFF is not yet implemented"); + case llvm::Triple::SPIRV: case llvm::Triple::XCOFF: - llvm_unreachable("XCOFF is not yet implemented"); + llvm_unreachable("unimplemented"); case llvm::Triple::COFF: case llvm::Triple::ELF: case llvm::Triple::Wasm: @@ -5245,7 +6223,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { } Entry.second = GV; - return ConstantAddress(GV, Alignment); + return ConstantAddress(GV, GV->getValueType(), Alignment); } bool CodeGenModule::getExpressionLocationsEnabled() const { @@ -5258,12 +6236,10 @@ QualType CodeGenModule::getObjCFastEnumerationStateType() { D->startDefinition(); QualType FieldTypes[] = { - Context.UnsignedLongTy, - Context.getPointerType(Context.getObjCIdType()), - Context.getPointerType(Context.UnsignedLongTy), - Context.getConstantArrayType(Context.UnsignedLongTy, - llvm::APInt(32, 5), nullptr, ArrayType::Normal, 0) - }; + Context.UnsignedLongTy, Context.getPointerType(Context.getObjCIdType()), + Context.getPointerType(Context.UnsignedLongTy), + Context.getConstantArrayType(Context.UnsignedLongTy, llvm::APInt(32, 5), + nullptr, ArraySizeModifier::Normal, 0)}; for (size_t i = 0; i < 4; ++i) { FieldDecl *Field = FieldDecl::Create(Context, @@ -5296,6 +6272,7 @@ CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) { // Resize the string to the right size, which is indicated by its type. const ConstantArrayType *CAT = Context.getAsConstantArrayType(E->getType()); + assert(CAT && "String literal not of constant array type!"); Str.resize(CAT->getSize().getZExtValue()); return llvm::ConstantDataArray::getString(VMContext, Str, false); } @@ -5360,10 +6337,10 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, if (!LangOpts.WritableStrings) { Entry = &ConstantStringMap[C]; if (auto GV = *Entry) { - if (Alignment.getQuantity() > GV->getAlignment()) + if (uint64_t(Alignment.getQuantity()) > GV->getAlignment()) GV->setAlignment(Alignment.getAsAlign()); return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } } @@ -5386,14 +6363,18 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S, } auto GV = GenerateStringLiteral(C, LT, *this, GlobalVariableName, Alignment); + + CGDebugInfo *DI = getModuleDebugInfo(); + if (DI && getCodeGenOpts().hasReducedDebugInfo()) + DI->AddStringLiteralDebugInfo(GV, S); + if (Entry) *Entry = GV; - SanitizerMD->reportGlobalToASan(GV, S->getStrTokenLoc(0), "<string literal>", - QualType()); + SanitizerMD->reportGlobal(GV, S->getStrTokenLoc(0), "<string literal>"); return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } /// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant @@ -5423,10 +6404,10 @@ ConstantAddress CodeGenModule::GetAddrOfConstantCString( if (!LangOpts.WritableStrings) { Entry = &ConstantStringMap[C]; if (auto GV = *Entry) { - if (Alignment.getQuantity() > GV->getAlignment()) + if (uint64_t(Alignment.getQuantity()) > GV->getAlignment()) GV->setAlignment(Alignment.getAsAlign()); return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } } @@ -5440,7 +6421,7 @@ ConstantAddress CodeGenModule::GetAddrOfConstantCString( *Entry = GV; return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV), - Alignment); + GV->getValueType(), Alignment); } ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( @@ -5470,7 +6451,11 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( getModule(), Type, false, llvm::GlobalVariable::InternalLinkage, nullptr); } - return ConstantAddress(InsertResult.first->second, Align); + return ConstantAddress(InsertResult.first->second, + llvm::cast<llvm::GlobalVariable>( + InsertResult.first->second->stripPointerCasts()) + ->getValueType(), + Align); } // FIXME: If an externally-visible declaration extends multiple temporaries, @@ -5482,7 +6467,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( VD, E->getManglingNumber(), Out); APValue *Value = nullptr; - if (E->getStorageDuration() == SD_Static && VD && VD->evaluateValue()) { + if (E->getStorageDuration() == SD_Static && VD->evaluateValue()) { // If the initializer of the extending declaration is a constant // initializer, we should have a cached constant initializer for this // temporary. Note that this might have a different value from the value @@ -5497,10 +6482,9 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( !EvalResult.hasSideEffects()) Value = &EvalResult.Val; - LangAS AddrSpace = - VD ? GetGlobalVarAddressSpace(VD) : MaterializedType.getAddressSpace(); + LangAS AddrSpace = GetGlobalVarAddressSpace(VD); - Optional<ConstantEmitter> emitter; + std::optional<ConstantEmitter> emitter; llvm::Constant *InitialValue = nullptr; bool Constant = false; llvm::Type *Type; @@ -5509,7 +6493,9 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( emitter.emplace(*this); InitialValue = emitter->emitForInitializer(*Value, AddrSpace, MaterializedType); - Constant = isTypeConstant(MaterializedType, /*ExcludeCtor*/Value); + Constant = + MaterializedType.isConstantStorage(getContext(), /*ExcludeCtor*/ Value, + /*ExcludeDtor*/ false); Type = InitialValue->getType(); } else { // No initializer, the initialization will be provided when we @@ -5518,8 +6504,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( } // Create a global variable for this lifetime-extended temporary. - llvm::GlobalValue::LinkageTypes Linkage = - getLLVMLinkageVarDefinition(VD, Constant); + llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD); if (Linkage == llvm::GlobalVariable::ExternalLinkage) { const VarDecl *InitVD; if (VD->isStaticDataMember() && VD->getAnyInitializer(InitVD) && @@ -5538,7 +6523,13 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( getModule(), Type, Constant, Linkage, InitialValue, Name.c_str(), /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS); if (emitter) emitter->finalize(GV); - setGVProperties(GV, VD); + // Don't assign dllimport or dllexport to local linkage globals. + if (!llvm::GlobalValue::isLocalLinkage(Linkage)) { + setGVProperties(GV, VD); + if (GV->getDLLStorageClass() == llvm::GlobalVariable::DLLExportStorageClass) + // The reference temporary should never be dllexport. + GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass); + } GV->setAlignment(Align.getAsAlign()); if (supportsCOMDAT() && GV->isWeakForLinker()) GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); @@ -5548,20 +6539,20 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( if (AddrSpace != LangAS::Default) CV = getTargetCodeGenInfo().performAddrSpaceCast( *this, GV, AddrSpace, LangAS::Default, - Type->getPointerTo( + llvm::PointerType::get( + getLLVMContext(), getContext().getTargetAddressSpace(LangAS::Default))); // Update the map with the new temporary. If we created a placeholder above, // replace it with the new global now. llvm::Constant *&Entry = MaterializedGlobalTemporaryMap[E]; if (Entry) { - Entry->replaceAllUsesWith( - llvm::ConstantExpr::getBitCast(CV, Entry->getType())); + Entry->replaceAllUsesWith(CV); llvm::cast<llvm::GlobalVariable>(Entry)->eraseFromParent(); } Entry = CV; - return ConstantAddress(CV, Align); + return ConstantAddress(CV, Type, Align); } /// EmitObjCPropertyImplementations - Emit information for synthesized @@ -5626,7 +6617,7 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { /*isInstance=*/true, /*isVariadic=*/false, /*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false, /*isImplicitlyDeclared=*/true, - /*isDefined=*/false, ObjCMethodDecl::Required); + /*isDefined=*/false, ObjCImplementationControl::Required); D->addInstanceMethod(DTORMethod); CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, DTORMethod, false); D->setHasDestructors(true); @@ -5647,7 +6638,7 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { /*isVariadic=*/false, /*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false, /*isImplicitlyDeclared=*/true, - /*isDefined=*/false, ObjCMethodDecl::Required); + /*isDefined=*/false, ObjCImplementationControl::Required); D->addInstanceMethod(CTORMethod); CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, CTORMethod, true); D->setHasNonZeroConstructors(true); @@ -5655,8 +6646,8 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) { // EmitLinkageSpec - Emit all declarations in a linkage spec. void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) { - if (LSD->getLanguage() != LinkageSpecDecl::lang_c && - LSD->getLanguage() != LinkageSpecDecl::lang_cxx) { + if (LSD->getLanguage() != LinkageSpecLanguageIDs::C && + LSD->getLanguage() != LinkageSpecLanguageIDs::CXX) { ErrorUnsupported(LSD, "linkage spec"); return; } @@ -5664,6 +6655,43 @@ void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) { EmitDeclContext(LSD); } +void CodeGenModule::EmitTopLevelStmt(const TopLevelStmtDecl *D) { + // Device code should not be at top level. + if (LangOpts.CUDA && LangOpts.CUDAIsDevice) + return; + + std::unique_ptr<CodeGenFunction> &CurCGF = + GlobalTopLevelStmtBlockInFlight.first; + + // We emitted a top-level stmt but after it there is initialization. + // Stop squashing the top-level stmts into a single function. + if (CurCGF && CXXGlobalInits.back() != CurCGF->CurFn) { + CurCGF->FinishFunction(D->getEndLoc()); + CurCGF = nullptr; + } + + if (!CurCGF) { + // void __stmts__N(void) + // FIXME: Ask the ABI name mangler to pick a name. + std::string Name = "__stmts__" + llvm::utostr(CXXGlobalInits.size()); + FunctionArgList Args; + QualType RetTy = getContext().VoidTy; + const CGFunctionInfo &FnInfo = + getTypes().arrangeBuiltinFunctionDeclaration(RetTy, Args); + llvm::FunctionType *FnTy = getTypes().GetFunctionType(FnInfo); + llvm::Function *Fn = llvm::Function::Create( + FnTy, llvm::GlobalValue::InternalLinkage, Name, &getModule()); + + CurCGF.reset(new CodeGenFunction(*this)); + GlobalTopLevelStmtBlockInFlight.second = D; + CurCGF->StartFunction(GlobalDecl(), RetTy, Fn, FnInfo, Args, + D->getBeginLoc(), D->getBeginLoc()); + CXXGlobalInits.push_back(Fn); + } + + CurCGF->EmitStmt(D->getStmt()); +} + void CodeGenModule::EmitDeclContext(const DeclContext *DC) { for (auto *I : DC->decls()) { // Unlike other DeclContexts, the contents of an ObjCImplDecl at TU scope @@ -5687,9 +6715,8 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { return; // Consteval function shouldn't be emitted. - if (auto *FD = dyn_cast<FunctionDecl>(D)) - if (FD->isConsteval()) - return; + if (auto *FD = dyn_cast<FunctionDecl>(D); FD && FD->isImmediateFunction()) + return; switch (D->getKind()) { case Decl::CXXConversion: @@ -5731,7 +6758,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { TSK_ExplicitInstantiationDefinition && Spec->hasDefinition()) DI->completeTemplateDefinition(*Spec); - } LLVM_FALLTHROUGH; + } [[fallthrough]]; case Decl::CXXRecord: { CXXRecordDecl *CRD = cast<CXXRecordDecl>(D); if (CGDebugInfo *DI = getModuleDebugInfo()) { @@ -5863,7 +6890,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { if (LangOpts.CUDA && LangOpts.CUDAIsDevice) break; // File-scope asm is ignored during device-side OpenMP compilation. - if (LangOpts.OpenMPIsDevice) + if (LangOpts.OpenMPIsTargetDevice) break; // File-scope asm is ignored during device-side SYCL compilation. if (LangOpts.SYCLIsDevice) @@ -5873,6 +6900,10 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { break; } + case Decl::TopLevelStmt: + EmitTopLevelStmt(cast<TopLevelStmtDecl>(D)); + break; + case Decl::Import: { auto *Import = cast<ImportDecl>(D); @@ -5886,6 +6917,16 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { DI->EmitImportDecl(*Import); } + // For C++ standard modules we are done - we will call the module + // initializer for imported modules, and that will likewise call those for + // any imports it has. + if (CXX20ModuleInits && Import->getImportedOwningModule() && + !Import->getImportedOwningModule()->isModuleMapModule()) + break; + + // For clang C++ module map modules the initializers for sub-modules are + // emitted here. + // Find all of the submodules and emit the module initializers. llvm::SmallPtrSet<clang::Module *, 16> Visited; SmallVector<clang::Module *, 16> Stack; @@ -5901,16 +6942,14 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { EmitTopLevelDecl(D); // Visit the submodules of this module. - for (clang::Module::submodule_iterator Sub = Mod->submodule_begin(), - SubEnd = Mod->submodule_end(); - Sub != SubEnd; ++Sub) { + for (auto *Submodule : Mod->submodules()) { // Skip explicit children; they need to be explicitly imported to emit // the initializers. - if ((*Sub)->IsExplicit) + if (Submodule->IsExplicit) continue; - if (Visited.insert(*Sub).second) - Stack.push_back(*Sub); + if (Visited.insert(Submodule).second) + Stack.push_back(Submodule); } } break; @@ -5959,6 +6998,10 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { DI->EmitAndRetainType(getContext().getEnumType(cast<EnumDecl>(D))); break; + case Decl::HLSLBuffer: + getHLSLRuntime().addBuffer(cast<HLSLBufferDecl>(D)); + break; + default: // Make sure we handled everything we should, every other kind is a // non-top-level decl. FIXME: Would be nice to have an isTopLevelDeclKind @@ -5984,9 +7027,7 @@ void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) { SourceManager &SM = getContext().getSourceManager(); if (LimitedCoverage && SM.getMainFileID() != SM.getFileID(D->getBeginLoc())) break; - auto I = DeferredEmptyCoverageMappingDecls.find(D); - if (I == DeferredEmptyCoverageMappingDecls.end()) - DeferredEmptyCoverageMappingDecls[D] = true; + DeferredEmptyCoverageMappingDecls.try_emplace(D, true); break; } default: @@ -6002,11 +7043,7 @@ void CodeGenModule::ClearUnusedCoverageMapping(const Decl *D) { if (Fn->isTemplateInstantiation()) ClearUnusedCoverageMapping(Fn->getTemplateInstantiationPattern()); } - auto I = DeferredEmptyCoverageMappingDecls.find(D); - if (I == DeferredEmptyCoverageMappingDecls.end()) - DeferredEmptyCoverageMappingDecls[D] = false; - else - I->second = false; + DeferredEmptyCoverageMappingDecls.insert_or_assign(D, false); } void CodeGenModule::EmitDeferredUnusedCoverageMappings() { @@ -6055,8 +7092,10 @@ void CodeGenModule::EmitMainVoidAlias() { // new-style no-argument main is in used. if (llvm::Function *F = getModule().getFunction("main")) { if (!F->isDeclaration() && F->arg_size() == 0 && !F->isVarArg() && - F->getReturnType()->isIntegerTy(Context.getTargetInfo().getIntWidth())) - addUsedGlobal(llvm::GlobalAlias::create("__main_void", F)); + F->getReturnType()->isIntegerTy(Context.getTargetInfo().getIntWidth())) { + auto *GA = llvm::GlobalAlias::create("__main_void", F); + GA->setVisibility(llvm::GlobalValue::HiddenVisibility); + } } } @@ -6083,6 +7122,72 @@ static void EmitGlobalDeclMetadata(CodeGenModule &CGM, GlobalMetadata->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops)); } +bool CodeGenModule::CheckAndReplaceExternCIFuncs(llvm::GlobalValue *Elem, + llvm::GlobalValue *CppFunc) { + // Store the list of ifuncs we need to replace uses in. + llvm::SmallVector<llvm::GlobalIFunc *> IFuncs; + // List of ConstantExprs that we should be able to delete when we're done + // here. + llvm::SmallVector<llvm::ConstantExpr *> CEs; + + // It isn't valid to replace the extern-C ifuncs if all we find is itself! + if (Elem == CppFunc) + return false; + + // First make sure that all users of this are ifuncs (or ifuncs via a + // bitcast), and collect the list of ifuncs and CEs so we can work on them + // later. + for (llvm::User *User : Elem->users()) { + // Users can either be a bitcast ConstExpr that is used by the ifuncs, OR an + // ifunc directly. In any other case, just give up, as we don't know what we + // could break by changing those. + if (auto *ConstExpr = dyn_cast<llvm::ConstantExpr>(User)) { + if (ConstExpr->getOpcode() != llvm::Instruction::BitCast) + return false; + + for (llvm::User *CEUser : ConstExpr->users()) { + if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(CEUser)) { + IFuncs.push_back(IFunc); + } else { + return false; + } + } + CEs.push_back(ConstExpr); + } else if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(User)) { + IFuncs.push_back(IFunc); + } else { + // This user is one we don't know how to handle, so fail redirection. This + // will result in an ifunc retaining a resolver name that will ultimately + // fail to be resolved to a defined function. + return false; + } + } + + // Now we know this is a valid case where we can do this alias replacement, we + // need to remove all of the references to Elem (and the bitcasts!) so we can + // delete it. + for (llvm::GlobalIFunc *IFunc : IFuncs) + IFunc->setResolver(nullptr); + for (llvm::ConstantExpr *ConstExpr : CEs) + ConstExpr->destroyConstant(); + + // We should now be out of uses for the 'old' version of this function, so we + // can erase it as well. + Elem->eraseFromParent(); + + for (llvm::GlobalIFunc *IFunc : IFuncs) { + // The type of the resolver is always just a function-type that returns the + // type of the IFunc, so create that here. If the type of the actual + // resolver doesn't match, it just gets bitcast to the right thing. + auto *ResolverTy = + llvm::FunctionType::get(IFunc->getType(), /*isVarArg*/ false); + llvm::Constant *Resolver = GetOrCreateLLVMFunction( + CppFunc->getName(), ResolverTy, {}, /*ForVTable*/ false); + IFunc->setResolver(Resolver); + } + return true; +} + /// For each function which is declared within an extern "C" region and marked /// as 'used', but has internal linkage, create an alias from the unmangled /// name to the mangled name if possible. People expect to be able to refer @@ -6094,7 +7199,19 @@ void CodeGenModule::EmitStaticExternCAliases() { for (auto &I : StaticExternCValues) { IdentifierInfo *Name = I.first; llvm::GlobalValue *Val = I.second; - if (Val && !getModule().getNamedValue(Name->getName())) + + // If Val is null, that implies there were multiple declarations that each + // had a claim to the unmangled name. In this case, generation of the alias + // is suppressed. See CodeGenModule::MaybeHandleStaticInExternC. + if (!Val) + break; + + llvm::GlobalValue *ExistingElem = + getModule().getNamedValue(Name->getName()); + + // If there is either not something already by this name, or we were able to + // replace all uses from IFuncs, create the alias. + if (!ExistingElem || CheckAndReplaceExternCIFuncs(ExistingElem, Val)) addCompilerUsedGlobal(llvm::GlobalAlias::create(Name->getName(), Val)); } } @@ -6175,10 +7292,6 @@ void CodeGenModule::EmitCommandLineMetadata() { } void CodeGenModule::EmitCoverageFile() { - if (getCodeGenOpts().CoverageDataFile.empty() && - getCodeGenOpts().CoverageNotesFile.empty()) - return; - llvm::NamedMDNode *CUNode = TheModule.getNamedMetadata("llvm.dbg.cu"); if (!CUNode) return; @@ -6201,10 +7314,8 @@ llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty, // Return a bogus pointer if RTTI is disabled, unless it's for EH. // FIXME: should we even be calling this method if RTTI is disabled // and it's not for EH? - if ((!ForEH && !getLangOpts().RTTI) || getLangOpts().CUDAIsDevice || - (getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice && - getTriple().isNVPTX())) - return llvm::Constant::getNullValue(Int8PtrTy); + if (!shouldEmitRTTI(ForEH)) + return llvm::Constant::getNullValue(GlobalsInt8PtrTy); if (ForEH && Ty->isObjCObjectPointerType() && LangOpts.ObjCRuntime.isGNUFamily()) @@ -6224,7 +7335,9 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) { !VD->getAnyInitializer()->isConstantInitializer(getContext(), /*ForRef=*/false); - Address Addr(GetAddrOfGlobalVar(VD), getContext().getDeclAlign(VD)); + Address Addr(GetAddrOfGlobalVar(VD), + getTypes().ConvertTypeForMem(VD->getType()), + getContext().getDeclAlign(VD)); if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition( VD, Addr, RefExpr->getBeginLoc(), PerformInit)) CXXGlobalInits.push_back(InitFunction); @@ -6234,6 +7347,11 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) { llvm::Metadata * CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, StringRef Suffix) { + if (auto *FnType = T->getAs<FunctionProtoType>()) + T = getContext().getFunctionType( + FnType->getReturnType(), FnType->getParamTypes(), + FnType->getExtProtoInfo().withExceptionSpec(EST_None)); + llvm::Metadata *&InternalId = Map[T.getCanonicalType()]; if (InternalId) return InternalId; @@ -6241,7 +7359,12 @@ CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, if (isExternallyVisible(T->getLinkage())) { std::string OutName; llvm::raw_string_ostream Out(OutName); - getCXXABI().getMangleContext().mangleTypeName(T, Out); + getCXXABI().getMangleContext().mangleCanonicalTypeName( + T, Out, getCodeGenOpts().SanitizeCfiICallNormalizeIntegers); + + if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers) + Out << ".normalized"; + Out << Suffix; InternalId = llvm::MDString::get(getLLVMContext(), Out.str()); @@ -6446,7 +7569,66 @@ bool CodeGenModule::stopAutoInit() { return false; } -void CodeGenModule::printPostfixForExternalizedStaticVar( - llvm::raw_ostream &OS) const { - OS << ".static." << getContext().getCUIDHash(); +void CodeGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &OS, + const Decl *D) const { + // ptxas does not allow '.' in symbol names. On the other hand, HIP prefers + // postfix beginning with '.' since the symbol name can be demangled. + if (LangOpts.HIP) + OS << (isa<VarDecl>(D) ? ".static." : ".intern."); + else + OS << (isa<VarDecl>(D) ? "__static__" : "__intern__"); + + // If the CUID is not specified we try to generate a unique postfix. + if (getLangOpts().CUID.empty()) { + SourceManager &SM = getContext().getSourceManager(); + PresumedLoc PLoc = SM.getPresumedLoc(D->getLocation()); + assert(PLoc.isValid() && "Source location is expected to be valid."); + + // Get the hash of the user defined macros. + llvm::MD5 Hash; + llvm::MD5::MD5Result Result; + for (const auto &Arg : PreprocessorOpts.Macros) + Hash.update(Arg.first); + Hash.final(Result); + + // Get the UniqueID for the file containing the decl. + llvm::sys::fs::UniqueID ID; + if (llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID)) { + PLoc = SM.getPresumedLoc(D->getLocation(), /*UseLineDirectives=*/false); + assert(PLoc.isValid() && "Source location is expected to be valid."); + if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID)) + SM.getDiagnostics().Report(diag::err_cannot_open_file) + << PLoc.getFilename() << EC.message(); + } + OS << llvm::format("%x", ID.getFile()) << llvm::format("%x", ID.getDevice()) + << "_" << llvm::utohexstr(Result.low(), /*LowerCase=*/true, /*Width=*/8); + } else { + OS << getContext().getCUIDHash(); + } +} + +void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) { + assert(DeferredDeclsToEmit.empty() && + "Should have emitted all decls deferred to emit."); + assert(NewBuilder->DeferredDecls.empty() && + "Newly created module should not have deferred decls"); + NewBuilder->DeferredDecls = std::move(DeferredDecls); + assert(EmittedDeferredDecls.empty() && + "Still have (unmerged) EmittedDeferredDecls deferred decls"); + + assert(NewBuilder->DeferredVTables.empty() && + "Newly created module should not have deferred vtables"); + NewBuilder->DeferredVTables = std::move(DeferredVTables); + + assert(NewBuilder->MangledDeclNames.empty() && + "Newly created module should not have mangled decl names"); + assert(NewBuilder->Manglings.empty() && + "Newly created module should not have manglings"); + NewBuilder->Manglings = std::move(Manglings); + + NewBuilder->WeakRefReferences = std::move(WeakRefReferences); + + NewBuilder->TBAA = std::move(TBAA); + + NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx); } |