diff options
Diffstat (limited to 'lib/CodeGen')
58 files changed, 11789 insertions, 5847 deletions
diff --git a/lib/CodeGen/ABIInfo.h b/lib/CodeGen/ABIInfo.h index 7e7f7fa20679..cc8652e169d8 100644 --- a/lib/CodeGen/ABIInfo.h +++ b/lib/CodeGen/ABIInfo.h @@ -87,6 +87,8 @@ namespace clang { virtual bool isHomogeneousAggregateSmallEnough(const Type *Base, uint64_t Members) const; + virtual bool shouldSignExtUnsignedType(QualType Ty) const; + bool isHomogeneousAggregate(QualType Ty, const Type *&Base, uint64_t &Members) const; diff --git a/lib/CodeGen/BackendUtil.cpp b/lib/CodeGen/BackendUtil.cpp index 25ecec586244..7f0c7bafc046 100644 --- a/lib/CodeGen/BackendUtil.cpp +++ b/lib/CodeGen/BackendUtil.cpp @@ -15,22 +15,22 @@ #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/Utils.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/IRPrintingPasses.h" +#include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" #include "llvm/MC/SubtargetFeature.h" -#include "llvm/PassManager.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/FormattedStream.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetLibraryInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetSubtargetInfo.h" @@ -55,44 +55,48 @@ class EmitAssemblyHelper { Timer CodeGenerationTime; - mutable PassManager *CodeGenPasses; - mutable PassManager *PerModulePasses; - mutable FunctionPassManager *PerFunctionPasses; + mutable legacy::PassManager *CodeGenPasses; + mutable legacy::PassManager *PerModulePasses; + mutable legacy::FunctionPassManager *PerFunctionPasses; private: - PassManager *getCodeGenPasses() const { + TargetIRAnalysis getTargetIRAnalysis() const { + if (TM) + return TM->getTargetIRAnalysis(); + + return TargetIRAnalysis(); + } + + legacy::PassManager *getCodeGenPasses() const { if (!CodeGenPasses) { - CodeGenPasses = new PassManager(); - CodeGenPasses->add(new DataLayoutPass()); - if (TM) - TM->addAnalysisPasses(*CodeGenPasses); + CodeGenPasses = new legacy::PassManager(); + CodeGenPasses->add( + createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); } return CodeGenPasses; } - PassManager *getPerModulePasses() const { + legacy::PassManager *getPerModulePasses() const { if (!PerModulePasses) { - PerModulePasses = new PassManager(); - PerModulePasses->add(new DataLayoutPass()); - if (TM) - TM->addAnalysisPasses(*PerModulePasses); + PerModulePasses = new legacy::PassManager(); + PerModulePasses->add( + createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); } return PerModulePasses; } - FunctionPassManager *getPerFunctionPasses() const { + legacy::FunctionPassManager *getPerFunctionPasses() const { if (!PerFunctionPasses) { - PerFunctionPasses = new FunctionPassManager(TheModule); - PerFunctionPasses->add(new DataLayoutPass()); - if (TM) - TM->addAnalysisPasses(*PerFunctionPasses); + PerFunctionPasses = new legacy::FunctionPassManager(TheModule); + PerFunctionPasses->add( + createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); } return PerFunctionPasses; } void CreatePasses(); - /// CreateTargetMachine - Generates the TargetMachine. + /// Generates the TargetMachine. /// Returns Null if it is unable to create the target machine. /// Some of our clang tests specify triples which are not built /// into clang. This is okay because these tests check the generated @@ -102,10 +106,10 @@ private: /// the requested target. TargetMachine *CreateTargetMachine(bool MustCreateTM); - /// AddEmitPasses - Add passes necessary to emit assembly or LLVM IR. + /// Add passes necessary to emit assembly or LLVM IR. /// /// \return True on success. - bool AddEmitPasses(BackendAction Action, formatted_raw_ostream &OS); + bool AddEmitPasses(BackendAction Action, raw_pwrite_stream &OS); public: EmitAssemblyHelper(DiagnosticsEngine &_Diags, @@ -128,7 +132,7 @@ public: std::unique_ptr<TargetMachine> TM; - void EmitAssembly(BackendAction Action, raw_ostream *OS); + void EmitAssembly(BackendAction Action, raw_pwrite_stream *OS); }; // We need this wrapper to access LangOpts and CGOpts from extension functions @@ -163,7 +167,7 @@ static void addObjCARCOptPass(const PassManagerBuilder &Builder, PassManagerBase } static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper &>(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); @@ -171,31 +175,38 @@ static void addSampleProfileLoaderPass(const PassManagerBuilder &Builder, } static void addAddDiscriminatorsPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { PM.add(createAddDiscriminatorsPass()); } static void addBoundsCheckingPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { PM.add(createBoundsCheckingPass()); } static void addSanitizerCoveragePass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper&>(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); - PM.add(createSanitizerCoverageModulePass(CGOpts.SanitizeCoverage)); + SanitizerCoverageOptions Opts; + Opts.CoverageType = + static_cast<SanitizerCoverageOptions::Type>(CGOpts.SanitizeCoverageType); + Opts.IndirectCalls = CGOpts.SanitizeCoverageIndirectCalls; + Opts.TraceBB = CGOpts.SanitizeCoverageTraceBB; + Opts.TraceCmp = CGOpts.SanitizeCoverageTraceCmp; + Opts.Use8bitCounters = CGOpts.SanitizeCoverage8bitCounters; + PM.add(createSanitizerCoverageModulePass(Opts)); } static void addAddressSanitizerPasses(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { PM.add(createAddressSanitizerFunctionPass()); PM.add(createAddressSanitizerModulePass()); } static void addMemorySanitizerPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper&>(Builder); const CodeGenOptions &CGOpts = BuilderWrapper.getCGOpts(); @@ -215,28 +226,36 @@ static void addMemorySanitizerPass(const PassManagerBuilder &Builder, } static void addThreadSanitizerPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { PM.add(createThreadSanitizerPass()); } static void addDataFlowSanitizerPass(const PassManagerBuilder &Builder, - PassManagerBase &PM) { + legacy::PassManagerBase &PM) { const PassManagerBuilderWrapper &BuilderWrapper = static_cast<const PassManagerBuilderWrapper&>(Builder); const LangOptions &LangOpts = BuilderWrapper.getLangOpts(); - PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFile)); + PM.add(createDataFlowSanitizerPass(LangOpts.SanitizerBlacklistFiles)); } -static TargetLibraryInfo *createTLI(llvm::Triple &TargetTriple, - const CodeGenOptions &CodeGenOpts) { - TargetLibraryInfo *TLI = new TargetLibraryInfo(TargetTriple); +static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple, + const CodeGenOptions &CodeGenOpts) { + TargetLibraryInfoImpl *TLII = new TargetLibraryInfoImpl(TargetTriple); if (!CodeGenOpts.SimplifyLibCalls) - TLI->disableAllFunctions(); - return TLI; + TLII->disableAllFunctions(); + + switch (CodeGenOpts.getVecLib()) { + case CodeGenOptions::Accelerate: + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate); + break; + default: + break; + } + return TLII; } static void addSymbolRewriterPass(const CodeGenOptions &Opts, - PassManager *MPM) { + legacy::PassManager *MPM) { llvm::SymbolRewriter::RewriteDescriptorList DL; llvm::SymbolRewriter::RewriteMapParser MapParser; @@ -294,7 +313,9 @@ void EmitAssemblyHelper::CreatePasses() { addBoundsCheckingPass); } - if (CodeGenOpts.SanitizeCoverage) { + if (CodeGenOpts.SanitizeCoverageType || + CodeGenOpts.SanitizeCoverageIndirectCalls || + CodeGenOpts.SanitizeCoverageTraceCmp) { PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast, addSanitizerCoveragePass); PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, @@ -331,7 +352,7 @@ void EmitAssemblyHelper::CreatePasses() { // Figure out TargetLibraryInfo. Triple TargetTriple(TheModule->getTargetTriple()); - PMBuilder.LibraryInfo = createTLI(TargetTriple, CodeGenOpts); + PMBuilder.LibraryInfo = createTLII(TargetTriple, CodeGenOpts); switch (Inlining) { case CodeGenOptions::NoInlining: break; @@ -351,17 +372,15 @@ void EmitAssemblyHelper::CreatePasses() { } // Set up the per-function pass manager. - FunctionPassManager *FPM = getPerFunctionPasses(); + legacy::FunctionPassManager *FPM = getPerFunctionPasses(); if (CodeGenOpts.VerifyModule) FPM->add(createVerifierPass()); PMBuilder.populateFunctionPassManager(*FPM); // Set up the per-module pass manager. - PassManager *MPM = getPerModulePasses(); + legacy::PassManager *MPM = getPerModulePasses(); if (!CodeGenOpts.RewriteMapFiles.empty()) addSymbolRewriterPass(CodeGenOpts, MPM); - if (CodeGenOpts.VerifyModule) - MPM->add(createDebugInfoVerifierPass()); if (!CodeGenOpts.DisableGCov && (CodeGenOpts.EmitGcovArcs || CodeGenOpts.EmitGcovNotes)) { @@ -375,6 +394,7 @@ void EmitAssemblyHelper::CreatePasses() { Options.NoRedZone = CodeGenOpts.DisableRedZone; Options.FunctionNamesInData = !CodeGenOpts.CoverageNoFunctionNamesInData; + Options.ExitBlockBeforeBody = CodeGenOpts.CoverageExitBlockBeforeBody; MPM->add(createGCOVProfilerPass(Options)); if (CodeGenOpts.getDebugInfo() == CodeGenOptions::NoDebugInfo) MPM->add(createStripSymbolsPass(true)); @@ -383,6 +403,7 @@ void EmitAssemblyHelper::CreatePasses() { if (CodeGenOpts.ProfileInstrGenerate) { InstrProfOptions Options; Options.NoRedZone = CodeGenOpts.DisableRedZone; + Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput; MPM->add(createInstrProfilingPass(Options)); } @@ -425,14 +446,12 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { BackendArgs.push_back("-time-passes"); for (unsigned i = 0, e = CodeGenOpts.BackendOptions.size(); i != e; ++i) BackendArgs.push_back(CodeGenOpts.BackendOptions[i].c_str()); - if (CodeGenOpts.NoGlobalMerge) - BackendArgs.push_back("-enable-global-merge=false"); BackendArgs.push_back(nullptr); llvm::cl::ParseCommandLineOptions(BackendArgs.size() - 1, BackendArgs.data()); std::string FeaturesStr; - if (TargetOpts.Features.size()) { + if (!TargetOpts.Features.empty()) { SubtargetFeatures Features; for (std::vector<std::string>::const_iterator it = TargetOpts.Features.begin(), @@ -472,15 +491,6 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { if (CodeGenOpts.CompressDebugSections) Options.CompressDebugSections = true; - // Set frame pointer elimination mode. - if (!CodeGenOpts.DisableFPElim) { - Options.NoFramePointerElim = false; - } else if (CodeGenOpts.OmitLeafFramePointer) { - Options.NoFramePointerElim = false; - } else { - Options.NoFramePointerElim = true; - } - if (CodeGenOpts.UseInitArray) Options.UseInitArray = true; @@ -512,13 +522,13 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { Options.NoNaNsFPMath = CodeGenOpts.NoNaNsFPMath; Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS; Options.UnsafeFPMath = CodeGenOpts.UnsafeFPMath; - Options.UseSoftFloat = CodeGenOpts.SoftFloat; Options.StackAlignmentOverride = CodeGenOpts.StackAlignment; Options.DisableTailCalls = CodeGenOpts.DisableTailCalls; Options.TrapFuncName = CodeGenOpts.TrapFuncName; Options.PositionIndependentExecutable = LangOpts.PIELevel != 0; Options.FunctionSections = CodeGenOpts.FunctionSections; Options.DataSections = CodeGenOpts.DataSections; + Options.UniqueSectionNames = CodeGenOpts.UniqueSectionNames; Options.MCOptions.MCRelaxAll = CodeGenOpts.RelaxAll; Options.MCOptions.MCSaveTempLabels = CodeGenOpts.SaveTempLabels; @@ -536,17 +546,16 @@ TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { } bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action, - formatted_raw_ostream &OS) { + raw_pwrite_stream &OS) { // Create the code generator passes. - PassManager *PM = getCodeGenPasses(); + legacy::PassManager *PM = getCodeGenPasses(); // Add LibraryInfo. llvm::Triple TargetTriple(TheModule->getTargetTriple()); - PM->add(createTLI(TargetTriple, CodeGenOpts)); - - // Add Target specific analysis passes. - TM->addAnalysisPasses(*PM); + std::unique_ptr<TargetLibraryInfoImpl> TLII( + createTLII(TargetTriple, CodeGenOpts)); + PM->add(new TargetLibraryInfoWrapperPass(*TLII)); // Normal mode, emit a .s or .o file by running the code generator. Note, // this also adds codegenerator level optimization passes. @@ -561,8 +570,7 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action, // Add ObjC ARC final-cleanup optimizations. This is done as part of the // "codegen" passes so that it isn't run multiple times when there is // inlining happening. - if (LangOpts.ObjCAutoRefCount && - CodeGenOpts.OptimizationLevel > 0) + if (CodeGenOpts.OptimizationLevel > 0) PM->add(createObjCARCContractPass()); if (TM->addPassesToEmitFile(*PM, OS, CGFT, @@ -574,9 +582,9 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action, return true; } -void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) { +void EmitAssemblyHelper::EmitAssembly(BackendAction Action, + raw_pwrite_stream *OS) { TimeRegion Region(llvm::TimePassesIsEnabled ? &CodeGenerationTime : nullptr); - llvm::formatted_raw_ostream FormattedOS; bool UsesCodeGen = (Action != Backend_EmitNothing && Action != Backend_EmitBC && @@ -592,17 +600,17 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) { break; case Backend_EmitBC: - getPerModulePasses()->add(createBitcodeWriterPass(*OS)); + getPerModulePasses()->add( + createBitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists)); break; case Backend_EmitLL: - FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM); - getPerModulePasses()->add(createPrintModulePass(FormattedOS)); + getPerModulePasses()->add( + createPrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists)); break; default: - FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM); - if (!AddEmitPasses(Action, FormattedOS)) + if (!AddEmitPasses(Action, *OS)) return; } @@ -639,7 +647,7 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags, const clang::TargetOptions &TOpts, const LangOptions &LOpts, StringRef TDesc, Module *M, BackendAction Action, - raw_ostream *OS) { + raw_pwrite_stream *OS) { EmitAssemblyHelper AsmHelper(Diags, CGOpts, TOpts, LOpts, M); AsmHelper.EmitAssembly(Action, OS); @@ -647,9 +655,8 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags, // If an optional clang TargetInfo description string was passed in, use it to // verify the LLVM TargetMachine's DataLayout. if (AsmHelper.TM && !TDesc.empty()) { - std::string DLDesc = AsmHelper.TM->getSubtargetImpl() - ->getDataLayout() - ->getStringRepresentation(); + std::string DLDesc = + AsmHelper.TM->getDataLayout()->getStringRepresentation(); if (DLDesc != TDesc) { unsigned DiagID = Diags.getCustomDiagID( DiagnosticsEngine::Error, "backend data layout '%0' does not match " diff --git a/lib/CodeGen/CGAtomic.cpp b/lib/CodeGen/CGAtomic.cpp index daac174c8e0c..da82249fe114 100644 --- a/lib/CodeGen/CGAtomic.cpp +++ b/lib/CodeGen/CGAtomic.cpp @@ -13,6 +13,7 @@ #include "CodeGenFunction.h" #include "CGCall.h" +#include "CGRecordLayout.h" #include "CodeGenModule.h" #include "clang/AST/ASTContext.h" #include "clang/CodeGen/CGFunctionInfo.h" @@ -36,34 +37,94 @@ namespace { CharUnits LValueAlign; TypeEvaluationKind EvaluationKind; bool UseLibcall; + LValue LVal; + CGBitFieldInfo BFI; public: - AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) { - assert(lvalue.isSimple()); - - AtomicTy = lvalue.getType(); - ValueTy = AtomicTy->castAs<AtomicType>()->getValueType(); - EvaluationKind = CGF.getEvaluationKind(ValueTy); - + AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) + : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), + EvaluationKind(TEK_Scalar), UseLibcall(true) { + assert(!lvalue.isGlobalReg()); ASTContext &C = CGF.getContext(); - - uint64_t ValueAlignInBits; - uint64_t AtomicAlignInBits; - TypeInfo ValueTI = C.getTypeInfo(ValueTy); - ValueSizeInBits = ValueTI.Width; - ValueAlignInBits = ValueTI.Align; - - TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); - AtomicSizeInBits = AtomicTI.Width; - AtomicAlignInBits = AtomicTI.Align; - - assert(ValueSizeInBits <= AtomicSizeInBits); - assert(ValueAlignInBits <= AtomicAlignInBits); - - AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); - ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); - if (lvalue.getAlignment().isZero()) - lvalue.setAlignment(AtomicAlign); - + if (lvalue.isSimple()) { + AtomicTy = lvalue.getType(); + if (auto *ATy = AtomicTy->getAs<AtomicType>()) + ValueTy = ATy->getValueType(); + else + ValueTy = AtomicTy; + EvaluationKind = CGF.getEvaluationKind(ValueTy); + + uint64_t ValueAlignInBits; + uint64_t AtomicAlignInBits; + TypeInfo ValueTI = C.getTypeInfo(ValueTy); + ValueSizeInBits = ValueTI.Width; + ValueAlignInBits = ValueTI.Align; + + TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); + AtomicSizeInBits = AtomicTI.Width; + AtomicAlignInBits = AtomicTI.Align; + + assert(ValueSizeInBits <= AtomicSizeInBits); + assert(ValueAlignInBits <= AtomicAlignInBits); + + AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); + ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); + if (lvalue.getAlignment().isZero()) + lvalue.setAlignment(AtomicAlign); + + LVal = lvalue; + } else if (lvalue.isBitField()) { + ValueTy = lvalue.getType(); + ValueSizeInBits = C.getTypeSize(ValueTy); + auto &OrigBFI = lvalue.getBitFieldInfo(); + auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); + AtomicSizeInBits = C.toBits( + C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) + .RoundUpToAlignment(lvalue.getAlignment())); + auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr()); + auto OffsetInChars = + (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * + lvalue.getAlignment(); + VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( + VoidPtrAddr, OffsetInChars.getQuantity()); + auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + VoidPtrAddr, + CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), + "atomic_bitfield_base"); + BFI = OrigBFI; + BFI.Offset = Offset; + BFI.StorageSize = AtomicSizeInBits; + LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(), + lvalue.getAlignment()); + LVal.setTBAAInfo(lvalue.getTBAAInfo()); + AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); + if (AtomicTy.isNull()) { + llvm::APInt Size( + /*numBits=*/32, + C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); + AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, + /*IndexTypeQuals=*/0); + } + AtomicAlign = ValueAlign = lvalue.getAlignment(); + } else if (lvalue.isVectorElt()) { + ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); + ValueSizeInBits = C.getTypeSize(ValueTy); + AtomicTy = lvalue.getType(); + AtomicSizeInBits = C.getTypeSize(AtomicTy); + AtomicAlign = ValueAlign = lvalue.getAlignment(); + LVal = lvalue; + } else { + assert(lvalue.isExtVectorElt()); + ValueTy = lvalue.getType(); + ValueSizeInBits = C.getTypeSize(ValueTy); + AtomicTy = ValueTy = CGF.getContext().getExtVectorType( + lvalue.getType(), lvalue.getExtVectorAddr() + ->getType() + ->getPointerElementType() + ->getVectorNumElements()); + AtomicSizeInBits = C.getTypeSize(AtomicTy); + AtomicAlign = ValueAlign = lvalue.getAlignment(); + LVal = lvalue; + } UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( AtomicSizeInBits, C.toBits(lvalue.getAlignment())); } @@ -76,6 +137,17 @@ namespace { uint64_t getValueSizeInBits() const { return ValueSizeInBits; } TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } bool shouldUseLibcall() const { return UseLibcall; } + const LValue &getAtomicLValue() const { return LVal; } + llvm::Value *getAtomicAddress() const { + if (LVal.isSimple()) + return LVal.getAddress(); + else if (LVal.isBitField()) + return LVal.getBitFieldAddr(); + else if (LVal.isVectorElt()) + return LVal.getVectorAddr(); + assert(LVal.isExtVectorElt()); + return LVal.getExtVectorAddr(); + } /// Is the atomic size larger than the underlying value type? /// @@ -87,7 +159,7 @@ namespace { return (ValueSizeInBits != AtomicSizeInBits); } - bool emitMemSetZeroIfNecessary(LValue dest) const; + bool emitMemSetZeroIfNecessary() const; llvm::Value *getAtomicSizeValue() const { CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); @@ -99,37 +171,141 @@ namespace { llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const; /// Turn an atomic-layout object into an r-value. - RValue convertTempToRValue(llvm::Value *addr, - AggValueSlot resultSlot, - SourceLocation loc) const; + RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot, + SourceLocation loc, bool AsValue) const; /// \brief Converts a rvalue to integer value. llvm::Value *convertRValueToInt(RValue RVal) const; - RValue convertIntToValue(llvm::Value *IntVal, AggValueSlot ResultSlot, - SourceLocation Loc) const; + RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, + AggValueSlot ResultSlot, + SourceLocation Loc, bool AsValue) const; /// Copy an atomic r-value into atomic-layout memory. - void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const; + void emitCopyIntoMemory(RValue rvalue) const; /// Project an l-value down to the value field. - LValue projectValue(LValue lvalue) const { - llvm::Value *addr = lvalue.getAddress(); + LValue projectValue() const { + assert(LVal.isSimple()); + llvm::Value *addr = getAtomicAddress(); if (hasPadding()) - addr = CGF.Builder.CreateStructGEP(addr, 0); + addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0); - return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(), - CGF.getContext(), lvalue.getTBAAInfo()); + return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(), + CGF.getContext(), LVal.getTBAAInfo()); } + /// \brief Emits atomic load. + /// \returns Loaded value. + RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, + bool AsValue, llvm::AtomicOrdering AO, + bool IsVolatile); + + /// \brief Emits atomic compare-and-exchange sequence. + /// \param Expected Expected value. + /// \param Desired Desired value. + /// \param Success Atomic ordering for success operation. + /// \param Failure Atomic ordering for failed operation. + /// \param IsWeak true if atomic operation is weak, false otherwise. + /// \returns Pair of values: previous value from storage (value type) and + /// boolean flag (i1 type) with true if success and false otherwise. + std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange( + RValue Expected, RValue Desired, + llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, + llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent, + bool IsWeak = false); + + /// \brief Emits atomic update. + /// \param AO Atomic ordering. + /// \param UpdateOp Update operation for the current lvalue. + void EmitAtomicUpdate(llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// \brief Emits atomic update. + /// \param AO Atomic ordering. + void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile); + /// Materialize an atomic r-value in atomic-layout memory. llvm::Value *materializeRValue(RValue rvalue) const; + /// \brief Translates LLVM atomic ordering to GNU atomic ordering for + /// libcalls. + static AtomicExpr::AtomicOrderingKind + translateAtomicOrdering(const llvm::AtomicOrdering AO); + private: bool requiresMemSetZero(llvm::Type *type) const; + + /// \brief Creates temp alloca for intermediate operations on atomic value. + llvm::Value *CreateTempAlloca() const; + + /// \brief Emits atomic load as a libcall. + void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, + llvm::AtomicOrdering AO, bool IsVolatile); + /// \brief Emits atomic load as LLVM instruction. + llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); + /// \brief Emits atomic compare-and-exchange op as a libcall. + llvm::Value *EmitAtomicCompareExchangeLibcall( + llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, + llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, + llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent); + /// \brief Emits atomic compare-and-exchange op as LLVM instruction. + std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( + llvm::Value *ExpectedVal, llvm::Value *DesiredVal, + llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, + llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent, + bool IsWeak = false); + /// \brief Emit atomic update as libcalls. + void + EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// \brief Emit atomic update as LLVM instructions. + void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// \brief Emit atomic update as libcalls. + void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile); + /// \brief Emit atomic update as LLVM instructions. + void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, + bool IsVolatile); }; } +AtomicExpr::AtomicOrderingKind +AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) { + switch (AO) { + case llvm::Unordered: + case llvm::NotAtomic: + case llvm::Monotonic: + return AtomicExpr::AO_ABI_memory_order_relaxed; + case llvm::Acquire: + return AtomicExpr::AO_ABI_memory_order_acquire; + case llvm::Release: + return AtomicExpr::AO_ABI_memory_order_release; + case llvm::AcquireRelease: + return AtomicExpr::AO_ABI_memory_order_acq_rel; + case llvm::SequentiallyConsistent: + return AtomicExpr::AO_ABI_memory_order_seq_cst; + } + llvm_unreachable("Unhandled AtomicOrdering"); +} + +llvm::Value *AtomicInfo::CreateTempAlloca() const { + auto *TempAlloca = CGF.CreateMemTemp( + (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy + : AtomicTy, + "atomic-temp"); + TempAlloca->setAlignment(getAtomicAlignment().getQuantity()); + // Cast to pointer to value type for bitfields. + if (LVal.isBitField()) + return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + TempAlloca, getAtomicAddress()->getType()); + return TempAlloca; +} + static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, @@ -172,14 +348,16 @@ bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { llvm_unreachable("bad evaluation kind"); } -bool AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const { - llvm::Value *addr = dest.getAddress(); +bool AtomicInfo::emitMemSetZeroIfNecessary() const { + assert(LVal.isSimple()); + llvm::Value *addr = LVal.getAddress(); if (!requiresMemSetZero(addr->getType()->getPointerElementType())) return false; - CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), - AtomicSizeInBits / 8, - dest.getAlignment().getQuantity()); + CGF.Builder.CreateMemSet( + addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), + CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), + LVal.getAlignment().getQuantity()); return true; } @@ -901,29 +1079,53 @@ llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const { RValue AtomicInfo::convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot, - SourceLocation loc) const { - if (EvaluationKind == TEK_Aggregate) - return resultSlot.asRValue(); - - // Drill into the padding structure if we have one. - if (hasPadding()) - addr = CGF.Builder.CreateStructGEP(addr, 0); - - // Otherwise, just convert the temporary to an r-value using the - // normal conversion routine. - return CGF.convertTempToRValue(addr, getValueType(), loc); + SourceLocation loc, bool AsValue) const { + if (LVal.isSimple()) { + if (EvaluationKind == TEK_Aggregate) + return resultSlot.asRValue(); + + // Drill into the padding structure if we have one. + if (hasPadding()) + addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0); + + // Otherwise, just convert the temporary to an r-value using the + // normal conversion routine. + return CGF.convertTempToRValue(addr, getValueType(), loc); + } + if (!AsValue) + // Get RValue from temp memory as atomic for non-simple lvalues + return RValue::get( + CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity())); + if (LVal.isBitField()) + return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield( + addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment())); + if (LVal.isVectorElt()) + return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(), + LVal.getType(), + LVal.getAlignment()), + loc); + assert(LVal.isExtVectorElt()); + return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( + addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment())); } -RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal, - AggValueSlot ResultSlot, - SourceLocation Loc) const { +RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, + AggValueSlot ResultSlot, + SourceLocation Loc, + bool AsValue) const { // Try not to in some easy cases. assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); - if (getEvaluationKind() == TEK_Scalar && !hasPadding()) { - auto *ValTy = CGF.ConvertTypeForMem(ValueTy); + if (getEvaluationKind() == TEK_Scalar && + (((!LVal.isBitField() || + LVal.getBitFieldInfo().Size == ValueSizeInBits) && + !hasPadding()) || + !AsValue)) { + auto *ValTy = AsValue + ? CGF.ConvertTypeForMem(ValueTy) + : getAtomicAddress()->getType()->getPointerElementType(); if (ValTy->isIntegerTy()) { assert(IntVal->getType() == ValTy && "Different integer types."); - return RValue::get(IntVal); + return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); } else if (ValTy->isPointerTy()) return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) @@ -935,13 +1137,13 @@ RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal, llvm::Value *Temp; bool TempIsVolatile = false; CharUnits TempAlignment; - if (getEvaluationKind() == TEK_Aggregate) { + if (AsValue && getEvaluationKind() == TEK_Aggregate) { assert(!ResultSlot.isIgnored()); Temp = ResultSlot.getAddr(); TempAlignment = getValueAlignment(); TempIsVolatile = ResultSlot.isVolatile(); } else { - Temp = CGF.CreateMemTemp(getAtomicType(), "atomic-temp"); + Temp = CreateTempAlloca(); TempAlignment = getAtomicAlignment(); } @@ -950,93 +1152,146 @@ RValue AtomicInfo::convertIntToValue(llvm::Value *IntVal, CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity()) ->setVolatile(TempIsVolatile); - return convertTempToRValue(Temp, ResultSlot, Loc); + return convertTempToRValue(Temp, ResultSlot, Loc, AsValue); } -/// Emit a load from an l-value of atomic type. Note that the r-value -/// we produce is an r-value of the atomic *value* type. -RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, - AggValueSlot resultSlot) { - AtomicInfo atomics(*this, src); +void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, + llvm::AtomicOrdering AO, bool) { + // void __atomic_load(size_t size, void *mem, void *return, int order); + CallArgList Args; + Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get( + llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))), + CGF.getContext().IntTy); + emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); +} - // Check whether we should use a library call. - if (atomics.shouldUseLibcall()) { - llvm::Value *tempAddr; - if (!resultSlot.isIgnored()) { - assert(atomics.getEvaluationKind() == TEK_Aggregate); - tempAddr = resultSlot.getAddr(); - } else { - tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); - } +llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, + bool IsVolatile) { + // Okay, we're doing this natively. + llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress()); + llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); + Load->setAtomic(AO); - // void __atomic_load(size_t size, void *mem, void *return, int order); - CallArgList args; - args.add(RValue::get(atomics.getAtomicSizeValue()), - getContext().getSizeType()); - args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())), - getContext().VoidPtrTy); - args.add(RValue::get(EmitCastToVoidPtr(tempAddr)), - getContext().VoidPtrTy); - args.add(RValue::get(llvm::ConstantInt::get( - IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), - getContext().IntTy); - emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args); + // Other decoration. + Load->setAlignment(getAtomicAlignment().getQuantity()); + if (IsVolatile) + Load->setVolatile(true); + if (LVal.getTBAAInfo()) + CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo()); + return Load; +} + +/// An LValue is a candidate for having its loads and stores be made atomic if +/// we are operating under /volatile:ms *and* the LValue itself is volatile and +/// performing such an operation can be performed without a libcall. +bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { + AtomicInfo AI(*this, LV); + bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); + // An atomic is inline if we don't need to use a libcall. + bool AtomicIsInline = !AI.shouldUseLibcall(); + return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline; +} - // Produce the r-value. - return atomics.convertTempToRValue(tempAddr, resultSlot, loc); +/// An type is a candidate for having its loads and stores be made atomic if +/// we are operating under /volatile:ms *and* we know the access is volatile and +/// performing such an operation can be performed without a libcall. +bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty, + bool IsVolatile) const { + // An atomic is inline if we don't need to use a libcall (e.g. it is builtin). + bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic( + getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty)); + return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline; +} + +RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, + AggValueSlot Slot) { + llvm::AtomicOrdering AO; + bool IsVolatile = LV.isVolatileQualified(); + if (LV.getType()->isAtomicType()) { + AO = llvm::SequentiallyConsistent; + } else { + AO = llvm::Acquire; + IsVolatile = true; } + return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); +} - // Okay, we're doing this natively. - llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress()); - llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load"); - load->setAtomic(llvm::SequentiallyConsistent); +RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, + bool AsValue, llvm::AtomicOrdering AO, + bool IsVolatile) { + // Check whether we should use a library call. + if (shouldUseLibcall()) { + llvm::Value *TempAddr; + if (LVal.isSimple() && !ResultSlot.isIgnored()) { + assert(getEvaluationKind() == TEK_Aggregate); + TempAddr = ResultSlot.getAddr(); + } else + TempAddr = CreateTempAlloca(); + + EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile); + + // Okay, turn that back into the original value or whole atomic (for + // non-simple lvalues) type. + return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue); + } - // Other decoration. - load->setAlignment(src.getAlignment().getQuantity()); - if (src.isVolatileQualified()) - load->setVolatile(true); - if (src.getTBAAInfo()) - CGM.DecorateInstruction(load, src.getTBAAInfo()); + // Okay, we're doing this natively. + auto *Load = EmitAtomicLoadOp(AO, IsVolatile); // If we're ignoring an aggregate return, don't do anything. - if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored()) + if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) return RValue::getAggregate(nullptr, false); - // Okay, turn that back into the original value type. - return atomics.convertIntToValue(load, resultSlot, loc); + // Okay, turn that back into the original value or atomic (for non-simple + // lvalues) type. + return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); } - +/// Emit a load from an l-value of atomic type. Note that the r-value +/// we produce is an r-value of the atomic *value* type. +RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, + llvm::AtomicOrdering AO, bool IsVolatile, + AggValueSlot resultSlot) { + AtomicInfo Atomics(*this, src); + return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, + IsVolatile); +} /// Copy an r-value into memory as part of storing to an atomic type. /// This needs to create a bit-pattern suitable for atomic operations. -void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const { +void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { + assert(LVal.isSimple()); // If we have an r-value, the rvalue should be of the atomic type, // which means that the caller is responsible for having zeroed // any padding. Just do an aggregate copy of that type. if (rvalue.isAggregate()) { - CGF.EmitAggregateCopy(dest.getAddress(), + CGF.EmitAggregateCopy(getAtomicAddress(), rvalue.getAggregateAddr(), getAtomicType(), (rvalue.isVolatileQualified() - || dest.isVolatileQualified()), - dest.getAlignment()); + || LVal.isVolatileQualified()), + LVal.getAlignment()); return; } // Okay, otherwise we're copying stuff. // Zero out the buffer if necessary. - emitMemSetZeroIfNecessary(dest); + emitMemSetZeroIfNecessary(); // Drill past the padding if present. - dest = projectValue(dest); + LValue TempLVal = projectValue(); // Okay, store the rvalue in. if (rvalue.isScalar()) { - CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true); + CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); } else { - CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true); + CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); } } @@ -1050,22 +1305,24 @@ llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const { return rvalue.getAggregateAddr(); // Otherwise, make a temporary and materialize into it. - llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp"); - LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment()); - emitCopyIntoMemory(rvalue, tempLV); - return temp; + LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(), + getAtomicAlignment()); + AtomicInfo Atomics(CGF, TempLV); + Atomics.emitCopyIntoMemory(rvalue); + return TempLV.getAddress(); } llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { // If we've got a scalar value of the right size, try to avoid going // through memory. - if (RVal.isScalar() && !hasPadding()) { + if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { llvm::Value *Value = RVal.getScalarVal(); if (isa<llvm::IntegerType>(Value->getType())) - return Value; + return CGF.EmitToMemory(Value, ValueTy); else { - llvm::IntegerType *InputIntTy = - llvm::IntegerType::get(CGF.getLLVMContext(), getValueSizeInBits()); + llvm::IntegerType *InputIntTy = llvm::IntegerType::get( + CGF.getLLVMContext(), + LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); if (isa<llvm::PointerType>(Value->getType())) return CGF.Builder.CreatePtrToInt(Value, InputIntTy); else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) @@ -1082,12 +1339,324 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { getAtomicAlignment().getQuantity()); } +std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( + llvm::Value *ExpectedVal, llvm::Value *DesiredVal, + llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { + // Do the atomic store. + auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress()); + auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal, + Success, Failure); + // Other decoration. + Inst->setVolatile(LVal.isVolatileQualified()); + Inst->setWeak(IsWeak); + + // Okay, turn that back into the original value type. + auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); + auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); + return std::make_pair(PreviousVal, SuccessFailureVal); +} + +llvm::Value * +AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, + llvm::Value *DesiredAddr, + llvm::AtomicOrdering Success, + llvm::AtomicOrdering Failure) { + // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, + // void *desired, int success, int failure); + CallArgList Args; + Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), + CGF.getContext().VoidPtrTy); + Args.add(RValue::get(llvm::ConstantInt::get( + CGF.IntTy, translateAtomicOrdering(Success))), + CGF.getContext().IntTy); + Args.add(RValue::get(llvm::ConstantInt::get( + CGF.IntTy, translateAtomicOrdering(Failure))), + CGF.getContext().IntTy); + auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", + CGF.getContext().BoolTy, Args); + + return SuccessFailureRVal.getScalarVal(); +} + +std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( + RValue Expected, RValue Desired, llvm::AtomicOrdering Success, + llvm::AtomicOrdering Failure, bool IsWeak) { + if (Failure >= Success) + // Don't assert on undefined behavior. + Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); + + // Check whether we should use a library call. + if (shouldUseLibcall()) { + // Produce a source address. + auto *ExpectedAddr = materializeRValue(Expected); + auto *DesiredAddr = materializeRValue(Desired); + auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, + Success, Failure); + return std::make_pair( + convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false), + Res); + } + + // If we've got a scalar value of the right size, try to avoid going + // through memory. + auto *ExpectedVal = convertRValueToInt(Expected); + auto *DesiredVal = convertRValueToInt(Desired); + auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, + Failure, IsWeak); + return std::make_pair( + ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false), + Res.second); +} + +static void +EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + llvm::Value *DesiredAddr) { + llvm::Value *Ptr = nullptr; + LValue UpdateLVal; + RValue UpRVal; + LValue AtomicLVal = Atomics.getAtomicLValue(); + LValue DesiredLVal; + if (AtomicLVal.isSimple()) { + UpRVal = OldRVal; + DesiredLVal = + LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(), + AtomicLVal.getAlignment(), CGF.CGM.getContext()); + } else { + // Build new lvalue for temp address + Ptr = Atomics.materializeRValue(OldRVal); + if (AtomicLVal.isBitField()) { + UpdateLVal = + LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + DesiredLVal = + LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + } else if (AtomicLVal.isVectorElt()) { + UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), + AtomicLVal.getType(), + AtomicLVal.getAlignment()); + DesiredLVal = LValue::MakeVectorElt( + DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), + AtomicLVal.getAlignment()); + } else { + assert(AtomicLVal.isExtVectorElt()); + UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), + AtomicLVal.getType(), + AtomicLVal.getAlignment()); + DesiredLVal = LValue::MakeExtVectorElt( + DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), + AtomicLVal.getAlignment()); + } + UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); + DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); + UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); + } + // Store new value in the corresponding memory area + RValue NewRVal = UpdateOp(UpRVal); + if (NewRVal.isScalar()) { + CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); + } else { + assert(NewRVal.isComplex()); + CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, + /*isInit=*/false); + } +} + +void AtomicInfo::EmitAtomicUpdateLibcall( + llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + llvm::Value *ExpectedAddr = CreateTempAlloca(); + + EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile); + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + CGF.EmitBlock(ContBB); + auto *DesiredAddr = CreateTempAlloca(); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + auto *OldVal = CGF.Builder.CreateAlignedLoad( + ExpectedAddr, getAtomicAlignment().getQuantity()); + CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr, + getAtomicAlignment().getQuantity()); + } + auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false); + EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); + auto *Res = + EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure); + CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +void AtomicInfo::EmitAtomicUpdateOp( + llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + // Do the atomic load. + auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); + // For non-simple lvalues perform compare-and-swap procedure. + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + auto *CurBB = CGF.Builder.GetInsertBlock(); + CGF.EmitBlock(ContBB); + llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), + /*NumReservedValues=*/2); + PHI->addIncoming(OldVal, CurBB); + auto *NewAtomicAddr = CreateTempAlloca(); + auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr, + getAtomicAlignment().getQuantity()); + } + auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), + SourceLocation(), /*AsValue=*/false); + EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); + auto *DesiredVal = CGF.Builder.CreateAlignedLoad( + NewAtomicIntAddr, getAtomicAlignment().getQuantity()); + // Try to write new value using cmpxchg operation + auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); + PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); + CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, + RValue UpdateRVal, llvm::Value *DesiredAddr) { + LValue AtomicLVal = Atomics.getAtomicLValue(); + LValue DesiredLVal; + // Build new lvalue for temp address + if (AtomicLVal.isBitField()) { + DesiredLVal = + LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + } else if (AtomicLVal.isVectorElt()) { + DesiredLVal = + LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), + AtomicLVal.getType(), AtomicLVal.getAlignment()); + } else { + assert(AtomicLVal.isExtVectorElt()); + DesiredLVal = LValue::MakeExtVectorElt( + DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), + AtomicLVal.getAlignment()); + } + DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); + // Store new value in the corresponding memory area + assert(UpdateRVal.isScalar()); + CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); +} + +void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, + RValue UpdateRVal, bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + llvm::Value *ExpectedAddr = CreateTempAlloca(); + + EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile); + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + CGF.EmitBlock(ContBB); + auto *DesiredAddr = CreateTempAlloca(); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + auto *OldVal = CGF.Builder.CreateAlignedLoad( + ExpectedAddr, getAtomicAlignment().getQuantity()); + CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr, + getAtomicAlignment().getQuantity()); + } + EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); + auto *Res = + EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure); + CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile) { + auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); + + // Do the atomic load. + auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); + // For non-simple lvalues perform compare-and-swap procedure. + auto *ContBB = CGF.createBasicBlock("atomic_cont"); + auto *ExitBB = CGF.createBasicBlock("atomic_exit"); + auto *CurBB = CGF.Builder.GetInsertBlock(); + CGF.EmitBlock(ContBB); + llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), + /*NumReservedValues=*/2); + PHI->addIncoming(OldVal, CurBB); + auto *NewAtomicAddr = CreateTempAlloca(); + auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); + if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || + requiresMemSetZero( + getAtomicAddress()->getType()->getPointerElementType())) { + CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr, + getAtomicAlignment().getQuantity()); + } + EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); + auto *DesiredVal = CGF.Builder.CreateAlignedLoad( + NewAtomicIntAddr, getAtomicAlignment().getQuantity()); + // Try to write new value using cmpxchg operation + auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); + PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); + CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); +} + +void AtomicInfo::EmitAtomicUpdate( + llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile) { + if (shouldUseLibcall()) { + EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); + } else { + EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); + } +} + +void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, + bool IsVolatile) { + if (shouldUseLibcall()) { + EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); + } else { + EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); + } +} + +void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, + bool isInit) { + bool IsVolatile = lvalue.isVolatileQualified(); + llvm::AtomicOrdering AO; + if (lvalue.getType()->isAtomicType()) { + AO = llvm::SequentiallyConsistent; + } else { + AO = llvm::Release; + IsVolatile = true; + } + return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); +} + /// Emit a store to an l-value of atomic type. /// /// Note that the r-value is expected to be an r-value *of the atomic /// type*; this means that for aggregate r-values, it should include /// storage for any padding that was necessary. -void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) { +void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, + llvm::AtomicOrdering AO, bool IsVolatile, + bool isInit) { // If this is an aggregate r-value, it should agree in type except // maybe for address-space qualification. assert(!rvalue.isAggregate() || @@ -1095,54 +1664,64 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, bool isInit) { == dest.getAddress()->getType()->getPointerElementType()); AtomicInfo atomics(*this, dest); + LValue LVal = atomics.getAtomicLValue(); // If this is an initialization, just put the value there normally. - if (isInit) { - atomics.emitCopyIntoMemory(rvalue, dest); - return; - } + if (LVal.isSimple()) { + if (isInit) { + atomics.emitCopyIntoMemory(rvalue); + return; + } - // Check whether we should use a library call. - if (atomics.shouldUseLibcall()) { - // Produce a source address. - llvm::Value *srcAddr = atomics.materializeRValue(rvalue); + // Check whether we should use a library call. + if (atomics.shouldUseLibcall()) { + // Produce a source address. + llvm::Value *srcAddr = atomics.materializeRValue(rvalue); - // void __atomic_store(size_t size, void *mem, void *val, int order) - CallArgList args; - args.add(RValue::get(atomics.getAtomicSizeValue()), - getContext().getSizeType()); - args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())), - getContext().VoidPtrTy); - args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), - getContext().VoidPtrTy); - args.add(RValue::get(llvm::ConstantInt::get( - IntTy, AtomicExpr::AO_ABI_memory_order_seq_cst)), - getContext().IntTy); - emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); + // void __atomic_store(size_t size, void *mem, void *val, int order) + CallArgList args; + args.add(RValue::get(atomics.getAtomicSizeValue()), + getContext().getSizeType()); + args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())), + getContext().VoidPtrTy); + args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy); + args.add(RValue::get(llvm::ConstantInt::get( + IntTy, AtomicInfo::translateAtomicOrdering(AO))), + getContext().IntTy); + emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); + return; + } + + // Okay, we're doing this natively. + llvm::Value *intValue = atomics.convertRValueToInt(rvalue); + + // Do the atomic store. + llvm::Value *addr = + atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); + intValue = Builder.CreateIntCast( + intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false); + llvm::StoreInst *store = Builder.CreateStore(intValue, addr); + + // Initializations don't need to be atomic. + if (!isInit) + store->setAtomic(AO); + + // Other decoration. + store->setAlignment(dest.getAlignment().getQuantity()); + if (IsVolatile) + store->setVolatile(true); + if (dest.getTBAAInfo()) + CGM.DecorateInstruction(store, dest.getTBAAInfo()); return; } - // Okay, we're doing this natively. - llvm::Value *intValue = atomics.convertRValueToInt(rvalue); - - // Do the atomic store. - llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress()); - llvm::StoreInst *store = Builder.CreateStore(intValue, addr); - - // Initializations don't need to be atomic. - if (!isInit) store->setAtomic(llvm::SequentiallyConsistent); - - // Other decoration. - store->setAlignment(dest.getAlignment().getQuantity()); - if (dest.isVolatileQualified()) - store->setVolatile(true); - if (dest.getTBAAInfo()) - CGM.DecorateInstruction(store, dest.getTBAAInfo()); + // Emit simple atomic update operation. + atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); } /// Emit a compare-and-exchange op for atomic type. /// -std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange( +std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, AggValueSlot Slot) { @@ -1156,56 +1735,15 @@ std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange( Obj.getAddress()->getType()->getPointerElementType()); AtomicInfo Atomics(*this, Obj); - if (Failure >= Success) - // Don't assert on undefined behavior. - Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); - - auto Alignment = Atomics.getValueAlignment(); - // Check whether we should use a library call. - if (Atomics.shouldUseLibcall()) { - auto *ExpectedAddr = Atomics.materializeRValue(Expected); - // Produce a source address. - auto *DesiredAddr = Atomics.materializeRValue(Desired); - // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, - // void *desired, int success, int failure); - CallArgList Args; - Args.add(RValue::get(Atomics.getAtomicSizeValue()), - getContext().getSizeType()); - Args.add(RValue::get(EmitCastToVoidPtr(Obj.getAddress())), - getContext().VoidPtrTy); - Args.add(RValue::get(EmitCastToVoidPtr(ExpectedAddr)), - getContext().VoidPtrTy); - Args.add(RValue::get(EmitCastToVoidPtr(DesiredAddr)), - getContext().VoidPtrTy); - Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Success)), - getContext().IntTy); - Args.add(RValue::get(llvm::ConstantInt::get(IntTy, Failure)), - getContext().IntTy); - auto SuccessFailureRVal = emitAtomicLibcall( - *this, "__atomic_compare_exchange", getContext().BoolTy, Args); - auto *PreviousVal = - Builder.CreateAlignedLoad(ExpectedAddr, Alignment.getQuantity()); - return std::make_pair(RValue::get(PreviousVal), SuccessFailureRVal); - } - - // If we've got a scalar value of the right size, try to avoid going - // through memory. - auto *ExpectedIntVal = Atomics.convertRValueToInt(Expected); - auto *DesiredIntVal = Atomics.convertRValueToInt(Desired); - - // Do the atomic store. - auto *Addr = Atomics.emitCastToAtomicIntPointer(Obj.getAddress()); - auto *Inst = Builder.CreateAtomicCmpXchg(Addr, ExpectedIntVal, DesiredIntVal, - Success, Failure); - // Other decoration. - Inst->setVolatile(Obj.isVolatileQualified()); - Inst->setWeak(IsWeak); + return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, + IsWeak); +} - // Okay, turn that back into the original value type. - auto *PreviousVal = Builder.CreateExtractValue(Inst, /*Idxs=*/0); - auto *SuccessFailureVal = Builder.CreateExtractValue(Inst, /*Idxs=*/1); - return std::make_pair(Atomics.convertIntToValue(PreviousVal, Slot, Loc), - RValue::get(SuccessFailureVal)); +void CodeGenFunction::EmitAtomicUpdate( + LValue LVal, llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { + AtomicInfo Atomics(*this, LVal); + Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); } void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { @@ -1214,13 +1752,13 @@ void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { switch (atomics.getEvaluationKind()) { case TEK_Scalar: { llvm::Value *value = EmitScalarExpr(init); - atomics.emitCopyIntoMemory(RValue::get(value), dest); + atomics.emitCopyIntoMemory(RValue::get(value)); return; } case TEK_Complex: { ComplexPairTy value = EmitComplexExpr(init); - atomics.emitCopyIntoMemory(RValue::getComplex(value), dest); + atomics.emitCopyIntoMemory(RValue::getComplex(value)); return; } @@ -1229,8 +1767,8 @@ void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { // of atomic type. bool Zeroed = false; if (!init->getType()->isAtomicType()) { - Zeroed = atomics.emitMemSetZeroIfNecessary(dest); - dest = atomics.projectValue(dest); + Zeroed = atomics.emitMemSetZeroIfNecessary(); + dest = atomics.projectValue(); } // Evaluate the expression directly into the destination. diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp index b98460a9ddd8..3fd344c389a5 100644 --- a/lib/CodeGen/CGBlocks.cpp +++ b/lib/CodeGen/CGBlocks.cpp @@ -621,8 +621,8 @@ static void enterBlockScope(CodeGenFunction &CGF, BlockDecl *block) { } // GEP down to the address. - llvm::Value *addr = CGF.Builder.CreateStructGEP(blockInfo.Address, - capture.getIndex()); + llvm::Value *addr = CGF.Builder.CreateStructGEP( + blockInfo.StructureType, blockInfo.Address, capture.getIndex()); // We can use that GEP as the dominating IP. if (!blockInfo.DominatingIP) @@ -721,6 +721,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // Build the block descriptor. llvm::Constant *descriptor = buildBlockDescriptor(CGM, blockInfo); + llvm::Type *blockTy = blockInfo.StructureType; llvm::AllocaInst *blockAddr = blockInfo.Address; assert(blockAddr && "block has no address!"); @@ -732,14 +733,17 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET; // Initialize the block literal. - Builder.CreateStore(isa, Builder.CreateStructGEP(blockAddr, 0, "block.isa")); - Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()), - Builder.CreateStructGEP(blockAddr, 1, "block.flags")); - Builder.CreateStore(llvm::ConstantInt::get(IntTy, 0), - Builder.CreateStructGEP(blockAddr, 2, "block.reserved")); - Builder.CreateStore(blockFn, Builder.CreateStructGEP(blockAddr, 3, - "block.invoke")); - Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockAddr, 4, + Builder.CreateStore( + isa, Builder.CreateStructGEP(blockTy, blockAddr, 0, "block.isa")); + Builder.CreateStore( + llvm::ConstantInt::get(IntTy, flags.getBitMask()), + Builder.CreateStructGEP(blockTy, blockAddr, 1, "block.flags")); + Builder.CreateStore( + llvm::ConstantInt::get(IntTy, 0), + Builder.CreateStructGEP(blockTy, blockAddr, 2, "block.reserved")); + Builder.CreateStore( + blockFn, Builder.CreateStructGEP(blockTy, blockAddr, 3, "block.invoke")); + Builder.CreateStore(descriptor, Builder.CreateStructGEP(blockTy, blockAddr, 4, "block.descriptor")); // Finally, capture all the values into the block. @@ -747,9 +751,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // First, 'this'. if (blockDecl->capturesCXXThis()) { - llvm::Value *addr = Builder.CreateStructGEP(blockAddr, - blockInfo.CXXThisIndex, - "block.captured-this.addr"); + llvm::Value *addr = Builder.CreateStructGEP( + blockTy, blockAddr, blockInfo.CXXThisIndex, "block.captured-this.addr"); Builder.CreateStore(LoadCXXThis(), addr); } @@ -766,9 +769,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { // This will be a [[type]]*, except that a byref entry will just be // an i8**. - llvm::Value *blockField = - Builder.CreateStructGEP(blockAddr, capture.getIndex(), - "block.captured"); + llvm::Value *blockField = Builder.CreateStructGEP( + blockTy, blockAddr, capture.getIndex(), "block.captured"); // Compute the address of the thing we're going to move into the // block literal. @@ -779,7 +781,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) { BlockInfo->getCapture(variable); // This is a [[type]]*, except that a byref entry wil just be an i8**. - src = Builder.CreateStructGEP(LoadBlockStruct(), + src = Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(), enclosingCapture.getIndex(), "block.capture.addr"); } else if (blockDecl->isConversionFromLambda()) { @@ -964,7 +966,8 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E, Builder.CreateBitCast(Callee, BlockLiteralTy, "block.literal"); // Get the function pointer from the literal. - llvm::Value *FuncPtr = Builder.CreateStructGEP(BlockLiteral, 3); + llvm::Value *FuncPtr = Builder.CreateStructGEP( + CGM.getGenericBlockLiteralType(), BlockLiteral, 3); BlockLiteral = Builder.CreateBitCast(BlockLiteral, VoidPtrTy); @@ -1004,26 +1007,27 @@ llvm::Value *CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable, if (capture.isConstant()) return LocalDeclMap[variable]; llvm::Value *addr = - Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(), - "block.capture.addr"); + Builder.CreateStructGEP(BlockInfo->StructureType, LoadBlockStruct(), + capture.getIndex(), "block.capture.addr"); if (isByRef) { // addr should be a void** right now. Load, then cast the result // to byref*. addr = Builder.CreateLoad(addr); - llvm::PointerType *byrefPointerType - = llvm::PointerType::get(BuildByRefType(variable), 0); + auto *byrefType = BuildByRefType(variable); + llvm::PointerType *byrefPointerType = llvm::PointerType::get(byrefType, 0); addr = Builder.CreateBitCast(addr, byrefPointerType, "byref.addr"); // Follow the forwarding pointer. - addr = Builder.CreateStructGEP(addr, 1, "byref.forwarding"); + addr = Builder.CreateStructGEP(byrefType, addr, 1, "byref.forwarding"); addr = Builder.CreateLoad(addr, "byref.addr.forwarded"); // Cast back to byref* and GEP over to the actual object. addr = Builder.CreateBitCast(addr, byrefPointerType); - addr = Builder.CreateStructGEP(addr, getByRefValueLLVMField(variable), + addr = Builder.CreateStructGEP(byrefType, addr, + getByRefValueLLVMField(variable).second, variable->getNameAsString()); } @@ -1136,8 +1140,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, args.push_back(&selfDecl); // Now add the rest of the parameters. - for (auto i : blockDecl->params()) - args.push_back(i); + args.append(blockDecl->param_begin(), blockDecl->param_end()); // Create the function declaration. const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType(); @@ -1178,7 +1181,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, Alloca->setAlignment(Align); // Set the DebugLocation to empty, so the store is recognized as a // frame setup instruction by llvm::DwarfDebug::beginFunction(). - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); Builder.CreateAlignedStore(BlockPointer, Alloca, Align); BlockPointerDbgLoc = Alloca; } @@ -1186,9 +1189,9 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, // If we have a C++ 'this' reference, go ahead and force it into // existence now. if (blockDecl->capturesCXXThis()) { - llvm::Value *addr = Builder.CreateStructGEP(BlockPointer, - blockInfo.CXXThisIndex, - "block.captured-this"); + llvm::Value *addr = + Builder.CreateStructGEP(blockInfo.StructureType, BlockPointer, + blockInfo.CXXThisIndex, "block.captured-this"); CXXThisValue = Builder.CreateLoad(addr, "this"); } @@ -1218,8 +1221,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, EmitLambdaBlockInvokeBody(); else { PGO.assignRegionCounters(blockDecl, fn); - RegionCounter Cnt = getPGORegionCounter(blockDecl->getBody()); - Cnt.beginRegion(Builder); + incrementProfileCounter(blockDecl->getBody()); EmitStmt(blockDecl->getBody()); } @@ -1328,11 +1330,10 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { nullptr, SC_Static, false, false); - // Create a scope with an artificial location for the body of this function. - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(FD, C.VoidTy, Fn, FI, args); - ArtificialLocation AL(*this); - + // Create a scope with an artificial location for the body of this function. + auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); llvm::Value *src = GetAddrOfLocalVar(&srcDecl); @@ -1404,8 +1405,10 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) { } unsigned index = capture.getIndex(); - llvm::Value *srcField = Builder.CreateStructGEP(src, index); - llvm::Value *dstField = Builder.CreateStructGEP(dst, index); + llvm::Value *srcField = + Builder.CreateStructGEP(blockInfo.StructureType, src, index); + llvm::Value *dstField = + Builder.CreateStructGEP(blockInfo.StructureType, dst, index); // If there's an explicit copy expression, we do that. if (copyExpr) { @@ -1500,9 +1503,9 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { nullptr, SC_Static, false, false); // Create a scope with an artificial location for the body of this function. - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(FD, C.VoidTy, Fn, FI, args); - ArtificialLocation AL(*this); + auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo(); @@ -1562,7 +1565,8 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) { } unsigned index = capture.getIndex(); - llvm::Value *srcField = Builder.CreateStructGEP(src, index); + llvm::Value *srcField = + Builder.CreateStructGEP(blockInfo.StructureType, src, index); // If there's an explicit copy expression, we do that. if (dtor) { @@ -1801,13 +1805,15 @@ generateByrefCopyHelper(CodeGenFunction &CGF, llvm::Value *destField = CGF.GetAddrOfLocalVar(&dst); destField = CGF.Builder.CreateLoad(destField); destField = CGF.Builder.CreateBitCast(destField, byrefPtrType); - destField = CGF.Builder.CreateStructGEP(destField, valueFieldIndex, "x"); + destField = CGF.Builder.CreateStructGEP(&byrefType, destField, + valueFieldIndex, "x"); // src->x llvm::Value *srcField = CGF.GetAddrOfLocalVar(&src); srcField = CGF.Builder.CreateLoad(srcField); srcField = CGF.Builder.CreateBitCast(srcField, byrefPtrType); - srcField = CGF.Builder.CreateStructGEP(srcField, valueFieldIndex, "x"); + srcField = + CGF.Builder.CreateStructGEP(&byrefType, srcField, valueFieldIndex, "x"); byrefInfo.emitCopy(CGF, destField, srcField); } @@ -1868,7 +1874,7 @@ generateByrefDisposeHelper(CodeGenFunction &CGF, llvm::Value *V = CGF.GetAddrOfLocalVar(&src); V = CGF.Builder.CreateLoad(V); V = CGF.Builder.CreateBitCast(V, byrefType.getPointerTo(0)); - V = CGF.Builder.CreateStructGEP(V, byrefValueIndex, "x"); + V = CGF.Builder.CreateStructGEP(&byrefType, V, byrefValueIndex, "x"); byrefInfo.emitDispose(CGF, V); } @@ -1925,7 +1931,7 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, const VarDecl &var = *emission.Variable; QualType type = var.getType(); - unsigned byrefValueIndex = getByRefValueLLVMField(&var); + unsigned byrefValueIndex = getByRefValueLLVMField(&var).second; if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) { const Expr *copyExpr = CGM.getContext().getBlockVarCopyInits(&var); @@ -1995,18 +2001,20 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType, return ::buildByrefHelpers(CGM, byrefType, byrefValueIndex, byrefInfo); } -unsigned CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const { +std::pair<llvm::Type *, unsigned> +CodeGenFunction::getByRefValueLLVMField(const ValueDecl *VD) const { assert(ByRefValueInfo.count(VD) && "Did not find value!"); - - return ByRefValueInfo.find(VD)->second.second; + + return ByRefValueInfo.find(VD)->second; } llvm::Value *CodeGenFunction::BuildBlockByrefAddress(llvm::Value *BaseAddr, const VarDecl *V) { - llvm::Value *Loc = Builder.CreateStructGEP(BaseAddr, 1, "forwarding"); + auto P = getByRefValueLLVMField(V); + llvm::Value *Loc = + Builder.CreateStructGEP(P.first, BaseAddr, 1, "forwarding"); Loc = Builder.CreateLoad(Loc); - Loc = Builder.CreateStructGEP(Loc, getByRefValueLLVMField(V), - V->getNameAsString()); + Loc = Builder.CreateStructGEP(P.first, Loc, P.second, V->getNameAsString()); return Loc; } @@ -2143,11 +2151,12 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { if (type.isObjCGCWeak()) isa = 1; V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa"); - Builder.CreateStore(V, Builder.CreateStructGEP(addr, 0, "byref.isa")); + Builder.CreateStore(V, + Builder.CreateStructGEP(nullptr, addr, 0, "byref.isa")); // Store the address of the variable into its own forwarding pointer. - Builder.CreateStore(addr, - Builder.CreateStructGEP(addr, 1, "byref.forwarding")); + Builder.CreateStore( + addr, Builder.CreateStructGEP(nullptr, addr, 1, "byref.forwarding")); // Blocks ABI: // c) the flags field is set to either 0 if no helper functions are @@ -2193,25 +2202,26 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) { printf("\n"); } } - + Builder.CreateStore(llvm::ConstantInt::get(IntTy, flags.getBitMask()), - Builder.CreateStructGEP(addr, 2, "byref.flags")); + Builder.CreateStructGEP(nullptr, addr, 2, "byref.flags")); CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType); V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity()); - Builder.CreateStore(V, Builder.CreateStructGEP(addr, 3, "byref.size")); + Builder.CreateStore(V, + Builder.CreateStructGEP(nullptr, addr, 3, "byref.size")); if (helpers) { - llvm::Value *copy_helper = Builder.CreateStructGEP(addr, 4); + llvm::Value *copy_helper = Builder.CreateStructGEP(nullptr, addr, 4); Builder.CreateStore(helpers->CopyHelper, copy_helper); - llvm::Value *destroy_helper = Builder.CreateStructGEP(addr, 5); + llvm::Value *destroy_helper = Builder.CreateStructGEP(nullptr, addr, 5); Builder.CreateStore(helpers->DisposeHelper, destroy_helper); } if (ByRefHasLifetime && HasByrefExtendedLayout) { llvm::Constant* ByrefLayoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type); - llvm::Value *ByrefInfoAddr = Builder.CreateStructGEP(addr, helpers ? 6 : 4, - "byref.layout"); + llvm::Value *ByrefInfoAddr = + Builder.CreateStructGEP(nullptr, addr, helpers ? 6 : 4, "byref.layout"); // cast destination to pointer to source type. llvm::Type *DesTy = ByrefLayoutInfo->getType(); DesTy = DesTy->getPointerTo(); diff --git a/lib/CodeGen/CGBuilder.h b/lib/CodeGen/CGBuilder.h index 72ba4faa3c7c..6610659131f7 100644 --- a/lib/CodeGen/CGBuilder.h +++ b/lib/CodeGen/CGBuilder.h @@ -33,7 +33,7 @@ protected: llvm::BasicBlock *BB, llvm::BasicBlock::iterator InsertPt) const; private: - void operator=(const CGBuilderInserter &) LLVM_DELETED_FUNCTION; + void operator=(const CGBuilderInserter &) = delete; CodeGenFunction *CGF; }; diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp index 635e34207de7..272baac80897 100644 --- a/lib/CodeGen/CGBuiltin.cpp +++ b/lib/CodeGen/CGBuiltin.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGCXXABI.h" #include "CGObjCRuntime.h" #include "CodeGenModule.h" #include "TargetInfo.h" @@ -21,9 +22,11 @@ #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Intrinsics.h" +#include <sstream> using namespace clang; using namespace CodeGen; @@ -156,6 +159,27 @@ static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) { return Call; } +/// Emit the computation of the sign bit for a floating point value. Returns +/// the i1 sign bit value. +static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) { + LLVMContext &C = CGF.CGM.getLLVMContext(); + + llvm::Type *Ty = V->getType(); + int Width = Ty->getPrimitiveSizeInBits(); + llvm::Type *IntTy = llvm::IntegerType::get(C, Width); + V = CGF.Builder.CreateBitCast(V, IntTy); + if (Ty->isPPC_FP128Ty()) { + // The higher-order double comes first, and so we need to truncate the + // pair to extract the overall sign. The order of the pair is the same + // in both little- and big-Endian modes. + Width >>= 1; + IntTy = llvm::IntegerType::get(C, Width); + V = CGF.Builder.CreateTrunc(V, IntTy); + } + Value *Zero = llvm::Constant::getNullValue(IntTy); + return CGF.Builder.CreateICmpSLT(V, Zero); +} + static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn, const CallExpr *E, llvm::Value *calleeValue) { return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E, @@ -181,7 +205,7 @@ static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF, "arguments have the same integer width?)"); llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType()); - llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y); + llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y}); Carry = CGF.Builder.CreateExtractValue(Tmp, 1); return CGF.Builder.CreateExtractValue(Tmp, 0); } @@ -230,8 +254,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, DstPtr = Builder.CreateBitCast(DstPtr, Type); SrcPtr = Builder.CreateBitCast(SrcPtr, Type); - return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy), - DstPtr, SrcPtr)); + return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), + {DstPtr, SrcPtr})); } case Builtin::BI__builtin_abs: case Builtin::BI__builtin_labs: @@ -309,7 +333,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Type *ResultType = ConvertType(E->getType()); Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); - Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); + Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); @@ -326,7 +350,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::Type *ResultType = ConvertType(E->getType()); Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); - Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef); + Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); @@ -342,9 +366,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue, - Builder.getTrue()), - llvm::ConstantInt::get(ArgType, 1)); + Value *Tmp = + Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}), + llvm::ConstantInt::get(ArgType, 1)); Value *Zero = llvm::Constant::getNullValue(ArgType); Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs"); @@ -389,11 +413,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *ArgValue = EmitScalarExpr(E->getArg(0)); llvm::Type *ArgType = ArgValue->getType(); - Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); Value *ExpectedValue = EmitScalarExpr(E->getArg(1)); + // Don't generate llvm.expect on -O0 as the backend won't use it for + // anything. + // Note, we still IRGen ExpectedValue because it could have side-effects. + if (CGM.getCodeGenOpts().OptimizationLevel == 0) + return RValue::get(ArgValue); - Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue, - "expval"); + Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType); + Value *Result = + Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval"); return RValue::get(Result); } case Builtin::BI__builtin_assume_aligned: { @@ -444,7 +473,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // FIXME: Get right address space. llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) }; Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys); - return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI)); + return RValue::get( + Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0)), CI})); } case Builtin::BI__builtin_prefetch: { Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0)); @@ -455,25 +485,25 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, llvm::ConstantInt::get(Int32Ty, 3); Value *Data = llvm::ConstantInt::get(Int32Ty, 1); Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data)); + return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data})); } case Builtin::BI__builtin_readcyclecounter: { Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter); - return RValue::get(Builder.CreateCall(F)); + return RValue::get(Builder.CreateCall(F, {})); } case Builtin::BI__builtin___clear_cache: { Value *Begin = EmitScalarExpr(E->getArg(0)); Value *End = EmitScalarExpr(E->getArg(1)); Value *F = CGM.getIntrinsic(Intrinsic::clear_cache); - return RValue::get(Builder.CreateCall2(F, Begin, End)); + return RValue::get(Builder.CreateCall(F, {Begin, End})); } case Builtin::BI__builtin_trap: { Value *F = CGM.getIntrinsic(Intrinsic::trap); - return RValue::get(Builder.CreateCall(F)); + return RValue::get(Builder.CreateCall(F, {})); } case Builtin::BI__debugbreak: { Value *F = CGM.getIntrinsic(Intrinsic::debugtrap); - return RValue::get(Builder.CreateCall(F)); + return RValue::get(Builder.CreateCall(F, {})); } case Builtin::BI__builtin_unreachable: { if (SanOpts.has(SanitizerKind::Unreachable)) { @@ -498,7 +528,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *Exponent = EmitScalarExpr(E->getArg(1)); llvm::Type *ArgType = Base->getType(); Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType); - return RValue::get(Builder.CreateCall2(F, Base, Exponent)); + return RValue::get(Builder.CreateCall(F, {Base, Exponent})); } case Builtin::BI__builtin_isgreater: @@ -551,8 +581,22 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType()))); } - // TODO: BI__builtin_isinf_sign - // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 + case Builtin::BI__builtin_isinf_sign: { + // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0 + Value *Arg = EmitScalarExpr(E->getArg(0)); + Value *AbsArg = EmitFAbs(*this, Arg); + Value *IsInf = Builder.CreateFCmpOEQ( + AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf"); + Value *IsNeg = EmitSignBit(*this, Arg); + + llvm::Type *IntTy = ConvertType(E->getType()); + Value *Zero = Constant::getNullValue(IntTy); + Value *One = ConstantInt::get(IntTy, 1); + Value *NegativeOne = ConstantInt::get(IntTy, -1); + Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One); + Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero); + return RValue::get(Result); + } case Builtin::BI__builtin_isnormal: { // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min @@ -815,7 +859,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32 : Intrinsic::eh_return_i64); - Builder.CreateCall2(F, Int, Ptr); + Builder.CreateCall(F, {Int, Ptr}); Builder.CreateUnreachable(); // We do need to preserve an insertion point. @@ -825,7 +869,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, } case Builtin::BI__builtin_unwind_init: { Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init); - return RValue::get(Builder.CreateCall(F)); + return RValue::get(Builder.CreateCall(F, {})); } case Builtin::BI__builtin_extend_pointer: { // Extends a pointer to the size of an _Unwind_Word, which is @@ -864,7 +908,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, // Store the stack pointer to the setjmp buffer. Value *StackAddr = - Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave)); + Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave), {}); Value *StackSaveSlot = Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2)); Builder.CreateStore(StackAddr, StackSaveSlot); @@ -1357,6 +1401,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, return RValue::get(Builder.CreateCall(F, Arg0)); } + case Builtin::BI__builtin_pow: + case Builtin::BI__builtin_powf: + case Builtin::BI__builtin_powl: case Builtin::BIpow: case Builtin::BIpowf: case Builtin::BIpowl: { @@ -1367,7 +1414,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *Exponent = EmitScalarExpr(E->getArg(1)); llvm::Type *ArgType = Base->getType(); Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType); - return RValue::get(Builder.CreateCall2(F, Base, Exponent)); + return RValue::get(Builder.CreateCall(F, {Base, Exponent})); } case Builtin::BIfma: @@ -1380,32 +1427,17 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Value *FirstArg = EmitScalarExpr(E->getArg(0)); llvm::Type *ArgType = FirstArg->getType(); Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType); - return RValue::get(Builder.CreateCall3(F, FirstArg, - EmitScalarExpr(E->getArg(1)), - EmitScalarExpr(E->getArg(2)))); + return RValue::get( + Builder.CreateCall(F, {FirstArg, EmitScalarExpr(E->getArg(1)), + EmitScalarExpr(E->getArg(2))})); } case Builtin::BI__builtin_signbit: case Builtin::BI__builtin_signbitf: case Builtin::BI__builtin_signbitl: { - LLVMContext &C = CGM.getLLVMContext(); - - Value *Arg = EmitScalarExpr(E->getArg(0)); - llvm::Type *ArgTy = Arg->getType(); - int ArgWidth = ArgTy->getPrimitiveSizeInBits(); - llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth); - Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy); - if (ArgTy->isPPC_FP128Ty()) { - // The higher-order double comes first, and so we need to truncate the - // pair to extract the overall sign. The order of the pair is the same - // in both little- and big-Endian modes. - ArgWidth >>= 1; - ArgIntTy = llvm::IntegerType::get(C, ArgWidth); - BCArg = Builder.CreateTrunc(BCArg, ArgIntTy); - } - Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy); - Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp); - return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType()))); + return RValue::get( + Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))), + ConvertType(E->getType()))); } case Builtin::BI__builtin_annotation: { llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0)); @@ -1650,6 +1682,76 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD, Builder.CreateAlignedLoad(IntToPtr, /*Align=*/4, /*isVolatile=*/true); return RValue::get(Load); } + + case Builtin::BI__exception_code: + case Builtin::BI_exception_code: + return RValue::get(EmitSEHExceptionCode()); + case Builtin::BI__exception_info: + case Builtin::BI_exception_info: + return RValue::get(EmitSEHExceptionInfo()); + case Builtin::BI__abnormal_termination: + case Builtin::BI_abnormal_termination: + return RValue::get(EmitSEHAbnormalTermination()); + case Builtin::BI_setjmpex: { + if (getTarget().getTriple().isOSMSVCRT()) { + llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; + llvm::AttributeSet ReturnsTwiceAttr = + AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, + llvm::Attribute::ReturnsTwice); + llvm::Constant *SetJmpEx = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), + "_setjmpex", ReturnsTwiceAttr); + llvm::Value *Buf = Builder.CreateBitOrPointerCast( + EmitScalarExpr(E->getArg(0)), Int8PtrTy); + llvm::Value *FrameAddr = + Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), + ConstantInt::get(Int32Ty, 0)); + llvm::Value *Args[] = {Buf, FrameAddr}; + llvm::CallSite CS = EmitRuntimeCallOrInvoke(SetJmpEx, Args); + CS.setAttributes(ReturnsTwiceAttr); + return RValue::get(CS.getInstruction()); + } + break; + } + case Builtin::BI_setjmp: { + if (getTarget().getTriple().isOSMSVCRT()) { + llvm::AttributeSet ReturnsTwiceAttr = + AttributeSet::get(getLLVMContext(), llvm::AttributeSet::FunctionIndex, + llvm::Attribute::ReturnsTwice); + llvm::Value *Buf = Builder.CreateBitOrPointerCast( + EmitScalarExpr(E->getArg(0)), Int8PtrTy); + llvm::CallSite CS; + if (getTarget().getTriple().getArch() == llvm::Triple::x86) { + llvm::Type *ArgTypes[] = {Int8PtrTy, IntTy}; + llvm::Constant *SetJmp3 = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/true), + "_setjmp3", ReturnsTwiceAttr); + llvm::Value *Count = ConstantInt::get(IntTy, 0); + llvm::Value *Args[] = {Buf, Count}; + CS = EmitRuntimeCallOrInvoke(SetJmp3, Args); + } else { + llvm::Type *ArgTypes[] = {Int8PtrTy, Int8PtrTy}; + llvm::Constant *SetJmp = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, ArgTypes, /*isVarArg=*/false), + "_setjmp", ReturnsTwiceAttr); + llvm::Value *FrameAddr = + Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress), + ConstantInt::get(Int32Ty, 0)); + llvm::Value *Args[] = {Buf, FrameAddr}; + CS = EmitRuntimeCallOrInvoke(SetJmp, Args); + } + CS.setAttributes(ReturnsTwiceAttr); + return RValue::get(CS.getInstruction()); + } + break; + } + + case Builtin::BI__GetExceptionInfo: { + if (llvm::GlobalVariable *GV = + CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType())) + return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy)); + break; + } } // If this is an alias for a lib function (e.g. __builtin_sin), emit @@ -1764,6 +1866,8 @@ Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID, case llvm::Triple::r600: case llvm::Triple::amdgcn: return EmitR600BuiltinExpr(BuiltinID, E); + case llvm::Triple::systemz: + return EmitSystemZBuiltinExpr(BuiltinID, E); default: return nullptr; } @@ -2534,7 +2638,7 @@ Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID, // Return type. SmallVector<llvm::Type *, 3> Tys; if (Modifier & AddRetType) { - llvm::Type *Ty = ConvertType(E->getCallReturnType()); + llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); if (Modifier & VectorizeRetType) Ty = llvm::VectorType::get( Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1); @@ -2812,7 +2916,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Ops[2] = Builder.CreateBitCast(Ops[2], Ty); // NEON intrinsic puts accumulator first, unlike the LLVM fma. - return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); + return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); } case NEON::BI__builtin_neon_vld1_v: case NEON::BI__builtin_neon_vld1q_v: @@ -2825,7 +2929,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( case NEON::BI__builtin_neon_vld4_v: case NEON::BI__builtin_neon_vld4q_v: { Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty); - Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint); + Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); return Builder.CreateStore(Ops[1], Ops[0]); @@ -3004,7 +3108,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Indices.push_back(Builder.getInt32(i+vi)); Indices.push_back(Builder.getInt32(i+e+vi)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); SV = Builder.CreateStore(SV, Addr); @@ -3032,7 +3136,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); SV = Builder.CreateStore(SV, Addr); @@ -3052,7 +3156,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr( Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); SV = Builder.CreateStore(SV, Addr); @@ -3185,7 +3289,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, : InlineAsm::get(FTy, ".inst 0x" + utohexstr(ZExtValue), "", /*SideEffects=*/true); - return Builder.CreateCall(Emit); + return Builder.CreateCall(Emit, {}); } if (BuiltinID == ARM::BI__builtin_arm_dbg) { @@ -3202,7 +3306,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Value *Locality = llvm::ConstantInt::get(Int32Ty, 3); Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return Builder.CreateCall4(F, Address, RW, Locality, IsData); + return Builder.CreateCall(F, {Address, RW, Locality, IsData}); } if (BuiltinID == ARM::BI__builtin_arm_rbit) { @@ -3300,7 +3404,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Value *Arg0 = Builder.CreateExtractValue(Val, 0); Value *Arg1 = Builder.CreateExtractValue(Val, 1); Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); - return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd"); + return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd"); } if (BuiltinID == ARM::BI__builtin_arm_strex || @@ -3324,12 +3428,12 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, ? Intrinsic::arm_stlex : Intrinsic::arm_strex, StoreAddr->getType()); - return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex"); + return Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex"); } if (BuiltinID == ARM::BI__builtin_arm_clrex) { Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex); - return Builder.CreateCall(F); + return Builder.CreateCall(F, {}); } // CRC32 @@ -3365,13 +3469,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty); Function *F = CGM.getIntrinsic(CRCIntrinsicID); - Value *Res = Builder.CreateCall2(F, Arg0, Arg1a); - return Builder.CreateCall2(F, Res, Arg1b); + Value *Res = Builder.CreateCall(F, {Arg0, Arg1a}); + return Builder.CreateCall(F, {Res, Arg1b}); } else { Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty); Function *F = CGM.getIntrinsic(CRCIntrinsicID); - return Builder.CreateCall2(F, Arg0, Arg1); + return Builder.CreateCall(F, {Arg0, Arg1}); } } @@ -3547,7 +3651,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, // Load the value as a one-element vector. Ty = llvm::VectorType::get(VTy->getElementType(), 1); Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty); - Value *Ld = Builder.CreateCall2(F, Ops[0], Align); + Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); // Combine them. SmallVector<Constant*, 2> Indices; Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane)); @@ -3582,7 +3686,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, default: llvm_unreachable("unknown vld_dup intrinsic?"); } Function *F = CGM.getIntrinsic(Int, Ty); - Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup"); + Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); Ops[0] = Builder.CreateBitCast(Ops[0], Ty); return Builder.CreateStore(Ops[1], Ops[0]); @@ -3651,7 +3755,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID, Ops[1] = Builder.CreateBitCast(Ops[1], Ty); Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts; - Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]); + Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); case NEON::BI__builtin_neon_vsri_n_v: case NEON::BI__builtin_neon_vsriq_n_v: @@ -3979,7 +4083,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify // PLDL3STRM or PLDL2STRM. Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return Builder.CreateCall4(F, Address, RW, Locality, IsData); + return Builder.CreateCall(F, {Address, RW, Locality, IsData}); } if (BuiltinID == AArch64::BI__builtin_arm_rbit) { @@ -4074,9 +4178,11 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Value *Arg1 = Builder.CreateExtractValue(Val, 1); Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy); - return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp"); - } else if (BuiltinID == AArch64::BI__builtin_arm_strex || - BuiltinID == AArch64::BI__builtin_arm_stlex) { + return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp"); + } + + if (BuiltinID == AArch64::BI__builtin_arm_strex || + BuiltinID == AArch64::BI__builtin_arm_stlex) { Value *StoreVal = EmitScalarExpr(E->getArg(0)); Value *StoreAddr = EmitScalarExpr(E->getArg(1)); @@ -4096,12 +4202,12 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr, StoreAddr->getType()); - return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr"); + return Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr"); } if (BuiltinID == AArch64::BI__builtin_arm_clrex) { Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex); - return Builder.CreateCall(F); + return Builder.CreateCall(F, {}); } // CRC32 @@ -4133,7 +4239,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, llvm::Type *DataTy = F->getFunctionType()->getParamType(1); Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy); - return Builder.CreateCall2(F, Arg0, Arg1); + return Builder.CreateCall(F, {Arg0, Arg1}); } llvm::SmallVector<Value*, 4> Ops; @@ -4248,36 +4354,36 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, case NEON::BI__builtin_neon_vceqzs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ, - ICmpInst::ICMP_EQ, "vceqz"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ, "vceqz"); case NEON::BI__builtin_neon_vcgezd_s64: case NEON::BI__builtin_neon_vcgezd_f64: case NEON::BI__builtin_neon_vcgezs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE, - ICmpInst::ICMP_SGE, "vcgez"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE, "vcgez"); case NEON::BI__builtin_neon_vclezd_s64: case NEON::BI__builtin_neon_vclezd_f64: case NEON::BI__builtin_neon_vclezs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE, - ICmpInst::ICMP_SLE, "vclez"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE, "vclez"); case NEON::BI__builtin_neon_vcgtzd_s64: case NEON::BI__builtin_neon_vcgtzd_f64: case NEON::BI__builtin_neon_vcgtzs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT, - ICmpInst::ICMP_SGT, "vcgtz"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT, "vcgtz"); case NEON::BI__builtin_neon_vcltzd_s64: case NEON::BI__builtin_neon_vcltzd_f64: case NEON::BI__builtin_neon_vcltzs_f32: Ops.push_back(EmitScalarExpr(E->getArg(0))); return EmitAArch64CompareBuiltinExpr( - Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT, - ICmpInst::ICMP_SLT, "vcltz"); + Ops[0], ConvertType(E->getCallReturnType(getContext())), + ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT, "vcltz"); case NEON::BI__builtin_neon_vceqzd_u64: { llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext()); @@ -4528,8 +4634,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, : Intrinsic::aarch64_neon_srshl; Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); - Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1], - Builder.CreateSExt(Ops[2], Int64Ty)); + Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), + {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); } case NEON::BI__builtin_neon_vshld_n_s64: @@ -4699,7 +4805,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[2] = Builder.CreateBitCast(Ops[2], VTy); Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy); - Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); + Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); return Builder.CreateBitCast(Result, Ty); } Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); @@ -4713,7 +4819,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, cast<ConstantInt>(Ops[3])); Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); - return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); + return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); } case NEON::BI__builtin_neon_vfmaq_laneq_v: { Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); @@ -4722,17 +4828,17 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Ops[2] = Builder.CreateBitCast(Ops[2], Ty); Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); - return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]); + return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); } case NEON::BI__builtin_neon_vfmas_lane_f32: case NEON::BI__builtin_neon_vfmas_laneq_f32: case NEON::BI__builtin_neon_vfmad_lane_f64: case NEON::BI__builtin_neon_vfmad_laneq_f64: { Ops.push_back(EmitScalarExpr(E->getArg(3))); - llvm::Type *Ty = ConvertType(E->getCallReturnType()); + llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext())); Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty); Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); - return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]); + return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); } case NEON::BI__builtin_neon_vfms_v: case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types @@ -5667,7 +5773,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Indices.push_back(ConstantInt::get(Int32Ty, i+vi)); Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); SV = Builder.CreateStore(SV, Addr); @@ -5686,7 +5792,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi)); - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); SV = Builder.CreateStore(SV, Addr); @@ -5706,7 +5812,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1)); Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e)); } - Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi); + Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); SV = llvm::ConstantVector::get(Indices); SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); SV = Builder.CreateStore(SV, Addr); @@ -5817,7 +5923,7 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Value *Locality = EmitScalarExpr(E->getArg(1)); Value *Data = ConstantInt::get(Int32Ty, 1); Value *F = CGM.getIntrinsic(Intrinsic::prefetch); - return Builder.CreateCall4(F, Address, RW, Locality, Data); + return Builder.CreateCall(F, {Address, RW, Locality, Data}); } case X86::BI__builtin_ia32_vec_init_v8qi: case X86::BI__builtin_ia32_vec_init_v4hi: @@ -5856,104 +5962,95 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); return Builder.CreateStore(Ops[1], Ops[0]); } - case X86::BI__builtin_ia32_palignr: { - unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); - - // If palignr is shifting the pair of input vectors less than 9 bytes, - // emit a shuffle instruction. - if (shiftVal <= 8) { - SmallVector<llvm::Constant*, 8> Indices; - for (unsigned i = 0; i != 8; ++i) - Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); - - Value* SV = llvm::ConstantVector::get(Indices); - return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); + case X86::BI__builtin_ia32_palignr128: + case X86::BI__builtin_ia32_palignr256: { + unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); + + unsigned NumElts = + cast<llvm::VectorType>(Ops[0]->getType())->getNumElements(); + assert(NumElts % 16 == 0); + unsigned NumLanes = NumElts / 16; + unsigned NumLaneElts = NumElts / NumLanes; + + // If palignr is shifting the pair of vectors more than the size of two + // lanes, emit zero. + if (ShiftVal >= (2 * NumLaneElts)) + return llvm::Constant::getNullValue(ConvertType(E->getType())); + + // If palignr is shifting the pair of input vectors more than one lane, + // but less than two lanes, convert to shifting in zeroes. + if (ShiftVal > NumLaneElts) { + ShiftVal -= NumLaneElts; + Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); } - // If palignr is shifting the pair of input vectors more than 8 but less - // than 16 bytes, emit a logical right shift of the destination. - if (shiftVal < 16) { - // MMX has these as 1 x i64 vectors for some odd optimization reasons. - llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1); - - Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); - Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8); - - // create i32 constant - llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q); - return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); + SmallVector<llvm::Constant*, 32> Indices; + // 256-bit palignr operates on 128-bit lanes so we need to handle that + for (unsigned l = 0; l != NumElts; l += NumLaneElts) { + for (unsigned i = 0; i != NumLaneElts; ++i) { + unsigned Idx = ShiftVal + i; + if (Idx >= NumLaneElts) + Idx += NumElts - NumLaneElts; // End of lane, switch operand. + Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); + } } - // If palignr is shifting the pair of vectors more than 16 bytes, emit zero. - return llvm::Constant::getNullValue(ConvertType(E->getType())); + Value* SV = llvm::ConstantVector::get(Indices); + return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); } - case X86::BI__builtin_ia32_palignr128: { - unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); + case X86::BI__builtin_ia32_pslldqi256: { + // Shift value is in bits so divide by 8. + unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; - // If palignr is shifting the pair of input vectors less than 17 bytes, - // emit a shuffle instruction. - if (shiftVal <= 16) { - SmallVector<llvm::Constant*, 16> Indices; - for (unsigned i = 0; i != 16; ++i) - Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i)); + // If pslldq is shifting the vector more than 15 bytes, emit zero. + if (shiftVal >= 16) + return llvm::Constant::getNullValue(ConvertType(E->getType())); - Value* SV = llvm::ConstantVector::get(Indices); - return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); + SmallVector<llvm::Constant*, 32> Indices; + // 256-bit pslldq operates on 128-bit lanes so we need to handle that + for (unsigned l = 0; l != 32; l += 16) { + for (unsigned i = 0; i != 16; ++i) { + unsigned Idx = 32 + i - shiftVal; + if (Idx < 32) Idx -= 16; // end of lane, switch operand. + Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); + } } - // If palignr is shifting the pair of input vectors more than 16 but less - // than 32 bytes, emit a logical right shift of the destination. - if (shiftVal < 32) { - llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2); - - Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); - Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); - - // create i32 constant - llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq); - return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); - } + llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); + Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); + Value *Zero = llvm::Constant::getNullValue(VecTy); - // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. - return llvm::Constant::getNullValue(ConvertType(E->getType())); - } - case X86::BI__builtin_ia32_palignr256: { - unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); - - // If palignr is shifting the pair of input vectors less than 17 bytes, - // emit a shuffle instruction. - if (shiftVal <= 16) { - SmallVector<llvm::Constant*, 32> Indices; - // 256-bit palignr operates on 128-bit lanes so we need to handle that - for (unsigned l = 0; l != 2; ++l) { - unsigned LaneStart = l * 16; - unsigned LaneEnd = (l+1) * 16; - for (unsigned i = 0; i != 16; ++i) { - unsigned Idx = shiftVal + i + LaneStart; - if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand - Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx)); - } + Value *SV = llvm::ConstantVector::get(Indices); + SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq"); + llvm::Type *ResultType = ConvertType(E->getType()); + return Builder.CreateBitCast(SV, ResultType, "cast"); + } + case X86::BI__builtin_ia32_psrldqi256: { + // Shift value is in bits so divide by 8. + unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; + + // If psrldq is shifting the vector more than 15 bytes, emit zero. + if (shiftVal >= 16) + return llvm::Constant::getNullValue(ConvertType(E->getType())); + + SmallVector<llvm::Constant*, 32> Indices; + // 256-bit psrldq operates on 128-bit lanes so we need to handle that + for (unsigned l = 0; l != 32; l += 16) { + for (unsigned i = 0; i != 16; ++i) { + unsigned Idx = i + shiftVal; + if (Idx >= 16) Idx += 16; // end of lane, switch operand. + Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx + l)); } - - Value* SV = llvm::ConstantVector::get(Indices); - return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); } - // If palignr is shifting the pair of input vectors more than 16 but less - // than 32 bytes, emit a logical right shift of the destination. - if (shiftVal < 32) { - llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4); - - Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); - Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8); - - // create i32 constant - llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq); - return Builder.CreateCall(F, makeArrayRef(Ops.data(), 2), "palignr"); - } + llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, 32); + Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); + Value *Zero = llvm::Constant::getNullValue(VecTy); - // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. - return llvm::Constant::getNullValue(ConvertType(E->getType())); + Value *SV = llvm::ConstantVector::get(Indices); + SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq"); + llvm::Type *ResultType = ConvertType(E->getType()); + return Builder.CreateBitCast(SV, ResultType, "cast"); } case X86::BI__builtin_ia32_movntps: case X86::BI__builtin_ia32_movntps256: @@ -5987,20 +6084,10 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, // 3DNow! case X86::BI__builtin_ia32_pswapdsf: case X86::BI__builtin_ia32_pswapdsi: { - const char *name; - Intrinsic::ID ID; - switch(BuiltinID) { - default: llvm_unreachable("Unsupported intrinsic!"); - case X86::BI__builtin_ia32_pswapdsf: - case X86::BI__builtin_ia32_pswapdsi: - name = "pswapd"; - ID = Intrinsic::x86_3dnowa_pswapd; - break; - } llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext()); Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); - llvm::Function *F = CGM.getIntrinsic(ID); - return Builder.CreateCall(F, Ops, name); + llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd); + return Builder.CreateCall(F, Ops, "pswapd"); } case X86::BI__builtin_ia32_rdrand16_step: case X86::BI__builtin_ia32_rdrand32_step: @@ -6031,17 +6118,10 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID, break; } - Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID)); + Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {}); Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]); return Builder.CreateExtractValue(Call, 1); } - // AVX2 broadcast - case X86::BI__builtin_ia32_vbroadcastsi256: { - Value *VecTmp = CreateMemTemp(E->getArg(0)->getType()); - Builder.CreateStore(Ops[0], VecTmp); - Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128); - return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy)); - } // SSE comparison intrisics case X86::BI__builtin_ia32_cmpeqps: case X86::BI__builtin_ia32_cmpltps: @@ -6318,7 +6398,7 @@ static Value *emitTernaryFPBuiltin(CodeGenFunction &CGF, llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2)); Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); - return CGF.Builder.CreateCall3(F, Src0, Src1, Src2); + return CGF.Builder.CreateCall(F, {Src0, Src1, Src2}); } // Emit an intrinsic that has 1 float or double operand, and 1 integer. @@ -6329,7 +6409,7 @@ static Value *emitFPIntBuiltin(CodeGenFunction &CGF, llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1)); Value *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType()); - return CGF.Builder.CreateCall2(F, Src0, Src1); + return CGF.Builder.CreateCall(F, {Src0, Src1}); } Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, @@ -6350,7 +6430,7 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::AMDGPU_div_scale, X->getType()); - llvm::Value *Tmp = Builder.CreateCall3(Callee, X, Y, Z); + llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z}); llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0); llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1); @@ -6373,7 +6453,7 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, llvm::Value *F = CGM.getIntrinsic(Intrinsic::AMDGPU_div_fmas, Src0->getType()); llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3); - return Builder.CreateCall4(F, Src0, Src1, Src2, Src3ToBool); + return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool}); } case R600::BI__builtin_amdgpu_div_fixup: case R600::BI__builtin_amdgpu_div_fixupf: @@ -6400,3 +6480,246 @@ Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID, return nullptr; } } + +/// Handle a SystemZ function in which the final argument is a pointer +/// to an int that receives the post-instruction CC value. At the LLVM level +/// this is represented as a function that returns a {result, cc} pair. +static Value *EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, + unsigned IntrinsicID, + const CallExpr *E) { + unsigned NumArgs = E->getNumArgs() - 1; + SmallVector<Value *, 8> Args(NumArgs); + for (unsigned I = 0; I < NumArgs; ++I) + Args[I] = CGF.EmitScalarExpr(E->getArg(I)); + Value *CCPtr = CGF.EmitScalarExpr(E->getArg(NumArgs)); + Value *F = CGF.CGM.getIntrinsic(IntrinsicID); + Value *Call = CGF.Builder.CreateCall(F, Args); + Value *CC = CGF.Builder.CreateExtractValue(Call, 1); + CGF.Builder.CreateStore(CC, CCPtr); + return CGF.Builder.CreateExtractValue(Call, 0); +} + +Value *CodeGenFunction::EmitSystemZBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + switch (BuiltinID) { + case SystemZ::BI__builtin_tbegin: { + Value *TDB = EmitScalarExpr(E->getArg(0)); + Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin); + return Builder.CreateCall(F, {TDB, Control}); + } + case SystemZ::BI__builtin_tbegin_nofloat: { + Value *TDB = EmitScalarExpr(E->getArg(0)); + Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat); + return Builder.CreateCall(F, {TDB, Control}); + } + case SystemZ::BI__builtin_tbeginc: { + Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy); + Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc); + return Builder.CreateCall(F, {TDB, Control}); + } + case SystemZ::BI__builtin_tabort: { + Value *Data = EmitScalarExpr(E->getArg(0)); + Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort); + return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort")); + } + case SystemZ::BI__builtin_non_tx_store: { + Value *Address = EmitScalarExpr(E->getArg(0)); + Value *Data = EmitScalarExpr(E->getArg(1)); + Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg); + return Builder.CreateCall(F, {Data, Address}); + } + + // Vector builtins. Note that most vector builtins are mapped automatically + // to target-specific LLVM intrinsics. The ones handled specially here can + // be represented via standard LLVM IR, which is preferable to enable common + // LLVM optimizations. + + case SystemZ::BI__builtin_s390_vpopctb: + case SystemZ::BI__builtin_s390_vpopcth: + case SystemZ::BI__builtin_s390_vpopctf: + case SystemZ::BI__builtin_s390_vpopctg: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType); + return Builder.CreateCall(F, X); + } + + case SystemZ::BI__builtin_s390_vclzb: + case SystemZ::BI__builtin_s390_vclzh: + case SystemZ::BI__builtin_s390_vclzf: + case SystemZ::BI__builtin_s390_vclzg: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); + Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType); + return Builder.CreateCall(F, {X, Undef}); + } + + case SystemZ::BI__builtin_s390_vctzb: + case SystemZ::BI__builtin_s390_vctzh: + case SystemZ::BI__builtin_s390_vctzf: + case SystemZ::BI__builtin_s390_vctzg: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false); + Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType); + return Builder.CreateCall(F, {X, Undef}); + } + + case SystemZ::BI__builtin_s390_vfsqdb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType); + return Builder.CreateCall(F, X); + } + case SystemZ::BI__builtin_s390_vfmadb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Y = EmitScalarExpr(E->getArg(1)); + Value *Z = EmitScalarExpr(E->getArg(2)); + Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); + return Builder.CreateCall(F, {X, Y, Z}); + } + case SystemZ::BI__builtin_s390_vfmsdb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Y = EmitScalarExpr(E->getArg(1)); + Value *Z = EmitScalarExpr(E->getArg(2)); + Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); + Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType); + return Builder.CreateCall(F, {X, Y, Builder.CreateFSub(Zero, Z, "sub")}); + } + case SystemZ::BI__builtin_s390_vflpdb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); + return Builder.CreateCall(F, X); + } + case SystemZ::BI__builtin_s390_vflndb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType); + Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType); + return Builder.CreateFSub(Zero, Builder.CreateCall(F, X), "sub"); + } + case SystemZ::BI__builtin_s390_vfidb: { + llvm::Type *ResultType = ConvertType(E->getType()); + Value *X = EmitScalarExpr(E->getArg(0)); + // Constant-fold the M4 and M5 mask arguments. + llvm::APSInt M4, M5; + bool IsConstM4 = E->getArg(1)->isIntegerConstantExpr(M4, getContext()); + bool IsConstM5 = E->getArg(2)->isIntegerConstantExpr(M5, getContext()); + assert(IsConstM4 && IsConstM5 && "Constant arg isn't actually constant?"); + (void)IsConstM4; (void)IsConstM5; + // Check whether this instance of vfidb can be represented via a LLVM + // standard intrinsic. We only support some combinations of M4 and M5. + Intrinsic::ID ID = Intrinsic::not_intrinsic; + switch (M4.getZExtValue()) { + default: break; + case 0: // IEEE-inexact exception allowed + switch (M5.getZExtValue()) { + default: break; + case 0: ID = Intrinsic::rint; break; + } + break; + case 4: // IEEE-inexact exception suppressed + switch (M5.getZExtValue()) { + default: break; + case 0: ID = Intrinsic::nearbyint; break; + case 1: ID = Intrinsic::round; break; + case 5: ID = Intrinsic::trunc; break; + case 6: ID = Intrinsic::ceil; break; + case 7: ID = Intrinsic::floor; break; + } + break; + } + if (ID != Intrinsic::not_intrinsic) { + Function *F = CGM.getIntrinsic(ID, ResultType); + return Builder.CreateCall(F, X); + } + Function *F = CGM.getIntrinsic(Intrinsic::s390_vfidb); + Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4); + Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5); + return Builder.CreateCall(F, {X, M4Value, M5Value}); + } + + // Vector intrisincs that output the post-instruction CC value. + +#define INTRINSIC_WITH_CC(NAME) \ + case SystemZ::BI__builtin_##NAME: \ + return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) + + INTRINSIC_WITH_CC(s390_vpkshs); + INTRINSIC_WITH_CC(s390_vpksfs); + INTRINSIC_WITH_CC(s390_vpksgs); + + INTRINSIC_WITH_CC(s390_vpklshs); + INTRINSIC_WITH_CC(s390_vpklsfs); + INTRINSIC_WITH_CC(s390_vpklsgs); + + INTRINSIC_WITH_CC(s390_vceqbs); + INTRINSIC_WITH_CC(s390_vceqhs); + INTRINSIC_WITH_CC(s390_vceqfs); + INTRINSIC_WITH_CC(s390_vceqgs); + + INTRINSIC_WITH_CC(s390_vchbs); + INTRINSIC_WITH_CC(s390_vchhs); + INTRINSIC_WITH_CC(s390_vchfs); + INTRINSIC_WITH_CC(s390_vchgs); + + INTRINSIC_WITH_CC(s390_vchlbs); + INTRINSIC_WITH_CC(s390_vchlhs); + INTRINSIC_WITH_CC(s390_vchlfs); + INTRINSIC_WITH_CC(s390_vchlgs); + + INTRINSIC_WITH_CC(s390_vfaebs); + INTRINSIC_WITH_CC(s390_vfaehs); + INTRINSIC_WITH_CC(s390_vfaefs); + + INTRINSIC_WITH_CC(s390_vfaezbs); + INTRINSIC_WITH_CC(s390_vfaezhs); + INTRINSIC_WITH_CC(s390_vfaezfs); + + INTRINSIC_WITH_CC(s390_vfeebs); + INTRINSIC_WITH_CC(s390_vfeehs); + INTRINSIC_WITH_CC(s390_vfeefs); + + INTRINSIC_WITH_CC(s390_vfeezbs); + INTRINSIC_WITH_CC(s390_vfeezhs); + INTRINSIC_WITH_CC(s390_vfeezfs); + + INTRINSIC_WITH_CC(s390_vfenebs); + INTRINSIC_WITH_CC(s390_vfenehs); + INTRINSIC_WITH_CC(s390_vfenefs); + + INTRINSIC_WITH_CC(s390_vfenezbs); + INTRINSIC_WITH_CC(s390_vfenezhs); + INTRINSIC_WITH_CC(s390_vfenezfs); + + INTRINSIC_WITH_CC(s390_vistrbs); + INTRINSIC_WITH_CC(s390_vistrhs); + INTRINSIC_WITH_CC(s390_vistrfs); + + INTRINSIC_WITH_CC(s390_vstrcbs); + INTRINSIC_WITH_CC(s390_vstrchs); + INTRINSIC_WITH_CC(s390_vstrcfs); + + INTRINSIC_WITH_CC(s390_vstrczbs); + INTRINSIC_WITH_CC(s390_vstrczhs); + INTRINSIC_WITH_CC(s390_vstrczfs); + + INTRINSIC_WITH_CC(s390_vfcedbs); + INTRINSIC_WITH_CC(s390_vfchdbs); + INTRINSIC_WITH_CC(s390_vfchedbs); + + INTRINSIC_WITH_CC(s390_vftcidb); + +#undef INTRINSIC_WITH_CC + + default: + return nullptr; + } +} diff --git a/lib/CodeGen/CGCUDANV.cpp b/lib/CodeGen/CGCUDANV.cpp index fb11751fc91e..67d0ab7a82f7 100644 --- a/lib/CodeGen/CGCUDANV.cpp +++ b/lib/CodeGen/CGCUDANV.cpp @@ -20,7 +20,6 @@ #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" -#include <vector> using namespace clang; using namespace CodeGen; @@ -30,29 +29,66 @@ namespace { class CGNVCUDARuntime : public CGCUDARuntime { private: - llvm::Type *IntTy, *SizeTy; - llvm::PointerType *CharPtrTy, *VoidPtrTy; + llvm::Type *IntTy, *SizeTy, *VoidTy; + llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy; + + /// Convenience reference to LLVM Context + llvm::LLVMContext &Context; + /// Convenience reference to the current module + llvm::Module &TheModule; + /// Keeps track of kernel launch stubs emitted in this module + llvm::SmallVector<llvm::Function *, 16> EmittedKernels; + /// Keeps track of variables containing handles of GPU binaries. Populated by + /// ModuleCtorFunction() and used to create corresponding cleanup calls in + /// ModuleDtorFunction() + llvm::SmallVector<llvm::GlobalVariable *, 16> GpuBinaryHandles; llvm::Constant *getSetupArgumentFn() const; llvm::Constant *getLaunchFn() const; + /// Creates a function to register all kernel stubs generated in this module. + llvm::Function *makeRegisterKernelsFn(); + + /// Helper function that generates a constant string and returns a pointer to + /// the start of the string. The result of this function can be used anywhere + /// where the C code specifies const char*. + llvm::Constant *makeConstantString(const std::string &Str, + const std::string &Name = "", + unsigned Alignment = 0) { + llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0), + llvm::ConstantInt::get(SizeTy, 0)}; + auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); + } + + void emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args); + public: CGNVCUDARuntime(CodeGenModule &CGM); - void EmitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args) override; + void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override; + /// Creates module constructor function + llvm::Function *makeModuleCtorFunction() override; + /// Creates module destructor function + llvm::Function *makeModuleDtorFunction() override; }; } -CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) : CGCUDARuntime(CGM) { +CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM) + : CGCUDARuntime(CGM), Context(CGM.getLLVMContext()), + TheModule(CGM.getModule()) { CodeGen::CodeGenTypes &Types = CGM.getTypes(); ASTContext &Ctx = CGM.getContext(); IntTy = Types.ConvertType(Ctx.IntTy); SizeTy = Types.ConvertType(Ctx.getSizeType()); + VoidTy = llvm::Type::getVoidTy(Context); CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy)); VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy)); + VoidPtrPtrTy = VoidPtrTy->getPointerTo(); } llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const { @@ -68,14 +104,17 @@ llvm::Constant *CGNVCUDARuntime::getSetupArgumentFn() const { llvm::Constant *CGNVCUDARuntime::getLaunchFn() const { // cudaError_t cudaLaunch(char *) - std::vector<llvm::Type*> Params; - Params.push_back(CharPtrTy); - return CGM.CreateRuntimeFunction(llvm::FunctionType::get(IntTy, - Params, false), - "cudaLaunch"); + return CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch"); +} + +void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF, + FunctionArgList &Args) { + EmittedKernels.push_back(CGF.CurFn); + emitDeviceStubBody(CGF, Args); } -void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF, +void CGNVCUDARuntime::emitDeviceStubBody(CodeGenFunction &CGF, FunctionArgList &Args) { // Build the argument value list and the argument stack struct type. SmallVector<llvm::Value *, 16> ArgValues; @@ -87,8 +126,7 @@ void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF, assert(isa<llvm::PointerType>(V->getType()) && "Arg type not PointerType"); ArgTypes.push_back(cast<llvm::PointerType>(V->getType())->getElementType()); } - llvm::StructType *ArgStackTy = llvm::StructType::get( - CGF.getLLVMContext(), ArgTypes); + llvm::StructType *ArgStackTy = llvm::StructType::get(Context, ArgTypes); llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end"); @@ -120,6 +158,160 @@ void CGNVCUDARuntime::EmitDeviceStubBody(CodeGenFunction &CGF, CGF.EmitBlock(EndBlock); } +/// Creates internal function to register all kernel stubs generated in this +/// module with the CUDA runtime. +/// \code +/// void __cuda_register_kernels(void** GpuBinaryHandle) { +/// __cudaRegisterFunction(GpuBinaryHandle,Kernel0,...); +/// ... +/// __cudaRegisterFunction(GpuBinaryHandle,KernelM,...); +/// } +/// \endcode +llvm::Function *CGNVCUDARuntime::makeRegisterKernelsFn() { + llvm::Function *RegisterKernelsFunc = llvm::Function::Create( + llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), + llvm::GlobalValue::InternalLinkage, "__cuda_register_kernels", &TheModule); + llvm::BasicBlock *EntryBB = + llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc); + CGBuilderTy Builder(Context); + Builder.SetInsertPoint(EntryBB); + + // void __cudaRegisterFunction(void **, const char *, char *, const char *, + // int, uint3*, uint3*, dim3*, dim3*, int*) + std::vector<llvm::Type *> RegisterFuncParams = { + VoidPtrPtrTy, CharPtrTy, CharPtrTy, CharPtrTy, IntTy, + VoidPtrTy, VoidPtrTy, VoidPtrTy, VoidPtrTy, IntTy->getPointerTo()}; + llvm::Constant *RegisterFunc = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(IntTy, RegisterFuncParams, false), + "__cudaRegisterFunction"); + + // Extract GpuBinaryHandle passed as the first argument passed to + // __cuda_register_kernels() and generate __cudaRegisterFunction() call for + // each emitted kernel. + llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin(); + for (llvm::Function *Kernel : EmittedKernels) { + llvm::Constant *KernelName = makeConstantString(Kernel->getName()); + llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy); + llvm::Value *args[] = { + &GpuBinaryHandlePtr, Builder.CreateBitCast(Kernel, VoidPtrTy), + KernelName, KernelName, llvm::ConstantInt::get(IntTy, -1), NullPtr, + NullPtr, NullPtr, NullPtr, + llvm::ConstantPointerNull::get(IntTy->getPointerTo())}; + Builder.CreateCall(RegisterFunc, args); + } + + Builder.CreateRetVoid(); + return RegisterKernelsFunc; +} + +/// Creates a global constructor function for the module: +/// \code +/// void __cuda_module_ctor(void*) { +/// Handle0 = __cudaRegisterFatBinary(GpuBinaryBlob0); +/// __cuda_register_kernels(Handle0); +/// ... +/// HandleN = __cudaRegisterFatBinary(GpuBinaryBlobN); +/// __cuda_register_kernels(HandleN); +/// } +/// \endcode +llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { + // void __cuda_register_kernels(void* handle); + llvm::Function *RegisterKernelsFunc = makeRegisterKernelsFn(); + // void ** __cudaRegisterFatBinary(void *); + llvm::Constant *RegisterFatbinFunc = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false), + "__cudaRegisterFatBinary"); + // struct { int magic, int version, void * gpu_binary, void * dont_care }; + llvm::StructType *FatbinWrapperTy = + llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy, nullptr); + + llvm::Function *ModuleCtorFunc = llvm::Function::Create( + llvm::FunctionType::get(VoidTy, VoidPtrTy, false), + llvm::GlobalValue::InternalLinkage, "__cuda_module_ctor", &TheModule); + llvm::BasicBlock *CtorEntryBB = + llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc); + CGBuilderTy CtorBuilder(Context); + + CtorBuilder.SetInsertPoint(CtorEntryBB); + + // For each GPU binary, register it with the CUDA runtime and store returned + // handle in a global variable and save the handle in GpuBinaryHandles vector + // to be cleaned up in destructor on exit. Then associate all known kernels + // with the GPU binary handle so CUDA runtime can figure out what to call on + // the GPU side. + for (const std::string &GpuBinaryFileName : + CGM.getCodeGenOpts().CudaGpuBinaryFileNames) { + llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> GpuBinaryOrErr = + llvm::MemoryBuffer::getFileOrSTDIN(GpuBinaryFileName); + if (std::error_code EC = GpuBinaryOrErr.getError()) { + CGM.getDiags().Report(diag::err_cannot_open_file) << GpuBinaryFileName + << EC.message(); + continue; + } + + // Create initialized wrapper structure that points to the loaded GPU binary + llvm::Constant *Values[] = { + llvm::ConstantInt::get(IntTy, 0x466243b1), // Fatbin wrapper magic. + llvm::ConstantInt::get(IntTy, 1), // Fatbin version. + makeConstantString(GpuBinaryOrErr.get()->getBuffer(), "", 16), // Data. + llvm::ConstantPointerNull::get(VoidPtrTy)}; // Unused in fatbin v1. + llvm::GlobalVariable *FatbinWrapper = new llvm::GlobalVariable( + TheModule, FatbinWrapperTy, true, llvm::GlobalValue::InternalLinkage, + llvm::ConstantStruct::get(FatbinWrapperTy, Values), + "__cuda_fatbin_wrapper"); + + // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper); + llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall( + RegisterFatbinFunc, + CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy)); + llvm::GlobalVariable *GpuBinaryHandle = new llvm::GlobalVariable( + TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage, + llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle"); + CtorBuilder.CreateStore(RegisterFatbinCall, GpuBinaryHandle, false); + + // Call __cuda_register_kernels(GpuBinaryHandle); + CtorBuilder.CreateCall(RegisterKernelsFunc, RegisterFatbinCall); + + // Save GpuBinaryHandle so we can unregister it in destructor. + GpuBinaryHandles.push_back(GpuBinaryHandle); + } + + CtorBuilder.CreateRetVoid(); + return ModuleCtorFunc; +} + +/// Creates a global destructor function that unregisters all GPU code blobs +/// registered by constructor. +/// \code +/// void __cuda_module_dtor(void*) { +/// __cudaUnregisterFatBinary(Handle0); +/// ... +/// __cudaUnregisterFatBinary(HandleN); +/// } +/// \endcode +llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() { + // void __cudaUnregisterFatBinary(void ** handle); + llvm::Constant *UnregisterFatbinFunc = CGM.CreateRuntimeFunction( + llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false), + "__cudaUnregisterFatBinary"); + + llvm::Function *ModuleDtorFunc = llvm::Function::Create( + llvm::FunctionType::get(VoidTy, VoidPtrTy, false), + llvm::GlobalValue::InternalLinkage, "__cuda_module_dtor", &TheModule); + llvm::BasicBlock *DtorEntryBB = + llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc); + CGBuilderTy DtorBuilder(Context); + DtorBuilder.SetInsertPoint(DtorEntryBB); + + for (llvm::GlobalVariable *GpuBinaryHandle : GpuBinaryHandles) { + DtorBuilder.CreateCall(UnregisterFatbinFunc, + DtorBuilder.CreateLoad(GpuBinaryHandle, false)); + } + + DtorBuilder.CreateRetVoid(); + return ModuleDtorFunc; +} + CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) { return new CGNVCUDARuntime(CGM); } diff --git a/lib/CodeGen/CGCUDARuntime.h b/lib/CodeGen/CGCUDARuntime.h index 8c162fb05ab9..dcacf9703277 100644 --- a/lib/CodeGen/CGCUDARuntime.h +++ b/lib/CodeGen/CGCUDARuntime.h @@ -16,6 +16,10 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGCUDARUNTIME_H #define LLVM_CLANG_LIB_CODEGEN_CGCUDARUNTIME_H +namespace llvm { +class Function; +} + namespace clang { class CUDAKernelCallExpr; @@ -39,10 +43,17 @@ public: virtual RValue EmitCUDAKernelCallExpr(CodeGenFunction &CGF, const CUDAKernelCallExpr *E, ReturnValueSlot ReturnValue); - - virtual void EmitDeviceStubBody(CodeGenFunction &CGF, - FunctionArgList &Args) = 0; + /// Emits a kernel launch stub. + virtual void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) = 0; + + /// Constructs and returns a module initialization function or nullptr if it's + /// not needed. Must be called after all kernels have been emitted. + virtual llvm::Function *makeModuleCtorFunction() = 0; + + /// Returns a module cleanup function or nullptr if it's not needed. + /// Must be called after ModuleCtorFunction + virtual llvm::Function *makeModuleDtorFunction() = 0; }; /// Creates an instance of a CUDA runtime class. diff --git a/lib/CodeGen/CGCXX.cpp b/lib/CodeGen/CGCXX.cpp index 9f0e67e42176..29a199d6973a 100644 --- a/lib/CodeGen/CGCXX.cpp +++ b/lib/CodeGen/CGCXX.cpp @@ -182,8 +182,8 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl, return true; // Create the alias with no name. - auto *Alias = llvm::GlobalAlias::create(AliasType->getElementType(), 0, - Linkage, "", Aliasee, &getModule()); + auto *Alias = + llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, &getModule()); // Switch any previous uses to the alias. if (Entry) { @@ -231,8 +231,7 @@ llvm::GlobalValue *CodeGenModule::getAddrOfCXXStructor( if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) { GD = GlobalDecl(CD, toCXXCtorType(Type)); } else { - auto *DD = dyn_cast<CXXDestructorDecl>(MD); - GD = GlobalDecl(DD, toCXXDtorType(Type)); + GD = GlobalDecl(cast<CXXDestructorDecl>(MD), toCXXDtorType(Type)); } StringRef Name = getMangledName(GD); diff --git a/lib/CodeGen/CGCXXABI.cpp b/lib/CodeGen/CGCXXABI.cpp index d31331de6868..cb7e6dfca1c5 100644 --- a/lib/CodeGen/CGCXXABI.cpp +++ b/lib/CodeGen/CGCXXABI.cpp @@ -302,3 +302,10 @@ CGCXXABI::EmitCtorCompleteObjectHandler(CodeGenFunction &CGF, bool CGCXXABI::NeedsVTTParameter(GlobalDecl GD) { return false; } + +llvm::CallInst * +CGCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn) { + // Just call std::terminate and ignore the violating exception. + return CGF.EmitNounwindRuntimeCall(CGF.CGM.getTerminateFn()); +} diff --git a/lib/CodeGen/CGCXXABI.h b/lib/CodeGen/CGCXXABI.h index cc5c1b2e0ae6..2c7392188f78 100644 --- a/lib/CodeGen/CGCXXABI.h +++ b/lib/CodeGen/CGCXXABI.h @@ -22,6 +22,7 @@ namespace llvm { class Constant; class Type; class Value; +class CallInst; } namespace clang { @@ -214,8 +215,18 @@ public: llvm::Value *Ptr, QualType ElementType, const CXXDestructorDecl *Dtor) = 0; virtual void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) = 0; + virtual void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) = 0; + virtual llvm::GlobalVariable *getThrowInfo(QualType T) { return nullptr; } + + virtual void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) = 0; + + virtual llvm::CallInst * + emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn); virtual llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) = 0; + virtual llvm::Constant * + getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) = 0; virtual bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) = 0; diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp index 6403fa99aa7b..f2340530d66d 100644 --- a/lib/CodeGen/CGCall.cpp +++ b/lib/CodeGen/CGCall.cpp @@ -30,7 +30,9 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/Transforms/Utils/Local.h" +#include <sstream> using namespace clang; using namespace CodeGen; @@ -51,6 +53,8 @@ static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) { case CC_X86Pascal: return llvm::CallingConv::C; // TODO: Add support for __vectorcall to LLVM. case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall; + case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC; + case CC_SpirKernel: return llvm::CallingConv::SPIR_KERNEL; } } @@ -96,8 +100,7 @@ arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod, CanQual<FunctionProtoType> FTP) { RequiredArgs required = RequiredArgs::forPrototypePlus(FTP, prefix.size()); // FIXME: Kill copy. - for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i) - prefix.push_back(FTP->getParamType(i)); + prefix.append(FTP->param_type_begin(), FTP->param_type_end()); CanQualType resultType = FTP->getReturnType().getUnqualifiedType(); return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod, /*chainCall=*/false, prefix, @@ -133,9 +136,6 @@ static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) { if (PcsAttr *PCS = D->getAttr<PcsAttr>()) return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP); - if (D->hasAttr<PnaclCallAttr>()) - return CC_PnaclCall; - if (D->hasAttr<IntelOclBiccAttr>()) return CC_IntelOclBicc; @@ -208,8 +208,7 @@ CodeGenTypes::arrangeCXXStructorDeclaration(const CXXMethodDecl *MD, CanQual<FunctionProtoType> FTP = GetFormalType(MD); // Add the formal parameters. - for (unsigned i = 0, e = FTP->getNumParams(); i != e; ++i) - argTypes.push_back(FTP->getParamType(i)); + argTypes.append(FTP->param_type_begin(), FTP->param_type_end()); TheCXXABI.buildStructorSignature(MD, Type, argTypes); @@ -349,6 +348,26 @@ CodeGenTypes::arrangeMSMemberPointerThunk(const CXXMethodDecl *MD) { FTP->getExtInfo(), RequiredArgs(1)); } +const CGFunctionInfo & +CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT) { + assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure); + + CanQual<FunctionProtoType> FTP = GetFormalType(CD); + SmallVector<CanQualType, 2> ArgTys; + const CXXRecordDecl *RD = CD->getParent(); + ArgTys.push_back(GetThisType(Context, RD)); + if (CT == Ctor_CopyingClosure) + ArgTys.push_back(*FTP->param_type_begin()); + if (RD->getNumVBases() > 0) + ArgTys.push_back(Context.IntTy); + CallingConv CC = Context.getDefaultCallingConvention( + /*IsVariadic=*/false, /*IsCXXMethod=*/true); + return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true, + /*chainCall=*/false, ArgTys, + FunctionType::ExtInfo(CC), RequiredArgs::All); +} + /// Arrange a call as unto a free function, except possibly with an /// additional number of formal parameters considered required. static const CGFunctionInfo & @@ -716,7 +735,8 @@ void CodeGenFunction::ExpandTypeFromArgs( auto Exp = getTypeExpansion(Ty, getContext()); if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { for (int i = 0, n = CAExp->NumElts; i < n; i++) { - llvm::Value *EltAddr = Builder.CreateConstGEP2_32(LV.getAddress(), 0, i); + llvm::Value *EltAddr = + Builder.CreateConstGEP2_32(nullptr, LV.getAddress(), 0, i); LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy); ExpandTypeFromArgs(CAExp->EltTy, LV, AI); } @@ -738,10 +758,12 @@ void CodeGenFunction::ExpandTypeFromArgs( ExpandTypeFromArgs(FD->getType(), SubLV, AI); } } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) { - llvm::Value *RealAddr = Builder.CreateStructGEP(LV.getAddress(), 0, "real"); + llvm::Value *RealAddr = + Builder.CreateStructGEP(nullptr, LV.getAddress(), 0, "real"); EmitStoreThroughLValue(RValue::get(*AI++), MakeAddrLValue(RealAddr, CExp->EltTy)); - llvm::Value *ImagAddr = Builder.CreateStructGEP(LV.getAddress(), 1, "imag"); + llvm::Value *ImagAddr = + Builder.CreateStructGEP(nullptr, LV.getAddress(), 1, "imag"); EmitStoreThroughLValue(RValue::get(*AI++), MakeAddrLValue(ImagAddr, CExp->EltTy)); } else { @@ -757,7 +779,7 @@ void CodeGenFunction::ExpandTypeToArgs( if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { llvm::Value *Addr = RV.getAggregateAddr(); for (int i = 0, n = CAExp->NumElts; i < n; i++) { - llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, i); + llvm::Value *EltAddr = Builder.CreateConstGEP2_32(nullptr, Addr, 0, i); RValue EltRV = convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()); ExpandTypeToArgs(CAExp->EltTy, EltRV, IRFuncTy, IRCallArgs, IRCallArgPos); @@ -825,7 +847,7 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr, return SrcPtr; // GEP into the first element. - SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive"); + SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcSTy, SrcPtr, 0, 0, "coerce.dive"); // If the first element is a struct, recurse. llvm::Type *SrcTy = @@ -963,7 +985,7 @@ static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val, if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) { for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i); + llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(STy, DestPtr, 0, i); llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i); llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile); @@ -1403,7 +1425,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, FuncAttrs.addAttribute(llvm::Attribute::ReadOnly); FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); } - if (TargetDecl->hasAttr<MallocAttr>()) + if (TargetDecl->hasAttr<RestrictAttr>()) RetAttrs.addAttribute(llvm::Attribute::NoAlias); if (TargetDecl->hasAttr<ReturnsNonNullAttr>()) RetAttrs.addAttribute(llvm::Attribute::NonNull); @@ -1458,6 +1480,26 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, if (!CodeGenOpts.StackRealignment) FuncAttrs.addAttribute("no-realign-stack"); + + // Add target-cpu and target-features work if they differ from the defaults. + std::string &CPU = getTarget().getTargetOpts().CPU; + if (CPU != "") + FuncAttrs.addAttribute("target-cpu", CPU); + + // TODO: Features gets us the features on the command line including + // feature dependencies. For canonicalization purposes we might want to + // avoid putting features in the target-features set if we know it'll be one + // of the default features in the backend, e.g. corei7-avx and +avx or figure + // out non-explicit dependencies. + std::vector<std::string> &Features = getTarget().getTargetOpts().Features; + if (!Features.empty()) { + std::stringstream S; + std::copy(Features.begin(), Features.end(), + std::ostream_iterator<std::string>(S, ",")); + // The drop_back gets rid of the trailing space. + FuncAttrs.addAttribute("target-features", + StringRef(S.str()).drop_back(1)); + } } ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI); @@ -1546,8 +1588,12 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI, case ABIArgInfo::Extend: if (ParamType->isSignedIntegerOrEnumerationType()) Attrs.addAttribute(llvm::Attribute::SExt); - else if (ParamType->isUnsignedIntegerOrEnumerationType()) - Attrs.addAttribute(llvm::Attribute::ZExt); + else if (ParamType->isUnsignedIntegerOrEnumerationType()) { + if (getTypes().getABIInfo().shouldSignExtUnsignedType(ParamType)) + Attrs.addAttribute(llvm::Attribute::SExt); + else + Attrs.addAttribute(llvm::Attribute::ZExt); + } // FALL THROUGH case ABIArgInfo::Direct: if (ArgNo == 0 && FI.isChainCall()) @@ -1734,8 +1780,9 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, switch (ArgI.getKind()) { case ABIArgInfo::InAlloca: { assert(NumIRArgs == 0); - llvm::Value *V = Builder.CreateStructGEP( - ArgStruct, ArgI.getInAllocaFieldIndex(), Arg->getName()); + llvm::Value *V = + Builder.CreateStructGEP(FI.getArgStruct(), ArgStruct, + ArgI.getInAllocaFieldIndex(), Arg->getName()); ArgVals.push_back(ValueAndIsPtr(V, HavePointer)); break; } @@ -1770,8 +1817,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, ArgVals.push_back(ValueAndIsPtr(V, HavePointer)); } else { // Load scalar value from indirect argument. - CharUnits Alignment = getContext().getTypeAlignInChars(Ty); - V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty, + V = EmitLoadOfScalar(V, false, ArgI.getIndirectAlign(), Ty, Arg->getLocStart()); if (isPromoted) @@ -1901,7 +1947,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, // If the value is offset in memory, apply the offset now. if (unsigned Offs = ArgI.getDirectOffset()) { Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy()); - Ptr = Builder.CreateConstGEP1_32(Ptr, Offs); + Ptr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), Ptr, Offs); Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ArgI.getCoerceToType())); } @@ -1923,7 +1969,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { auto AI = FnArgs[FirstIRArg + i]; AI->setName(Arg->getName() + ".coerce" + Twine(i)); - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i); + llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, Ptr, 0, i); Builder.CreateStore(AI, EltPtr); } } else { @@ -1936,7 +1982,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { auto AI = FnArgs[FirstIRArg + i]; AI->setName(Arg->getName() + ".coerce" + Twine(i)); - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(TempV, 0, i); + llvm::Value *EltPtr = + Builder.CreateConstGEP2_32(ArgI.getCoerceToType(), TempV, 0, i); Builder.CreateStore(AI, EltPtr); } @@ -2173,7 +2220,29 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { if (!CGF.ReturnValue->hasOneUse()) { llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); if (IP->empty()) return nullptr; - llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(&IP->back()); + llvm::Instruction *I = &IP->back(); + + // Skip lifetime markers + for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(), + IE = IP->rend(); + II != IE; ++II) { + if (llvm::IntrinsicInst *Intrinsic = + dyn_cast<llvm::IntrinsicInst>(&*II)) { + if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) { + const llvm::Value *CastAddr = Intrinsic->getArgOperand(1); + ++II; + if (isa<llvm::BitCastInst>(&*II)) { + if (CastAddr == &*II) { + continue; + } + } + } + } + I = &*II; + break; + } + + llvm::StoreInst *store = dyn_cast<llvm::StoreInst>(I); if (!store) return nullptr; if (store->getPointerOperand() != CGF.ReturnValue) return nullptr; assert(!store->isAtomic() && !store->isVolatile()); // see below @@ -2231,8 +2300,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; llvm::Value *ArgStruct = EI; - llvm::Value *SRet = - Builder.CreateStructGEP(ArgStruct, RetAI.getInAllocaFieldIndex()); + llvm::Value *SRet = Builder.CreateStructGEP( + nullptr, ArgStruct, RetAI.getInAllocaFieldIndex()); RV = Builder.CreateLoad(SRet, "sret"); } break; @@ -2271,7 +2340,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, // If there is a dominating store to ReturnValue, we can elide // the load, zap the store, and usually zap the alloca. - if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(*this)) { + if (llvm::StoreInst *SI = + findDominatingStoreToReturnValue(*this)) { // Reuse the debug location from the store unless there is // cleanup code to be emitted between the store and return // instruction. @@ -2296,7 +2366,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, // If the value is offset in memory, apply the offset now. if (unsigned Offs = RetAI.getDirectOffset()) { V = Builder.CreateBitCast(V, Builder.getInt8PtrTy()); - V = Builder.CreateConstGEP1_32(V, Offs); + V = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), V, Offs); V = Builder.CreateBitCast(V, llvm::PointerType::getUnqual(RetAI.getCoerceToType())); } @@ -2342,8 +2412,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, Ret = Builder.CreateRetVoid(); } - if (!RetDbgLoc.isUnknown()) - Ret->setDebugLoc(RetDbgLoc); + if (RetDbgLoc) + Ret->setDebugLoc(std::move(RetDbgLoc)); } static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) { @@ -2626,7 +2696,7 @@ void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) { // Save the stack. llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave); - StackBase = CGF.Builder.CreateCall(F, "inalloca.save"); + StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save"); // Control gets really tied up in landing pads, so we have to spill the // stacksave to an alloca to avoid violating SSA form. @@ -2678,12 +2748,7 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, CallExpr::const_arg_iterator ArgBeg, CallExpr::const_arg_iterator ArgEnd, const FunctionDecl *CalleeDecl, - unsigned ParamsToSkip, - bool ForceColumnInfo) { - CGDebugInfo *DI = getDebugInfo(); - SourceLocation CallLoc; - if (DI) CallLoc = DI->getLocation(); - + unsigned ParamsToSkip) { // We *have* to evaluate arguments from right to left in the MS C++ ABI, // because arguments are destroyed left to right in the callee. if (CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { @@ -2704,8 +2769,6 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, EmitCallArg(Args, *Arg, ArgTypes[I]); emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(), CalleeDecl, ParamsToSkip + I); - // Restore the debug location. - if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo); } // Un-reverse the arguments we just evaluated so they match up with the LLVM @@ -2720,8 +2783,6 @@ void CodeGenFunction::EmitCallArgs(CallArgList &Args, EmitCallArg(Args, *Arg, ArgTypes[I]); emitNonNullArgCheck(*this, Args.back().RV, ArgTypes[I], Arg->getExprLoc(), CalleeDecl, ParamsToSkip + I); - // Restore the debug location. - if (DI) DI->EmitLocation(Builder, CallLoc, ForceColumnInfo); } } @@ -2744,8 +2805,22 @@ struct DestroyUnpassedArg : EHScopeStack::Cleanup { } +struct DisableDebugLocationUpdates { + CodeGenFunction &CGF; + bool disabledDebugInfo; + DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) { + if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo())) + CGF.disableDebugInfo(); + } + ~DisableDebugLocationUpdates() { + if (disabledDebugInfo) + CGF.enableDebugInfo(); + } +}; + void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, QualType type) { + DisableDebugLocationUpdates Dis(*this, E); if (const ObjCIndirectCopyRestoreExpr *CRE = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) { assert(getLangOpts().ObjCAutoRefCount); @@ -2900,7 +2975,6 @@ void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(llvm::Value *callee, call->setCallingConv(getRuntimeCC()); Builder.CreateUnreachable(); } - PGO.setCurrentRegionUnreachable(); } /// Emits a call or invoke instruction to the given nullary runtime @@ -2949,7 +3023,7 @@ CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee, if (CGM.getLangOpts().ObjCAutoRefCount) AddObjCARCExceptionMetadata(Inst); - return Inst; + return llvm::CallSite(Inst); } /// \brief Store a non-aggregate value to an address to initialize it. For @@ -2986,7 +3060,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If we're using inalloca, insert the allocation after the stack save. // FIXME: Do this earlier rather than hacking it in here! - llvm::Value *ArgMemory = nullptr; + llvm::AllocaInst *ArgMemory = nullptr; if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { llvm::Instruction *IP = CallArgs.getStackBase(); llvm::AllocaInst *AI; @@ -3015,7 +3089,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr; } else { llvm::Value *Addr = - Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); + Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, + RetAI.getInAllocaFieldIndex()); Builder.CreateStore(SRetPtr, Addr); } } @@ -3049,14 +3124,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, cast<llvm::Instruction>(RV.getAggregateAddr()); CGBuilderTy::InsertPoint IP = Builder.saveIP(); Builder.SetInsertPoint(Placeholder); - llvm::Value *Addr = Builder.CreateStructGEP( - ArgMemory, ArgInfo.getInAllocaFieldIndex()); + llvm::Value *Addr = + Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, + ArgInfo.getInAllocaFieldIndex()); Builder.restoreIP(IP); deferPlaceholderReplacement(Placeholder, Addr); } else { // Store the RValue into the argument struct. llvm::Value *Addr = - Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex()); + Builder.CreateStructGEP(ArgMemory->getAllocatedType(), ArgMemory, + ArgInfo.getInAllocaFieldIndex()); unsigned AS = Addr->getType()->getPointerAddressSpace(); llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS); // There are some cases where a trivial bitcast is not avoidable. The @@ -3100,8 +3177,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, : 0); if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) || (ArgInfo.getIndirectByVal() && TypeAlign.getQuantity() < Align && - llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align) || - (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) { + llvm::getOrEnforceKnownAlignment(Addr, Align, *TD) < Align) || + (ArgInfo.getIndirectByVal() && (RVAddrSpace != ArgAddrSpace))) { // Create an aligned temporary, and copy to it. llvm::AllocaInst *AI = CreateMemTemp(I->Ty); if (Align > AI->getAlignment()) @@ -3158,7 +3235,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the value is offset in memory, apply the offset now. if (unsigned Offs = ArgInfo.getDirectOffset()) { SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy()); - SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs); + SrcPtr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), SrcPtr, Offs); SrcPtr = Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(ArgInfo.getCoerceToType())); @@ -3190,7 +3267,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, assert(NumIRArgs == STy->getNumElements()); for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) { - llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i); + llvm::Value *EltPtr = Builder.CreateConstGEP2_32(STy, SrcPtr, 0, i); llvm::LoadInst *LI = Builder.CreateLoad(EltPtr); // We don't know what we're loading from. LI->setAlignment(1); @@ -3300,7 +3377,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::BasicBlock *InvokeDest = nullptr; if (!Attrs.hasAttribute(llvm::AttributeSet::FunctionIndex, - llvm::Attribute::NoUnwind)) + llvm::Attribute::NoUnwind) || + currentFunctionUsesSEHTry()) InvokeDest = getInvokeDest(); llvm::CallSite CS; @@ -3320,6 +3398,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex, llvm::Attribute::AlwaysInline); + // Disable inlining inside SEH __try blocks. + if (isSEHTryScope()) + Attrs = + Attrs.addAttribute(getLLVMContext(), llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoInline); + CS.setAttributes(Attrs); CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv)); @@ -3413,7 +3497,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, llvm::Value *StorePtr = DestPtr; if (unsigned Offs = RetAI.getDirectOffset()) { StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy()); - StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs); + StorePtr = + Builder.CreateConstGEP1_32(Builder.getInt8Ty(), StorePtr, Offs); StorePtr = Builder.CreateBitCast(StorePtr, llvm::PointerType::getUnqual(RetAI.getCoerceToType())); } diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp index d72eda95d7cf..cd75da276287 100644 --- a/lib/CodeGen/CGClass.cpp +++ b/lib/CodeGen/CGClass.cpp @@ -24,35 +24,36 @@ #include "clang/Basic/TargetBuiltins.h" #include "clang/CodeGen/CGFunctionInfo.h" #include "clang/Frontend/CodeGenOptions.h" +#include "llvm/IR/Intrinsics.h" using namespace clang; using namespace CodeGen; -static CharUnits -ComputeNonVirtualBaseClassOffset(ASTContext &Context, +static CharUnits +ComputeNonVirtualBaseClassOffset(ASTContext &Context, const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start, CastExpr::path_const_iterator End) { CharUnits Offset = CharUnits::Zero(); - + const CXXRecordDecl *RD = DerivedClass; - + for (CastExpr::path_const_iterator I = Start; I != End; ++I) { const CXXBaseSpecifier *Base = *I; assert(!Base->isVirtual() && "Should not see virtual bases here!"); // Get the layout. const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); - - const CXXRecordDecl *BaseDecl = + + const CXXRecordDecl *BaseDecl = cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl()); - + // Add the offset. Offset += Layout.getBaseClassOffset(BaseDecl); - + RD = BaseDecl; } - + return Offset; } @@ -62,15 +63,15 @@ CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl, CastExpr::path_const_iterator PathEnd) { assert(PathBegin != PathEnd && "Base path should not be empty!"); - CharUnits Offset = + CharUnits Offset = ComputeNonVirtualBaseClassOffset(getContext(), ClassDecl, PathBegin, PathEnd); if (Offset.isZero()) return nullptr; - llvm::Type *PtrDiffTy = + llvm::Type *PtrDiffTy = Types.ConvertType(getContext().getPointerDiffType()); - + return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity()); } @@ -127,7 +128,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ptr, } else { baseOffset = virtualOffset; } - + // Apply the base offset. ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8PtrTy); ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr"); @@ -149,7 +150,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( // *start* with a step down to the correct virtual base subobject, // and hence will not require any further steps. if ((*Start)->isVirtual()) { - VBase = + VBase = cast<CXXRecordDecl>((*Start)->getType()->getAs<RecordType>()->getDecl()); ++Start; } @@ -157,7 +158,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( // Compute the static offset of the ultimate destination within its // allocating subobject (the virtual base, if there is one, or else // the "complete" object that we see). - CharUnits NonVirtualOffset = + CharUnits NonVirtualOffset = ComputeNonVirtualBaseClassOffset(getContext(), VBase ? VBase : Derived, Start, PathEnd); @@ -172,7 +173,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( } // Get the base pointer type. - llvm::Type *BasePtrTy = + llvm::Type *BasePtrTy = ConvertType((PathEnd[-1])->getType())->getPointerTo(); QualType DerivedTy = getContext().getRecordType(Derived); @@ -197,7 +198,7 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( origBB = Builder.GetInsertBlock(); llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull"); endBB = createBasicBlock("cast.end"); - + llvm::Value *isNull = Builder.CreateIsNull(Value); Builder.CreateCondBr(isNull, endBB, notNullBB); EmitBlock(notNullBB); @@ -216,10 +217,10 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( } // Apply both offsets. - Value = ApplyNonVirtualAndVirtualOffset(*this, Value, + Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset, VirtualOffset); - + // Cast to the destination type. Value = Builder.CreateBitCast(Value, BasePtrTy); @@ -228,13 +229,13 @@ llvm::Value *CodeGenFunction::GetAddressOfBaseClass( llvm::BasicBlock *notNullBB = Builder.GetInsertBlock(); Builder.CreateBr(endBB); EmitBlock(endBB); - + llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result"); PHI->addIncoming(Value, notNullBB); PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB); Value = PHI; } - + return Value; } @@ -252,7 +253,7 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, llvm::Value *NonVirtualOffset = CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd); - + if (!NonVirtualOffset) { // No offset, we can just cast back. return Builder.CreateBitCast(Value, DerivedPtrTy); @@ -266,12 +267,12 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, CastNull = createBasicBlock("cast.null"); CastNotNull = createBasicBlock("cast.notnull"); CastEnd = createBasicBlock("cast.end"); - + llvm::Value *IsNull = Builder.CreateIsNull(Value); Builder.CreateCondBr(IsNull, CastNull, CastNotNull); EmitBlock(CastNotNull); } - + // Apply the offset. Value = Builder.CreateBitCast(Value, Int8PtrTy); Value = Builder.CreateGEP(Value, Builder.CreateNeg(NonVirtualOffset), @@ -285,14 +286,14 @@ CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value, EmitBlock(CastNull); Builder.CreateBr(CastEnd); EmitBlock(CastEnd); - + llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2); PHI->addIncoming(Value, CastNotNull); - PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), + PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull); Value = PHI; } - + return Value; } @@ -303,7 +304,7 @@ llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, // This constructor/destructor does not need a VTT parameter. return nullptr; } - + const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent(); const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent(); @@ -323,15 +324,15 @@ llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD, SubVTTIndex = 0; } else { const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); - CharUnits BaseOffset = ForVirtualBase ? - Layout.getVBaseClassOffset(Base) : + CharUnits BaseOffset = ForVirtualBase ? + Layout.getVBaseClassOffset(Base) : Layout.getBaseClassOffset(Base); - SubVTTIndex = + SubVTTIndex = CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset)); assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!"); } - + if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) { // A VTT parameter was passed to the constructor, use it. VTT = LoadCXXVTT(); @@ -358,7 +359,7 @@ namespace { cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent(); const CXXDestructorDecl *D = BaseClass->getDestructor(); - llvm::Value *Addr = + llvm::Value *Addr = CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThis(), DerivedClass, BaseClass, BaseIsVirtual); @@ -391,7 +392,7 @@ static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) { return Checker.UsesThis; } -static void EmitBaseInitializer(CodeGenFunction &CGF, +static void EmitBaseInitializer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl, CXXCtorInitializer *BaseInit, CXXCtorType CtorType) { @@ -399,7 +400,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, "Must have base initializer!"); llvm::Value *ThisPtr = CGF.LoadCXXThis(); - + const Type *BaseType = BaseInit->getBaseClass(); CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getAs<RecordType>()->getDecl()); @@ -418,7 +419,7 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, // We can pretend to be a complete class because it only matters for // virtual bases, and we only do virtual bases for complete ctors. - llvm::Value *V = + llvm::Value *V = CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl, BaseClassDecl, isBaseVirtual); @@ -430,8 +431,8 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, AggValueSlot::IsNotAliased); CGF.EmitAggExpr(BaseInit->getInit(), AggSlot); - - if (CGF.CGM.getLangOpts().Exceptions && + + if (CGF.CGM.getLangOpts().Exceptions && !BaseClassDecl->hasTrivialDestructor()) CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl, isBaseVirtual); @@ -490,17 +491,17 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, llvm::Value *IndexVar = CGF.GetAddrOfLocalVar(ArrayIndexes[Index]); assert(IndexVar && "Array index variable not loaded"); - + // Initialize this index variable to zero. llvm::Value* Zero = llvm::Constant::getNullValue( CGF.ConvertType(CGF.getContext().getSizeType())); CGF.Builder.CreateStore(Zero, IndexVar); - + // Start the loop with a block that tests the condition. llvm::BasicBlock *CondBlock = CGF.createBasicBlock("for.cond"); llvm::BasicBlock *AfterFor = CGF.createBasicBlock("for.end"); - + CGF.EmitBlock(CondBlock); llvm::BasicBlock *ForBody = CGF.createBasicBlock("for.body"); @@ -512,7 +513,7 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, llvm::ConstantInt::get(Counter->getType(), NumElements); llvm::Value *IsLess = CGF.Builder.CreateICmpULT(Counter, NumElementsPtr, "isless"); - + // If the condition is true, execute the body. CGF.Builder.CreateCondBr(IsLess, ForBody, AfterFor); @@ -539,6 +540,23 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF, CGF.EmitBlock(AfterFor, true); } +static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) { + auto *CD = dyn_cast<CXXConstructorDecl>(D); + if (!(CD && CD->isCopyOrMoveConstructor()) && + !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator()) + return false; + + // We can emit a memcpy for a trivial copy or move constructor/assignment. + if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) + return true; + + // We *must* emit a memcpy for a defaulted union copy or move op. + if (D->getParent()->isUnion() && D->isDefaulted()) + return true; + + return false; +} + static void EmitMemberInitializer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl, CXXCtorInitializer *MemberInit, @@ -548,7 +566,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, assert(MemberInit->isAnyMemberInitializer() && "Must have member initializer!"); assert(MemberInit->getInit() && "Must have initializer!"); - + // non-static data member initializers. FieldDecl *Field = MemberInit->getAnyMember(); QualType FieldType = Field->getType(); @@ -580,14 +598,14 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, QualType BaseElementTy = CGF.getContext().getBaseElementType(Array); CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit()); if (BaseElementTy.isPODType(CGF.getContext()) || - (CE && CE->getConstructor()->isTrivial())) { + (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) { unsigned SrcArgIndex = CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args); llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex])); LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy); LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field); - + // Copy the aggregate. CGF.EmitAggregateCopy(LHS.getAddress(), Src.getAddress(), FieldType, LHS.isVolatileQualified()); @@ -621,28 +639,28 @@ void CodeGenFunction::EmitInitializerForField( llvm::Value *ArrayIndexVar = nullptr; if (ArrayIndexes.size()) { llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - + // The LHS is a pointer to the first object we'll be constructing, as // a flat array. QualType BaseElementTy = getContext().getBaseElementType(FieldType); llvm::Type *BasePtr = ConvertType(BaseElementTy); BasePtr = llvm::PointerType::getUnqual(BasePtr); - llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), + llvm::Value *BaseAddrPtr = Builder.CreateBitCast(LHS.getAddress(), BasePtr); LHS = MakeAddrLValue(BaseAddrPtr, BaseElementTy); - + // Create an array index that will be used to walk over all of the // objects we're constructing. ArrayIndexVar = CreateTempAlloca(SizeTy, "object.index"); llvm::Value *Zero = llvm::Constant::getNullValue(SizeTy); Builder.CreateStore(Zero, ArrayIndexVar); - - + + // Emit the block variables for the array indices, if any. for (unsigned I = 0, N = ArrayIndexes.size(); I != N; ++I) EmitAutoVarDecl(*ArrayIndexes[I]); } - + EmitAggMemberInitializer(*this, LHS, Init, ArrayIndexVar, FieldType, ArrayIndexes, 0); } @@ -762,9 +780,9 @@ void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) { if (PoisonSize < AsanAlignment || !SSV[i].Size || (NextField % AsanAlignment) != 0) continue; - Builder.CreateCall2( - F, Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)), - Builder.getIntN(PtrSize, PoisonSize)); + Builder.CreateCall( + F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)), + Builder.getIntN(PtrSize, PoisonSize)}); } } @@ -796,8 +814,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) { if (IsTryBody) EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true); - RegionCounter Cnt = getPGORegionCounter(Body); - Cnt.beginRegion(Builder); + incrementProfileCounter(Body); RunCleanupsScope RunCleanups(*this); @@ -850,7 +867,7 @@ namespace { public: FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl, const VarDecl *SrcRec) - : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec), + : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec), RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)), FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0), LastFieldOffset(0), LastAddedFieldIndex(0) {} @@ -876,7 +893,7 @@ namespace { unsigned LastFieldSize = LastField->isBitField() ? LastField->getBitWidthValue(CGF.getContext()) : - CGF.getContext().getTypeSize(LastField->getType()); + CGF.getContext().getTypeSize(LastField->getType()); uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize - FirstByteOffset + CGF.getContext().getCharWidth() - 1; @@ -1021,8 +1038,8 @@ namespace { QualType FieldType = Field->getType(); CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit()); - // Bail out on non-POD, not-trivially-constructable members. - if (!(CE && CE->getConstructor()->isTrivial()) && + // Bail out on non-memcpyable, not-trivially-copyable members. + if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) && !(FieldType.isTriviallyCopyableType(CGF.getContext()) || FieldType->isReferenceType())) return false; @@ -1127,9 +1144,7 @@ namespace { return Field; } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) { CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl()); - if (!(MD && (MD->isCopyAssignmentOperator() || - MD->isMoveAssignmentOperator()) && - MD->isTrivial())) + if (!(MD && isMemcpyEquivalentSpecialMember(MD))) return nullptr; MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument()); if (!IOA) @@ -1189,7 +1204,7 @@ namespace { if (F) { addMemcpyableField(F); AggregatedStmts.push_back(S); - } else { + } else { emitAggregatedStmts(); CGF.EmitStmt(S); } @@ -1274,7 +1289,7 @@ static bool FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field); static bool -HasTrivialDestructorBody(ASTContext &Context, +HasTrivialDestructorBody(ASTContext &Context, const CXXRecordDecl *BaseClassDecl, const CXXRecordDecl *MostDerivedClassDecl) { @@ -1309,7 +1324,7 @@ HasTrivialDestructorBody(ASTContext &Context, cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl()); if (!HasTrivialDestructorBody(Context, VirtualBase, MostDerivedClassDecl)) - return false; + return false; } } @@ -1325,7 +1340,7 @@ FieldHasTrivialDestructorBody(ASTContext &Context, const RecordType *RT = FieldBaseElementType->getAs<RecordType>(); if (!RT) return true; - + CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl()); return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl); } @@ -1351,6 +1366,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl()); CXXDtorType DtorType = CurGD.getDtorType(); + Stmt *Body = Dtor->getBody(); + if (Body) + incrementProfileCounter(Body); + // The call to operator delete in a deleting destructor happens // outside of the function-try-block, which means it's always // possible to delegate the destructor body to the complete @@ -1363,8 +1382,6 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { return; } - Stmt *Body = Dtor->getBody(); - // If the body is a function-try-block, enter the try before // anything else. bool isTryBody = (Body && isa<CXXTryStmt>(Body)); @@ -1374,11 +1391,11 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { // Enter the epilogue cleanups. RunCleanupsScope DtorEpilogue(*this); - + // If this is the complete variant, just invoke the base variant; // the epilogue will destruct the virtual bases. But we can't do // this optimization if the body is a function-try-block, because - // we'd introduce *two* handler blocks. In the Microsoft ABI, we + // we'd introduce *two* handler blocks. In the Microsoft ABI, we // always delegate because we might not have a definition in this TU. switch (DtorType) { case Dtor_Comdat: @@ -1399,13 +1416,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) { break; } // Fallthrough: act like we're in the base variant. - + case Dtor_Base: assert(Body); - RegionCounter Cnt = getPGORegionCounter(Body); - Cnt.beginRegion(Builder); - // Enter the cleanup scopes for fields and non-virtual bases. EnterDtorCleanups(Dtor, Dtor_Base); @@ -1447,7 +1461,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) AssignmentMemcpyizer AM(*this, AssignOp, Args); for (auto *I : RootCS->body()) - AM.emitAssignment(I); + AM.emitAssignment(I); AM.finish(); } @@ -1508,7 +1522,7 @@ namespace { LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy); LValue LV = CGF.EmitLValueForField(ThisLV, field); assert(LV.isSimple()); - + CGF.emitDestroy(LV.getAddress(), field->getType(), destroyer, flags.isForNormalCleanup() && useEHCleanupForArray); } @@ -1526,7 +1540,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, // The deleting-destructor phase just needs to call the appropriate // operator delete that Sema picked up. if (DtorType == Dtor_Deleting) { - assert(DD->getOperatorDelete() && + assert(DD->getOperatorDelete() && "operator delete missing - EnterDtorCleanups"); if (CXXStructorImplicitParamValue) { // If there is an implicit param to the deleting dtor, it's a boolean @@ -1553,7 +1567,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, for (const auto &Base : ClassDecl->vbases()) { CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl()); - + // Ignore trivial destructors. if (BaseClassDecl->hasTrivialDestructor()) continue; @@ -1567,15 +1581,15 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD, } assert(DtorType == Dtor_Base); - + // Destroy non-virtual bases. for (const auto &Base : ClassDecl->bases()) { // Ignore virtual bases. if (Base.isVirtual()) continue; - + CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl(); - + // Ignore trivial destructors. if (BaseClassDecl->hasTrivialDestructor()) continue; @@ -1656,7 +1670,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB); EmitBlock(loopBB); } - + // Find the end of the array. llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements, "arrayctor.end"); @@ -1676,15 +1690,15 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor, // Zero initialize the storage, if requested. if (zeroInitialize) EmitNullInitialization(cur, type); - - // C++ [class.temporary]p4: + + // C++ [class.temporary]p4: // There are two contexts in which temporaries are destroyed at a different // point than the end of the full-expression. The first context is when a - // default constructor is called to initialize an element of an array. - // If the constructor has one or more default arguments, the destruction of - // every temporary created in a default argument expression is sequenced + // default constructor is called to initialize an element of an array. + // If the constructor has one or more default arguments, the destruction of + // every temporary created in a default argument expression is sequenced // before the construction of the next array element, if any. - + { RunCleanupsScope Scope(*this); @@ -1733,33 +1747,32 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, bool ForVirtualBase, bool Delegating, llvm::Value *This, const CXXConstructExpr *E) { - // If this is a trivial constructor, just emit what's needed. - if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) { - if (E->getNumArgs() == 0) { - // Trivial default constructor, no codegen required. - assert(D->isDefaultConstructor() && - "trivial 0-arg ctor not a default ctor"); - return; - } + // C++11 [class.mfct.non-static]p2: + // If a non-static member function of a class X is called for an object that + // is not of type X, or of a type derived from X, the behavior is undefined. + // FIXME: Provide a source location here. + EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This, + getContext().getRecordType(D->getParent())); + + if (D->isTrivial() && D->isDefaultConstructor()) { + assert(E->getNumArgs() == 0 && "trivial default ctor with args"); + return; + } + // If this is a trivial constructor, just emit what's needed. If this is a + // union copy constructor, we must emit a memcpy, because the AST does not + // model that copy. + if (isMemcpyEquivalentSpecialMember(D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); - assert(D->isCopyOrMoveConstructor() && - "trivial 1-arg ctor not a copy/move ctor"); const Expr *Arg = E->getArg(0); - QualType Ty = Arg->getType(); + QualType SrcTy = Arg->getType(); llvm::Value *Src = EmitLValue(Arg).getAddress(); - EmitAggregateCopy(This, Src, Ty); + QualType DestTy = getContext().getTypeDeclType(D->getParent()); + EmitAggregateCopyCtor(This, Src, DestTy, SrcTy); return; } - // C++11 [class.mfct.non-static]p2: - // If a non-static member function of a class X is called for an object that - // is not of type X, or of a type derived from X, the behavior is undefined. - // FIXME: Provide a source location here. - EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, SourceLocation(), This, - getContext().getRecordType(D->getParent())); - CallArgList Args; // Push the this ptr. @@ -1784,25 +1797,26 @@ void CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, llvm::Value *This, llvm::Value *Src, const CXXConstructExpr *E) { - if (D->isTrivial() && - !D->getParent()->mayInsertExtraPadding()) { + if (isMemcpyEquivalentSpecialMember(D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); assert(D->isCopyOrMoveConstructor() && "trivial 1-arg ctor not a copy/move ctor"); - EmitAggregateCopy(This, Src, E->arg_begin()->getType()); + EmitAggregateCopyCtor(This, Src, + getContext().getTypeDeclType(D->getParent()), + E->arg_begin()->getType()); return; } llvm::Value *Callee = CGM.getAddrOfCXXStructor(D, StructorType::Complete); assert(D->isInstance() && "Trying to emit a member call expr on a static method!"); - + const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>(); - + CallArgList Args; - + // Push the this ptr. Args.add(RValue::get(This), D->getThisType(getContext())); - + // Push the src ptr. QualType QT = *(FPT->param_type_begin()); llvm::Type *t = CGM.getTypes().ConvertType(QT); @@ -1945,10 +1959,18 @@ void CodeGenFunction::PushDestructorCleanup(QualType T, llvm::Value *Addr) { } void -CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, +CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, const CXXRecordDecl *NearestVBase, CharUnits OffsetFromNearestVBase, const CXXRecordDecl *VTableClass) { + const CXXRecordDecl *RD = Base.getBase(); + + // Don't initialize the vtable pointer if the class is marked with the + // 'novtable' attribute. + if ((RD == VTableClass || RD == NearestVBase) && + VTableClass->hasAttr<MSNoVTableAttr>()) + return; + // Compute the address point. bool NeedsVirtualOffset; llvm::Value *VTableAddressPoint = @@ -1960,7 +1982,7 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, // Compute where to store the address point. llvm::Value *VirtualOffset = nullptr; CharUnits NonVirtualOffset = CharUnits::Zero(); - + if (NeedsVirtualOffset) { // We need to use the virtual base offset offset because the virtual base // might have a different offset in the most derived class. @@ -1973,12 +1995,12 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, // We can just use the base offset in the complete class. NonVirtualOffset = Base.getBaseOffset(); } - + // Apply the offsets. llvm::Value *VTableField = LoadCXXThis(); - + if (!NonVirtualOffset.isZero() || VirtualOffset) - VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField, + VTableField = ApplyNonVirtualAndVirtualOffset(*this, VTableField, NonVirtualOffset, VirtualOffset); @@ -1995,7 +2017,7 @@ CodeGenFunction::InitializeVTablePointer(BaseSubobject Base, } void -CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, +CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase, CharUnits OffsetFromNearestVBase, bool BaseIsNonVirtualPrimaryBase, @@ -2008,7 +2030,7 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, InitializeVTablePointer(Base, NearestVBase, OffsetFromNearestVBase, VTableClass); } - + const CXXRecordDecl *RD = Base.getBase(); // Traverse bases. @@ -2029,7 +2051,7 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, if (!VBases.insert(BaseDecl).second) continue; - const ASTRecordLayout &Layout = + const ASTRecordLayout &Layout = getContext().getASTRecordLayout(VTableClass); BaseOffset = Layout.getVBaseClassOffset(BaseDecl); @@ -2039,15 +2061,15 @@ CodeGenFunction::InitializeVTablePointers(BaseSubobject Base, const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl); - BaseOffsetFromNearestVBase = + BaseOffsetFromNearestVBase = OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl); BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl; } - - InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset), + + InitializeVTablePointers(BaseSubobject(BaseDecl, BaseOffset), I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase, - BaseDeclIsNonVirtualPrimaryBase, + BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases); } } @@ -2059,7 +2081,7 @@ void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) { // Initialize the vtable pointers for this class and all of its bases. VisitedVirtualBasesSetTy VBases; - InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()), + InitializeVTablePointers(BaseSubobject(RD, CharUnits::Zero()), /*NearestVBase=*/nullptr, /*OffsetFromNearestVBase=*/CharUnits::Zero(), /*BaseIsNonVirtualPrimaryBase=*/false, RD, VBases); @@ -2076,6 +2098,128 @@ llvm::Value *CodeGenFunction::GetVTablePtr(llvm::Value *This, return VTable; } +// If a class has a single non-virtual base and does not introduce or override +// virtual member functions or fields, it will have the same layout as its base. +// This function returns the least derived such class. +// +// Casting an instance of a base class to such a derived class is technically +// undefined behavior, but it is a relatively common hack for introducing member +// functions on class instances with specific properties (e.g. llvm::Operator) +// that works under most compilers and should not have security implications, so +// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict. +static const CXXRecordDecl * +LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) { + if (!RD->field_empty()) + return RD; + + if (RD->getNumVBases() != 0) + return RD; + + if (RD->getNumBases() != 1) + return RD; + + for (const CXXMethodDecl *MD : RD->methods()) { + if (MD->isVirtual()) { + // Virtual member functions are only ok if they are implicit destructors + // because the implicit destructor will have the same semantics as the + // base class's destructor if no fields are added. + if (isa<CXXDestructorDecl>(MD) && MD->isImplicit()) + continue; + return RD; + } + } + + return LeastDerivedClassWithSameLayout( + RD->bases_begin()->getType()->getAsCXXRecordDecl()); +} + +void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, + llvm::Value *VTable) { + const CXXRecordDecl *ClassDecl = MD->getParent(); + if (!SanOpts.has(SanitizerKind::CFICastStrict)) + ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl); + + EmitVTablePtrCheck(ClassDecl, VTable); +} + +void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, + llvm::Value *Derived, + bool MayBeNull) { + if (!getLangOpts().CPlusPlus) + return; + + auto *ClassTy = T->getAs<RecordType>(); + if (!ClassTy) + return; + + const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl()); + + if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass()) + return; + + SmallString<64> MangledName; + llvm::raw_svector_ostream Out(MangledName); + CGM.getCXXABI().getMangleContext().mangleCXXRTTI(T.getUnqualifiedType(), + Out); + + // Blacklist based on the mangled type. + if (CGM.getContext().getSanitizerBlacklist().isBlacklistedType(Out.str())) + return; + + if (!SanOpts.has(SanitizerKind::CFICastStrict)) + ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl); + + llvm::BasicBlock *ContBlock = 0; + + if (MayBeNull) { + llvm::Value *DerivedNotNull = + Builder.CreateIsNotNull(Derived, "cast.nonnull"); + + llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check"); + ContBlock = createBasicBlock("cast.cont"); + + Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock); + + EmitBlock(CheckBlock); + } + + llvm::Value *VTable = GetVTablePtr(Derived, Int8PtrTy); + EmitVTablePtrCheck(ClassDecl, VTable); + + if (MayBeNull) { + Builder.CreateBr(ContBlock); + EmitBlock(ContBlock); + } +} + +void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD, + llvm::Value *VTable) { + // FIXME: Add blacklisting scheme. + if (RD->isInStdNamespace()) + return; + + std::string OutName; + llvm::raw_string_ostream Out(OutName); + CGM.getCXXABI().getMangleContext().mangleCXXVTableBitSet(RD, Out); + + llvm::Value *BitSetName = llvm::MetadataAsValue::get( + getLLVMContext(), llvm::MDString::get(getLLVMContext(), Out.str())); + + llvm::Value *BitSetTest = Builder.CreateCall( + CGM.getIntrinsic(llvm::Intrinsic::bitset_test), + {Builder.CreateBitCast(VTable, CGM.Int8PtrTy), BitSetName}); + + llvm::BasicBlock *ContBlock = createBasicBlock("vtable.check.cont"); + llvm::BasicBlock *TrapBlock = createBasicBlock("vtable.check.trap"); + + Builder.CreateCondBr(BitSetTest, ContBlock, TrapBlock); + + EmitBlock(TrapBlock); + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap), {}); + Builder.CreateUnreachable(); + + EmitBlock(ContBlock); +} // FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do // quite what we want. @@ -2140,7 +2284,7 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, // This is a record decl. We know the type and can devirtualize it. return VD->getType()->isRecordType(); } - + return false; } @@ -2154,14 +2298,14 @@ CodeGenFunction::CanDevirtualizeMemberFunctionCall(const Expr *Base, // We can always devirtualize calls on temporary object expressions. if (isa<CXXConstructExpr>(Base)) return true; - + // And calls on bound temporaries. if (isa<CXXBindTemporaryExpr>(Base)) return true; - + // Check if this is a call expr that returns a record type. if (const CallExpr *CE = dyn_cast<CallExpr>(Base)) - return CE->getCallReturnType()->isRecordType(); + return CE->getCallReturnType(getContext())->isRecordType(); // We can't devirtualize the call. return false; @@ -2190,7 +2334,7 @@ void CodeGenFunction::EmitForwardingCallToLambda( // We don't need to separately arrange the call arguments because // the call can't be variadic anyway --- it's impossible to forward // variadic arguments. - + // Now emit our call. RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs, callOperator); @@ -2218,7 +2362,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() { for (auto param : BD->params()) EmitDelegateCallArg(CallArgs, param, param->getLocStart()); - assert(!Lambda->isGenericLambda() && + assert(!Lambda->isGenericLambda() && "generic lambda interconversion to block not implemented"); EmitForwardingCallToLambda(Lambda->getLambdaCallOperator(), CallArgs); } @@ -2256,7 +2400,7 @@ void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) { const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs(); FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate(); void *InsertPos = nullptr; - FunctionDecl *CorrespondingCallOpSpecialization = + FunctionDecl *CorrespondingCallOpSpecialization = CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos); assert(CorrespondingCallOpSpecialization); CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization); diff --git a/lib/CodeGen/CGCleanup.cpp b/lib/CodeGen/CGCleanup.cpp index 18ed3e543d20..d97e40554ef2 100644 --- a/lib/CodeGen/CGCleanup.cpp +++ b/lib/CodeGen/CGCleanup.cpp @@ -52,8 +52,10 @@ DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) { llvm::StructType::get(V.first->getType(), V.second->getType(), (void*) nullptr); llvm::Value *addr = CGF.CreateTempAlloca(ComplexTy, "saved-complex"); - CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0)); - CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1)); + CGF.Builder.CreateStore(V.first, + CGF.Builder.CreateStructGEP(ComplexTy, addr, 0)); + CGF.Builder.CreateStore(V.second, + CGF.Builder.CreateStructGEP(ComplexTy, addr, 1)); return saved_type(addr, ComplexAddress); } @@ -82,9 +84,9 @@ RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) { return RValue::getAggregate(CGF.Builder.CreateLoad(Value)); case ComplexAddress: { llvm::Value *real = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(Value, 0)); + CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 0)); llvm::Value *imag = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(Value, 1)); + CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(nullptr, Value, 1)); return RValue::getComplex(real, imag); } } @@ -123,6 +125,17 @@ char *EHScopeStack::allocate(size_t Size) { return StartOfData; } +bool EHScopeStack::containsOnlyLifetimeMarkers( + EHScopeStack::stable_iterator Old) const { + for (EHScopeStack::iterator it = begin(); stabilize(it) != Old; it++) { + EHCleanupScope *cleanup = dyn_cast<EHCleanupScope>(&*it); + if (!cleanup || !cleanup->isLifetimeMarker()) + return false; + } + + return true; +} + EHScopeStack::stable_iterator EHScopeStack::getInnermostActiveNormalCleanup() const { for (stable_iterator si = getInnermostNormalCleanup(), se = stable_end(); @@ -469,8 +482,14 @@ static void EmitCleanup(CodeGenFunction &CGF, EHScopeStack::Cleanup *Fn, EHScopeStack::Cleanup::Flags flags, llvm::Value *ActiveFlag) { - // EH cleanups always occur within a terminate scope. - if (flags.isForEHCleanup()) CGF.EHStack.pushTerminate(); + // Itanium EH cleanups occur within a terminate scope. Microsoft SEH doesn't + // have this behavior, and the Microsoft C++ runtime will call terminate for + // us if the cleanup throws. + bool PushedTerminate = false; + if (flags.isForEHCleanup() && !CGF.getTarget().getCXXABI().isMicrosoft()) { + CGF.EHStack.pushTerminate(); + PushedTerminate = true; + } // If there's an active flag, load it and skip the cleanup if it's // false. @@ -493,7 +512,8 @@ static void EmitCleanup(CodeGenFunction &CGF, CGF.EmitBlock(ContBB); // Leave the terminate scope. - if (flags.isForEHCleanup()) CGF.EHStack.popTerminate(); + if (PushedTerminate) + CGF.EHStack.popTerminate(); } static void ForwardPrebranchedFallthrough(llvm::BasicBlock *Exit, @@ -739,7 +759,15 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { Scope.getNumBranchAfters() == 1) { assert(!BranchThroughDest || !IsActive); - // TODO: clean up the possibly dead stores to the cleanup dest slot. + // Clean up the possibly dead store to the cleanup dest slot. + llvm::Instruction *NormalCleanupDestSlot = + cast<llvm::Instruction>(getNormalCleanupDestSlot()); + if (NormalCleanupDestSlot->hasOneUse()) { + NormalCleanupDestSlot->user_back()->eraseFromParent(); + NormalCleanupDestSlot->eraseFromParent(); + NormalCleanupDest = nullptr; + } + llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0); InstsToAppend.push_back(llvm::BranchInst::Create(BranchAfter)); @@ -861,8 +889,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) { // Emit the EH cleanup if required. if (RequiresEHCleanup) { - ApplyDebugLocation AutoRestoreLocation(*this, CurEHLocation); - CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP(); EmitBlock(EHEntry); diff --git a/lib/CodeGen/CGCleanup.h b/lib/CodeGen/CGCleanup.h index dd156c696ad3..81c64123dfdb 100644 --- a/lib/CodeGen/CGCleanup.h +++ b/lib/CodeGen/CGCleanup.h @@ -62,6 +62,9 @@ protected: /// Whether this cleanup is currently active. unsigned IsActive : 1; + /// Whether this cleanup is a lifetime marker + unsigned IsLifetimeMarker : 1; + /// Whether the normal cleanup should test the activation flag. unsigned TestFlagInNormalCleanup : 1; @@ -75,7 +78,7 @@ protected: /// The number of fixups required by enclosing scopes (not including /// this one). If this is the top cleanup scope, all the fixups /// from this index onwards belong to this scope. - unsigned FixupDepth : 32 - 17 - NumCommonBits; // currently 13 + unsigned FixupDepth : 32 - 18 - NumCommonBits; // currently 13 }; class FilterBitFields { @@ -272,6 +275,7 @@ public: CleanupBits.IsNormalCleanup = isNormal; CleanupBits.IsEHCleanup = isEH; CleanupBits.IsActive = isActive; + CleanupBits.IsLifetimeMarker = false; CleanupBits.TestFlagInNormalCleanup = false; CleanupBits.TestFlagInEHCleanup = false; CleanupBits.CleanupSize = cleanupSize; @@ -284,19 +288,20 @@ public: delete ExtInfo; } // Objects of EHCleanupScope are not destructed. Use Destroy(). - ~EHCleanupScope() LLVM_DELETED_FUNCTION; + ~EHCleanupScope() = delete; bool isNormalCleanup() const { return CleanupBits.IsNormalCleanup; } llvm::BasicBlock *getNormalBlock() const { return NormalBlock; } void setNormalBlock(llvm::BasicBlock *BB) { NormalBlock = BB; } bool isEHCleanup() const { return CleanupBits.IsEHCleanup; } - llvm::BasicBlock *getEHBlock() const { return getCachedEHDispatchBlock(); } - void setEHBlock(llvm::BasicBlock *BB) { setCachedEHDispatchBlock(BB); } bool isActive() const { return CleanupBits.IsActive; } void setActive(bool A) { CleanupBits.IsActive = A; } + bool isLifetimeMarker() const { return CleanupBits.IsLifetimeMarker; } + void setLifetimeMarker() { CleanupBits.IsLifetimeMarker = true; } + llvm::AllocaInst *getActiveFlag() const { return ActiveFlag; } void setActiveFlag(llvm::AllocaInst *Var) { ActiveFlag = Var; } diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp index 978e1bb5b81f..48458dbd6003 100644 --- a/lib/CodeGen/CGDebugInfo.cpp +++ b/lib/CodeGen/CGDebugInfo.cpp @@ -52,35 +52,48 @@ CGDebugInfo::~CGDebugInfo() { "Region stack mismatch, stack not empty!"); } -ArtificialLocation::ArtificialLocation(CodeGenFunction &CGF) - : ApplyDebugLocation(CGF) { - if (auto *DI = CGF.getDebugInfo()) { - // Construct a location that has a valid scope, but no line info. - assert(!DI->LexicalBlockStack.empty()); - llvm::DIDescriptor Scope(DI->LexicalBlockStack.back()); - CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc::get(0, 0, Scope)); - } +ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, + SourceLocation TemporaryLocation) + : CGF(CGF) { + init(TemporaryLocation); } ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, - SourceLocation TemporaryLocation, - bool ForceColumnInfo) + bool DefaultToEmpty, + SourceLocation TemporaryLocation) : CGF(CGF) { + init(TemporaryLocation, DefaultToEmpty); +} + +void ApplyDebugLocation::init(SourceLocation TemporaryLocation, + bool DefaultToEmpty) { if (auto *DI = CGF.getDebugInfo()) { OriginalLocation = CGF.Builder.getCurrentDebugLocation(); - if (TemporaryLocation.isInvalid()) - CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc()); - else - DI->EmitLocation(CGF.Builder, TemporaryLocation, ForceColumnInfo); + if (TemporaryLocation.isInvalid()) { + if (DefaultToEmpty) + CGF.Builder.SetCurrentDebugLocation(llvm::DebugLoc()); + else { + // Construct a location that has a valid scope, but no line info. + assert(!DI->LexicalBlockStack.empty()); + CGF.Builder.SetCurrentDebugLocation( + llvm::DebugLoc::get(0, 0, DI->LexicalBlockStack.back())); + } + } else + DI->EmitLocation(CGF.Builder, TemporaryLocation); } } +ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E) + : CGF(CGF) { + init(E->getExprLoc()); +} + ApplyDebugLocation::ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc) : CGF(CGF) { if (CGF.getDebugInfo()) { OriginalLocation = CGF.Builder.getCurrentDebugLocation(); - if (!Loc.isUnknown()) - CGF.Builder.SetCurrentDebugLocation(Loc); + if (Loc) + CGF.Builder.SetCurrentDebugLocation(std::move(Loc)); } } @@ -88,7 +101,7 @@ ApplyDebugLocation::~ApplyDebugLocation() { // Query CGF so the location isn't overwritten when location updates are // temporarily disabled (for C++ default function arguments) if (CGF.getDebugInfo()) - CGF.Builder.SetCurrentDebugLocation(OriginalLocation); + CGF.Builder.SetCurrentDebugLocation(std::move(OriginalLocation)); } /// ArtificialLocation - An RAII object that temporarily switches to @@ -107,37 +120,33 @@ void CGDebugInfo::setLocation(SourceLocation Loc) { return; SourceManager &SM = CGM.getContext().getSourceManager(); - llvm::DIScope Scope(LexicalBlockStack.back()); + auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back()); PresumedLoc PCLoc = SM.getPresumedLoc(CurLoc); - if (PCLoc.isInvalid() || Scope.getFilename() == PCLoc.getFilename()) + if (PCLoc.isInvalid() || Scope->getFilename() == PCLoc.getFilename()) return; - if (Scope.isLexicalBlockFile()) { - llvm::DILexicalBlockFile LBF = llvm::DILexicalBlockFile(Scope); - llvm::DIDescriptor D = DBuilder.createLexicalBlockFile( - LBF.getScope(), getOrCreateFile(CurLoc)); - llvm::MDNode *N = D; + if (auto *LBF = dyn_cast<llvm::DILexicalBlockFile>(Scope)) { LexicalBlockStack.pop_back(); - LexicalBlockStack.emplace_back(N); - } else if (Scope.isLexicalBlock() || Scope.isSubprogram()) { - llvm::DIDescriptor D = - DBuilder.createLexicalBlockFile(Scope, getOrCreateFile(CurLoc)); - llvm::MDNode *N = D; + LexicalBlockStack.emplace_back(DBuilder.createLexicalBlockFile( + LBF->getScope(), getOrCreateFile(CurLoc))); + } else if (isa<llvm::DILexicalBlock>(Scope) || + isa<llvm::DISubprogram>(Scope)) { LexicalBlockStack.pop_back(); - LexicalBlockStack.emplace_back(N); + LexicalBlockStack.emplace_back( + DBuilder.createLexicalBlockFile(Scope, getOrCreateFile(CurLoc))); } } /// getContextDescriptor - Get context info for the decl. -llvm::DIScope CGDebugInfo::getContextDescriptor(const Decl *Context) { +llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context) { if (!Context) return TheCU; auto I = RegionMap.find(Context); if (I != RegionMap.end()) { llvm::Metadata *V = I->second; - return llvm::DIScope(dyn_cast_or_null<llvm::MDNode>(V)); + return dyn_cast_or_null<llvm::DIScope>(V); } // Check namespace. @@ -234,17 +243,17 @@ StringRef CGDebugInfo::getClassName(const RecordDecl *RD) { } /// getOrCreateFile - Get the file debug info descriptor for the input location. -llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) { +llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) { if (!Loc.isValid()) // If Location is not valid then use main input file. - return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory()); + return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); SourceManager &SM = CGM.getContext().getSourceManager(); PresumedLoc PLoc = SM.getPresumedLoc(Loc); if (PLoc.isInvalid() || StringRef(PLoc.getFilename()).empty()) // If the location is not valid then use main input file. - return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory()); + return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); // Cache the results. const char *fname = PLoc.getFilename(); @@ -253,18 +262,19 @@ llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) { if (it != DIFileCache.end()) { // Verify that the information still exists. if (llvm::Metadata *V = it->second) - return llvm::DIFile(cast<llvm::MDNode>(V)); + return cast<llvm::DIFile>(V); } - llvm::DIFile F = DBuilder.createFile(PLoc.getFilename(), getCurrentDirname()); + llvm::DIFile *F = + DBuilder.createFile(PLoc.getFilename(), getCurrentDirname()); DIFileCache[fname].reset(F); return F; } /// getOrCreateMainFile - Get the file info for main compile unit. -llvm::DIFile CGDebugInfo::getOrCreateMainFile() { - return DBuilder.createFile(TheCU.getFilename(), TheCU.getDirectory()); +llvm::DIFile *CGDebugInfo::getOrCreateMainFile() { + return DBuilder.createFile(TheCU->getFilename(), TheCU->getDirectory()); } /// getLineNumber - Get line number for the location. If location is invalid @@ -369,12 +379,13 @@ void CGDebugInfo::CreateCompileUnit() { DebugKind <= CodeGenOptions::DebugLineTablesOnly ? llvm::DIBuilder::LineTablesOnly : llvm::DIBuilder::FullDebug, + 0 /* DWOid */, DebugKind != CodeGenOptions::LocTrackingOnly); } /// CreateType - Get the Basic type from the cache or create a new /// one if necessary. -llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { +llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) { llvm::dwarf::TypeKind Encoding; StringRef BTName; switch (BT->getKind()) { @@ -386,7 +397,7 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { case BuiltinType::NullPtr: return DBuilder.createNullPtrType(); case BuiltinType::Void: - return llvm::DIType(); + return nullptr; case BuiltinType::ObjCClass: if (!ClassTy) ClassTy = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, @@ -409,11 +420,11 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy); - llvm::DIType ISATy = DBuilder.createPointerType(ClassTy, Size); + auto *ISATy = DBuilder.createPointerType(ClassTy, Size); ObjTy = DBuilder.createStructType(TheCU, "objc_object", getOrCreateMainFile(), - 0, 0, 0, 0, llvm::DIType(), llvm::DIArray()); + 0, 0, 0, 0, nullptr, llvm::DINodeArray()); DBuilder.replaceArrays( ObjTy, @@ -510,11 +521,10 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) { // Bit size, align and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(BT); uint64_t Align = CGM.getContext().getTypeAlign(BT); - llvm::DIType DbgTy = DBuilder.createBasicType(BTName, Size, Align, Encoding); - return DbgTy; + return DBuilder.createBasicType(BTName, Size, Align, Encoding); } -llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty) { +llvm::DIType *CGDebugInfo::CreateType(const ComplexType *Ty) { // Bit size, align and offset of the type. llvm::dwarf::TypeKind Encoding = llvm::dwarf::DW_ATE_complex_float; if (Ty->isComplexIntegerType()) @@ -522,15 +532,13 @@ llvm::DIType CGDebugInfo::CreateType(const ComplexType *Ty) { uint64_t Size = CGM.getContext().getTypeSize(Ty); uint64_t Align = CGM.getContext().getTypeAlign(Ty); - llvm::DIType DbgTy = - DBuilder.createBasicType("complex", Size, Align, Encoding); - - return DbgTy; + return DBuilder.createBasicType("complex", Size, Align, Encoding); } /// CreateCVRType - Get the qualified type from the cache or create /// a new one if necessary. -llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateQualifiedType(QualType Ty, + llvm::DIFile *Unit) { QualifierCollector Qc; const Type *T = Qc.strip(Ty); @@ -556,17 +564,15 @@ llvm::DIType CGDebugInfo::CreateQualifiedType(QualType Ty, llvm::DIFile Unit) { return getOrCreateType(QualType(T, 0), Unit); } - llvm::DIType FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit); + auto *FromTy = getOrCreateType(Qc.apply(CGM.getContext(), T), Unit); // No need to fill in the Name, Line, Size, Alignment, Offset in case of // CVR derived types. - llvm::DIType DbgTy = DBuilder.createQualifiedType(Tag, FromTy); - - return DbgTy; + return DBuilder.createQualifiedType(Tag, FromTy); } -llvm::DIType CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty, + llvm::DIFile *Unit) { // The frontend treats 'id' as a typedef to an ObjCObjectType, // whereas 'id<protocol>' is treated as an ObjCPointerType. For the @@ -574,12 +580,12 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCObjectPointerType *Ty, if (Ty->isObjCQualifiedIdType()) return getOrCreateType(CGM.getContext().getObjCIdType(), Unit); - llvm::DIType DbgTy = CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, - Ty, Ty->getPointeeType(), Unit); - return DbgTy; + return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty, + Ty->getPointeeType(), Unit); } -llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const PointerType *Ty, + llvm::DIFile *Unit) { return CreatePointerLikeType(llvm::dwarf::DW_TAG_pointer_type, Ty, Ty->getPointeeType(), Unit); } @@ -588,12 +594,12 @@ llvm::DIType CGDebugInfo::CreateType(const PointerType *Ty, llvm::DIFile Unit) { /// on their mangled names, if they're external. static SmallString<256> getUniqueTagTypeName(const TagType *Ty, CodeGenModule &CGM, - llvm::DICompileUnit TheCU) { + llvm::DICompileUnit *TheCU) { SmallString<256> FullName; // FIXME: ODR should apply to ObjC++ exactly the same wasy it does to C++. // For now, only apply ODR with C++. const TagDecl *TD = Ty->getDecl(); - if (TheCU.getLanguage() != llvm::dwarf::DW_LANG_C_plus_plus || + if (TheCU->getSourceLanguage() != llvm::dwarf::DW_LANG_C_plus_plus || !TD->isExternallyVisible()) return FullName; // Microsoft Mangler does not have support for mangleCXXRTTIName yet. @@ -608,41 +614,56 @@ static SmallString<256> getUniqueTagTypeName(const TagType *Ty, return FullName; } -// Creates a forward declaration for a RecordDecl in the given context. -llvm::DICompositeType -CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty, - llvm::DIDescriptor Ctx) { - const RecordDecl *RD = Ty->getDecl(); - if (llvm::DIType T = getTypeOrNull(CGM.getContext().getRecordType(RD))) - return llvm::DICompositeType(T); - llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation()); - unsigned Line = getLineNumber(RD->getLocation()); - StringRef RDName = getClassName(RD); - - llvm::dwarf::Tag Tag; +static llvm::dwarf::Tag getTagForRecord(const RecordDecl *RD) { + llvm::dwarf::Tag Tag; if (RD->isStruct() || RD->isInterface()) Tag = llvm::dwarf::DW_TAG_structure_type; else if (RD->isUnion()) Tag = llvm::dwarf::DW_TAG_union_type; else { + // FIXME: This could be a struct type giving a default visibility different + // than C++ class type, but needs llvm metadata changes first. assert(RD->isClass()); Tag = llvm::dwarf::DW_TAG_class_type; } + return Tag; +} + +// Creates a forward declaration for a RecordDecl in the given context. +llvm::DICompositeType * +CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty, + llvm::DIScope *Ctx) { + const RecordDecl *RD = Ty->getDecl(); + if (llvm::DIType *T = getTypeOrNull(CGM.getContext().getRecordType(RD))) + return cast<llvm::DICompositeType>(T); + llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation()); + unsigned Line = getLineNumber(RD->getLocation()); + StringRef RDName = getClassName(RD); + + uint64_t Size = 0; + uint64_t Align = 0; + + const RecordDecl *D = RD->getDefinition(); + if (D && D->isCompleteDefinition()) { + Size = CGM.getContext().getTypeSize(Ty); + Align = CGM.getContext().getTypeAlign(Ty); + } // Create the type. SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); - llvm::DICompositeType RetTy = DBuilder.createReplaceableForwardDecl( - Tag, RDName, Ctx, DefUnit, Line, 0, 0, 0, FullName); + llvm::DICompositeType *RetTy = DBuilder.createReplaceableCompositeType( + getTagForRecord(RD), RDName, Ctx, DefUnit, Line, 0, Size, Align, + llvm::DINode::FlagFwdDecl, FullName); ReplaceMap.emplace_back( std::piecewise_construct, std::make_tuple(Ty), std::make_tuple(static_cast<llvm::Metadata *>(RetTy))); return RetTy; } -llvm::DIType CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, - const Type *Ty, - QualType PointeeTy, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, + const Type *Ty, + QualType PointeeTy, + llvm::DIFile *Unit) { if (Tag == llvm::dwarf::DW_TAG_reference_type || Tag == llvm::dwarf::DW_TAG_rvalue_reference_type) return DBuilder.createReferenceType(Tag, getOrCreateType(PointeeTy, Unit)); @@ -658,8 +679,8 @@ llvm::DIType CGDebugInfo::CreatePointerLikeType(llvm::dwarf::Tag Tag, Align); } -llvm::DIType CGDebugInfo::getOrCreateStructPtrType(StringRef Name, - llvm::DIType &Cache) { +llvm::DIType *CGDebugInfo::getOrCreateStructPtrType(StringRef Name, + llvm::DIType *&Cache) { if (Cache) return Cache; Cache = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type, Name, @@ -669,18 +690,16 @@ llvm::DIType CGDebugInfo::getOrCreateStructPtrType(StringRef Name, return Cache; } -llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const BlockPointerType *Ty, + llvm::DIFile *Unit) { if (BlockLiteralGeneric) return BlockLiteralGeneric; SmallVector<llvm::Metadata *, 8> EltTys; - llvm::DIType FieldTy; QualType FType; uint64_t FieldSize, FieldOffset; unsigned FieldAlign; - llvm::DIArray Elements; - llvm::DIType EltTy, DescTy; + llvm::DINodeArray Elements; FieldOffset = 0; FType = CGM.getContext().UnsignedLongTy; @@ -690,17 +709,17 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty, Elements = DBuilder.getOrCreateArray(EltTys); EltTys.clear(); - unsigned Flags = llvm::DIDescriptor::FlagAppleBlock; + unsigned Flags = llvm::DINode::FlagAppleBlock; unsigned LineNo = getLineNumber(CurLoc); - EltTy = DBuilder.createStructType(Unit, "__block_descriptor", Unit, LineNo, - FieldOffset, 0, Flags, llvm::DIType(), - Elements); + auto *EltTy = + DBuilder.createStructType(Unit, "__block_descriptor", Unit, LineNo, + FieldOffset, 0, Flags, nullptr, Elements); // Bit size, align and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(Ty); - DescTy = DBuilder.createPointerType(EltTy, Size); + auto *DescTy = DBuilder.createPointerType(EltTy, Size); FieldOffset = 0; FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy); @@ -712,29 +731,27 @@ llvm::DIType CGDebugInfo::CreateType(const BlockPointerType *Ty, EltTys.push_back(CreateMemberType(Unit, FType, "__FuncPtr", &FieldOffset)); FType = CGM.getContext().getPointerType(CGM.getContext().VoidTy); - FieldTy = DescTy; FieldSize = CGM.getContext().getTypeSize(Ty); FieldAlign = CGM.getContext().getTypeAlign(Ty); - FieldTy = - DBuilder.createMemberType(Unit, "__descriptor", Unit, LineNo, FieldSize, - FieldAlign, FieldOffset, 0, FieldTy); - EltTys.push_back(FieldTy); + EltTys.push_back(DBuilder.createMemberType(Unit, "__descriptor", Unit, LineNo, + FieldSize, FieldAlign, FieldOffset, + 0, DescTy)); FieldOffset += FieldSize; Elements = DBuilder.getOrCreateArray(EltTys); - EltTy = DBuilder.createStructType(Unit, "__block_literal_generic", Unit, - LineNo, FieldOffset, 0, Flags, - llvm::DIType(), Elements); + EltTy = + DBuilder.createStructType(Unit, "__block_literal_generic", Unit, LineNo, + FieldOffset, 0, Flags, nullptr, Elements); BlockLiteralGeneric = DBuilder.createPointerType(EltTy, Size); return BlockLiteralGeneric; } -llvm::DIType CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, + llvm::DIFile *Unit) { assert(Ty->isTypeAlias()); - llvm::DIType Src = getOrCreateType(Ty->getAliasedType(), Unit); + llvm::DIType *Src = getOrCreateType(Ty->getAliasedType(), Unit); SmallString<128> NS; llvm::raw_svector_ostream OS(NS); @@ -749,35 +766,26 @@ llvm::DIType CGDebugInfo::CreateType(const TemplateSpecializationType *Ty, Ty->getTemplateName().getAsTemplateDecl())->getTemplatedDecl(); SourceLocation Loc = AliasDecl->getLocation(); - llvm::DIFile File = getOrCreateFile(Loc); - unsigned Line = getLineNumber(Loc); - - llvm::DIDescriptor Ctxt = - getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext())); - - return DBuilder.createTypedef(Src, internString(OS.str()), File, Line, Ctxt); + return DBuilder.createTypedef( + Src, internString(OS.str()), getOrCreateFile(Loc), getLineNumber(Loc), + getContextDescriptor(cast<Decl>(AliasDecl->getDeclContext()))); } -llvm::DIType CGDebugInfo::CreateType(const TypedefType *Ty, llvm::DIFile Unit) { - // Typedefs are derived from some other type. If we have a typedef of a - // typedef, make sure to emit the whole chain. - llvm::DIType Src = getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit); +llvm::DIType *CGDebugInfo::CreateType(const TypedefType *Ty, + llvm::DIFile *Unit) { // We don't set size information, but do specify where the typedef was // declared. SourceLocation Loc = Ty->getDecl()->getLocation(); - llvm::DIFile File = getOrCreateFile(Loc); - unsigned Line = getLineNumber(Loc); - const TypedefNameDecl *TyDecl = Ty->getDecl(); - llvm::DIDescriptor TypedefContext = - getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext())); - - return DBuilder.createTypedef(Src, TyDecl->getName(), File, Line, - TypedefContext); + // Typedefs are derived from some other type. + return DBuilder.createTypedef( + getOrCreateType(Ty->getDecl()->getUnderlyingType(), Unit), + Ty->getDecl()->getName(), getOrCreateFile(Loc), getLineNumber(Loc), + getContextDescriptor(cast<Decl>(Ty->getDecl()->getDeclContext()))); } -llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const FunctionType *Ty, + llvm::DIFile *Unit) { SmallVector<llvm::Metadata *, 16> EltTys; // Add the result type at least. @@ -794,11 +802,11 @@ llvm::DIType CGDebugInfo::CreateType(const FunctionType *Ty, EltTys.push_back(DBuilder.createUnspecifiedParameter()); } - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); return DBuilder.createSubroutineType(Unit, EltTypeArray); } -/// Convert an AccessSpecifier into the corresponding DIDescriptor flag. +/// Convert an AccessSpecifier into the corresponding DINode flag. /// As an optimization, return 0 if the access specifier equals the /// default for the containing type. static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) { @@ -813,25 +821,25 @@ static unsigned getAccessFlag(AccessSpecifier Access, const RecordDecl *RD) { switch (Access) { case clang::AS_private: - return llvm::DIDescriptor::FlagPrivate; + return llvm::DINode::FlagPrivate; case clang::AS_protected: - return llvm::DIDescriptor::FlagProtected; + return llvm::DINode::FlagProtected; case clang::AS_public: - return llvm::DIDescriptor::FlagPublic; + return llvm::DINode::FlagPublic; case clang::AS_none: return 0; } llvm_unreachable("unexpected access enumerator"); } -llvm::DIType CGDebugInfo::createFieldType( +llvm::DIType *CGDebugInfo::createFieldType( StringRef name, QualType type, uint64_t sizeInBitsOverride, SourceLocation loc, AccessSpecifier AS, uint64_t offsetInBits, - llvm::DIFile tunit, llvm::DIScope scope, const RecordDecl *RD) { - llvm::DIType debugType = getOrCreateType(type, tunit); + llvm::DIFile *tunit, llvm::DIScope *scope, const RecordDecl *RD) { + llvm::DIType *debugType = getOrCreateType(type, tunit); // Get the location for the field. - llvm::DIFile file = getOrCreateFile(loc); + llvm::DIFile *file = getOrCreateFile(loc); unsigned line = getLineNumber(loc); uint64_t SizeInBits = 0; @@ -853,7 +861,7 @@ llvm::DIType CGDebugInfo::createFieldType( /// CollectRecordLambdaFields - Helper for CollectRecordFields. void CGDebugInfo::CollectRecordLambdaFields( const CXXRecordDecl *CXXDecl, SmallVectorImpl<llvm::Metadata *> &elements, - llvm::DIType RecordTy) { + llvm::DIType *RecordTy) { // For C++11 Lambdas a Field will be the same as a Capture, but the Capture // has the name and the location of the variable so we should iterate over // both concurrently. @@ -866,14 +874,14 @@ void CGDebugInfo::CollectRecordLambdaFields( const LambdaCapture &C = *I; if (C.capturesVariable()) { VarDecl *V = C.getCapturedVar(); - llvm::DIFile VUnit = getOrCreateFile(C.getLocation()); + llvm::DIFile *VUnit = getOrCreateFile(C.getLocation()); StringRef VName = V->getName(); uint64_t SizeInBitsOverride = 0; if (Field->isBitField()) { SizeInBitsOverride = Field->getBitWidthValue(CGM.getContext()); assert(SizeInBitsOverride && "found named 0-width bitfield"); } - llvm::DIType fieldType = createFieldType( + llvm::DIType *fieldType = createFieldType( VName, Field->getType(), SizeInBitsOverride, C.getLocation(), Field->getAccess(), layout.getFieldOffset(fieldno), VUnit, RecordTy, CXXDecl); @@ -884,9 +892,9 @@ void CGDebugInfo::CollectRecordLambdaFields( // by using AT_object_pointer for the function and having that be // used as 'this' for semantic references. FieldDecl *f = *Field; - llvm::DIFile VUnit = getOrCreateFile(f->getLocation()); + llvm::DIFile *VUnit = getOrCreateFile(f->getLocation()); QualType type = f->getType(); - llvm::DIType fieldType = createFieldType( + llvm::DIType *fieldType = createFieldType( "this", type, 0, f->getLocation(), f->getAccess(), layout.getFieldOffset(fieldno), VUnit, RecordTy, CXXDecl); @@ -896,14 +904,14 @@ void CGDebugInfo::CollectRecordLambdaFields( } /// Helper for CollectRecordFields. -llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, - llvm::DIType RecordTy, - const RecordDecl *RD) { +llvm::DIDerivedType * +CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, llvm::DIType *RecordTy, + const RecordDecl *RD) { // Create the descriptor for the static variable, with or without // constant initializers. Var = Var->getCanonicalDecl(); - llvm::DIFile VUnit = getOrCreateFile(Var->getLocation()); - llvm::DIType VTy = getOrCreateType(Var->getType(), VUnit); + llvm::DIFile *VUnit = getOrCreateFile(Var->getLocation()); + llvm::DIType *VTy = getOrCreateType(Var->getType(), VUnit); unsigned LineNumber = getLineNumber(Var->getLocation()); StringRef VName = Var->getName(); @@ -919,7 +927,7 @@ llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, } unsigned Flags = getAccessFlag(Var->getAccess(), RD); - llvm::DIDerivedType GV = DBuilder.createStaticMemberType( + llvm::DIDerivedType *GV = DBuilder.createStaticMemberType( RecordTy, VName, VUnit, LineNumber, VTy, Flags, C); StaticDataMemberCache[Var->getCanonicalDecl()].reset(GV); return GV; @@ -927,8 +935,8 @@ llvm::DIDerivedType CGDebugInfo::CreateRecordStaticField(const VarDecl *Var, /// CollectRecordNormalField - Helper for CollectRecordFields. void CGDebugInfo::CollectRecordNormalField( - const FieldDecl *field, uint64_t OffsetInBits, llvm::DIFile tunit, - SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType RecordTy, + const FieldDecl *field, uint64_t OffsetInBits, llvm::DIFile *tunit, + SmallVectorImpl<llvm::Metadata *> &elements, llvm::DIType *RecordTy, const RecordDecl *RD) { StringRef name = field->getName(); QualType type = field->getType(); @@ -943,7 +951,7 @@ void CGDebugInfo::CollectRecordNormalField( assert(SizeInBitsOverride && "found named 0-width bitfield"); } - llvm::DIType fieldType = + llvm::DIType *fieldType = createFieldType(name, type, SizeInBitsOverride, field->getLocation(), field->getAccess(), OffsetInBits, tunit, RecordTy, RD); @@ -953,9 +961,9 @@ void CGDebugInfo::CollectRecordNormalField( /// CollectRecordFields - A helper function to collect debug info for /// record fields. This is used while creating debug info entry for a Record. void CGDebugInfo::CollectRecordFields( - const RecordDecl *record, llvm::DIFile tunit, + const RecordDecl *record, llvm::DIFile *tunit, SmallVectorImpl<llvm::Metadata *> &elements, - llvm::DICompositeType RecordTy) { + llvm::DICompositeType *RecordTy) { const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(record); if (CXXDecl && CXXDecl->isLambda()) @@ -975,8 +983,7 @@ void CGDebugInfo::CollectRecordFields( if (MI != StaticDataMemberCache.end()) { assert(MI->second && "Static data member declaration should still exist"); - elements.push_back( - llvm::DIDerivedType(cast<llvm::MDNode>(MI->second))); + elements.push_back(cast<llvm::DIDerivedTypeBase>(MI->second)); } else { auto Field = CreateRecordStaticField(V, RecordTy, record); elements.push_back(Field); @@ -994,27 +1001,29 @@ void CGDebugInfo::CollectRecordFields( /// getOrCreateMethodType - CXXMethodDecl's type is a FunctionType. This /// function type is not updated to include implicit "this" pointer. Use this /// routine to get a method type which includes "this" pointer. -llvm::DICompositeType +llvm::DISubroutineType * CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method, - llvm::DIFile Unit) { + llvm::DIFile *Unit) { const FunctionProtoType *Func = Method->getType()->getAs<FunctionProtoType>(); if (Method->isStatic()) - return llvm::DICompositeType(getOrCreateType(QualType(Func, 0), Unit)); + return cast_or_null<llvm::DISubroutineType>( + getOrCreateType(QualType(Func, 0), Unit)); return getOrCreateInstanceMethodType(Method->getThisType(CGM.getContext()), Func, Unit); } -llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType( - QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile Unit) { +llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType( + QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile *Unit) { // Add "this" pointer. - llvm::DITypeArray Args = llvm::DISubroutineType( - getOrCreateType(QualType(Func, 0), Unit)).getTypeArray(); - assert(Args.getNumElements() && "Invalid number of arguments!"); + llvm::DITypeRefArray Args( + cast<llvm::DISubroutineType>(getOrCreateType(QualType(Func, 0), Unit)) + ->getTypeArray()); + assert(Args.size() && "Invalid number of arguments!"); SmallVector<llvm::Metadata *, 16> Elts; // First element is always return type. For 'void' functions it is NULL. - Elts.push_back(Args.getElement(0)); + Elts.push_back(Args[0]); // "this" pointer is always first argument. const CXXRecordDecl *RD = ThisPtr->getPointeeCXXRecordDecl(); @@ -1025,8 +1034,8 @@ llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType( unsigned AS = CGM.getContext().getTargetAddressSpace(PointeeTy); uint64_t Size = CGM.getTarget().getPointerWidth(AS); uint64_t Align = CGM.getContext().getTypeAlign(ThisPtrTy); - llvm::DIType PointeeType = getOrCreateType(PointeeTy, Unit); - llvm::DIType ThisPtrType = + llvm::DIType *PointeeType = getOrCreateType(PointeeTy, Unit); + llvm::DIType *ThisPtrType = DBuilder.createPointerType(PointeeType, Size, Align); TypeCache[ThisPtr.getAsOpaquePtr()].reset(ThisPtrType); // TODO: This and the artificial type below are misleading, the @@ -1035,23 +1044,23 @@ llvm::DICompositeType CGDebugInfo::getOrCreateInstanceMethodType( ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType); Elts.push_back(ThisPtrType); } else { - llvm::DIType ThisPtrType = getOrCreateType(ThisPtr, Unit); + llvm::DIType *ThisPtrType = getOrCreateType(ThisPtr, Unit); TypeCache[ThisPtr.getAsOpaquePtr()].reset(ThisPtrType); ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType); Elts.push_back(ThisPtrType); } // Copy rest of the arguments. - for (unsigned i = 1, e = Args.getNumElements(); i != e; ++i) - Elts.push_back(Args.getElement(i)); + for (unsigned i = 1, e = Args.size(); i != e; ++i) + Elts.push_back(Args[i]); - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); unsigned Flags = 0; if (Func->getExtProtoInfo().RefQualifier == RQ_LValue) - Flags |= llvm::DIDescriptor::FlagLValueReference; + Flags |= llvm::DINode::FlagLValueReference; if (Func->getExtProtoInfo().RefQualifier == RQ_RValue) - Flags |= llvm::DIDescriptor::FlagRValueReference; + Flags |= llvm::DINode::FlagRValueReference; return DBuilder.createSubroutineType(Unit, EltTypeArray, Flags); } @@ -1066,16 +1075,15 @@ static bool isFunctionLocalClass(const CXXRecordDecl *RD) { return false; } -/// CreateCXXMemberFunction - A helper function to create a DISubprogram for +/// CreateCXXMemberFunction - A helper function to create a subprogram for /// a single member function GlobalDecl. -llvm::DISubprogram -CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, - llvm::DIFile Unit, llvm::DIType RecordTy) { +llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction( + const CXXMethodDecl *Method, llvm::DIFile *Unit, llvm::DIType *RecordTy) { bool IsCtorOrDtor = isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method); StringRef MethodName = getFunctionName(Method); - llvm::DICompositeType MethodTy = getOrCreateMethodType(Method, Unit); + llvm::DISubroutineType *MethodTy = getOrCreateMethodType(Method, Unit); // Since a single ctor/dtor corresponds to multiple functions, it doesn't // make sense to give a single ctor/dtor a linkage name. @@ -1084,7 +1092,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, MethodLinkageName = CGM.getMangledName(Method); // Get the location for the method. - llvm::DIFile MethodDefUnit; + llvm::DIFile *MethodDefUnit = nullptr; unsigned MethodLine = 0; if (!Method->isImplicit()) { MethodDefUnit = getOrCreateFile(Method->getLocation()); @@ -1092,7 +1100,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, } // Collect virtual method info. - llvm::DIType ContainingType; + llvm::DIType *ContainingType = nullptr; unsigned Virtuality = 0; unsigned VIndex = 0; @@ -1115,29 +1123,29 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, unsigned Flags = 0; if (Method->isImplicit()) - Flags |= llvm::DIDescriptor::FlagArtificial; + Flags |= llvm::DINode::FlagArtificial; Flags |= getAccessFlag(Method->getAccess(), Method->getParent()); if (const CXXConstructorDecl *CXXC = dyn_cast<CXXConstructorDecl>(Method)) { if (CXXC->isExplicit()) - Flags |= llvm::DIDescriptor::FlagExplicit; + Flags |= llvm::DINode::FlagExplicit; } else if (const CXXConversionDecl *CXXC = dyn_cast<CXXConversionDecl>(Method)) { if (CXXC->isExplicit()) - Flags |= llvm::DIDescriptor::FlagExplicit; + Flags |= llvm::DINode::FlagExplicit; } if (Method->hasPrototype()) - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; if (Method->getRefQualifier() == RQ_LValue) - Flags |= llvm::DIDescriptor::FlagLValueReference; + Flags |= llvm::DINode::FlagLValueReference; if (Method->getRefQualifier() == RQ_RValue) - Flags |= llvm::DIDescriptor::FlagRValueReference; + Flags |= llvm::DINode::FlagRValueReference; - llvm::DIArray TParamsArray = CollectFunctionTemplateParams(Method, Unit); - llvm::DISubprogram SP = DBuilder.createMethod( + llvm::DINodeArray TParamsArray = CollectFunctionTemplateParams(Method, Unit); + llvm::DISubprogram *SP = DBuilder.createMethod( RecordTy, MethodName, MethodLinkageName, MethodDefUnit, MethodLine, MethodTy, /*isLocalToUnit=*/false, /* isDefinition=*/false, Virtuality, VIndex, ContainingType, Flags, - CGM.getLangOpts().Optimize, nullptr, TParamsArray); + CGM.getLangOpts().Optimize, nullptr, TParamsArray.get()); SPCache[Method->getCanonicalDecl()].reset(SP); @@ -1148,8 +1156,8 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method, /// C++ member functions. This is used while creating debug info entry for /// a Record. void CGDebugInfo::CollectCXXMemberFunctions( - const CXXRecordDecl *RD, llvm::DIFile Unit, - SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType RecordTy) { + const CXXRecordDecl *RD, llvm::DIFile *Unit, + SmallVectorImpl<llvm::Metadata *> &EltTys, llvm::DIType *RecordTy) { // Since we want more than just the individual member decls if we // have templated functions iterate over every declaration to gather @@ -1187,10 +1195,9 @@ void CGDebugInfo::CollectCXXMemberFunctions( /// CollectCXXBases - A helper function to collect debug info for /// C++ base classes. This is used while creating debug info entry for /// a Record. -void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit, +void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile *Unit, SmallVectorImpl<llvm::Metadata *> &EltTys, - llvm::DIType RecordTy) { - + llvm::DIType *RecordTy) { const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); for (const auto &BI : RD->bases()) { unsigned BFlags = 0; @@ -1212,24 +1219,24 @@ void CGDebugInfo::CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit, BaseOffset = 4 * CGM.getMicrosoftVTableContext().getVBTableIndex(RD, Base); } - BFlags = llvm::DIDescriptor::FlagVirtual; + BFlags = llvm::DINode::FlagVirtual; } else BaseOffset = CGM.getContext().toBits(RL.getBaseClassOffset(Base)); // FIXME: Inconsistent units for BaseOffset. It is in bytes when // BI->isVirtual() and bits when not. BFlags |= getAccessFlag(BI.getAccessSpecifier(), RD); - llvm::DIType DTy = DBuilder.createInheritance( + llvm::DIType *DTy = DBuilder.createInheritance( RecordTy, getOrCreateType(BI.getType(), Unit), BaseOffset, BFlags); EltTys.push_back(DTy); } } /// CollectTemplateParams - A helper function to collect template parameters. -llvm::DIArray +llvm::DINodeArray CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, ArrayRef<TemplateArgument> TAList, - llvm::DIFile Unit) { + llvm::DIFile *Unit) { SmallVector<llvm::Metadata *, 16> TemplateParams; for (unsigned i = 0, e = TAList.size(); i != e; ++i) { const TemplateArgument &TA = TAList[i]; @@ -1238,23 +1245,20 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, Name = TPList->getParam(i)->getName(); switch (TA.getKind()) { case TemplateArgument::Type: { - llvm::DIType TTy = getOrCreateType(TA.getAsType(), Unit); - llvm::DITemplateTypeParameter TTP = - DBuilder.createTemplateTypeParameter(TheCU, Name, TTy); - TemplateParams.push_back(TTP); + llvm::DIType *TTy = getOrCreateType(TA.getAsType(), Unit); + TemplateParams.push_back( + DBuilder.createTemplateTypeParameter(TheCU, Name, TTy)); } break; case TemplateArgument::Integral: { - llvm::DIType TTy = getOrCreateType(TA.getIntegralType(), Unit); - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter( - TheCU, Name, TTy, - llvm::ConstantInt::get(CGM.getLLVMContext(), TA.getAsIntegral())); - TemplateParams.push_back(TVP); + llvm::DIType *TTy = getOrCreateType(TA.getIntegralType(), Unit); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, + llvm::ConstantInt::get(CGM.getLLVMContext(), TA.getAsIntegral()))); } break; case TemplateArgument::Declaration: { const ValueDecl *D = TA.getAsDecl(); QualType T = TA.getParamTypeForDecl().getDesugaredType(CGM.getContext()); - llvm::DIType TTy = getOrCreateType(T, Unit); + llvm::DIType *TTy = getOrCreateType(T, Unit); llvm::Constant *V = nullptr; const CXXMethodDecl *MD; // Variable pointer template parameters have a value that is the address @@ -1278,15 +1282,13 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, CGM.getContext().toCharUnitsFromBits((int64_t)fieldOffset); V = CGM.getCXXABI().EmitMemberDataPointer(MPT, chars); } - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter( - TheCU, Name, TTy, - cast_or_null<llvm::Constant>(V->stripPointerCasts())); - TemplateParams.push_back(TVP); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, + cast_or_null<llvm::Constant>(V->stripPointerCasts()))); } break; case TemplateArgument::NullPtr: { QualType T = TA.getNullPtrType(); - llvm::DIType TTy = getOrCreateType(T, Unit); + llvm::DIType *TTy = getOrCreateType(T, Unit); llvm::Constant *V = nullptr; // Special case member data pointer null values since they're actually -1 // instead of zero. @@ -1301,24 +1303,19 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, V = CGM.getCXXABI().EmitNullMemberPointer(MPT); if (!V) V = llvm::ConstantInt::get(CGM.Int8Ty, 0); - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter(TheCU, Name, TTy, - cast<llvm::Constant>(V)); - TemplateParams.push_back(TVP); - } break; - case TemplateArgument::Template: { - llvm::DITemplateValueParameter - TVP = DBuilder.createTemplateTemplateParameter( - TheCU, Name, llvm::DIType(), - TA.getAsTemplate().getAsTemplateDecl()->getQualifiedNameAsString()); - TemplateParams.push_back(TVP); - } break; - case TemplateArgument::Pack: { - llvm::DITemplateValueParameter TVP = DBuilder.createTemplateParameterPack( - TheCU, Name, llvm::DIType(), - CollectTemplateParams(nullptr, TA.getPackAsArray(), Unit)); - TemplateParams.push_back(TVP); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, cast<llvm::Constant>(V))); } break; + case TemplateArgument::Template: + TemplateParams.push_back(DBuilder.createTemplateTemplateParameter( + TheCU, Name, nullptr, + TA.getAsTemplate().getAsTemplateDecl()->getQualifiedNameAsString())); + break; + case TemplateArgument::Pack: + TemplateParams.push_back(DBuilder.createTemplateParameterPack( + TheCU, Name, nullptr, + CollectTemplateParams(nullptr, TA.getPackAsArray(), Unit))); + break; case TemplateArgument::Expression: { const Expr *E = TA.getAsExpr(); QualType T = E->getType(); @@ -1326,11 +1323,9 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, T = CGM.getContext().getLValueReferenceType(T); llvm::Constant *V = CGM.EmitConstantExpr(E, T); assert(V && "Expression in template argument isn't constant"); - llvm::DIType TTy = getOrCreateType(T, Unit); - llvm::DITemplateValueParameter TVP = - DBuilder.createTemplateValueParameter( - TheCU, Name, TTy, cast<llvm::Constant>(V->stripPointerCasts())); - TemplateParams.push_back(TVP); + llvm::DIType *TTy = getOrCreateType(T, Unit); + TemplateParams.push_back(DBuilder.createTemplateValueParameter( + TheCU, Name, TTy, cast<llvm::Constant>(V->stripPointerCasts()))); } break; // And the following should never occur: case TemplateArgument::TemplateExpansion: @@ -1344,8 +1339,9 @@ CGDebugInfo::CollectTemplateParams(const TemplateParameterList *TPList, /// CollectFunctionTemplateParams - A helper function to collect debug /// info for function template parameters. -llvm::DIArray CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD, - llvm::DIFile Unit) { +llvm::DINodeArray +CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD, + llvm::DIFile *Unit) { if (FD->getTemplatedKind() == FunctionDecl::TK_FunctionTemplateSpecialization) { const TemplateParameterList *TList = FD->getTemplateSpecializationInfo() @@ -1354,13 +1350,13 @@ llvm::DIArray CGDebugInfo::CollectFunctionTemplateParams(const FunctionDecl *FD, return CollectTemplateParams( TList, FD->getTemplateSpecializationArgs()->asArray(), Unit); } - return llvm::DIArray(); + return llvm::DINodeArray(); } /// CollectCXXTemplateParams - A helper function to collect debug info for /// template parameters. -llvm::DIArray CGDebugInfo::CollectCXXTemplateParams( - const ClassTemplateSpecializationDecl *TSpecial, llvm::DIFile Unit) { +llvm::DINodeArray CGDebugInfo::CollectCXXTemplateParams( + const ClassTemplateSpecializationDecl *TSpecial, llvm::DIFile *Unit) { // Always get the full list of parameters, not just the ones from // the specialization. TemplateParameterList *TPList = @@ -1370,18 +1366,18 @@ llvm::DIArray CGDebugInfo::CollectCXXTemplateParams( } /// getOrCreateVTablePtrType - Return debug info descriptor for vtable. -llvm::DIType CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile Unit) { - if (VTablePtrType.isValid()) +llvm::DIType *CGDebugInfo::getOrCreateVTablePtrType(llvm::DIFile *Unit) { + if (VTablePtrType) return VTablePtrType; ASTContext &Context = CGM.getContext(); /* Function type */ llvm::Metadata *STy = getOrCreateType(Context.IntTy, Unit); - llvm::DITypeArray SElements = DBuilder.getOrCreateTypeArray(STy); - llvm::DIType SubTy = DBuilder.createSubroutineType(Unit, SElements); + llvm::DITypeRefArray SElements = DBuilder.getOrCreateTypeArray(STy); + llvm::DIType *SubTy = DBuilder.createSubroutineType(Unit, SElements); unsigned Size = Context.getTypeSize(Context.VoidPtrTy); - llvm::DIType vtbl_ptr_type = + llvm::DIType *vtbl_ptr_type = DBuilder.createPointerType(SubTy, Size, 0, "__vtbl_ptr_type"); VTablePtrType = DBuilder.createPointerType(vtbl_ptr_type, Size); return VTablePtrType; @@ -1395,7 +1391,7 @@ StringRef CGDebugInfo::getVTableName(const CXXRecordDecl *RD) { /// CollectVTableInfo - If the C++ class has vtable info then insert appropriate /// debug info entry in EltTys vector. -void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit, +void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile *Unit, SmallVectorImpl<llvm::Metadata *> &EltTys) { const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); @@ -1408,26 +1404,26 @@ void CGDebugInfo::CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit, return; unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy); - llvm::DIType VPTR = DBuilder.createMemberType( + llvm::DIType *VPTR = DBuilder.createMemberType( Unit, getVTableName(RD), Unit, 0, Size, 0, 0, - llvm::DIDescriptor::FlagArtificial, getOrCreateVTablePtrType(Unit)); + llvm::DINode::FlagArtificial, getOrCreateVTablePtrType(Unit)); EltTys.push_back(VPTR); } /// getOrCreateRecordType - Emit record type's standalone debug info. -llvm::DIType CGDebugInfo::getOrCreateRecordType(QualType RTy, - SourceLocation Loc) { +llvm::DIType *CGDebugInfo::getOrCreateRecordType(QualType RTy, + SourceLocation Loc) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); - llvm::DIType T = getOrCreateType(RTy, getOrCreateFile(Loc)); + llvm::DIType *T = getOrCreateType(RTy, getOrCreateFile(Loc)); return T; } /// getOrCreateInterfaceType - Emit an objective c interface type standalone /// debug info. -llvm::DIType CGDebugInfo::getOrCreateInterfaceType(QualType D, - SourceLocation Loc) { +llvm::DIType *CGDebugInfo::getOrCreateInterfaceType(QualType D, + SourceLocation Loc) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); - llvm::DIType T = getOrCreateType(D, getOrCreateFile(Loc)); + llvm::DIType *T = getOrCreateType(D, getOrCreateFile(Loc)); RetainedTypes.push_back(D.getAsOpaquePtr()); return T; } @@ -1438,11 +1434,10 @@ void CGDebugInfo::completeType(const EnumDecl *ED) { QualType Ty = CGM.getContext().getEnumType(ED); void *TyPtr = Ty.getAsOpaquePtr(); auto I = TypeCache.find(TyPtr); - if (I == TypeCache.end() || - !llvm::DIType(cast<llvm::MDNode>(I->second)).isForwardDecl()) + if (I == TypeCache.end() || !cast<llvm::DIType>(I->second)->isForwardDecl()) return; - llvm::DIType Res = CreateTypeDefinition(Ty->castAs<EnumType>()); - assert(!Res.isForwardDecl()); + llvm::DIType *Res = CreateTypeDefinition(Ty->castAs<EnumType>()); + assert(!Res->isForwardDecl()); TypeCache[TyPtr].reset(Res); } @@ -1461,8 +1456,8 @@ void CGDebugInfo::completeRequiredType(const RecordDecl *RD) { return; QualType Ty = CGM.getContext().getRecordType(RD); - llvm::DIType T = getTypeOrNull(Ty); - if (T && T.isForwardDecl()) + llvm::DIType *T = getTypeOrNull(Ty); + if (T && T->isForwardDecl()) completeClassData(RD); } @@ -1472,11 +1467,10 @@ void CGDebugInfo::completeClassData(const RecordDecl *RD) { QualType Ty = CGM.getContext().getRecordType(RD); void *TyPtr = Ty.getAsOpaquePtr(); auto I = TypeCache.find(TyPtr); - if (I != TypeCache.end() && - !llvm::DIType(cast<llvm::MDNode>(I->second)).isForwardDecl()) + if (I != TypeCache.end() && !cast<llvm::DIType>(I->second)->isForwardDecl()) return; - llvm::DIType Res = CreateTypeDefinition(Ty->castAs<RecordType>()); - assert(!Res.isForwardDecl()); + llvm::DIType *Res = CreateTypeDefinition(Ty->castAs<RecordType>()); + assert(!Res->isForwardDecl()); TypeCache[TyPtr].reset(Res); } @@ -1524,9 +1518,9 @@ static bool shouldOmitDefinition(CodeGenOptions::DebugInfoKind DebugKind, } /// CreateType - get structure or union type. -llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) { +llvm::DIType *CGDebugInfo::CreateType(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); - llvm::DICompositeType T(getTypeOrNull(QualType(Ty, 0))); + llvm::DIType *T = cast_or_null<llvm::DIType>(getTypeOrNull(QualType(Ty, 0))); if (T || shouldOmitDefinition(DebugKind, RD, CGM.getLangOpts())) { if (!T) T = getOrCreateRecordFwdDecl( @@ -1537,11 +1531,11 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty) { return CreateTypeDefinition(Ty); } -llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { +llvm::DIType *CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); // Get overall information about the record type for the debug info. - llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation()); // Records and classes and unions can all be recursive. To handle them, we // first generate a debug descriptor for the struct as a forward declaration. @@ -1550,11 +1544,11 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { // may refer to the forward decl if the struct is recursive) and replace all // uses of the forward declaration with the final definition. - llvm::DICompositeType FwdDecl(getOrCreateLimitedType(Ty, DefUnit)); - assert(FwdDecl.isCompositeType() && - "The debug type of a RecordType should be a llvm::DICompositeType"); + auto *FwdDecl = + cast<llvm::DICompositeType>(getOrCreateLimitedType(Ty, DefUnit)); - if (FwdDecl.isForwardDecl()) + const RecordDecl *D = RD->getDefinition(); + if (!D || !D->isCompleteDefinition()) return FwdDecl; if (const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(RD)) @@ -1586,16 +1580,20 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) { LexicalBlockStack.pop_back(); RegionMap.erase(Ty->getDecl()); - llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys); + llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys); DBuilder.replaceArrays(FwdDecl, Elements); + if (FwdDecl->isTemporary()) + FwdDecl = + llvm::MDNode::replaceWithPermanent(llvm::TempDICompositeType(FwdDecl)); + RegionMap[Ty->getDecl()].reset(FwdDecl); return FwdDecl; } /// CreateType - get objective-c object type. -llvm::DIType CGDebugInfo::CreateType(const ObjCObjectType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ObjCObjectType *Ty, + llvm::DIFile *Unit) { // Ignore protocols. return getOrCreateType(Ty->getBaseType(), Unit); } @@ -1625,22 +1623,23 @@ static bool hasDefaultSetterName(const ObjCPropertyDecl *PD, } /// CreateType - get objective-c interface type. -llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, + llvm::DIFile *Unit) { ObjCInterfaceDecl *ID = Ty->getDecl(); if (!ID) - return llvm::DIType(); + return nullptr; // Get overall information about the record type for the debug info. - llvm::DIFile DefUnit = getOrCreateFile(ID->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation()); unsigned Line = getLineNumber(ID->getLocation()); - llvm::dwarf::SourceLanguage RuntimeLang = TheCU.getLanguage(); + auto RuntimeLang = + static_cast<llvm::dwarf::SourceLanguage>(TheCU->getSourceLanguage()); // If this is just a forward declaration return a special forward-declaration // debug type since we won't be able to lay out the entire type. ObjCInterfaceDecl *Def = ID->getDefinition(); if (!Def || !Def->getImplementation()) { - llvm::DIType FwdDecl = DBuilder.createReplaceableForwardDecl( + llvm::DIType *FwdDecl = DBuilder.createReplaceableCompositeType( llvm::dwarf::DW_TAG_structure_type, ID->getName(), TheCU, DefUnit, Line, RuntimeLang); ObjCInterfaceCache.push_back(ObjCInterfaceCacheEntry(Ty, FwdDecl, Unit)); @@ -1650,12 +1649,12 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty, return CreateTypeDefinition(Ty, Unit); } -llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, + llvm::DIFile *Unit) { ObjCInterfaceDecl *ID = Ty->getDecl(); - llvm::DIFile DefUnit = getOrCreateFile(ID->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(ID->getLocation()); unsigned Line = getLineNumber(ID->getLocation()); - unsigned RuntimeLang = TheCU.getLanguage(); + unsigned RuntimeLang = TheCU->getSourceLanguage(); // Bit size, align and offset of the type. uint64_t Size = CGM.getContext().getTypeSize(Ty); @@ -1663,17 +1662,17 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, unsigned Flags = 0; if (ID->getImplementation()) - Flags |= llvm::DIDescriptor::FlagObjcClassComplete; + Flags |= llvm::DINode::FlagObjcClassComplete; - llvm::DICompositeType RealDecl = DBuilder.createStructType( - Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, llvm::DIType(), - llvm::DIArray(), RuntimeLang); + llvm::DICompositeType *RealDecl = DBuilder.createStructType( + Unit, ID->getName(), DefUnit, Line, Size, Align, Flags, nullptr, + llvm::DINodeArray(), RuntimeLang); QualType QTy(Ty, 0); TypeCache[QTy.getAsOpaquePtr()].reset(RealDecl); // Push the struct on region stack. - LexicalBlockStack.emplace_back(static_cast<llvm::MDNode *>(RealDecl)); + LexicalBlockStack.emplace_back(RealDecl); RegionMap[Ty->getDecl()].reset(RealDecl); // Convert all the elements. @@ -1681,19 +1680,19 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, ObjCInterfaceDecl *SClass = ID->getSuperClass(); if (SClass) { - llvm::DIType SClassTy = + llvm::DIType *SClassTy = getOrCreateType(CGM.getContext().getObjCInterfaceType(SClass), Unit); - if (!SClassTy.isValid()) - return llvm::DIType(); + if (!SClassTy) + return nullptr; - llvm::DIType InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, 0); + llvm::DIType *InhTag = DBuilder.createInheritance(RealDecl, SClassTy, 0, 0); EltTys.push_back(InhTag); } // Create entries for all of the properties. for (const auto *PD : ID->properties()) { SourceLocation Loc = PD->getLocation(); - llvm::DIFile PUnit = getOrCreateFile(Loc); + llvm::DIFile *PUnit = getOrCreateFile(Loc); unsigned PLine = getLineNumber(Loc); ObjCMethodDecl *Getter = PD->getGetterMethodDecl(); ObjCMethodDecl *Setter = PD->getSetterMethodDecl(); @@ -1711,9 +1710,9 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, unsigned FieldNo = 0; for (ObjCIvarDecl *Field = ID->all_declared_ivar_begin(); Field; Field = Field->getNextIvar(), ++FieldNo) { - llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit); - if (!FieldTy.isValid()) - return llvm::DIType(); + llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit); + if (!FieldTy) + return nullptr; StringRef FieldName = Field->getName(); @@ -1722,7 +1721,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, continue; // Get the location for the field. - llvm::DIFile FieldDefUnit = getOrCreateFile(Field->getLocation()); + llvm::DIFile *FieldDefUnit = getOrCreateFile(Field->getLocation()); unsigned FieldLine = getLineNumber(Field->getLocation()); QualType FType = Field->getType(); uint64_t FieldSize = 0; @@ -1755,11 +1754,11 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, unsigned Flags = 0; if (Field->getAccessControl() == ObjCIvarDecl::Protected) - Flags = llvm::DIDescriptor::FlagProtected; + Flags = llvm::DINode::FlagProtected; else if (Field->getAccessControl() == ObjCIvarDecl::Private) - Flags = llvm::DIDescriptor::FlagPrivate; + Flags = llvm::DINode::FlagPrivate; else if (Field->getAccessControl() == ObjCIvarDecl::Public) - Flags = llvm::DIDescriptor::FlagPublic; + Flags = llvm::DINode::FlagPublic; llvm::MDNode *PropertyNode = nullptr; if (ObjCImplementationDecl *ImpD = ID->getImplementation()) { @@ -1767,7 +1766,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, ImpD->FindPropertyImplIvarDecl(Field->getIdentifier())) { if (ObjCPropertyDecl *PD = PImpD->getPropertyDecl()) { SourceLocation Loc = PD->getLocation(); - llvm::DIFile PUnit = getOrCreateFile(Loc); + llvm::DIFile *PUnit = getOrCreateFile(Loc); unsigned PLine = getLineNumber(Loc); ObjCMethodDecl *Getter = PD->getGetterMethodDecl(); ObjCMethodDecl *Setter = PD->getSetterMethodDecl(); @@ -1788,15 +1787,16 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const ObjCInterfaceType *Ty, EltTys.push_back(FieldTy); } - llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys); + llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys); DBuilder.replaceArrays(RealDecl, Elements); LexicalBlockStack.pop_back(); return RealDecl; } -llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) { - llvm::DIType ElementTy = getOrCreateType(Ty->getElementType(), Unit); +llvm::DIType *CGDebugInfo::CreateType(const VectorType *Ty, + llvm::DIFile *Unit) { + llvm::DIType *ElementTy = getOrCreateType(Ty->getElementType(), Unit); int64_t Count = Ty->getNumElements(); if (Count == 0) // If number of elements are not known then this is an unbounded array. @@ -1804,7 +1804,7 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) { Count = -1; llvm::Metadata *Subscript = DBuilder.getOrCreateSubrange(0, Count); - llvm::DIArray SubscriptArray = DBuilder.getOrCreateArray(Subscript); + llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscript); uint64_t Size = CGM.getContext().getTypeSize(Ty); uint64_t Align = CGM.getContext().getTypeAlign(Ty); @@ -1812,7 +1812,7 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty, llvm::DIFile Unit) { return DBuilder.createVectorType(Size, Align, ElementTy, SubscriptArray); } -llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) { uint64_t Size; uint64_t Align; @@ -1858,32 +1858,33 @@ llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile Unit) { EltTy = Ty->getElementType(); } - llvm::DIArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts); + llvm::DINodeArray SubscriptArray = DBuilder.getOrCreateArray(Subscripts); - llvm::DIType DbgTy = DBuilder.createArrayType( - Size, Align, getOrCreateType(EltTy, Unit), SubscriptArray); - return DbgTy; + return DBuilder.createArrayType(Size, Align, getOrCreateType(EltTy, Unit), + SubscriptArray); } -llvm::DIType CGDebugInfo::CreateType(const LValueReferenceType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const LValueReferenceType *Ty, + llvm::DIFile *Unit) { return CreatePointerLikeType(llvm::dwarf::DW_TAG_reference_type, Ty, Ty->getPointeeType(), Unit); } -llvm::DIType CGDebugInfo::CreateType(const RValueReferenceType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateType(const RValueReferenceType *Ty, + llvm::DIFile *Unit) { return CreatePointerLikeType(llvm::dwarf::DW_TAG_rvalue_reference_type, Ty, Ty->getPointeeType(), Unit); } -llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty, - llvm::DIFile U) { - llvm::DIType ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U); - if (!Ty->getPointeeType()->isFunctionType()) +llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty, + llvm::DIFile *U) { + uint64_t Size = CGM.getCXXABI().isTypeInfoCalculable(QualType(Ty, 0)) + ? CGM.getContext().getTypeSize(Ty) + : 0; + llvm::DIType *ClassType = getOrCreateType(QualType(Ty->getClass(), 0), U); + if (Ty->isMemberDataPointerType()) return DBuilder.createMemberPointerType( - getOrCreateType(Ty->getPointeeType(), U), ClassType, - CGM.getContext().getTypeSize(Ty)); + getOrCreateType(Ty->getPointeeType(), U), ClassType, Size); const FunctionProtoType *FPT = Ty->getPointeeType()->getAs<FunctionProtoType>(); @@ -1891,17 +1892,17 @@ llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty, getOrCreateInstanceMethodType(CGM.getContext().getPointerType(QualType( Ty->getClass(), FPT->getTypeQuals())), FPT, U), - ClassType, CGM.getContext().getTypeSize(Ty)); + ClassType, Size); } -llvm::DIType CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile U) { +llvm::DIType *CGDebugInfo::CreateType(const AtomicType *Ty, llvm::DIFile *U) { // Ignore the atomic wrapping // FIXME: What is the correct representation? return getOrCreateType(Ty->getValueType(), U); } /// CreateEnumType - get enumeration type. -llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) { +llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) { const EnumDecl *ED = Ty->getDecl(); uint64_t Size = 0; uint64_t Align = 0; @@ -1915,14 +1916,14 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) { // If this is just a forward declaration, construct an appropriately // marked node and just return it. if (!ED->getDefinition()) { - llvm::DIDescriptor EDContext; - EDContext = getContextDescriptor(cast<Decl>(ED->getDeclContext())); - llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation()); + llvm::DIScope *EDContext = + getContextDescriptor(cast<Decl>(ED->getDeclContext())); + llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation()); unsigned Line = getLineNumber(ED->getLocation()); StringRef EDName = ED->getName(); - llvm::DIType RetTy = DBuilder.createReplaceableForwardDecl( + llvm::DIType *RetTy = DBuilder.createReplaceableCompositeType( llvm::dwarf::DW_TAG_enumeration_type, EDName, EDContext, DefUnit, Line, - 0, Size, Align, FullName); + 0, Size, Align, llvm::DINode::FlagFwdDecl, FullName); ReplaceMap.emplace_back( std::piecewise_construct, std::make_tuple(Ty), std::make_tuple(static_cast<llvm::Metadata *>(RetTy))); @@ -1932,7 +1933,7 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumType *Ty) { return CreateTypeDefinition(Ty); } -llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { +llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { const EnumDecl *ED = Ty->getDecl(); uint64_t Size = 0; uint64_t Align = 0; @@ -1943,7 +1944,7 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); - // Create DIEnumerator elements for each enumerator. + // Create elements for each enumerator. SmallVector<llvm::Metadata *, 16> Enumerators; ED = ED->getDefinition(); for (const auto *Enum : ED->enumerators()) { @@ -1952,19 +1953,17 @@ llvm::DIType CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) { } // Return a CompositeType for the enum itself. - llvm::DIArray EltArray = DBuilder.getOrCreateArray(Enumerators); + llvm::DINodeArray EltArray = DBuilder.getOrCreateArray(Enumerators); - llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(ED->getLocation()); unsigned Line = getLineNumber(ED->getLocation()); - llvm::DIDescriptor EnumContext = + llvm::DIScope *EnumContext = getContextDescriptor(cast<Decl>(ED->getDeclContext())); - llvm::DIType ClassTy = ED->isFixed() - ? getOrCreateType(ED->getIntegerType(), DefUnit) - : llvm::DIType(); - llvm::DIType DbgTy = - DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit, Line, - Size, Align, EltArray, ClassTy, FullName); - return DbgTy; + llvm::DIType *ClassTy = + ED->isFixed() ? getOrCreateType(ED->getIntegerType(), DefUnit) : nullptr; + return DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit, + Line, Size, Align, EltArray, ClassTy, + FullName); } static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { @@ -2024,7 +2023,7 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) { /// getType - Get the type from the cache or return null type if it doesn't /// exist. -llvm::DIType CGDebugInfo::getTypeOrNull(QualType Ty) { +llvm::DIType *CGDebugInfo::getTypeOrNull(QualType Ty) { // Unwrap the type as needed for debug information. Ty = UnwrapTypeForDebugInfo(Ty, CGM.getContext()); @@ -2033,10 +2032,10 @@ llvm::DIType CGDebugInfo::getTypeOrNull(QualType Ty) { if (it != TypeCache.end()) { // Verify that the debug info still exists. if (llvm::Metadata *V = it->second) - return llvm::DIType(cast<llvm::MDNode>(V)); + return cast<llvm::DIType>(V); } - return llvm::DIType(); + return nullptr; } void CGDebugInfo::completeTemplateDefinition( @@ -2052,18 +2051,18 @@ void CGDebugInfo::completeTemplateDefinition( /// getOrCreateType - Get the type from the cache or create a new /// one if necessary. -llvm::DIType CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) { if (Ty.isNull()) - return llvm::DIType(); + return nullptr; // Unwrap the type as needed for debug information. Ty = UnwrapTypeForDebugInfo(Ty, CGM.getContext()); - if (llvm::DIType T = getTypeOrNull(Ty)) + if (auto *T = getTypeOrNull(Ty)) return T; // Otherwise create the type. - llvm::DIType Res = CreateTypeNode(Ty, Unit); + llvm::DIType *Res = CreateTypeNode(Ty, Unit); void *TyPtr = Ty.getAsOpaquePtr(); // And update the type cache. @@ -2099,7 +2098,7 @@ ObjCInterfaceDecl *CGDebugInfo::getObjCInterfaceDecl(QualType Ty) { } /// CreateTypeNode - Create a new debug type node. -llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) { // Handle qualifiers, which recursively handles what they refer to. if (Ty.hasLocalQualifiers()) return CreateQualifiedType(Ty, Unit); @@ -2181,25 +2180,25 @@ llvm::DIType CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile Unit) { /// getOrCreateLimitedType - Get the type from the cache or create a new /// limited type if necessary. -llvm::DIType CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, - llvm::DIFile Unit) { +llvm::DIType *CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, + llvm::DIFile *Unit) { QualType QTy(Ty, 0); - llvm::DICompositeType T(getTypeOrNull(QTy)); + auto *T = cast_or_null<llvm::DICompositeTypeBase>(getTypeOrNull(QTy)); // We may have cached a forward decl when we could have created // a non-forward decl. Go ahead and create a non-forward decl // now. - if (T && !T.isForwardDecl()) + if (T && !T->isForwardDecl()) return T; // Otherwise create the type. - llvm::DICompositeType Res = CreateLimitedType(Ty); + llvm::DICompositeType *Res = CreateLimitedType(Ty); // Propagate members from the declaration to the definition // CreateType(const RecordType*) will overwrite this with the members in the // correct order if the full type is needed. - DBuilder.replaceArrays(Res, T.getElements()); + DBuilder.replaceArrays(Res, T ? T->getElements() : llvm::DINodeArray()); // And update the type cache. TypeCache[QTy.getAsOpaquePtr()].reset(Res); @@ -2207,21 +2206,22 @@ llvm::DIType CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty, } // TODO: Currently used for context chains when limiting debug info. -llvm::DICompositeType CGDebugInfo::CreateLimitedType(const RecordType *Ty) { +llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) { RecordDecl *RD = Ty->getDecl(); // Get overall information about the record type for the debug info. - llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation()); + llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation()); unsigned Line = getLineNumber(RD->getLocation()); StringRef RDName = getClassName(RD); - llvm::DIDescriptor RDContext = + llvm::DIScope *RDContext = getContextDescriptor(cast<Decl>(RD->getDeclContext())); // If we ended up creating the type during the context chain construction, // just return that. - llvm::DICompositeType T(getTypeOrNull(CGM.getContext().getRecordType(RD))); - if (T && (!T.isForwardDecl() || !RD->getDefinition())) + auto *T = cast_or_null<llvm::DICompositeType>( + getTypeOrNull(CGM.getContext().getRecordType(RD))); + if (T && (!T->isForwardDecl() || !RD->getDefinition())) return T; // If this is just a forward or incomplete declaration, construct an @@ -2232,38 +2232,27 @@ llvm::DICompositeType CGDebugInfo::CreateLimitedType(const RecordType *Ty) { uint64_t Size = CGM.getContext().getTypeSize(Ty); uint64_t Align = CGM.getContext().getTypeAlign(Ty); - llvm::DICompositeType RealDecl; SmallString<256> FullName = getUniqueTagTypeName(Ty, CGM, TheCU); - if (RD->isUnion()) - RealDecl = DBuilder.createUnionType(RDContext, RDName, DefUnit, Line, Size, - Align, 0, llvm::DIArray(), 0, FullName); - else if (RD->isClass()) { - // FIXME: This could be a struct type giving a default visibility different - // than C++ class type, but needs llvm metadata changes first. - RealDecl = DBuilder.createClassType( - RDContext, RDName, DefUnit, Line, Size, Align, 0, 0, llvm::DIType(), - llvm::DIArray(), llvm::DIType(), llvm::DIArray(), FullName); - } else - RealDecl = DBuilder.createStructType( - RDContext, RDName, DefUnit, Line, Size, Align, 0, llvm::DIType(), - llvm::DIArray(), 0, llvm::DIType(), FullName); + llvm::DICompositeType *RealDecl = DBuilder.createReplaceableCompositeType( + getTagForRecord(RD), RDName, RDContext, DefUnit, Line, 0, Size, Align, 0, + FullName); RegionMap[Ty->getDecl()].reset(RealDecl); TypeCache[QualType(Ty, 0).getAsOpaquePtr()].reset(RealDecl); if (const ClassTemplateSpecializationDecl *TSpecial = dyn_cast<ClassTemplateSpecializationDecl>(RD)) - DBuilder.replaceArrays(RealDecl, llvm::DIArray(), + DBuilder.replaceArrays(RealDecl, llvm::DINodeArray(), CollectCXXTemplateParams(TSpecial, DefUnit)); return RealDecl; } void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD, - llvm::DICompositeType RealDecl) { + llvm::DICompositeType *RealDecl) { // A class's primary base or the class itself contains the vtable. - llvm::DICompositeType ContainingType; + llvm::DICompositeType *ContainingType = nullptr; const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD); if (const CXXRecordDecl *PBase = RL.getPrimaryBase()) { // Seek non-virtual primary base root. @@ -2275,7 +2264,7 @@ void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD, else break; } - ContainingType = llvm::DICompositeType( + ContainingType = cast<llvm::DICompositeType>( getOrCreateType(QualType(PBase->getTypeForDecl(), 0), getOrCreateFile(RD->getLocation()))); } else if (RD->isDynamicClass()) @@ -2285,29 +2274,29 @@ void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD, } /// CreateMemberType - Create new member and increase Offset by FType's size. -llvm::DIType CGDebugInfo::CreateMemberType(llvm::DIFile Unit, QualType FType, - StringRef Name, uint64_t *Offset) { - llvm::DIType FieldTy = CGDebugInfo::getOrCreateType(FType, Unit); +llvm::DIType *CGDebugInfo::CreateMemberType(llvm::DIFile *Unit, QualType FType, + StringRef Name, uint64_t *Offset) { + llvm::DIType *FieldTy = CGDebugInfo::getOrCreateType(FType, Unit); uint64_t FieldSize = CGM.getContext().getTypeSize(FType); unsigned FieldAlign = CGM.getContext().getTypeAlign(FType); - llvm::DIType Ty = DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize, - FieldAlign, *Offset, 0, FieldTy); + llvm::DIType *Ty = DBuilder.createMemberType(Unit, Name, Unit, 0, FieldSize, + FieldAlign, *Offset, 0, FieldTy); *Offset += FieldSize; return Ty; } -void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, - llvm::DIFile Unit, - StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &FDContext, - llvm::DIArray &TParamsArray, +void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit, + StringRef &Name, + StringRef &LinkageName, + llvm::DIScope *&FDContext, + llvm::DINodeArray &TParamsArray, unsigned &Flags) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); Name = getFunctionName(FD); // Use mangled name as linkage name for C/C++ functions. if (FD->hasPrototype()) { LinkageName = CGM.getMangledName(GD); - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; } // No need to replicate the linkage name if it isn't different from the // subprogram name, no need to have it at all unless coverage is enabled or @@ -2330,10 +2319,10 @@ void CGDebugInfo::collectFunctionDeclProps(GlobalDecl GD, } } -void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit, +void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit, unsigned &LineNo, QualType &T, StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &VDContext) { + llvm::DIScope *&VDContext) { Unit = getOrCreateFile(VD->getLocation()); LineNo = getLineNumber(VD->getLocation()); @@ -2362,19 +2351,27 @@ void CGDebugInfo::collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit, // FIXME: Generalize this for even non-member global variables where the // declaration and definition may have different lexical decl contexts, once // we have support for emitting declarations of (non-member) global variables. - VDContext = getContextDescriptor( - dyn_cast<Decl>(VD->isStaticDataMember() ? VD->getLexicalDeclContext() - : VD->getDeclContext())); -} - -llvm::DISubprogram + const DeclContext *DC = VD->isStaticDataMember() ? VD->getLexicalDeclContext() + : VD->getDeclContext(); + // When a record type contains an in-line initialization of a static data + // member, and the record type is marked as __declspec(dllexport), an implicit + // definition of the member will be created in the record context. DWARF + // doesn't seem to have a nice way to describe this in a form that consumers + // are likely to understand, so fake the "normal" situation of a definition + // outside the class by putting it in the global scope. + if (DC->isRecord()) + DC = CGM.getContext().getTranslationUnitDecl(); + VDContext = getContextDescriptor(dyn_cast<Decl>(DC)); +} + +llvm::DISubprogram * CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) { - llvm::DIArray TParamsArray; + llvm::DINodeArray TParamsArray; StringRef Name, LinkageName; unsigned Flags = 0; SourceLocation Loc = FD->getLocation(); - llvm::DIFile Unit = getOrCreateFile(Loc); - llvm::DIDescriptor DContext(Unit); + llvm::DIFile *Unit = getOrCreateFile(Loc); + llvm::DIScope *DContext = Unit; unsigned Line = getLineNumber(Loc); collectFunctionDeclProps(FD, Unit, Name, LinkageName, DContext, @@ -2386,35 +2383,31 @@ CGDebugInfo::getFunctionForwardDeclaration(const FunctionDecl *FD) { QualType FnType = CGM.getContext().getFunctionType(FD->getReturnType(), ArgTypes, FunctionProtoType::ExtProtoInfo()); - llvm::DISubprogram SP = - DBuilder.createTempFunctionFwdDecl(DContext, Name, LinkageName, Unit, Line, - getOrCreateFunctionType(FD, FnType, Unit), - !FD->isExternallyVisible(), - false /*declaration*/, 0, Flags, - CGM.getLangOpts().Optimize, nullptr, - TParamsArray, getFunctionDeclaration(FD)); + llvm::DISubprogram *SP = DBuilder.createTempFunctionFwdDecl( + DContext, Name, LinkageName, Unit, Line, + getOrCreateFunctionType(FD, FnType, Unit), !FD->isExternallyVisible(), + false /*declaration*/, 0, Flags, CGM.getLangOpts().Optimize, nullptr, + TParamsArray.get(), getFunctionDeclaration(FD)); const FunctionDecl *CanonDecl = cast<FunctionDecl>(FD->getCanonicalDecl()); - FwdDeclReplaceMap.emplace_back( - std::piecewise_construct, std::make_tuple(CanonDecl), - std::make_tuple(static_cast<llvm::Metadata *>(SP))); + FwdDeclReplaceMap.emplace_back(std::piecewise_construct, + std::make_tuple(CanonDecl), + std::make_tuple(SP)); return SP; } -llvm::DIGlobalVariable +llvm::DIGlobalVariable * CGDebugInfo::getGlobalVariableForwardDeclaration(const VarDecl *VD) { QualType T; StringRef Name, LinkageName; SourceLocation Loc = VD->getLocation(); - llvm::DIFile Unit = getOrCreateFile(Loc); - llvm::DIDescriptor DContext(Unit); + llvm::DIFile *Unit = getOrCreateFile(Loc); + llvm::DIScope *DContext = Unit; unsigned Line = getLineNumber(Loc); collectVarDeclProps(VD, Unit, Line, T, Name, LinkageName, DContext); - llvm::DIGlobalVariable GV = - DBuilder.createTempGlobalVariableFwdDecl(DContext, Name, LinkageName, Unit, - Line, getOrCreateType(T, Unit), - !VD->isExternallyVisible(), - nullptr, nullptr); + auto *GV = DBuilder.createTempGlobalVariableFwdDecl( + DContext, Name, LinkageName, Unit, Line, getOrCreateType(T, Unit), + !VD->isExternallyVisible(), nullptr, nullptr); FwdDeclReplaceMap.emplace_back( std::piecewise_construct, std::make_tuple(cast<VarDecl>(VD->getCanonicalDecl())), @@ -2422,7 +2415,7 @@ CGDebugInfo::getGlobalVariableForwardDeclaration(const VarDecl *VD) { return GV; } -llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { +llvm::DINode *CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { // We only need a declaration (not a definition) of the type - so use whatever // we would otherwise do to get a type for a pointee. (forward declarations in // limited debug info, full definitions (if the type definition is available) @@ -2433,7 +2426,7 @@ llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { auto I = DeclCache.find(D->getCanonicalDecl()); if (I != DeclCache.end()) - return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(I->second)); + return dyn_cast_or_null<llvm::DINode>(I->second); // No definition for now. Emit a forward definition that might be // merged with a potential upcoming definition. @@ -2442,59 +2435,55 @@ llvm::DIDescriptor CGDebugInfo::getDeclarationOrDefinition(const Decl *D) { else if (const auto *VD = dyn_cast<VarDecl>(D)) return getGlobalVariableForwardDeclaration(VD); - return llvm::DIDescriptor(); + return nullptr; } /// getFunctionDeclaration - Return debug info descriptor to describe method /// declaration for the given method definition. -llvm::DISubprogram CGDebugInfo::getFunctionDeclaration(const Decl *D) { +llvm::DISubprogram *CGDebugInfo::getFunctionDeclaration(const Decl *D) { if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly) - return llvm::DISubprogram(); + return nullptr; const FunctionDecl *FD = dyn_cast<FunctionDecl>(D); if (!FD) - return llvm::DISubprogram(); + return nullptr; // Setup context. - llvm::DIScope S = getContextDescriptor(cast<Decl>(D->getDeclContext())); + auto *S = getContextDescriptor(cast<Decl>(D->getDeclContext())); auto MI = SPCache.find(FD->getCanonicalDecl()); if (MI == SPCache.end()) { if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD->getCanonicalDecl())) { - llvm::DICompositeType T(S); - llvm::DISubprogram SP = - CreateCXXMemberFunction(MD, getOrCreateFile(MD->getLocation()), T); - return SP; + return CreateCXXMemberFunction(MD, getOrCreateFile(MD->getLocation()), + cast<llvm::DICompositeType>(S)); } } if (MI != SPCache.end()) { - llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(MI->second)); - if (SP.isSubprogram() && !SP.isDefinition()) + auto *SP = dyn_cast_or_null<llvm::DISubprogram>(MI->second); + if (SP && !SP->isDefinition()) return SP; } for (auto NextFD : FD->redecls()) { auto MI = SPCache.find(NextFD->getCanonicalDecl()); if (MI != SPCache.end()) { - llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(MI->second)); - if (SP.isSubprogram() && !SP.isDefinition()) + auto *SP = dyn_cast_or_null<llvm::DISubprogram>(MI->second); + if (SP && !SP->isDefinition()) return SP; } } - return llvm::DISubprogram(); + return nullptr; } -// getOrCreateFunctionType - Construct DIType. If it is a c++ method, include +// getOrCreateFunctionType - Construct type. If it is a c++ method, include // implicit parameter "this". -llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, - QualType FnType, - llvm::DIFile F) { +llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D, + QualType FnType, + llvm::DIFile *F) { if (!D || DebugKind <= CodeGenOptions::DebugLineTablesOnly) - // Create fake but valid subroutine type. Otherwise - // llvm::DISubprogram::Verify() would return false, and - // subprogram DIE will miss DW_AT_decl_file and - // DW_AT_decl_line fields. + // Create fake but valid subroutine type. Otherwise -verify would fail, and + // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields. return DBuilder.createSubroutineType(F, DBuilder.getOrCreateTypeArray(None)); @@ -2515,11 +2504,10 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, Elts.push_back(getOrCreateType(ResultTy, F)); // "self" pointer is always first argument. QualType SelfDeclTy = OMethod->getSelfDecl()->getType(); - llvm::DIType SelfTy = getOrCreateType(SelfDeclTy, F); - Elts.push_back(CreateSelfType(SelfDeclTy, SelfTy)); + Elts.push_back(CreateSelfType(SelfDeclTy, getOrCreateType(SelfDeclTy, F))); // "_cmd" pointer is always second argument. - llvm::DIType CmdTy = getOrCreateType(OMethod->getCmdDecl()->getType(), F); - Elts.push_back(DBuilder.createArtificialType(CmdTy)); + Elts.push_back(DBuilder.createArtificialType( + getOrCreateType(OMethod->getCmdDecl()->getType(), F))); // Get rest of the arguments. for (const auto *PI : OMethod->params()) Elts.push_back(getOrCreateType(PI->getType(), F)); @@ -2527,7 +2515,7 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, if (OMethod->isVariadic()) Elts.push_back(DBuilder.createUnspecifiedParameter()); - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(Elts); return DBuilder.createSubroutineType(F, EltTypeArray); } @@ -2541,11 +2529,11 @@ llvm::DICompositeType CGDebugInfo::getOrCreateFunctionType(const Decl *D, for (unsigned i = 0, e = FPT->getNumParams(); i != e; ++i) EltTys.push_back(getOrCreateType(FPT->getParamType(i), F)); EltTys.push_back(DBuilder.createUnspecifiedParameter()); - llvm::DITypeArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); + llvm::DITypeRefArray EltTypeArray = DBuilder.getOrCreateTypeArray(EltTys); return DBuilder.createSubroutineType(F, EltTypeArray); } - return llvm::DICompositeType(getOrCreateType(FnType, F)); + return cast<llvm::DISubroutineType>(getOrCreateType(FnType, F)); } /// EmitFunctionStart - Constructs the debug code for entering a function. @@ -2562,20 +2550,19 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, bool HasDecl = (D != nullptr); unsigned Flags = 0; - llvm::DIFile Unit = getOrCreateFile(Loc); - llvm::DIDescriptor FDContext(Unit); - llvm::DIArray TParamsArray; + llvm::DIFile *Unit = getOrCreateFile(Loc); + llvm::DIScope *FDContext = Unit; + llvm::DINodeArray TParamsArray; if (!HasDecl) { // Use llvm function name. LinkageName = Fn->getName(); } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) { - // If there is a DISubprogram for this function available then use it. + // If there is a subprogram for this function available then use it. auto FI = SPCache.find(FD->getCanonicalDecl()); if (FI != SPCache.end()) { - llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(FI->second)); - if (SP.isSubprogram() && llvm::DISubprogram(SP).isDefinition()) { - llvm::MDNode *SPN = SP; - LexicalBlockStack.emplace_back(SPN); + auto *SP = dyn_cast_or_null<llvm::DISubprogram>(FI->second); + if (SP && SP->isDefinition()) { + LexicalBlockStack.emplace_back(SP); RegionMap[D].reset(SP); return; } @@ -2584,17 +2571,17 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, TParamsArray, Flags); } else if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D)) { Name = getObjCMethodName(OMD); - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; } else { // Use llvm function name. Name = Fn->getName(); - Flags |= llvm::DIDescriptor::FlagPrototyped; + Flags |= llvm::DINode::FlagPrototyped; } if (!Name.empty() && Name[0] == '\01') Name = Name.substr(1); if (!HasDecl || D->isImplicit()) { - Flags |= llvm::DIDescriptor::FlagArtificial; + Flags |= llvm::DINode::FlagArtificial; // Artificial functions without a location should not silently reuse CurLoc. if (Loc.isInvalid()) CurLoc = SourceLocation(); @@ -2607,11 +2594,11 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, // FunctionDecls. When/if we fix this we can have FDContext be TheCU/null for // all subprograms instead of the actual context since subprogram definitions // are emitted as CU level entities by the backend. - llvm::DISubprogram SP = DBuilder.createFunction( + llvm::DISubprogram *SP = DBuilder.createFunction( FDContext, Name, LinkageName, Unit, LineNo, getOrCreateFunctionType(D, FnType, Unit), Fn->hasInternalLinkage(), true /*definition*/, ScopeLine, Flags, CGM.getLangOpts().Optimize, Fn, - TParamsArray, getFunctionDeclaration(D)); + TParamsArray.get(), getFunctionDeclaration(D)); // We might get here with a VarDecl in the case we're generating // code for the initialization of globals. Do not record these decls // as they will overwrite the actual VarDecl Decl in the cache. @@ -2619,8 +2606,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, DeclCache[D->getCanonicalDecl()].reset(static_cast<llvm::Metadata *>(SP)); // Push the function onto the lexical block stack. - llvm::MDNode *SPN = SP; - LexicalBlockStack.emplace_back(SPN); + LexicalBlockStack.emplace_back(SP); if (HasDecl) RegionMap[D].reset(SP); @@ -2629,8 +2615,7 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, SourceLocation Loc, /// EmitLocation - Emit metadata to indicate a change in line/column /// information in the source file. If the location is invalid, the /// previous location will be reused. -void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc, - bool ForceColumnInfo) { +void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc) { // Update our current location setLocation(Loc); @@ -2639,7 +2624,7 @@ void CGDebugInfo::EmitLocation(CGBuilderTy &Builder, SourceLocation Loc, llvm::MDNode *Scope = LexicalBlockStack.back(); Builder.SetCurrentDebugLocation(llvm::DebugLoc::get( - getLineNumber(CurLoc), getColumnNumber(CurLoc, ForceColumnInfo), Scope)); + getLineNumber(CurLoc), getColumnNumber(CurLoc), Scope)); } /// CreateLexicalBlock - Creates a new lexical block node and pushes it on @@ -2648,11 +2633,9 @@ void CGDebugInfo::CreateLexicalBlock(SourceLocation Loc) { llvm::MDNode *Back = nullptr; if (!LexicalBlockStack.empty()) Back = LexicalBlockStack.back().get(); - llvm::DIDescriptor D = DBuilder.createLexicalBlock( - llvm::DIDescriptor(Back), getOrCreateFile(CurLoc), getLineNumber(CurLoc), - getColumnNumber(CurLoc)); - llvm::MDNode *DN = D; - LexicalBlockStack.emplace_back(DN); + LexicalBlockStack.emplace_back(DBuilder.createLexicalBlock( + cast<llvm::DIScope>(Back), getOrCreateFile(CurLoc), getLineNumber(CurLoc), + getColumnNumber(CurLoc))); } /// EmitLexicalBlockStart - Constructs the debug code for entering a declarative @@ -2705,15 +2688,15 @@ void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) { // EmitTypeForVarWithBlocksAttr - Build up structure info for the byref. // See BuildByRefType. -llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, - uint64_t *XOffset) { +llvm::DIType *CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, + uint64_t *XOffset) { SmallVector<llvm::Metadata *, 5> EltTys; QualType FType; uint64_t FieldSize, FieldOffset; unsigned FieldAlign; - llvm::DIFile Unit = getOrCreateFile(VD->getLocation()); + llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); QualType Type = VD->getType(); FieldOffset = 0; @@ -2760,7 +2743,7 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, } FType = Type; - llvm::DIType FieldTy = getOrCreateType(FType, Unit); + llvm::DIType *FieldTy = getOrCreateType(FType, Unit); FieldSize = CGM.getContext().getTypeSize(FType); FieldAlign = CGM.getContext().toBits(Align); @@ -2770,16 +2753,16 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const VarDecl *VD, EltTys.push_back(FieldTy); FieldOffset += FieldSize; - llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys); + llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys); - unsigned Flags = llvm::DIDescriptor::FlagBlockByrefStruct; + unsigned Flags = llvm::DINode::FlagBlockByrefStruct; return DBuilder.createStructType(Unit, "", Unit, 0, FieldOffset, 0, Flags, - llvm::DIType(), Elements); + nullptr, Elements); } /// EmitDeclare - Emit local variable declaration debug info. -void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, +void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::Tag Tag, llvm::Value *Storage, unsigned ArgNo, CGBuilderTy &Builder) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); @@ -2788,10 +2771,10 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, bool Unwritten = VD->isImplicit() || (isa<Decl>(VD->getDeclContext()) && cast<Decl>(VD->getDeclContext())->isImplicit()); - llvm::DIFile Unit; + llvm::DIFile *Unit = nullptr; if (!Unwritten) Unit = getOrCreateFile(VD->getLocation()); - llvm::DIType Ty; + llvm::DIType *Ty; uint64_t XOffset = 0; if (VD->hasAttr<BlocksAttr>()) Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset); @@ -2810,58 +2793,64 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, Line = getLineNumber(VD->getLocation()); Column = getColumnNumber(VD->getLocation()); } + SmallVector<int64_t, 9> Expr; unsigned Flags = 0; if (VD->isImplicit()) - Flags |= llvm::DIDescriptor::FlagArtificial; + Flags |= llvm::DINode::FlagArtificial; // If this is the first argument and it is implicit then // give it an object pointer flag. // FIXME: There has to be a better way to do this, but for static // functions there won't be an implicit param at arg1 and // otherwise it is 'self' or 'this'. if (isa<ImplicitParamDecl>(VD) && ArgNo == 1) - Flags |= llvm::DIDescriptor::FlagObjectPointer; + Flags |= llvm::DINode::FlagObjectPointer; if (llvm::Argument *Arg = dyn_cast<llvm::Argument>(Storage)) if (Arg->getType()->isPointerTy() && !Arg->hasByValAttr() && !VD->getType()->isPointerType()) - Flags |= llvm::DIDescriptor::FlagIndirectVariable; + Expr.push_back(llvm::dwarf::DW_OP_deref); - llvm::MDNode *Scope = LexicalBlockStack.back(); + auto *Scope = cast<llvm::DIScope>(LexicalBlockStack.back()); StringRef Name = VD->getName(); if (!Name.empty()) { if (VD->hasAttr<BlocksAttr>()) { CharUnits offset = CharUnits::fromQuantity(32); - SmallVector<int64_t, 9> addr; - addr.push_back(llvm::dwarf::DW_OP_plus); + Expr.push_back(llvm::dwarf::DW_OP_plus); // offset of __forwarding field offset = CGM.getContext().toCharUnitsFromBits( CGM.getTarget().getPointerWidth(0)); - addr.push_back(offset.getQuantity()); - addr.push_back(llvm::dwarf::DW_OP_deref); - addr.push_back(llvm::dwarf::DW_OP_plus); + Expr.push_back(offset.getQuantity()); + Expr.push_back(llvm::dwarf::DW_OP_deref); + Expr.push_back(llvm::dwarf::DW_OP_plus); // offset of x field offset = CGM.getContext().toCharUnitsFromBits(XOffset); - addr.push_back(offset.getQuantity()); + Expr.push_back(offset.getQuantity()); // Create the descriptor for the variable. - llvm::DIVariable D = DBuilder.createLocalVariable( - Tag, llvm::DIDescriptor(Scope), VD->getName(), Unit, Line, Ty, ArgNo); + auto *D = DBuilder.createLocalVariable(Tag, Scope, VD->getName(), Unit, + Line, Ty, ArgNo); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = - DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), - Builder.GetInsertBlock()); - Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope)); + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), + llvm::DebugLoc::get(Line, Column, Scope), + Builder.GetInsertBlock()); return; } else if (isa<VariableArrayType>(VD->getType())) - Flags |= llvm::DIDescriptor::FlagIndirectVariable; + Expr.push_back(llvm::dwarf::DW_OP_deref); } else if (const RecordType *RT = dyn_cast<RecordType>(VD->getType())) { // If VD is an anonymous union then Storage represents value for // all union fields. const RecordDecl *RD = cast<RecordDecl>(RT->getDecl()); if (RD->isUnion() && RD->isAnonymousStructOrUnion()) { + // GDB has trouble finding local variables in anonymous unions, so we emit + // artifical local variables for each of the members. + // + // FIXME: Remove this code as soon as GDB supports this. + // The debug info verifier in LLVM operates based on the assumption that a + // variable has the same size as its storage and we had to disable the check + // for artificial variables. for (const auto *Field : RD->fields()) { - llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit); + llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit); StringRef FieldName = Field->getName(); // Ignore unnamed fields. Do not ignore unnamed records. @@ -2869,28 +2858,28 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, llvm::dwarf::LLVMConstants Tag, continue; // Use VarDecl's Tag, Scope and Line number. - llvm::DIVariable D = DBuilder.createLocalVariable( - Tag, llvm::DIDescriptor(Scope), FieldName, Unit, Line, FieldTy, - CGM.getLangOpts().Optimize, Flags, ArgNo); + auto *D = DBuilder.createLocalVariable( + Tag, Scope, FieldName, Unit, Line, FieldTy, + CGM.getLangOpts().Optimize, Flags | llvm::DINode::FlagArtificial, + ArgNo); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = DBuilder.insertDeclare( - Storage, D, DBuilder.createExpression(), Builder.GetInsertBlock()); - Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope)); + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), + llvm::DebugLoc::get(Line, Column, Scope), + Builder.GetInsertBlock()); } - return; } } // Create the descriptor for the variable. - llvm::DIVariable D = DBuilder.createLocalVariable( - Tag, llvm::DIDescriptor(Scope), Name, Unit, Line, Ty, - CGM.getLangOpts().Optimize, Flags, ArgNo); + auto *D = + DBuilder.createLocalVariable(Tag, Scope, Name, Unit, Line, Ty, + CGM.getLangOpts().Optimize, Flags, ArgNo); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = DBuilder.insertDeclare( - Storage, D, DBuilder.createExpression(), Builder.GetInsertBlock()); - Call->setDebugLoc(llvm::DebugLoc::get(Line, Column, Scope)); + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(Expr), + llvm::DebugLoc::get(Line, Column, Scope), + Builder.GetInsertBlock()); } void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD, @@ -2906,9 +2895,9 @@ void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD, /// never happen though, since creating a type for the implicit self /// argument implies that we already parsed the interface definition /// and the ivar declarations in the implementation. -llvm::DIType CGDebugInfo::CreateSelfType(const QualType &QualTy, - llvm::DIType Ty) { - llvm::DIType CachedTy = getTypeOrNull(QualTy); +llvm::DIType *CGDebugInfo::CreateSelfType(const QualType &QualTy, + llvm::DIType *Ty) { + llvm::DIType *CachedTy = getTypeOrNull(QualTy); if (CachedTy) Ty = CachedTy; return DBuilder.createObjectPointerType(Ty); @@ -2926,8 +2915,8 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable( bool isByRef = VD->hasAttr<BlocksAttr>(); uint64_t XOffset = 0; - llvm::DIFile Unit = getOrCreateFile(VD->getLocation()); - llvm::DIType Ty; + llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); + llvm::DIType *Ty; if (isByRef) Ty = EmitTypeForVarWithBlocksAttr(VD, &XOffset); else @@ -2968,19 +2957,19 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable( } // Create the descriptor for the variable. - llvm::DIVariable D = - DBuilder.createLocalVariable(llvm::dwarf::DW_TAG_auto_variable, - llvm::DIDescriptor(LexicalBlockStack.back()), - VD->getName(), Unit, Line, Ty); + auto *D = DBuilder.createLocalVariable( + llvm::dwarf::DW_TAG_auto_variable, + cast<llvm::DILocalScope>(LexicalBlockStack.back()), VD->getName(), Unit, + Line, Ty); // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *Call = InsertPoint ? - DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), - InsertPoint) - : DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), - Builder.GetInsertBlock()); - Call->setDebugLoc( - llvm::DebugLoc::get(Line, Column, LexicalBlockStack.back())); + auto DL = llvm::DebugLoc::get(Line, Column, LexicalBlockStack.back()); + if (InsertPoint) + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), DL, + InsertPoint); + else + DBuilder.insertDeclare(Storage, D, DBuilder.createExpression(addr), DL, + Builder.GetInsertBlock()); } /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument @@ -3013,7 +3002,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, // Collect some general information about the block's location. SourceLocation loc = blockDecl->getCaretLocation(); - llvm::DIFile tunit = getOrCreateFile(loc); + llvm::DIFile *tunit = getOrCreateFile(loc); unsigned line = getLineNumber(loc); unsigned column = getColumnNumber(loc); @@ -3096,7 +3085,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, const VarDecl *variable = capture->getVariable(); StringRef name = variable->getName(); - llvm::DIType fieldType; + llvm::DIType *fieldType; if (capture->isByRef()) { TypeInfo PtrInfo = C.getTypeInfo(C.VoidPtrTy); @@ -3118,70 +3107,67 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, llvm::raw_svector_ostream(typeName) << "__block_literal_" << CGM.getUniqueBlockCount(); - llvm::DIArray fieldsArray = DBuilder.getOrCreateArray(fields); + llvm::DINodeArray fieldsArray = DBuilder.getOrCreateArray(fields); - llvm::DIType type = - DBuilder.createStructType(tunit, typeName.str(), tunit, line, - CGM.getContext().toBits(block.BlockSize), - CGM.getContext().toBits(block.BlockAlign), 0, - llvm::DIType(), fieldsArray); + llvm::DIType *type = DBuilder.createStructType( + tunit, typeName.str(), tunit, line, + CGM.getContext().toBits(block.BlockSize), + CGM.getContext().toBits(block.BlockAlign), 0, nullptr, fieldsArray); type = DBuilder.createPointerType(type, CGM.PointerWidthInBits); // Get overall information about the block. - unsigned flags = llvm::DIDescriptor::FlagArtificial; - llvm::MDNode *scope = LexicalBlockStack.back(); + unsigned flags = llvm::DINode::FlagArtificial; + auto *scope = cast<llvm::DILocalScope>(LexicalBlockStack.back()); // Create the descriptor for the parameter. - llvm::DIVariable debugVar = DBuilder.createLocalVariable( - llvm::dwarf::DW_TAG_arg_variable, llvm::DIDescriptor(scope), - Arg->getName(), tunit, line, type, CGM.getLangOpts().Optimize, flags, - ArgNo); + auto *debugVar = DBuilder.createLocalVariable( + llvm::dwarf::DW_TAG_arg_variable, scope, Arg->getName(), tunit, line, + type, CGM.getLangOpts().Optimize, flags, ArgNo); if (LocalAddr) { // Insert an llvm.dbg.value into the current block. - llvm::Instruction *DbgVal = DBuilder.insertDbgValueIntrinsic( + DBuilder.insertDbgValueIntrinsic( LocalAddr, 0, debugVar, DBuilder.createExpression(), - Builder.GetInsertBlock()); - DbgVal->setDebugLoc(llvm::DebugLoc::get(line, column, scope)); + llvm::DebugLoc::get(line, column, scope), Builder.GetInsertBlock()); } // Insert an llvm.dbg.declare into the current block. - llvm::Instruction *DbgDecl = DBuilder.insertDeclare( - Arg, debugVar, DBuilder.createExpression(), Builder.GetInsertBlock()); - DbgDecl->setDebugLoc(llvm::DebugLoc::get(line, column, scope)); + DBuilder.insertDeclare(Arg, debugVar, DBuilder.createExpression(), + llvm::DebugLoc::get(line, column, scope), + Builder.GetInsertBlock()); } /// If D is an out-of-class definition of a static data member of a class, find /// its corresponding in-class declaration. -llvm::DIDerivedType +llvm::DIDerivedType * CGDebugInfo::getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D) { if (!D->isStaticDataMember()) - return llvm::DIDerivedType(); + return nullptr; + auto MI = StaticDataMemberCache.find(D->getCanonicalDecl()); if (MI != StaticDataMemberCache.end()) { assert(MI->second && "Static data member declaration should still exist"); - return llvm::DIDerivedType(cast<llvm::MDNode>(MI->second)); + return cast<llvm::DIDerivedType>(MI->second); } // If the member wasn't found in the cache, lazily construct and add it to the // type (used when a limited form of the type is emitted). auto DC = D->getDeclContext(); - llvm::DICompositeType Ctxt(getContextDescriptor(cast<Decl>(DC))); + auto *Ctxt = + cast<llvm::DICompositeType>(getContextDescriptor(cast<Decl>(DC))); return CreateRecordStaticField(D, Ctxt, cast<RecordDecl>(DC)); } /// Recursively collect all of the member fields of a global anonymous decl and /// create static variables for them. The first time this is called it needs /// to be on a union and then from there we can have additional unnamed fields. -llvm::DIGlobalVariable -CGDebugInfo::CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit, - unsigned LineNo, StringRef LinkageName, - llvm::GlobalVariable *Var, - llvm::DIDescriptor DContext) { - llvm::DIGlobalVariable GV; +llvm::DIGlobalVariable *CGDebugInfo::CollectAnonRecordDecls( + const RecordDecl *RD, llvm::DIFile *Unit, unsigned LineNo, + StringRef LinkageName, llvm::GlobalVariable *Var, llvm::DIScope *DContext) { + llvm::DIGlobalVariable *GV = nullptr; for (const auto *Field : RD->fields()) { - llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit); + llvm::DIType *FieldTy = getOrCreateType(Field->getType(), Unit); StringRef FieldName = Field->getName(); // Ignore unnamed fields, but recurse into anonymous records. @@ -3193,9 +3179,9 @@ CGDebugInfo::CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit, continue; } // Use VarDecl's Tag, Scope and Line number. - GV = DBuilder.createGlobalVariable( - DContext, FieldName, LinkageName, Unit, LineNo, FieldTy, - Var->hasInternalLinkage(), Var, llvm::DIDerivedType()); + GV = DBuilder.createGlobalVariable(DContext, FieldName, LinkageName, Unit, + LineNo, FieldTy, + Var->hasInternalLinkage(), Var, nullptr); } return GV; } @@ -3205,8 +3191,8 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, const VarDecl *D) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); // Create global variable debug descriptor. - llvm::DIFile Unit; - llvm::DIDescriptor DContext; + llvm::DIFile *Unit = nullptr; + llvm::DIScope *DContext = nullptr; unsigned LineNo; StringRef DeclName, LinkageName; QualType T; @@ -3214,7 +3200,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var, // Attempt to store one global variable for the declaration - even if we // emit a lot of fields. - llvm::DIGlobalVariable GV; + llvm::DIGlobalVariable *GV = nullptr; // If this is an anonymous union then we'll want to emit a global // variable for each member of the anonymous union so that it's possible @@ -3238,16 +3224,18 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init) { assert(DebugKind >= CodeGenOptions::LimitedDebugInfo); // Create the descriptor for the variable. - llvm::DIFile Unit = getOrCreateFile(VD->getLocation()); + llvm::DIFile *Unit = getOrCreateFile(VD->getLocation()); StringRef Name = VD->getName(); - llvm::DIType Ty = getOrCreateType(VD->getType(), Unit); + llvm::DIType *Ty = getOrCreateType(VD->getType(), Unit); if (const EnumConstantDecl *ECD = dyn_cast<EnumConstantDecl>(VD)) { const EnumDecl *ED = cast<EnumDecl>(ECD->getDeclContext()); assert(isa<EnumType>(ED->getTypeForDecl()) && "Enum without EnumType?"); Ty = getOrCreateType(QualType(ED->getTypeForDecl(), 0), Unit); } - // Do not use DIGlobalVariable for enums. - if (Ty.getTag() == llvm::dwarf::DW_TAG_enumeration_type) + // Do not use global variables for enums. + // + // FIXME: why not? + if (Ty->getTag() == llvm::dwarf::DW_TAG_enumeration_type) return; // Do not emit separate definitions for function local const/statics. if (isa<FunctionDecl>(VD->getDeclContext())) @@ -3263,7 +3251,7 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, return; } - llvm::DIDescriptor DContext = + llvm::DIScope *DContext = getContextDescriptor(dyn_cast<Decl>(VD->getDeclContext())); auto &GV = DeclCache[VD]; @@ -3274,9 +3262,9 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, true, Init, getOrCreateStaticDataMemberDeclarationOrNull(VarD))); } -llvm::DIScope CGDebugInfo::getCurrentContextDescriptor(const Decl *D) { +llvm::DIScope *CGDebugInfo::getCurrentContextDescriptor(const Decl *D) { if (!LexicalBlockStack.empty()) - return llvm::DIScope(LexicalBlockStack.back()); + return LexicalBlockStack.back(); return getContextDescriptor(D); } @@ -3297,21 +3285,21 @@ void CGDebugInfo::EmitUsingDecl(const UsingDecl &UD) { // Emitting one decl is sufficient - debuggers can detect that this is an // overloaded name & provide lookup for all the overloads. const UsingShadowDecl &USD = **UD.shadow_begin(); - if (llvm::DIDescriptor Target = + if (llvm::DINode *Target = getDeclarationOrDefinition(USD.getUnderlyingDecl())) DBuilder.createImportedDeclaration( getCurrentContextDescriptor(cast<Decl>(USD.getDeclContext())), Target, getLineNumber(USD.getLocation())); } -llvm::DIImportedEntity +llvm::DIImportedEntity * CGDebugInfo::EmitNamespaceAlias(const NamespaceAliasDecl &NA) { if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo) - return llvm::DIImportedEntity(nullptr); + return nullptr; auto &VH = NamespaceAliasCache[&NA]; if (VH) - return llvm::DIImportedEntity(cast<llvm::MDNode>(VH)); - llvm::DIImportedEntity R(nullptr); + return cast<llvm::DIImportedEntity>(VH); + llvm::DIImportedEntity *R; if (const NamespaceAliasDecl *Underlying = dyn_cast<NamespaceAliasDecl>(NA.getAliasedNamespace())) // This could cache & dedup here rather than relying on metadata deduping. @@ -3330,19 +3318,19 @@ CGDebugInfo::EmitNamespaceAlias(const NamespaceAliasDecl &NA) { /// getOrCreateNamesSpace - Return namespace descriptor for the given /// namespace decl. -llvm::DINameSpace +llvm::DINamespace * CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl) { NSDecl = NSDecl->getCanonicalDecl(); auto I = NameSpaceCache.find(NSDecl); if (I != NameSpaceCache.end()) - return llvm::DINameSpace(cast<llvm::MDNode>(I->second)); + return cast<llvm::DINamespace>(I->second); unsigned LineNo = getLineNumber(NSDecl->getLocation()); - llvm::DIFile FileD = getOrCreateFile(NSDecl->getLocation()); - llvm::DIDescriptor Context = - getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext())); - llvm::DINameSpace NS = - DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo); + llvm::DIFile *FileD = getOrCreateFile(NSDecl->getLocation()); + llvm::DIScope *Context = + getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext())); + llvm::DINamespace *NS = + DBuilder.createNameSpace(Context, NSDecl->getName(), FileD, LineNo); NameSpaceCache[NSDecl].reset(NS); return NS; } @@ -3352,28 +3340,28 @@ void CGDebugInfo::finalize() { // element and the size(), so don't cache/reference them. for (size_t i = 0; i != ObjCInterfaceCache.size(); ++i) { ObjCInterfaceCacheEntry E = ObjCInterfaceCache[i]; - E.Decl.replaceAllUsesWith(CGM.getLLVMContext(), - E.Type->getDecl()->getDefinition() - ? CreateTypeDefinition(E.Type, E.Unit) - : E.Decl); + llvm::DIType *Ty = E.Type->getDecl()->getDefinition() + ? CreateTypeDefinition(E.Type, E.Unit) + : E.Decl; + DBuilder.replaceTemporary(llvm::TempDIType(E.Decl), Ty); } for (auto p : ReplaceMap) { assert(p.second); - llvm::DIType Ty(cast<llvm::MDNode>(p.second)); - assert(Ty.isForwardDecl()); + auto *Ty = cast<llvm::DIType>(p.second); + assert(Ty->isForwardDecl()); auto it = TypeCache.find(p.first); assert(it != TypeCache.end()); assert(it->second); - llvm::DIType RepTy(cast<llvm::MDNode>(it->second)); - Ty.replaceAllUsesWith(CGM.getLLVMContext(), RepTy); + DBuilder.replaceTemporary(llvm::TempDIType(Ty), + cast<llvm::DIType>(it->second)); } for (const auto &p : FwdDeclReplaceMap) { assert(p.second); - llvm::DIDescriptor FwdDecl(cast<llvm::MDNode>(p.second)); + llvm::TempMDNode FwdDecl(cast<llvm::MDNode>(p.second)); llvm::Metadata *Repl; auto it = DeclCache.find(p.first); @@ -3385,15 +3373,14 @@ void CGDebugInfo::finalize() { else Repl = it->second; - FwdDecl.replaceAllUsesWith(CGM.getLLVMContext(), - llvm::DIDescriptor(cast<llvm::MDNode>(Repl))); + DBuilder.replaceTemporary(std::move(FwdDecl), cast<llvm::MDNode>(Repl)); } // We keep our own list of retained types, because we need to look // up the final type in the type cache. for (std::vector<void *>::const_iterator RI = RetainedTypes.begin(), RE = RetainedTypes.end(); RI != RE; ++RI) - DBuilder.retainType(llvm::DIType(cast<llvm::MDNode>(TypeCache[*RI]))); + DBuilder.retainType(cast<llvm::DIType>(TypeCache[*RI])); DBuilder.finalize(); } @@ -3401,7 +3388,8 @@ void CGDebugInfo::finalize() { void CGDebugInfo::EmitExplicitCastType(QualType Ty) { if (CGM.getCodeGenOpts().getDebugInfo() < CodeGenOptions::LimitedDebugInfo) return; - llvm::DIType DieTy = getOrCreateType(Ty, getOrCreateMainFile()); - // Don't ignore in case of explicit cast where it is referenced indirectly. - DBuilder.retainType(DieTy); + + if (auto *DieTy = getOrCreateType(Ty, getOrCreateMainFile())) + // Don't ignore in case of explicit cast where it is referenced indirectly. + DBuilder.retainType(DieTy); } diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h index 0be032c1d790..8509e0770db6 100644 --- a/lib/CodeGen/CGDebugInfo.h +++ b/lib/CodeGen/CGDebugInfo.h @@ -43,47 +43,50 @@ namespace CodeGen { class CodeGenFunction; class CGBlockInfo; -/// CGDebugInfo - This class gathers all debug information during compilation +/// \brief This class gathers all debug information during compilation /// and is responsible for emitting to llvm globals or pass directly to /// the backend. class CGDebugInfo { - friend class ArtificialLocation; + friend class ApplyDebugLocation; friend class SaveAndRestoreLocation; CodeGenModule &CGM; const CodeGenOptions::DebugInfoKind DebugKind; llvm::DIBuilder DBuilder; - llvm::DICompileUnit TheCU; + llvm::DICompileUnit *TheCU = nullptr; SourceLocation CurLoc; - llvm::DIType VTablePtrType; - llvm::DIType ClassTy; - llvm::DICompositeType ObjTy; - llvm::DIType SelTy; - llvm::DIType OCLImage1dDITy, OCLImage1dArrayDITy, OCLImage1dBufferDITy; - llvm::DIType OCLImage2dDITy, OCLImage2dArrayDITy; - llvm::DIType OCLImage3dDITy; - llvm::DIType OCLEventDITy; - llvm::DIType BlockLiteralGeneric; - - /// TypeCache - Cache of previously constructed Types. + llvm::DIType *VTablePtrType = nullptr; + llvm::DIType *ClassTy = nullptr; + llvm::DICompositeType *ObjTy = nullptr; + llvm::DIType *SelTy = nullptr; + llvm::DIType *OCLImage1dDITy = nullptr; + llvm::DIType *OCLImage1dArrayDITy = nullptr; + llvm::DIType *OCLImage1dBufferDITy = nullptr; + llvm::DIType *OCLImage2dDITy = nullptr; + llvm::DIType *OCLImage2dArrayDITy = nullptr; + llvm::DIType *OCLImage3dDITy = nullptr; + llvm::DIType *OCLEventDITy = nullptr; + llvm::DIType *BlockLiteralGeneric = nullptr; + + /// \brief Cache of previously constructed Types. llvm::DenseMap<const void *, llvm::TrackingMDRef> TypeCache; struct ObjCInterfaceCacheEntry { const ObjCInterfaceType *Type; - llvm::DIType Decl; - llvm::DIFile Unit; - ObjCInterfaceCacheEntry(const ObjCInterfaceType *Type, llvm::DIType Decl, - llvm::DIFile Unit) + llvm::DIType *Decl; + llvm::DIFile *Unit; + ObjCInterfaceCacheEntry(const ObjCInterfaceType *Type, llvm::DIType *Decl, + llvm::DIFile *Unit) : Type(Type), Decl(Decl), Unit(Unit) {} }; - /// ObjCInterfaceCache - Cache of previously constructed interfaces + /// \brief Cache of previously constructed interfaces /// which may change. llvm::SmallVector<ObjCInterfaceCacheEntry, 32> ObjCInterfaceCache; - /// RetainedTypes - list of interfaces we want to keep even if orphaned. + /// \brief list of interfaces we want to keep even if orphaned. std::vector<void *> RetainedTypes; - /// ReplaceMap - Cache of forward declared types to RAUW at the end of + /// \brief Cache of forward declared types to RAUW at the end of /// compilation. std::vector<std::pair<const TagType *, llvm::TrackingMDRef>> ReplaceMap; @@ -93,14 +96,14 @@ class CGDebugInfo { FwdDeclReplaceMap; // LexicalBlockStack - Keep track of our current nested lexical block. - std::vector<llvm::TrackingMDNodeRef> LexicalBlockStack; + std::vector<llvm::TypedTrackingMDRef<llvm::DIScope>> LexicalBlockStack; llvm::DenseMap<const Decl *, llvm::TrackingMDRef> RegionMap; // FnBeginRegionCount - Keep track of LexicalBlockStack counter at the // beginning of a function. This is used to pop unbalanced regions at // the end of a function. std::vector<unsigned> FnBeginRegionCount; - /// DebugInfoNames - This is a storage for names that are + /// \brief This is a storage for names that are /// constructed on demand. For example, C++ destructors, C++ operators etc.. llvm::BumpPtrAllocator DebugInfoNames; StringRef CWDName; @@ -117,95 +120,94 @@ class CGDebugInfo { /// Helper functions for getOrCreateType. unsigned Checksum(const ObjCInterfaceDecl *InterfaceDecl); - llvm::DIType CreateType(const BuiltinType *Ty); - llvm::DIType CreateType(const ComplexType *Ty); - llvm::DIType CreateQualifiedType(QualType Ty, llvm::DIFile Fg); - llvm::DIType CreateType(const TypedefType *Ty, llvm::DIFile Fg); - llvm::DIType CreateType(const TemplateSpecializationType *Ty, llvm::DIFile Fg); - llvm::DIType CreateType(const ObjCObjectPointerType *Ty, - llvm::DIFile F); - llvm::DIType CreateType(const PointerType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const BlockPointerType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const FunctionType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const RecordType *Tyg); - llvm::DIType CreateTypeDefinition(const RecordType *Ty); - llvm::DICompositeType CreateLimitedType(const RecordType *Ty); - void CollectContainingType(const CXXRecordDecl *RD, llvm::DICompositeType CT); - llvm::DIType CreateType(const ObjCInterfaceType *Ty, llvm::DIFile F); - llvm::DIType CreateTypeDefinition(const ObjCInterfaceType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const ObjCObjectType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const VectorType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const ArrayType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const RValueReferenceType *Ty, llvm::DIFile Unit); - llvm::DIType CreateType(const MemberPointerType *Ty, llvm::DIFile F); - llvm::DIType CreateType(const AtomicType *Ty, llvm::DIFile F); - llvm::DIType CreateEnumType(const EnumType *Ty); - llvm::DIType CreateTypeDefinition(const EnumType *Ty); - llvm::DIType CreateSelfType(const QualType &QualTy, llvm::DIType Ty); - llvm::DIType getTypeOrNull(const QualType); - llvm::DICompositeType getOrCreateMethodType(const CXXMethodDecl *Method, - llvm::DIFile F); - llvm::DICompositeType getOrCreateInstanceMethodType( - QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile Unit); - llvm::DICompositeType getOrCreateFunctionType(const Decl *D, QualType FnType, - llvm::DIFile F); - llvm::DIType getOrCreateVTablePtrType(llvm::DIFile F); - llvm::DINameSpace getOrCreateNameSpace(const NamespaceDecl *N); - llvm::DIType getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile F); - llvm::DIType CreatePointerLikeType(llvm::dwarf::Tag Tag, - const Type *Ty, QualType PointeeTy, - llvm::DIFile F); + llvm::DIType *CreateType(const BuiltinType *Ty); + llvm::DIType *CreateType(const ComplexType *Ty); + llvm::DIType *CreateQualifiedType(QualType Ty, llvm::DIFile *Fg); + llvm::DIType *CreateType(const TypedefType *Ty, llvm::DIFile *Fg); + llvm::DIType *CreateType(const TemplateSpecializationType *Ty, + llvm::DIFile *Fg); + llvm::DIType *CreateType(const ObjCObjectPointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const PointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const BlockPointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const FunctionType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const RecordType *Tyg); + llvm::DIType *CreateTypeDefinition(const RecordType *Ty); + llvm::DICompositeType *CreateLimitedType(const RecordType *Ty); + void CollectContainingType(const CXXRecordDecl *RD, + llvm::DICompositeType *CT); + llvm::DIType *CreateType(const ObjCInterfaceType *Ty, llvm::DIFile *F); + llvm::DIType *CreateTypeDefinition(const ObjCInterfaceType *Ty, + llvm::DIFile *F); + llvm::DIType *CreateType(const ObjCObjectType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const VectorType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const ArrayType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const LValueReferenceType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const RValueReferenceType *Ty, llvm::DIFile *Unit); + llvm::DIType *CreateType(const MemberPointerType *Ty, llvm::DIFile *F); + llvm::DIType *CreateType(const AtomicType *Ty, llvm::DIFile *F); + llvm::DIType *CreateEnumType(const EnumType *Ty); + llvm::DIType *CreateTypeDefinition(const EnumType *Ty); + llvm::DIType *CreateSelfType(const QualType &QualTy, llvm::DIType *Ty); + llvm::DIType *getTypeOrNull(const QualType); + llvm::DISubroutineType *getOrCreateMethodType(const CXXMethodDecl *Method, + llvm::DIFile *F); + llvm::DISubroutineType * + getOrCreateInstanceMethodType(QualType ThisPtr, const FunctionProtoType *Func, + llvm::DIFile *Unit); + llvm::DISubroutineType * + getOrCreateFunctionType(const Decl *D, QualType FnType, llvm::DIFile *F); + llvm::DIType *getOrCreateVTablePtrType(llvm::DIFile *F); + llvm::DINamespace *getOrCreateNameSpace(const NamespaceDecl *N); + llvm::DIType *getOrCreateTypeDeclaration(QualType PointeeTy, llvm::DIFile *F); + llvm::DIType *CreatePointerLikeType(llvm::dwarf::Tag Tag, const Type *Ty, + QualType PointeeTy, llvm::DIFile *F); llvm::Value *getCachedInterfaceTypeOrNull(const QualType Ty); - llvm::DIType getOrCreateStructPtrType(StringRef Name, llvm::DIType &Cache); + llvm::DIType *getOrCreateStructPtrType(StringRef Name, llvm::DIType *&Cache); - llvm::DISubprogram CreateCXXMemberFunction(const CXXMethodDecl *Method, - llvm::DIFile F, - llvm::DIType RecordTy); + llvm::DISubprogram *CreateCXXMemberFunction(const CXXMethodDecl *Method, + llvm::DIFile *F, + llvm::DIType *RecordTy); - void CollectCXXMemberFunctions(const CXXRecordDecl *Decl, llvm::DIFile F, + void CollectCXXMemberFunctions(const CXXRecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DIType T); + llvm::DIType *T); - void CollectCXXBases(const CXXRecordDecl *Decl, llvm::DIFile F, + void CollectCXXBases(const CXXRecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &EltTys, - llvm::DIType RecordTy); - - llvm::DIArray - CollectTemplateParams(const TemplateParameterList *TPList, - ArrayRef<TemplateArgument> TAList, - llvm::DIFile Unit); - llvm::DIArray - CollectFunctionTemplateParams(const FunctionDecl *FD, llvm::DIFile Unit); - llvm::DIArray + llvm::DIType *RecordTy); + + llvm::DINodeArray CollectTemplateParams(const TemplateParameterList *TPList, + ArrayRef<TemplateArgument> TAList, + llvm::DIFile *Unit); + llvm::DINodeArray CollectFunctionTemplateParams(const FunctionDecl *FD, + llvm::DIFile *Unit); + llvm::DINodeArray CollectCXXTemplateParams(const ClassTemplateSpecializationDecl *TS, - llvm::DIFile F); + llvm::DIFile *F); - llvm::DIType createFieldType(StringRef name, QualType type, - uint64_t sizeInBitsOverride, SourceLocation loc, - AccessSpecifier AS, - uint64_t offsetInBits, - llvm::DIFile tunit, - llvm::DIScope scope, - const RecordDecl* RD = nullptr); + llvm::DIType *createFieldType(StringRef name, QualType type, + uint64_t sizeInBitsOverride, SourceLocation loc, + AccessSpecifier AS, uint64_t offsetInBits, + llvm::DIFile *tunit, llvm::DIScope *scope, + const RecordDecl *RD = nullptr); // Helpers for collecting fields of a record. void CollectRecordLambdaFields(const CXXRecordDecl *CXXDecl, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DIType RecordTy); - llvm::DIDerivedType CreateRecordStaticField(const VarDecl *Var, - llvm::DIType RecordTy, - const RecordDecl* RD); + llvm::DIType *RecordTy); + llvm::DIDerivedType *CreateRecordStaticField(const VarDecl *Var, + llvm::DIType *RecordTy, + const RecordDecl *RD); void CollectRecordNormalField(const FieldDecl *Field, uint64_t OffsetInBits, - llvm::DIFile F, + llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DIType RecordTy, const RecordDecl *RD); - void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile F, + llvm::DIType *RecordTy, const RecordDecl *RD); + void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &E, - llvm::DICompositeType RecordTy); + llvm::DICompositeType *RecordTy); - void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile F, + void CollectVTableInfo(const CXXRecordDecl *Decl, llvm::DIFile *F, SmallVectorImpl<llvm::Metadata *> &EltTys); // CreateLexicalBlock - Create a new lexical block node and push it on @@ -218,20 +220,15 @@ public: void finalize(); - /// setLocation - Update the current source location. If \arg loc is + /// \brief Update the current source location. If \arg loc is /// invalid it is ignored. void setLocation(SourceLocation Loc); - /// getLocation - Return the current source location. - SourceLocation getLocation() const { return CurLoc; } - - /// EmitLocation - Emit metadata to indicate a change in line/column + /// \brief Emit metadata to indicate a change in line/column /// information in the source file. - /// \param ForceColumnInfo Assume DebugColumnInfo option is true. - void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc, - bool ForceColumnInfo = false); + void EmitLocation(CGBuilderTy &Builder, SourceLocation Loc); - /// EmitFunctionStart - Emit a call to llvm.dbg.function.start to indicate + /// \brief Emit a call to llvm.dbg.function.start to indicate /// start of a new function. /// \param Loc The location of the function header. /// \param ScopeLoc The location of the function body. @@ -240,23 +237,23 @@ public: QualType FnType, llvm::Function *Fn, CGBuilderTy &Builder); - /// EmitFunctionEnd - Constructs the debug code for exiting a function. + /// \brief Constructs the debug code for exiting a function. void EmitFunctionEnd(CGBuilderTy &Builder); - /// EmitLexicalBlockStart - Emit metadata to indicate the beginning of a + /// \brief Emit metadata to indicate the beginning of a /// new lexical block and push the block onto the stack. void EmitLexicalBlockStart(CGBuilderTy &Builder, SourceLocation Loc); - /// EmitLexicalBlockEnd - Emit metadata to indicate the end of a new lexical + /// \brief Emit metadata to indicate the end of a new lexical /// block and pop the current block. void EmitLexicalBlockEnd(CGBuilderTy &Builder, SourceLocation Loc); - /// EmitDeclareOfAutoVariable - Emit call to llvm.dbg.declare for an automatic + /// \brief Emit call to llvm.dbg.declare for an automatic /// variable declaration. void EmitDeclareOfAutoVariable(const VarDecl *Decl, llvm::Value *AI, CGBuilderTy &Builder); - /// EmitDeclareOfBlockDeclRefVariable - Emit call to llvm.dbg.declare for an + /// \brief Emit call to llvm.dbg.declare for an /// imported variable declaration in a block. void EmitDeclareOfBlockDeclRefVariable(const VarDecl *variable, llvm::Value *storage, @@ -264,12 +261,12 @@ public: const CGBlockInfo &blockInfo, llvm::Instruction *InsertPoint = 0); - /// EmitDeclareOfArgVariable - Emit call to llvm.dbg.declare for an argument + /// \brief Emit call to llvm.dbg.declare for an argument /// variable declaration. void EmitDeclareOfArgVariable(const VarDecl *Decl, llvm::Value *AI, unsigned ArgNo, CGBuilderTy &Builder); - /// EmitDeclareOfBlockLiteralArgVariable - Emit call to + /// \brief Emit call to /// llvm.dbg.declare for the block-literal argument to a block /// invocation function. void EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block, @@ -277,31 +274,30 @@ public: llvm::Value *LocalAddr, CGBuilderTy &Builder); - /// EmitGlobalVariable - Emit information about a global variable. + /// \brief Emit information about a global variable. void EmitGlobalVariable(llvm::GlobalVariable *GV, const VarDecl *Decl); - /// EmitGlobalVariable - Emit global variable's debug info. + /// \brief Emit global variable's debug info. void EmitGlobalVariable(const ValueDecl *VD, llvm::Constant *Init); - /// \brief - Emit C++ using directive. + /// \brief Emit C++ using directive. void EmitUsingDirective(const UsingDirectiveDecl &UD); - /// EmitExplicitCastType - Emit the type explicitly casted to. + /// \brief Emit the type explicitly casted to. void EmitExplicitCastType(QualType Ty); - /// \brief - Emit C++ using declaration. + /// \brief Emit C++ using declaration. void EmitUsingDecl(const UsingDecl &UD); - /// \brief - Emit C++ namespace alias. - llvm::DIImportedEntity EmitNamespaceAlias(const NamespaceAliasDecl &NA); + /// \brief Emit C++ namespace alias. + llvm::DIImportedEntity *EmitNamespaceAlias(const NamespaceAliasDecl &NA); - /// getOrCreateRecordType - Emit record type's standalone debug info. - llvm::DIType getOrCreateRecordType(QualType Ty, SourceLocation L); + /// \brief Emit record type's standalone debug info. + llvm::DIType *getOrCreateRecordType(QualType Ty, SourceLocation L); - /// getOrCreateInterfaceType - Emit an objective c interface type standalone + /// \brief Emit an objective c interface type standalone /// debug info. - llvm::DIType getOrCreateInterfaceType(QualType Ty, - SourceLocation Loc); + llvm::DIType *getOrCreateInterfaceType(QualType Ty, SourceLocation Loc); void completeType(const EnumDecl *ED); void completeType(const RecordDecl *RD); @@ -311,133 +307,129 @@ public: void completeTemplateDefinition(const ClassTemplateSpecializationDecl &SD); private: - /// EmitDeclare - Emit call to llvm.dbg.declare for a variable declaration. + /// \brief Emit call to llvm.dbg.declare for a variable declaration. /// Tag accepts custom types DW_TAG_arg_variable and DW_TAG_auto_variable, /// otherwise would be of type llvm::dwarf::Tag. - void EmitDeclare(const VarDecl *decl, llvm::dwarf::LLVMConstants Tag, - llvm::Value *AI, unsigned ArgNo, CGBuilderTy &Builder); + void EmitDeclare(const VarDecl *decl, llvm::dwarf::Tag Tag, llvm::Value *AI, + unsigned ArgNo, CGBuilderTy &Builder); // EmitTypeForVarWithBlocksAttr - Build up structure info for the byref. // See BuildByRefType. - llvm::DIType EmitTypeForVarWithBlocksAttr(const VarDecl *VD, - uint64_t *OffSet); + llvm::DIType *EmitTypeForVarWithBlocksAttr(const VarDecl *VD, + uint64_t *OffSet); - /// getContextDescriptor - Get context info for the decl. - llvm::DIScope getContextDescriptor(const Decl *Decl); + /// \brief Get context info for the decl. + llvm::DIScope *getContextDescriptor(const Decl *Decl); - llvm::DIScope getCurrentContextDescriptor(const Decl *Decl); + llvm::DIScope *getCurrentContextDescriptor(const Decl *Decl); /// \brief Create a forward decl for a RecordType in a given context. - llvm::DICompositeType getOrCreateRecordFwdDecl(const RecordType *, - llvm::DIDescriptor); - - /// createContextChain - Create a set of decls for the context chain. - llvm::DIDescriptor createContextChain(const Decl *Decl); + llvm::DICompositeType *getOrCreateRecordFwdDecl(const RecordType *, + llvm::DIScope *); - /// getCurrentDirname - Return current directory name. + /// \brief Return current directory name. StringRef getCurrentDirname(); - /// CreateCompileUnit - Create new compile unit. + /// \brief Create new compile unit. void CreateCompileUnit(); - /// getOrCreateFile - Get the file debug info descriptor for the input + /// \brief Get the file debug info descriptor for the input /// location. - llvm::DIFile getOrCreateFile(SourceLocation Loc); + llvm::DIFile *getOrCreateFile(SourceLocation Loc); - /// getOrCreateMainFile - Get the file info for main compile unit. - llvm::DIFile getOrCreateMainFile(); + /// \brief Get the file info for main compile unit. + llvm::DIFile *getOrCreateMainFile(); - /// getOrCreateType - Get the type from the cache or create a new type if + /// \brief Get the type from the cache or create a new type if /// necessary. - llvm::DIType getOrCreateType(QualType Ty, llvm::DIFile Fg); + llvm::DIType *getOrCreateType(QualType Ty, llvm::DIFile *Fg); - /// getOrCreateLimitedType - Get the type from the cache or create a new + /// \brief Get the type from the cache or create a new /// partial type if necessary. - llvm::DIType getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile F); + llvm::DIType *getOrCreateLimitedType(const RecordType *Ty, llvm::DIFile *F); - /// CreateTypeNode - Create type metadata for a source language type. - llvm::DIType CreateTypeNode(QualType Ty, llvm::DIFile Fg); + /// \brief Create type metadata for a source language type. + llvm::DIType *CreateTypeNode(QualType Ty, llvm::DIFile *Fg); - /// getObjCInterfaceDecl - return the underlying ObjCInterfaceDecl + /// \brief return the underlying ObjCInterfaceDecl /// if Ty is an ObjCInterface or a pointer to one. ObjCInterfaceDecl* getObjCInterfaceDecl(QualType Ty); - /// CreateMemberType - Create new member and increase Offset by FType's size. - llvm::DIType CreateMemberType(llvm::DIFile Unit, QualType FType, - StringRef Name, uint64_t *Offset); + /// \brief Create new member and increase Offset by FType's size. + llvm::DIType *CreateMemberType(llvm::DIFile *Unit, QualType FType, + StringRef Name, uint64_t *Offset); /// \brief Retrieve the DIDescriptor, if any, for the canonical form of this /// declaration. - llvm::DIDescriptor getDeclarationOrDefinition(const Decl *D); + llvm::DINode *getDeclarationOrDefinition(const Decl *D); - /// getFunctionDeclaration - Return debug info descriptor to describe method + /// \brief Return debug info descriptor to describe method /// declaration for the given method definition. - llvm::DISubprogram getFunctionDeclaration(const Decl *D); + llvm::DISubprogram *getFunctionDeclaration(const Decl *D); /// Return debug info descriptor to describe in-class static data member /// declaration for the given out-of-class definition. - llvm::DIDerivedType + llvm::DIDerivedType * getOrCreateStaticDataMemberDeclarationOrNull(const VarDecl *D); - /// \brief Create a DISubprogram describing the forward + /// \brief Create a subprogram describing the forward /// decalration represented in the given FunctionDecl. - llvm::DISubprogram getFunctionForwardDeclaration(const FunctionDecl *FD); + llvm::DISubprogram *getFunctionForwardDeclaration(const FunctionDecl *FD); - /// \brief Create a DIGlobalVariable describing the forward - /// decalration represented in the given VarDecl. - llvm::DIGlobalVariable getGlobalVariableForwardDeclaration(const VarDecl *VD); + /// \brief Create a global variable describing the forward decalration + /// represented in the given VarDecl. + llvm::DIGlobalVariable * + getGlobalVariableForwardDeclaration(const VarDecl *VD); /// Return a global variable that represents one of the collection of /// global variables created for an anonmyous union. - llvm::DIGlobalVariable - CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile Unit, unsigned LineNo, - StringRef LinkageName, llvm::GlobalVariable *Var, - llvm::DIDescriptor DContext); + llvm::DIGlobalVariable * + CollectAnonRecordDecls(const RecordDecl *RD, llvm::DIFile *Unit, + unsigned LineNo, StringRef LinkageName, + llvm::GlobalVariable *Var, llvm::DIScope *DContext); - /// getFunctionName - Get function name for the given FunctionDecl. If the + /// \brief Get function name for the given FunctionDecl. If the /// name is constructed on demand (e.g. C++ destructor) then the name /// is stored on the side. StringRef getFunctionName(const FunctionDecl *FD); - /// getObjCMethodName - Returns the unmangled name of an Objective-C method. + /// \brief Returns the unmangled name of an Objective-C method. /// This is the display name for the debugging info. StringRef getObjCMethodName(const ObjCMethodDecl *FD); - /// getSelectorName - Return selector name. This is used for debugging + /// \brief Return selector name. This is used for debugging /// info. StringRef getSelectorName(Selector S); - /// getClassName - Get class name including template argument list. + /// \brief Get class name including template argument list. StringRef getClassName(const RecordDecl *RD); - /// getVTableName - Get vtable name for the given Class. + /// \brief Get vtable name for the given Class. StringRef getVTableName(const CXXRecordDecl *Decl); - /// getLineNumber - Get line number for the location. If location is invalid + /// \brief Get line number for the location. If location is invalid /// then use current location. unsigned getLineNumber(SourceLocation Loc); - /// getColumnNumber - Get column number for the location. If location is + /// \brief Get column number for the location. If location is /// invalid then use current location. /// \param Force Assume DebugColumnInfo option is true. unsigned getColumnNumber(SourceLocation Loc, bool Force=false); /// \brief Collect various properties of a FunctionDecl. /// \param GD A GlobalDecl whose getDecl() must return a FunctionDecl. - void collectFunctionDeclProps(GlobalDecl GD, - llvm::DIFile Unit, + void collectFunctionDeclProps(GlobalDecl GD, llvm::DIFile *Unit, StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &FDContext, - llvm::DIArray &TParamsArray, + llvm::DIScope *&FDContext, + llvm::DINodeArray &TParamsArray, unsigned &Flags); /// \brief Collect various properties of a VarDecl. - void collectVarDeclProps(const VarDecl *VD, llvm::DIFile &Unit, - unsigned &LineNo, QualType &T, - StringRef &Name, StringRef &LinkageName, - llvm::DIDescriptor &VDContext); + void collectVarDeclProps(const VarDecl *VD, llvm::DIFile *&Unit, + unsigned &LineNo, QualType &T, StringRef &Name, + StringRef &LinkageName, llvm::DIScope *&VDContext); - /// internString - Allocate a copy of \p A using the DebugInfoNames allocator + /// \brief Allocate a copy of \p A using the DebugInfoNames allocator /// and return a reference to it. If multiple arguments are given the strings /// are concatenated. StringRef internString(StringRef A, StringRef B = StringRef()) { @@ -448,33 +440,60 @@ private: } }; +/// \brief A scoped helper to set the current debug location to the specified +/// location or preferred location of the specified Expr. class ApplyDebugLocation { -protected: +private: + void init(SourceLocation TemporaryLocation, bool DefaultToEmpty = false); + ApplyDebugLocation(CodeGenFunction &CGF, bool DefaultToEmpty, + SourceLocation TemporaryLocation); + llvm::DebugLoc OriginalLocation; CodeGenFunction &CGF; - public: - ApplyDebugLocation(CodeGenFunction &CGF, - SourceLocation TemporaryLocation = SourceLocation(), - bool ForceColumnInfo = false); + + /// \brief Set the location to the (valid) TemporaryLocation. + ApplyDebugLocation(CodeGenFunction &CGF, SourceLocation TemporaryLocation); + ApplyDebugLocation(CodeGenFunction &CGF, const Expr *E); ApplyDebugLocation(CodeGenFunction &CGF, llvm::DebugLoc Loc); + ~ApplyDebugLocation(); -}; -/// ArtificialLocation - An RAII object that temporarily switches to -/// an artificial debug location that has a valid scope, but no line -/// information. This is useful when emitting compiler-generated -/// helper functions that have no source location associated with -/// them. The DWARF specification allows the compiler to use the -/// special line number 0 to indicate code that can not be attributed -/// to any source location. -/// -/// This is necessary because passing an empty SourceLocation to -/// CGDebugInfo::setLocation() will result in the last valid location -/// being reused. -class ArtificialLocation : public ApplyDebugLocation { -public: - ArtificialLocation(CodeGenFunction &CGF); + /// \brief Apply TemporaryLocation if it is valid. Otherwise switch to an + /// artificial debug location that has a valid scope, but no line information. + /// + /// Artificial locations are useful when emitting compiler-generated helper + /// functions that have no source location associated with them. The DWARF + /// specification allows the compiler to use the special line number 0 to + /// indicate code that can not be attributed to any source location. Note that + /// passing an empty SourceLocation to CGDebugInfo::setLocation() will result + /// in the last valid location being reused. + static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF) { + return ApplyDebugLocation(CGF, false, SourceLocation()); + } + /// \brief Apply TemporaryLocation if it is valid. Otherwise switch to an + /// artificial debug location that has a valid scope, but no line information. + static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, + SourceLocation TemporaryLocation) { + return ApplyDebugLocation(CGF, false, TemporaryLocation); + } + + /// \brief Set the IRBuilder to not attach debug locations. Note that passing + /// an empty SourceLocation to CGDebugInfo::setLocation() will result in the + /// last valid location being reused. Note that all instructions that do not + /// have a location at the beginning of a function are counted towards to + /// funciton prologue. + static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF) { + return ApplyDebugLocation(CGF, true, SourceLocation()); + } + + /// \brief Apply TemporaryLocation if it is valid. Otherwise set the IRBuilder + /// to not attach debug locations. + static ApplyDebugLocation CreateDefaultEmpty(CodeGenFunction &CGF, + SourceLocation TemporaryLocation) { + return ApplyDebugLocation(CGF, true, TemporaryLocation); + } + }; diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp index 15a1a7fb5f12..579a04145567 100644 --- a/lib/CodeGen/CGDecl.cpp +++ b/lib/CodeGen/CGDecl.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGCleanup.h" #include "CGDebugInfo.h" #include "CGOpenCLRuntime.h" #include "CodeGenModule.h" @@ -34,6 +35,7 @@ using namespace CodeGen; void CodeGenFunction::EmitDecl(const Decl &D) { switch (D.getKind()) { case Decl::TranslationUnit: + case Decl::ExternCContext: case Decl::Namespace: case Decl::UnresolvedUsingTypename: case Decl::ClassTemplateSpecialization: @@ -154,6 +156,8 @@ static std::string getStaticDeclName(CodeGenModule &CGM, const VarDecl &D) { assert(!D.isExternallyVisible() && "name shouldn't matter"); std::string ContextName; const DeclContext *DC = D.getDeclContext(); + if (auto *CD = dyn_cast<CapturedDecl>(DC)) + DC = cast<DeclContext>(CD->getNonClosureContext()); if (const auto *FD = dyn_cast<FunctionDecl>(DC)) ContextName = CGM.getMangledName(FD); else if (const auto *BD = dyn_cast<BlockDecl>(DC)) @@ -206,6 +210,9 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl( GV->setAlignment(getContext().getDeclAlign(&D).getQuantity()); setGlobalVisibility(GV, &D); + if (supportsCOMDAT() && GV->isWeakForLinker()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); + if (D.getTLSKind()) setTLSMode(GV, D); @@ -512,10 +519,7 @@ namespace { : Addr(addr), Size(size) {} void Emit(CodeGenFunction &CGF, Flags flags) override { - llvm::Value *castAddr = CGF.Builder.CreateBitCast(Addr, CGF.Int8PtrTy); - CGF.Builder.CreateCall2(CGF.CGM.getLLVMLifetimeEndFn(), - Size, castAddr) - ->setDoesNotThrow(); + CGF.EmitLifetimeEnd(Size, Addr); } }; } @@ -631,8 +635,9 @@ void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D, if (capturedByInit) { // We can use a simple GEP for this because it can't have been // moved yet. - tempLV.setAddress(Builder.CreateStructGEP(tempLV.getAddress(), - getByRefValueLLVMField(cast<VarDecl>(D)))); + tempLV.setAddress(Builder.CreateStructGEP( + nullptr, tempLV.getAddress(), + getByRefValueLLVMField(cast<VarDecl>(D)).second)); } llvm::PointerType *ty @@ -793,8 +798,9 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc, // If necessary, get a pointer to the element and emit it. if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt)) - emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i), - isVolatile, Builder); + emitStoresForInitAfterMemset( + Elt, Builder.CreateConstGEP2_32(Init->getType(), Loc, 0, i), + isVolatile, Builder); } return; } @@ -807,8 +813,9 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc, // If necessary, get a pointer to the element and emit it. if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt)) - emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i), - isVolatile, Builder); + emitStoresForInitAfterMemset( + Elt, Builder.CreateConstGEP2_32(Init->getType(), Loc, 0, i), + isVolatile, Builder); } } @@ -833,21 +840,6 @@ static bool shouldUseMemSetPlusStoresToInitialize(llvm::Constant *Init, canEmitInitWithFewStoresAfterMemset(Init, StoreBudget); } -/// Should we use the LLVM lifetime intrinsics for the given local variable? -static bool shouldUseLifetimeMarkers(CodeGenFunction &CGF, const VarDecl &D, - unsigned Size) { - // For now, only in optimized builds. - if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) - return false; - - // Limit the size of marked objects to 32 bytes. We don't want to increase - // compile time by marking tiny objects. - unsigned SizeThreshold = 32; - - return Size > SizeThreshold; -} - - /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a /// variable declaration with auto, register, or no storage class specifier. /// These turn into simple stack objects, or GlobalValues depending on target. @@ -857,6 +849,38 @@ void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) { EmitAutoVarCleanups(emission); } +/// Emit a lifetime.begin marker if some criteria are satisfied. +/// \return a pointer to the temporary size Value if a marker was emitted, null +/// otherwise +llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size, + llvm::Value *Addr) { + // For now, only in optimized builds. + if (CGM.getCodeGenOpts().OptimizationLevel == 0) + return nullptr; + + // Disable lifetime markers in msan builds. + // FIXME: Remove this when msan works with lifetime markers. + if (getLangOpts().Sanitize.has(SanitizerKind::Memory)) + return nullptr; + + llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size); + llvm::Value *Args[] = { + SizeV, + new llvm::BitCastInst(Addr, Int8PtrTy, "", Builder.GetInsertBlock())}; + llvm::CallInst *C = llvm::CallInst::Create(CGM.getLLVMLifetimeStartFn(), Args, + "", Builder.GetInsertBlock()); + C->setDoesNotThrow(); + return SizeV; +} + +void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) { + llvm::Value *Args[] = {Size, new llvm::BitCastInst(Addr, Int8PtrTy, "", + Builder.GetInsertBlock())}; + llvm::CallInst *C = llvm::CallInst::Create(CGM.getLLVMLifetimeEndFn(), Args, + "", Builder.GetInsertBlock()); + C->setDoesNotThrow(); +} + /// EmitAutoVarAlloca - Emit the alloca and debug information for a /// local variable. Does not emit initialization or destruction. CodeGenFunction::AutoVarEmission @@ -952,13 +976,8 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { // Emit a lifetime intrinsic if meaningful. There's no point // in doing this if we don't have a valid insertion point (?). uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy); - if (HaveInsertPoint() && shouldUseLifetimeMarkers(*this, D, size)) { - llvm::Value *sizeV = llvm::ConstantInt::get(Int64Ty, size); - - emission.SizeForLifetimeMarkers = sizeV; - llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy); - Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), sizeV, castAddr) - ->setDoesNotThrow(); + if (HaveInsertPoint()) { + emission.SizeForLifetimeMarkers = EmitLifetimeStart(size, Alloc); } else { assert(!emission.useLifetimeMarkers()); } @@ -971,7 +990,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) { llvm::Value *Stack = CreateTempAlloca(Int8PtrTy, "saved_stack"); llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::stacksave); - llvm::Value *V = Builder.CreateCall(F); + llvm::Value *V = Builder.CreateCall(F, {}); Builder.CreateStore(V, Stack); @@ -1087,7 +1106,7 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) { if (emission.wasEmittedAsGlobal()) return; const VarDecl &D = *emission.Variable; - ApplyDebugLocation DL(*this, D.getLocation()); + auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, D.getLocation()); QualType type = D.getType(); // If this local has an initializer, emit it now. @@ -1304,6 +1323,8 @@ void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) { EHStack.pushCleanup<CallLifetimeEnd>(NormalCleanup, emission.getAllocatedAddress(), emission.getSizeForLifetimeMarkers()); + EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin()); + cleanup.setLifetimeMarker(); } // Check the type for a cleanup. diff --git a/lib/CodeGen/CGDeclCXX.cpp b/lib/CodeGen/CGDeclCXX.cpp index 19e4bdd00cfe..06d157bd82e7 100644 --- a/lib/CodeGen/CGDeclCXX.cpp +++ b/lib/CodeGen/CGDeclCXX.cpp @@ -139,9 +139,32 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D, const Expr *Init = D.getInit(); QualType T = D.getType(); + // The address space of a static local variable (DeclPtr) may be different + // from the address space of the "this" argument of the constructor. In that + // case, we need an addrspacecast before calling the constructor. + // + // struct StructWithCtor { + // __device__ StructWithCtor() {...} + // }; + // __device__ void foo() { + // __shared__ StructWithCtor s; + // ... + // } + // + // For example, in the above CUDA code, the static local variable s has a + // "shared" address space qualifier, but the constructor of StructWithCtor + // expects "this" in the "generic" address space. + unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T); + unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace(); + if (ActualAddrSpace != ExpectedAddrSpace) { + llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T); + llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace); + DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy); + } + if (!T->isReferenceType()) { if (getLangOpts().OpenMP && D.hasAttr<OMPThreadPrivateDeclAttr>()) - (void)CGM.getOpenMPRuntime().EmitOMPThreadPrivateVarDefinition( + (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition( &D, DeclPtr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(), PerformInit, this); if (PerformInit) @@ -236,6 +259,8 @@ llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction( Fn->setSection(Section); } + SetLLVMFunctionAttributes(nullptr, getTypes().arrangeNullaryFunction(), Fn); + Fn->setCallingConv(getRuntimeCC()); if (!getLangOpts().Exceptions) @@ -267,15 +292,7 @@ void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D, addUsedGlobal(PtrArray); // If the GV is already in a comdat group, then we have to join it. - llvm::Comdat *C = GV->getComdat(); - - // LinkOnce and Weak linkage are lowered down to a single-member comdat group. - // Make an explicit group so we can join it. - if (!C && (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage())) { - C = TheModule.getOrInsertComdat(GV->getName()); - GV->setComdat(C); - } - if (C) + if (llvm::Comdat *C = GV->getComdat()) PtrArray->setComdat(C); } @@ -283,6 +300,11 @@ void CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, llvm::GlobalVariable *Addr, bool PerformInit) { + // Check if we've already initialized this decl. + auto I = DelayedCXXInitPosition.find(D); + if (I != DelayedCXXInitPosition.end() && I->second == ~0U) + return; + llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false); SmallString<256> FnName; { @@ -312,11 +334,9 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, CXXThreadLocalInitVars.push_back(Addr); } else if (PerformInit && ISA) { EmitPointerToInitFunc(D, Addr, Fn, ISA); - DelayedCXXInitPosition.erase(D); } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) { OrderGlobalInits Key(IPA->getPriority(), PrioritizedCXXGlobalInits.size()); PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn)); - DelayedCXXInitPosition.erase(D); } else if (isTemplateInstantiation(D->getTemplateSpecializationKind())) { // C++ [basic.start.init]p2: // Definitions of explicitly specialized class template static data @@ -331,24 +351,24 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D, // minor startup time optimization. In the MS C++ ABI, there are no guard // variables, so this COMDAT key is required for correctness. AddGlobalCtor(Fn, 65535, COMDATKey); - DelayedCXXInitPosition.erase(D); } else if (D->hasAttr<SelectAnyAttr>()) { // SelectAny globals will be comdat-folded. Put the initializer into a // COMDAT group associated with the global, so the initializers get folded // too. AddGlobalCtor(Fn, 65535, COMDATKey); - DelayedCXXInitPosition.erase(D); } else { - llvm::DenseMap<const Decl *, unsigned>::iterator I = - DelayedCXXInitPosition.find(D); + I = DelayedCXXInitPosition.find(D); // Re-do lookup in case of re-hash. if (I == DelayedCXXInitPosition.end()) { CXXGlobalInits.push_back(Fn); - } else { - assert(CXXGlobalInits[I->second] == nullptr); + } else if (I->second != ~0U) { + assert(I->second < CXXGlobalInits.size() && + CXXGlobalInits[I->second] == nullptr); CXXGlobalInits[I->second] = Fn; - DelayedCXXInitPosition.erase(I); } } + + // Remember that we already emitted the initializer for this global. + DelayedCXXInitPosition[D] = ~0U; } void CodeGenModule::EmitCXXThreadLocalInitFunc() { @@ -411,7 +431,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() { // priority emitted above. FileName = llvm::sys::path::filename(MainFile->getName()); } else { - FileName = SmallString<128>("<null>"); + FileName = "<null>"; } for (size_t i = 0; i < FileName.size(); ++i) { @@ -477,11 +497,11 @@ CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn, ArrayRef<llvm::Function *> Decls, llvm::GlobalVariable *Guard) { { - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(GlobalDecl(), getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(), FunctionArgList()); // Emit an artificial location for this function. - ArtificialLocation AL(*this); + auto AL = ApplyDebugLocation::CreateArtificial(*this); llvm::BasicBlock *ExitBlock = nullptr; if (Guard) { @@ -528,11 +548,11 @@ void CodeGenFunction::GenerateCXXGlobalDtorsFunc(llvm::Function *Fn, const std::vector<std::pair<llvm::WeakVH, llvm::Constant*> > &DtorsAndObjects) { { - ApplyDebugLocation NL(*this); + auto NL = ApplyDebugLocation::CreateEmpty(*this); StartFunction(GlobalDecl(), getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(), FunctionArgList()); // Emit an artificial location for this function. - ArtificialLocation AL(*this); + auto AL = ApplyDebugLocation::CreateArtificial(*this); // Emit the dtors, in reverse order from construction. for (unsigned i = 0, e = DtorsAndObjects.size(); i != e; ++i) { diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp index cb8eb8fa490c..d9a3f0b252a5 100644 --- a/lib/CodeGen/CGException.cpp +++ b/lib/CodeGen/CGException.cpp @@ -16,23 +16,18 @@ #include "CGCleanup.h" #include "CGObjCRuntime.h" #include "TargetInfo.h" +#include "clang/AST/Mangle.h" #include "clang/AST/StmtCXX.h" #include "clang/AST/StmtObjC.h" +#include "clang/AST/StmtVisitor.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/Support/SaveAndRestore.h" using namespace clang; using namespace CodeGen; -static llvm::Constant *getAllocateExceptionFn(CodeGenModule &CGM) { - // void *__cxa_allocate_exception(size_t thrown_size); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.Int8PtrTy, CGM.SizeTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_allocate_exception"); -} - static llvm::Constant *getFreeExceptionFn(CodeGenModule &CGM) { // void __cxa_free_exception(void *thrown_exception); @@ -42,44 +37,6 @@ static llvm::Constant *getFreeExceptionFn(CodeGenModule &CGM) { return CGM.CreateRuntimeFunction(FTy, "__cxa_free_exception"); } -static llvm::Constant *getThrowFn(CodeGenModule &CGM) { - // void __cxa_throw(void *thrown_exception, std::type_info *tinfo, - // void (*dest) (void *)); - - llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.Int8PtrTy, CGM.Int8PtrTy }; - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, Args, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_throw"); -} - -static llvm::Constant *getGetExceptionPtrFn(CodeGenModule &CGM) { - // void *__cxa_get_exception_ptr(void*); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_get_exception_ptr"); -} - -static llvm::Constant *getBeginCatchFn(CodeGenModule &CGM) { - // void *__cxa_begin_catch(void*); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_begin_catch"); -} - -static llvm::Constant *getEndCatchFn(CodeGenModule &CGM) { - // void __cxa_end_catch(); - - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false); - - return CGM.CreateRuntimeFunction(FTy, "__cxa_end_catch"); -} - static llvm::Constant *getUnexpectedFn(CodeGenModule &CGM) { // void __cxa_call_unexpected(void *thrown_exception); @@ -89,23 +46,30 @@ static llvm::Constant *getUnexpectedFn(CodeGenModule &CGM) { return CGM.CreateRuntimeFunction(FTy, "__cxa_call_unexpected"); } -static llvm::Constant *getTerminateFn(CodeGenModule &CGM) { +llvm::Constant *CodeGenModule::getTerminateFn() { // void __terminate(); llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false); + llvm::FunctionType::get(VoidTy, /*IsVarArgs=*/false); StringRef name; // In C++, use std::terminate(). - if (CGM.getLangOpts().CPlusPlus) - name = "_ZSt9terminatev"; // FIXME: mangling! - else if (CGM.getLangOpts().ObjC1 && - CGM.getLangOpts().ObjCRuntime.hasTerminate()) + if (getLangOpts().CPlusPlus && + getTarget().getCXXABI().isItaniumFamily()) { + name = "_ZSt9terminatev"; + } else if (getLangOpts().CPlusPlus && + getTarget().getCXXABI().isMicrosoft()) { + if (getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2015)) + name = "__std_terminate"; + else + name = "\01?terminate@@YAXXZ"; + } else if (getLangOpts().ObjC1 && + getLangOpts().ObjCRuntime.hasTerminate()) name = "objc_terminate"; else name = "abort"; - return CGM.CreateRuntimeFunction(FTy, name); + return CreateRuntimeFunction(FTy, name); } static llvm::Constant *getCatchallRethrowFn(CodeGenModule &CGM, @@ -126,7 +90,12 @@ namespace { // This function must have prototype void(void*). const char *CatchallRethrowFn; - static const EHPersonality &get(CodeGenModule &CGM); + static const EHPersonality &get(CodeGenModule &CGM, + const FunctionDecl *FD); + static const EHPersonality &get(CodeGenFunction &CGF) { + return get(CGF.CGM, dyn_cast_or_null<FunctionDecl>(CGF.CurCodeDecl)); + } + static const EHPersonality GNU_C; static const EHPersonality GNU_C_SJLJ; static const EHPersonality GNU_C_SEH; @@ -137,6 +106,9 @@ namespace { static const EHPersonality GNU_CPlusPlus; static const EHPersonality GNU_CPlusPlus_SJLJ; static const EHPersonality GNU_CPlusPlus_SEH; + static const EHPersonality MSVC_except_handler; + static const EHPersonality MSVC_C_specific_handler; + static const EHPersonality MSVC_CxxFrameHandler3; }; } @@ -159,6 +131,12 @@ const EHPersonality EHPersonality::GNU_ObjCXX = { "__gnustep_objcxx_personality_v0", nullptr }; const EHPersonality EHPersonality::GNUstep_ObjC = { "__gnustep_objc_personality_v0", nullptr }; +const EHPersonality +EHPersonality::MSVC_except_handler = { "_except_handler3", nullptr }; +const EHPersonality +EHPersonality::MSVC_C_specific_handler = { "__C_specific_handler", nullptr }; +const EHPersonality +EHPersonality::MSVC_CxxFrameHandler3 = { "__CxxFrameHandler3", nullptr }; /// On Win64, use libgcc's SEH personality function. We fall back to dwarf on /// other platforms, unless the user asked for SjLj exceptions. @@ -231,9 +209,29 @@ static const EHPersonality &getObjCXXPersonality(const llvm::Triple &T, llvm_unreachable("bad runtime kind"); } -const EHPersonality &EHPersonality::get(CodeGenModule &CGM) { +static const EHPersonality &getSEHPersonalityMSVC(const llvm::Triple &T) { + if (T.getArch() == llvm::Triple::x86) + return EHPersonality::MSVC_except_handler; + return EHPersonality::MSVC_C_specific_handler; +} + +const EHPersonality &EHPersonality::get(CodeGenModule &CGM, + const FunctionDecl *FD) { const llvm::Triple &T = CGM.getTarget().getTriple(); const LangOptions &L = CGM.getLangOpts(); + + // Try to pick a personality function that is compatible with MSVC if we're + // not compiling Obj-C. Obj-C users better have an Obj-C runtime that supports + // the GCC-style personality function. + if (T.isWindowsMSVCEnvironment() && !L.ObjC1) { + if (L.SjLjExceptions) + return EHPersonality::GNU_CPlusPlus_SJLJ; + else if (FD && FD->usesSEHTry()) + return getSEHPersonalityMSVC(T); + else + return EHPersonality::MSVC_CxxFrameHandler3; + } + if (L.CPlusPlus && L.ObjC1) return getObjCXXPersonality(T, L); else if (L.CPlusPlus) @@ -318,7 +316,7 @@ void CodeGenModule::SimplifyPersonality() { if (!LangOpts.ObjCRuntime.isNeXTFamily()) return; - const EHPersonality &ObjCXX = EHPersonality::get(*this); + const EHPersonality &ObjCXX = EHPersonality::get(*this, /*FD=*/nullptr); const EHPersonality &CXX = getCXXPersonality(getTarget().getTriple(), LangOpts); if (&ObjCXX == &CXX) @@ -369,17 +367,16 @@ namespace { // differs from EmitAnyExprToMem only in that, if a final copy-ctor // call is required, an exception within that copy ctor causes // std::terminate to be invoked. -static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e, - llvm::Value *addr) { +void CodeGenFunction::EmitAnyExprToExn(const Expr *e, llvm::Value *addr) { // Make sure the exception object is cleaned up if there's an // exception during initialization. - CGF.pushFullExprCleanup<FreeException>(EHCleanup, addr); - EHScopeStack::stable_iterator cleanup = CGF.EHStack.stable_begin(); + pushFullExprCleanup<FreeException>(EHCleanup, addr); + EHScopeStack::stable_iterator cleanup = EHStack.stable_begin(); // __cxa_allocate_exception returns a void*; we need to cast this // to the appropriate type for the object. - llvm::Type *ty = CGF.ConvertTypeForMem(e->getType())->getPointerTo(); - llvm::Value *typedAddr = CGF.Builder.CreateBitCast(addr, ty); + llvm::Type *ty = ConvertTypeForMem(e->getType())->getPointerTo(); + llvm::Value *typedAddr = Builder.CreateBitCast(addr, ty); // FIXME: this isn't quite right! If there's a final unelided call // to a copy constructor, then according to [except.terminate]p1 we @@ -388,11 +385,11 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *e, // evaluated but before the exception is caught. But the best way // to handle that is to teach EmitAggExpr to do the final copy // differently if it can't be elided. - CGF.EmitAnyExprToMem(e, typedAddr, e->getType().getQualifiers(), - /*IsInit*/ true); + EmitAnyExprToMem(e, typedAddr, e->getType().getQualifiers(), + /*IsInit*/ true); // Deactivate the cleanup block. - CGF.DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr)); + DeactivateCleanupBlock(cleanup, cast<llvm::Instruction>(typedAddr)); } llvm::Value *CodeGenFunction::getExceptionSlot() { @@ -417,67 +414,18 @@ llvm::Value *CodeGenFunction::getSelectorFromSlot() { void CodeGenFunction::EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint) { - if (!E->getSubExpr()) { - CGM.getCXXABI().emitRethrow(*this, /*isNoReturn*/true); - - // throw is an expression, and the expression emitters expect us - // to leave ourselves at a valid insertion point. - if (KeepInsertionPoint) - EmitBlock(createBasicBlock("throw.cont")); - - return; - } - - if (CGM.getTarget().getTriple().isKnownWindowsMSVCEnvironment()) { - ErrorUnsupported(E, "throw expression"); - return; - } - - QualType ThrowType = E->getSubExpr()->getType(); - - if (ThrowType->isObjCObjectPointerType()) { - const Stmt *ThrowStmt = E->getSubExpr(); - const ObjCAtThrowStmt S(E->getExprLoc(), - const_cast<Stmt *>(ThrowStmt)); - CGM.getObjCRuntime().EmitThrowStmt(*this, S, false); - // This will clear insertion point which was not cleared in - // call to EmitThrowStmt. - if (KeepInsertionPoint) - EmitBlock(createBasicBlock("throw.cont")); - return; - } - - // Now allocate the exception object. - llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity(); - - llvm::Constant *AllocExceptionFn = getAllocateExceptionFn(CGM); - llvm::CallInst *ExceptionPtr = - EmitNounwindRuntimeCall(AllocExceptionFn, - llvm::ConstantInt::get(SizeTy, TypeSize), - "exception"); - - EmitAnyExprToExn(*this, E->getSubExpr(), ExceptionPtr); - - // Now throw the exception. - llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, - /*ForEH=*/true); - - // The address of the destructor. If the exception type has a - // trivial destructor (or isn't a record), we just pass null. - llvm::Constant *Dtor = nullptr; - if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) { - CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl()); - if (!Record->hasTrivialDestructor()) { - CXXDestructorDecl *DtorD = Record->getDestructor(); - Dtor = CGM.getAddrOfCXXStructor(DtorD, StructorType::Complete); - Dtor = llvm::ConstantExpr::getBitCast(Dtor, Int8PtrTy); + if (const Expr *SubExpr = E->getSubExpr()) { + QualType ThrowType = SubExpr->getType(); + if (ThrowType->isObjCObjectPointerType()) { + const Stmt *ThrowStmt = E->getSubExpr(); + const ObjCAtThrowStmt S(E->getExprLoc(), const_cast<Stmt *>(ThrowStmt)); + CGM.getObjCRuntime().EmitThrowStmt(*this, S, false); + } else { + CGM.getCXXABI().emitThrow(*this, E); } + } else { + CGM.getCXXABI().emitRethrow(*this, /*isNoReturn=*/true); } - if (!Dtor) Dtor = llvm::Constant::getNullValue(Int8PtrTy); - - llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor }; - EmitNoreturnRuntimeCallOrInvoke(getThrowFn(CGM), args); // throw is an expression, and the expression emitters expect us // to leave ourselves at a valid insertion point. @@ -509,6 +457,10 @@ void CodeGenFunction::EmitStartEHSpec(const Decl *D) { EHStack.pushTerminate(); } } else if (EST == EST_Dynamic || EST == EST_DynamicNone) { + // TODO: Revisit exception specifications for the MS ABI. There is a way to + // encode these in an object file but MSVC doesn't do anything with it. + if (getTarget().getCXXABI().isMicrosoft()) + return; unsigned NumExceptions = Proto->getNumExceptions(); EHFilterScope *Filter = EHStack.pushFilter(NumExceptions); @@ -543,8 +495,9 @@ static void emitFilterDispatchBlock(CodeGenFunction &CGF, llvm::Value *zero = CGF.Builder.getInt32(0); llvm::Value *failsFilter = - CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails"); - CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, CGF.getEHResumeBlock(false)); + CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails"); + CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, + CGF.getEHResumeBlock(false)); CGF.EmitBlock(unexpectedBB); } @@ -582,6 +535,10 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) { EHStack.popTerminate(); } } else if (EST == EST_Dynamic || EST == EST_DynamicNone) { + // TODO: Revisit exception specifications for the MS ABI. There is a way to + // encode these in an object file but MSVC doesn't do anything with it. + if (getTarget().getCXXABI().isMicrosoft()) + return; EHFilterScope &filterScope = cast<EHFilterScope>(*EHStack.begin()); emitFilterDispatchBlock(*this, filterScope); EHStack.popFilter(); @@ -589,11 +546,6 @@ void CodeGenFunction::EmitEndEHSpec(const Decl *D) { } void CodeGenFunction::EmitCXXTryStmt(const CXXTryStmt &S) { - if (CGM.getTarget().getTriple().isKnownWindowsMSVCEnvironment()) { - ErrorUnsupported(&S, "try statement"); - return; - } - EnterCXXTryStmt(S); EmitStmt(S.getTryBlock()); ExitCXXTryStmt(S); @@ -622,7 +574,8 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { if (CaughtType->isObjCObjectPointerType()) TypeInfo = CGM.getObjCRuntime().GetEHType(CaughtType); else - TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, /*ForEH=*/true); + TypeInfo = + CGM.getAddrOfCXXCatchHandlerType(CaughtType, C->getCaughtType()); CatchScope->setHandler(I, TypeInfo, Handler); } else { // No exception decl indicates '...', a catch-all. @@ -695,8 +648,15 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() { assert(EHStack.requiresLandingPad()); assert(!EHStack.empty()); - if (!CGM.getLangOpts().Exceptions) - return nullptr; + // If exceptions are disabled, there are usually no landingpads. However, when + // SEH is enabled, functions using SEH still get landingpads. + const LangOptions &LO = CGM.getLangOpts(); + if (!LO.Exceptions) { + if (!LO.Borland && !LO.MicrosoftExt) + return nullptr; + if (!currentFunctionUsesSEHTry()) + return nullptr; + } // Check the innermost scope for a cached landing pad. If this is // a non-EH cleanup, we'll check enclosing scopes in EmitLandingPad. @@ -734,9 +694,9 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { // Save the current IR generation state. CGBuilderTy::InsertPoint savedIP = Builder.saveAndClearIP(); - ApplyDebugLocation AutoRestoreLocation(*this, CurEHLocation); + auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, CurEHLocation); - const EHPersonality &personality = EHPersonality::get(CGM); + const EHPersonality &personality = EHPersonality::get(*this); // Create and configure the landing pad. llvm::BasicBlock *lpad = createBasicBlock("lpad"); @@ -762,8 +722,8 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { bool hasFilter = false; SmallVector<llvm::Value*, 4> filterTypes; llvm::SmallPtrSet<llvm::Value*, 4> catchTypes; - for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end(); - I != E; ++I) { + for (EHScopeStack::iterator I = EHStack.begin(), E = EHStack.end(); I != E; + ++I) { switch (I->getKind()) { case EHScope::Cleanup: @@ -857,263 +817,6 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() { return lpad; } -namespace { - /// A cleanup to call __cxa_end_catch. In many cases, the caught - /// exception type lets us state definitively that the thrown exception - /// type does not have a destructor. In particular: - /// - Catch-alls tell us nothing, so we have to conservatively - /// assume that the thrown exception might have a destructor. - /// - Catches by reference behave according to their base types. - /// - Catches of non-record types will only trigger for exceptions - /// of non-record types, which never have destructors. - /// - Catches of record types can trigger for arbitrary subclasses - /// of the caught type, so we have to assume the actual thrown - /// exception type might have a throwing destructor, even if the - /// caught type's destructor is trivial or nothrow. - struct CallEndCatch : EHScopeStack::Cleanup { - CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {} - bool MightThrow; - - void Emit(CodeGenFunction &CGF, Flags flags) override { - if (!MightThrow) { - CGF.EmitNounwindRuntimeCall(getEndCatchFn(CGF.CGM)); - return; - } - - CGF.EmitRuntimeCallOrInvoke(getEndCatchFn(CGF.CGM)); - } - }; -} - -/// Emits a call to __cxa_begin_catch and enters a cleanup to call -/// __cxa_end_catch. -/// -/// \param EndMightThrow - true if __cxa_end_catch might throw -static llvm::Value *CallBeginCatch(CodeGenFunction &CGF, - llvm::Value *Exn, - bool EndMightThrow) { - llvm::CallInst *call = - CGF.EmitNounwindRuntimeCall(getBeginCatchFn(CGF.CGM), Exn); - - CGF.EHStack.pushCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow); - - return call; -} - -/// A "special initializer" callback for initializing a catch -/// parameter during catch initialization. -static void InitCatchParam(CodeGenFunction &CGF, - const VarDecl &CatchParam, - llvm::Value *ParamAddr, - SourceLocation Loc) { - // Load the exception from where the landing pad saved it. - llvm::Value *Exn = CGF.getExceptionFromSlot(); - - CanQualType CatchType = - CGF.CGM.getContext().getCanonicalType(CatchParam.getType()); - llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType); - - // If we're catching by reference, we can just cast the object - // pointer to the appropriate pointer. - if (isa<ReferenceType>(CatchType)) { - QualType CaughtType = cast<ReferenceType>(CatchType)->getPointeeType(); - bool EndCatchMightThrow = CaughtType->isRecordType(); - - // __cxa_begin_catch returns the adjusted object pointer. - llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndCatchMightThrow); - - // We have no way to tell the personality function that we're - // catching by reference, so if we're catching a pointer, - // __cxa_begin_catch will actually return that pointer by value. - if (const PointerType *PT = dyn_cast<PointerType>(CaughtType)) { - QualType PointeeType = PT->getPointeeType(); - - // When catching by reference, generally we should just ignore - // this by-value pointer and use the exception object instead. - if (!PointeeType->isRecordType()) { - - // Exn points to the struct _Unwind_Exception header, which - // we have to skip past in order to reach the exception data. - unsigned HeaderSize = - CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException(); - AdjustedExn = CGF.Builder.CreateConstGEP1_32(Exn, HeaderSize); - - // However, if we're catching a pointer-to-record type that won't - // work, because the personality function might have adjusted - // the pointer. There's actually no way for us to fully satisfy - // the language/ABI contract here: we can't use Exn because it - // might have the wrong adjustment, but we can't use the by-value - // pointer because it's off by a level of abstraction. - // - // The current solution is to dump the adjusted pointer into an - // alloca, which breaks language semantics (because changing the - // pointer doesn't change the exception) but at least works. - // The better solution would be to filter out non-exact matches - // and rethrow them, but this is tricky because the rethrow - // really needs to be catchable by other sites at this landing - // pad. The best solution is to fix the personality function. - } else { - // Pull the pointer for the reference type off. - llvm::Type *PtrTy = - cast<llvm::PointerType>(LLVMCatchTy)->getElementType(); - - // Create the temporary and write the adjusted pointer into it. - llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp"); - llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); - CGF.Builder.CreateStore(Casted, ExnPtrTmp); - - // Bind the reference to the temporary. - AdjustedExn = ExnPtrTmp; - } - } - - llvm::Value *ExnCast = - CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref"); - CGF.Builder.CreateStore(ExnCast, ParamAddr); - return; - } - - // Scalars and complexes. - TypeEvaluationKind TEK = CGF.getEvaluationKind(CatchType); - if (TEK != TEK_Aggregate) { - llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, false); - - // If the catch type is a pointer type, __cxa_begin_catch returns - // the pointer by value. - if (CatchType->hasPointerRepresentation()) { - llvm::Value *CastExn = - CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.casted"); - - switch (CatchType.getQualifiers().getObjCLifetime()) { - case Qualifiers::OCL_Strong: - CastExn = CGF.EmitARCRetainNonBlock(CastExn); - // fallthrough - - case Qualifiers::OCL_None: - case Qualifiers::OCL_ExplicitNone: - case Qualifiers::OCL_Autoreleasing: - CGF.Builder.CreateStore(CastExn, ParamAddr); - return; - - case Qualifiers::OCL_Weak: - CGF.EmitARCInitWeak(ParamAddr, CastExn); - return; - } - llvm_unreachable("bad ownership qualifier!"); - } - - // Otherwise, it returns a pointer into the exception object. - - llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok - llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); - - LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType); - LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType, - CGF.getContext().getDeclAlign(&CatchParam)); - switch (TEK) { - case TEK_Complex: - CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV, - /*init*/ true); - return; - case TEK_Scalar: { - llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(srcLV, Loc); - CGF.EmitStoreOfScalar(ExnLoad, destLV, /*init*/ true); - return; - } - case TEK_Aggregate: - llvm_unreachable("evaluation kind filtered out!"); - } - llvm_unreachable("bad evaluation kind"); - } - - assert(isa<RecordType>(CatchType) && "unexpected catch type!"); - - llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok - - // Check for a copy expression. If we don't have a copy expression, - // that means a trivial copy is okay. - const Expr *copyExpr = CatchParam.getInit(); - if (!copyExpr) { - llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true); - llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); - CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType); - return; - } - - // We have to call __cxa_get_exception_ptr to get the adjusted - // pointer before copying. - llvm::CallInst *rawAdjustedExn = - CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn); - - // Cast that to the appropriate type. - llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); - - // The copy expression is defined in terms of an OpaqueValueExpr. - // Find it and map it to the adjusted expression. - CodeGenFunction::OpaqueValueMapping - opaque(CGF, OpaqueValueExpr::findInCopyConstruct(copyExpr), - CGF.MakeAddrLValue(adjustedExn, CatchParam.getType())); - - // Call the copy ctor in a terminate scope. - CGF.EHStack.pushTerminate(); - - // Perform the copy construction. - CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam); - CGF.EmitAggExpr(copyExpr, - AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(), - AggValueSlot::IsNotDestructed, - AggValueSlot::DoesNotNeedGCBarriers, - AggValueSlot::IsNotAliased)); - - // Leave the terminate scope. - CGF.EHStack.popTerminate(); - - // Undo the opaque value mapping. - opaque.pop(); - - // Finally we can call __cxa_begin_catch. - CallBeginCatch(CGF, Exn, true); -} - -/// Begins a catch statement by initializing the catch variable and -/// calling __cxa_begin_catch. -static void BeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *S) { - // We have to be very careful with the ordering of cleanups here: - // C++ [except.throw]p4: - // The destruction [of the exception temporary] occurs - // immediately after the destruction of the object declared in - // the exception-declaration in the handler. - // - // So the precise ordering is: - // 1. Construct catch variable. - // 2. __cxa_begin_catch - // 3. Enter __cxa_end_catch cleanup - // 4. Enter dtor cleanup - // - // We do this by using a slightly abnormal initialization process. - // Delegation sequence: - // - ExitCXXTryStmt opens a RunCleanupsScope - // - EmitAutoVarAlloca creates the variable and debug info - // - InitCatchParam initializes the variable from the exception - // - CallBeginCatch calls __cxa_begin_catch - // - CallBeginCatch enters the __cxa_end_catch cleanup - // - EmitAutoVarCleanups enters the variable destructor cleanup - // - EmitCXXTryStmt emits the code for the catch body - // - EmitCXXTryStmt close the RunCleanupsScope - - VarDecl *CatchParam = S->getExceptionDecl(); - if (!CatchParam) { - llvm::Value *Exn = CGF.getExceptionFromSlot(); - CallBeginCatch(CGF, Exn, true); - return; - } - - // Emit the local. - CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); - InitCatchParam(CGF, *CatchParam, var.getObjectAddress(CGF), S->getLocStart()); - CGF.EmitAutoVarCleanups(var); -} - /// Emit the structure of the dispatch block for the given catch scope. /// It is an invariant that the dispatch block already exists. static void emitCatchDispatchBlock(CodeGenFunction &CGF, @@ -1252,11 +955,10 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { RunCleanupsScope CatchScope(*this); // Initialize the catch variable and set up the cleanups. - BeginCatch(*this, C); + CGM.getCXXABI().emitBeginCatch(*this, C); // Emit the PGO counter increment. - RegionCounter CatchCnt = getPGORegionCounter(C); - CatchCnt.beginRegion(Builder); + incrementProfileCounter(C); // Perform the body of the catch. EmitStmt(C->getHandlerBlock()); @@ -1284,9 +986,8 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) { Builder.CreateBr(ContBB); } - RegionCounter ContCnt = getPGORegionCounter(&S); EmitBlock(ContBB); - ContCnt.beginRegion(Builder); + incrementProfileCounter(&S); } namespace { @@ -1480,68 +1181,6 @@ void CodeGenFunction::FinallyInfo::exit(CodeGenFunction &CGF) { CGF.PopCleanupBlock(); } -/// In a terminate landing pad, should we use __clang__call_terminate -/// or just a naked call to std::terminate? -/// -/// __clang_call_terminate calls __cxa_begin_catch, which then allows -/// std::terminate to usefully report something about the -/// violating exception. -static bool useClangCallTerminate(CodeGenModule &CGM) { - // Only do this for Itanium-family ABIs in C++ mode. - return (CGM.getLangOpts().CPlusPlus && - CGM.getTarget().getCXXABI().isItaniumFamily()); -} - -/// Get or define the following function: -/// void @__clang_call_terminate(i8* %exn) nounwind noreturn -/// This code is used only in C++. -static llvm::Constant *getClangCallTerminateFn(CodeGenModule &CGM) { - llvm::FunctionType *fnTy = - llvm::FunctionType::get(CGM.VoidTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); - llvm::Constant *fnRef = - CGM.CreateRuntimeFunction(fnTy, "__clang_call_terminate"); - - llvm::Function *fn = dyn_cast<llvm::Function>(fnRef); - if (fn && fn->empty()) { - fn->setDoesNotThrow(); - fn->setDoesNotReturn(); - - // What we really want is to massively penalize inlining without - // forbidding it completely. The difference between that and - // 'noinline' is negligible. - fn->addFnAttr(llvm::Attribute::NoInline); - - // Allow this function to be shared across translation units, but - // we don't want it to turn into an exported symbol. - fn->setLinkage(llvm::Function::LinkOnceODRLinkage); - fn->setVisibility(llvm::Function::HiddenVisibility); - - // Set up the function. - llvm::BasicBlock *entry = - llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn); - CGBuilderTy builder(entry); - - // Pull the exception pointer out of the parameter list. - llvm::Value *exn = &*fn->arg_begin(); - - // Call __cxa_begin_catch(exn). - llvm::CallInst *catchCall = builder.CreateCall(getBeginCatchFn(CGM), exn); - catchCall->setDoesNotThrow(); - catchCall->setCallingConv(CGM.getRuntimeCC()); - - // Call std::terminate(). - llvm::CallInst *termCall = builder.CreateCall(getTerminateFn(CGM)); - termCall->setDoesNotThrow(); - termCall->setDoesNotReturn(); - termCall->setCallingConv(CGM.getRuntimeCC()); - - // std::terminate cannot return. - builder.CreateUnreachable(); - } - - return fnRef; -} - llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() { if (TerminateLandingPad) return TerminateLandingPad; @@ -1553,20 +1192,17 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() { Builder.SetInsertPoint(TerminateLandingPad); // Tell the backend that this is a landing pad. - const EHPersonality &Personality = EHPersonality::get(CGM); + const EHPersonality &Personality = EHPersonality::get(*this); llvm::LandingPadInst *LPadInst = Builder.CreateLandingPad(llvm::StructType::get(Int8PtrTy, Int32Ty, nullptr), getOpaquePersonalityFn(CGM, Personality), 0); LPadInst->addClause(getCatchAllValue(*this)); - llvm::CallInst *terminateCall; - if (useClangCallTerminate(CGM)) { - // Extract out the exception pointer. - llvm::Value *exn = Builder.CreateExtractValue(LPadInst, 0); - terminateCall = EmitNounwindRuntimeCall(getClangCallTerminateFn(CGM), exn); - } else { - terminateCall = EmitNounwindRuntimeCall(getTerminateFn(CGM)); - } + llvm::Value *Exn = 0; + if (getLangOpts().CPlusPlus) + Exn = Builder.CreateExtractValue(LPadInst, 0); + llvm::CallInst *terminateCall = + CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn); terminateCall->setDoesNotReturn(); Builder.CreateUnreachable(); @@ -1586,14 +1222,11 @@ llvm::BasicBlock *CodeGenFunction::getTerminateHandler() { // end of the function by FinishFunction. TerminateHandler = createBasicBlock("terminate.handler"); Builder.SetInsertPoint(TerminateHandler); - llvm::CallInst *terminateCall; - if (useClangCallTerminate(CGM)) { - // Load the exception pointer. - llvm::Value *exn = getExceptionFromSlot(); - terminateCall = EmitNounwindRuntimeCall(getClangCallTerminateFn(CGM), exn); - } else { - terminateCall = EmitNounwindRuntimeCall(getTerminateFn(CGM)); - } + llvm::Value *Exn = 0; + if (getLangOpts().CPlusPlus) + Exn = getExceptionFromSlot(); + llvm::CallInst *terminateCall = + CGM.getCXXABI().emitTerminateForUnexpectedException(*this, Exn); terminateCall->setDoesNotReturn(); Builder.CreateUnreachable(); @@ -1612,15 +1245,14 @@ llvm::BasicBlock *CodeGenFunction::getEHResumeBlock(bool isCleanup) { EHResumeBlock = createBasicBlock("eh.resume"); Builder.SetInsertPoint(EHResumeBlock); - const EHPersonality &Personality = EHPersonality::get(CGM); + const EHPersonality &Personality = EHPersonality::get(*this); // This can always be a call because we necessarily didn't find // anything on the EH stack which needs our help. const char *RethrowName = Personality.CatchallRethrowFn; if (RethrowName != nullptr && !isCleanup) { EmitRuntimeCall(getCatchallRethrowFn(CGM, RethrowName), - getExceptionFromSlot()) - ->setDoesNotReturn(); + getExceptionFromSlot())->setDoesNotReturn(); Builder.CreateUnreachable(); Builder.restoreIP(SavedIP); return EHResumeBlock; @@ -1642,9 +1274,433 @@ llvm::BasicBlock *CodeGenFunction::getEHResumeBlock(bool isCleanup) { } void CodeGenFunction::EmitSEHTryStmt(const SEHTryStmt &S) { - CGM.ErrorUnsupported(&S, "SEH __try"); + // FIXME: Implement SEH on other architectures. + const llvm::Triple &T = CGM.getTarget().getTriple(); + if (T.getArch() != llvm::Triple::x86_64 || + !T.isKnownWindowsMSVCEnvironment()) { + ErrorUnsupported(&S, "__try statement"); + return; + } + + EnterSEHTryStmt(S); + { + JumpDest TryExit = getJumpDestInCurrentScope("__try.__leave"); + + SEHTryEpilogueStack.push_back(&TryExit); + EmitStmt(S.getTryBlock()); + SEHTryEpilogueStack.pop_back(); + + if (!TryExit.getBlock()->use_empty()) + EmitBlock(TryExit.getBlock(), /*IsFinished=*/true); + else + delete TryExit.getBlock(); + } + ExitSEHTryStmt(S); +} + +namespace { +struct PerformSEHFinally : EHScopeStack::Cleanup { + llvm::Function *OutlinedFinally; + PerformSEHFinally(llvm::Function *OutlinedFinally) + : OutlinedFinally(OutlinedFinally) {} + + void Emit(CodeGenFunction &CGF, Flags F) override { + ASTContext &Context = CGF.getContext(); + QualType ArgTys[2] = {Context.UnsignedCharTy, Context.VoidPtrTy}; + FunctionProtoType::ExtProtoInfo EPI; + const auto *FTP = cast<FunctionType>( + Context.getFunctionType(Context.VoidTy, ArgTys, EPI)); + + CallArgList Args; + llvm::Value *IsForEH = + llvm::ConstantInt::get(CGF.ConvertType(ArgTys[0]), F.isForEHCleanup()); + Args.add(RValue::get(IsForEH), ArgTys[0]); + + CodeGenModule &CGM = CGF.CGM; + llvm::Value *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0); + llvm::Value *FrameAddr = CGM.getIntrinsic(llvm::Intrinsic::frameaddress); + llvm::Value *FP = CGF.Builder.CreateCall(FrameAddr, Zero); + Args.add(RValue::get(FP), ArgTys[1]); + + const CGFunctionInfo &FnInfo = + CGM.getTypes().arrangeFreeFunctionCall(Args, FTP, /*chainCall=*/false); + CGF.EmitCall(FnInfo, OutlinedFinally, ReturnValueSlot(), Args); + } +}; +} + +namespace { +/// Find all local variable captures in the statement. +struct CaptureFinder : ConstStmtVisitor<CaptureFinder> { + CodeGenFunction &ParentCGF; + const VarDecl *ParentThis; + SmallVector<const VarDecl *, 4> Captures; + CaptureFinder(CodeGenFunction &ParentCGF, const VarDecl *ParentThis) + : ParentCGF(ParentCGF), ParentThis(ParentThis) {} + + void Visit(const Stmt *S) { + // See if this is a capture, then recurse. + ConstStmtVisitor<CaptureFinder>::Visit(S); + for (const Stmt *Child : S->children()) + if (Child) + Visit(Child); + } + + void VisitDeclRefExpr(const DeclRefExpr *E) { + // If this is already a capture, just make sure we capture 'this'. + if (E->refersToEnclosingVariableOrCapture()) { + Captures.push_back(ParentThis); + return; + } + + const auto *D = dyn_cast<VarDecl>(E->getDecl()); + if (D && D->isLocalVarDeclOrParm() && D->hasLocalStorage()) + Captures.push_back(D); + } + + void VisitCXXThisExpr(const CXXThisExpr *E) { + Captures.push_back(ParentThis); + } +}; +} + +void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF, + const Stmt *OutlinedStmt, + llvm::Value *ParentFP) { + // Find all captures in the Stmt. + CaptureFinder Finder(ParentCGF, ParentCGF.CXXABIThisDecl); + Finder.Visit(OutlinedStmt); + + // Typically there are no captures and we can exit early. + if (Finder.Captures.empty()) + return; + + // Prepare the first two arguments to llvm.framerecover. + llvm::Function *FrameRecoverFn = llvm::Intrinsic::getDeclaration( + &CGM.getModule(), llvm::Intrinsic::framerecover); + llvm::Constant *ParentI8Fn = + llvm::ConstantExpr::getBitCast(ParentCGF.CurFn, Int8PtrTy); + + // Create llvm.framerecover calls for all captures. + for (const VarDecl *VD : Finder.Captures) { + if (isa<ImplicitParamDecl>(VD)) { + CGM.ErrorUnsupported(VD, "'this' captured by SEH"); + CXXThisValue = llvm::UndefValue::get(ConvertTypeForMem(VD->getType())); + continue; + } + if (VD->getType()->isVariablyModifiedType()) { + CGM.ErrorUnsupported(VD, "VLA captured by SEH"); + continue; + } + assert((isa<ImplicitParamDecl>(VD) || VD->isLocalVarDeclOrParm()) && + "captured non-local variable"); + + // If this decl hasn't been declared yet, it will be declared in the + // OutlinedStmt. + auto I = ParentCGF.LocalDeclMap.find(VD); + if (I == ParentCGF.LocalDeclMap.end()) + continue; + llvm::Value *ParentVar = I->second; + + llvm::CallInst *RecoverCall = nullptr; + CGBuilderTy Builder(AllocaInsertPt); + if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar)) { + // Mark the variable escaped if nobody else referenced it and compute the + // frameescape index. + auto InsertPair = + ParentCGF.EscapedLocals.insert(std::make_pair(ParentAlloca, -1)); + if (InsertPair.second) + InsertPair.first->second = ParentCGF.EscapedLocals.size() - 1; + int FrameEscapeIdx = InsertPair.first->second; + // call i8* @llvm.framerecover(i8* bitcast(@parentFn), i8* %fp, i32 N) + RecoverCall = Builder.CreateCall( + FrameRecoverFn, {ParentI8Fn, ParentFP, + llvm::ConstantInt::get(Int32Ty, FrameEscapeIdx)}); + + } else { + // If the parent didn't have an alloca, we're doing some nested outlining. + // Just clone the existing framerecover call, but tweak the FP argument to + // use our FP value. All other arguments are constants. + auto *ParentRecover = + cast<llvm::IntrinsicInst>(ParentVar->stripPointerCasts()); + assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::framerecover && + "expected alloca or framerecover in parent LocalDeclMap"); + RecoverCall = cast<llvm::CallInst>(ParentRecover->clone()); + RecoverCall->setArgOperand(1, ParentFP); + RecoverCall->insertBefore(AllocaInsertPt); + } + + // Bitcast the variable, rename it, and insert it in the local decl map. + llvm::Value *ChildVar = + Builder.CreateBitCast(RecoverCall, ParentVar->getType()); + ChildVar->setName(ParentVar->getName()); + LocalDeclMap[VD] = ChildVar; + } +} + +/// Arrange a function prototype that can be called by Windows exception +/// handling personalities. On Win64, the prototype looks like: +/// RetTy func(void *EHPtrs, void *ParentFP); +void CodeGenFunction::startOutlinedSEHHelper(CodeGenFunction &ParentCGF, + StringRef Name, QualType RetTy, + FunctionArgList &Args, + const Stmt *OutlinedStmt) { + llvm::Function *ParentFn = ParentCGF.CurFn; + const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionDeclaration( + RetTy, Args, FunctionType::ExtInfo(), /*isVariadic=*/false); + + llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); + llvm::Function *Fn = llvm::Function::Create( + FnTy, llvm::GlobalValue::InternalLinkage, Name.str(), &CGM.getModule()); + // The filter is either in the same comdat as the function, or it's internal. + if (llvm::Comdat *C = ParentFn->getComdat()) { + Fn->setComdat(C); + } else if (ParentFn->hasWeakLinkage() || ParentFn->hasLinkOnceLinkage()) { + llvm::Comdat *C = CGM.getModule().getOrInsertComdat(ParentFn->getName()); + ParentFn->setComdat(C); + Fn->setComdat(C); + } else { + Fn->setLinkage(llvm::GlobalValue::InternalLinkage); + } + + IsOutlinedSEHHelper = true; + + StartFunction(GlobalDecl(), RetTy, Fn, FnInfo, Args, + OutlinedStmt->getLocStart(), OutlinedStmt->getLocStart()); + + CGM.SetLLVMFunctionAttributes(nullptr, FnInfo, CurFn); + + auto AI = Fn->arg_begin(); + ++AI; + EmitCapturedLocals(ParentCGF, OutlinedStmt, &*AI); +} + +/// Create a stub filter function that will ultimately hold the code of the +/// filter expression. The EH preparation passes in LLVM will outline the code +/// from the main function body into this stub. +llvm::Function * +CodeGenFunction::GenerateSEHFilterFunction(CodeGenFunction &ParentCGF, + const SEHExceptStmt &Except) { + const Expr *FilterExpr = Except.getFilterExpr(); + SourceLocation StartLoc = FilterExpr->getLocStart(); + + SEHPointersDecl = ImplicitParamDecl::Create( + getContext(), nullptr, StartLoc, + &getContext().Idents.get("exception_pointers"), getContext().VoidPtrTy); + FunctionArgList Args; + Args.push_back(SEHPointersDecl); + Args.push_back(ImplicitParamDecl::Create( + getContext(), nullptr, StartLoc, + &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy)); + + // Get the mangled function name. + SmallString<128> Name; + { + llvm::raw_svector_ostream OS(Name); + const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl; + const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl); + assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH"); + CGM.getCXXABI().getMangleContext().mangleSEHFilterExpression(Parent, OS); + } + + startOutlinedSEHHelper(ParentCGF, Name, getContext().LongTy, Args, + FilterExpr); + + // Mark finally block calls as nounwind and noinline to make LLVM's job a + // little easier. + // FIXME: Remove these restrictions in the future. + CurFn->addFnAttr(llvm::Attribute::NoUnwind); + CurFn->addFnAttr(llvm::Attribute::NoInline); + + EmitSEHExceptionCodeSave(); + + // Emit the original filter expression, convert to i32, and return. + llvm::Value *R = EmitScalarExpr(FilterExpr); + R = Builder.CreateIntCast(R, ConvertType(getContext().LongTy), + FilterExpr->getType()->isSignedIntegerType()); + Builder.CreateStore(R, ReturnValue); + + FinishFunction(FilterExpr->getLocEnd()); + + return CurFn; +} + +llvm::Function * +CodeGenFunction::GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF, + const SEHFinallyStmt &Finally) { + const Stmt *FinallyBlock = Finally.getBlock(); + SourceLocation StartLoc = FinallyBlock->getLocStart(); + + FunctionArgList Args; + Args.push_back(ImplicitParamDecl::Create( + getContext(), nullptr, StartLoc, + &getContext().Idents.get("abnormal_termination"), + getContext().UnsignedCharTy)); + Args.push_back(ImplicitParamDecl::Create( + getContext(), nullptr, StartLoc, + &getContext().Idents.get("frame_pointer"), getContext().VoidPtrTy)); + + // Get the mangled function name. + SmallString<128> Name; + { + llvm::raw_svector_ostream OS(Name); + const Decl *ParentCodeDecl = ParentCGF.CurCodeDecl; + const NamedDecl *Parent = dyn_cast_or_null<NamedDecl>(ParentCodeDecl); + assert(Parent && "FIXME: handle unnamed decls (lambdas, blocks) with SEH"); + CGM.getCXXABI().getMangleContext().mangleSEHFinallyBlock(Parent, OS); + } + + startOutlinedSEHHelper(ParentCGF, Name, getContext().VoidTy, Args, + FinallyBlock); + + // Emit the original filter expression, convert to i32, and return. + EmitStmt(FinallyBlock); + + FinishFunction(FinallyBlock->getLocEnd()); + + return CurFn; +} + +void CodeGenFunction::EmitSEHExceptionCodeSave() { + // Save the exception code in the exception slot to unify exception access in + // the filter function and the landing pad. + // struct EXCEPTION_POINTERS { + // EXCEPTION_RECORD *ExceptionRecord; + // CONTEXT *ContextRecord; + // }; + // void *exn.slot = + // (void *)(uintptr_t)exception_pointers->ExceptionRecord->ExceptionCode; + llvm::Value *Ptrs = Builder.CreateLoad(GetAddrOfLocalVar(SEHPointersDecl)); + llvm::Type *RecordTy = CGM.Int32Ty->getPointerTo(); + llvm::Type *PtrsTy = llvm::StructType::get(RecordTy, CGM.VoidPtrTy, nullptr); + Ptrs = Builder.CreateBitCast(Ptrs, PtrsTy->getPointerTo()); + llvm::Value *Rec = Builder.CreateStructGEP(PtrsTy, Ptrs, 0); + Rec = Builder.CreateLoad(Rec); + llvm::Value *Code = Builder.CreateLoad(Rec); + Code = Builder.CreateZExt(Code, CGM.IntPtrTy); + // FIXME: Change landing pads to produce {i32, i32} and make the exception + // slot an i32. + Code = Builder.CreateIntToPtr(Code, CGM.VoidPtrTy); + Builder.CreateStore(Code, getExceptionSlot()); +} + +llvm::Value *CodeGenFunction::EmitSEHExceptionInfo() { + // Sema should diagnose calling this builtin outside of a filter context, but + // don't crash if we screw up. + if (!SEHPointersDecl) + return llvm::UndefValue::get(Int8PtrTy); + return Builder.CreateLoad(GetAddrOfLocalVar(SEHPointersDecl)); +} + +llvm::Value *CodeGenFunction::EmitSEHExceptionCode() { + // If we're in a landing pad or filter function, the exception slot contains + // the code. + assert(ExceptionSlot); + llvm::Value *Code = + Builder.CreatePtrToInt(getExceptionFromSlot(), CGM.IntPtrTy); + return Builder.CreateTrunc(Code, CGM.Int32Ty); +} + +llvm::Value *CodeGenFunction::EmitSEHAbnormalTermination() { + // Abnormal termination is just the first parameter to the outlined finally + // helper. + auto AI = CurFn->arg_begin(); + return Builder.CreateZExt(&*AI, Int32Ty); +} + +void CodeGenFunction::EnterSEHTryStmt(const SEHTryStmt &S) { + CodeGenFunction HelperCGF(CGM, /*suppressNewContext=*/true); + if (const SEHFinallyStmt *Finally = S.getFinallyHandler()) { + // Push a cleanup for __finally blocks. + llvm::Function *FinallyFunc = + HelperCGF.GenerateSEHFinallyFunction(*this, *Finally); + EHStack.pushCleanup<PerformSEHFinally>(NormalAndEHCleanup, FinallyFunc); + return; + } + + // Otherwise, we must have an __except block. + const SEHExceptStmt *Except = S.getExceptHandler(); + assert(Except); + EHCatchScope *CatchScope = EHStack.pushCatch(1); + + // If the filter is known to evaluate to 1, then we can use the clause "catch + // i8* null". + llvm::Constant *C = + CGM.EmitConstantExpr(Except->getFilterExpr(), getContext().IntTy, this); + if (C && C->isOneValue()) { + CatchScope->setCatchAllHandler(0, createBasicBlock("__except")); + return; + } + + // In general, we have to emit an outlined filter function. Use the function + // in place of the RTTI typeinfo global that C++ EH uses. + llvm::Function *FilterFunc = + HelperCGF.GenerateSEHFilterFunction(*this, *Except); + llvm::Constant *OpaqueFunc = + llvm::ConstantExpr::getBitCast(FilterFunc, Int8PtrTy); + CatchScope->setHandler(0, OpaqueFunc, createBasicBlock("__except")); +} + +void CodeGenFunction::ExitSEHTryStmt(const SEHTryStmt &S) { + // Just pop the cleanup if it's a __finally block. + if (S.getFinallyHandler()) { + PopCleanupBlock(); + return; + } + + // Otherwise, we must have an __except block. + const SEHExceptStmt *Except = S.getExceptHandler(); + assert(Except && "__try must have __finally xor __except"); + EHCatchScope &CatchScope = cast<EHCatchScope>(*EHStack.begin()); + + // Don't emit the __except block if the __try block lacked invokes. + // TODO: Model unwind edges from instructions, either with iload / istore or + // a try body function. + if (!CatchScope.hasEHBranches()) { + CatchScope.clearHandlerBlocks(); + EHStack.popCatch(); + return; + } + + // The fall-through block. + llvm::BasicBlock *ContBB = createBasicBlock("__try.cont"); + + // We just emitted the body of the __try; jump to the continue block. + if (HaveInsertPoint()) + Builder.CreateBr(ContBB); + + // Check if our filter function returned true. + emitCatchDispatchBlock(*this, CatchScope); + + // Grab the block before we pop the handler. + llvm::BasicBlock *ExceptBB = CatchScope.getHandler(0).Block; + EHStack.popCatch(); + + EmitBlockAfterUses(ExceptBB); + + // Emit the __except body. + EmitStmt(Except->getBlock()); + + if (HaveInsertPoint()) + Builder.CreateBr(ContBB); + + EmitBlock(ContBB); } void CodeGenFunction::EmitSEHLeaveStmt(const SEHLeaveStmt &S) { - CGM.ErrorUnsupported(&S, "SEH __leave"); + // If this code is reachable then emit a stop point (if generating + // debug info). We have to do this ourselves because we are on the + // "simple" statement path. + if (HaveInsertPoint()) + EmitStopPoint(&S); + + // This must be a __leave from a __finally block, which we warn on and is UB. + // Just emit unreachable. + if (!isSEHTryScope()) { + Builder.CreateUnreachable(); + Builder.ClearInsertionPoint(); + return; + } + + EmitBranchThroughCleanup(*SEHTryEpilogueStack.back()); } diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp index ce7679c836e4..1ed45a33d0fd 100644 --- a/lib/CodeGen/CGExpr.cpp +++ b/lib/CodeGen/CGExpr.cpp @@ -31,6 +31,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/Support/ConvertUTF.h" +#include "llvm/Support/MathExtras.h" using namespace clang; using namespace CodeGen; @@ -300,9 +301,26 @@ createReferenceTemporary(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, const Expr *Inner) { switch (M->getStorageDuration()) { case SD_FullExpression: - case SD_Automatic: - return CGF.CreateMemTemp(Inner->getType(), "ref.tmp"); - + case SD_Automatic: { + // If we have a constant temporary array or record try to promote it into a + // constant global under the same rules a normal constant would've been + // promoted. This is easier on the optimizer and generally emits fewer + // instructions. + QualType Ty = Inner->getType(); + if (CGF.CGM.getCodeGenOpts().MergeAllConstants && + (Ty->isArrayType() || Ty->isRecordType()) && + CGF.CGM.isTypeConstant(Ty, true)) + if (llvm::Constant *Init = CGF.CGM.EmitConstantExpr(Inner, Ty, &CGF)) { + auto *GV = new llvm::GlobalVariable( + CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, + llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp"); + GV->setAlignment( + CGF.getContext().getTypeAlignInChars(Ty).getQuantity()); + // FIXME: Should we put the new global into a COMDAT? + return GV; + } + return CGF.CreateMemTemp(Ty, "ref.tmp"); + } case SD_Thread: case SD_Static: return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner); @@ -324,14 +342,15 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { M->getType().getObjCLifetime() != Qualifiers::OCL_None && M->getType().getObjCLifetime() != Qualifiers::OCL_ExplicitNone) { llvm::Value *Object = createReferenceTemporary(*this, M, E); - LValue RefTempDst = MakeAddrLValue(Object, M->getType()); - if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) { + Object = llvm::ConstantExpr::getBitCast( + Var, ConvertTypeForMem(E->getType())->getPointerTo()); // We should not have emitted the initializer for this temporary as a // constant. assert(!Var->hasInitializer()); Var->setInitializer(CGM.EmitNullConstant(E->getType())); } + LValue RefTempDst = MakeAddrLValue(Object, M->getType()); switch (getEvaluationKind(E->getType())) { default: llvm_unreachable("expected scalar or aggregate expression"); @@ -370,8 +389,11 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { // Create and initialize the reference temporary. llvm::Value *Object = createReferenceTemporary(*this, M, E); if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object)) { - // If the temporary is a global and has a constant initializer, we may - // have already initialized it. + Object = llvm::ConstantExpr::getBitCast( + Var, ConvertTypeForMem(E->getType())->getPointerTo()); + // If the temporary is a global and has a constant initializer or is a + // constant temporary that we promoted to a global, we may have already + // initialized it. if (!Var->hasInitializer()) { Var->setInitializer(CGM.EmitNullConstant(E->getType())); EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true); @@ -478,7 +500,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, SanitizerScope SanScope(this); - SmallVector<std::pair<llvm::Value *, SanitizerKind>, 3> Checks; + SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks; llvm::BasicBlock *Done = nullptr; bool AllowNullPointers = TCK == TCK_DowncastPointer || TCK == TCK_Upcast || @@ -513,7 +535,7 @@ void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *Min = Builder.getFalse(); llvm::Value *CastAddr = Builder.CreateBitCast(Address, Int8PtrTy); llvm::Value *LargeEnough = - Builder.CreateICmpUGE(Builder.CreateCall2(F, CastAddr, Min), + Builder.CreateICmpUGE(Builder.CreateCall(F, {CastAddr, Min}), llvm::ConstantInt::get(IntPtrTy, Size)); Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize)); } @@ -807,6 +829,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { /// length type, this is not possible. /// LValue CodeGenFunction::EmitLValue(const Expr *E) { + ApplyDebugLocation DL(*this, E); switch (E->getStmtClass()) { default: return EmitUnsupportedLValue(E, "l-value expression"); @@ -819,10 +842,14 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) { return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E)); case Expr::BinaryOperatorClass: return EmitBinaryOperatorLValue(cast<BinaryOperator>(E)); - case Expr::CompoundAssignOperatorClass: - if (!E->getType()->isAnyComplexType()) + case Expr::CompoundAssignOperatorClass: { + QualType Ty = E->getType(); + if (const AtomicType *AT = Ty->getAs<AtomicType>()) + Ty = AT->getValueType(); + if (!Ty->isAnyComplexType()) return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); + } case Expr::CallExprClass: case Expr::CXXMemberCallExprClass: case Expr::CXXOperatorCallExprClass: @@ -1135,7 +1162,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile, } // Atomic operations have to be done on integral types. - if (Ty->isAtomicType()) { + if (Ty->isAtomicType() || typeIsSuitableForInlineAtomic(Ty, Volatile)) { LValue lvalue = LValue::MakeAddr(Addr, Ty, CharUnits::fromQuantity(Alignment), getContext(), TBAAInfo); @@ -1178,7 +1205,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile, EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty) }; - SanitizerKind Kind = NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool; + SanitizerMask Kind = NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool; EmitCheck(std::make_pair(Check, Kind), "load_invalid_value", StaticArgs, EmitCheckValue(Load)); } @@ -1254,7 +1281,8 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr, Value = EmitToMemory(Value, Ty); - if (Ty->isAtomicType()) { + if (Ty->isAtomicType() || + (!isInit && typeIsSuitableForInlineAtomic(Ty, Volatile))) { EmitAtomicStore(RValue::get(Value), LValue::MakeAddr(Addr, Ty, CharUnits::fromQuantity(Alignment), @@ -1692,8 +1720,8 @@ void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) { llvm::Value *Value = Src.getScalarVal(); if (OrigTy->isPointerTy()) Value = Builder.CreatePtrToInt(Value, Ty); - Builder.CreateCall2(F, llvm::MetadataAsValue::get(Ty->getContext(), RegName), - Value); + Builder.CreateCall( + F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value}); } // setObjCGCLValueClass - sets class of the lvalue for the purpose of @@ -1807,7 +1835,7 @@ EmitBitCastOfLValueToProperType(CodeGenFunction &CGF, static LValue EmitThreadPrivateVarDeclLValue( CodeGenFunction &CGF, const VarDecl *VD, QualType T, llvm::Value *V, llvm::Type *RealVarTy, CharUnits Alignment, SourceLocation Loc) { - V = CGF.CGM.getOpenMPRuntime().getOMPAddrOfThreadPrivate(CGF, VD, V, Loc); + V = CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, V, Loc); V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); return CGF.MakeAddrLValue(V, T, Alignment); } @@ -2050,9 +2078,8 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { assert(E->getSubExpr()->getType()->isAnyComplexType()); unsigned Idx = E->getOpcode() == UO_Imag; - return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(), - Idx, "idx"), - ExprTy); + return MakeAddrLValue( + Builder.CreateStructGEP(nullptr, LV.getAddress(), Idx, "idx"), ExprTy); } case UO_PreInc: case UO_PreDec: { @@ -2217,7 +2244,8 @@ enum class CheckRecoverableKind { }; } -static CheckRecoverableKind getRecoverableKind(SanitizerKind Kind) { +static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) { + assert(llvm::countPopulation(Kind) == 1); switch (Kind) { case SanitizerKind::Vptr: return CheckRecoverableKind::AlwaysRecoverable; @@ -2264,7 +2292,7 @@ static void emitCheckHandlerCall(CodeGenFunction &CGF, } void CodeGenFunction::EmitCheck( - ArrayRef<std::pair<llvm::Value *, SanitizerKind>> Checked, + ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked, StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs, ArrayRef<llvm::Value *> DynamicArgs) { assert(IsSanitizerScope); @@ -2376,7 +2404,7 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) { Builder.CreateCondBr(Checked, Cont, TrapBB); EmitBlock(TrapBB); llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap); - llvm::CallInst *TrapCall = Builder.CreateCall(F); + llvm::CallInst *TrapCall = Builder.CreateCall(F, {}); TrapCall->setDoesNotReturn(); TrapCall->setDoesNotThrow(); Builder.CreateUnreachable(); @@ -2648,7 +2676,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, unsigned Idx = RL.getLLVMFieldNo(field); if (Idx != 0) // For structs, we GEP to the field that the record layout suggests. - Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + Addr = Builder.CreateStructGEP(nullptr, Addr, Idx, field->getName()); // Get the access type. llvm::Type *PtrTy = llvm::Type::getIntNPtrTy( getLLVMContext(), Info.StorageSize, @@ -2683,7 +2711,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, } else { // For structs, we GEP to the field that the record layout suggests. unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); - addr = Builder.CreateStructGEP(addr, idx, field->getName()); + addr = Builder.CreateStructGEP(nullptr, addr, idx, field->getName()); // If this is a reference field, load the reference right now. if (const ReferenceType *refType = type->getAs<ReferenceType>()) { @@ -2762,7 +2790,7 @@ CodeGenFunction::EmitLValueForFieldInitialization(LValue Base, const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(Field->getParent()); unsigned idx = RL.getLLVMFieldNo(Field); - llvm::Value *V = Builder.CreateStructGEP(Base.getAddress(), idx); + llvm::Value *V = Builder.CreateStructGEP(nullptr, Base.getAddress(), idx); assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs"); // Make sure that the address is pointing to the right type. This is critical @@ -2834,7 +2862,6 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { } OpaqueValueMapping binding(*this, expr); - RegionCounter Cnt = getPGORegionCounter(expr); const Expr *condExpr = expr->getCond(); bool CondExprBool; @@ -2845,7 +2872,7 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { if (!ContainsLabel(dead)) { // If the true case is live, we need to track its region. if (CondExprBool) - Cnt.beginRegion(Builder); + incrementProfileCounter(expr); return EmitLValue(live); } } @@ -2855,11 +2882,11 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) { llvm::BasicBlock *contBlock = createBasicBlock("cond.end"); ConditionalEvaluation eval(*this); - EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, Cnt.getCount()); + EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock, getProfileCount(expr)); // Any temporaries created here are conditional. EmitBlock(lhsBlock); - Cnt.beginRegion(Builder); + incrementProfileCounter(expr); eval.begin(*this); Optional<LValue> lhs = EmitLValueOrThrowExpression(*this, expr->getTrueExpr()); @@ -3007,6 +3034,9 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived, E->getType()); + if (SanOpts.has(SanitizerKind::CFIDerivedCast)) + EmitVTablePtrCheckForCast(E->getType(), Derived, /*MayBeNull=*/false); + return MakeAddrLValue(Derived, E->getType()); } case CK_LValueBitCast: { @@ -3016,6 +3046,10 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { LValue LV = EmitLValue(E->getSubExpr()); llvm::Value *V = Builder.CreateBitCast(LV.getAddress(), ConvertType(CE->getTypeAsWritten())); + + if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) + EmitVTablePtrCheckForCast(E->getType(), V, /*MayBeNull=*/false); + return MakeAddrLValue(V, E->getType()); } case CK_ObjCObjectLValueCast: { @@ -3059,16 +3093,6 @@ RValue CodeGenFunction::EmitRValueForField(LValue LV, RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue) { - // Force column info to be generated so we can differentiate - // multiple call sites on the same line in the debug info. - // FIXME: This is insufficient. Two calls coming from the same macro - // expansion will still get the same line/column and break debug info. It's - // possible that LLVM can be fixed to not rely on this uniqueness, at which - // point this workaround can be removed. - ApplyDebugLocation DL(*this, E->getLocStart(), - E->getDirectCallee() && - E->getDirectCallee()->isInlineSpecified()); - // Builtins never have block type. if (E->getCallee()->getType()->isBlockPointerType()) return EmitBlockCallExpr(E, ReturnValue); @@ -3202,7 +3226,7 @@ LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) { if (!RV.isScalar()) return MakeAddrLValue(RV.getAggregateAddr(), E->getType()); - assert(E->getCallReturnType()->isReferenceType() && + assert(E->getCallReturnType(getContext())->isReferenceType() && "Can't have a scalar return unless the return type is a " "reference type!"); @@ -3328,16 +3352,6 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, const auto *FnType = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType()); - // Force column info to differentiate multiple inlined call sites on - // the same line, analoguous to EmitCallExpr. - // FIXME: This is insufficient. Two calls coming from the same macro expansion - // will still get the same line/column and break debug info. It's possible - // that LLVM can be fixed to not rely on this uniqueness, at which point this - // workaround can be removed. - bool ForceColumnInfo = false; - if (const FunctionDecl* FD = dyn_cast_or_null<const FunctionDecl>(TargetDecl)) - ForceColumnInfo = FD->isInlineSpecified(); - if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) && (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { if (llvm::Constant *PrefixSig = @@ -3355,7 +3369,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, llvm::Value *CalleePrefixStruct = Builder.CreateBitCast( Callee, llvm::PointerType::getUnqual(PrefixStructTy)); llvm::Value *CalleeSigPtr = - Builder.CreateConstGEP2_32(CalleePrefixStruct, 0, 0); + Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0); llvm::Value *CalleeSig = Builder.CreateLoad(CalleeSigPtr); llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig); @@ -3365,7 +3379,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, EmitBlock(TypeCheck); llvm::Value *CalleeRTTIPtr = - Builder.CreateConstGEP2_32(CalleePrefixStruct, 0, 1); + Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1); llvm::Value *CalleeRTTI = Builder.CreateLoad(CalleeRTTIPtr); llvm::Value *CalleeRTTIMatch = Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst); @@ -3386,8 +3400,7 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee, Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), CGM.getContext().VoidPtrTy); EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arg_begin(), - E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0, - ForceColumnInfo); + E->arg_end(), E->getDirectCallee(), /*ParamsToSkip*/ 0); const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall( Args, FnType, /*isChainCall=*/Chain); diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp index 4cf94c033bb4..6fedf0efda9d 100644 --- a/lib/CodeGen/CGExprAgg.cpp +++ b/lib/CodeGen/CGExprAgg.cpp @@ -98,6 +98,11 @@ public: // Visitor Methods //===--------------------------------------------------------------------===// + void Visit(Expr *E) { + ApplyDebugLocation DL(CGF, E); + StmtVisitor<AggExprEmitter>::Visit(E); + } + void VisitStmt(Stmt *S) { CGF.ErrorUnsupported(S, "aggregate expression"); } @@ -207,7 +212,7 @@ void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) { LValue LV = CGF.EmitLValue(E); // If the type of the l-value is atomic, then do an atomic load. - if (LV.getType()->isAtomicType()) { + if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) { CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest); return; } @@ -579,7 +584,12 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { } case CK_ToUnion: { - if (Dest.isIgnored()) break; + // Evaluate even if the destination is ignored. + if (Dest.isIgnored()) { + CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(), + /*ignoreResult=*/true); + break; + } // GCC union extension QualType Ty = E->getSubExpr()->getType(); @@ -640,7 +650,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { // Build a GEP to refer to the subobject. llvm::Value *valueAddr = - CGF.Builder.CreateStructGEP(valueDest.getAddr(), 0); + CGF.Builder.CreateStructGEP(nullptr, valueDest.getAddr(), 0); valueDest = AggValueSlot::forAddr(valueAddr, valueDest.getAlignment(), valueDest.getQualifiers(), @@ -661,7 +671,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { CGF.EmitAggExpr(E->getSubExpr(), atomicSlot); llvm::Value *valueAddr = - Builder.CreateStructGEP(atomicSlot.getAddr(), 0); + Builder.CreateStructGEP(nullptr, atomicSlot.getAddr(), 0); RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile()); return EmitFinalDestCopy(valueType, rvalue); } @@ -736,7 +746,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) { } void AggExprEmitter::VisitCallExpr(const CallExpr *E) { - if (E->getCallReturnType()->isReferenceType()) { + if (E->getCallReturnType(CGF.getContext())->isReferenceType()) { EmitAggLoadOfLValue(E); return; } @@ -860,7 +870,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store); // That copy is an atomic copy if the LHS is atomic. - if (LHS.getType()->isAtomicType()) { + if (LHS.getType()->isAtomicType() || + CGF.LValueIsSuitableForInlineAtomic(LHS)) { CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); return; } @@ -877,7 +888,8 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { // If we have an atomic type, evaluate into the destination and then // do an atomic copy. - if (LHS.getType()->isAtomicType()) { + if (LHS.getType()->isAtomicType() || + CGF.LValueIsSuitableForInlineAtomic(LHS)) { EnsureDest(E->getRHS()->getType()); Visit(E->getRHS()); CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); @@ -909,16 +921,16 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // Bind the common expression if necessary. CodeGenFunction::OpaqueValueMapping binding(CGF, E); - RegionCounter Cnt = CGF.getPGORegionCounter(E); CodeGenFunction::ConditionalEvaluation eval(CGF); - CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, + CGF.getProfileCount(E)); // Save whether the destination's lifetime is externally managed. bool isExternallyDestructed = Dest.isExternallyDestructed(); eval.begin(CGF); CGF.EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Visit(E->getTrueExpr()); eval.end(CGF); @@ -1408,7 +1420,8 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, assert((Record->hasTrivialCopyConstructor() || Record->hasTrivialCopyAssignment() || Record->hasTrivialMoveConstructor() || - Record->hasTrivialMoveAssignment()) && + Record->hasTrivialMoveAssignment() || + Record->isUnion()) && "Trying to aggregate-copy a type without a trivial copy/move " "constructor or assignment operator"); // Ignore empty classes in C++. @@ -1439,7 +1452,34 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, if (alignment.isZero()) alignment = TypeInfo.second; - // FIXME: Handle variable sized types. + llvm::Value *SizeVal = nullptr; + if (TypeInfo.first.isZero()) { + // But note that getTypeInfo returns 0 for a VLA. + if (auto *VAT = dyn_cast_or_null<VariableArrayType>( + getContext().getAsArrayType(Ty))) { + QualType BaseEltTy; + SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr); + TypeInfo = getContext().getTypeInfoDataSizeInChars(BaseEltTy); + std::pair<CharUnits, CharUnits> LastElementTypeInfo; + if (!isAssignment) + LastElementTypeInfo = getContext().getTypeInfoInChars(BaseEltTy); + assert(!TypeInfo.first.isZero()); + SizeVal = Builder.CreateNUWMul( + SizeVal, + llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity())); + if (!isAssignment) { + SizeVal = Builder.CreateNUWSub( + SizeVal, + llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity())); + SizeVal = Builder.CreateNUWAdd( + SizeVal, llvm::ConstantInt::get( + SizeTy, LastElementTypeInfo.first.getQuantity())); + } + } + } + if (!SizeVal) { + SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()); + } // FIXME: If we have a volatile struct, the optimizer can remove what might // appear to be `extra' memory ops: @@ -1470,9 +1510,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) { RecordDecl *Record = RecordTy->getDecl(); if (Record->hasObjectMember()) { - CharUnits size = TypeInfo.first; - llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - llvm::Value *SizeVal = llvm::ConstantInt::get(SizeTy, size.getQuantity()); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, SizeVal); return; @@ -1481,10 +1518,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, QualType BaseType = getContext().getBaseElementType(Ty); if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) { if (RecordTy->getDecl()->hasObjectMember()) { - CharUnits size = TypeInfo.first; - llvm::Type *SizeTy = ConvertType(getContext().getSizeType()); - llvm::Value *SizeVal = - llvm::ConstantInt::get(SizeTy, size.getQuantity()); CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, SizeVal); return; @@ -1497,9 +1530,6 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr, // the optimizer wishes to expand it in to scalar memory operations. llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty); - Builder.CreateMemCpy(DestPtr, SrcPtr, - llvm::ConstantInt::get(IntPtrTy, - TypeInfo.first.getQuantity()), - alignment.getQuantity(), isVolatile, - /*TBAATag=*/nullptr, TBAAStructTag); + Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, alignment.getQuantity(), + isVolatile, /*TBAATag=*/nullptr, TBAAStructTag); } diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp index 6d63b3ae9cf8..13dfbb38816f 100644 --- a/lib/CodeGen/CGExprCXX.cpp +++ b/lib/CodeGen/CGExprCXX.cpp @@ -173,7 +173,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( This = EmitLValue(Base).getAddress(); - if (MD->isTrivial()) { + if (MD->isTrivial() || (MD->isDefaulted() && MD->getParent()->isUnion())) { if (isa<CXXDestructorDecl>(MD)) return RValue::get(nullptr); if (isa<CXXConstructorDecl>(MD) && cast<CXXConstructorDecl>(MD)->isDefaultConstructor()) @@ -256,6 +256,12 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( } else if (UseVirtualCall) { Callee = CGM.getCXXABI().getVirtualFunctionPointer(*this, MD, This, Ty); } else { + if (SanOpts.has(SanitizerKind::CFINVCall) && + MD->getParent()->isDynamicClass()) { + llvm::Value *VTable = GetVTablePtr(This, Int8PtrTy); + EmitVTablePtrCheckForCall(MD, VTable); + } + if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier) Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty); else if (!DevirtualizedMethod) @@ -684,7 +690,7 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF, llvm::Value *tsmV = llvm::ConstantInt::get(CGF.SizeTy, typeSizeMultiplier); llvm::Value *result = - CGF.Builder.CreateCall2(umul_with_overflow, size, tsmV); + CGF.Builder.CreateCall(umul_with_overflow, {size, tsmV}); llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1); if (hasOverflow) @@ -723,7 +729,7 @@ static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF, llvm::Value *cookieSizeV = llvm::ConstantInt::get(CGF.SizeTy, cookieSize); llvm::Value *result = - CGF.Builder.CreateCall2(uadd_with_overflow, size, cookieSizeV); + CGF.Builder.CreateCall(uadd_with_overflow, {size, cookieSizeV}); llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1); if (hasOverflow) @@ -778,12 +784,10 @@ static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init, llvm_unreachable("bad evaluation kind"); } -void -CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, - QualType ElementType, - llvm::Value *BeginPtr, - llvm::Value *NumElements, - llvm::Value *AllocSizeWithoutCookie) { +void CodeGenFunction::EmitNewArrayInitializer( + const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy, + llvm::Value *BeginPtr, llvm::Value *NumElements, + llvm::Value *AllocSizeWithoutCookie) { // If we have a type with trivial initialization and no initializer, // there's nothing to do. if (!E->hasInitializer()) @@ -809,7 +813,8 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>( AllocType->getAsArrayTypeUnsafe())) { unsigned AS = CurPtr->getType()->getPointerAddressSpace(); - llvm::Type *AllocPtrTy = ConvertTypeForMem(AllocType)->getPointerTo(AS); + ElementTy = ConvertTypeForMem(AllocType); + llvm::Type *AllocPtrTy = ElementTy->getPointerTo(AS); CurPtr = Builder.CreateBitCast(CurPtr, AllocPtrTy); InitListElements *= getContext().getConstantArrayElementCount(CAT); } @@ -839,7 +844,8 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, // initialization loops. StoreAnyExprIntoOneUnit(*this, ILE->getInit(i), ILE->getInit(i)->getType(), CurPtr); - CurPtr = Builder.CreateConstInBoundsGEP1_32(CurPtr, 1, "array.exp.next"); + CurPtr = Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, + "array.exp.next"); } // The remaining elements are filled with the array filler expression. @@ -1000,7 +1006,7 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, // Advance to the next element by adjusting the pointer type as necessary. llvm::Value *NextPtr = - Builder.CreateConstInBoundsGEP1_32(CurPtr, 1, "array.next"); + Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr, 1, "array.next"); // Check whether we've gotten to the end of the array and, if so, // exit the loop. @@ -1012,13 +1018,12 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E, } static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E, - QualType ElementType, - llvm::Value *NewPtr, - llvm::Value *NumElements, + QualType ElementType, llvm::Type *ElementTy, + llvm::Value *NewPtr, llvm::Value *NumElements, llvm::Value *AllocSizeWithoutCookie) { - ApplyDebugLocation DL(CGF, E->getStartLoc()); + ApplyDebugLocation DL(CGF, E); if (E->isArray()) - CGF.EmitNewArrayInitializer(E, ElementType, NewPtr, NumElements, + CGF.EmitNewArrayInitializer(E, ElementType, ElementTy, NewPtr, NumElements, AllocSizeWithoutCookie); else if (const Expr *Init = E->getInitializer()) StoreAnyExprIntoOneUnit(CGF, Init, E->getAllocatedType(), NewPtr); @@ -1279,10 +1284,9 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { // Emit a null check on the allocation result if the allocation // function is allowed to return null (because it has a non-throwing - // exception spec; for this part, we inline - // CXXNewExpr::shouldNullCheckAllocation()) and we have an + // exception spec or is the reserved placement new) and we have an // interesting initializer. - bool nullCheck = allocatorType->isNothrow(getContext()) && + bool nullCheck = E->shouldNullCheckAllocation(getContext()) && (!allocType.isPODType(getContext()) || E->hasInitializer()); llvm::BasicBlock *nullCheckBB = nullptr; @@ -1327,11 +1331,11 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) { E, allocType); } - llvm::Type *elementPtrTy - = ConvertTypeForMem(allocType)->getPointerTo(AS); + llvm::Type *elementTy = ConvertTypeForMem(allocType); + llvm::Type *elementPtrTy = elementTy->getPointerTo(AS); llvm::Value *result = Builder.CreateBitCast(allocation, elementPtrTy); - EmitNewInitializer(*this, E, allocType, result, numElements, + EmitNewInitializer(*this, E, allocType, elementTy, result, numElements, allocSizeWithoutCookie); if (E->isArray()) { // NewPtr is a pointer to the base element type. If we're diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp index 1580bbe6a294..27d1c689966b 100644 --- a/lib/CodeGen/CGExprComplex.cpp +++ b/lib/CodeGen/CGExprComplex.cpp @@ -95,6 +95,7 @@ public: //===--------------------------------------------------------------------===// ComplexPairTy Visit(Expr *E) { + ApplyDebugLocation DL(CGF, E); return StmtVisitor<ComplexExprEmitter, ComplexPairTy>::Visit(E); } @@ -316,14 +317,14 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue, llvm::Value *Real=nullptr, *Imag=nullptr; if (!IgnoreReal || isVolatile) { - llvm::Value *RealP = Builder.CreateStructGEP(SrcPtr, 0, + llvm::Value *RealP = Builder.CreateStructGEP(nullptr, SrcPtr, 0, SrcPtr->getName() + ".realp"); Real = Builder.CreateAlignedLoad(RealP, AlignR, isVolatile, SrcPtr->getName() + ".real"); } if (!IgnoreImag || isVolatile) { - llvm::Value *ImagP = Builder.CreateStructGEP(SrcPtr, 1, + llvm::Value *ImagP = Builder.CreateStructGEP(nullptr, SrcPtr, 1, SrcPtr->getName() + ".imagp"); Imag = Builder.CreateAlignedLoad(ImagP, AlignI, isVolatile, SrcPtr->getName() + ".imag"); @@ -335,12 +336,13 @@ ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue, /// specified value pointer. void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue, bool isInit) { - if (lvalue.getType()->isAtomicType()) + if (lvalue.getType()->isAtomicType() || + (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue))) return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit); llvm::Value *Ptr = lvalue.getAddress(); - llvm::Value *RealPtr = Builder.CreateStructGEP(Ptr, 0, "real"); - llvm::Value *ImagPtr = Builder.CreateStructGEP(Ptr, 1, "imag"); + llvm::Value *RealPtr = Builder.CreateStructGEP(nullptr, Ptr, 0, "real"); + llvm::Value *ImagPtr = Builder.CreateStructGEP(nullptr, Ptr, 1, "imag"); unsigned AlignR = lvalue.getAlignment().getQuantity(); ASTContext &C = CGF.getContext(); QualType ComplexTy = lvalue.getType(); @@ -375,7 +377,7 @@ VisitImaginaryLiteral(const ImaginaryLiteral *IL) { ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) { - if (E->getCallReturnType()->isReferenceType()) + if (E->getCallReturnType(CGF.getContext())->isReferenceType()) return EmitLoadOfLValue(E); return CGF.EmitCallExpr(E).getComplexVal(); @@ -818,6 +820,8 @@ EmitCompoundAssignLValue(const CompoundAssignOperator *E, TestAndClearIgnoreReal(); TestAndClearIgnoreImag(); QualType LHSTy = E->getLHS()->getType(); + if (const AtomicType *AT = LHSTy->getAs<AtomicType>()) + LHSTy = AT->getValueType(); BinOpInfo OpInfo; @@ -945,13 +949,14 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // Bind the common expression if necessary. CodeGenFunction::OpaqueValueMapping binding(CGF, E); - RegionCounter Cnt = CGF.getPGORegionCounter(E); + CodeGenFunction::ConditionalEvaluation eval(CGF); - CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, + CGF.getProfileCount(E)); eval.begin(CGF); CGF.EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); ComplexPairTy LHS = Visit(E->getTrueExpr()); LHSBlock = Builder.GetInsertBlock(); CGF.EmitBranch(ContBlock); @@ -1033,7 +1038,7 @@ ComplexPairTy CodeGenFunction::EmitComplexExpr(const Expr *E, bool IgnoreReal, "Invalid complex expression to emit"); return ComplexExprEmitter(*this, IgnoreReal, IgnoreImag) - .Visit(const_cast<Expr*>(E)); + .Visit(const_cast<Expr *>(E)); } void CodeGenFunction::EmitComplexExprIntoLValue(const Expr *E, LValue dest, @@ -1085,8 +1090,8 @@ EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E) { } LValue CodeGenFunction:: -EmitScalarCompooundAssignWithComplex(const CompoundAssignOperator *E, - llvm::Value *&Result) { +EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E, + llvm::Value *&Result) { CompoundFunc Op = getComplexOp(E->getOpcode()); RValue Val; LValue Ret = ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val); diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp index 54f7eee6791e..73ca0cc1c3d5 100644 --- a/lib/CodeGen/CGExprConstant.cpp +++ b/lib/CodeGen/CGExprConstant.cpp @@ -383,14 +383,19 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) { if (!EltInit) return false; - + if (!Field->isBitField()) { // Handle non-bitfield members. AppendField(*Field, Layout.getFieldOffset(FieldNo), EltInit); } else { // Otherwise we have a bitfield. - AppendBitField(*Field, Layout.getFieldOffset(FieldNo), - cast<llvm::ConstantInt>(EltInit)); + if (auto *CI = dyn_cast<llvm::ConstantInt>(EltInit)) { + AppendBitField(*Field, Layout.getFieldOffset(FieldNo), CI); + } else { + // We are trying to initialize a bitfield with a non-trivial constant, + // this must require run-time code. + return false; + } } } @@ -1110,7 +1115,7 @@ llvm::Constant *CodeGenModule::EmitConstantValue(const APValue &Value, unsigned AS = C->getType()->getPointerAddressSpace(); llvm::Type *CharPtrTy = Int8Ty->getPointerTo(AS); llvm::Constant *Casted = llvm::ConstantExpr::getBitCast(C, CharPtrTy); - Casted = llvm::ConstantExpr::getGetElementPtr(Casted, Offset); + Casted = llvm::ConstantExpr::getGetElementPtr(Int8Ty, Casted, Offset); C = llvm::ConstantExpr::getPointerCast(Casted, C->getType()); } @@ -1403,10 +1408,6 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { llvm::Constant *Element = EmitNullConstant(ElementTy); unsigned NumElements = CAT->getSize().getZExtValue(); - - if (Element->isNullValue()) - return llvm::ConstantAggregateZero::get(ATy); - SmallVector<llvm::Constant *, 8> Array(NumElements, Element); return llvm::ConstantArray::get(ATy, Array); } @@ -1416,8 +1417,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) { return ::EmitNullConstant(*this, RD, /*complete object*/ true); } - assert(T->isMemberPointerType() && "Should only see member pointers here!"); - assert(!T->getAs<MemberPointerType>()->getPointeeType()->isFunctionType() && + assert(T->isMemberDataPointerType() && "Should only see pointers to data members here!"); return getCXXABI().EmitNullMemberPointer(T->castAs<MemberPointerType>()); diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp index 3be14c8c3ee6..08c81c012c76 100644 --- a/lib/CodeGen/CGExprScalar.cpp +++ b/lib/CodeGen/CGExprScalar.cpp @@ -85,7 +85,7 @@ public: return CGF.EmitCheckedLValue(E, TCK); } - void EmitBinOpCheck(ArrayRef<std::pair<Value *, SanitizerKind>> Checks, + void EmitBinOpCheck(ArrayRef<std::pair<Value *, SanitizerMask>> Checks, const BinOpInfo &Info); Value *EmitLoadOfLValue(LValue LV, SourceLocation Loc) { @@ -196,7 +196,7 @@ public: //===--------------------------------------------------------------------===// Value *Visit(Expr *E) { - ApplyDebugLocation DL(CGF, E->getLocStart()); + ApplyDebugLocation DL(CGF, E); return StmtVisitor<ScalarExprEmitter, Value*>::Visit(E); } @@ -320,7 +320,7 @@ public: Value *VisitCastExpr(CastExpr *E); Value *VisitCallExpr(const CallExpr *E) { - if (E->getCallReturnType()->isReferenceType()) + if (E->getCallReturnType(CGF.getContext())->isReferenceType()) return EmitLoadOfLValue(E); Value *V = CGF.EmitCallExpr(E).getScalarVal(); @@ -349,10 +349,9 @@ public: return EmitScalarPrePostIncDec(E, LV, true, true); } - llvm::Value *EmitAddConsiderOverflowBehavior(const UnaryOperator *E, - llvm::Value *InVal, - llvm::Value *NextVal, - bool IsInc); + llvm::Value *EmitIncDecConsiderOverflowBehavior(const UnaryOperator *E, + llvm::Value *InVal, + bool IsInc); llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, bool isInc, bool isPre); @@ -745,23 +744,37 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, QualType OrigSrcType = SrcType; llvm::Type *SrcTy = Src->getType(); - // If casting to/from storage-only half FP, use special intrinsics. - if (SrcType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) { - Src = Builder.CreateCall( - CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, - CGF.CGM.FloatTy), - Src); - SrcType = CGF.getContext().FloatTy; - SrcTy = CGF.FloatTy; - } - // Handle conversions to bool first, they are special: comparisons against 0. if (DstType->isBooleanType()) return EmitConversionToBool(Src, SrcType); llvm::Type *DstTy = ConvertType(DstType); + // Cast from half through float if half isn't a native type. + if (SrcType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { + // Cast to FP using the intrinsic if the half type itself isn't supported. + if (DstTy->isFloatingPointTy()) { + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) + return Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, DstTy), + Src); + } else { + // Cast to other types through float, using either the intrinsic or FPExt, + // depending on whether the half type itself is supported + // (as opposed to operations on half, available with NativeHalfType). + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + Src = Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, + CGF.CGM.FloatTy), + Src); + } else { + Src = Builder.CreateFPExt(Src, CGF.CGM.FloatTy, "conv"); + } + SrcType = CGF.getContext().FloatTy; + SrcTy = CGF.FloatTy; + } + } + // Ignore conversions like int -> uint. if (SrcTy == DstTy) return Src; @@ -818,10 +831,20 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType, DstTy); - // Cast to half via float - if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) + // Cast to half through float if half isn't a native type. + if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { + // Make sure we cast in a single step if from another FP type. + if (SrcTy->isFloatingPointTy()) { + // Use the intrinsic if the half type itself isn't supported + // (as opposed to operations on half, available with NativeHalfType). + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) + return Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, SrcTy), Src); + // If the half type is supported, just use an fptrunc. + return Builder.CreateFPTrunc(Src, DstTy); + } DstTy = CGF.FloatTy; + } if (isa<llvm::IntegerType>(SrcTy)) { bool InputSigned = SrcType->isSignedIntegerOrEnumerationType(); @@ -847,10 +870,14 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType, } if (DstTy != ResTy) { - assert(ResTy->isIntegerTy(16) && "Only half FP requires extra conversion"); - Res = Builder.CreateCall( + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + assert(ResTy->isIntegerTy(16) && "Only half FP requires extra conversion"); + Res = Builder.CreateCall( CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, CGF.CGM.FloatTy), Res); + } else { + Res = Builder.CreateFPTrunc(Res, ResTy, "conv"); + } } return Res; @@ -889,7 +916,7 @@ Value *ScalarExprEmitter::EmitNullValue(QualType Ty) { /// operation). The check passes if all values in \p Checks (which are \c i1), /// are \c true. void ScalarExprEmitter::EmitBinOpCheck( - ArrayRef<std::pair<Value *, SanitizerKind>> Checks, const BinOpInfo &Info) { + ArrayRef<std::pair<Value *, SanitizerMask>> Checks, const BinOpInfo &Info) { assert(CGF.IsSanitizerScope); StringRef CheckName; SmallVector<llvm::Constant *, 4> StaticData; @@ -1355,6 +1382,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { llvm_unreachable("wrong cast for pointers in different address spaces" "(must be an address space cast)!"); } + + if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast)) { + if (auto PT = DestTy->getAs<PointerType>()) + CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Src, + /*MayBeNull=*/true); + } + return Builder.CreateBitCast(Src, DstTy); } case CK_AddressSpaceConversion: { @@ -1384,6 +1418,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(), Derived, DestTy->getPointeeType()); + if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast)) + CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived, + /*MayBeNull=*/true); + return Derived; } case CK_UncheckedDerivedToBase: @@ -1412,13 +1450,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) { // anything here. if (!E->getType()->isVariableArrayType()) { assert(isa<llvm::PointerType>(V->getType()) && "Expected pointer"); + llvm::Type *NewTy = ConvertType(E->getType()); V = CGF.Builder.CreatePointerCast( - V, ConvertType(E->getType())->getPointerTo( - V->getType()->getPointerAddressSpace())); + V, NewTy->getPointerTo(V->getType()->getPointerAddressSpace())); assert(isa<llvm::ArrayType>(V->getType()->getPointerElementType()) && "Expected pointer to array"); - V = Builder.CreateStructGEP(V, 0, "arraydecay"); + V = Builder.CreateStructGEP(NewTy, V, 0, "arraydecay"); } // Make sure the array decay ends up being the right type. This matters if @@ -1571,26 +1609,32 @@ Value *ScalarExprEmitter::VisitStmtExpr(const StmtExpr *E) { // Unary Operators //===----------------------------------------------------------------------===// -llvm::Value *ScalarExprEmitter:: -EmitAddConsiderOverflowBehavior(const UnaryOperator *E, - llvm::Value *InVal, - llvm::Value *NextVal, bool IsInc) { +static BinOpInfo createBinOpInfoFromIncDec(const UnaryOperator *E, + llvm::Value *InVal, bool IsInc) { + BinOpInfo BinOp; + BinOp.LHS = InVal; + BinOp.RHS = llvm::ConstantInt::get(InVal->getType(), 1, false); + BinOp.Ty = E->getType(); + BinOp.Opcode = IsInc ? BO_Add : BO_Sub; + BinOp.FPContractable = false; + BinOp.E = E; + return BinOp; +} + +llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior( + const UnaryOperator *E, llvm::Value *InVal, bool IsInc) { + llvm::Value *Amount = + llvm::ConstantInt::get(InVal->getType(), IsInc ? 1 : -1, true); + StringRef Name = IsInc ? "inc" : "dec"; switch (CGF.getLangOpts().getSignedOverflowBehavior()) { case LangOptions::SOB_Defined: - return Builder.CreateAdd(InVal, NextVal, IsInc ? "inc" : "dec"); + return Builder.CreateAdd(InVal, Amount, Name); case LangOptions::SOB_Undefined: if (!CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) - return Builder.CreateNSWAdd(InVal, NextVal, IsInc ? "inc" : "dec"); + return Builder.CreateNSWAdd(InVal, Amount, Name); // Fall through. case LangOptions::SOB_Trapping: - BinOpInfo BinOp; - BinOp.LHS = InVal; - BinOp.RHS = NextVal; - BinOp.Ty = E->getType(); - BinOp.Opcode = BO_Add; - BinOp.FPContractable = false; - BinOp.E = E; - return EmitOverflowCheckedBinOp(BinOp); + return EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(E, InVal, IsInc)); } llvm_unreachable("Unknown SignedOverflowBehaviorTy"); } @@ -1668,27 +1712,20 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, // Most common case by far: integer increment. } else if (type->isIntegerType()) { - - llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true); - // Note that signed integer inc/dec with width less than int can't // overflow because of promotion rules; we're just eliding a few steps here. bool CanOverflow = value->getType()->getIntegerBitWidth() >= CGF.IntTy->getIntegerBitWidth(); if (CanOverflow && type->isSignedIntegerOrEnumerationType()) { - value = EmitAddConsiderOverflowBehavior(E, value, amt, isInc); + value = EmitIncDecConsiderOverflowBehavior(E, value, isInc); } else if (CanOverflow && type->isUnsignedIntegerType() && CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) { - BinOpInfo BinOp; - BinOp.LHS = value; - BinOp.RHS = llvm::ConstantInt::get(value->getType(), 1, false); - BinOp.Ty = E->getType(); - BinOp.Opcode = isInc ? BO_Add : BO_Sub; - BinOp.FPContractable = false; - BinOp.E = E; - value = EmitOverflowCheckedBinOp(BinOp); - } else + value = + EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(E, value, isInc)); + } else { + llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true); value = Builder.CreateAdd(value, amt, isInc ? "inc" : "dec"); + } // Next most common: pointer increment. } else if (const PointerType *ptr = type->getAs<PointerType>()) { @@ -1742,13 +1779,16 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, // Add the inc/dec to the real part. llvm::Value *amt; - if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) { + if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { // Another special case: half FP increment should be done via float - value = Builder.CreateCall( - CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, - CGF.CGM.FloatTy), - input); + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + value = Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_from_fp16, + CGF.CGM.FloatTy), + input, "incdec.conv"); + } else { + value = Builder.CreateFPExt(input, CGF.CGM.FloatTy, "incdec.conv"); + } } if (value->getType()->isFloatTy()) @@ -1758,20 +1798,29 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, amt = llvm::ConstantFP::get(VMContext, llvm::APFloat(static_cast<double>(amount))); else { + // Remaining types are either Half or LongDouble. Convert from float. llvm::APFloat F(static_cast<float>(amount)); bool ignored; - F.convert(CGF.getTarget().getLongDoubleFormat(), + // Don't use getFloatTypeSemantics because Half isn't + // necessarily represented using the "half" LLVM type. + F.convert(value->getType()->isHalfTy() + ? CGF.getTarget().getHalfFormat() + : CGF.getTarget().getLongDoubleFormat(), llvm::APFloat::rmTowardZero, &ignored); amt = llvm::ConstantFP::get(VMContext, F); } value = Builder.CreateFAdd(value, amt, isInc ? "inc" : "dec"); - if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType && - !CGF.getContext().getLangOpts().HalfArgsAndReturns) - value = Builder.CreateCall( - CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, - CGF.CGM.FloatTy), - value); + if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) { + if (!CGF.getContext().getLangOpts().HalfArgsAndReturns) { + value = Builder.CreateCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::convert_to_fp16, + CGF.CGM.FloatTy), + value, "incdec.conv"); + } else { + value = Builder.CreateFPTrunc(value, input->getType(), "incdec.conv"); + } + } // Objective-C pointer types. } else { @@ -1794,10 +1843,9 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV, llvm::BasicBlock *opBB = Builder.GetInsertBlock(); llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn); auto Pair = CGF.EmitAtomicCompareExchange( - LV, RValue::get(atomicPHI), RValue::get(CGF.EmitToMemory(value, type)), - E->getExprLoc()); - llvm::Value *old = Pair.first.getScalarVal(); - llvm::Value *success = Pair.second.getScalarVal(); + LV, RValue::get(atomicPHI), RValue::get(value), E->getExprLoc()); + llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), type); + llvm::Value *success = Pair.second; atomicPHI->addIncoming(old, opBB); Builder.CreateCondBr(success, contBB, opBB); Builder.SetInsertPoint(contBB); @@ -2056,7 +2104,7 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( BinOpInfo OpInfo; if (E->getComputationResultType()->isAnyComplexType()) - return CGF.EmitScalarCompooundAssignWithComplex(E, Result); + return CGF.EmitScalarCompoundAssignWithComplex(E, Result); // Emit the RHS first. __block variables need to have the rhs evaluated // first, plus this should improve codegen a little. @@ -2138,10 +2186,9 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue( llvm::BasicBlock *opBB = Builder.GetInsertBlock(); llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn); auto Pair = CGF.EmitAtomicCompareExchange( - LHSLV, RValue::get(atomicPHI), - RValue::get(CGF.EmitToMemory(Result, LHSTy)), E->getExprLoc()); - llvm::Value *old = Pair.first.getScalarVal(); - llvm::Value *success = Pair.second.getScalarVal(); + LHSLV, RValue::get(atomicPHI), RValue::get(Result), E->getExprLoc()); + llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), LHSTy); + llvm::Value *success = Pair.second; atomicPHI->addIncoming(old, opBB); Builder.CreateCondBr(success, contBB, opBB); Builder.SetInsertPoint(contBB); @@ -2184,7 +2231,7 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E, void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck( const BinOpInfo &Ops, llvm::Value *Zero, bool isDiv) { - SmallVector<std::pair<llvm::Value *, SanitizerKind>, 2> Checks; + SmallVector<std::pair<llvm::Value *, SanitizerMask>, 2> Checks; if (CGF.SanOpts.has(SanitizerKind::IntegerDivideByZero)) { Checks.push_back(std::make_pair(Builder.CreateICmpNE(Ops.RHS, Zero), @@ -2296,7 +2343,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) { llvm::Function *intrinsic = CGF.CGM.getIntrinsic(IID, opTy); - Value *resultAndOverflow = Builder.CreateCall2(intrinsic, Ops.LHS, Ops.RHS); + Value *resultAndOverflow = Builder.CreateCall(intrinsic, {Ops.LHS, Ops.RHS}); Value *result = Builder.CreateExtractValue(resultAndOverflow, 0); Value *overflow = Builder.CreateExtractValue(resultAndOverflow, 1); @@ -2309,7 +2356,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) { if (!isSigned || CGF.SanOpts.has(SanitizerKind::SignedIntegerOverflow)) { CodeGenFunction::SanitizerScope SanScope(&CGF); llvm::Value *NotOverflow = Builder.CreateNot(overflow); - SanitizerKind Kind = isSigned ? SanitizerKind::SignedIntegerOverflow + SanitizerMask Kind = isSigned ? SanitizerKind::SignedIntegerOverflow : SanitizerKind::UnsignedIntegerOverflow; EmitBinOpCheck(std::make_pair(NotOverflow, Kind), Ops); } else @@ -2476,10 +2523,9 @@ static Value* buildFMulAdd(llvm::BinaryOperator *MulOp, Value *Addend, "neg"); } - Value *FMulAdd = - Builder.CreateCall3( + Value *FMulAdd = Builder.CreateCall( CGF.CGM.getIntrinsic(llvm::Intrinsic::fmuladd, Addend->getType()), - MulOp0, MulOp1, Addend); + {MulOp0, MulOp1, Addend}); MulOp->eraseFromParent(); return FMulAdd; @@ -2664,21 +2710,34 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) { if (Ops.LHS->getType() != RHS->getType()) RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom"); - if (CGF.SanOpts.has(SanitizerKind::Shift) && !CGF.getLangOpts().OpenCL && - isa<llvm::IntegerType>(Ops.LHS->getType())) { + bool SanitizeBase = CGF.SanOpts.has(SanitizerKind::ShiftBase) && + Ops.Ty->hasSignedIntegerRepresentation(); + bool SanitizeExponent = CGF.SanOpts.has(SanitizerKind::ShiftExponent); + // OpenCL 6.3j: shift values are effectively % word size of LHS. + if (CGF.getLangOpts().OpenCL) + RHS = + Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shl.mask"); + else if ((SanitizeBase || SanitizeExponent) && + isa<llvm::IntegerType>(Ops.LHS->getType())) { CodeGenFunction::SanitizerScope SanScope(&CGF); + SmallVector<std::pair<Value *, SanitizerMask>, 2> Checks; llvm::Value *WidthMinusOne = GetWidthMinusOneValue(Ops.LHS, RHS); - llvm::Value *Valid = Builder.CreateICmpULE(RHS, WidthMinusOne); + llvm::Value *ValidExponent = Builder.CreateICmpULE(RHS, WidthMinusOne); - if (Ops.Ty->hasSignedIntegerRepresentation()) { - llvm::BasicBlock *Orig = Builder.GetInsertBlock(); - llvm::BasicBlock *Cont = CGF.createBasicBlock("cont"); - llvm::BasicBlock *CheckBitsShifted = CGF.createBasicBlock("check"); - Builder.CreateCondBr(Valid, CheckBitsShifted, Cont); + if (SanitizeExponent) { + Checks.push_back( + std::make_pair(ValidExponent, SanitizerKind::ShiftExponent)); + } + if (SanitizeBase) { // Check whether we are shifting any non-zero bits off the top of the - // integer. - CGF.EmitBlock(CheckBitsShifted); + // integer. We only emit this check if exponent is valid - otherwise + // instructions below will have undefined behavior themselves. + llvm::BasicBlock *Orig = Builder.GetInsertBlock(); + llvm::BasicBlock *Cont = CGF.createBasicBlock("cont"); + llvm::BasicBlock *CheckShiftBase = CGF.createBasicBlock("check"); + Builder.CreateCondBr(ValidExponent, CheckShiftBase, Cont); + CGF.EmitBlock(CheckShiftBase); llvm::Value *BitsShiftedOff = Builder.CreateLShr(Ops.LHS, Builder.CreateSub(WidthMinusOne, RHS, "shl.zeros", @@ -2693,19 +2752,17 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) { BitsShiftedOff = Builder.CreateLShr(BitsShiftedOff, One); } llvm::Value *Zero = llvm::ConstantInt::get(BitsShiftedOff->getType(), 0); - llvm::Value *SecondCheck = Builder.CreateICmpEQ(BitsShiftedOff, Zero); + llvm::Value *ValidBase = Builder.CreateICmpEQ(BitsShiftedOff, Zero); CGF.EmitBlock(Cont); - llvm::PHINode *P = Builder.CreatePHI(Valid->getType(), 2); - P->addIncoming(Valid, Orig); - P->addIncoming(SecondCheck, CheckBitsShifted); - Valid = P; + llvm::PHINode *BaseCheck = Builder.CreatePHI(ValidBase->getType(), 2); + BaseCheck->addIncoming(Builder.getTrue(), Orig); + BaseCheck->addIncoming(ValidBase, CheckShiftBase); + Checks.push_back(std::make_pair(BaseCheck, SanitizerKind::ShiftBase)); } - EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::Shift), Ops); + assert(!Checks.empty()); + EmitBinOpCheck(Checks, Ops); } - // OpenCL 6.3j: shift values are effectively % word size of LHS. - if (CGF.getLangOpts().OpenCL) - RHS = Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shl.mask"); return Builder.CreateShl(Ops.LHS, RHS, "shl"); } @@ -2717,18 +2774,18 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) { if (Ops.LHS->getType() != RHS->getType()) RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom"); - if (CGF.SanOpts.has(SanitizerKind::Shift) && !CGF.getLangOpts().OpenCL && - isa<llvm::IntegerType>(Ops.LHS->getType())) { + // OpenCL 6.3j: shift values are effectively % word size of LHS. + if (CGF.getLangOpts().OpenCL) + RHS = + Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shr.mask"); + else if (CGF.SanOpts.has(SanitizerKind::ShiftExponent) && + isa<llvm::IntegerType>(Ops.LHS->getType())) { CodeGenFunction::SanitizerScope SanScope(&CGF); llvm::Value *Valid = Builder.CreateICmpULE(RHS, GetWidthMinusOneValue(Ops.LHS, RHS)); - EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::Shift), Ops); + EmitBinOpCheck(std::make_pair(Valid, SanitizerKind::ShiftExponent), Ops); } - // OpenCL 6.3j: shift values are effectively % word size of LHS. - if (CGF.getLangOpts().OpenCL) - RHS = Builder.CreateAnd(RHS, GetWidthMinusOneValue(Ops.LHS, RHS), "shr.mask"); - if (Ops.Ty->hasUnsignedIntegerRepresentation()) return Builder.CreateLShr(Ops.LHS, RHS, "shr"); return Builder.CreateAShr(Ops.LHS, RHS, "shr"); @@ -2846,7 +2903,7 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc, Value *CR6Param = Builder.getInt32(CR6); llvm::Function *F = CGF.CGM.getIntrinsic(ID); - Result = Builder.CreateCall3(F, CR6Param, FirstVecArg, SecondVecArg, ""); + Result = Builder.CreateCall(F, {CR6Param, FirstVecArg, SecondVecArg}); return EmitScalarConversion(Result, CGF.getContext().BoolTy, E->getType()); } @@ -2975,11 +3032,9 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) { } Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { - RegionCounter Cnt = CGF.getPGORegionCounter(E); - // Perform vector logical and on comparisons with zero vectors. if (E->getType()->isVectorType()) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *LHS = Visit(E->getLHS()); Value *RHS = Visit(E->getRHS()); @@ -3002,7 +3057,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { bool LHSCondVal; if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) { if (LHSCondVal) { // If we have 1 && X, just emit X. - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); // ZExt result to int or bool. @@ -3020,7 +3075,8 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { CodeGenFunction::ConditionalEvaluation eval(CGF); // Branch on the LHS first. If it is false, go to the failure (cont) block. - CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(E->getLHS(), RHSBlock, ContBlock, + CGF.getProfileCount(E->getRHS())); // Any edges into the ContBlock are now from an (indeterminate number of) // edges from this first condition. All of these values will be false. Start @@ -3033,7 +3089,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { eval.begin(CGF); CGF.EmitBlock(RHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); eval.end(CGF); @@ -3043,7 +3099,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { // Emit an unconditional branch from this block to ContBlock. { // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(CGF); + auto NL = ApplyDebugLocation::CreateEmpty(CGF); CGF.EmitBlock(ContBlock); } // Insert an entry into the phi node for the edge with the value of RHSCond. @@ -3054,11 +3110,9 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) { } Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { - RegionCounter Cnt = CGF.getPGORegionCounter(E); - // Perform vector logical or on comparisons with zero vectors. if (E->getType()->isVectorType()) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *LHS = Visit(E->getLHS()); Value *RHS = Visit(E->getRHS()); @@ -3081,7 +3135,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { bool LHSCondVal; if (CGF.ConstantFoldsToSimpleInteger(E->getLHS(), LHSCondVal)) { if (!LHSCondVal) { // If we have 0 || X, just emit X. - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); // ZExt result to int or bool. @@ -3100,7 +3154,8 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { // Branch on the LHS first. If it is true, go to the success (cont) block. CGF.EmitBranchOnBoolExpr(E->getLHS(), ContBlock, RHSBlock, - Cnt.getParentCount() - Cnt.getCount()); + CGF.getCurrentProfileCount() - + CGF.getProfileCount(E->getRHS())); // Any edges into the ContBlock are now from an (indeterminate number of) // edges from this first condition. All of these values will be true. Start @@ -3115,7 +3170,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) { // Emit the RHS condition as a bool value. CGF.EmitBlock(RHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS()); eval.end(CGF); @@ -3166,7 +3221,6 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // Bind the common expression if necessary. CodeGenFunction::OpaqueValueMapping binding(CGF, E); - RegionCounter Cnt = CGF.getPGORegionCounter(E); Expr *condExpr = E->getCond(); Expr *lhsExpr = E->getTrueExpr(); @@ -3182,7 +3236,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // If the dead side doesn't have labels we need, just emit the Live part. if (!CGF.ContainsLabel(dead)) { if (CondExprBool) - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); Value *Result = Visit(live); // If the live part is a throw expression, it acts like it has a void @@ -3199,7 +3253,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // the select function. if (CGF.getLangOpts().OpenCL && condExpr->getType()->isVectorType()) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); llvm::Value *CondV = CGF.EmitScalarExpr(condExpr); llvm::Value *LHS = Visit(lhsExpr); @@ -3244,7 +3298,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { // safe to evaluate the LHS and RHS unconditionally. if (isCheapEnoughToEvaluateUnconditionally(lhsExpr, CGF) && isCheapEnoughToEvaluateUnconditionally(rhsExpr, CGF)) { - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); llvm::Value *CondV = CGF.EvaluateExprAsBool(condExpr); llvm::Value *LHS = Visit(lhsExpr); @@ -3262,10 +3316,11 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end"); CodeGenFunction::ConditionalEvaluation eval(CGF); - CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, Cnt.getCount()); + CGF.EmitBranchOnBoolExpr(condExpr, LHSBlock, RHSBlock, + CGF.getProfileCount(lhsExpr)); CGF.EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); + CGF.incrementProfileCounter(E); eval.begin(CGF); Value *LHS = Visit(lhsExpr); eval.end(CGF); @@ -3393,14 +3448,8 @@ Value *CodeGenFunction::EmitScalarExpr(const Expr *E, bool IgnoreResultAssign) { assert(E && hasScalarEvaluationKind(E->getType()) && "Invalid scalar expression to emit"); - bool hasDebugInfo = getDebugInfo(); - if (isa<CXXDefaultArgExpr>(E)) - disableDebugInfo(); - Value *V = ScalarExprEmitter(*this, IgnoreResultAssign) - .Visit(const_cast<Expr*>(E)); - if (isa<CXXDefaultArgExpr>(E) && hasDebugInfo) - enableDebugInfo(); - return V; + return ScalarExprEmitter(*this, IgnoreResultAssign) + .Visit(const_cast<Expr *>(E)); } /// EmitScalarConversion - Emit a conversion from the specified type to the diff --git a/lib/CodeGen/CGLoopInfo.cpp b/lib/CodeGen/CGLoopInfo.cpp index 89f43c281590..011ae7e836ae 100644 --- a/lib/CodeGen/CGLoopInfo.cpp +++ b/lib/CodeGen/CGLoopInfo.cpp @@ -26,8 +26,8 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { SmallVector<Metadata *, 4> Args; // Reserve operand 0 for loop id self reference. - MDNode *TempNode = MDNode::getTemporary(Ctx, None); - Args.push_back(TempNode); + auto TempNode = MDNode::getTemporary(Ctx, None); + Args.push_back(TempNode.get()); // Setting vectorizer.width if (Attrs.VectorizerWidth > 0) { @@ -58,7 +58,6 @@ static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { // Set the first operand to itself. MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); - MDNode::deleteTemporary(TempNode); return LoopID; } diff --git a/lib/CodeGen/CGLoopInfo.h b/lib/CodeGen/CGLoopInfo.h index b1693996507e..aee162118a2f 100644 --- a/lib/CodeGen/CGLoopInfo.h +++ b/lib/CodeGen/CGLoopInfo.h @@ -78,8 +78,8 @@ private: /// This stack can be used to prepare attributes which are applied when a loop /// is emitted. class LoopInfoStack { - LoopInfoStack(const LoopInfoStack &) LLVM_DELETED_FUNCTION; - void operator=(const LoopInfoStack &) LLVM_DELETED_FUNCTION; + LoopInfoStack(const LoopInfoStack &) = delete; + void operator=(const LoopInfoStack &) = delete; public: LoopInfoStack() {} diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp index 34c6d94f8817..ef9a92dfc35b 100644 --- a/lib/CodeGen/CGObjC.cpp +++ b/lib/CodeGen/CGObjC.cpp @@ -102,8 +102,8 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, ArrayType::Normal, /*IndexTypeQuals=*/0); // Allocate the temporary array(s). - llvm::Value *Objects = CreateMemTemp(ElementArrayType, "objects"); - llvm::Value *Keys = nullptr; + llvm::AllocaInst *Objects = CreateMemTemp(ElementArrayType, "objects"); + llvm::AllocaInst *Keys = nullptr; if (DLE) Keys = CreateMemTemp(ElementArrayType, "keys"); @@ -119,10 +119,9 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, if (ALE) { // Emit the element and store it to the appropriate array slot. const Expr *Rhs = ALE->getElement(i); - LValue LV = LValue::MakeAddr(Builder.CreateStructGEP(Objects, i), - ElementType, - Context.getTypeAlignInChars(Rhs->getType()), - Context); + LValue LV = LValue::MakeAddr( + Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i), + ElementType, Context.getTypeAlignInChars(Rhs->getType()), Context); llvm::Value *value = EmitScalarExpr(Rhs); EmitStoreThroughLValue(RValue::get(value), LV, true); @@ -132,19 +131,17 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E, } else { // Emit the key and store it to the appropriate array slot. const Expr *Key = DLE->getKeyValueElement(i).Key; - LValue KeyLV = LValue::MakeAddr(Builder.CreateStructGEP(Keys, i), - ElementType, - Context.getTypeAlignInChars(Key->getType()), - Context); + LValue KeyLV = LValue::MakeAddr( + Builder.CreateStructGEP(Keys->getAllocatedType(), Keys, i), + ElementType, Context.getTypeAlignInChars(Key->getType()), Context); llvm::Value *keyValue = EmitScalarExpr(Key); EmitStoreThroughLValue(RValue::get(keyValue), KeyLV, /*isInit=*/true); // Emit the value and store it to the appropriate array slot. - const Expr *Value = DLE->getKeyValueElement(i).Value; - LValue ValueLV = LValue::MakeAddr(Builder.CreateStructGEP(Objects, i), - ElementType, - Context.getTypeAlignInChars(Value->getType()), - Context); + const Expr *Value = DLE->getKeyValueElement(i).Value; + LValue ValueLV = LValue::MakeAddr( + Builder.CreateStructGEP(Objects->getAllocatedType(), Objects, i), + ElementType, Context.getTypeAlignInChars(Value->getType()), Context); llvm::Value *valueValue = EmitScalarExpr(Value); EmitStoreThroughLValue(RValue::get(valueValue), ValueLV, /*isInit=*/true); if (TrackNeededObjects) { @@ -472,8 +469,7 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD, args.push_back(OMD->getSelfDecl()); args.push_back(OMD->getCmdDecl()); - for (const auto *PI : OMD->params()) - args.push_back(PI); + args.append(OMD->param_begin(), OMD->param_end()); CurGD = OMD; CurEHLocation = OMD->getLocEnd(); @@ -501,8 +497,7 @@ void CodeGenFunction::GenerateObjCMethod(const ObjCMethodDecl *OMD) { StartObjCMethod(OMD, OMD->getClassInterface()); PGO.assignRegionCounters(OMD, CurFn); assert(isa<CompoundStmt>(OMD->getBody())); - RegionCounter Cnt = getPGORegionCounter(OMD->getBody()); - Cnt.beginRegion(Builder); + incrementProfileCounter(OMD->getBody()); EmitCompoundStmtWithoutScope(*cast<CompoundStmt>(OMD->getBody())); FinishFunction(OMD->getBodyRBrace()); } @@ -1435,7 +1430,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Fast enumeration state. QualType StateTy = CGM.getObjCFastEnumerationStateType(); - llvm::Value *StatePtr = CreateMemTemp(StateTy, "state.ptr"); + llvm::AllocaInst *StatePtr = CreateMemTemp(StateTy, "state.ptr"); EmitNullInitialization(StatePtr, StateTy); // Number of elements in the items array. @@ -1507,11 +1502,11 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // If the limit pointer was zero to begin with, the collection is // empty; skip all this. Set the branch weight assuming this has the same // probability of exiting the loop as any other loop exit. - uint64_t EntryCount = PGO.getCurrentRegionCount(); - RegionCounter Cnt = getPGORegionCounter(&S); - Builder.CreateCondBr(Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"), - EmptyBB, LoopInitBB, - PGO.createBranchWeights(EntryCount, Cnt.getCount())); + uint64_t EntryCount = getCurrentProfileCount(); + Builder.CreateCondBr( + Builder.CreateICmpEQ(initialBufferLimit, zero, "iszero"), EmptyBB, + LoopInitBB, + createProfileWeights(EntryCount, getProfileCount(S.getBody()))); // Otherwise, initialize the loop. EmitBlock(LoopInitBB); @@ -1519,8 +1514,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Save the initial mutations value. This is the value at an // address that was written into the state object by // countByEnumeratingWithState:objects:count:. - llvm::Value *StateMutationsPtrPtr = - Builder.CreateStructGEP(StatePtr, 2, "mutationsptr.ptr"); + llvm::Value *StateMutationsPtrPtr = Builder.CreateStructGEP( + StatePtr->getAllocatedType(), StatePtr, 2, "mutationsptr.ptr"); llvm::Value *StateMutationsPtr = Builder.CreateLoad(StateMutationsPtrPtr, "mutationsptr"); @@ -1540,7 +1535,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ llvm::PHINode *count = Builder.CreatePHI(UnsignedLongLTy, 3, "forcoll.count"); count->addIncoming(initialBufferLimit, LoopInitBB); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); // Check whether the mutations value has changed from where it was // at start. StateMutationsPtr should actually be invariant between @@ -1600,8 +1595,8 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Fetch the buffer out of the enumeration state. // TODO: this pointer should actually be invariant between // refreshes, which would help us do certain loop optimizations. - llvm::Value *StateItemsPtr = - Builder.CreateStructGEP(StatePtr, 1, "stateitems.ptr"); + llvm::Value *StateItemsPtr = Builder.CreateStructGEP( + StatePtr->getAllocatedType(), StatePtr, 1, "stateitems.ptr"); llvm::Value *EnumStateItems = Builder.CreateLoad(StateItemsPtr, "stateitems"); @@ -1652,9 +1647,9 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){ // Set the branch weights based on the simplifying assumption that this is // like a while-loop, i.e., ignoring that the false branch fetches more // elements and then returns to the loop. - Builder.CreateCondBr(Builder.CreateICmpULT(indexPlusOne, count), - LoopBodyBB, FetchMoreBB, - PGO.createBranchWeights(Cnt.getCount(), EntryCount)); + Builder.CreateCondBr( + Builder.CreateICmpULT(indexPlusOne, count), LoopBodyBB, FetchMoreBB, + createProfileWeights(getProfileCount(S.getBody()), EntryCount)); index->addIncoming(indexPlusOne, AfterBody.getBlock()); count->addIncoming(count, AfterBody.getBlock()); @@ -1985,7 +1980,8 @@ CodeGenFunction::EmitARCRetainAutoreleasedReturnValue(llvm::Value *value) { } // Call the marker asm if we made one, which we do only at -O0. - if (marker) Builder.CreateCall(marker); + if (marker) + Builder.CreateCall(marker, {}); return emitARCValueOperation(*this, value, CGM.getARCEntrypoints().objc_retainAutoreleasedReturnValue, diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp index c0dc3b8002d8..1580c777ea47 100644 --- a/lib/CodeGen/CGObjCGNU.cpp +++ b/lib/CodeGen/CGObjCGNU.cpp @@ -46,54 +46,49 @@ namespace { /// avoids constructing the type more than once if it's used more than once. class LazyRuntimeFunction { CodeGenModule *CGM; - std::vector<llvm::Type*> ArgTys; + llvm::FunctionType *FTy; const char *FunctionName; llvm::Constant *Function; - public: - /// Constructor leaves this class uninitialized, because it is intended to - /// be used as a field in another class and not all of the types that are - /// used as arguments will necessarily be available at construction time. - LazyRuntimeFunction() - : CGM(nullptr), FunctionName(nullptr), Function(nullptr) {} - /// Initialises the lazy function with the name, return type, and the types - /// of the arguments. - LLVM_END_WITH_NULL - void init(CodeGenModule *Mod, const char *name, - llvm::Type *RetTy, ...) { - CGM =Mod; - FunctionName = name; - Function = nullptr; - ArgTys.clear(); - va_list Args; - va_start(Args, RetTy); - while (llvm::Type *ArgTy = va_arg(Args, llvm::Type*)) - ArgTys.push_back(ArgTy); - va_end(Args); - // Push the return type on at the end so we can pop it off easily - ArgTys.push_back(RetTy); - } - /// Overloaded cast operator, allows the class to be implicitly cast to an - /// LLVM constant. - operator llvm::Constant*() { - if (!Function) { - if (!FunctionName) return nullptr; - // We put the return type on the end of the vector, so pop it back off - llvm::Type *RetTy = ArgTys.back(); - ArgTys.pop_back(); - llvm::FunctionType *FTy = llvm::FunctionType::get(RetTy, ArgTys, false); - Function = - cast<llvm::Constant>(CGM->CreateRuntimeFunction(FTy, FunctionName)); - // We won't need to use the types again, so we may as well clean up the - // vector now - ArgTys.resize(0); - } - return Function; - } - operator llvm::Function*() { - return cast<llvm::Function>((llvm::Constant*)*this); - } +public: + /// Constructor leaves this class uninitialized, because it is intended to + /// be used as a field in another class and not all of the types that are + /// used as arguments will necessarily be available at construction time. + LazyRuntimeFunction() + : CGM(nullptr), FunctionName(nullptr), Function(nullptr) {} + /// Initialises the lazy function with the name, return type, and the types + /// of the arguments. + LLVM_END_WITH_NULL + void init(CodeGenModule *Mod, const char *name, llvm::Type *RetTy, ...) { + CGM = Mod; + FunctionName = name; + Function = nullptr; + std::vector<llvm::Type *> ArgTys; + va_list Args; + va_start(Args, RetTy); + while (llvm::Type *ArgTy = va_arg(Args, llvm::Type *)) + ArgTys.push_back(ArgTy); + va_end(Args); + FTy = llvm::FunctionType::get(RetTy, ArgTys, false); + } + + llvm::FunctionType *getType() { return FTy; } + + /// Overloaded cast operator, allows the class to be implicitly cast to an + /// LLVM constant. + operator llvm::Constant *() { + if (!Function) { + if (!FunctionName) + return nullptr; + Function = + cast<llvm::Constant>(CGM->CreateRuntimeFunction(FTy, FunctionName)); + } + return Function; + } + operator llvm::Function *() { + return cast<llvm::Function>((llvm::Constant *)*this); + } }; @@ -171,8 +166,9 @@ protected: /// where the C code specifies const char*. llvm::Constant *MakeConstantString(const std::string &Str, const std::string &Name="") { - llvm::Constant *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); - return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros); + auto *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str()); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); } /// Emits a linkonce_odr string, whose name is the prefix followed by the /// string value. This allows the linker to combine the strings between @@ -181,13 +177,14 @@ protected: llvm::Constant *ExportUniqueString(const std::string &Str, const std::string prefix) { std::string name = prefix + Str; - llvm::Constant *ConstStr = TheModule.getGlobalVariable(name); + auto *ConstStr = TheModule.getGlobalVariable(name); if (!ConstStr) { llvm::Constant *value = llvm::ConstantDataArray::getString(VMContext,Str); ConstStr = new llvm::GlobalVariable(TheModule, value->getType(), true, llvm::GlobalValue::LinkOnceODRLinkage, value, prefix + Str); } - return llvm::ConstantExpr::getGetElementPtr(ConstStr, Zeros); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); } /// Generates a global structure, initialized by the elements in the vector. /// The element types must match the types of the structure elements in the @@ -237,8 +234,9 @@ protected: NameAndAttributes += TypeStr; NameAndAttributes += '\0'; NameAndAttributes += PD->getNameAsString(); - return llvm::ConstantExpr::getGetElementPtr( - CGM.GetAddrOfConstantCString(NameAndAttributes), Zeros); + auto *ConstStr = CGM.GetAddrOfConstantCString(NameAndAttributes); + return llvm::ConstantExpr::getGetElementPtr(ConstStr->getValueType(), + ConstStr, Zeros); } return MakeConstantString(PD->getNameAsString()); } @@ -672,8 +670,8 @@ class CGObjCGNUstep : public CGObjCGNU { slot->setMetadata(msgSendMDKind, node); // Load the imp from the slot - llvm::Value *imp = - Builder.CreateLoad(Builder.CreateStructGEP(slot.getInstruction(), 4)); + llvm::Value *imp = Builder.CreateLoad( + Builder.CreateStructGEP(nullptr, slot.getInstruction(), 4)); // The lookup function may have changed the receiver, so make sure we use // the new one. @@ -690,7 +688,7 @@ class CGObjCGNUstep : public CGObjCGNU { CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs); slot->setOnlyReadsMemory(); - return Builder.CreateLoad(Builder.CreateStructGEP(slot, 4)); + return Builder.CreateLoad(Builder.CreateStructGEP(nullptr, slot, 4)); } public: CGObjCGNUstep(CodeGenModule &Mod) : CGObjCGNU(Mod, 9, 3) { @@ -1013,7 +1011,7 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion, llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF, const std::string &Name, bool isWeak) { - llvm::Value *ClassName = CGM.GetAddrOfConstantCString(Name); + llvm::GlobalVariable *ClassNameGV = CGM.GetAddrOfConstantCString(Name); // With the incompatible ABI, this will need to be replaced with a direct // reference to the class symbol. For the compatible nonfragile ABI we are // still performing this lookup at run time but emitting the symbol for the @@ -1023,7 +1021,8 @@ llvm::Value *CGObjCGNU::GetClassNamed(CodeGenFunction &CGF, // with memoized versions or with static references if it's safe to do so. if (!isWeak) EmitClassRef(Name); - ClassName = CGF.Builder.CreateStructGEP(ClassName, 0); + llvm::Value *ClassName = + CGF.Builder.CreateStructGEP(ClassNameGV->getValueType(), ClassNameGV, 0); llvm::Constant *ClassLookupFn = CGM.CreateRuntimeFunction(llvm::FunctionType::get(IdTy, PtrToInt8Ty, true), @@ -1056,7 +1055,7 @@ llvm::Value *CGObjCGNU::GetSelector(CodeGenFunction &CGF, Selector Sel, } if (!SelValue) { SelValue = llvm::GlobalAlias::create( - SelectorTy->getElementType(), 0, llvm::GlobalValue::PrivateLinkage, + SelectorTy, llvm::GlobalValue::PrivateLinkage, ".objc_selector_" + Sel.getAsString(), &TheModule); Types.push_back(TypedSelector(TypeEncoding, SelValue)); } @@ -1143,21 +1142,22 @@ llvm::Constant *CGObjCGNUstep::GetEHType(QualType T) { // It's quite ugly hard-coding this. Ideally we'd generate it using the host // platform's name mangling. const char *vtableName = "_ZTVN7gnustep7libobjc22__objc_class_type_infoE"; - llvm::Constant *Vtable = TheModule.getGlobalVariable(vtableName); + auto *Vtable = TheModule.getGlobalVariable(vtableName); if (!Vtable) { Vtable = new llvm::GlobalVariable(TheModule, PtrToInt8Ty, true, llvm::GlobalValue::ExternalLinkage, nullptr, vtableName); } llvm::Constant *Two = llvm::ConstantInt::get(IntTy, 2); - Vtable = llvm::ConstantExpr::getGetElementPtr(Vtable, Two); - Vtable = llvm::ConstantExpr::getBitCast(Vtable, PtrToInt8Ty); + auto *BVtable = llvm::ConstantExpr::getBitCast( + llvm::ConstantExpr::getGetElementPtr(Vtable->getValueType(), Vtable, Two), + PtrToInt8Ty); llvm::Constant *typeName = ExportUniqueString(className, "__objc_eh_typename_"); std::vector<llvm::Constant*> fields; - fields.push_back(Vtable); + fields.push_back(BVtable); fields.push_back(typeName); llvm::Constant *TI = MakeGlobal(llvm::StructType::get(PtrToInt8Ty, PtrToInt8Ty, @@ -1261,25 +1261,25 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, if (IsClassMessage) { if (!MetaClassPtrAlias) { MetaClassPtrAlias = llvm::GlobalAlias::create( - IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage, + IdTy, llvm::GlobalValue::InternalLinkage, ".objc_metaclass_ref" + Class->getNameAsString(), &TheModule); } ReceiverClass = MetaClassPtrAlias; } else { if (!ClassPtrAlias) { ClassPtrAlias = llvm::GlobalAlias::create( - IdTy->getElementType(), 0, llvm::GlobalValue::InternalLinkage, + IdTy, llvm::GlobalValue::InternalLinkage, ".objc_class_ref" + Class->getNameAsString(), &TheModule); } ReceiverClass = ClassPtrAlias; } } // Cast the pointer to a simplified version of the class structure + llvm::Type *CastTy = llvm::StructType::get(IdTy, IdTy, nullptr); ReceiverClass = Builder.CreateBitCast(ReceiverClass, - llvm::PointerType::getUnqual( - llvm::StructType::get(IdTy, IdTy, nullptr))); + llvm::PointerType::getUnqual(CastTy)); // Get the superclass pointer - ReceiverClass = Builder.CreateStructGEP(ReceiverClass, 1); + ReceiverClass = Builder.CreateStructGEP(CastTy, ReceiverClass, 1); // Load the superclass pointer ReceiverClass = Builder.CreateLoad(ReceiverClass); // Construct the structure used to look up the IMP @@ -1287,8 +1287,10 @@ CGObjCGNU::GenerateMessageSendSuper(CodeGenFunction &CGF, Receiver->getType(), IdTy, nullptr); llvm::Value *ObjCSuper = Builder.CreateAlloca(ObjCSuperTy); - Builder.CreateStore(Receiver, Builder.CreateStructGEP(ObjCSuper, 0)); - Builder.CreateStore(ReceiverClass, Builder.CreateStructGEP(ObjCSuper, 1)); + Builder.CreateStore(Receiver, + Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 0)); + Builder.CreateStore(ReceiverClass, + Builder.CreateStructGEP(ObjCSuperTy, ObjCSuper, 1)); ObjCSuper = EnforceType(Builder, ObjCSuper, PtrToObjCSuperTy); @@ -2294,7 +2296,8 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) { offsetPointerIndexes[2] = llvm::ConstantInt::get(IndexTy, ivarIndex); // Get the correct ivar field llvm::Constant *offsetValue = llvm::ConstantExpr::getGetElementPtr( - IvarList, offsetPointerIndexes); + cast<llvm::GlobalVariable>(IvarList)->getValueType(), IvarList, + offsetPointerIndexes); // Get the existing variable, if one exists. llvm::GlobalVariable *offset = TheModule.getNamedGlobal(Name); if (offset) { @@ -2366,7 +2369,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { std::vector<llvm::Constant*> Elements; llvm::Constant *Statics = NULLPtr; // Generate statics list: - if (ConstantStrings.size()) { + if (!ConstantStrings.empty()) { llvm::ArrayType *StaticsArrayTy = llvm::ArrayType::get(PtrToInt8Ty, ConstantStrings.size() + 1); ConstantStrings.push_back(NULLPtr); @@ -2439,8 +2442,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { // Number of static selectors Elements.push_back(llvm::ConstantInt::get(LongTy, SelectorCount)); - llvm::Constant *SelectorList = MakeGlobalArray(SelStructTy, Selectors, - ".objc_selector_list"); + llvm::GlobalVariable *SelectorList = + MakeGlobalArray(SelStructTy, Selectors, ".objc_selector_list"); Elements.push_back(llvm::ConstantExpr::getBitCast(SelectorList, SelStructPtrTy)); @@ -2450,8 +2453,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { llvm::Constant *Idxs[] = {Zeros[0], llvm::ConstantInt::get(Int32Ty, i), Zeros[0]}; // FIXME: We're generating redundant loads and stores here! - llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr(SelectorList, - makeArrayRef(Idxs, 2)); + llvm::Constant *SelPtr = llvm::ConstantExpr::getGetElementPtr( + SelectorList->getValueType(), SelectorList, makeArrayRef(Idxs, 2)); // If selectors are defined as an opaque type, cast the pointer to this // type. SelPtr = llvm::ConstantExpr::getBitCast(SelPtr, SelectorTy); @@ -2562,8 +2565,8 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() { true); if (TheClass) { TheClass = llvm::ConstantExpr::getBitCast(TheClass, PtrTy); - Builder.CreateCall2(RegisterAlias, TheClass, - MakeConstantString(iter->second)); + Builder.CreateCall(RegisterAlias, + {TheClass, MakeConstantString(iter->second)}); } } // Jump to end: @@ -2679,7 +2682,7 @@ llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF, llvm::Value *AddrWeakObj) { CGBuilderTy &B = CGF.Builder; AddrWeakObj = EnforceType(B, AddrWeakObj, PtrToIdTy); - return B.CreateCall(WeakReadFn, AddrWeakObj); + return B.CreateCall(WeakReadFn.getType(), WeakReadFn, AddrWeakObj); } void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, @@ -2687,7 +2690,7 @@ void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - B.CreateCall2(WeakAssignFn, src, dst); + B.CreateCall(WeakAssignFn.getType(), WeakAssignFn, {src, dst}); } void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, @@ -2696,11 +2699,9 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - if (!threadlocal) - B.CreateCall2(GlobalAssignFn, src, dst); - else - // FIXME. Add threadloca assign API - llvm_unreachable("EmitObjCGlobalAssign - Threal Local API NYI"); + // FIXME. Add threadloca assign API + assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI"); + B.CreateCall(GlobalAssignFn.getType(), GlobalAssignFn, {src, dst}); } void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, @@ -2709,7 +2710,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, IdTy); - B.CreateCall3(IvarAssignFn, src, dst, ivarOffset); + B.CreateCall(IvarAssignFn.getType(), IvarAssignFn, {src, dst, ivarOffset}); } void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, @@ -2717,7 +2718,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF, CGBuilderTy &B = CGF.Builder; src = EnforceType(B, src, IdTy); dst = EnforceType(B, dst, PtrToIdTy); - B.CreateCall2(StrongCastAssignFn, src, dst); + B.CreateCall(StrongCastAssignFn.getType(), StrongCastAssignFn, {src, dst}); } void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, @@ -2728,7 +2729,7 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF, DestPtr = EnforceType(B, DestPtr, PtrTy); SrcPtr = EnforceType(B, SrcPtr, PtrTy); - B.CreateCall3(MemMoveFn, DestPtr, SrcPtr, Size); + B.CreateCall(MemMoveFn.getType(), MemMoveFn, {DestPtr, SrcPtr, Size}); } llvm::GlobalVariable *CGObjCGNU::ObjCIvarOffsetVariable( diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp index f91e8e15b039..a45446a7065a 100644 --- a/lib/CodeGen/CGObjCMac.cpp +++ b/lib/CodeGen/CGObjCMac.cpp @@ -486,7 +486,6 @@ public: } ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm); - ~ObjCCommonTypesHelper(){} }; /// ObjCTypesHelper - Helper class that encapsulates lazy @@ -595,7 +594,6 @@ public: public: ObjCTypesHelper(CodeGen::CodeGenModule &cgm); - ~ObjCTypesHelper() {} }; /// ObjCNonFragileABITypesHelper - will have all types needed by objective-c's @@ -733,7 +731,6 @@ public: llvm::Type *EHTypePtrTy; ObjCNonFragileABITypesHelper(CodeGen::CodeGenModule &cgm); - ~ObjCNonFragileABITypesHelper(){} }; class CGObjCCommonMac : public CodeGen::CGObjCRuntime { @@ -1678,14 +1675,13 @@ struct NullReturnState { /// getConstantGEP() - Help routine to construct simple GEPs. static llvm::Constant *getConstantGEP(llvm::LLVMContext &VMContext, - llvm::Constant *C, - unsigned idx0, + llvm::GlobalVariable *C, unsigned idx0, unsigned idx1) { llvm::Value *Idxs[] = { llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx0), llvm::ConstantInt::get(llvm::Type::getInt32Ty(VMContext), idx1) }; - return llvm::ConstantExpr::getGetElementPtr(C, Idxs); + return llvm::ConstantExpr::getGetElementPtr(C->getValueType(), C, Idxs); } /// hasObjCExceptionAttribute - Return true if this class or any super @@ -1791,8 +1787,9 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, CGF.CreateTempAlloca(ObjCTypes.SuperTy, "objc_super"); llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); - CGF.Builder.CreateStore(ReceiverAsObject, - CGF.Builder.CreateStructGEP(ObjCSuper, 0)); + CGF.Builder.CreateStore( + ReceiverAsObject, + CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0)); // If this is a class message the metaclass is passed as the target. llvm::Value *Target; @@ -1805,20 +1802,20 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, // the class's "isa" pointer. The following assumes that // isa" is the first ivar in a class (which it must be). Target = EmitClassRef(CGF, Class->getSuperClass()); - Target = CGF.Builder.CreateStructGEP(Target, 0); + Target = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, Target, 0); Target = CGF.Builder.CreateLoad(Target); } else { - llvm::Value *MetaClassPtr = EmitMetaClassRef(Class); - llvm::Value *SuperPtr = CGF.Builder.CreateStructGEP(MetaClassPtr, 1); + llvm::Constant *MetaClassPtr = EmitMetaClassRef(Class); + llvm::Value *SuperPtr = + CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, MetaClassPtr, 1); llvm::Value *Super = CGF.Builder.CreateLoad(SuperPtr); Target = Super; } - } - else if (isCategoryImpl) + } else if (isCategoryImpl) Target = EmitClassRef(CGF, Class->getSuperClass()); else { llvm::Value *ClassPtr = EmitSuperClassRef(Class); - ClassPtr = CGF.Builder.CreateStructGEP(ClassPtr, 1); + ClassPtr = CGF.Builder.CreateStructGEP(ObjCTypes.ClassTy, ClassPtr, 1); Target = CGF.Builder.CreateLoad(ClassPtr); } // FIXME: We shouldn't need to do this cast, rectify the ASTContext and @@ -1826,8 +1823,8 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, llvm::Type *ClassTy = CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType()); Target = CGF.Builder.CreateBitCast(Target, ClassTy); - CGF.Builder.CreateStore(Target, - CGF.Builder.CreateStructGEP(ObjCSuper, 1)); + CGF.Builder.CreateStore( + Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1)); return EmitMessageSend(CGF, Return, ResultType, EmitSelector(CGF, Sel), ObjCSuper, ObjCTypes.SuperPtrCTy, @@ -3810,15 +3807,16 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF, // Enter a try block: // - Call objc_exception_try_enter to push ExceptionData on top of // the EH stack. - CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), ExceptionData); + CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(), + ExceptionData); // - Call setjmp on the exception data buffer. llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0); llvm::Value *GEPIndexes[] = { Zero, Zero, Zero }; - llvm::Value *SetJmpBuffer = - CGF.Builder.CreateGEP(ExceptionData, GEPIndexes, "setjmp_buffer"); - llvm::CallInst *SetJmpResult = - CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); + llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP( + ObjCTypes.ExceptionDataTy, ExceptionData, GEPIndexes, "setjmp_buffer"); + llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall( + ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result"); SetJmpResult->setCanReturnTwice(); // If setjmp returned 0, enter the protected block; otherwise, @@ -5263,6 +5261,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul // const uint32_t size; // sizeof(struct _protocol_t) // const uint32_t flags; // = 0 // const char ** extendedMethodTypes; + // const char *demangledName; // } // Holder for struct _protocol_list_t * @@ -5275,6 +5274,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(CodeGen::CodeGenModul MethodListnfABIPtrTy, MethodListnfABIPtrTy, MethodListnfABIPtrTy, MethodListnfABIPtrTy, PropertyListPtrTy, IntTy, IntTy, Int8PtrPtrTy, + Int8PtrTy, nullptr); // struct _protocol_t* @@ -6207,6 +6207,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocolRef( /// const uint32_t size; // sizeof(struct _protocol_t) /// const uint32_t flags; // = 0 /// const char ** extendedMethodTypes; +/// const char *demangledName; /// } /// @endcode /// @@ -6258,7 +6259,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol( MethodTypesExt.insert(MethodTypesExt.end(), OptMethodTypesExt.begin(), OptMethodTypesExt.end()); - llvm::Constant *Values[11]; + llvm::Constant *Values[12]; // isa is NULL Values[0] = llvm::Constant::getNullValue(ObjCTypes.ObjectPtrTy); Values[1] = GetClassName(PD->getObjCRuntimeNameAsString()); @@ -6291,6 +6292,9 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol( Values[10] = EmitProtocolMethodTypes("\01l_OBJC_$_PROTOCOL_METHOD_TYPES_" + PD->getObjCRuntimeNameAsString(), MethodTypesExt, ObjCTypes); + // const char *demangledName; + Values[11] = llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy); + llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ProtocolnfABITy, Values); @@ -6562,7 +6566,8 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF, args[1].RV = RValue::get(mref); // Load the function to call from the message ref table. - llvm::Value *callee = CGF.Builder.CreateStructGEP(mref, 0); + llvm::Value *callee = + CGF.Builder.CreateStructGEP(ObjCTypes.MessageRefTy, mref, 0); callee = CGF.Builder.CreateLoad(callee, "msgSend_fn"); callee = CGF.Builder.CreateBitCast(callee, MSI.MessengerType); @@ -6727,8 +6732,9 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, llvm::Value *ReceiverAsObject = CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy); - CGF.Builder.CreateStore(ReceiverAsObject, - CGF.Builder.CreateStructGEP(ObjCSuper, 0)); + CGF.Builder.CreateStore( + ReceiverAsObject, + CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 0)); // If this is a class message the metaclass is passed as the target. llvm::Value *Target; @@ -6742,8 +6748,8 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF, llvm::Type *ClassTy = CGM.getTypes().ConvertType(CGF.getContext().getObjCClassType()); Target = CGF.Builder.CreateBitCast(Target, ClassTy); - CGF.Builder.CreateStore(Target, - CGF.Builder.CreateStructGEP(ObjCSuper, 1)); + CGF.Builder.CreateStore( + Target, CGF.Builder.CreateStructGEP(ObjCTypes.SuperTy, ObjCSuper, 1)); return (isVTableDispatchedSelector(Sel)) ? EmitVTableMessageSend(CGF, Return, ResultType, Sel, @@ -6992,10 +6998,10 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID, llvm::Value *VTableIdx = llvm::ConstantInt::get(CGM.Int32Ty, 2); llvm::Constant *Values[] = { - llvm::ConstantExpr::getGetElementPtr(VTableGV, VTableIdx), - GetClassName(ID->getObjCRuntimeNameAsString()), - GetClassGlobal(ClassName.str()) - }; + llvm::ConstantExpr::getGetElementPtr(VTableGV->getValueType(), VTableGV, + VTableIdx), + GetClassName(ID->getObjCRuntimeNameAsString()), + GetClassGlobal(ClassName.str())}; llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.EHTypeTy, Values); diff --git a/lib/CodeGen/CGObjCRuntime.cpp b/lib/CodeGen/CGObjCRuntime.cpp index 3d013da51e65..5290a87cebf2 100644 --- a/lib/CodeGen/CGObjCRuntime.cpp +++ b/lib/CodeGen/CGObjCRuntime.cpp @@ -160,7 +160,7 @@ namespace { void Emit(CodeGenFunction &CGF, Flags flags) override { if (!MightThrow) { - CGF.Builder.CreateCall(Fn)->setDoesNotThrow(); + CGF.Builder.CreateCall(Fn, {})->setDoesNotThrow(); return; } diff --git a/lib/CodeGen/CGOpenMPRuntime.cpp b/lib/CodeGen/CGOpenMPRuntime.cpp index 22ee00f2c7ae..1238accf42d7 100644 --- a/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/lib/CodeGen/CGOpenMPRuntime.cpp @@ -13,6 +13,7 @@ #include "CGOpenMPRuntime.h" #include "CodeGenFunction.h" +#include "CGCleanup.h" #include "clang/AST/Decl.h" #include "clang/AST/StmtOpenMP.h" #include "llvm/ADT/ArrayRef.h" @@ -27,61 +28,236 @@ using namespace clang; using namespace CodeGen; namespace { -/// \brief API for captured statement code generation in OpenMP constructs. +/// \brief Base class for handling code generation inside OpenMP regions. class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { public: - CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &CS, - const VarDecl *ThreadIDVar) - : CGCapturedStmtInfo(CS, CR_OpenMP), ThreadIDVar(ThreadIDVar), - Directive(D) { - assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); - } + /// \brief Kinds of OpenMP regions used in codegen. + enum CGOpenMPRegionKind { + /// \brief Region with outlined function for standalone 'parallel' + /// directive. + ParallelOutlinedRegion, + /// \brief Region with outlined function for standalone 'task' directive. + TaskOutlinedRegion, + /// \brief Region for constructs that do not require function outlining, + /// like 'for', 'sections', 'atomic' etc. directives. + InlinedRegion, + }; + + CGOpenMPRegionInfo(const CapturedStmt &CS, + const CGOpenMPRegionKind RegionKind, + const RegionCodeGenTy &CodeGen) + : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind), + CodeGen(CodeGen) {} + + CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind, + const RegionCodeGenTy &CodeGen) + : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), + CodeGen(CodeGen) {} - /// \brief Gets a variable or parameter for storing global thread id + /// \brief Get a variable or parameter for storing global thread id /// inside OpenMP construct. - const VarDecl *getThreadIDVariable() const { return ThreadIDVar; } + virtual const VarDecl *getThreadIDVariable() const = 0; - /// \brief Gets an LValue for the current ThreadID variable. - LValue getThreadIDVariableLValue(CodeGenFunction &CGF); + /// \brief Emit the captured statement body. + virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override; + + /// \brief Get an LValue for the current ThreadID variable. + /// \return LValue for thread id variable. This LValue always has type int32*. + virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF); + + CGOpenMPRegionKind getRegionKind() const { return RegionKind; } static bool classof(const CGCapturedStmtInfo *Info) { return Info->getKind() == CR_OpenMP; } - /// \brief Emit the captured statement body. - void EmitBody(CodeGenFunction &CGF, Stmt *S) override; +protected: + CGOpenMPRegionKind RegionKind; + const RegionCodeGenTy &CodeGen; +}; + +/// \brief API for captured statement code generation in OpenMP constructs. +class CGOpenMPOutlinedRegionInfo : public CGOpenMPRegionInfo { +public: + CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen) + : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen), + ThreadIDVar(ThreadIDVar) { + assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); + } + /// \brief Get a variable or parameter for storing global thread id + /// inside OpenMP construct. + const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } /// \brief Get the name of the capture helper. StringRef getHelperName() const override { return ".omp_outlined."; } + static bool classof(const CGCapturedStmtInfo *Info) { + return CGOpenMPRegionInfo::classof(Info) && + cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == + ParallelOutlinedRegion; + } + private: /// \brief A variable or parameter storing global thread id for OpenMP /// constructs. const VarDecl *ThreadIDVar; - /// \brief OpenMP executable directive associated with the region. - const OMPExecutableDirective &Directive; }; + +/// \brief API for captured statement code generation in OpenMP constructs. +class CGOpenMPTaskOutlinedRegionInfo : public CGOpenMPRegionInfo { +public: + CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS, + const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen) + : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen), + ThreadIDVar(ThreadIDVar) { + assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); + } + /// \brief Get a variable or parameter for storing global thread id + /// inside OpenMP construct. + const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } + + /// \brief Get an LValue for the current ThreadID variable. + LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override; + + /// \brief Get the name of the capture helper. + StringRef getHelperName() const override { return ".omp_outlined."; } + + static bool classof(const CGCapturedStmtInfo *Info) { + return CGOpenMPRegionInfo::classof(Info) && + cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == + TaskOutlinedRegion; + } + +private: + /// \brief A variable or parameter storing global thread id for OpenMP + /// constructs. + const VarDecl *ThreadIDVar; +}; + +/// \brief API for inlined captured statement code generation in OpenMP +/// constructs. +class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo { +public: + CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI, + const RegionCodeGenTy &CodeGen) + : CGOpenMPRegionInfo(InlinedRegion, CodeGen), OldCSI(OldCSI), + OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {} + // \brief Retrieve the value of the context parameter. + llvm::Value *getContextValue() const override { + if (OuterRegionInfo) + return OuterRegionInfo->getContextValue(); + llvm_unreachable("No context value for inlined OpenMP region"); + } + virtual void setContextValue(llvm::Value *V) override { + if (OuterRegionInfo) { + OuterRegionInfo->setContextValue(V); + return; + } + llvm_unreachable("No context value for inlined OpenMP region"); + } + /// \brief Lookup the captured field decl for a variable. + const FieldDecl *lookup(const VarDecl *VD) const override { + if (OuterRegionInfo) + return OuterRegionInfo->lookup(VD); + // If there is no outer outlined region,no need to lookup in a list of + // captured variables, we can use the original one. + return nullptr; + } + FieldDecl *getThisFieldDecl() const override { + if (OuterRegionInfo) + return OuterRegionInfo->getThisFieldDecl(); + return nullptr; + } + /// \brief Get a variable or parameter for storing global thread id + /// inside OpenMP construct. + const VarDecl *getThreadIDVariable() const override { + if (OuterRegionInfo) + return OuterRegionInfo->getThreadIDVariable(); + return nullptr; + } + + /// \brief Get the name of the capture helper. + StringRef getHelperName() const override { + if (auto *OuterRegionInfo = getOldCSI()) + return OuterRegionInfo->getHelperName(); + llvm_unreachable("No helper name for inlined OpenMP construct"); + } + + CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; } + + static bool classof(const CGCapturedStmtInfo *Info) { + return CGOpenMPRegionInfo::classof(Info) && + cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion; + } + +private: + /// \brief CodeGen info about outer OpenMP region. + CodeGenFunction::CGCapturedStmtInfo *OldCSI; + CGOpenMPRegionInfo *OuterRegionInfo; +}; + +/// \brief RAII for emitting code of OpenMP constructs. +class InlinedOpenMPRegionRAII { + CodeGenFunction &CGF; + +public: + /// \brief Constructs region for combined constructs. + /// \param CodeGen Code generation sequence for combined directives. Includes + /// a list of functions used for code generation of implicitly inlined + /// regions. + InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen) + : CGF(CGF) { + // Start emission for the construct. + CGF.CapturedStmtInfo = + new CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, CodeGen); + } + ~InlinedOpenMPRegionRAII() { + // Restore original CapturedStmtInfo only if we're done with code emission. + auto *OldCSI = + cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); + delete CGF.CapturedStmtInfo; + CGF.CapturedStmtInfo = OldCSI; + } +}; + } // namespace LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { return CGF.MakeNaturalAlignAddrLValue( - CGF.GetAddrOfLocalVar(ThreadIDVar), - CGF.getContext().getPointerType(ThreadIDVar->getType())); + CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(getThreadIDVariable()), + CGF.PointerAlignInBytes), + getThreadIDVariable() + ->getType() + ->castAs<PointerType>() + ->getPointeeType()); } -void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, Stmt *S) { - CodeGenFunction::OMPPrivateScope PrivateScope(CGF); - CGF.EmitOMPPrivateClause(Directive, PrivateScope); - CGF.EmitOMPFirstprivateClause(Directive, PrivateScope); - if (PrivateScope.Privatize()) - // Emit implicit barrier to synchronize threads and avoid data races. - CGF.CGM.getOpenMPRuntime().EmitOMPBarrierCall(CGF, Directive.getLocStart(), - /*IsExplicit=*/false); - CGCapturedStmtInfo::EmitBody(CGF, S); +void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { + // 1.2.2 OpenMP Language Terminology + // Structured block - An executable statement with a single entry at the + // top and a single exit at the bottom. + // The point of exit cannot be a branch out of the structured block. + // longjmp() and throw() must not violate the entry/exit criteria. + CGF.EHStack.pushTerminate(); + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CodeGen(CGF); + } + CGF.EHStack.popTerminate(); +} + +LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue( + CodeGenFunction &CGF) { + return CGF.MakeNaturalAlignAddrLValue( + CGF.GetAddrOfLocalVar(getThreadIDVariable()), + getThreadIDVariable()->getType()); } CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) - : CGM(CGM), DefaultOpenMPPSource(nullptr) { + : CGM(CGM), DefaultOpenMPPSource(nullptr), KmpRoutineEntryPtrTy(nullptr) { IdentTy = llvm::StructType::create( "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */, CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */, @@ -93,18 +269,38 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8); } +void CGOpenMPRuntime::clear() { + InternalVars.clear(); +} + llvm::Value * -CGOpenMPRuntime::EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D, - const VarDecl *ThreadIDVar) { +CGOpenMPRuntime::emitParallelOutlinedFunction(const OMPExecutableDirective &D, + const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen) { + assert(ThreadIDVar->getType()->isPointerType() && + "thread id variable must be of type kmp_int32 *"); const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt()); CodeGenFunction CGF(CGM, true); - CGOpenMPRegionInfo CGInfo(D, *CS, ThreadIDVar); + CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen); CGF.CapturedStmtInfo = &CGInfo; return CGF.GenerateCapturedStmtFunction(*CS); } llvm::Value * -CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) { +CGOpenMPRuntime::emitTaskOutlinedFunction(const OMPExecutableDirective &D, + const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen) { + assert(!ThreadIDVar->getType()->isPointerType() && + "thread id variable must be of type kmp_int32 for tasks"); + auto *CS = cast<CapturedStmt>(D.getAssociatedStmt()); + CodeGenFunction CGF(CGM, true); + CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen); + CGF.CapturedStmtInfo = &CGInfo; + return CGF.GenerateCapturedStmtFunction(*CS); +} + +llvm::Value * +CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) { llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags); if (!Entry) { if (!DefaultOpenMPPSource) { @@ -134,12 +330,13 @@ CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) { return Entry; } -llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation( - CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags) { +llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, + SourceLocation Loc, + OpenMPLocationFlags Flags) { // If no debug info is generated - return global default location. if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo || Loc.isInvalid()) - return GetOrCreateDefaultOpenMPLocation(Flags); + return getOrCreateDefaultLocation(Flags); assert(CGF.CurFn && "No function in current CodeGenFunction."); @@ -159,14 +356,14 @@ llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation( CGBuilderTy::InsertPointGuard IPG(CGF.Builder); CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); - CGF.Builder.CreateMemCpy(LocValue, GetOrCreateDefaultOpenMPLocation(Flags), + CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags), llvm::ConstantExpr::getSizeOf(IdentTy), CGM.PointerAlignInBytes); } // char **psource = &.kmpc_loc_<flags>.addr.psource; - auto *PSource = - CGF.Builder.CreateConstInBoundsGEP2_32(LocValue, 0, IdentField_PSource); + auto *PSource = CGF.Builder.CreateConstInBoundsGEP2_32(IdentTy, LocValue, 0, + IdentField_PSource); auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding()); if (OMPDebugLoc == nullptr) { @@ -189,8 +386,8 @@ llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation( return LocValue; } -llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF, - SourceLocation Loc) { +llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, + SourceLocation Loc) { assert(CGF.CurFn && "No function in current CodeGenFunction."); llvm::Value *ThreadID = nullptr; @@ -204,36 +401,35 @@ llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF, } if (auto OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { - // Check if this an outlined function with thread id passed as argument. - auto ThreadIDVar = OMPRegionInfo->getThreadIDVariable(); - auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); - auto RVal = CGF.EmitLoadOfLValue(LVal, Loc); - LVal = CGF.MakeNaturalAlignAddrLValue(RVal.getScalarVal(), - ThreadIDVar->getType()); - ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal(); - // If value loaded in entry block, cache it and use it everywhere in - // function. - if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { - auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); - Elem.second.ThreadID = ThreadID; + if (OMPRegionInfo->getThreadIDVariable()) { + // Check if this an outlined function with thread id passed as argument. + auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); + ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal(); + // If value loaded in entry block, cache it and use it everywhere in + // function. + if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { + auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); + Elem.second.ThreadID = ThreadID; + } + return ThreadID; } - } else { - // This is not an outlined function region - need to call __kmpc_int32 - // kmpc_global_thread_num(ident_t *loc). - // Generate thread id value and cache this value for use across the - // function. - CGBuilderTy::InsertPointGuard IPG(CGF.Builder); - CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc)}; - ThreadID = CGF.EmitRuntimeCall( - CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), Args); - auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); - Elem.second.ThreadID = ThreadID; } + + // This is not an outlined function region - need to call __kmpc_int32 + // kmpc_global_thread_num(ident_t *loc). + // Generate thread id value and cache this value for use across the + // function. + CGBuilderTy::InsertPointGuard IPG(CGF.Builder); + CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); + ThreadID = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), + emitUpdateLocation(CGF, Loc)); + auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); + Elem.second.ThreadID = ThreadID; return ThreadID; } -void CGOpenMPRuntime::FunctionFinished(CodeGenFunction &CGF) { +void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { assert(CGF.CurFn && "No function in current CodeGenFunction."); if (OpenMPLocThreadIDMap.count(CGF.CurFn)) OpenMPLocThreadIDMap.erase(CGF.CurFn); @@ -248,7 +444,7 @@ llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() { } llvm::Constant * -CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { +CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) { llvm::Constant *RTLFn = nullptr; switch (Function) { case OMPRTL__kmpc_fork_call: { @@ -334,87 +530,6 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier"); break; } - // Build __kmpc_for_static_init*( - // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, - // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, - // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, - // kmp_int[32|64] incr, kmp_int[32|64] chunk); - case OMPRTL__kmpc_for_static_init_4: { - auto ITy = CGM.Int32Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; - llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_4"); - break; - } - case OMPRTL__kmpc_for_static_init_4u: { - auto ITy = CGM.Int32Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; - llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_4u"); - break; - } - case OMPRTL__kmpc_for_static_init_8: { - auto ITy = CGM.Int64Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; - llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_8"); - break; - } - case OMPRTL__kmpc_for_static_init_8u: { - auto ITy = CGM.Int64Ty; - auto PtrTy = llvm::PointerType::getUnqual(ITy); - llvm::Type *TypeParams[] = { - getIdentTyPointerTy(), // loc - CGM.Int32Ty, // tid - CGM.Int32Ty, // schedtype - llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter - PtrTy, // p_lower - PtrTy, // p_upper - PtrTy, // p_stride - ITy, // incr - ITy // chunk - }; - llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); - RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_init_8u"); - break; - } case OMPRTL__kmpc_for_static_fini: { // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid); llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; @@ -452,10 +567,10 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { break; } case OMPRTL__kmpc_flush: { - // Build void __kmpc_flush(ident_t *loc, ...); + // Build void __kmpc_flush(ident_t *loc); llvm::Type *TypeParams[] = {getIdentTyPointerTy()}; llvm::FunctionType *FnTy = - llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true); + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush"); break; } @@ -475,38 +590,291 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master"); break; } + case OMPRTL__kmpc_omp_taskyield: { + // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid, + // int end_part); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield"); + break; + } + case OMPRTL__kmpc_single: { + // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single"); + break; + } + case OMPRTL__kmpc_end_single: { + // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single"); + break; + } + case OMPRTL__kmpc_omp_task_alloc: { + // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, + // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + // kmp_routine_entry_t *task_entry); + assert(KmpRoutineEntryPtrTy != nullptr && + "Type kmp_routine_entry_t must be created."); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, + CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy}; + // Return void * and then cast to particular kmp_task_t type. + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc"); + break; + } + case OMPRTL__kmpc_omp_task: { + // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task"); + break; + } + case OMPRTL__kmpc_copyprivate: { + // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid, + // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), + // kmp_int32 didit); + llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; + auto *CpyFnTy = + llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy, + CGM.VoidPtrTy, CpyFnTy->getPointerTo(), + CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate"); + break; + } + case OMPRTL__kmpc_reduce: { + // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void + // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck); + llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; + auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams, + /*isVarArg=*/false); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy, + CGM.VoidPtrTy, ReduceFnTy->getPointerTo(), + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce"); + break; + } + case OMPRTL__kmpc_reduce_nowait: { + // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 + // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, + // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name + // *lck); + llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; + auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams, + /*isVarArg=*/false); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy, + CGM.VoidPtrTy, ReduceFnTy->getPointerTo(), + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait"); + break; + } + case OMPRTL__kmpc_end_reduce: { + // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce"); + break; + } + case OMPRTL__kmpc_end_reduce_nowait: { + // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), CGM.Int32Ty, + llvm::PointerType::getUnqual(KmpCriticalNameTy)}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = + CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait"); + break; + } + case OMPRTL__kmpc_omp_task_begin_if0: { + // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = + CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0"); + break; + } + case OMPRTL__kmpc_omp_task_complete_if0: { + // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, + CGM.VoidPtrTy}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, + /*Name=*/"__kmpc_omp_task_complete_if0"); + break; + } + case OMPRTL__kmpc_ordered: { + // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered"); + break; + } + case OMPRTL__kmpc_end_ordered: { + // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered"); + break; + } + case OMPRTL__kmpc_omp_taskwait: { + // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid); + llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); + RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait"); + break; + } } return RTLFn; } +llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4" + : "__kmpc_for_static_init_4u") + : (IVSigned ? "__kmpc_for_static_init_8" + : "__kmpc_for_static_init_8u"); + auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; + auto PtrTy = llvm::PointerType::getUnqual(ITy); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + CGM.Int32Ty, // schedtype + llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter + PtrTy, // p_lower + PtrTy, // p_upper + PtrTy, // p_stride + ITy, // incr + ITy // chunk + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + +llvm::Constant *CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = + IVSize == 32 + ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u") + : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u"); + auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; + llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + CGM.Int32Ty, // schedtype + ITy, // lower + ITy, // upper + ITy, // stride + ITy // chunk + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + +llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = + IVSize == 32 + ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u") + : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u"); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + +llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, + bool IVSigned) { + assert((IVSize == 32 || IVSize == 64) && + "IV size is not compatible with the omp runtime"); + auto Name = + IVSize == 32 + ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u") + : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u"); + auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; + auto PtrTy = llvm::PointerType::getUnqual(ITy); + llvm::Type *TypeParams[] = { + getIdentTyPointerTy(), // loc + CGM.Int32Ty, // tid + llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter + PtrTy, // p_lower + PtrTy, // p_upper + PtrTy // p_stride + }; + llvm::FunctionType *FnTy = + llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); + return CGM.CreateRuntimeFunction(FnTy, Name); +} + llvm::Constant * CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) { // Lookup the entry, lazily creating it if necessary. - return GetOrCreateInternalVariable(CGM.Int8PtrPtrTy, + return getOrCreateInternalVariable(CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)) + ".cache."); } -llvm::Value *CGOpenMPRuntime::getOMPAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - llvm::Value *VDAddr, - SourceLocation Loc) { +llvm::Value *CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, + const VarDecl *VD, + llvm::Value *VDAddr, + SourceLocation Loc) { auto VarTy = VDAddr->getType()->getPointerElementType(); - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - GetOpenMPThreadID(CGF, Loc), + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy), CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), getOrCreateThreadPrivateCache(VD)}; return CGF.EmitRuntimeCall( - CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args); + createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args); } -void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit( +void CGOpenMPRuntime::emitThreadPrivateVarInit( CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) { // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime // library. - auto OMPLoc = EmitOpenMPUpdateLocation(CGF, Loc); - CGF.EmitRuntimeCall(CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), + auto OMPLoc = emitUpdateLocation(CGF, Loc); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), OMPLoc); // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) // to register constructor/destructor for variable. @@ -514,10 +882,10 @@ void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit( CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy), Ctor, CopyCtor, Dtor}; CGF.EmitRuntimeCall( - CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args); + createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args); } -llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition( +llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF) { VD = VD->getDefinition(CGM.getContext()); @@ -620,54 +988,121 @@ llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition( InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction, CGM.getTypes().arrangeNullaryFunction(), ArgList, Loc); - EmitOMPThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); + emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); InitCGF.FinishFunction(); return InitFunction; } - EmitOMPThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); + emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); } return nullptr; } -void CGOpenMPRuntime::EmitOMPParallelCall(CodeGenFunction &CGF, - SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct) { - // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/) - llvm::Value *Args[] = { - EmitOpenMPUpdateLocation(CGF, Loc), - CGF.Builder.getInt32(1), // Number of arguments after 'microtask' argument - // (there is only one additional argument - 'context') - CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()), - CGF.EmitCastToVoidPtr(CapturedStruct)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_fork_call); - CGF.EmitRuntimeCall(RTLFn, Args); -} - -void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF, - SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct) { - auto ThreadID = GetOpenMPThreadID(CGF, Loc); - // Build calls: - // __kmpc_serialized_parallel(&Loc, GTid); - llvm::Value *SerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_serialized_parallel); - CGF.EmitRuntimeCall(RTLFn, SerArgs); - - // OutlinedFn(>id, &zero, CapturedStruct); - auto ThreadIDAddr = EmitThreadIDAddress(CGF, Loc); - auto Int32Ty = - CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); - auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr"); - CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); - llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct}; - CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); +/// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen +/// function. Here is the logic: +/// if (Cond) { +/// ThenGen(); +/// } else { +/// ElseGen(); +/// } +static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, + const RegionCodeGenTy &ThenGen, + const RegionCodeGenTy &ElseGen) { + CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); + + // If the condition constant folds and can be elided, try to avoid emitting + // the condition and the dead arm of the if/else. + bool CondConstant; + if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { + CodeGenFunction::RunCleanupsScope Scope(CGF); + if (CondConstant) { + ThenGen(CGF); + } else { + ElseGen(CGF); + } + return; + } + + // Otherwise, the condition did not fold, or we couldn't elide it. Just + // emit the conditional branch. + auto ThenBlock = CGF.createBasicBlock("omp_if.then"); + auto ElseBlock = CGF.createBasicBlock("omp_if.else"); + auto ContBlock = CGF.createBasicBlock("omp_if.end"); + CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); - // __kmpc_end_serialized_parallel(&Loc, GTid); - llvm::Value *EndSerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID}; - RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel); - CGF.EmitRuntimeCall(RTLFn, EndSerArgs); + // Emit the 'then' code. + CGF.EmitBlock(ThenBlock); + { + CodeGenFunction::RunCleanupsScope ThenScope(CGF); + ThenGen(CGF); + } + CGF.EmitBranch(ContBlock); + // Emit the 'else' code if present. + { + // There is no need to emit line number for unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(CGF); + CGF.EmitBlock(ElseBlock); + } + { + CodeGenFunction::RunCleanupsScope ThenScope(CGF); + ElseGen(CGF); + } + { + // There is no need to emit line number for unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(CGF); + CGF.EmitBranch(ContBlock); + } + // Emit the continuation block for code after the if. + CGF.EmitBlock(ContBlock, /*IsFinished=*/true); +} + +void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, + llvm::Value *OutlinedFn, + llvm::Value *CapturedStruct, + const Expr *IfCond) { + auto *RTLoc = emitUpdateLocation(CGF, Loc); + auto &&ThenGen = + [this, OutlinedFn, CapturedStruct, RTLoc](CodeGenFunction &CGF) { + // Build call __kmpc_fork_call(loc, 1, microtask, + // captured_struct/*context*/) + llvm::Value *Args[] = { + RTLoc, + CGF.Builder.getInt32( + 1), // Number of arguments after 'microtask' argument + // (there is only one additional argument - 'context') + CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()), + CGF.EmitCastToVoidPtr(CapturedStruct)}; + auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call); + CGF.EmitRuntimeCall(RTLFn, Args); + }; + auto &&ElseGen = [this, OutlinedFn, CapturedStruct, RTLoc, Loc]( + CodeGenFunction &CGF) { + auto ThreadID = getThreadID(CGF, Loc); + // Build calls: + // __kmpc_serialized_parallel(&Loc, GTid); + llvm::Value *Args[] = {RTLoc, ThreadID}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), + Args); + + // OutlinedFn(>id, &zero, CapturedStruct); + auto ThreadIDAddr = emitThreadIDAddress(CGF, Loc); + auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, + /*Signed*/ true); + auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr"); + CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); + llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct}; + CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); + + // __kmpc_end_serialized_parallel(&Loc, GTid); + llvm::Value *EndArgs[] = {emitUpdateLocation(CGF, Loc), ThreadID}; + CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel), EndArgs); + }; + if (IfCond) { + emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); + ThenGen(CGF); + } } // If we're inside an (outlined) parallel region, use the region info's @@ -676,13 +1111,14 @@ void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF, // regular serial code region, get thread ID by calling kmp_int32 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and // return the address of that temp. -llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF, +llvm::Value *CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc) { if (auto OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) - return CGF.EmitLoadOfLValue(OMPRegionInfo->getThreadIDVariableLValue(CGF), - SourceLocation()).getScalarVal(); - auto ThreadID = GetOpenMPThreadID(CGF, Loc); + if (OMPRegionInfo->getThreadIDVariable()) + return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(); + + auto ThreadID = getThreadID(CGF, Loc); auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); @@ -693,7 +1129,7 @@ llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF, } llvm::Constant * -CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty, +CGOpenMPRuntime::getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name) { SmallString<256> Buffer; llvm::raw_svector_ostream Out(Buffer); @@ -712,31 +1148,51 @@ CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty, Elem.first()); } -llvm::Value *CGOpenMPRuntime::GetCriticalRegionLock(StringRef CriticalName) { +llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) { llvm::Twine Name(".gomp_critical_user_", CriticalName); - return GetOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var")); + return getOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var")); } -void CGOpenMPRuntime::EmitOMPCriticalRegion( - CodeGenFunction &CGF, StringRef CriticalName, - const std::function<void()> &CriticalOpGen, SourceLocation Loc) { - auto RegionLock = GetCriticalRegionLock(CriticalName); +namespace { +template <size_t N> class CallEndCleanup : public EHScopeStack::Cleanup { + llvm::Value *Callee; + llvm::Value *Args[N]; + +public: + CallEndCleanup(llvm::Value *Callee, ArrayRef<llvm::Value *> CleanupArgs) + : Callee(Callee) { + assert(CleanupArgs.size() == N); + std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args)); + } + void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { + CGF.EmitRuntimeCall(Callee, Args); + } +}; +} // namespace + +void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, + StringRef CriticalName, + const RegionCodeGenTy &CriticalOpGen, + SourceLocation Loc) { // __kmpc_critical(ident_t *, gtid, Lock); // CriticalOpGen(); // __kmpc_end_critical(ident_t *, gtid, Lock); // Prepare arguments and build a call to __kmpc_critical - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - GetOpenMPThreadID(CGF, Loc), RegionLock}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_critical); - CGF.EmitRuntimeCall(RTLFn, Args); - CriticalOpGen(); - // Build a call to __kmpc_end_critical - RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_critical); - CGF.EmitRuntimeCall(RTLFn, Args); -} - -static void EmitOMPIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, - const std::function<void()> &BodyOpGen) { + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + getCriticalRegionLock(CriticalName)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args); + // Build a call to __kmpc_end_critical + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, CriticalOpGen); + } +} + +static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, + const RegionCodeGenTy &BodyOpGen) { llvm::Value *CallBool = CGF.EmitScalarConversion( IfCond, CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true), @@ -747,61 +1203,234 @@ static void EmitOMPIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, // Generate the branch (If-stmt) CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); CGF.EmitBlock(ThenBlock); - BodyOpGen(); + CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, BodyOpGen); // Emit the rest of bblocks/branches CGF.EmitBranch(ContBlock); CGF.EmitBlock(ContBlock, true); } -void CGOpenMPRuntime::EmitOMPMasterRegion( - CodeGenFunction &CGF, const std::function<void()> &MasterOpGen, - SourceLocation Loc) { +void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &MasterOpGen, + SourceLocation Loc) { // if(__kmpc_master(ident_t *, gtid)) { // MasterOpGen(); // __kmpc_end_master(ident_t *, gtid); // } // Prepare arguments and build a call to __kmpc_master - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - GetOpenMPThreadID(CGF, Loc)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_master); - auto *IsMaster = CGF.EmitRuntimeCall(RTLFn, Args); - EmitOMPIfStmt(CGF, IsMaster, [&]() -> void { - MasterOpGen(); - // Build a call to __kmpc_end_master. - // OpenMP [1.2.2 OpenMP Language Terminology] - // For C/C++, an executable statement, possibly compound, with a single - // entry at the top and a single exit at the bottom, or an OpenMP construct. - // * Access to the structured block must not be the result of a branch. - // * The point of exit cannot be a branch out of the structured block. - // * The point of entry must not be a call to setjmp(). - // * longjmp() and throw() must not violate the entry/exit criteria. - // * An expression statement, iteration statement, selection statement, or - // try block is considered to be a structured block if the corresponding - // compound statement obtained by enclosing it in { and } would be a - // structured block. - // It is analyzed in Sema, so we can just call __kmpc_end_master() on - // fallthrough rather than pushing a normal cleanup for it. - RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_master); - CGF.EmitRuntimeCall(RTLFn, Args); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + auto *IsMaster = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args); + typedef CallEndCleanup<std::extent<decltype(Args)>::value> + MasterCallEndCleanup; + emitIfStmt(CGF, IsMaster, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<MasterCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master), + llvm::makeArrayRef(Args)); + MasterOpGen(CGF); }); } -void CGOpenMPRuntime::EmitOMPBarrierCall(CodeGenFunction &CGF, - SourceLocation Loc, bool IsExplicit) { +void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, + SourceLocation Loc) { + // Build call __kmpc_omp_taskyield(loc, thread_id, 0); + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), + llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args); +} + +static llvm::Value *emitCopyprivateCopyFunction( + CodeGenModule &CGM, llvm::Type *ArgsType, + ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, + ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) { + auto &C = CGM.getContext(); + // void copy_func(void *LHSArg, void *RHSArg); + FunctionArgList Args; + ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + Args.push_back(&LHSArg); + Args.push_back(&RHSArg); + FunctionType::ExtInfo EI; + auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration( + C.VoidTy, Args, EI, /*isVariadic=*/false); + auto *Fn = llvm::Function::Create( + CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, + ".omp.copyprivate.copy_func", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn); + CodeGenFunction CGF(CGM); + CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); + // Dest = (void*[n])(LHSArg); + // Src = (void*[n])(RHSArg); + auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg), + CGF.PointerAlignInBytes), + ArgsType); + auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg), + CGF.PointerAlignInBytes), + ArgsType); + // *(Type0*)Dst[0] = *(Type0*)Src[0]; + // *(Type1*)Dst[1] = *(Type1*)Src[1]; + // ... + // *(Typen*)Dst[n] = *(Typen*)Src[n]; + for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) { + auto *DestAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(nullptr, LHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType()))); + auto *SrcAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(nullptr, RHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType()))); + auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl(); + QualType Type = VD->getType(); + CGF.EmitOMPCopy(CGF, Type, DestAddr, SrcAddr, + cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()), + AssignmentOps[I]); + } + CGF.FinishFunction(); + return Fn; +} + +void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &SingleOpGen, + SourceLocation Loc, + ArrayRef<const Expr *> CopyprivateVars, + ArrayRef<const Expr *> SrcExprs, + ArrayRef<const Expr *> DstExprs, + ArrayRef<const Expr *> AssignmentOps) { + assert(CopyprivateVars.size() == SrcExprs.size() && + CopyprivateVars.size() == DstExprs.size() && + CopyprivateVars.size() == AssignmentOps.size()); + auto &C = CGM.getContext(); + // int32 did_it = 0; + // if(__kmpc_single(ident_t *, gtid)) { + // SingleOpGen(); + // __kmpc_end_single(ident_t *, gtid); + // did_it = 1; + // } + // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, + // <copy_func>, did_it); + + llvm::AllocaInst *DidIt = nullptr; + if (!CopyprivateVars.empty()) { + // int32 did_it = 0; + auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); + DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); + CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(0), DidIt, + DidIt->getAlignment()); + } + // Prepare arguments and build a call to __kmpc_single + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + auto *IsSingle = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args); + typedef CallEndCleanup<std::extent<decltype(Args)>::value> + SingleCallEndCleanup; + emitIfStmt(CGF, IsSingle, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<SingleCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single), + llvm::makeArrayRef(Args)); + SingleOpGen(CGF); + if (DidIt) { + // did_it = 1; + CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(1), DidIt, + DidIt->getAlignment()); + } + }); + // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, + // <copy_func>, did_it); + if (DidIt) { + llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size()); + auto CopyprivateArrayTy = + C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal, + /*IndexTypeQuals=*/0); + // Create a list of all private variables for copyprivate. + auto *CopyprivateList = + CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); + for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) { + auto *Elem = CGF.Builder.CreateStructGEP( + CopyprivateList->getAllocatedType(), CopyprivateList, I); + CGF.Builder.CreateAlignedStore( + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.EmitLValue(CopyprivateVars[I]).getAddress(), CGF.VoidPtrTy), + Elem, CGM.PointerAlignInBytes); + } + // Build function that copies private values from single region to all other + // threads in the corresponding parallel region. + auto *CpyFn = emitCopyprivateCopyFunction( + CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), + CopyprivateVars, SrcExprs, DstExprs, AssignmentOps); + auto *BufSize = llvm::ConstantInt::get( + CGM.SizeTy, C.getTypeSizeInChars(CopyprivateArrayTy).getQuantity()); + auto *CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, + CGF.VoidPtrTy); + auto *DidItVal = + CGF.Builder.CreateAlignedLoad(DidIt, CGF.PointerAlignInBytes); + llvm::Value *Args[] = { + emitUpdateLocation(CGF, Loc), // ident_t *<loc> + getThreadID(CGF, Loc), // i32 <gtid> + BufSize, // size_t <buf_size> + CL, // void *<copyprivate list> + CpyFn, // void (*) (void *, void *) <copy_func> + DidItVal // i32 did_it + }; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args); + } +} + +void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &OrderedOpGen, + SourceLocation Loc) { + // __kmpc_ordered(ident_t *, gtid); + // OrderedOpGen(); + // __kmpc_end_ordered(ident_t *, gtid); + // Prepare arguments and build a call to __kmpc_ordered + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args); + // Build a call to __kmpc_end_ordered + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, OrderedOpGen); + } +} + +void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPDirectiveKind Kind) { // Build call __kmpc_cancel_barrier(loc, thread_id); - auto Flags = static_cast<OpenMPLocationFlags>( - OMP_IDENT_KMPC | - (IsExplicit ? OMP_IDENT_BARRIER_EXPL : OMP_IDENT_BARRIER_IMPL)); + OpenMPLocationFlags Flags = OMP_IDENT_KMPC; + if (Kind == OMPD_for) { + Flags = + static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_FOR); + } else if (Kind == OMPD_sections) { + Flags = static_cast<OpenMPLocationFlags>(Flags | + OMP_IDENT_BARRIER_IMPL_SECTIONS); + } else if (Kind == OMPD_single) { + Flags = + static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_SINGLE); + } else if (Kind == OMPD_barrier) { + Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_EXPL); + } else { + Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL); + } // Build call __kmpc_cancel_barrier(loc, thread_id); // Replace __kmpc_barrier() function by __kmpc_cancel_barrier() because this // one provides the same functionality and adds initial support for // cancellation constructs introduced in OpenMP 4.0. __kmpc_cancel_barrier() // is provided default by the runtime library so it safe to make such // replacement. - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, Flags), - GetOpenMPThreadID(CGF, Loc)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_cancel_barrier); - CGF.EmitRuntimeCall(RTLFn, Args); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), + getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args); } /// \brief Schedule types for 'omp for' loops (these enumerators are taken from @@ -817,106 +1446,1017 @@ enum OpenMPSchedType { OMP_sch_auto = 38, /// \brief Lower bound for 'ordered' versions. OMP_ord_lower = 64, - /// \brief Lower bound for 'nomerge' versions. - OMP_nm_lower = 160, + OMP_ord_static_chunked = 65, + OMP_ord_static = 66, + OMP_ord_dynamic_chunked = 67, + OMP_ord_guided_chunked = 68, + OMP_ord_runtime = 69, + OMP_ord_auto = 70, + OMP_sch_default = OMP_sch_static, }; /// \brief Map the OpenMP loop schedule to the runtime enumeration. static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, - bool Chunked) { + bool Chunked, bool Ordered) { switch (ScheduleKind) { case OMPC_SCHEDULE_static: - return Chunked ? OMP_sch_static_chunked : OMP_sch_static; + return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked) + : (Ordered ? OMP_ord_static : OMP_sch_static); case OMPC_SCHEDULE_dynamic: - return OMP_sch_dynamic_chunked; + return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked; case OMPC_SCHEDULE_guided: - return OMP_sch_guided_chunked; - case OMPC_SCHEDULE_auto: - return OMP_sch_auto; + return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked; case OMPC_SCHEDULE_runtime: - return OMP_sch_runtime; + return Ordered ? OMP_ord_runtime : OMP_sch_runtime; + case OMPC_SCHEDULE_auto: + return Ordered ? OMP_ord_auto : OMP_sch_auto; case OMPC_SCHEDULE_unknown: assert(!Chunked && "chunk was specified but schedule kind not known"); - return OMP_sch_static; + return Ordered ? OMP_ord_static : OMP_sch_static; } llvm_unreachable("Unexpected runtime schedule"); } bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const { - auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked); + auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false); return Schedule == OMP_sch_static; } -void CGOpenMPRuntime::EmitOMPForInit(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind ScheduleKind, - unsigned IVSize, bool IVSigned, - llvm::Value *IL, llvm::Value *LB, - llvm::Value *UB, llvm::Value *ST, - llvm::Value *Chunk) { - OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunk != nullptr); - // Call __kmpc_for_static_init( - // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, - // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, - // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, - // kmp_int[32|64] incr, kmp_int[32|64] chunk); - // TODO: Implement dynamic schedule. - - // If the Chunk was not specified in the clause - use default value 1. - if (Chunk == nullptr) - Chunk = CGF.Builder.getIntN(IVSize, /*C*/ 1); +bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const { + auto Schedule = + getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false); + assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here"); + return Schedule != OMP_sch_static; +} + +void CGOpenMPRuntime::emitForInit(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPScheduleClauseKind ScheduleKind, + unsigned IVSize, bool IVSigned, bool Ordered, + llvm::Value *IL, llvm::Value *LB, + llvm::Value *UB, llvm::Value *ST, + llvm::Value *Chunk) { + OpenMPSchedType Schedule = + getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered); + if (Ordered || + (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && + Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked)) { + // Call __kmpc_dispatch_init( + // ident_t *loc, kmp_int32 tid, kmp_int32 schedule, + // kmp_int[32|64] lower, kmp_int[32|64] upper, + // kmp_int[32|64] stride, kmp_int[32|64] chunk); + + // If the Chunk was not specified in the clause - use default value 1. + if (Chunk == nullptr) + Chunk = CGF.Builder.getIntN(IVSize, 1); + llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc), + CGF.Builder.getInt32(Schedule), // Schedule type + CGF.Builder.getIntN(IVSize, 0), // Lower + UB, // Upper + CGF.Builder.getIntN(IVSize, 1), // Stride + Chunk // Chunk + }; + CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); + } else { + // Call __kmpc_for_static_init( + // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, + // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, + // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, + // kmp_int[32|64] incr, kmp_int[32|64] chunk); + if (Chunk == nullptr) { + assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) && + "expected static non-chunked schedule"); + // If the Chunk was not specified in the clause - use default value 1. + Chunk = CGF.Builder.getIntN(IVSize, 1); + } else + assert((Schedule == OMP_sch_static_chunked || + Schedule == OMP_ord_static_chunked) && + "expected static chunked schedule"); + llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc), + CGF.Builder.getInt32(Schedule), // Schedule type + IL, // &isLastIter + LB, // &LB + UB, // &UB + ST, // &Stride + CGF.Builder.getIntN(IVSize, 1), // Incr + Chunk // Chunk + }; + CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args); + } +} +void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, + SourceLocation Loc) { + // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini), + Args); +} + +void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, + SourceLocation Loc, + unsigned IVSize, + bool IVSigned) { + // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), + getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); +} + +llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, + SourceLocation Loc, unsigned IVSize, + bool IVSigned, llvm::Value *IL, + llvm::Value *LB, llvm::Value *UB, + llvm::Value *ST) { + // Call __kmpc_dispatch_next( + // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, + // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, + // kmp_int[32|64] *p_stride); llvm::Value *Args[] = { - EmitOpenMPUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), - GetOpenMPThreadID(CGF, Loc), - CGF.Builder.getInt32(Schedule), // Schedule type - IL, // &isLastIter - LB, // &LB - UB, // &UB - ST, // &Stride - CGF.Builder.getIntN(IVSize, 1), // Incr - Chunk // Chunk + emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc), + IL, // &isLastIter + LB, // &Lower + UB, // &Upper + ST // &Stride }; - assert((IVSize == 32 || IVSize == 64) && - "Index size is not compatible with the omp runtime"); - auto F = IVSize == 32 ? (IVSigned ? OMPRTL__kmpc_for_static_init_4 - : OMPRTL__kmpc_for_static_init_4u) - : (IVSigned ? OMPRTL__kmpc_for_static_init_8 - : OMPRTL__kmpc_for_static_init_8u); - auto RTLFn = CreateRuntimeFunction(F); - CGF.EmitRuntimeCall(RTLFn, Args); -} - -void CGOpenMPRuntime::EmitOMPForFinish(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind ScheduleKind) { - assert((ScheduleKind == OMPC_SCHEDULE_static || - ScheduleKind == OMPC_SCHEDULE_unknown) && - "Non-static schedule kinds are not yet implemented"); - // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), - GetOpenMPThreadID(CGF, Loc)}; - auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_for_static_fini); - CGF.EmitRuntimeCall(RTLFn, Args); + llvm::Value *Call = + CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); + return CGF.EmitScalarConversion( + Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true), + CGF.getContext().BoolTy); } -void CGOpenMPRuntime::EmitOMPNumThreadsClause(CodeGenFunction &CGF, - llvm::Value *NumThreads, - SourceLocation Loc) { +void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, + llvm::Value *NumThreads, + SourceLocation Loc) { // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads) llvm::Value *Args[] = { - EmitOpenMPUpdateLocation(CGF, Loc), GetOpenMPThreadID(CGF, Loc), + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; - llvm::Constant *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_push_num_threads); - CGF.EmitRuntimeCall(RTLFn, Args); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads), + Args); +} + +void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, + SourceLocation Loc) { + // Build call void __kmpc_flush(ident_t *loc) + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush), + emitUpdateLocation(CGF, Loc)); +} + +namespace { +/// \brief Indexes of fields for type kmp_task_t. +enum KmpTaskTFields { + /// \brief List of shared variables. + KmpTaskTShareds, + /// \brief Task routine. + KmpTaskTRoutine, + /// \brief Partition id for the untied tasks. + KmpTaskTPartId, + /// \brief Function with call of destructors for private variables. + KmpTaskTDestructors, +}; +} // namespace + +void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) { + if (!KmpRoutineEntryPtrTy) { + // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type. + auto &C = CGM.getContext(); + QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy}; + FunctionProtoType::ExtProtoInfo EPI; + KmpRoutineEntryPtrQTy = C.getPointerType( + C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI)); + KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy); + } +} + +static void addFieldToRecordDecl(ASTContext &C, DeclContext *DC, + QualType FieldTy) { + auto *Field = FieldDecl::Create( + C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy, + C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()), + /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit); + Field->setAccess(AS_public); + DC->addDecl(Field); +} + +namespace { +struct PrivateHelpersTy { + PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy, + const VarDecl *PrivateElemInit) + : Original(Original), PrivateCopy(PrivateCopy), + PrivateElemInit(PrivateElemInit) {} + const VarDecl *Original; + const VarDecl *PrivateCopy; + const VarDecl *PrivateElemInit; +}; +typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy; +} // namespace + +static RecordDecl * +createPrivatesRecordDecl(CodeGenModule &CGM, + const ArrayRef<PrivateDataTy> Privates) { + if (!Privates.empty()) { + auto &C = CGM.getContext(); + // Build struct .kmp_privates_t. { + // /* private vars */ + // }; + auto *RD = C.buildImplicitRecord(".kmp_privates.t"); + RD->startDefinition(); + for (auto &&Pair : Privates) { + auto Type = Pair.second.Original->getType(); + Type = Type.getNonReferenceType(); + addFieldToRecordDecl(C, RD, Type); + } + RD->completeDefinition(); + return RD; + } + return nullptr; } -void CGOpenMPRuntime::EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, - SourceLocation Loc) { - // Build call void __kmpc_flush(ident_t *loc, ...) - // FIXME: List of variables is ignored by libiomp5 runtime, no need to - // generate it, just request full memory fence. - llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), - llvm::ConstantInt::get(CGM.Int32Ty, 0)}; - auto *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_flush); - CGF.EmitRuntimeCall(RTLFn, Args); +static RecordDecl * +createKmpTaskTRecordDecl(CodeGenModule &CGM, QualType KmpInt32Ty, + QualType KmpRoutineEntryPointerQTy) { + auto &C = CGM.getContext(); + // Build struct kmp_task_t { + // void * shareds; + // kmp_routine_entry_t routine; + // kmp_int32 part_id; + // kmp_routine_entry_t destructors; + // }; + auto *RD = C.buildImplicitRecord("kmp_task_t"); + RD->startDefinition(); + addFieldToRecordDecl(C, RD, C.VoidPtrTy); + addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy); + addFieldToRecordDecl(C, RD, KmpInt32Ty); + addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy); + RD->completeDefinition(); + return RD; +} + +static RecordDecl * +createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, + const ArrayRef<PrivateDataTy> Privates) { + auto &C = CGM.getContext(); + // Build struct kmp_task_t_with_privates { + // kmp_task_t task_data; + // .kmp_privates_t. privates; + // }; + auto *RD = C.buildImplicitRecord("kmp_task_t_with_privates"); + RD->startDefinition(); + addFieldToRecordDecl(C, RD, KmpTaskTQTy); + if (auto *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) { + addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD)); + } + RD->completeDefinition(); + return RD; } + +/// \brief Emit a proxy function which accepts kmp_task_t as the second +/// argument. +/// \code +/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { +/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, +/// tt->shareds); +/// return 0; +/// } +/// \endcode +static llvm::Value * +emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, + QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, + QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, + QualType SharedsPtrTy, llvm::Value *TaskFunction, + llvm::Value *TaskPrivatesMap) { + auto &C = CGM.getContext(); + FunctionArgList Args; + ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty); + ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy); + Args.push_back(&GtidArg); + Args.push_back(&TaskTypeArg); + FunctionType::ExtInfo Info; + auto &TaskEntryFnInfo = + CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info, + /*isVariadic=*/false); + auto *TaskEntryTy = CGM.getTypes().GetFunctionType(TaskEntryFnInfo); + auto *TaskEntry = + llvm::Function::Create(TaskEntryTy, llvm::GlobalValue::InternalLinkage, + ".omp_task_entry.", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskEntryFnInfo, TaskEntry); + CodeGenFunction CGF(CGM); + CGF.disableDebugInfo(); + CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args); + + // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map, + // tt->task_data.shareds); + auto *GtidParam = CGF.EmitLoadOfScalar( + CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, + C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc); + auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes); + LValue TDBase = + CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesQTyRD = + cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); + LValue Base = + CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); + auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl()); + auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId); + auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); + auto *PartidParam = CGF.EmitLoadOfLValue(PartIdLVal, Loc).getScalarVal(); + + auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds); + auto SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); + auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(), + CGF.ConvertTypeForMem(SharedsPtrTy)); + + auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1); + llvm::Value *PrivatesParam; + if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) { + auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); + PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + PrivatesLVal.getAddress(), CGF.VoidPtrTy); + } else { + PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); + } + + llvm::Value *CallArgs[] = {GtidParam, PartidParam, PrivatesParam, + TaskPrivatesMap, SharedsParam}; + CGF.EmitCallOrInvoke(TaskFunction, CallArgs); + CGF.EmitStoreThroughLValue( + RValue::get(CGF.Builder.getInt32(/*C=*/0)), + CGF.MakeNaturalAlignAddrLValue(CGF.ReturnValue, KmpInt32Ty)); + CGF.FinishFunction(); + return TaskEntry; +} + +static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, + SourceLocation Loc, + QualType KmpInt32Ty, + QualType KmpTaskTWithPrivatesPtrQTy, + QualType KmpTaskTWithPrivatesQTy) { + auto &C = CGM.getContext(); + FunctionArgList Args; + ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty); + ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy); + Args.push_back(&GtidArg); + Args.push_back(&TaskTypeArg); + FunctionType::ExtInfo Info; + auto &DestructorFnInfo = + CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info, + /*isVariadic=*/false); + auto *DestructorFnTy = CGM.getTypes().GetFunctionType(DestructorFnInfo); + auto *DestructorFn = + llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage, + ".omp_task_destructor.", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, DestructorFnInfo, DestructorFn); + CodeGenFunction CGF(CGM); + CGF.disableDebugInfo(); + CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, + Args); + + auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes); + LValue Base = + CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesQTyRD = + cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); + auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); + Base = CGF.EmitLValueForField(Base, *FI); + for (auto *Field : + cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) { + if (auto DtorKind = Field->getType().isDestructedType()) { + auto FieldLValue = CGF.EmitLValueForField(Base, Field); + CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType()); + } + } + CGF.FinishFunction(); + return DestructorFn; +} + +/// \brief Emit a privates mapping function for correct handling of private and +/// firstprivate variables. +/// \code +/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1> +/// **noalias priv1,..., <tyn> **noalias privn) { +/// *priv1 = &.privates.priv1; +/// ...; +/// *privn = &.privates.privn; +/// } +/// \endcode +static llvm::Value * +emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, + const ArrayRef<const Expr *> PrivateVars, + const ArrayRef<const Expr *> FirstprivateVars, + QualType PrivatesQTy, + const ArrayRef<PrivateDataTy> Privates) { + auto &C = CGM.getContext(); + FunctionArgList Args; + ImplicitParamDecl TaskPrivatesArg( + C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, + C.getPointerType(PrivatesQTy).withConst().withRestrict()); + Args.push_back(&TaskPrivatesArg); + llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos; + unsigned Counter = 1; + for (auto *E: PrivateVars) { + Args.push_back(ImplicitParamDecl::Create( + C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType())) + .withConst() + .withRestrict())); + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + PrivateVarsPos[VD] = Counter; + ++Counter; + } + for (auto *E : FirstprivateVars) { + Args.push_back(ImplicitParamDecl::Create( + C, /*DC=*/nullptr, Loc, + /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType())) + .withConst() + .withRestrict())); + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + PrivateVarsPos[VD] = Counter; + ++Counter; + } + FunctionType::ExtInfo Info; + auto &TaskPrivatesMapFnInfo = + CGM.getTypes().arrangeFreeFunctionDeclaration(C.VoidTy, Args, Info, + /*isVariadic=*/false); + auto *TaskPrivatesMapTy = + CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo); + auto *TaskPrivatesMap = llvm::Function::Create( + TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, + ".omp_task_privates_map.", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo, + TaskPrivatesMap); + TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline); + CodeGenFunction CGF(CGM); + CGF.disableDebugInfo(); + CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, + TaskPrivatesMapFnInfo, Args); + + // *privi = &.privates.privi; + auto *TaskPrivatesArgAddr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(&TaskPrivatesArg), CGM.PointerAlignInBytes); + LValue Base = + CGF.MakeNaturalAlignAddrLValue(TaskPrivatesArgAddr, PrivatesQTy); + auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl()); + Counter = 0; + for (auto *Field : PrivatesQTyRD->fields()) { + auto FieldLVal = CGF.EmitLValueForField(Base, Field); + auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]]; + auto RefLVal = CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(VD), + VD->getType()); + auto RefLoadRVal = CGF.EmitLoadOfLValue(RefLVal, Loc); + CGF.EmitStoreOfScalar( + FieldLVal.getAddress(), + CGF.MakeNaturalAlignAddrLValue(RefLoadRVal.getScalarVal(), + RefLVal.getType()->getPointeeType())); + ++Counter; + } + CGF.FinishFunction(); + return TaskPrivatesMap; +} + +static int array_pod_sort_comparator(const PrivateDataTy *P1, + const PrivateDataTy *P2) { + return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0); +} + +void CGOpenMPRuntime::emitTaskCall( + CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, + bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final, + llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds, + const Expr *IfCond, const ArrayRef<const Expr *> PrivateVars, + const ArrayRef<const Expr *> PrivateCopies, + const ArrayRef<const Expr *> FirstprivateVars, + const ArrayRef<const Expr *> FirstprivateCopies, + const ArrayRef<const Expr *> FirstprivateInits) { + auto &C = CGM.getContext(); + llvm::SmallVector<PrivateDataTy, 8> Privates; + // Aggregate privates and sort them by the alignment. + auto I = PrivateCopies.begin(); + for (auto *E : PrivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + Privates.push_back(std::make_pair( + C.getTypeAlignInChars(VD->getType()), + PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), + /*PrivateElemInit=*/nullptr))); + ++I; + } + I = FirstprivateCopies.begin(); + auto IElemInitRef = FirstprivateInits.begin(); + for (auto *E : FirstprivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + Privates.push_back(std::make_pair( + C.getTypeAlignInChars(VD->getType()), + PrivateHelpersTy( + VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), + cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())))); + ++I, ++IElemInitRef; + } + llvm::array_pod_sort(Privates.begin(), Privates.end(), + array_pod_sort_comparator); + auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); + // Build type kmp_routine_entry_t (if not built yet). + emitKmpRoutineEntryT(KmpInt32Ty); + // Build type kmp_task_t (if not built yet). + if (KmpTaskTQTy.isNull()) { + KmpTaskTQTy = C.getRecordType( + createKmpTaskTRecordDecl(CGM, KmpInt32Ty, KmpRoutineEntryPtrQTy)); + } + auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl()); + // Build particular struct kmp_task_t for the given task. + auto *KmpTaskTWithPrivatesQTyRD = + createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates); + auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD); + QualType KmpTaskTWithPrivatesPtrQTy = + C.getPointerType(KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); + auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo(); + auto KmpTaskTWithPrivatesTySize = + CGM.getSize(C.getTypeSizeInChars(KmpTaskTWithPrivatesQTy)); + QualType SharedsPtrTy = C.getPointerType(SharedsTy); + + // Emit initial values for private copies (if any). + llvm::Value *TaskPrivatesMap = nullptr; + auto *TaskPrivatesMapTy = + std::next(cast<llvm::Function>(TaskFunction)->getArgumentList().begin(), + 3) + ->getType(); + if (!Privates.empty()) { + auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); + TaskPrivatesMap = emitTaskPrivateMappingFunction( + CGM, Loc, PrivateVars, FirstprivateVars, FI->getType(), Privates); + TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + TaskPrivatesMap, TaskPrivatesMapTy); + } else { + TaskPrivatesMap = llvm::ConstantPointerNull::get( + cast<llvm::PointerType>(TaskPrivatesMapTy)); + } + // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid, + // kmp_task_t *tt); + auto *TaskEntry = emitProxyTaskFunction( + CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTy, + KmpTaskTQTy, SharedsPtrTy, TaskFunction, TaskPrivatesMap); + + // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, + // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + // kmp_routine_entry_t *task_entry); + // Task flags. Format is taken from + // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h, + // description of kmp_tasking_flags struct. + const unsigned TiedFlag = 0x1; + const unsigned FinalFlag = 0x2; + unsigned Flags = Tied ? TiedFlag : 0; + auto *TaskFlags = + Final.getPointer() + ? CGF.Builder.CreateSelect(Final.getPointer(), + CGF.Builder.getInt32(FinalFlag), + CGF.Builder.getInt32(/*C=*/0)) + : CGF.Builder.getInt32(Final.getInt() ? FinalFlag : 0); + TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); + auto SharedsSize = C.getTypeSizeInChars(SharedsTy); + llvm::Value *AllocArgs[] = { + emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), TaskFlags, + KmpTaskTWithPrivatesTySize, CGM.getSize(SharedsSize), + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskEntry, + KmpRoutineEntryPtrTy)}; + auto *NewTask = CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs); + auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + NewTask, KmpTaskTWithPrivatesPtrTy); + LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, + KmpTaskTWithPrivatesQTy); + LValue TDBase = + CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); + // Fill the data in the resulting kmp_task_t record. + // Copy shareds if there are any. + llvm::Value *KmpTaskSharedsPtr = nullptr; + if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) { + KmpTaskSharedsPtr = CGF.EmitLoadOfScalar( + CGF.EmitLValueForField( + TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)), + Loc); + CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy); + } + // Emit initial values for private copies (if any). + bool NeedsCleanup = false; + if (!Privates.empty()) { + auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); + auto PrivatesBase = CGF.EmitLValueForField(Base, *FI); + FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin(); + LValue SharedsBase; + if (!FirstprivateVars.empty()) { + SharedsBase = CGF.MakeNaturalAlignAddrLValue( + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), + SharedsTy); + } + CodeGenFunction::CGCapturedStmtInfo CapturesInfo( + cast<CapturedStmt>(*D.getAssociatedStmt())); + for (auto &&Pair : Privates) { + auto *VD = Pair.second.PrivateCopy; + auto *Init = VD->getAnyInitializer(); + LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); + if (Init) { + if (auto *Elem = Pair.second.PrivateElemInit) { + auto *OriginalVD = Pair.second.Original; + auto *SharedField = CapturesInfo.lookup(OriginalVD); + auto SharedRefLValue = + CGF.EmitLValueForField(SharedsBase, SharedField); + QualType Type = OriginalVD->getType(); + if (Type->isArrayType()) { + // Initialize firstprivate array. + if (!isa<CXXConstructExpr>(Init) || + CGF.isTrivialInitializer(Init)) { + // Perform simple memcpy. + CGF.EmitAggregateAssign(PrivateLValue.getAddress(), + SharedRefLValue.getAddress(), Type); + } else { + // Initialize firstprivate array using element-by-element + // intialization. + CGF.EmitOMPAggregateAssign( + PrivateLValue.getAddress(), SharedRefLValue.getAddress(), + Type, [&CGF, Elem, Init, &CapturesInfo]( + llvm::Value *DestElement, llvm::Value *SrcElement) { + // Clean up any temporaries needed by the initialization. + CodeGenFunction::OMPPrivateScope InitScope(CGF); + InitScope.addPrivate(Elem, [SrcElement]() -> llvm::Value *{ + return SrcElement; + }); + (void)InitScope.Privatize(); + // Emit initialization for single element. + auto *OldCapturedStmtInfo = CGF.CapturedStmtInfo; + CGF.CapturedStmtInfo = &CapturesInfo; + CGF.EmitAnyExprToMem(Init, DestElement, + Init->getType().getQualifiers(), + /*IsInitializer=*/false); + CGF.CapturedStmtInfo = OldCapturedStmtInfo; + }); + } + } else { + CodeGenFunction::OMPPrivateScope InitScope(CGF); + InitScope.addPrivate(Elem, [SharedRefLValue]() -> llvm::Value *{ + return SharedRefLValue.getAddress(); + }); + (void)InitScope.Privatize(); + auto *OldCapturedStmtInfo = CGF.CapturedStmtInfo; + CGF.CapturedStmtInfo = &CapturesInfo; + CGF.EmitExprAsInit(Init, VD, PrivateLValue, + /*capturedByInit=*/false); + CGF.CapturedStmtInfo = OldCapturedStmtInfo; + } + } else { + CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); + } + } + NeedsCleanup = NeedsCleanup || FI->getType().isDestructedType(); + ++FI; + } + } + // Provide pointer to function with destructors for privates. + llvm::Value *DestructorFn = + NeedsCleanup ? emitDestructorsFunction(CGM, Loc, KmpInt32Ty, + KmpTaskTWithPrivatesPtrQTy, + KmpTaskTWithPrivatesQTy) + : llvm::ConstantPointerNull::get( + cast<llvm::PointerType>(KmpRoutineEntryPtrTy)); + LValue Destructor = CGF.EmitLValueForField( + TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTDestructors)); + CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + DestructorFn, KmpRoutineEntryPtrTy), + Destructor); + // NOTE: routine and part_id fields are intialized by __kmpc_omp_task_alloc() + // libcall. + // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t + // *new_task); + auto *ThreadID = getThreadID(CGF, Loc); + llvm::Value *TaskArgs[] = {emitUpdateLocation(CGF, Loc), ThreadID, NewTask}; + auto &&ThenCodeGen = [this, &TaskArgs](CodeGenFunction &CGF) { + // TODO: add check for untied tasks. + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs); + }; + typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value> + IfCallEndCleanup; + auto &&ElseCodeGen = + [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry]( + CodeGenFunction &CGF) { + CodeGenFunction::RunCleanupsScope LocalScope(CGF); + CGF.EmitRuntimeCall( + createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs); + // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, + // kmp_task_t *new_task); + CGF.EHStack.pushCleanup<IfCallEndCleanup>( + NormalAndEHCleanup, + createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), + llvm::makeArrayRef(TaskArgs)); + + // Call proxy_task_entry(gtid, new_task); + llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy}; + CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs); + }; + if (IfCond) { + emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); + ThenCodeGen(CGF); + } +} + +static llvm::Value *emitReductionFunction(CodeGenModule &CGM, + llvm::Type *ArgsType, + ArrayRef<const Expr *> LHSExprs, + ArrayRef<const Expr *> RHSExprs, + ArrayRef<const Expr *> ReductionOps) { + auto &C = CGM.getContext(); + + // void reduction_func(void *LHSArg, void *RHSArg); + FunctionArgList Args; + ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr, + C.VoidPtrTy); + Args.push_back(&LHSArg); + Args.push_back(&RHSArg); + FunctionType::ExtInfo EI; + auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration( + C.VoidTy, Args, EI, /*isVariadic=*/false); + auto *Fn = llvm::Function::Create( + CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage, + ".omp.reduction.reduction_func", &CGM.getModule()); + CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn); + CodeGenFunction CGF(CGM); + CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); + + // Dst = (void*[n])(LHSArg); + // Src = (void*[n])(RHSArg); + auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg), + CGF.PointerAlignInBytes), + ArgsType); + auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg), + CGF.PointerAlignInBytes), + ArgsType); + + // ... + // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); + // ... + CodeGenFunction::OMPPrivateScope Scope(CGF); + for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I) { + Scope.addPrivate( + cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()), + [&]() -> llvm::Value *{ + return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, RHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(RHSExprs[I]->getType()))); + }); + Scope.addPrivate( + cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()), + [&]() -> llvm::Value *{ + return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.Builder.CreateAlignedLoad( + CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, LHS, I), + CGM.PointerAlignInBytes), + CGF.ConvertTypeForMem(C.getPointerType(LHSExprs[I]->getType()))); + }); + } + Scope.Privatize(); + for (auto *E : ReductionOps) { + CGF.EmitIgnoredExpr(E); + } + Scope.ForceCleanup(); + CGF.FinishFunction(); + return Fn; +} + +void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, + ArrayRef<const Expr *> LHSExprs, + ArrayRef<const Expr *> RHSExprs, + ArrayRef<const Expr *> ReductionOps, + bool WithNowait) { + // Next code should be emitted for reduction: + // + // static kmp_critical_name lock = { 0 }; + // + // void reduce_func(void *lhs[<n>], void *rhs[<n>]) { + // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]); + // ... + // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1], + // *(Type<n>-1*)rhs[<n>-1]); + // } + // + // ... + // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; + // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), + // RedList, reduce_func, &<lock>)) { + // case 1: + // ... + // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + // ... + // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + // break; + // case 2: + // ... + // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); + // ... + // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);] + // break; + // default:; + // } + + auto &C = CGM.getContext(); + + // 1. Build a list of reduction variables. + // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; + llvm::APInt ArraySize(/*unsigned int numBits=*/32, RHSExprs.size()); + QualType ReductionArrayTy = + C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal, + /*IndexTypeQuals=*/0); + auto *ReductionList = + CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); + for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I) { + auto *Elem = CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, ReductionList, I); + CGF.Builder.CreateAlignedStore( + CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( + CGF.EmitLValue(RHSExprs[I]).getAddress(), CGF.VoidPtrTy), + Elem, CGM.PointerAlignInBytes); + } + + // 2. Emit reduce_func(). + auto *ReductionFn = emitReductionFunction( + CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), LHSExprs, + RHSExprs, ReductionOps); + + // 3. Create static kmp_critical_name lock = { 0 }; + auto *Lock = getCriticalRegionLock(".reduction"); + + // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), + // RedList, reduce_func, &<lock>); + auto *IdentTLoc = emitUpdateLocation( + CGF, Loc, + static_cast<OpenMPLocationFlags>(OMP_IDENT_KMPC | OMP_ATOMIC_REDUCE)); + auto *ThreadId = getThreadID(CGF, Loc); + auto *ReductionArrayTySize = llvm::ConstantInt::get( + CGM.SizeTy, C.getTypeSizeInChars(ReductionArrayTy).getQuantity()); + auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList, + CGF.VoidPtrTy); + llvm::Value *Args[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> + ReductionArrayTySize, // size_type sizeof(RedList) + RL, // void *RedList + ReductionFn, // void (*) (void *, void *) <reduce_func> + Lock // kmp_critical_name *&<lock> + }; + auto Res = CGF.EmitRuntimeCall( + createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait + : OMPRTL__kmpc_reduce), + Args); + + // 5. Build switch(res) + auto *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); + auto *SwInst = CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); + + // 6. Build case 1: + // ... + // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + // ... + // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + // break; + auto *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); + SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); + CGF.EmitBlock(Case1BB); + + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + llvm::Value *EndArgs[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + Lock // kmp_critical_name *&<lock> + }; + CGF.EHStack + .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>( + NormalAndEHCleanup, + createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait + : OMPRTL__kmpc_end_reduce), + llvm::makeArrayRef(EndArgs)); + for (auto *E : ReductionOps) { + CGF.EmitIgnoredExpr(E); + } + } + + CGF.EmitBranch(DefaultBB); + + // 7. Build case 2: + // ... + // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); + // ... + // break; + auto *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); + SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); + CGF.EmitBlock(Case2BB); + + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + if (!WithNowait) { + // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>); + llvm::Value *EndArgs[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + Lock // kmp_critical_name *&<lock> + }; + CGF.EHStack + .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>( + NormalAndEHCleanup, + createRuntimeFunction(OMPRTL__kmpc_end_reduce), + llvm::makeArrayRef(EndArgs)); + } + auto I = LHSExprs.begin(); + for (auto *E : ReductionOps) { + const Expr *XExpr = nullptr; + const Expr *EExpr = nullptr; + const Expr *UpExpr = nullptr; + BinaryOperatorKind BO = BO_Comma; + if (auto *BO = dyn_cast<BinaryOperator>(E)) { + if (BO->getOpcode() == BO_Assign) { + XExpr = BO->getLHS(); + UpExpr = BO->getRHS(); + } + } + // Try to emit update expression as a simple atomic. + auto *RHSExpr = UpExpr; + if (RHSExpr) { + // Analyze RHS part of the whole expression. + if (auto *ACO = dyn_cast<AbstractConditionalOperator>( + RHSExpr->IgnoreParenImpCasts())) { + // If this is a conditional operator, analyze its condition for + // min/max reduction operator. + RHSExpr = ACO->getCond(); + } + if (auto *BORHS = + dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { + EExpr = BORHS->getRHS(); + BO = BORHS->getOpcode(); + } + } + if (XExpr) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); + LValue X = CGF.EmitLValue(XExpr); + RValue E; + if (EExpr) + E = CGF.EmitAnyExpr(EExpr); + CGF.EmitOMPAtomicSimpleUpdateExpr( + X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc, + [&CGF, UpExpr, VD](RValue XRValue) { + CodeGenFunction::OMPPrivateScope PrivateScope(CGF); + PrivateScope.addPrivate( + VD, [&CGF, VD, XRValue]() -> llvm::Value *{ + auto *LHSTemp = CGF.CreateMemTemp(VD->getType()); + CGF.EmitStoreThroughLValue( + XRValue, + CGF.MakeNaturalAlignAddrLValue(LHSTemp, VD->getType())); + return LHSTemp; + }); + (void)PrivateScope.Privatize(); + return CGF.EmitAnyExpr(UpExpr); + }); + } else { + // Emit as a critical region. + emitCriticalRegion(CGF, ".atomic_reduction", [E](CodeGenFunction &CGF) { + CGF.EmitIgnoredExpr(E); + }, Loc); + } + ++I; + } + } + + CGF.EmitBranch(DefaultBB); + CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); +} + +void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, + SourceLocation Loc) { + // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 + // global_tid); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + // Ignore return result until untied tasks are supported. + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args); +} + +void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, + const RegionCodeGenTy &CodeGen) { + InlinedOpenMPRegionRAII Region(CGF, CodeGen); + CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); +} + diff --git a/lib/CodeGen/CGOpenMPRuntime.h b/lib/CodeGen/CGOpenMPRuntime.h index 6daf8179c14e..f5aa4a51df93 100644 --- a/lib/CodeGen/CGOpenMPRuntime.h +++ b/lib/CodeGen/CGOpenMPRuntime.h @@ -14,6 +14,7 @@ #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H +#include "clang/AST/Type.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/SourceLocation.h" #include "llvm/ADT/DenseMap.h" @@ -42,9 +43,9 @@ namespace CodeGen { class CodeGenFunction; class CodeGenModule; -class CGOpenMPRuntime { -public: +typedef llvm::function_ref<void(CodeGenFunction &)> RegionCodeGenTy; +class CGOpenMPRuntime { private: enum OpenMPRTLFunction { /// \brief Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, @@ -67,11 +68,7 @@ private: // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32 // global_tid); OMPRTL__kmpc_cancel_barrier, - // Calls for static scheduling 'omp for' loops. - OMPRTL__kmpc_for_static_init_4, - OMPRTL__kmpc_for_static_init_4u, - OMPRTL__kmpc_for_static_init_8, - OMPRTL__kmpc_for_static_init_8u, + // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid); OMPRTL__kmpc_for_static_fini, // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 // global_tid); @@ -82,12 +79,58 @@ private: // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, // kmp_int32 num_threads); OMPRTL__kmpc_push_num_threads, - // Call to void __kmpc_flush(ident_t *loc, ...); + // Call to void __kmpc_flush(ident_t *loc); OMPRTL__kmpc_flush, // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid); OMPRTL__kmpc_master, // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid); OMPRTL__kmpc_end_master, + // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid, + // int end_part); + OMPRTL__kmpc_omp_taskyield, + // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid); + OMPRTL__kmpc_single, + // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid); + OMPRTL__kmpc_end_single, + // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, + // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + // kmp_routine_entry_t *task_entry); + OMPRTL__kmpc_omp_task_alloc, + // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t * + // new_task); + OMPRTL__kmpc_omp_task, + // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid, + // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), + // kmp_int32 didit); + OMPRTL__kmpc_copyprivate, + // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void + // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck); + OMPRTL__kmpc_reduce, + // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 + // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, + // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name + // *lck); + OMPRTL__kmpc_reduce_nowait, + // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + OMPRTL__kmpc_end_reduce, + // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, + // kmp_critical_name *lck); + OMPRTL__kmpc_end_reduce_nowait, + // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, + // kmp_task_t * new_task); + OMPRTL__kmpc_omp_task_begin_if0, + // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, + // kmp_task_t * new_task); + OMPRTL__kmpc_omp_task_complete_if0, + // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_ordered, + // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid); + OMPRTL__kmpc_end_ordered, + // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 + // global_tid); + OMPRTL__kmpc_omp_taskwait, }; /// \brief Values for bit flags used in the ident_t to describe the fields. @@ -118,7 +161,7 @@ private: /// \brief Map of flags and corresponding default locations. typedef llvm::DenseMap<unsigned, llvm::Value *> OpenMPDefaultLocMapTy; OpenMPDefaultLocMapTy OpenMPDefaultLocMap; - llvm::Value *GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags); + llvm::Value *getOrCreateDefaultLocation(OpenMPLocationFlags Flags); /// \brief Describes ident structure that describes a source location. /// All descriptions are taken from /// http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h @@ -186,13 +229,28 @@ private: /// variables. llvm::StringMap<llvm::AssertingVH<llvm::Constant>, llvm::BumpPtrAllocator> InternalVars; + /// \brief Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); + llvm::Type *KmpRoutineEntryPtrTy; + QualType KmpRoutineEntryPtrQTy; + /// \brief Type typedef struct kmp_task { + /// void * shareds; /**< pointer to block of pointers to + /// shared vars */ + /// kmp_routine_entry_t routine; /**< pointer to routine to call for + /// executing task */ + /// kmp_int32 part_id; /**< part id for the task */ + /// kmp_routine_entry_t destructors; /* pointer to function to invoke + /// deconstructors of firstprivate C++ objects */ + /// } kmp_task_t; + QualType KmpTaskTQTy; + + /// \brief Build type kmp_routine_entry_t (if not built yet). + void emitKmpRoutineEntryT(QualType KmpInt32Ty); /// \brief Emits object of ident_t type with info for source location. /// \param Flags Flags for OpenMP location. /// - llvm::Value * - EmitOpenMPUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPLocationFlags Flags = OMP_IDENT_KMPC); + llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPLocationFlags Flags = OMP_IDENT_KMPC); /// \brief Returns pointer to ident_t type. llvm::Type *getIdentTyPointerTy(); @@ -203,7 +261,23 @@ private: /// \brief Returns specified OpenMP runtime function. /// \param Function OpenMP runtime function. /// \return Specified function. - llvm::Constant *CreateRuntimeFunction(OpenMPRTLFunction Function); + llvm::Constant *createRuntimeFunction(OpenMPRTLFunction Function); + + /// \brief Returns __kmpc_for_static_init_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createForStaticInitFunction(unsigned IVSize, bool IVSigned); + + /// \brief Returns __kmpc_dispatch_init_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createDispatchInitFunction(unsigned IVSize, bool IVSigned); + + /// \brief Returns __kmpc_dispatch_next_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createDispatchNextFunction(unsigned IVSize, bool IVSigned); + + /// \brief Returns __kmpc_dispatch_fini_* runtime function for the specified + /// size \a IVSize and sign \a IVSigned. + llvm::Constant *createDispatchFiniFunction(unsigned IVSize, bool IVSigned); /// \brief If the specified mangled name is not in the module, create and /// return threadprivate cache object. This object is a pointer's worth of @@ -214,12 +288,12 @@ private: /// \brief Emits address of the word in a memory where current thread id is /// stored. - virtual llvm::Value *EmitThreadIDAddress(CodeGenFunction &CGF, + virtual llvm::Value *emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc); /// \brief Gets thread id value for the current thread. /// - llvm::Value *GetOpenMPThreadID(CodeGenFunction &CGF, SourceLocation Loc); + llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc); /// \brief Gets (if variable with the given name already exist) or creates /// internal global variable with the specified Name. The created variable has @@ -227,7 +301,7 @@ private: /// \param Ty Type of the global variable. If it is exist already the type /// must be the same. /// \param Name Name of the variable. - llvm::Constant *GetOrCreateInternalVariable(llvm::Type *Ty, + llvm::Constant *getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name); /// \brief Set of threadprivate variables with the generated initializer. @@ -239,78 +313,105 @@ private: /// \param CopyCtor Pointer to a global copy function for \a VD. /// \param Dtor Pointer to a global destructor function for \a VD. /// \param Loc Location of threadprivate declaration. - void EmitOMPThreadPrivateVarInit(CodeGenFunction &CGF, llvm::Value *VDAddr, - llvm::Value *Ctor, llvm::Value *CopyCtor, - llvm::Value *Dtor, SourceLocation Loc); + void emitThreadPrivateVarInit(CodeGenFunction &CGF, llvm::Value *VDAddr, + llvm::Value *Ctor, llvm::Value *CopyCtor, + llvm::Value *Dtor, SourceLocation Loc); /// \brief Returns corresponding lock object for the specified critical region /// name. If the lock object does not exist it is created, otherwise the /// reference to the existing copy is returned. /// \param CriticalName Name of the critical region. /// - llvm::Value *GetCriticalRegionLock(StringRef CriticalName); + llvm::Value *getCriticalRegionLock(StringRef CriticalName); public: explicit CGOpenMPRuntime(CodeGenModule &CGM); virtual ~CGOpenMPRuntime() {} + virtual void clear(); - /// \brief Emits outlined function for the specified OpenMP directive \a D - /// (required for parallel and task directives). This outlined function has - /// type void(*)(kmp_int32 /*ThreadID*/, kmp_int32 /*BoundID*/, struct - /// context_vars*). + /// \brief Emits outlined function for the specified OpenMP parallel directive + /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, + /// kmp_int32 BoundID, struct context_vars*). /// \param D OpenMP directive. /// \param ThreadIDVar Variable for thread id in the current OpenMP region. - /// + /// \param CodeGen Code generation sequence for the \a D directive. virtual llvm::Value * - EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D, - const VarDecl *ThreadIDVar); + emitParallelOutlinedFunction(const OMPExecutableDirective &D, + const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen); - /// \brief Cleans up references to the objects in finished function. + /// \brief Emits outlined function for the OpenMP task directive \a D. This + /// outlined function has type void(*)(kmp_int32 ThreadID, kmp_int32 + /// PartID, struct context_vars*). + /// \param D OpenMP directive. + /// \param ThreadIDVar Variable for thread id in the current OpenMP region. + /// \param CodeGen Code generation sequence for the \a D directive. /// - void FunctionFinished(CodeGenFunction &CGF); + virtual llvm::Value *emitTaskOutlinedFunction(const OMPExecutableDirective &D, + const VarDecl *ThreadIDVar, + const RegionCodeGenTy &CodeGen); - /// \brief Emits code for parallel call of the \a OutlinedFn with variables - /// captured in a record which address is stored in \a CapturedStruct. - /// \param OutlinedFn Outlined function to be run in parallel threads. Type of - /// this function is void(*)(kmp_int32, kmp_int32, struct context_vars*). - /// \param CapturedStruct A pointer to the record with the references to - /// variables used in \a OutlinedFn function. + /// \brief Cleans up references to the objects in finished function. /// - virtual void EmitOMPParallelCall(CodeGenFunction &CGF, SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct); + void functionFinished(CodeGenFunction &CGF); - /// \brief Emits code for serial call of the \a OutlinedFn with variables - /// captured in a record which address is stored in \a CapturedStruct. - /// \param OutlinedFn Outlined function to be run in serial mode. + /// \brief Emits code for parallel or serial call of the \a OutlinedFn with + /// variables captured in a record which address is stored in \a + /// CapturedStruct. + /// \param OutlinedFn Outlined function to be run in parallel threads. Type of + /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). /// \param CapturedStruct A pointer to the record with the references to /// variables used in \a OutlinedFn function. + /// \param IfCond Condition in the associated 'if' clause, if it was + /// specified, nullptr otherwise. /// - virtual void EmitOMPSerialCall(CodeGenFunction &CGF, SourceLocation Loc, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct); + virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, + llvm::Value *OutlinedFn, + llvm::Value *CapturedStruct, + const Expr *IfCond); /// \brief Emits a critical region. /// \param CriticalName Name of the critical region. /// \param CriticalOpGen Generator for the statement associated with the given /// critical region. - virtual void EmitOMPCriticalRegion(CodeGenFunction &CGF, - StringRef CriticalName, - const std::function<void()> &CriticalOpGen, - SourceLocation Loc); + virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, + const RegionCodeGenTy &CriticalOpGen, + SourceLocation Loc); /// \brief Emits a master region. /// \param MasterOpGen Generator for the statement associated with the given /// master region. - virtual void EmitOMPMasterRegion(CodeGenFunction &CGF, - const std::function<void()> &MasterOpGen, - SourceLocation Loc); + virtual void emitMasterRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &MasterOpGen, + SourceLocation Loc); + + /// \brief Emits code for a taskyield directive. + virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc); + + /// \brief Emits a single region. + /// \param SingleOpGen Generator for the statement associated with the given + /// single region. + virtual void emitSingleRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &SingleOpGen, + SourceLocation Loc, + ArrayRef<const Expr *> CopyprivateVars, + ArrayRef<const Expr *> DestExprs, + ArrayRef<const Expr *> SrcExprs, + ArrayRef<const Expr *> AssignmentOps); + + /// \brief Emit an ordered region. + /// \param OrderedOpGen Generator for the statement associated with the given + /// critical region. + virtual void emitOrderedRegion(CodeGenFunction &CGF, + const RegionCodeGenTy &OrderedOpGen, + SourceLocation Loc); - /// \brief Emits explicit barrier for OpenMP threads. - /// \param IsExplicit true, if it is explicitly specified barrier. + /// \brief Emit an implicit/explicit barrier for OpenMP threads. + /// \param Kind Directive for which this implicit barrier call must be + /// generated. Must be OMPD_barrier for explicit barrier generation. /// - virtual void EmitOMPBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, - bool IsExplicit = true); + virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPDirectiveKind Kind); /// \brief Check if the specified \a ScheduleKind is static non-chunked. /// This kind of worksharing directive is emitted without outer loop. @@ -320,6 +421,12 @@ public: virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const; + /// \brief Check if the specified \a ScheduleKind is dynamic. + /// This kind of worksharing directive is emitted without outer loop. + /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause. + /// + virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const; + /// \brief Call the appropriate runtime routine to initialize it before start /// of loop. /// @@ -332,6 +439,7 @@ public: /// \param SchedKind Schedule kind, specified by the 'schedule' clause. /// \param IVSize Size of the iteration variable in bits. /// \param IVSigned Sign of the interation variable. + /// \param Ordered true if loop is ordered, false otherwise. /// \param IL Address of the output variable in which the flag of the /// last iteration is returned. /// \param LB Address of the output variable in which the lower iteration @@ -343,29 +451,58 @@ public: /// \param Chunk Value of the chunk for the static_chunked scheduled loop. /// For the default (nullptr) value, the chunk 1 will be used. /// - virtual void EmitOMPForInit(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind SchedKind, - unsigned IVSize, bool IVSigned, llvm::Value *IL, - llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, - llvm::Value *Chunk = nullptr); + virtual void emitForInit(CodeGenFunction &CGF, SourceLocation Loc, + OpenMPScheduleClauseKind SchedKind, unsigned IVSize, + bool IVSigned, bool Ordered, llvm::Value *IL, + llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, + llvm::Value *Chunk = nullptr); + + /// \brief Call the appropriate runtime routine to notify that we finished + /// iteration of the ordered loop with the dynamic scheduling. + /// + /// \param CGF Reference to current CodeGenFunction. + /// \param Loc Clang source location. + /// \param IVSize Size of the iteration variable in bits. + /// \param IVSigned Sign of the interation variable. + /// + virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, + SourceLocation Loc, unsigned IVSize, + bool IVSigned); /// \brief Call the appropriate runtime routine to notify that we finished /// all the work with current loop. /// /// \param CGF Reference to current CodeGenFunction. /// \param Loc Clang source location. - /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause. /// - virtual void EmitOMPForFinish(CodeGenFunction &CGF, SourceLocation Loc, - OpenMPScheduleClauseKind ScheduleKind); + virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc); + + /// Call __kmpc_dispatch_next( + /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, + /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, + /// kmp_int[32|64] *p_stride); + /// \param IVSize Size of the iteration variable in bits. + /// \param IVSigned Sign of the interation variable. + /// \param IL Address of the output variable in which the flag of the + /// last iteration is returned. + /// \param LB Address of the output variable in which the lower iteration + /// number is returned. + /// \param UB Address of the output variable in which the upper iteration + /// number is returned. + /// \param ST Address of the output variable in which the stride value is + /// returned. + virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc, + unsigned IVSize, bool IVSigned, + llvm::Value *IL, llvm::Value *LB, + llvm::Value *UB, llvm::Value *ST); /// \brief Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' /// clause. /// \param NumThreads An integer value of threads. - virtual void EmitOMPNumThreadsClause(CodeGenFunction &CGF, - llvm::Value *NumThreads, - SourceLocation Loc); + virtual void emitNumThreadsClause(CodeGenFunction &CGF, + llvm::Value *NumThreads, + SourceLocation Loc); /// \brief Returns address of the threadprivate variable for the current /// thread. @@ -373,10 +510,10 @@ public: /// \param VDAddr Address of the global variable \a VD. /// \param Loc Location of the reference to threadprivate var. /// \return Address of the threadprivate variable for the current thread. - virtual llvm::Value *getOMPAddrOfThreadPrivate(CodeGenFunction &CGF, - const VarDecl *VD, - llvm::Value *VDAddr, - SourceLocation Loc); + virtual llvm::Value *getAddrOfThreadPrivate(CodeGenFunction &CGF, + const VarDecl *VD, + llvm::Value *VDAddr, + SourceLocation Loc); /// \brief Emit a code for initialization of threadprivate variable. It emits /// a call to runtime library which adds initial value to the newly created @@ -387,15 +524,120 @@ public: /// \param Loc Location of threadprivate declaration. /// \param PerformInit true if initialization expression is not constant. virtual llvm::Function * - EmitOMPThreadPrivateVarDefinition(const VarDecl *VD, llvm::Value *VDAddr, - SourceLocation Loc, bool PerformInit, - CodeGenFunction *CGF = nullptr); + emitThreadPrivateVarDefinition(const VarDecl *VD, llvm::Value *VDAddr, + SourceLocation Loc, bool PerformInit, + CodeGenFunction *CGF = nullptr); /// \brief Emit flush of the variables specified in 'omp flush' directive. /// \param Vars List of variables to flush. - virtual void EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, - SourceLocation Loc); + virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, + SourceLocation Loc); + + /// \brief Emit task region for the task directive. The task region is + /// emitted in several steps: + /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 + /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, + /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the + /// function: + /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { + /// TaskFunction(gtid, tt->part_id, tt->shareds); + /// return 0; + /// } + /// 2. Copy a list of shared variables to field shareds of the resulting + /// structure kmp_task_t returned by the previous call (if any). + /// 3. Copy a pointer to destructions function to field destructions of the + /// resulting structure kmp_task_t. + /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, + /// kmp_task_t *new_task), where new_task is a resulting structure from + /// previous items. + /// \param D Current task directive. + /// \param Tied true if the task is tied (the task is tied to the thread that + /// can suspend its task region), false - untied (the task is not tied to any + /// thread). + /// \param Final Contains either constant bool value, or llvm::Value * of i1 + /// type for final clause. If the value is true, the task forces all of its + /// child tasks to become final and included tasks. + /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32 + /// /*part_id*/, captured_struct */*__context*/); + /// \param SharedsTy A type which contains references the shared variables. + /// \param Shareds Context with the list of shared variables from the \a + /// TaskFunction. + /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr + /// otherwise. + /// \param PrivateVars List of references to private variables for the task + /// directive. + /// \param PrivateCopies List of private copies for each private variable in + /// \p PrivateVars. + /// \param FirstprivateVars List of references to private variables for the + /// task directive. + /// \param FirstprivateCopies List of private copies for each private variable + /// in \p FirstprivateVars. + /// \param FirstprivateInits List of references to auto generated variables + /// used for initialization of a single array element. Used if firstprivate + /// variable is of array type. + virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, + const OMPExecutableDirective &D, bool Tied, + llvm::PointerIntPair<llvm::Value *, 1, bool> Final, + llvm::Value *TaskFunction, QualType SharedsTy, + llvm::Value *Shareds, const Expr *IfCond, + const ArrayRef<const Expr *> PrivateVars, + const ArrayRef<const Expr *> PrivateCopies, + const ArrayRef<const Expr *> FirstprivateVars, + const ArrayRef<const Expr *> FirstprivateCopies, + const ArrayRef<const Expr *> FirstprivateInits); + + /// \brief Emit code for the directive that does not require outlining. + /// + /// \param CodeGen Code generation sequence for the \a D directive. + virtual void emitInlinedDirective(CodeGenFunction &CGF, + const RegionCodeGenTy &CodeGen); + /// \brief Emit a code for reduction clause. Next code should be emitted for + /// reduction: + /// \code + /// + /// static kmp_critical_name lock = { 0 }; + /// + /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) { + /// ... + /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); + /// ... + /// } + /// + /// ... + /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; + /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), + /// RedList, reduce_func, &<lock>)) { + /// case 1: + /// ... + /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); + /// ... + /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + /// break; + /// case 2: + /// ... + /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); + /// ... + /// break; + /// default:; + /// } + /// \endcode + /// + /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. + /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. + /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' + /// or 'operator binop(LHS, RHS)'. + /// \param WithNowait true if parent directive has also nowait clause, false + /// otherwise. + virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, + ArrayRef<const Expr *> LHSExprs, + ArrayRef<const Expr *> RHSExprs, + ArrayRef<const Expr *> ReductionOps, + bool WithNowait); + + /// \brief Emit code for 'taskwait' directive. + virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc); }; + } // namespace CodeGen } // namespace clang diff --git a/lib/CodeGen/CGRecordLayout.h b/lib/CodeGen/CGRecordLayout.h index 2de0b2f87fc1..c15f9fde06f5 100644 --- a/lib/CodeGen/CGRecordLayout.h +++ b/lib/CodeGen/CGRecordLayout.h @@ -109,8 +109,8 @@ struct CGBitFieldInfo { class CGRecordLayout { friend class CodeGenTypes; - CGRecordLayout(const CGRecordLayout &) LLVM_DELETED_FUNCTION; - void operator=(const CGRecordLayout &) LLVM_DELETED_FUNCTION; + CGRecordLayout(const CGRecordLayout &) = delete; + void operator=(const CGRecordLayout &) = delete; private: /// The LLVM type corresponding to this record layout; used when diff --git a/lib/CodeGen/CGRecordLayoutBuilder.cpp b/lib/CodeGen/CGRecordLayoutBuilder.cpp index 7ad394b5eeb0..72ecd65c28a9 100644 --- a/lib/CodeGen/CGRecordLayoutBuilder.cpp +++ b/lib/CodeGen/CGRecordLayoutBuilder.cpp @@ -99,10 +99,25 @@ struct CGRecordLowering { MemberInfo StorageInfo(CharUnits Offset, llvm::Type *Data) { return MemberInfo(Offset, MemberInfo::Field, Data); } - bool useMSABI() { + + /// The Microsoft bitfield layout rule allocates discrete storage + /// units of the field's formal type and only combines adjacent + /// fields of the same formal type. We want to emit a layout with + /// these discrete storage units instead of combining them into a + /// continuous run. + bool isDiscreteBitFieldABI() { return Context.getTargetInfo().getCXXABI().isMicrosoft() || D->isMsStruct(Context); } + + /// The Itanium base layout rule allows virtual bases to overlap + /// other bases, which complicates layout in specific ways. + /// + /// Note specifically that the ms_struct attribute doesn't change this. + bool isOverlappingVBaseABI() { + return !Context.getTargetInfo().getCXXABI().isMicrosoft(); + } + /// \brief Wraps llvm::Type::getIntNTy with some implicit arguments. llvm::Type *getIntNType(uint64_t NumBits) { return llvm::Type::getIntNTy(Types.getLLVMContext(), @@ -119,8 +134,9 @@ struct CGRecordLowering { /// for itanium bitfields that are smaller than their declared type. llvm::Type *getStorageType(const FieldDecl *FD) { llvm::Type *Type = Types.ConvertTypeForMem(FD->getType()); - return useMSABI() || !FD->isBitField() ? Type : - getIntNType(std::min(FD->getBitWidthValue(Context), + if (!FD->isBitField()) return Type; + if (isDiscreteBitFieldABI()) return Type; + return getIntNType(std::min(FD->getBitWidthValue(Context), (unsigned)Context.toBits(getSize(Type)))); } /// \brief Gets the llvm Basesubobject type from a CXXRecordDecl. @@ -137,15 +153,10 @@ struct CGRecordLowering { return CharUnits::fromQuantity(DataLayout.getABITypeAlignment(Type)); } bool isZeroInitializable(const FieldDecl *FD) { - const Type *Type = FD->getType()->getBaseElementTypeUnsafe(); - if (const MemberPointerType *MPT = Type->getAs<MemberPointerType>()) - return Types.getCXXABI().isZeroInitializable(MPT); - if (const RecordType *RT = Type->getAs<RecordType>()) - return isZeroInitializable(RT->getDecl()); - return true; + return Types.isZeroInitializable(FD->getType()); } bool isZeroInitializable(const RecordDecl *RD) { - return Types.getCGRecordLayout(RD).isZeroInitializable(); + return Types.isZeroInitializable(RD); } void appendPaddingBytes(CharUnits Size) { if (!Size.isZero()) @@ -198,8 +209,8 @@ struct CGRecordLowering { bool IsZeroInitializableAsBase : 1; bool Packed : 1; private: - CGRecordLowering(const CGRecordLowering &) LLVM_DELETED_FUNCTION; - void operator =(const CGRecordLowering &) LLVM_DELETED_FUNCTION; + CGRecordLowering(const CGRecordLowering &) = delete; + void operator =(const CGRecordLowering &) = delete; }; } // namespace { @@ -365,7 +376,7 @@ CGRecordLowering::accumulateBitFields(RecordDecl::field_iterator Field, // used to determine if the ASTRecordLayout is treating these two bitfields as // contiguous. StartBitOffset is offset of the beginning of the Run. uint64_t StartBitOffset, Tail = 0; - if (useMSABI()) { + if (isDiscreteBitFieldABI()) { for (; Field != FieldEnd; ++Field) { uint64_t BitOffset = getFieldBitOffset(*Field); // Zero-width bitfields end runs. @@ -438,8 +449,12 @@ void CGRecordLowering::accumulateBases() { for (const auto &Base : RD->bases()) { if (Base.isVirtual()) continue; + + // Bases can be zero-sized even if not technically empty if they + // contain only a trailing array member. const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); - if (!BaseDecl->isEmpty()) + if (!BaseDecl->isEmpty() && + !Context.getASTRecordLayout(BaseDecl).getSize().isZero()) Members.push_back(MemberInfo(Layout.getBaseClassOffset(BaseDecl), MemberInfo::Base, getStorageType(BaseDecl), BaseDecl)); } @@ -461,7 +476,7 @@ void CGRecordLowering::accumulateVBases() { // smaller than the nvsize. Here we check to see if such a base is placed // before the nvsize and set the scissor offset to that, instead of the // nvsize. - if (!useMSABI()) + if (isOverlappingVBaseABI()) for (const auto &Base : RD->vbases()) { const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); if (BaseDecl->isEmpty()) @@ -482,7 +497,8 @@ void CGRecordLowering::accumulateVBases() { CharUnits Offset = Layout.getVBaseClassOffset(BaseDecl); // If the vbase is a primary virtual base of some base, then it doesn't // get its own storage location but instead lives inside of that base. - if (!useMSABI() && Context.isNearlyEmpty(BaseDecl) && + if (isOverlappingVBaseABI() && + Context.isNearlyEmpty(BaseDecl) && !hasOwnStorage(RD, BaseDecl)) { Members.push_back(MemberInfo(Offset, MemberInfo::VBase, nullptr, BaseDecl)); diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp index e3bdf8661ef7..c879750015bf 100644 --- a/lib/CodeGen/CGStmt.cpp +++ b/lib/CodeGen/CGStmt.cpp @@ -88,6 +88,7 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::ContinueStmtClass: case Stmt::DefaultStmtClass: case Stmt::CaseStmtClass: + case Stmt::SEHLeaveStmtClass: llvm_unreachable("should have emitted these statements as simple"); #define STMT(Type, Base) @@ -173,9 +174,6 @@ void CodeGenFunction::EmitStmt(const Stmt *S) { case Stmt::SEHTryStmtClass: EmitSEHTryStmt(cast<SEHTryStmt>(*S)); break; - case Stmt::SEHLeaveStmtClass: - EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S)); - break; case Stmt::OMPParallelDirectiveClass: EmitOMPParallelDirective(cast<OMPParallelDirective>(*S)); break; @@ -256,6 +254,7 @@ bool CodeGenFunction::EmitSimpleStmt(const Stmt *S) { case Stmt::ContinueStmtClass: EmitContinueStmt(cast<ContinueStmt>(*S)); break; case Stmt::DefaultStmtClass: EmitDefaultStmt(cast<DefaultStmt>(*S)); break; case Stmt::CaseStmtClass: EmitCaseStmt(cast<CaseStmt>(*S)); break; + case Stmt::SEHLeaveStmtClass: EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S)); break; } return true; @@ -424,9 +423,8 @@ void CodeGenFunction::EmitLabel(const LabelDecl *D) { ResolveBranchFixups(Dest.getBlock()); } - RegionCounter Cnt = getPGORegionCounter(D->getStmt()); EmitBlock(Dest.getBlock()); - Cnt.beginRegion(Builder); + incrementProfileCounter(D->getStmt()); } /// Change the cleanup scope of the labels in this lexical scope to @@ -514,7 +512,6 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // C99 6.8.4.1: The first substatement is executed if the expression compares // unequal to 0. The condition must be a scalar type. LexicalScope ConditionScope(*this, S.getCond()->getSourceRange()); - RegionCounter Cnt = getPGORegionCounter(&S); if (S.getConditionVariable()) EmitAutoVarDecl(*S.getConditionVariable()); @@ -533,7 +530,7 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // This avoids emitting dead code and simplifies the CFG substantially. if (!ContainsLabel(Skipped)) { if (CondConstant) - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); if (Executed) { RunCleanupsScope ExecutedScope(*this); EmitStmt(Executed); @@ -550,11 +547,12 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { if (S.getElse()) ElseBlock = createBasicBlock("if.else"); - EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, Cnt.getCount()); + EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, + getProfileCount(S.getThen())); // Emit the 'then' code. EmitBlock(ThenBlock); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); { RunCleanupsScope ThenScope(*this); EmitStmt(S.getThen()); @@ -564,8 +562,8 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // Emit the 'else' code if present. if (const Stmt *Else = S.getElse()) { { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(*this); + // There is no need to emit line number for an unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(*this); EmitBlock(ElseBlock); } { @@ -573,8 +571,8 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { EmitStmt(Else); } { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(*this); + // There is no need to emit line number for an unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(*this); EmitBranch(ContBlock); } } @@ -679,8 +677,6 @@ void CodeGenFunction::EmitCondBrHints(llvm::LLVMContext &Context, void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> WhileAttrs) { - RegionCounter Cnt = getPGORegionCounter(&S); - // Emit the header for the loop, which will also become // the continue target. JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); @@ -725,9 +721,9 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - llvm::BranchInst *CondBr = - Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, - PGO.createLoopWeights(S.getCond(), Cnt)); + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, LoopBody, ExitBlock, + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -743,7 +739,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S, { RunCleanupsScope BodyScope(*this); EmitBlock(LoopBody); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); EmitStmt(S.getBody()); } @@ -772,7 +768,7 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, JumpDest LoopExit = getJumpDestInCurrentScope("do.end"); JumpDest LoopCond = getJumpDestInCurrentScope("do.cond"); - RegionCounter Cnt = getPGORegionCounter(&S); + uint64_t ParentCount = getCurrentProfileCount(); // Store the blocks to use for break and continue. BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond)); @@ -782,7 +778,7 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, LoopStack.push(LoopBody); - EmitBlockWithFallThrough(LoopBody, Cnt); + EmitBlockWithFallThrough(LoopBody, &S); { RunCleanupsScope BodyScope(*this); EmitStmt(S.getBody()); @@ -809,9 +805,10 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S, // As long as the condition is true, iterate the loop. if (EmitBoolCondBranch) { - llvm::BranchInst *CondBr = - Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(), - PGO.createLoopWeights(S.getCond(), Cnt)); + uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount; + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, LoopBody, LoopExit.getBlock(), + createProfileWeightsForLoop(S.getCond(), BackedgeCount)); // Attach metadata to loop body conditional branch. EmitCondBrHints(LoopBody->getContext(), CondBr, DoAttrs); @@ -838,8 +835,6 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, if (S.getInit()) EmitStmt(S.getInit()); - RegionCounter Cnt = getPGORegionCounter(&S); - // Start the loop with a block that tests the condition. // If there's an increment, the continue scope will be overwritten // later. @@ -881,9 +876,9 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - llvm::BranchInst *CondBr = - Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, - PGO.createLoopWeights(S.getCond(), Cnt)); + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, ForBody, ExitBlock, + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); // Attach metadata to loop body conditional branch. EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs); @@ -898,7 +893,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S, // Treat it as a non-zero constant. Don't even create a new block for the // body, just fall into it. } - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); { // Create a separate cleanup scope for the body, in case it is not @@ -939,8 +934,6 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, EmitStmt(S.getRangeStmt()); EmitStmt(S.getBeginEndStmt()); - RegionCounter Cnt = getPGORegionCounter(&S); - // Start the loop with a block that tests the condition. // If there's an increment, the continue scope will be overwritten // later. @@ -962,7 +955,8 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); llvm::BranchInst *CondBr = Builder.CreateCondBr( - BoolCondVal, ForBody, ExitBlock, PGO.createLoopWeights(S.getCond(), Cnt)); + BoolCondVal, ForBody, ExitBlock, + createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()))); // Attach metadata to loop body conditional branch. EmitCondBrHints(ForBody->getContext(), CondBr, ForAttrs); @@ -973,7 +967,7 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S, } EmitBlock(ForBody); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); // Create a block for the increment. In case of a 'continue', we jump there. JumpDest Continue = getJumpDestInCurrentScope("for.inc"); @@ -1022,6 +1016,12 @@ void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) { /// if the function returns void, or may be missing one if the function returns /// non-void. Fun stuff :). void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { + // Returning from an outlined SEH helper is UB, and we already warn on it. + if (IsOutlinedSEHHelper) { + Builder.CreateUnreachable(); + Builder.ClearInsertionPoint(); + } + // Emit the result value, even if unused, to evalute the side effects. const Expr *RV = S.getRetValue(); @@ -1133,13 +1133,11 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext()); llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext()); - RegionCounter CaseCnt = getPGORegionCounter(&S); - // Emit the code for this case. We do this first to make sure it is // properly chained from our predecessor before generating the // switch machinery to enter this block. llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb"); - EmitBlockWithFallThrough(CaseDest, CaseCnt); + EmitBlockWithFallThrough(CaseDest, &S); EmitStmt(S.getSubStmt()); // If range is empty, do nothing. @@ -1150,7 +1148,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { // FIXME: parameters such as this should not be hardcoded. if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) { // Range is small enough to add multiple switch instruction cases. - uint64_t Total = CaseCnt.getCount(); + uint64_t Total = getProfileCount(&S); unsigned NCases = Range.getZExtValue() + 1; // We only have one region counter for the entire set of cases here, so we // need to divide the weights evenly between the generated cases, ensuring @@ -1189,9 +1187,9 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { llvm::MDNode *Weights = nullptr; if (SwitchWeights) { - uint64_t ThisCount = CaseCnt.getCount(); + uint64_t ThisCount = getProfileCount(&S); uint64_t DefaultCount = (*SwitchWeights)[0]; - Weights = PGO.createBranchWeights(ThisCount, DefaultCount); + Weights = createProfileWeights(ThisCount, DefaultCount); // Since we're chaining the switch default through each large case range, we // need to update the weight for the default, ie, the first case, to include @@ -1224,7 +1222,6 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { return; } - RegionCounter CaseCnt = getPGORegionCounter(&S); llvm::ConstantInt *CaseVal = Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext())); @@ -1239,7 +1236,7 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { // Only do this optimization if there are no cleanups that need emitting. if (isObviouslyBranchWithoutCleanups(Block)) { if (SwitchWeights) - SwitchWeights->push_back(CaseCnt.getCount()); + SwitchWeights->push_back(getProfileCount(&S)); SwitchInsn->addCase(CaseVal, Block.getBlock()); // If there was a fallthrough into this case, make sure to redirect it to @@ -1253,9 +1250,9 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { } llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb"); - EmitBlockWithFallThrough(CaseDest, CaseCnt); + EmitBlockWithFallThrough(CaseDest, &S); if (SwitchWeights) - SwitchWeights->push_back(CaseCnt.getCount()); + SwitchWeights->push_back(getProfileCount(&S)); SwitchInsn->addCase(CaseVal, CaseDest); // Recursively emitting the statement is acceptable, but is not wonderful for @@ -1276,12 +1273,11 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { llvm::ConstantInt *CaseVal = Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext())); - CaseCnt = getPGORegionCounter(NextCase); if (SwitchWeights) - SwitchWeights->push_back(CaseCnt.getCount()); + SwitchWeights->push_back(getProfileCount(NextCase)); if (CGM.getCodeGenOpts().ProfileInstrGenerate) { CaseDest = createBasicBlock("sw.bb"); - EmitBlockWithFallThrough(CaseDest, CaseCnt); + EmitBlockWithFallThrough(CaseDest, &S); } SwitchInsn->addCase(CaseVal, CaseDest); @@ -1297,8 +1293,7 @@ void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S) { assert(DefaultBlock->empty() && "EmitDefaultStmt: Default block already defined?"); - RegionCounter Cnt = getPGORegionCounter(&S); - EmitBlockWithFallThrough(DefaultBlock, Cnt); + EmitBlockWithFallThrough(DefaultBlock, &S); EmitStmt(S.getSubStmt()); } @@ -1520,10 +1515,8 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { const SwitchCase *Case = nullptr; if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts, getContext(), Case)) { - if (Case) { - RegionCounter CaseCnt = getPGORegionCounter(Case); - CaseCnt.beginRegion(Builder); - } + if (Case) + incrementProfileCounter(Case); RunCleanupsScope ExecutedScope(*this); // Emit the condition variable if needed inside the entire cleanup scope @@ -1540,8 +1533,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // specified series of statements and we're good. for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i) EmitStmt(CaseStmts[i]); - RegionCounter ExitCnt = getPGORegionCounter(&S); - ExitCnt.beginRegion(Builder); + incrementProfileCounter(&S); // Now we want to restore the saved switch instance so that nested // switches continue to function properly @@ -1572,7 +1564,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { Case; Case = Case->getNextSwitchCase()) { if (isa<DefaultStmt>(Case)) - DefaultCount = getPGORegionCounter(Case).getCount(); + DefaultCount = getProfileCount(Case); NumCases += 1; } SwitchWeights = new SmallVector<uint64_t, 16>(); @@ -1621,8 +1613,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // Emit continuation. EmitBlock(SwitchExit.getBlock(), true); - RegionCounter ExitCnt = getPGORegionCounter(&S); - ExitCnt.beginRegion(Builder); + incrementProfileCounter(&S); if (SwitchWeights) { assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() && @@ -1630,7 +1621,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // If there's only one jump destination there's no sense weighting it. if (SwitchWeights->size() > 1) SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof, - PGO.createBranchWeights(*SwitchWeights)); + createProfileWeights(*SwitchWeights)); delete SwitchWeights; } SwitchInsn = SavedSwitchInsn; @@ -1696,7 +1687,7 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target, static std::string AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr, const TargetInfo &Target, CodeGenModule &CGM, - const AsmStmt &Stmt) { + const AsmStmt &Stmt, const bool EarlyClobber) { const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr); if (!AsmDeclRef) return Constraint; @@ -1721,7 +1712,7 @@ AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr, } // Canonicalize the register here before returning it. Register = Target.getNormalizedGCCRegisterName(Register); - return "{" + Register.str() + "}"; + return (EarlyClobber ? "&{" : "{") + Register.str() + "}"; } llvm::Value* @@ -1854,7 +1845,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { OutExpr = OutExpr->IgnoreParenNoopCasts(getContext()); OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr, - getTarget(), CGM, S); + getTarget(), CGM, S, + Info.earlyClobber()); LValue Dest = EmitLValue(OutExpr); if (!Constraints.empty()) @@ -1956,10 +1948,9 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) { InputConstraint = SimplifyConstraint(InputConstraint.c_str(), getTarget(), &OutputConstraintInfos); - InputConstraint = - AddVariableConstraints(InputConstraint, - *InputExpr->IgnoreParenNoopCasts(getContext()), - getTarget(), CGM, S); + InputConstraint = AddVariableConstraints( + InputConstraint, *InputExpr->IgnoreParenNoopCasts(getContext()), + getTarget(), CGM, S, false /* No EarlyClobber */); llvm::Value *Arg = EmitAsmInput(Info, InputExpr, Constraints); @@ -2187,6 +2178,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) { llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage, CapturedStmtInfo->getHelperName(), &CGM.getModule()); CGM.SetInternalFunctionAttributes(CD, F, FuncInfo); + if (CD->isNothrow()) + F->addFnAttr(llvm::Attribute::NoUnwind); // Generate the function. StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, diff --git a/lib/CodeGen/CGStmtOpenMP.cpp b/lib/CodeGen/CGStmtOpenMP.cpp index 78fd37ce6562..07fc6e966af6 100644 --- a/lib/CodeGen/CGStmtOpenMP.cpp +++ b/lib/CodeGen/CGStmtOpenMP.cpp @@ -20,258 +20,473 @@ using namespace clang; using namespace CodeGen; -namespace { -/// \brief RAII for emitting code of CapturedStmt without function outlining. -class InlinedOpenMPRegion { - CodeGenFunction &CGF; - CodeGenFunction::CGCapturedStmtInfo *PrevCapturedStmtInfo; - const Decl *StoredCurCodeDecl; - - /// \brief A class to emit CapturedStmt construct as inlined statement without - /// generating a function for outlined code. - class CGInlinedOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { - public: - CGInlinedOpenMPRegionInfo() : CGCapturedStmtInfo() {} - }; - -public: - InlinedOpenMPRegion(CodeGenFunction &CGF, const Stmt *S) - : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo), - StoredCurCodeDecl(CGF.CurCodeDecl) { - CGF.CurCodeDecl = cast<CapturedStmt>(S)->getCapturedDecl(); - CGF.CapturedStmtInfo = new CGInlinedOpenMPRegionInfo(); - } - ~InlinedOpenMPRegion() { - delete CGF.CapturedStmtInfo; - CGF.CapturedStmtInfo = PrevCapturedStmtInfo; - CGF.CurCodeDecl = StoredCurCodeDecl; - } -}; -} // namespace - //===----------------------------------------------------------------------===// // OpenMP Directive Emission //===----------------------------------------------------------------------===// +void CodeGenFunction::EmitOMPAggregateAssign( + llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, + const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen) { + // Perform element-by-element initialization. + QualType ElementTy; + auto SrcBegin = SrcAddr; + auto DestBegin = DestAddr; + auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); + auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); + // Cast from pointer to array type to pointer to single element. + SrcBegin = Builder.CreatePointerBitCastOrAddrSpaceCast(SrcBegin, + DestBegin->getType()); + auto DestEnd = Builder.CreateGEP(DestBegin, NumElements); + // The basic structure here is a while-do loop. + auto BodyBB = createBasicBlock("omp.arraycpy.body"); + auto DoneBB = createBasicBlock("omp.arraycpy.done"); + auto IsEmpty = + Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); + Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); + + // Enter the loop body, making that address the current address. + auto EntryBB = Builder.GetInsertBlock(); + EmitBlock(BodyBB); + auto SrcElementCurrent = + Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast"); + SrcElementCurrent->addIncoming(SrcBegin, EntryBB); + auto DestElementCurrent = Builder.CreatePHI(DestBegin->getType(), 2, + "omp.arraycpy.destElementPast"); + DestElementCurrent->addIncoming(DestBegin, EntryBB); + + // Emit copy. + CopyGen(DestElementCurrent, SrcElementCurrent); + + // Shift the address forward by one element. + auto DestElementNext = Builder.CreateConstGEP1_32( + DestElementCurrent, /*Idx0=*/1, "omp.arraycpy.dest.element"); + auto SrcElementNext = Builder.CreateConstGEP1_32( + SrcElementCurrent, /*Idx0=*/1, "omp.arraycpy.src.element"); + // Check whether we've reached the end. + auto Done = + Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); + Builder.CreateCondBr(Done, DoneBB, BodyBB); + DestElementCurrent->addIncoming(DestElementNext, Builder.GetInsertBlock()); + SrcElementCurrent->addIncoming(SrcElementNext, Builder.GetInsertBlock()); + + // Done. + EmitBlock(DoneBB, /*IsFinished=*/true); +} -/// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen -/// function. Here is the logic: -/// if (Cond) { -/// CodeGen(true); -/// } else { -/// CodeGen(false); -/// } -static void EmitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, - const std::function<void(bool)> &CodeGen) { - CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); - - // If the condition constant folds and can be elided, try to avoid emitting - // the condition and the dead arm of the if/else. - bool CondConstant; - if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { - CodeGen(CondConstant); - return; - } - - // Otherwise, the condition did not fold, or we couldn't elide it. Just - // emit the conditional branch. - auto ThenBlock = CGF.createBasicBlock(/*name*/ "omp_if.then"); - auto ElseBlock = CGF.createBasicBlock(/*name*/ "omp_if.else"); - auto ContBlock = CGF.createBasicBlock(/*name*/ "omp_if.end"); - CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount*/ 0); - - // Emit the 'then' code. - CGF.EmitBlock(ThenBlock); - CodeGen(/*ThenBlock*/ true); - CGF.EmitBranch(ContBlock); - // Emit the 'else' code if present. - { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(CGF); - CGF.EmitBlock(ElseBlock); - } - CodeGen(/*ThenBlock*/ false); - { - // There is no need to emit line number for unconditional branch. - ApplyDebugLocation DL(CGF); - CGF.EmitBranch(ContBlock); - } - // Emit the continuation block for code after the if. - CGF.EmitBlock(ContBlock, /*IsFinished*/ true); -} - -void CodeGenFunction::EmitOMPAggregateAssign(LValue OriginalAddr, - llvm::Value *PrivateAddr, - const Expr *AssignExpr, - QualType OriginalType, - const VarDecl *VDInit) { - EmitBlock(createBasicBlock(".omp.assign.begin.")); - if (!isa<CXXConstructExpr>(AssignExpr) || isTrivialInitializer(AssignExpr)) { - // Perform simple memcpy. - EmitAggregateAssign(PrivateAddr, OriginalAddr.getAddress(), - AssignExpr->getType()); - } else { - // Perform element-by-element initialization. - QualType ElementTy; - auto SrcBegin = OriginalAddr.getAddress(); - auto DestBegin = PrivateAddr; - auto ArrayTy = OriginalType->getAsArrayTypeUnsafe(); - auto SrcNumElements = emitArrayLength(ArrayTy, ElementTy, SrcBegin); - auto DestNumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin); - auto SrcEnd = Builder.CreateGEP(SrcBegin, SrcNumElements); - auto DestEnd = Builder.CreateGEP(DestBegin, DestNumElements); - // The basic structure here is a do-while loop, because we don't - // need to check for the zero-element case. - auto BodyBB = createBasicBlock("omp.arraycpy.body"); - auto DoneBB = createBasicBlock("omp.arraycpy.done"); - auto IsEmpty = - Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty"); - Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); - - // Enter the loop body, making that address the current address. - auto EntryBB = Builder.GetInsertBlock(); - EmitBlock(BodyBB); - auto SrcElementPast = Builder.CreatePHI(SrcBegin->getType(), 2, - "omp.arraycpy.srcElementPast"); - SrcElementPast->addIncoming(SrcEnd, EntryBB); - auto DestElementPast = Builder.CreatePHI(DestBegin->getType(), 2, - "omp.arraycpy.destElementPast"); - DestElementPast->addIncoming(DestEnd, EntryBB); - - // Shift the address back by one element. - auto NegativeOne = llvm::ConstantInt::get(SizeTy, -1, true); - auto DestElement = Builder.CreateGEP(DestElementPast, NegativeOne, - "omp.arraycpy.dest.element"); - auto SrcElement = Builder.CreateGEP(SrcElementPast, NegativeOne, - "omp.arraycpy.src.element"); - { - // Create RunCleanScope to cleanup possible temps. - CodeGenFunction::RunCleanupsScope Init(*this); - // Emit initialization for single element. - LocalDeclMap[VDInit] = SrcElement; - EmitAnyExprToMem(AssignExpr, DestElement, - AssignExpr->getType().getQualifiers(), - /*IsInitializer*/ false); - LocalDeclMap.erase(VDInit); +void CodeGenFunction::EmitOMPCopy(CodeGenFunction &CGF, + QualType OriginalType, llvm::Value *DestAddr, + llvm::Value *SrcAddr, const VarDecl *DestVD, + const VarDecl *SrcVD, const Expr *Copy) { + if (OriginalType->isArrayType()) { + auto *BO = dyn_cast<BinaryOperator>(Copy); + if (BO && BO->getOpcode() == BO_Assign) { + // Perform simple memcpy for simple copying. + CGF.EmitAggregateAssign(DestAddr, SrcAddr, OriginalType); + } else { + // For arrays with complex element types perform element by element + // copying. + CGF.EmitOMPAggregateAssign( + DestAddr, SrcAddr, OriginalType, + [&CGF, Copy, SrcVD, DestVD](llvm::Value *DestElement, + llvm::Value *SrcElement) { + // Working with the single array element, so have to remap + // destination and source variables to corresponding array + // elements. + CodeGenFunction::OMPPrivateScope Remap(CGF); + Remap.addPrivate(DestVD, [DestElement]() -> llvm::Value *{ + return DestElement; + }); + Remap.addPrivate( + SrcVD, [SrcElement]() -> llvm::Value *{ return SrcElement; }); + (void)Remap.Privatize(); + CGF.EmitIgnoredExpr(Copy); + }); } - - // Check whether we've reached the end. - auto Done = - Builder.CreateICmpEQ(DestElement, DestBegin, "omp.arraycpy.done"); - Builder.CreateCondBr(Done, DoneBB, BodyBB); - DestElementPast->addIncoming(DestElement, Builder.GetInsertBlock()); - SrcElementPast->addIncoming(SrcElement, Builder.GetInsertBlock()); - - // Done. - EmitBlock(DoneBB, true); + } else { + // Remap pseudo source variable to private copy. + CodeGenFunction::OMPPrivateScope Remap(CGF); + Remap.addPrivate(SrcVD, [SrcAddr]() -> llvm::Value *{ return SrcAddr; }); + Remap.addPrivate(DestVD, [DestAddr]() -> llvm::Value *{ return DestAddr; }); + (void)Remap.Privatize(); + // Emit copying of the whole variable. + CGF.EmitIgnoredExpr(Copy); } - EmitBlock(createBasicBlock(".omp.assign.end.")); } -void CodeGenFunction::EmitOMPFirstprivateClause( - const OMPExecutableDirective &D, - CodeGenFunction::OMPPrivateScope &PrivateScope) { - auto PrivateFilter = [](const OMPClause *C) -> bool { - return C->getClauseKind() == OMPC_firstprivate; - }; - for (OMPExecutableDirective::filtered_clause_iterator<decltype(PrivateFilter)> - I(D.clauses(), PrivateFilter); I; ++I) { +bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D, + OMPPrivateScope &PrivateScope) { + llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate; + for (auto &&I = D.getClausesOfKind(OMPC_firstprivate); I; ++I) { auto *C = cast<OMPFirstprivateClause>(*I); auto IRef = C->varlist_begin(); auto InitsRef = C->inits().begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); - auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); - bool IsRegistered; - if (*InitsRef != nullptr) { - // Emit VarDecl with copy init for arrays. - auto *FD = CapturedStmtInfo->lookup(OrigVD); - LValue Base = MakeNaturalAlignAddrLValue( - CapturedStmtInfo->getContextValue(), - getContext().getTagDeclType(FD->getParent())); - auto OriginalAddr = EmitLValueForField(Base, FD); - auto VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl()); - IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { - auto Emission = EmitAutoVarAlloca(*VD); - // Emit initialization of aggregate firstprivate vars. - EmitOMPAggregateAssign(OriginalAddr, Emission.getAllocatedAddress(), - VD->getInit(), (*IRef)->getType(), VDInit); - EmitAutoVarCleanups(Emission); - return Emission.getAllocatedAddress(); - }); - } else - IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { - // Emit private VarDecl with copy init. - EmitDecl(*VD); - return GetAddrOfLocalVar(VD); - }); - assert(IsRegistered && "counter already registered as private"); - // Silence the warning about unused variable. - (void)IsRegistered; + if (EmittedAsFirstprivate.count(OrigVD) == 0) { + EmittedAsFirstprivate.insert(OrigVD); + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl()); + bool IsRegistered; + DeclRefExpr DRE( + const_cast<VarDecl *>(OrigVD), + /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( + OrigVD) != nullptr, + (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); + auto *OriginalAddr = EmitLValue(&DRE).getAddress(); + QualType Type = OrigVD->getType(); + if (Type->isArrayType()) { + // Emit VarDecl with copy init for arrays. + // Get the address of the original variable captured in current + // captured region. + IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + auto Emission = EmitAutoVarAlloca(*VD); + auto *Init = VD->getInit(); + if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) { + // Perform simple memcpy. + EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr, + Type); + } else { + EmitOMPAggregateAssign( + Emission.getAllocatedAddress(), OriginalAddr, Type, + [this, VDInit, Init](llvm::Value *DestElement, + llvm::Value *SrcElement) { + // Clean up any temporaries needed by the initialization. + RunCleanupsScope InitScope(*this); + // Emit initialization for single element. + LocalDeclMap[VDInit] = SrcElement; + EmitAnyExprToMem(Init, DestElement, + Init->getType().getQualifiers(), + /*IsInitializer*/ false); + LocalDeclMap.erase(VDInit); + }); + } + EmitAutoVarCleanups(Emission); + return Emission.getAllocatedAddress(); + }); + } else { + IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + // Emit private VarDecl with copy init. + // Remap temp VDInit variable to the address of the original + // variable + // (for proper handling of captured global variables). + LocalDeclMap[VDInit] = OriginalAddr; + EmitDecl(*VD); + LocalDeclMap.erase(VDInit); + return GetAddrOfLocalVar(VD); + }); + } + assert(IsRegistered && + "firstprivate var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + } ++IRef, ++InitsRef; } } + return !EmittedAsFirstprivate.empty(); } void CodeGenFunction::EmitOMPPrivateClause( const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) { - auto PrivateFilter = [](const OMPClause *C) -> bool { - return C->getClauseKind() == OMPC_private; - }; - for (OMPExecutableDirective::filtered_clause_iterator<decltype(PrivateFilter)> - I(D.clauses(), PrivateFilter); I; ++I) { + llvm::DenseSet<const VarDecl *> EmittedAsPrivate; + for (auto &&I = D.getClausesOfKind(OMPC_private); I; ++I) { auto *C = cast<OMPPrivateClause>(*I); auto IRef = C->varlist_begin(); for (auto IInit : C->private_copies()) { auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); - auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { + auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + bool IsRegistered = + PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + // Emit private VarDecl with copy init. + EmitDecl(*VD); + return GetAddrOfLocalVar(VD); + }); + assert(IsRegistered && "private var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + } + ++IRef; + } + } +} + +bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) { + // threadprivate_var1 = master_threadprivate_var1; + // operator=(threadprivate_var2, master_threadprivate_var2); + // ... + // __kmpc_barrier(&loc, global_tid); + llvm::DenseSet<const VarDecl *> CopiedVars; + llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr; + for (auto &&I = D.getClausesOfKind(OMPC_copyin); I; ++I) { + auto *C = cast<OMPCopyinClause>(*I); + auto IRef = C->varlist_begin(); + auto ISrcRef = C->source_exprs().begin(); + auto IDestRef = C->destination_exprs().begin(); + for (auto *AssignOp : C->assignment_ops()) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + QualType Type = VD->getType(); + if (CopiedVars.insert(VD->getCanonicalDecl()).second) { + // Get the address of the master variable. + auto *MasterAddr = VD->isStaticLocal() + ? CGM.getStaticLocalDeclAddress(VD) + : CGM.GetAddrOfGlobal(VD); + // Get the address of the threadprivate variable. + auto *PrivateAddr = EmitLValue(*IRef).getAddress(); + if (CopiedVars.size() == 1) { + // At first check if current thread is a master thread. If it is, no + // need to copy data. + CopyBegin = createBasicBlock("copyin.not.master"); + CopyEnd = createBasicBlock("copyin.not.master.end"); + Builder.CreateCondBr( + Builder.CreateICmpNE( + Builder.CreatePtrToInt(MasterAddr, CGM.IntPtrTy), + Builder.CreatePtrToInt(PrivateAddr, CGM.IntPtrTy)), + CopyBegin, CopyEnd); + EmitBlock(CopyBegin); + } + auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); + auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); + EmitOMPCopy(*this, Type, PrivateAddr, MasterAddr, DestVD, SrcVD, + AssignOp); + } + ++IRef; + ++ISrcRef; + ++IDestRef; + } + } + if (CopyEnd) { + // Exit out of copying procedure for non-master thread. + EmitBlock(CopyEnd, /*IsFinished=*/true); + return true; + } + return false; +} + +bool CodeGenFunction::EmitOMPLastprivateClauseInit( + const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) { + bool HasAtLeastOneLastprivate = false; + llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; + for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) { + HasAtLeastOneLastprivate = true; + auto *C = cast<OMPLastprivateClause>(*I); + auto IRef = C->varlist_begin(); + auto IDestRef = C->destination_exprs().begin(); + for (auto *IInit : C->private_copies()) { + // Keep the address of the original variable for future update at the end + // of the loop. + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) { + auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); + PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> llvm::Value *{ + DeclRefExpr DRE( + const_cast<VarDecl *>(OrigVD), + /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup( + OrigVD) != nullptr, + (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc()); + return EmitLValue(&DRE).getAddress(); + }); + // Check if the variable is also a firstprivate: in this case IInit is + // not generated. Initialization of this variable will happen in codegen + // for 'firstprivate' clause. + if (IInit) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl()); + bool IsRegistered = + PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value *{ + // Emit private VarDecl with copy init. + EmitDecl(*VD); + return GetAddrOfLocalVar(VD); + }); + assert(IsRegistered && + "lastprivate var already registered as private"); + (void)IsRegistered; + } + } + ++IRef, ++IDestRef; + } + } + return HasAtLeastOneLastprivate; +} + +void CodeGenFunction::EmitOMPLastprivateClauseFinal( + const OMPExecutableDirective &D, llvm::Value *IsLastIterCond) { + // Emit following code: + // if (<IsLastIterCond>) { + // orig_var1 = private_orig_var1; + // ... + // orig_varn = private_orig_varn; + // } + auto *ThenBB = createBasicBlock(".omp.lastprivate.then"); + auto *DoneBB = createBasicBlock(".omp.lastprivate.done"); + Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB); + EmitBlock(ThenBB); + llvm::DenseMap<const Decl *, const Expr *> LoopCountersAndUpdates; + const Expr *LastIterVal = nullptr; + const Expr *IVExpr = nullptr; + const Expr *IncExpr = nullptr; + if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) { + LastIterVal = + cast<VarDecl>(cast<DeclRefExpr>(LoopDirective->getUpperBoundVariable()) + ->getDecl()) + ->getAnyInitializer(); + IVExpr = LoopDirective->getIterationVariable(); + IncExpr = LoopDirective->getInc(); + auto IUpdate = LoopDirective->updates().begin(); + for (auto *E : LoopDirective->counters()) { + auto *D = cast<DeclRefExpr>(E)->getDecl()->getCanonicalDecl(); + LoopCountersAndUpdates[D] = *IUpdate; + ++IUpdate; + } + } + { + llvm::DenseSet<const VarDecl *> AlreadyEmittedVars; + bool FirstLCV = true; + for (auto &&I = D.getClausesOfKind(OMPC_lastprivate); I; ++I) { + auto *C = cast<OMPLastprivateClause>(*I); + auto IRef = C->varlist_begin(); + auto ISrcRef = C->source_exprs().begin(); + auto IDestRef = C->destination_exprs().begin(); + for (auto *AssignOp : C->assignment_ops()) { + auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + QualType Type = PrivateVD->getType(); + auto *CanonicalVD = PrivateVD->getCanonicalDecl(); + if (AlreadyEmittedVars.insert(CanonicalVD).second) { + // If lastprivate variable is a loop control variable for loop-based + // directive, update its value before copyin back to original + // variable. + if (auto *UpExpr = LoopCountersAndUpdates.lookup(CanonicalVD)) { + if (FirstLCV) { + EmitAnyExprToMem(LastIterVal, EmitLValue(IVExpr).getAddress(), + IVExpr->getType().getQualifiers(), + /*IsInitializer=*/false); + EmitIgnoredExpr(IncExpr); + FirstLCV = false; + } + EmitIgnoredExpr(UpExpr); + } + auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl()); + auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl()); + // Get the address of the original variable. + auto *OriginalAddr = GetAddrOfLocalVar(DestVD); + // Get the address of the private variable. + auto *PrivateAddr = GetAddrOfLocalVar(PrivateVD); + EmitOMPCopy(*this, Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, + AssignOp); + } + ++IRef; + ++ISrcRef; + ++IDestRef; + } + } + } + EmitBlock(DoneBB, /*IsFinished=*/true); +} + +void CodeGenFunction::EmitOMPReductionClauseInit( + const OMPExecutableDirective &D, + CodeGenFunction::OMPPrivateScope &PrivateScope) { + for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) { + auto *C = cast<OMPReductionClause>(*I); + auto ILHS = C->lhs_exprs().begin(); + auto IRHS = C->rhs_exprs().begin(); + for (auto IRef : C->varlists()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl()); + auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + // Store the address of the original variable associated with the LHS + // implicit variable. + PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> llvm::Value *{ + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + IRef->getType(), VK_LValue, IRef->getExprLoc()); + return EmitLValue(&DRE).getAddress(); + }); + // Emit reduction copy. bool IsRegistered = - PrivateScope.addPrivate(OrigVD, [&]() -> llvm::Value * { - // Emit private VarDecl with copy init. - EmitDecl(*VD); - return GetAddrOfLocalVar(VD); + PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> llvm::Value *{ + // Emit private VarDecl with reduction init. + EmitDecl(*PrivateVD); + return GetAddrOfLocalVar(PrivateVD); }); - assert(IsRegistered && "counter already registered as private"); + assert(IsRegistered && "private var already registered as private"); // Silence the warning about unused variable. (void)IsRegistered; - ++IRef; + ++ILHS, ++IRHS; } } } -/// \brief Emits code for OpenMP parallel directive in the parallel region. -static void EmitOMPParallelCall(CodeGenFunction &CGF, - const OMPParallelDirective &S, - llvm::Value *OutlinedFn, - llvm::Value *CapturedStruct) { - if (auto C = S.getSingleClause(/*K*/ OMPC_num_threads)) { +void CodeGenFunction::EmitOMPReductionClauseFinal( + const OMPExecutableDirective &D) { + llvm::SmallVector<const Expr *, 8> LHSExprs; + llvm::SmallVector<const Expr *, 8> RHSExprs; + llvm::SmallVector<const Expr *, 8> ReductionOps; + bool HasAtLeastOneReduction = false; + for (auto &&I = D.getClausesOfKind(OMPC_reduction); I; ++I) { + HasAtLeastOneReduction = true; + auto *C = cast<OMPReductionClause>(*I); + LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end()); + RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end()); + ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end()); + } + if (HasAtLeastOneReduction) { + // Emit nowait reduction if nowait clause is present or directive is a + // parallel directive (it always has implicit barrier). + CGM.getOpenMPRuntime().emitReduction( + *this, D.getLocEnd(), LHSExprs, RHSExprs, ReductionOps, + D.getSingleClause(OMPC_nowait) || + isOpenMPParallelDirective(D.getDirectiveKind())); + } +} + +static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, + const OMPExecutableDirective &S, + const RegionCodeGenTy &CodeGen) { + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + auto CapturedStruct = CGF.GenerateCapturedStmtArgument(*CS); + auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( + S, *CS->getCapturedDecl()->param_begin(), CodeGen); + if (auto C = S.getSingleClause(OMPC_num_threads)) { CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); auto NumThreadsClause = cast<OMPNumThreadsClause>(C); auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), /*IgnoreResultAssign*/ true); - CGF.CGM.getOpenMPRuntime().EmitOMPNumThreadsClause( + CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( CGF, NumThreads, NumThreadsClause->getLocStart()); } - CGF.CGM.getOpenMPRuntime().EmitOMPParallelCall(CGF, S.getLocStart(), - OutlinedFn, CapturedStruct); + const Expr *IfCond = nullptr; + if (auto C = S.getSingleClause(OMPC_if)) { + IfCond = cast<OMPIfClause>(C)->getCondition(); + } + CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn, + CapturedStruct, IfCond); } void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) { - auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); - auto CapturedStruct = GenerateCapturedStmtArgument(*CS); - auto OutlinedFn = CGM.getOpenMPRuntime().EmitOpenMPOutlinedFunction( - S, *CS->getCapturedDecl()->param_begin()); - if (auto C = S.getSingleClause(/*K*/ OMPC_if)) { - auto Cond = cast<OMPIfClause>(C)->getCondition(); - EmitOMPIfClause(*this, Cond, [&](bool ThenBlock) { - if (ThenBlock) - EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); - else - CGM.getOpenMPRuntime().EmitOMPSerialCall(*this, S.getLocStart(), - OutlinedFn, CapturedStruct); - }); - } else - EmitOMPParallelCall(*this, S, OutlinedFn, CapturedStruct); + LexicalScope Scope(*this, S.getSourceRange()); + // Emit parallel region as a standalone region. + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + OMPPrivateScope PrivateScope(CGF); + bool Copyins = CGF.EmitOMPCopyinClause(S); + bool Firstprivates = CGF.EmitOMPFirstprivateClause(S, PrivateScope); + if (Copyins || Firstprivates) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables or propagation master's thread + // values of threadprivate variables to local instances of that variables + // of all other implicit threads. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + } + CGF.EmitOMPPrivateClause(S, PrivateScope); + CGF.EmitOMPReductionClauseInit(S, PrivateScope); + (void)PrivateScope.Privatize(); + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EmitOMPReductionClauseFinal(S); + // Emit implicit barrier at the end of the 'parallel' directive. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + }; + emitCommonOMPParallelDirective(*this, S, CodeGen); } void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, @@ -281,6 +496,14 @@ void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, for (auto I : S.updates()) { EmitIgnoredExpr(I); } + // Update the linear variables. + for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + for (auto U : C->updates()) { + EmitIgnoredExpr(U); + } + } + // On a continue in the body, jump to the end. auto Continue = getJumpDestInCurrentScope("omp.body.continue"); BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue)); @@ -297,11 +520,12 @@ void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S, } } -void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, - OMPPrivateScope &LoopScope, - bool SeparateIter) { +void CodeGenFunction::EmitOMPInnerLoop( + const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, + const Expr *IncExpr, + const llvm::function_ref<void(CodeGenFunction &)> &BodyGen, + const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) { auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end"); - auto Cnt = getPGORegionCounter(&S); // Start the loop with a block that tests the condition. auto CondBlock = createBasicBlock("omp.inner.for.cond"); @@ -311,35 +535,31 @@ void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, // If there are any cleanups between here and the loop-exit scope, // create a block to stage a loop exit along. auto ExitBlock = LoopExit.getBlock(); - if (LoopScope.requiresCleanups()) + if (RequiresCleanup) ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup"); auto LoopBody = createBasicBlock("omp.inner.for.body"); - // Emit condition: "IV < LastIteration + 1 [ - 1]" - // ("- 1" when lastprivate clause is present - separate one iteration). - llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond(SeparateIter)); - Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, - PGO.createLoopWeights(S.getCond(SeparateIter), Cnt)); - + // Emit condition. + EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S)); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); } EmitBlock(LoopBody); - Cnt.beginRegion(Builder); + incrementProfileCounter(&S); // Create a block for the increment. auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc"); BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); - EmitOMPLoopBody(S); - EmitStopPoint(&S); + BodyGen(*this); // Emit "IV = IV + 1" and a back-edge to the condition block. EmitBlock(Continue.getBlock()); - EmitIgnoredExpr(S.getInc()); + EmitIgnoredExpr(IncExpr); + PostIncGen(*this); BreakContinueStack.pop_back(); EmitBranch(CondBlock); LoopStack.pop(); @@ -350,11 +570,38 @@ void CodeGenFunction::EmitOMPInnerLoop(const OMPLoopDirective &S, void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &S) { auto IC = S.counters().begin(); for (auto F : S.finals()) { - if (LocalDeclMap.lookup(cast<DeclRefExpr>((*IC))->getDecl())) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl()); + if (LocalDeclMap.lookup(OrigVD)) { + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); + auto *OrigAddr = EmitLValue(&DRE).getAddress(); + OMPPrivateScope VarScope(*this); + VarScope.addPrivate(OrigVD, + [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); + (void)VarScope.Privatize(); EmitIgnoredExpr(F); } ++IC; } + // Emit the final values of the linear variables. + for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + auto IC = C->varlist_begin(); + for (auto F : C->finals()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl()); + DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), + CapturedStmtInfo->lookup(OrigVD) != nullptr, + (*IC)->getType(), VK_LValue, (*IC)->getExprLoc()); + auto *OrigAddr = EmitLValue(&DRE).getAddress(); + OMPPrivateScope VarScope(*this); + VarScope.addPrivate(OrigVD, + [OrigAddr]() -> llvm::Value *{ return OrigAddr; }); + (void)VarScope.Privatize(); + EmitIgnoredExpr(F); + ++IC; + } + } } static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM, @@ -388,116 +635,348 @@ static void EmitPrivateLoopCounters(CodeGenFunction &CGF, ArrayRef<Expr *> Counters) { for (auto *E : Counters) { auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); - bool IsRegistered = LoopScope.addPrivate(VD, [&]() -> llvm::Value * { + (void)LoopScope.addPrivate(VD, [&]() -> llvm::Value *{ // Emit var without initialization. auto VarEmission = CGF.EmitAutoVarAlloca(*VD); CGF.EmitAutoVarCleanups(VarEmission); return VarEmission.getAllocatedAddress(); }); - assert(IsRegistered && "counter already registered as private"); - // Silence the warning about unused variable. - (void)IsRegistered; } - (void)LoopScope.Privatize(); +} + +static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, + const Expr *Cond, llvm::BasicBlock *TrueBlock, + llvm::BasicBlock *FalseBlock, uint64_t TrueCount) { + CodeGenFunction::OMPPrivateScope PreCondScope(CGF); + EmitPrivateLoopCounters(CGF, PreCondScope, S.counters()); + const VarDecl *IVDecl = + cast<VarDecl>(cast<DeclRefExpr>(S.getIterationVariable())->getDecl()); + bool IsRegistered = PreCondScope.addPrivate(IVDecl, [&]() -> llvm::Value *{ + // Emit var without initialization. + auto VarEmission = CGF.EmitAutoVarAlloca(*IVDecl); + CGF.EmitAutoVarCleanups(VarEmission); + return VarEmission.getAllocatedAddress(); + }); + assert(IsRegistered && "counter already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + (void)PreCondScope.Privatize(); + // Initialize internal counter to 0 to calculate initial values of real + // counters. + LValue IV = CGF.EmitLValue(S.getIterationVariable()); + CGF.EmitStoreOfScalar( + llvm::ConstantInt::getNullValue( + IV.getAddress()->getType()->getPointerElementType()), + CGF.EmitLValue(S.getIterationVariable()), /*isInit=*/true); + // Get initial values of real counters. + for (auto I : S.updates()) { + CGF.EmitIgnoredExpr(I); + } + // Check that loop is executed at least one time. + CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); +} + +static void +EmitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, + CodeGenFunction::OMPPrivateScope &PrivateScope) { + for (auto &&I = D.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + for (auto *E : C->varlists()) { + auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + bool IsRegistered = PrivateScope.addPrivate(VD, [&]()->llvm::Value * { + // Emit var without initialization. + auto VarEmission = CGF.EmitAutoVarAlloca(*VD); + CGF.EmitAutoVarCleanups(VarEmission); + return VarEmission.getAllocatedAddress(); + }); + assert(IsRegistered && "linear var already registered as private"); + // Silence the warning about unused variable. + (void)IsRegistered; + } + } } void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) { - // Pragma 'simd' code depends on presence of 'lastprivate'. - // If present, we have to separate last iteration of the loop: + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + // Pragma 'simd' code depends on presence of 'lastprivate'. + // If present, we have to separate last iteration of the loop: + // + // if (PreCond) { + // for (IV in 0..LastIteration-1) BODY; + // BODY with updates of lastprivate vars; + // <Final counter/linear vars updates>; + // } + // + // otherwise (when there's no lastprivate): + // + // if (PreCond) { + // for (IV in 0..LastIteration) BODY; + // <Final counter/linear vars updates>; + // } + // + + // Emit: if (PreCond) - begin. + // If the condition constant folds and can be elided, avoid emitting the + // whole loop. + bool CondConstant; + llvm::BasicBlock *ContBlock = nullptr; + if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { + if (!CondConstant) + return; + } else { + auto *ThenBlock = CGF.createBasicBlock("simd.if.then"); + ContBlock = CGF.createBasicBlock("simd.if.end"); + emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, + CGF.getProfileCount(&S)); + CGF.EmitBlock(ThenBlock); + CGF.incrementProfileCounter(&S); + } + // Walk clauses and process safelen/lastprivate. + bool SeparateIter = false; + CGF.LoopStack.setParallel(); + CGF.LoopStack.setVectorizerEnable(true); + for (auto C : S.clauses()) { + switch (C->getClauseKind()) { + case OMPC_safelen: { + RValue Len = CGF.EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(), + AggValueSlot::ignored(), true); + llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); + CGF.LoopStack.setVectorizerWidth(Val->getZExtValue()); + // In presence of finite 'safelen', it may be unsafe to mark all + // the memory instructions parallel, because loop-carried + // dependences of 'safelen' iterations are possible. + CGF.LoopStack.setParallel(false); + break; + } + case OMPC_aligned: + EmitOMPAlignedClause(CGF, CGF.CGM, cast<OMPAlignedClause>(*C)); + break; + case OMPC_lastprivate: + SeparateIter = true; + break; + default: + // Not handled yet + ; + } + } + + // Emit inits for the linear variables. + for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + for (auto Init : C->inits()) { + auto *D = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl()); + CGF.EmitVarDecl(*D); + } + } + + // Emit the loop iteration variable. + const Expr *IVExpr = S.getIterationVariable(); + const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); + CGF.EmitVarDecl(*IVDecl); + CGF.EmitIgnoredExpr(S.getInit()); + + // Emit the iterations count variable. + // If it is not a variable, Sema decided to calculate iterations count on + // each iteration (e.g., it is foldable into a constant). + if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { + CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); + // Emit calculation of the iterations count. + CGF.EmitIgnoredExpr(S.getCalcLastIteration()); + } + + // Emit the linear steps for the linear clauses. + // If a step is not constant, it is pre-calculated before the loop. + for (auto &&I = S.getClausesOfKind(OMPC_linear); I; ++I) { + auto *C = cast<OMPLinearClause>(*I); + if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep())) + if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) { + CGF.EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl())); + // Emit calculation of the linear step. + CGF.EmitIgnoredExpr(CS); + } + } + + { + OMPPrivateScope LoopScope(CGF); + EmitPrivateLoopCounters(CGF, LoopScope, S.counters()); + EmitPrivateLinearVars(CGF, S, LoopScope); + CGF.EmitOMPPrivateClause(S, LoopScope); + (void)LoopScope.Privatize(); + CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), + S.getCond(SeparateIter), S.getInc(), + [&S](CodeGenFunction &CGF) { + CGF.EmitOMPLoopBody(S); + CGF.EmitStopPoint(&S); + }, + [](CodeGenFunction &) {}); + if (SeparateIter) { + CGF.EmitOMPLoopBody(S, /*SeparateIter=*/true); + } + } + CGF.EmitOMPSimdFinal(S); + // Emit: if (PreCond) - end. + if (ContBlock) { + CGF.EmitBranch(ContBlock); + CGF.EmitBlock(ContBlock, true); + } + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); +} + +void CodeGenFunction::EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, + const OMPLoopDirective &S, + OMPPrivateScope &LoopScope, + bool Ordered, llvm::Value *LB, + llvm::Value *UB, llvm::Value *ST, + llvm::Value *IL, llvm::Value *Chunk) { + auto &RT = CGM.getOpenMPRuntime(); + + // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime). + const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind); + + assert((Ordered || + !RT.isStaticNonchunked(ScheduleKind, /*Chunked=*/Chunk != nullptr)) && + "static non-chunked schedule does not need outer loop"); + + // Emit outer loop. + // + // OpenMP [2.7.1, Loop Construct, Description, table 2-1] + // When schedule(dynamic,chunk_size) is specified, the iterations are + // distributed to threads in the team in chunks as the threads request them. + // Each thread executes a chunk of iterations, then requests another chunk, + // until no chunks remain to be distributed. Each chunk contains chunk_size + // iterations, except for the last chunk to be distributed, which may have + // fewer iterations. When no chunk_size is specified, it defaults to 1. // - // if (LastIteration != 0) { - // for (IV in 0..LastIteration-1) BODY; - // BODY with updates of lastprivate vars; - // <Final counter/linear vars updates>; + // When schedule(guided,chunk_size) is specified, the iterations are assigned + // to threads in the team in chunks as the executing threads request them. + // Each thread executes a chunk of iterations, then requests another chunk, + // until no chunks remain to be assigned. For a chunk_size of 1, the size of + // each chunk is proportional to the number of unassigned iterations divided + // by the number of threads in the team, decreasing to 1. For a chunk_size + // with value k (greater than 1), the size of each chunk is determined in the + // same way, with the restriction that the chunks do not contain fewer than k + // iterations (except for the last chunk to be assigned, which may have fewer + // than k iterations). + // + // When schedule(auto) is specified, the decision regarding scheduling is + // delegated to the compiler and/or runtime system. The programmer gives the + // implementation the freedom to choose any possible mapping of iterations to + // threads in the team. + // + // When schedule(runtime) is specified, the decision regarding scheduling is + // deferred until run time, and the schedule and chunk size are taken from the + // run-sched-var ICV. If the ICV is set to auto, the schedule is + // implementation defined + // + // while(__kmpc_dispatch_next(&LB, &UB)) { + // idx = LB; + // while (idx <= UB) { BODY; ++idx; + // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only. + // } // inner loop // } // - // otherwise (when there's no lastprivate): + // OpenMP [2.7.1, Loop Construct, Description, table 2-1] + // When schedule(static, chunk_size) is specified, iterations are divided into + // chunks of size chunk_size, and the chunks are assigned to the threads in + // the team in a round-robin fashion in the order of the thread number. // - // for (IV in 0..LastIteration) BODY; - // <Final counter/linear vars updates>; + // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) { + // while (idx <= UB) { BODY; ++idx; } // inner loop + // LB = LB + ST; + // UB = UB + ST; + // } // - // Walk clauses and process safelen/lastprivate. - bool SeparateIter = false; - LoopStack.setParallel(); - LoopStack.setVectorizerEnable(true); - for (auto C : S.clauses()) { - switch (C->getClauseKind()) { - case OMPC_safelen: { - RValue Len = EmitAnyExpr(cast<OMPSafelenClause>(C)->getSafelen(), - AggValueSlot::ignored(), true); - llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal()); - LoopStack.setVectorizerWidth(Val->getZExtValue()); - // In presence of finite 'safelen', it may be unsafe to mark all - // the memory instructions parallel, because loop-carried - // dependences of 'safelen' iterations are possible. - LoopStack.setParallel(false); - break; - } - case OMPC_aligned: - EmitOMPAlignedClause(*this, CGM, cast<OMPAlignedClause>(*C)); - break; - case OMPC_lastprivate: - SeparateIter = true; - break; - default: - // Not handled yet - ; - } - } + const Expr *IVExpr = S.getIterationVariable(); + const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); + const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope DirectiveScope(*this); + RT.emitForInit( + *this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, Ordered, IL, LB, + (DynamicOrOrdered ? EmitAnyExpr(S.getLastIteration()).getScalarVal() + : UB), + ST, Chunk); - CGDebugInfo *DI = getDebugInfo(); - if (DI) - DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); + auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end"); - // Emit the loop iteration variable. - const Expr *IVExpr = S.getIterationVariable(); - const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl()); - EmitVarDecl(*IVDecl); - EmitIgnoredExpr(S.getInit()); + // Start the loop with a block that tests the condition. + auto CondBlock = createBasicBlock("omp.dispatch.cond"); + EmitBlock(CondBlock); + LoopStack.push(CondBlock); - // Emit the iterations count variable. - // If it is not a variable, Sema decided to calculate iterations count on each - // iteration (e.g., it is foldable into a constant). - if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) { - EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); - // Emit calculation of the iterations count. - EmitIgnoredExpr(S.getCalcLastIteration()); + llvm::Value *BoolCondVal = nullptr; + if (!DynamicOrOrdered) { + // UB = min(UB, GlobalUB) + EmitIgnoredExpr(S.getEnsureUpperBound()); + // IV = LB + EmitIgnoredExpr(S.getInit()); + // IV < UB + BoolCondVal = EvaluateExprAsBool(S.getCond(false)); + } else { + BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, + IL, LB, UB, ST); } - if (SeparateIter) { - // Emit: if (LastIteration > 0) - begin. - RegionCounter Cnt = getPGORegionCounter(&S); - auto ThenBlock = createBasicBlock("simd.if.then"); - auto ContBlock = createBasicBlock("simd.if.end"); - EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); - EmitBlock(ThenBlock); - Cnt.beginRegion(Builder); - // Emit 'then' code. - { - OMPPrivateScope LoopScope(*this); - EmitPrivateLoopCounters(*this, LoopScope, S.counters()); - EmitOMPInnerLoop(S, LoopScope, /* SeparateIter */ true); - EmitOMPLoopBody(S, /* SeparateIter */ true); - } - EmitOMPSimdFinal(S); - // Emit: if (LastIteration != 0) - end. - EmitBranch(ContBlock); - EmitBlock(ContBlock, true); - } else { - { - OMPPrivateScope LoopScope(*this); - EmitPrivateLoopCounters(*this, LoopScope, S.counters()); - EmitOMPInnerLoop(S, LoopScope); - } - EmitOMPSimdFinal(S); + // If there are any cleanups between here and the loop-exit scope, + // create a block to stage a loop exit along. + auto ExitBlock = LoopExit.getBlock(); + if (LoopScope.requiresCleanups()) + ExitBlock = createBasicBlock("omp.dispatch.cleanup"); + + auto LoopBody = createBasicBlock("omp.dispatch.body"); + Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); + if (ExitBlock != LoopExit.getBlock()) { + EmitBlock(ExitBlock); + EmitBranchThroughCleanup(LoopExit); + } + EmitBlock(LoopBody); + + // Emit "IV = LB" (in case of static schedule, we have already calculated new + // LB for loop condition and emitted it above). + if (DynamicOrOrdered) + EmitIgnoredExpr(S.getInit()); + + // Create a block for the increment. + auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc"); + BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); + + SourceLocation Loc = S.getLocStart(); + // Generate !llvm.loop.parallel metadata for loads and stores for loops with + // dynamic/guided scheduling and without ordered clause. + LoopStack.setParallel((ScheduleKind == OMPC_SCHEDULE_dynamic || + ScheduleKind == OMPC_SCHEDULE_guided) && + !Ordered); + EmitOMPInnerLoop( + S, LoopScope.requiresCleanups(), S.getCond(/*SeparateIter=*/false), + S.getInc(), + [&S](CodeGenFunction &CGF) { + CGF.EmitOMPLoopBody(S); + CGF.EmitStopPoint(&S); + }, + [Ordered, IVSize, IVSigned, Loc](CodeGenFunction &CGF) { + if (Ordered) { + CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd( + CGF, Loc, IVSize, IVSigned); + } + }); + + EmitBlock(Continue.getBlock()); + BreakContinueStack.pop_back(); + if (!DynamicOrOrdered) { + // Emit "LB = LB + Stride", "UB = UB + Stride". + EmitIgnoredExpr(S.getNextLowerBound()); + EmitIgnoredExpr(S.getNextUpperBound()); } - if (DI) - DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); + EmitBranch(CondBlock); + LoopStack.pop(); + // Emit the fall-through block. + EmitBlock(LoopExit.getBlock()); + + // Tell the runtime we are done. + if (!DynamicOrOrdered) + RT.emitForStaticFinish(*this, S.getLocEnd()); } /// \brief Emit a helper variable and return corresponding lvalue. @@ -508,7 +987,39 @@ static LValue EmitOMPHelperVar(CodeGenFunction &CGF, return CGF.EmitLValue(Helper); } -void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { +static std::pair<llvm::Value * /*Chunk*/, OpenMPScheduleClauseKind> +emitScheduleClause(CodeGenFunction &CGF, const OMPLoopDirective &S, + bool OuterRegion) { + // Detect the loop schedule kind and chunk. + auto ScheduleKind = OMPC_SCHEDULE_unknown; + llvm::Value *Chunk = nullptr; + if (auto *C = + cast_or_null<OMPScheduleClause>(S.getSingleClause(OMPC_schedule))) { + ScheduleKind = C->getScheduleKind(); + if (const auto *Ch = C->getChunkSize()) { + if (auto *ImpRef = cast_or_null<DeclRefExpr>(C->getHelperChunkSize())) { + if (OuterRegion) { + const VarDecl *ImpVar = cast<VarDecl>(ImpRef->getDecl()); + CGF.EmitVarDecl(*ImpVar); + CGF.EmitStoreThroughLValue( + CGF.EmitAnyExpr(Ch), + CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(ImpVar), + ImpVar->getType())); + } else { + Ch = ImpRef; + } + } + if (!C->getHelperChunkSize() || !OuterRegion) { + Chunk = CGF.EmitScalarExpr(Ch); + Chunk = CGF.EmitScalarConversion(Chunk, Ch->getType(), + S.getIterationVariable()->getType()); + } + } + } + return std::make_pair(Chunk, ScheduleKind); +} + +bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { // Emit the loop iteration variable. auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable()); auto IVDecl = cast<VarDecl>(IVExpr->getDecl()); @@ -525,15 +1036,25 @@ void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { auto &RT = CGM.getOpenMPRuntime(); + bool HasLastprivateClause; // Check pre-condition. { // Skip the entire loop if we don't meet the precondition. - RegionCounter Cnt = getPGORegionCounter(&S); - auto ThenBlock = createBasicBlock("omp.precond.then"); - auto ContBlock = createBasicBlock("omp.precond.end"); - EmitBranchOnBoolExpr(S.getPreCond(), ThenBlock, ContBlock, Cnt.getCount()); - EmitBlock(ThenBlock); - Cnt.beginRegion(Builder); + // If the condition constant folds and can be elided, avoid emitting the + // whole loop. + bool CondConstant; + llvm::BasicBlock *ContBlock = nullptr; + if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { + if (!CondConstant) + return false; + } else { + auto *ThenBlock = createBasicBlock("omp.precond.then"); + ContBlock = createBasicBlock("omp.precond.end"); + emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock, + getProfileCount(&S)); + EmitBlock(ThenBlock); + incrementProfileCounter(&S); + } // Emit 'then' code. { // Emit helper vars inits. @@ -547,105 +1068,342 @@ void CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) { EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); OMPPrivateScope LoopScope(*this); + if (EmitOMPFirstprivateClause(S, LoopScope)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), + OMPD_unknown); + } + EmitOMPPrivateClause(S, LoopScope); + HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); + EmitOMPReductionClauseInit(S, LoopScope); EmitPrivateLoopCounters(*this, LoopScope, S.counters()); + (void)LoopScope.Privatize(); // Detect the loop schedule kind and chunk. - auto ScheduleKind = OMPC_SCHEDULE_unknown; - llvm::Value *Chunk = nullptr; - if (auto C = cast_or_null<OMPScheduleClause>( - S.getSingleClause(OMPC_schedule))) { - ScheduleKind = C->getScheduleKind(); - if (auto Ch = C->getChunkSize()) { - Chunk = EmitScalarExpr(Ch); - Chunk = EmitScalarConversion(Chunk, Ch->getType(), - S.getIterationVariable()->getType()); - } - } + llvm::Value *Chunk; + OpenMPScheduleClauseKind ScheduleKind; + auto ScheduleInfo = + emitScheduleClause(*this, S, /*OuterRegion=*/false); + Chunk = ScheduleInfo.first; + ScheduleKind = ScheduleInfo.second; const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); + const bool Ordered = S.getSingleClause(OMPC_ordered) != nullptr; if (RT.isStaticNonchunked(ScheduleKind, - /* Chunked */ Chunk != nullptr)) { + /* Chunked */ Chunk != nullptr) && + !Ordered) { // OpenMP [2.7.1, Loop Construct, Description, table 2-1] // When no chunk_size is specified, the iteration space is divided into // chunks that are approximately equal in size, and at most one chunk is // distributed to each thread. Note that the size of the chunks is // unspecified in this case. - RT.EmitOMPForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, - IL.getAddress(), LB.getAddress(), UB.getAddress(), - ST.getAddress()); + RT.emitForInit(*this, S.getLocStart(), ScheduleKind, IVSize, IVSigned, + Ordered, IL.getAddress(), LB.getAddress(), + UB.getAddress(), ST.getAddress()); // UB = min(UB, GlobalUB); EmitIgnoredExpr(S.getEnsureUpperBound()); // IV = LB; EmitIgnoredExpr(S.getInit()); // while (idx <= UB) { BODY; ++idx; } - EmitOMPInnerLoop(S, LoopScope); + EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), + S.getCond(/*SeparateIter=*/false), S.getInc(), + [&S](CodeGenFunction &CGF) { + CGF.EmitOMPLoopBody(S); + CGF.EmitStopPoint(&S); + }, + [](CodeGenFunction &) {}); // Tell the runtime we are done. - RT.EmitOMPForFinish(*this, S.getLocStart(), ScheduleKind); - } else - ErrorUnsupported(&S, "OpenMP loop with requested schedule"); + RT.emitForStaticFinish(*this, S.getLocStart()); + } else { + // Emit the outer loop, which requests its work chunk [LB..UB] from + // runtime and runs the inner loop to process it. + EmitOMPForOuterLoop(ScheduleKind, S, LoopScope, Ordered, + LB.getAddress(), UB.getAddress(), ST.getAddress(), + IL.getAddress(), Chunk); + } + EmitOMPReductionClauseFinal(S); + // Emit final copy of the lastprivate variables if IsLastIter != 0. + if (HasLastprivateClause) + EmitOMPLastprivateClauseFinal( + S, Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart()))); } // We're now done with the loop, so jump to the continuation block. - EmitBranch(ContBlock); - EmitBlock(ContBlock, true); + if (ContBlock) { + EmitBranch(ContBlock); + EmitBlock(ContBlock, true); + } } + return HasLastprivateClause; } void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) { - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope DirectiveScope(*this); - - CGDebugInfo *DI = getDebugInfo(); - if (DI) - DI->EmitLexicalBlockStart(Builder, S.getSourceRange().getBegin()); - - EmitOMPWorksharingLoop(S); + LexicalScope Scope(*this, S.getSourceRange()); + bool HasLastprivates = false; + auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) { + HasLastprivates = CGF.EmitOMPWorksharingLoop(S); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); // Emit an implicit barrier at the end. - CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart(), - /*IsExplicit*/ false); - if (DI) - DI->EmitLexicalBlockEnd(Builder, S.getSourceRange().getEnd()); + if (!S.getSingleClause(OMPC_nowait) || HasLastprivates) { + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for); + } } void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &) { llvm_unreachable("CodeGen for 'omp for simd' is not supported yet."); } -void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &) { - llvm_unreachable("CodeGen for 'omp sections' is not supported yet."); +static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, + const Twine &Name, + llvm::Value *Init = nullptr) { + auto LVal = CGF.MakeNaturalAlignAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); + if (Init) + CGF.EmitScalarInit(Init, LVal); + return LVal; } -void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &) { - llvm_unreachable("CodeGen for 'omp section' is not supported yet."); +static OpenMPDirectiveKind emitSections(CodeGenFunction &CGF, + const OMPExecutableDirective &S) { + auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt(); + auto *CS = dyn_cast<CompoundStmt>(Stmt); + if (CS && CS->size() > 1) { + bool HasLastprivates = false; + auto &&CodeGen = [&S, CS, &HasLastprivates](CodeGenFunction &CGF) { + auto &C = CGF.CGM.getContext(); + auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); + // Emit helper vars inits. + LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", + CGF.Builder.getInt32(0)); + auto *GlobalUBVal = CGF.Builder.getInt32(CS->size() - 1); + LValue UB = + createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); + LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", + CGF.Builder.getInt32(1)); + LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", + CGF.Builder.getInt32(0)); + // Loop counter. + LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); + OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); + CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); + OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue); + CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); + // Generate condition for loop. + BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, + OK_Ordinary, S.getLocStart(), + /*fpContractable=*/false); + // Increment for loop counter. + UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, + OK_Ordinary, S.getLocStart()); + auto BodyGen = [CS, &S, &IV](CodeGenFunction &CGF) { + // Iterate through all sections and emit a switch construct: + // switch (IV) { + // case 0: + // <SectionStmt[0]>; + // break; + // ... + // case <NumSection> - 1: + // <SectionStmt[<NumSection> - 1]>; + // break; + // } + // .omp.sections.exit: + auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); + auto *SwitchStmt = CGF.Builder.CreateSwitch( + CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB, + CS->size()); + unsigned CaseNumber = 0; + for (auto C = CS->children(); C; ++C, ++CaseNumber) { + auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); + CGF.EmitBlock(CaseBB); + SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); + CGF.EmitStmt(*C); + CGF.EmitBranch(ExitBB); + } + CGF.EmitBlock(ExitBB, /*IsFinished=*/true); + }; + + CodeGenFunction::OMPPrivateScope LoopScope(CGF); + if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + } + CGF.EmitOMPPrivateClause(S, LoopScope); + HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); + CGF.EmitOMPReductionClauseInit(S, LoopScope); + (void)LoopScope.Privatize(); + + // Emit static non-chunked loop. + CGF.CGM.getOpenMPRuntime().emitForInit( + CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32, + /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), + LB.getAddress(), UB.getAddress(), ST.getAddress()); + // UB = min(UB, GlobalUB); + auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart()); + auto *MinUBGlobalUB = CGF.Builder.CreateSelect( + CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); + CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); + // IV = LB; + CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV); + // while (idx <= UB) { BODY; ++idx; } + CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen, + [](CodeGenFunction &) {}); + // Tell the runtime we are done. + CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart()); + CGF.EmitOMPReductionClauseFinal(S); + + // Emit final copy of the lastprivate variables if IsLastIter != 0. + if (HasLastprivates) + CGF.EmitOMPLastprivateClauseFinal( + S, CGF.Builder.CreateIsNotNull( + CGF.EmitLoadOfScalar(IL, S.getLocStart()))); + }; + + CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, CodeGen); + // Emit barrier for lastprivates only if 'sections' directive has 'nowait' + // clause. Otherwise the barrier will be generated by the codegen for the + // directive. + if (HasLastprivates && S.getSingleClause(OMPC_nowait)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + } + return OMPD_sections; + } + // If only one section is found - no need to generate loop, emit as a single + // region. + bool HasFirstprivates; + // No need to generate reductions for sections with single section region, we + // can use original shared variables for all operations. + bool HasReductions = !S.getClausesOfKind(OMPC_reduction).empty(); + // No need to generate lastprivates for sections with single section region, + // we can use original shared variable for all calculations with barrier at + // the end of the sections. + bool HasLastprivates = !S.getClausesOfKind(OMPC_lastprivate).empty(); + auto &&CodeGen = [Stmt, &S, &HasFirstprivates](CodeGenFunction &CGF) { + CodeGenFunction::OMPPrivateScope SingleScope(CGF); + HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope); + CGF.EmitOMPPrivateClause(S, SingleScope); + (void)SingleScope.Privatize(); + + CGF.EmitStmt(Stmt); + CGF.EnsureInsertPoint(); + }; + CGF.CGM.getOpenMPRuntime().emitSingleRegion(CGF, CodeGen, S.getLocStart(), + llvm::None, llvm::None, + llvm::None, llvm::None); + // Emit barrier for firstprivates, lastprivates or reductions only if + // 'sections' directive has 'nowait' clause. Otherwise the barrier will be + // generated by the codegen for the directive. + if ((HasFirstprivates || HasLastprivates || HasReductions) && + S.getSingleClause(OMPC_nowait)) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_unknown); + } + return OMPD_single; +} + +void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + OpenMPDirectiveKind EmittedAs = emitSections(*this, S); + // Emit an implicit barrier at the end. + if (!S.getSingleClause(OMPC_nowait)) { + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), EmittedAs); + } } -void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &) { - llvm_unreachable("CodeGen for 'omp single' is not supported yet."); +void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); +} + +void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) { + llvm::SmallVector<const Expr *, 8> CopyprivateVars; + llvm::SmallVector<const Expr *, 8> DestExprs; + llvm::SmallVector<const Expr *, 8> SrcExprs; + llvm::SmallVector<const Expr *, 8> AssignmentOps; + // Check if there are any 'copyprivate' clauses associated with this + // 'single' + // construct. + // Build a list of copyprivate variables along with helper expressions + // (<source>, <destination>, <destination>=<source> expressions) + for (auto &&I = S.getClausesOfKind(OMPC_copyprivate); I; ++I) { + auto *C = cast<OMPCopyprivateClause>(*I); + CopyprivateVars.append(C->varlists().begin(), C->varlists().end()); + DestExprs.append(C->destination_exprs().begin(), + C->destination_exprs().end()); + SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end()); + AssignmentOps.append(C->assignment_ops().begin(), + C->assignment_ops().end()); + } + LexicalScope Scope(*this, S.getSourceRange()); + // Emit code for 'single' region along with 'copyprivate' clauses + bool HasFirstprivates; + auto &&CodeGen = [&S, &HasFirstprivates](CodeGenFunction &CGF) { + CodeGenFunction::OMPPrivateScope SingleScope(CGF); + HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope); + CGF.EmitOMPPrivateClause(S, SingleScope); + (void)SingleScope.Privatize(); + + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(), + CopyprivateVars, DestExprs, SrcExprs, + AssignmentOps); + // Emit an implicit barrier at the end (to avoid data race on firstprivate + // init or if no 'nowait' clause was specified and no 'copyprivate' clause). + if ((!S.getSingleClause(OMPC_nowait) || HasFirstprivates) && + CopyprivateVars.empty()) { + CGM.getOpenMPRuntime().emitBarrierCall( + *this, S.getLocStart(), + S.getSingleClause(OMPC_nowait) ? OMPD_unknown : OMPD_single); + } } void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) { - CGM.getOpenMPRuntime().EmitOMPMasterRegion(*this, [&]() -> void { - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope Scope(*this); - EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - EnsureInsertPoint(); - }, S.getLocStart()); + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart()); } void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) { - CGM.getOpenMPRuntime().EmitOMPCriticalRegion( - *this, S.getDirectiveName().getAsString(), [&]() -> void { - InlinedOpenMPRegion Region(*this, S.getAssociatedStmt()); - RunCleanupsScope Scope(*this); - EmitStmt( - cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); - EnsureInsertPoint(); - }, S.getLocStart()); + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitCriticalRegion( + *this, S.getDirectiveName().getAsString(), CodeGen, S.getLocStart()); } -void -CodeGenFunction::EmitOMPParallelForDirective(const OMPParallelForDirective &) { - llvm_unreachable("CodeGen for 'omp parallel for' is not supported yet."); +void CodeGenFunction::EmitOMPParallelForDirective( + const OMPParallelForDirective &S) { + // Emit directive as a combined directive that consists of two implicit + // directives: 'parallel' with 'for' directive. + LexicalScope Scope(*this, S.getSourceRange()); + (void)emitScheduleClause(*this, S, /*OuterRegion=*/true); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitOMPWorksharingLoop(S); + // Emit implicit barrier at the end of parallel region, but this barrier + // is at the end of 'for' directive, so emit it as the implicit barrier for + // this 'for' directive. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_parallel); + }; + emitCommonOMPParallelDirective(*this, S, CodeGen); } void CodeGenFunction::EmitOMPParallelForSimdDirective( @@ -654,45 +1412,600 @@ void CodeGenFunction::EmitOMPParallelForSimdDirective( } void CodeGenFunction::EmitOMPParallelSectionsDirective( - const OMPParallelSectionsDirective &) { - llvm_unreachable("CodeGen for 'omp parallel sections' is not supported yet."); + const OMPParallelSectionsDirective &S) { + // Emit directive as a combined directive that consists of two implicit + // directives: 'parallel' with 'sections' directive. + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + (void)emitSections(CGF, S); + // Emit implicit barrier at the end of parallel region. + CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getLocStart(), + OMPD_parallel); + }; + emitCommonOMPParallelDirective(*this, S, CodeGen); } -void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &) { - llvm_unreachable("CodeGen for 'omp task' is not supported yet."); +void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) { + // Emit outlined function for task construct. + LexicalScope Scope(*this, S.getSourceRange()); + auto CS = cast<CapturedStmt>(S.getAssociatedStmt()); + auto CapturedStruct = GenerateCapturedStmtArgument(*CS); + auto *I = CS->getCapturedDecl()->param_begin(); + auto *PartId = std::next(I); + // The first function argument for tasks is a thread id, the second one is a + // part id (0 for tied tasks, >=0 for untied task). + llvm::DenseSet<const VarDecl *> EmittedAsPrivate; + // Get list of private variables. + llvm::SmallVector<const Expr *, 8> PrivateVars; + llvm::SmallVector<const Expr *, 8> PrivateCopies; + for (auto &&I = S.getClausesOfKind(OMPC_private); I; ++I) { + auto *C = cast<OMPPrivateClause>(*I); + auto IRef = C->varlist_begin(); + for (auto *IInit : C->private_copies()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { + PrivateVars.push_back(*IRef); + PrivateCopies.push_back(IInit); + } + ++IRef; + } + } + EmittedAsPrivate.clear(); + // Get list of firstprivate variables. + llvm::SmallVector<const Expr *, 8> FirstprivateVars; + llvm::SmallVector<const Expr *, 8> FirstprivateCopies; + llvm::SmallVector<const Expr *, 8> FirstprivateInits; + for (auto &&I = S.getClausesOfKind(OMPC_firstprivate); I; ++I) { + auto *C = cast<OMPFirstprivateClause>(*I); + auto IRef = C->varlist_begin(); + auto IElemInitRef = C->inits().begin(); + for (auto *IInit : C->private_copies()) { + auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()); + if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) { + FirstprivateVars.push_back(*IRef); + FirstprivateCopies.push_back(IInit); + FirstprivateInits.push_back(*IElemInitRef); + } + ++IRef, ++IElemInitRef; + } + } + auto &&CodeGen = [PartId, &S, &PrivateVars, &FirstprivateVars]( + CodeGenFunction &CGF) { + // Set proper addresses for generated private copies. + auto *CS = cast<CapturedStmt>(S.getAssociatedStmt()); + OMPPrivateScope Scope(CGF); + if (!PrivateVars.empty() || !FirstprivateVars.empty()) { + auto *CopyFn = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)), + CGF.PointerAlignInBytes); + auto *PrivatesPtr = CGF.Builder.CreateAlignedLoad( + CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)), + CGF.PointerAlignInBytes); + // Map privates. + llvm::SmallVector<std::pair<const VarDecl *, llvm::Value *>, 16> + PrivatePtrs; + llvm::SmallVector<llvm::Value *, 16> CallArgs; + CallArgs.push_back(PrivatesPtr); + for (auto *E : PrivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + auto *PrivatePtr = + CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); + PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); + CallArgs.push_back(PrivatePtr); + } + for (auto *E : FirstprivateVars) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); + auto *PrivatePtr = + CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType())); + PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr)); + CallArgs.push_back(PrivatePtr); + } + CGF.EmitRuntimeCall(CopyFn, CallArgs); + for (auto &&Pair : PrivatePtrs) { + auto *Replacement = + CGF.Builder.CreateAlignedLoad(Pair.second, CGF.PointerAlignInBytes); + Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; }); + } + } + (void)Scope.Privatize(); + if (*PartId) { + // TODO: emit code for untied tasks. + } + CGF.EmitStmt(CS->getCapturedStmt()); + }; + auto OutlinedFn = + CGM.getOpenMPRuntime().emitTaskOutlinedFunction(S, *I, CodeGen); + // Check if we should emit tied or untied task. + bool Tied = !S.getSingleClause(OMPC_untied); + // Check if the task is final + llvm::PointerIntPair<llvm::Value *, 1, bool> Final; + if (auto *Clause = S.getSingleClause(OMPC_final)) { + // If the condition constant folds and can be elided, try to avoid emitting + // the condition and the dead arm of the if/else. + auto *Cond = cast<OMPFinalClause>(Clause)->getCondition(); + bool CondConstant; + if (ConstantFoldsToSimpleInteger(Cond, CondConstant)) + Final.setInt(CondConstant); + else + Final.setPointer(EvaluateExprAsBool(Cond)); + } else { + // By default the task is not final. + Final.setInt(/*IntVal=*/false); + } + auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl()); + const Expr *IfCond = nullptr; + if (auto C = S.getSingleClause(OMPC_if)) { + IfCond = cast<OMPIfClause>(C)->getCondition(); + } + CGM.getOpenMPRuntime().emitTaskCall( + *this, S.getLocStart(), S, Tied, Final, OutlinedFn, SharedsTy, + CapturedStruct, IfCond, PrivateVars, PrivateCopies, FirstprivateVars, + FirstprivateCopies, FirstprivateInits); } -void CodeGenFunction::EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &) { - llvm_unreachable("CodeGen for 'omp taskyield' is not supported yet."); +void CodeGenFunction::EmitOMPTaskyieldDirective( + const OMPTaskyieldDirective &S) { + CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart()); } void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) { - CGM.getOpenMPRuntime().EmitOMPBarrierCall(*this, S.getLocStart()); + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier); } -void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &) { - llvm_unreachable("CodeGen for 'omp taskwait' is not supported yet."); +void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) { + CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart()); } void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) { - CGM.getOpenMPRuntime().EmitOMPFlush( - *this, [&]() -> ArrayRef<const Expr *> { - if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { - auto FlushClause = cast<OMPFlushClause>(C); - return llvm::makeArrayRef(FlushClause->varlist_begin(), - FlushClause->varlist_end()); - } - return llvm::None; - }(), - S.getLocStart()); + CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> { + if (auto C = S.getSingleClause(/*K*/ OMPC_flush)) { + auto FlushClause = cast<OMPFlushClause>(C); + return llvm::makeArrayRef(FlushClause->varlist_begin(), + FlushClause->varlist_end()); + } + return llvm::None; + }(), S.getLocStart()); +} + +void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) { + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S](CodeGenFunction &CGF) { + CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt()); + CGF.EnsureInsertPoint(); + }; + CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart()); +} + +static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, + QualType SrcType, QualType DestType) { + assert(CGF.hasScalarEvaluationKind(DestType) && + "DestType must have scalar evaluation kind."); + assert(!Val.isAggregate() && "Must be a scalar or complex."); + return Val.isScalar() + ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType) + : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType, + DestType); +} + +static CodeGenFunction::ComplexPairTy +convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, + QualType DestType) { + assert(CGF.getEvaluationKind(DestType) == TEK_Complex && + "DestType must have complex evaluation kind."); + CodeGenFunction::ComplexPairTy ComplexVal; + if (Val.isScalar()) { + // Convert the input element to the element type of the complex. + auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); + auto ScalarVal = + CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestElementType); + ComplexVal = CodeGenFunction::ComplexPairTy( + ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType())); + } else { + assert(Val.isComplex() && "Must be a scalar or complex."); + auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType(); + auto DestElementType = DestType->castAs<ComplexType>()->getElementType(); + ComplexVal.first = CGF.EmitScalarConversion( + Val.getComplexVal().first, SrcElementType, DestElementType); + ComplexVal.second = CGF.EmitScalarConversion( + Val.getComplexVal().second, SrcElementType, DestElementType); + } + return ComplexVal; +} + +static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, + LValue LVal, RValue RVal) { + if (LVal.isGlobalReg()) { + CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); + } else { + CGF.EmitAtomicStore(RVal, LVal, IsSeqCst ? llvm::SequentiallyConsistent + : llvm::Monotonic, + LVal.isVolatile(), /*IsInit=*/false); + } +} + +static void emitSimpleStore(CodeGenFunction &CGF, LValue LVal, RValue RVal, + QualType RValTy) { + switch (CGF.getEvaluationKind(LVal.getType())) { + case TEK_Scalar: + CGF.EmitStoreThroughLValue( + RValue::get(convertToScalarValue(CGF, RVal, RValTy, LVal.getType())), + LVal); + break; + case TEK_Complex: + CGF.EmitStoreOfComplex( + convertToComplexValue(CGF, RVal, RValTy, LVal.getType()), LVal, + /*isInit=*/false); + break; + case TEK_Aggregate: + llvm_unreachable("Must be a scalar or complex."); + } } -void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &) { - llvm_unreachable("CodeGen for 'omp ordered' is not supported yet."); +static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, + const Expr *X, const Expr *V, + SourceLocation Loc) { + // v = x; + assert(V->isLValue() && "V of 'omp atomic read' is not lvalue"); + assert(X->isLValue() && "X of 'omp atomic read' is not lvalue"); + LValue XLValue = CGF.EmitLValue(X); + LValue VLValue = CGF.EmitLValue(V); + RValue Res = XLValue.isGlobalReg() + ? CGF.EmitLoadOfLValue(XLValue, Loc) + : CGF.EmitAtomicLoad(XLValue, Loc, + IsSeqCst ? llvm::SequentiallyConsistent + : llvm::Monotonic, + XLValue.isVolatile()); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); + emitSimpleStore(CGF,VLValue, Res, X->getType().getNonReferenceType()); } -void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &) { - llvm_unreachable("CodeGen for 'omp atomic' is not supported yet."); +static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, + const Expr *X, const Expr *E, + SourceLocation Loc) { + // x = expr; + assert(X->isLValue() && "X of 'omp atomic write' is not lvalue"); + emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); +} + +static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, + RValue Update, + BinaryOperatorKind BO, + llvm::AtomicOrdering AO, + bool IsXLHSInRHSPart) { + auto &Context = CGF.CGM.getContext(); + // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x' + // expression is simple and atomic is allowed for the given type for the + // target platform. + if (BO == BO_Comma || !Update.isScalar() || + !Update.getScalarVal()->getType()->isIntegerTy() || + !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) && + (Update.getScalarVal()->getType() != + X.getAddress()->getType()->getPointerElementType())) || + !X.getAddress()->getType()->getPointerElementType()->isIntegerTy() || + !Context.getTargetInfo().hasBuiltinAtomic( + Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment()))) + return std::make_pair(false, RValue::get(nullptr)); + + llvm::AtomicRMWInst::BinOp RMWOp; + switch (BO) { + case BO_Add: + RMWOp = llvm::AtomicRMWInst::Add; + break; + case BO_Sub: + if (!IsXLHSInRHSPart) + return std::make_pair(false, RValue::get(nullptr)); + RMWOp = llvm::AtomicRMWInst::Sub; + break; + case BO_And: + RMWOp = llvm::AtomicRMWInst::And; + break; + case BO_Or: + RMWOp = llvm::AtomicRMWInst::Or; + break; + case BO_Xor: + RMWOp = llvm::AtomicRMWInst::Xor; + break; + case BO_LT: + RMWOp = X.getType()->hasSignedIntegerRepresentation() + ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min + : llvm::AtomicRMWInst::Max) + : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin + : llvm::AtomicRMWInst::UMax); + break; + case BO_GT: + RMWOp = X.getType()->hasSignedIntegerRepresentation() + ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max + : llvm::AtomicRMWInst::Min) + : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax + : llvm::AtomicRMWInst::UMin); + break; + case BO_Assign: + RMWOp = llvm::AtomicRMWInst::Xchg; + break; + case BO_Mul: + case BO_Div: + case BO_Rem: + case BO_Shl: + case BO_Shr: + case BO_LAnd: + case BO_LOr: + return std::make_pair(false, RValue::get(nullptr)); + case BO_PtrMemD: + case BO_PtrMemI: + case BO_LE: + case BO_GE: + case BO_EQ: + case BO_NE: + case BO_AddAssign: + case BO_SubAssign: + case BO_AndAssign: + case BO_OrAssign: + case BO_XorAssign: + case BO_MulAssign: + case BO_DivAssign: + case BO_RemAssign: + case BO_ShlAssign: + case BO_ShrAssign: + case BO_Comma: + llvm_unreachable("Unsupported atomic update operation"); + } + auto *UpdateVal = Update.getScalarVal(); + if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) { + UpdateVal = CGF.Builder.CreateIntCast( + IC, X.getAddress()->getType()->getPointerElementType(), + X.getType()->hasSignedIntegerRepresentation()); + } + auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getAddress(), UpdateVal, AO); + return std::make_pair(true, RValue::get(Res)); +} + +std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr( + LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, + llvm::AtomicOrdering AO, SourceLocation Loc, + const llvm::function_ref<RValue(RValue)> &CommonGen) { + // Update expressions are allowed to have the following forms: + // x binop= expr; -> xrval + expr; + // x++, ++x -> xrval + 1; + // x--, --x -> xrval - 1; + // x = x binop expr; -> xrval binop expr + // x = expr Op x; - > expr binop xrval; + auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart); + if (!Res.first) { + if (X.isGlobalReg()) { + // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop + // 'xrval'. + EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X); + } else { + // Perform compare-and-swap procedure. + EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified()); + } + } + return Res; +} + +static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, + const Expr *X, const Expr *E, + const Expr *UE, bool IsXLHSInRHSPart, + SourceLocation Loc) { + assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) && + "Update expr in 'atomic update' must be a binary operator."); + auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); + // Update expressions are allowed to have the following forms: + // x binop= expr; -> xrval + expr; + // x++, ++x -> xrval + 1; + // x--, --x -> xrval - 1; + // x = x binop expr; -> xrval binop expr + // x = expr Op x; - > expr binop xrval; + assert(X->isLValue() && "X of 'omp atomic update' is not lvalue"); + LValue XLValue = CGF.EmitLValue(X); + RValue ExprRValue = CGF.EmitAnyExpr(E); + auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic; + auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); + auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); + auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; + auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; + auto Gen = + [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue { + CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); + CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); + return CGF.EmitAnyExpr(UE); + }; + (void)CGF.EmitOMPAtomicSimpleUpdateExpr( + XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); +} + +static RValue convertToType(CodeGenFunction &CGF, RValue Value, + QualType SourceType, QualType ResType) { + switch (CGF.getEvaluationKind(ResType)) { + case TEK_Scalar: + return RValue::get(convertToScalarValue(CGF, Value, SourceType, ResType)); + case TEK_Complex: { + auto Res = convertToComplexValue(CGF, Value, SourceType, ResType); + return RValue::getComplex(Res.first, Res.second); + } + case TEK_Aggregate: + break; + } + llvm_unreachable("Must be a scalar or complex."); +} + +static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, + bool IsPostfixUpdate, const Expr *V, + const Expr *X, const Expr *E, + const Expr *UE, bool IsXLHSInRHSPart, + SourceLocation Loc) { + assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue"); + assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue"); + RValue NewVVal; + LValue VLValue = CGF.EmitLValue(V); + LValue XLValue = CGF.EmitLValue(X); + RValue ExprRValue = CGF.EmitAnyExpr(E); + auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic; + QualType NewVValType; + if (UE) { + // 'x' is updated with some additional value. + assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) && + "Update expr in 'atomic capture' must be a binary operator."); + auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts()); + // Update expressions are allowed to have the following forms: + // x binop= expr; -> xrval + expr; + // x++, ++x -> xrval + 1; + // x--, --x -> xrval - 1; + // x = x binop expr; -> xrval binop expr + // x = expr Op x; - > expr binop xrval; + auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts()); + auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts()); + auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS; + NewVValType = XRValExpr->getType(); + auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS; + auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, + IsSeqCst, IsPostfixUpdate](RValue XRValue) -> RValue { + CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); + CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); + RValue Res = CGF.EmitAnyExpr(UE); + NewVVal = IsPostfixUpdate ? XRValue : Res; + return Res; + }; + auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( + XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen); + if (Res.first) { + // 'atomicrmw' instruction was generated. + if (IsPostfixUpdate) { + // Use old value from 'atomicrmw'. + NewVVal = Res.second; + } else { + // 'atomicrmw' does not provide new value, so evaluate it using old + // value of 'x'. + CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); + CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); + NewVVal = CGF.EmitAnyExpr(UE); + } + } + } else { + // 'x' is simply rewritten with some 'expr'. + NewVValType = X->getType().getNonReferenceType(); + ExprRValue = convertToType(CGF, ExprRValue, E->getType(), + X->getType().getNonReferenceType()); + auto &&Gen = [&CGF, &NewVVal, ExprRValue](RValue XRValue) -> RValue { + NewVVal = XRValue; + return ExprRValue; + }; + // Try to perform atomicrmw xchg, otherwise simple exchange. + auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( + XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO, + Loc, Gen); + if (Res.first) { + // 'atomicrmw' instruction was generated. + NewVVal = IsPostfixUpdate ? Res.second : ExprRValue; + } + } + // Emit post-update store to 'v' of old/new 'x' value. + emitSimpleStore(CGF, VLValue, NewVVal, NewVValType); + // OpenMP, 2.12.6, atomic Construct + // Any atomic construct with a seq_cst clause forces the atomically + // performed operation to include an implicit flush operation without a + // list. + if (IsSeqCst) + CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc); +} + +static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, + bool IsSeqCst, bool IsPostfixUpdate, + const Expr *X, const Expr *V, const Expr *E, + const Expr *UE, bool IsXLHSInRHSPart, + SourceLocation Loc) { + switch (Kind) { + case OMPC_read: + EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc); + break; + case OMPC_write: + EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc); + break; + case OMPC_unknown: + case OMPC_update: + EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc); + break; + case OMPC_capture: + EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE, + IsXLHSInRHSPart, Loc); + break; + case OMPC_if: + case OMPC_final: + case OMPC_num_threads: + case OMPC_private: + case OMPC_firstprivate: + case OMPC_lastprivate: + case OMPC_reduction: + case OMPC_safelen: + case OMPC_collapse: + case OMPC_default: + case OMPC_seq_cst: + case OMPC_shared: + case OMPC_linear: + case OMPC_aligned: + case OMPC_copyin: + case OMPC_copyprivate: + case OMPC_flush: + case OMPC_proc_bind: + case OMPC_schedule: + case OMPC_ordered: + case OMPC_nowait: + case OMPC_untied: + case OMPC_threadprivate: + case OMPC_mergeable: + llvm_unreachable("Clause is not allowed in 'omp atomic'."); + } +} + +void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) { + bool IsSeqCst = S.getSingleClause(/*K=*/OMPC_seq_cst); + OpenMPClauseKind Kind = OMPC_unknown; + for (auto *C : S.clauses()) { + // Find first clause (skip seq_cst clause, if it is first). + if (C->getClauseKind() != OMPC_seq_cst) { + Kind = C->getClauseKind(); + break; + } + } + + const auto *CS = + S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); + if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) { + enterFullExpression(EWC); + } + // Processing for statements under 'atomic capture'. + if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) { + for (const auto *C : Compound->body()) { + if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) { + enterFullExpression(EWC); + } + } + } + + LexicalScope Scope(*this, S.getSourceRange()); + auto &&CodeGen = [&S, Kind, IsSeqCst](CodeGenFunction &CGF) { + EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(), + S.getV(), S.getExpr(), S.getUpdateExpr(), + S.isXLHSInRHSPart(), S.getLocStart()); + }; + CGM.getOpenMPRuntime().emitInlinedDirective(*this, CodeGen); } void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) { @@ -702,4 +2015,3 @@ void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) { void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &) { llvm_unreachable("CodeGen for 'omp teams' is not supported yet."); } - diff --git a/lib/CodeGen/CGVTT.cpp b/lib/CodeGen/CGVTT.cpp index bd280ea14e8d..e3df5a4c05e3 100644 --- a/lib/CodeGen/CGVTT.cpp +++ b/lib/CodeGen/CGVTT.cpp @@ -18,7 +18,7 @@ using namespace clang; using namespace CodeGen; -static llvm::Constant * +static llvm::GlobalVariable * GetAddrOfVTTVTable(CodeGenVTables &CGVT, CodeGenModule &CGM, const CXXRecordDecl *MostDerivedClass, const VTTVTable &VTable, @@ -47,8 +47,8 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, llvm::Type *Int8PtrTy = CGM.Int8PtrTy, *Int64Ty = CGM.Int64Ty; llvm::ArrayType *ArrayType = llvm::ArrayType::get(Int8PtrTy, Builder.getVTTComponents().size()); - - SmallVector<llvm::Constant *, 8> VTables; + + SmallVector<llvm::GlobalVariable *, 8> VTables; SmallVector<VTableAddressPointsMapTy, 8> VTableAddressPoints; for (const VTTVTable *i = Builder.getVTTVTables().begin(), *e = Builder.getVTTVTables().end(); i != e; ++i) { @@ -61,7 +61,7 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, for (const VTTComponent *i = Builder.getVTTComponents().begin(), *e = Builder.getVTTComponents().end(); i != e; ++i) { const VTTVTable &VTTVT = Builder.getVTTVTables()[i->VTableIndex]; - llvm::Constant *VTable = VTables[i->VTableIndex]; + llvm::GlobalVariable *VTable = VTables[i->VTableIndex]; uint64_t AddressPoint; if (VTTVT.getBase() == RD) { // Just get the address point for the regular vtable. @@ -79,8 +79,8 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, llvm::ConstantInt::get(Int64Ty, AddressPoint) }; - llvm::Constant *Init = - llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Idxs); + llvm::Constant *Init = llvm::ConstantExpr::getInBoundsGetElementPtr( + VTable->getValueType(), VTable, Idxs); Init = llvm::ConstantExpr::getBitCast(Init, Int8PtrTy); @@ -94,6 +94,9 @@ CodeGenVTables::EmitVTTDefinition(llvm::GlobalVariable *VTT, // Set the correct linkage. VTT->setLinkage(Linkage); + if (CGM.supportsCOMDAT() && VTT->isWeakForLinker()) + VTT->setComdat(CGM.getModule().getOrInsertComdat(VTT->getName())); + // Set the right visibility. CGM.setGlobalVisibility(VTT, RD); } @@ -174,4 +177,3 @@ CodeGenVTables::getSecondaryVirtualPointerIndex(const CXXRecordDecl *RD, return I->second; } - diff --git a/lib/CodeGen/CGVTables.cpp b/lib/CodeGen/CGVTables.cpp index acb2a56fab30..57370a6faa2c 100644 --- a/lib/CodeGen/CGVTables.cpp +++ b/lib/CodeGen/CGVTables.cpp @@ -377,7 +377,10 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, // Set the right linkage. CGM.setFunctionLinkage(GD, Fn); - + + if (CGM.supportsCOMDAT() && Fn->isWeakForLinker()) + Fn->setComdat(CGM.getModule().getOrInsertComdat(Fn->getName())); + // Set the right visibility. const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); setThunkVisibility(CGM, MD, Thunk, Fn); @@ -666,6 +669,8 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD, VTLayout->getNumVTableThunks(), RTTI); VTable->setInitializer(Init); + CGM.EmitVTableBitSetEntries(VTable, *VTLayout.get()); + return VTable; } @@ -738,7 +743,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { return DiscardableODRLinkage; case TSK_ExplicitInstantiationDeclaration: - llvm_unreachable("Should not have been asked to emit this"); + return llvm::GlobalVariable::ExternalLinkage; case TSK_ExplicitInstantiationDefinition: return NonDiscardableODRLinkage; @@ -747,19 +752,13 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) { llvm_unreachable("Invalid TemplateSpecializationKind!"); } -/// This is a callback from Sema to tell us that it believes that a -/// particular v-table is required to be emitted in this translation -/// unit. +/// This is a callback from Sema to tell us that that a particular v-table is +/// required to be emitted in this translation unit. /// -/// The reason we don't simply trust this callback is because Sema -/// will happily report that something is used even when it's used -/// only in code that we don't actually have to emit. -/// -/// \param isRequired - if true, the v-table is mandatory, e.g. -/// because the translation unit defines the key function -void CodeGenModule::EmitVTable(CXXRecordDecl *theClass, bool isRequired) { - if (!isRequired) return; - +/// This is only called for vtables that _must_ be emitted (mainly due to key +/// functions). For weak vtables, CodeGen tracks when they are needed and +/// emits them as-needed. +void CodeGenModule::EmitVTable(CXXRecordDecl *theClass) { VTables.GenerateClassData(theClass); } @@ -840,3 +839,68 @@ void CodeGenModule::EmitDeferredVTables() { "deferred extra v-tables during v-table emission?"); DeferredVTables.clear(); } + +void CodeGenModule::EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, + const VTableLayout &VTLayout) { + if (!LangOpts.Sanitize.has(SanitizerKind::CFIVCall) && + !LangOpts.Sanitize.has(SanitizerKind::CFINVCall) && + !LangOpts.Sanitize.has(SanitizerKind::CFIDerivedCast) && + !LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast)) + return; + + llvm::Metadata *VTableMD = llvm::ConstantAsMetadata::get(VTable); + + std::vector<llvm::MDTuple *> BitsetEntries; + // Create a bit set entry for each address point. + for (auto &&AP : VTLayout.getAddressPoints()) { + // FIXME: Add blacklisting scheme. + if (AP.first.getBase()->isInStdNamespace()) + continue; + + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleCXXVTableBitSet(AP.first.getBase(), + Out); + + CharUnits PointerWidth = + Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0)); + uint64_t AddrPointOffset = AP.second * PointerWidth.getQuantity(); + + llvm::Metadata *BitsetOps[] = { + llvm::MDString::get(getLLVMContext(), Out.str()), + VTableMD, + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(Int64Ty, AddrPointOffset))}; + llvm::MDTuple *BitsetEntry = + llvm::MDTuple::get(getLLVMContext(), BitsetOps); + BitsetEntries.push_back(BitsetEntry); + } + + // Sort the bit set entries for determinism. + std::sort(BitsetEntries.begin(), BitsetEntries.end(), [](llvm::MDTuple *T1, + llvm::MDTuple *T2) { + if (T1 == T2) + return false; + + StringRef S1 = cast<llvm::MDString>(T1->getOperand(0))->getString(); + StringRef S2 = cast<llvm::MDString>(T2->getOperand(0))->getString(); + if (S1 < S2) + return true; + if (S1 != S2) + return false; + + uint64_t Offset1 = cast<llvm::ConstantInt>( + cast<llvm::ConstantAsMetadata>(T1->getOperand(2)) + ->getValue())->getZExtValue(); + uint64_t Offset2 = cast<llvm::ConstantInt>( + cast<llvm::ConstantAsMetadata>(T2->getOperand(2)) + ->getValue())->getZExtValue(); + assert(Offset1 != Offset2); + return Offset1 < Offset2; + }); + + llvm::NamedMDNode *BitsetsMD = + getModule().getOrInsertNamedMetadata("llvm.bitsets"); + for (auto BitsetEntry : BitsetEntries) + BitsetsMD->addOperand(BitsetEntry); +} diff --git a/lib/CodeGen/CGValue.h b/lib/CodeGen/CGValue.h index 82cd9496e00c..92055917dba9 100644 --- a/lib/CodeGen/CGValue.h +++ b/lib/CodeGen/CGValue.h @@ -19,6 +19,7 @@ #include "clang/AST/CharUnits.h" #include "clang/AST/Type.h" #include "llvm/IR/Value.h" +#include "llvm/IR/Type.h" namespace llvm { class Constant; @@ -299,6 +300,7 @@ public: LValue R; R.LVType = Simple; + assert(address->getType()->isPointerTy()); R.V = address; R.Initialize(type, qs, alignment, TBAAInfo); return R; diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index 10c2409f6bff..18f505d8c592 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -1,4 +1,5 @@ set(LLVM_LINK_COMPONENTS + Analysis BitReader BitWriter Core @@ -12,7 +13,6 @@ set(LLVM_LINK_COMPONENTS ProfileData ScalarOpts Support - Target TransformUtils ) diff --git a/lib/CodeGen/CodeGenAction.cpp b/lib/CodeGen/CodeGenAction.cpp index a6f6fdef335c..7e82fcc4b22a 100644 --- a/lib/CodeGen/CodeGenAction.cpp +++ b/lib/CodeGen/CodeGenAction.cpp @@ -46,7 +46,7 @@ namespace clang { const CodeGenOptions &CodeGenOpts; const TargetOptions &TargetOpts; const LangOptions &LangOpts; - raw_ostream *AsmOutStream; + raw_pwrite_stream *AsmOutStream; ASTContext *Context; Timer LLVMIRGeneration; @@ -56,18 +56,17 @@ namespace clang { std::unique_ptr<llvm::Module> TheModule, LinkModule; public: - BackendConsumer(BackendAction action, DiagnosticsEngine &_Diags, - const CodeGenOptions &compopts, - const TargetOptions &targetopts, - const LangOptions &langopts, bool TimePasses, - const std::string &infile, llvm::Module *LinkModule, - raw_ostream *OS, LLVMContext &C, + BackendConsumer(BackendAction Action, DiagnosticsEngine &Diags, + const CodeGenOptions &CodeGenOpts, + const TargetOptions &TargetOpts, + const LangOptions &LangOpts, bool TimePasses, + const std::string &InFile, llvm::Module *LinkModule, + raw_pwrite_stream *OS, LLVMContext &C, CoverageSourceInfo *CoverageInfo = nullptr) - : Diags(_Diags), Action(action), CodeGenOpts(compopts), - TargetOpts(targetopts), LangOpts(langopts), AsmOutStream(OS), + : Diags(Diags), Action(Action), CodeGenOpts(CodeGenOpts), + TargetOpts(TargetOpts), LangOpts(LangOpts), AsmOutStream(OS), Context(nullptr), LLVMIRGeneration("LLVM IR Generation Time"), - Gen(CreateLLVMCodeGen(Diags, infile, compopts, - targetopts, C, CoverageInfo)), + Gen(CreateLLVMCodeGen(Diags, InFile, CodeGenOpts, C, CoverageInfo)), LinkModule(LinkModule) { llvm::TimePassesIsEnabled = TimePasses; } @@ -80,6 +79,11 @@ namespace clang { } void Initialize(ASTContext &Ctx) override { + if (Context) { + assert(Context == &Ctx); + return; + } + Context = &Ctx; if (llvm::TimePassesIsEnabled) @@ -196,8 +200,8 @@ namespace clang { Gen->CompleteTentativeDefinition(D); } - void HandleVTable(CXXRecordDecl *RD, bool DefinitionRequired) override { - Gen->HandleVTable(RD, DefinitionRequired); + void HandleVTable(CXXRecordDecl *RD) override { + Gen->HandleVTable(RD); } void HandleLinkerOptionPragma(llvm::StringRef Opts) override { @@ -430,13 +434,16 @@ void BackendConsumer::EmitOptimizationMessage( FileManager &FileMgr = SourceMgr.getFileManager(); StringRef Filename; unsigned Line, Column; - D.getLocation(&Filename, &Line, &Column); SourceLocation DILoc; - const FileEntry *FE = FileMgr.getFile(Filename); - if (FE && Line > 0) { - // If -gcolumn-info was not used, Column will be 0. This upsets the - // source manager, so pass 1 if Column is not set. - DILoc = SourceMgr.translateFileLineCol(FE, Line, Column ? Column : 1); + + if (D.isLocationAvailable()) { + D.getLocation(&Filename, &Line, &Column); + const FileEntry *FE = FileMgr.getFile(Filename); + if (FE && Line > 0) { + // If -gcolumn-info was not used, Column will be 0. This upsets the + // source manager, so pass 1 if Column is not set. + DILoc = SourceMgr.translateFileLineCol(FE, Line, Column ? Column : 1); + } } // If a location isn't available, try to approximate it using the associated @@ -451,7 +458,7 @@ void BackendConsumer::EmitOptimizationMessage( << AddFlagValue(D.getPassName() ? D.getPassName() : "") << D.getMsg().str(); - if (DILoc.isInvalid()) + if (DILoc.isInvalid() && D.isLocationAvailable()) // If we were not able to translate the file:line:col information // back to a SourceLocation, at least emit a note stating that // we could not translate this location. This can happen in the @@ -602,9 +609,8 @@ llvm::LLVMContext *CodeGenAction::takeLLVMContext() { return VMContext; } -static raw_ostream *GetOutputStream(CompilerInstance &CI, - StringRef InFile, - BackendAction Action) { +static raw_pwrite_stream * +GetOutputStream(CompilerInstance &CI, StringRef InFile, BackendAction Action) { switch (Action) { case Backend_EmitAssembly: return CI.createDefaultOutputFile(false, InFile, "s"); @@ -626,7 +632,7 @@ static raw_ostream *GetOutputStream(CompilerInstance &CI, std::unique_ptr<ASTConsumer> CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { BackendAction BA = static_cast<BackendAction>(Act); - std::unique_ptr<raw_ostream> OS(GetOutputStream(CI, InFile, BA)); + raw_pwrite_stream *OS = GetOutputStream(CI, InFile, BA); if (BA != Backend_EmitNothing && !OS) return nullptr; @@ -663,17 +669,23 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { std::unique_ptr<BackendConsumer> Result(new BackendConsumer( BA, CI.getDiagnostics(), CI.getCodeGenOpts(), CI.getTargetOpts(), CI.getLangOpts(), CI.getFrontendOpts().ShowTimers, InFile, - LinkModuleToUse, OS.release(), *VMContext, CoverageInfo)); + LinkModuleToUse, OS, *VMContext, CoverageInfo)); BEConsumer = Result.get(); return std::move(Result); } +static void BitcodeInlineAsmDiagHandler(const llvm::SMDiagnostic &SM, + void *Context, + unsigned LocCookie) { + SM.print(nullptr, llvm::errs()); +} + void CodeGenAction::ExecuteAction() { // If this is an IR file, we have to treat it specially. if (getCurrentFileKind() == IK_LLVM_IR) { BackendAction BA = static_cast<BackendAction>(Act); CompilerInstance &CI = getCompilerInstance(); - raw_ostream *OS = GetOutputStream(CI, getCurrentFile(), BA); + raw_pwrite_stream *OS = GetOutputStream(CI, getCurrentFile(), BA); if (BA != Backend_EmitNothing && !OS) return; @@ -710,14 +722,14 @@ void CodeGenAction::ExecuteAction() { } const TargetOptions &TargetOpts = CI.getTargetOpts(); if (TheModule->getTargetTriple() != TargetOpts.Triple) { - unsigned DiagID = CI.getDiagnostics().getCustomDiagID( - DiagnosticsEngine::Warning, - "overriding the module target triple with %0"); - - CI.getDiagnostics().Report(SourceLocation(), DiagID) << TargetOpts.Triple; + CI.getDiagnostics().Report(SourceLocation(), + diag::warn_fe_override_module) + << TargetOpts.Triple; TheModule->setTargetTriple(TargetOpts.Triple); } + LLVMContext &Ctx = TheModule->getContext(); + Ctx.setInlineAsmDiagnosticHandler(BitcodeInlineAsmDiagHandler); EmitBackendOutput(CI.getDiagnostics(), CI.getCodeGenOpts(), TargetOpts, CI.getLangOpts(), CI.getTarget().getTargetDescription(), TheModule.get(), BA, OS); diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp index 826171a46e23..01da75005610 100644 --- a/lib/CodeGen/CodeGenFunction.cpp +++ b/lib/CodeGen/CodeGenFunction.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "CodeGenFunction.h" +#include "CGCleanup.h" #include "CGCUDARuntime.h" #include "CGCXXABI.h" #include "CGDebugInfo.h" @@ -40,10 +41,11 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) CurFn(nullptr), CapturedStmtInfo(nullptr), SanOpts(CGM.getLangOpts().Sanitize), IsSanitizerScope(false), CurFuncIsThunk(false), AutoreleaseResult(false), SawAsmBlock(false), - BlockInfo(nullptr), BlockPointer(nullptr), + IsOutlinedSEHHelper(false), BlockInfo(nullptr), BlockPointer(nullptr), LambdaThisCaptureField(nullptr), NormalCleanupDest(nullptr), NextCleanupDestIndex(1), FirstBlockInfo(nullptr), EHResumeBlock(nullptr), ExceptionSlot(nullptr), EHSelectorSlot(nullptr), + AbnormalTerminationSlot(nullptr), SEHPointersDecl(nullptr), DebugInfo(CGM.getModuleDebugInfo()), DisableDebugInfo(false), DidCallStackSave(false), IndirectBranch(nullptr), PGO(cgm), SwitchInsn(nullptr), SwitchWeights(nullptr), CaseRangeBlock(nullptr), @@ -69,6 +71,9 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext) if (CGM.getCodeGenOpts().NoSignedZeros) { FMF.setNoSignedZeros(); } + if (CGM.getCodeGenOpts().ReciprocalMath) { + FMF.setAllowReciprocal(); + } Builder.SetFastMathFlags(FMF); } @@ -82,7 +87,7 @@ CodeGenFunction::~CodeGenFunction() { destroyBlockInfos(FirstBlockInfo); if (getLangOpts().OpenMP) { - CGM.getOpenMPRuntime().FunctionFinished(*this); + CGM.getOpenMPRuntime().functionFinished(*this); } } @@ -239,17 +244,18 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { // parameters. Do this in whatever block we're currently in; it's // important to do this before we enter the return block or return // edges will be *really* confused. - bool EmitRetDbgLoc = true; - if (EHStack.stable_begin() != PrologueCleanupDepth) { - PopCleanupBlocks(PrologueCleanupDepth); - + bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth; + bool HasOnlyLifetimeMarkers = + HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth); + bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers; + if (HasCleanups) { // Make sure the line table doesn't jump back into the body for // the ret after it's been at EndLoc. - EmitRetDbgLoc = false; - if (CGDebugInfo *DI = getDebugInfo()) if (OnlySimpleReturnStmts) DI->EmitLocation(Builder, EndLoc); + + PopCleanupBlocks(PrologueCleanupDepth); } // Emit function epilog (to return). @@ -278,6 +284,20 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) { Builder.ClearInsertionPoint(); } + // If some of our locals escaped, insert a call to llvm.frameescape in the + // entry block. + if (!EscapedLocals.empty()) { + // Invert the map from local to index into a simple vector. There should be + // no holes. + SmallVector<llvm::Value *, 4> EscapeArgs; + EscapeArgs.resize(EscapedLocals.size()); + for (auto &Pair : EscapedLocals) + EscapeArgs[Pair.second] = Pair.first; + llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration( + &CGM.getModule(), llvm::Intrinsic::frameescape); + CGBuilderTy(AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs); + } + // Remove the AllocaInsertPt instruction, which is just a convenience for us. llvm::Instruction *Ptr = AllocaInsertPt; AllocaInsertPt = nullptr; @@ -588,6 +608,20 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, if (CGM.isInSanitizerBlacklist(Fn, Loc)) SanOpts.clear(); + if (D) { + // Apply the no_sanitize* attributes to SanOpts. + for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) + SanOpts.Mask &= ~Attr->getMask(); + } + + // Apply sanitizer attributes to the function. + if (SanOpts.has(SanitizerKind::Address)) + Fn->addFnAttr(llvm::Attribute::SanitizeAddress); + if (SanOpts.has(SanitizerKind::Thread)) + Fn->addFnAttr(llvm::Attribute::SanitizeThread); + if (SanOpts.has(SanitizerKind::Memory)) + Fn->addFnAttr(llvm::Attribute::SanitizeMemory); + // Pass inline keyword to optimizer if it appears explicitly on any // declaration. Also, in the case of -fno-inline attach NoInline // attribute to all function that are not marked AlwaysInline. @@ -679,7 +713,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex(); llvm::Function::arg_iterator EI = CurFn->arg_end(); --EI; - llvm::Value *Addr = Builder.CreateStructGEP(EI, Idx); + llvm::Value *Addr = Builder.CreateStructGEP(nullptr, EI, Idx); ReturnValue = Builder.CreateLoad(Addr, "agg.result"); } else { ReturnValue = CreateIRTemp(RetTy, "retval"); @@ -753,8 +787,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args, const Stmt *Body) { - RegionCounter Cnt = getPGORegionCounter(Body); - Cnt.beginRegion(Builder); + incrementProfileCounter(Body); if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body)) EmitCompoundStmtWithoutScope(*S); else @@ -766,7 +799,7 @@ void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args, /// emit a branch around the instrumentation code. When not instrumenting, /// this just calls EmitBlock(). void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB, - RegionCounter &Cnt) { + const Stmt *S) { llvm::BasicBlock *SkipCountBB = nullptr; if (HaveInsertPoint() && CGM.getCodeGenOpts().ProfileInstrGenerate) { // When instrumenting for profiling, the fallthrough to certain @@ -776,7 +809,9 @@ void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB, EmitBranch(SkipCountBB); } EmitBlock(BB); - Cnt.beginRegion(Builder, /*AddIncomingFallThrough=*/true); + uint64_t CurrentCount = getCurrentProfileCount(); + incrementProfileCounter(S); + setCurrentProfileCount(getCurrentProfileCount() + CurrentCount); if (SkipCountBB) EmitBlock(SkipCountBB); } @@ -801,17 +836,6 @@ static void TryMarkNoThrow(llvm::Function *F) { F->setDoesNotThrow(); } -static void EmitSizedDeallocationFunction(CodeGenFunction &CGF, - const FunctionDecl *UnsizedDealloc) { - // This is a weak discardable definition of the sized deallocation function. - CGF.CurFn->setLinkage(llvm::Function::LinkOnceAnyLinkage); - - // Call the unsized deallocation function and forward the first argument - // unchanged. - llvm::Constant *Unsized = CGF.CGM.GetAddrOfFunction(UnsizedDealloc); - CGF.Builder.CreateCall(Unsized, &*CGF.CurFn->arg_begin()); -} - void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, const CGFunctionInfo &FnInfo) { const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); @@ -832,7 +856,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, ResTy = CGM.getContext().VoidPtrTy; CGM.getCXXABI().buildThisParam(*this, Args); } - + Args.append(FD->param_begin(), FD->param_end()); if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD))) @@ -866,9 +890,9 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, else if (isa<CXXConstructorDecl>(FD)) EmitConstructorBody(Args); else if (getLangOpts().CUDA && - !CGM.getCodeGenOpts().CUDAIsDevice && + !getLangOpts().CUDAIsDevice && FD->hasAttr<CUDAGlobalAttr>()) - CGM.getCUDARuntime().EmitDeviceStubBody(*this, Args); + CGM.getCUDARuntime().emitDeviceStub(*this, Args); else if (isa<CXXConversionDecl>(FD) && cast<CXXConversionDecl>(FD)->isLambdaToBlockPointerConversion()) { // The lambda conversion to block pointer is special; the semantics can't be @@ -887,11 +911,6 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, emitImplicitAssignmentOperatorBody(Args); } else if (Stmt *Body = FD->getBody()) { EmitFunctionBody(Args, Body); - } else if (FunctionDecl *UnsizedDealloc = - FD->getCorrespondingUnsizedGlobalDeallocationFunction()) { - // Global sized deallocation functions get an implicit weak definition if - // they don't have an explicit definition. - EmitSizedDeallocationFunction(*this, UnsizedDealloc); } else llvm_unreachable("no definition for emitted function"); @@ -910,7 +929,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn, "missing_return", EmitCheckSourceLocation(FD->getLocation()), None); } else if (CGM.getCodeGenOpts().OptimizationLevel == 0) - Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap)); + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap), {}); Builder.CreateUnreachable(); Builder.ClearInsertionPoint(); } @@ -1028,15 +1047,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // Handle X && Y in a condition. if (CondBOp->getOpcode() == BO_LAnd) { - RegionCounter Cnt = getPGORegionCounter(CondBOp); - // If we have "1 && X", simplify the code. "0 && X" would have constant // folded if the case was simple enough. bool ConstantBool = false; if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) && ConstantBool) { // br(1 && X) -> br(X). - Cnt.beginRegion(Builder); + incrementProfileCounter(CondBOp); return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount); } @@ -1055,14 +1072,19 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true"); // The counter tells us how often we evaluate RHS, and all of TrueCount // can be propagated to that branch. - uint64_t RHSCount = Cnt.getCount(); + uint64_t RHSCount = getProfileCount(CondBOp->getRHS()); ConditionalEvaluation eval(*this); - EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount); - EmitBlock(LHSTrue); + { + ApplyDebugLocation DL(*this, Cond); + EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount); + EmitBlock(LHSTrue); + } + + incrementProfileCounter(CondBOp); + setCurrentProfileCount(getProfileCount(CondBOp->getRHS())); // Any temporaries created here are conditional. - Cnt.beginRegion(Builder); eval.begin(*this); EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount); eval.end(*this); @@ -1071,15 +1093,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, } if (CondBOp->getOpcode() == BO_LOr) { - RegionCounter Cnt = getPGORegionCounter(CondBOp); - // If we have "0 || X", simplify the code. "1 || X" would have constant // folded if the case was simple enough. bool ConstantBool = false; if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) && !ConstantBool) { // br(0 || X) -> br(X). - Cnt.beginRegion(Builder); + incrementProfileCounter(CondBOp); return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount); } @@ -1099,15 +1119,21 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // We have the count for entry to the RHS and for the whole expression // being true, so we can divy up True count between the short circuit and // the RHS. - uint64_t LHSCount = Cnt.getParentCount() - Cnt.getCount(); + uint64_t LHSCount = + getCurrentProfileCount() - getProfileCount(CondBOp->getRHS()); uint64_t RHSCount = TrueCount - LHSCount; ConditionalEvaluation eval(*this); - EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount); - EmitBlock(LHSFalse); + { + ApplyDebugLocation DL(*this, Cond); + EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount); + EmitBlock(LHSFalse); + } + + incrementProfileCounter(CondBOp); + setCurrentProfileCount(getProfileCount(CondBOp->getRHS())); // Any temporaries created here are conditional. - Cnt.beginRegion(Builder); eval.begin(*this); EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, RHSCount); @@ -1121,7 +1147,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // br(!x, t, f) -> br(x, f, t) if (CondUOp->getOpcode() == UO_LNot) { // Negate the count. - uint64_t FalseCount = PGO.getCurrentRegionCount() - TrueCount; + uint64_t FalseCount = getCurrentProfileCount() - TrueCount; // Negate the condition and swap the destination blocks. return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock, FalseCount); @@ -1133,9 +1159,9 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true"); llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false"); - RegionCounter Cnt = getPGORegionCounter(CondOp); ConditionalEvaluation cond(*this); - EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock, Cnt.getCount()); + EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock, + getProfileCount(CondOp)); // When computing PGO branch weights, we only know the overall count for // the true block. This code is essentially doing tail duplication of the @@ -1144,15 +1170,19 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // the conditional operator. uint64_t LHSScaledTrueCount = 0; if (TrueCount) { - double LHSRatio = Cnt.getCount() / (double) Cnt.getParentCount(); + double LHSRatio = + getProfileCount(CondOp) / (double)getCurrentProfileCount(); LHSScaledTrueCount = TrueCount * LHSRatio; } cond.begin(*this); EmitBlock(LHSBlock); - Cnt.beginRegion(Builder); - EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock, - LHSScaledTrueCount); + incrementProfileCounter(CondOp); + { + ApplyDebugLocation DL(*this, Cond); + EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock, + LHSScaledTrueCount); + } cond.end(*this); cond.begin(*this); @@ -1176,12 +1206,16 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, // Create branch weights based on the number of times we get here and the // number of times the condition should be true. - uint64_t CurrentCount = std::max(PGO.getCurrentRegionCount(), TrueCount); - llvm::MDNode *Weights = PGO.createBranchWeights(TrueCount, - CurrentCount - TrueCount); + uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount); + llvm::MDNode *Weights = + createProfileWeights(TrueCount, CurrentCount - TrueCount); // Emit the code with the fully general case. - llvm::Value *CondV = EvaluateExprAsBool(Cond); + llvm::Value *CondV; + { + ApplyDebugLocation DL(*this, Cond); + CondV = EvaluateExprAsBool(Cond); + } Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights); } @@ -1231,7 +1265,8 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, /*volatile*/ false); // Go to the next element. - llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(cur, 1, "vla.next"); + llvm::Value *next = Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), + cur, 1, "vla.next"); // Leave if that's the end of the VLA. llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone"); diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h index 3a990d21490d..650ad7b04009 100644 --- a/lib/CodeGen/CodeGenFunction.h +++ b/lib/CodeGen/CodeGenFunction.h @@ -27,6 +27,7 @@ #include "clang/AST/Type.h" #include "clang/Basic/ABI.h" #include "clang/Basic/CapturedStmt.h" +#include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CodeGenOptions.h" #include "llvm/ADT/ArrayRef.h" @@ -96,8 +97,8 @@ enum TypeEvaluationKind { /// CodeGenFunction - This class organizes the per-function state that is used /// while generating LLVM code. class CodeGenFunction : public CodeGenTypeCache { - CodeGenFunction(const CodeGenFunction &) LLVM_DELETED_FUNCTION; - void operator=(const CodeGenFunction &) LLVM_DELETED_FUNCTION; + CodeGenFunction(const CodeGenFunction &) = delete; + void operator=(const CodeGenFunction &) = delete; friend class CGCXXABI; public: @@ -191,26 +192,25 @@ public: CapturedRegionKind getKind() const { return Kind; } - void setContextValue(llvm::Value *V) { ThisValue = V; } + virtual void setContextValue(llvm::Value *V) { ThisValue = V; } // \brief Retrieve the value of the context parameter. - llvm::Value *getContextValue() const { return ThisValue; } + virtual llvm::Value *getContextValue() const { return ThisValue; } /// \brief Lookup the captured field decl for a variable. - const FieldDecl *lookup(const VarDecl *VD) const { + virtual const FieldDecl *lookup(const VarDecl *VD) const { return CaptureFields.lookup(VD); } - bool isCXXThisExprCaptured() const { return CXXThisFieldDecl != nullptr; } - FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; } + bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; } + virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; } static bool classof(const CGCapturedStmtInfo *) { return true; } /// \brief Emit the captured statement body. - virtual void EmitBody(CodeGenFunction &CGF, Stmt *S) { - RegionCounter Cnt = CGF.getPGORegionCounter(S); - Cnt.beginRegion(CGF.Builder); + virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) { + CGF.incrementProfileCounter(S); CGF.EmitStmt(S); } @@ -262,6 +262,10 @@ public: /// potentially set the return value. bool SawAsmBlock; + /// True if the current function is an outlined SEH helper. This can be a + /// finally block or filter expression. + bool IsOutlinedSEHHelper; + const CodeGen::CGBlockInfo *BlockInfo; llvm::Value *BlockPointer; @@ -274,6 +278,7 @@ public: EHScopeStack EHStack; llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack; + llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack; /// Header for data within LifetimeExtendedCleanupStack. struct LifetimeExtendedCleanupHeader { @@ -305,6 +310,12 @@ public: /// write the current selector value into this alloca. llvm::AllocaInst *EHSelectorSlot; + llvm::AllocaInst *AbnormalTerminationSlot; + + /// The implicit parameter to SEH filter functions of type + /// 'EXCEPTION_POINTERS*'. + ImplicitParamDecl *SEHPointersDecl; + /// Emits a landing pad for the current EH stack. llvm::BasicBlock *EmitLandingPad(); @@ -343,87 +354,32 @@ public: void exit(CodeGenFunction &CGF); }; - /// pushFullExprCleanup - Push a cleanup to be run at the end of the - /// current full-expression. Safe against the possibility that - /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0> - void pushFullExprCleanup(CleanupKind kind, A0 a0) { - // If we're not in a conditional branch, or if none of the - // arguments requires saving, then use the unconditional cleanup. - if (!isInConditionalBranch()) - return EHStack.pushCleanup<T>(kind, a0); - - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - - typedef EHScopeStack::ConditionalCleanup1<T, A0> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved); - initFullExprCleanup(); - } + /// Returns true inside SEH __try blocks. + bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); } /// pushFullExprCleanup - Push a cleanup to be run at the end of the /// current full-expression. Safe against the possibility that /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0, class A1> - void pushFullExprCleanup(CleanupKind kind, A0 a0, A1 a1) { + template <class T, class... As> + void pushFullExprCleanup(CleanupKind kind, As... A) { // If we're not in a conditional branch, or if none of the // arguments requires saving, then use the unconditional cleanup. if (!isInConditionalBranch()) - return EHStack.pushCleanup<T>(kind, a0, a1); + return EHStack.pushCleanup<T>(kind, A...); - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - typename DominatingValue<A1>::saved_type a1_saved = saveValueInCond(a1); + // Stash values in a tuple so we can guarantee the order of saves. + typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple; + SavedTuple Saved{saveValueInCond(A)...}; - typedef EHScopeStack::ConditionalCleanup2<T, A0, A1> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved, a1_saved); - initFullExprCleanup(); - } - - /// pushFullExprCleanup - Push a cleanup to be run at the end of the - /// current full-expression. Safe against the possibility that - /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0, class A1, class A2> - void pushFullExprCleanup(CleanupKind kind, A0 a0, A1 a1, A2 a2) { - // If we're not in a conditional branch, or if none of the - // arguments requires saving, then use the unconditional cleanup. - if (!isInConditionalBranch()) { - return EHStack.pushCleanup<T>(kind, a0, a1, a2); - } - - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - typename DominatingValue<A1>::saved_type a1_saved = saveValueInCond(a1); - typename DominatingValue<A2>::saved_type a2_saved = saveValueInCond(a2); - - typedef EHScopeStack::ConditionalCleanup3<T, A0, A1, A2> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved, a1_saved, a2_saved); - initFullExprCleanup(); - } - - /// pushFullExprCleanup - Push a cleanup to be run at the end of the - /// current full-expression. Safe against the possibility that - /// we're currently inside a conditionally-evaluated expression. - template <class T, class A0, class A1, class A2, class A3> - void pushFullExprCleanup(CleanupKind kind, A0 a0, A1 a1, A2 a2, A3 a3) { - // If we're not in a conditional branch, or if none of the - // arguments requires saving, then use the unconditional cleanup. - if (!isInConditionalBranch()) { - return EHStack.pushCleanup<T>(kind, a0, a1, a2, a3); - } - - typename DominatingValue<A0>::saved_type a0_saved = saveValueInCond(a0); - typename DominatingValue<A1>::saved_type a1_saved = saveValueInCond(a1); - typename DominatingValue<A2>::saved_type a2_saved = saveValueInCond(a2); - typename DominatingValue<A3>::saved_type a3_saved = saveValueInCond(a3); - - typedef EHScopeStack::ConditionalCleanup4<T, A0, A1, A2, A3> CleanupType; - EHStack.pushCleanup<CleanupType>(kind, a0_saved, a1_saved, - a2_saved, a3_saved); + typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType; + EHStack.pushCleanupTuple<CleanupType>(kind, Saved); initFullExprCleanup(); } /// \brief Queue a cleanup to be pushed after finishing the current /// full-expression. - template <class T, class A0, class A1, class A2, class A3> - void pushCleanupAfterFullExpr(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) { + template <class T, class... As> + void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) { assert(!isInConditionalBranch() && "can't defer conditional cleanup"); LifetimeExtendedCleanupHeader Header = { sizeof(T), Kind }; @@ -434,7 +390,7 @@ public: char *Buffer = &LifetimeExtendedCleanupStack[OldSize]; new (Buffer) LifetimeExtendedCleanupHeader(Header); - new (Buffer + sizeof(Header)) T(a0, a1, a2, a3); + new (Buffer + sizeof(Header)) T(A...); } /// Set up the last cleaup that was pushed as a conditional @@ -488,8 +444,8 @@ public: bool PerformCleanup; private: - RunCleanupsScope(const RunCleanupsScope &) LLVM_DELETED_FUNCTION; - void operator=(const RunCleanupsScope &) LLVM_DELETED_FUNCTION; + RunCleanupsScope(const RunCleanupsScope &) = delete; + void operator=(const RunCleanupsScope &) = delete; protected: CodeGenFunction& CGF; @@ -537,8 +493,8 @@ public: SmallVector<const LabelDecl*, 4> Labels; LexicalScope *ParentScope; - LexicalScope(const LexicalScope &) LLVM_DELETED_FUNCTION; - void operator=(const LexicalScope &) LLVM_DELETED_FUNCTION; + LexicalScope(const LexicalScope &) = delete; + void operator=(const LexicalScope &) = delete; public: /// \brief Enter a new cleanup scope. @@ -562,7 +518,10 @@ public: // If we should perform a cleanup, force them now. Note that // this ends the cleanup scope before rescoping any labels. - if (PerformCleanup) ForceCleanup(); + if (PerformCleanup) { + ApplyDebugLocation DL(CGF, Range.getEnd()); + ForceCleanup(); + } } /// \brief Force the emission of cleanups now, instead of waiting @@ -587,8 +546,8 @@ public: VarDeclMapTy SavedPrivates; private: - OMPPrivateScope(const OMPPrivateScope &) LLVM_DELETED_FUNCTION; - void operator=(const OMPPrivateScope &) LLVM_DELETED_FUNCTION; + OMPPrivateScope(const OMPPrivateScope &) = delete; + void operator=(const OMPPrivateScope &) = delete; public: /// \brief Enter a new OpenMP private scope. @@ -637,7 +596,10 @@ public: } /// \brief Exit scope - all the mapped variables are restored. - ~OMPPrivateScope() { ForceCleanup(); } + ~OMPPrivateScope() { + if (PerformCleanup) + ForceCleanup(); + } }; /// \brief Takes the old cleanup stack size and emits the cleanup blocks @@ -881,7 +843,8 @@ public: /// getByrefValueFieldNumber - Given a declaration, returns the LLVM field /// number that holds the value. - unsigned getByRefValueLLVMField(const ValueDecl *VD) const; + std::pair<llvm::Type *, unsigned> + getByRefValueLLVMField(const ValueDecl *VD) const; /// BuildBlockByrefAddress - Computes address location of the /// variable which is declared as __block. @@ -906,6 +869,10 @@ private: typedef llvm::DenseMap<const Decl*, llvm::Value*> DeclMapTy; DeclMapTy LocalDeclMap; + /// Track escaped local variables with auto storage. Used during SEH + /// outlining to produce a call to llvm.frameescape. + llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals; + /// LabelMap - This keeps track of the LLVM basic block for each C label. llvm::DenseMap<const LabelDecl*, JumpDest> LabelMap; @@ -922,12 +889,39 @@ private: CodeGenPGO PGO; + /// Calculate branch weights appropriate for PGO data + llvm::MDNode *createProfileWeights(uint64_t TrueCount, uint64_t FalseCount); + llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights); + llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond, + uint64_t LoopCount); + public: - /// Get a counter for instrumentation of the region associated with the given - /// statement. - RegionCounter getPGORegionCounter(const Stmt *S) { - return RegionCounter(PGO, S); + /// Increment the profiler's counter for the given statement. + void incrementProfileCounter(const Stmt *S) { + if (CGM.getCodeGenOpts().ProfileInstrGenerate) + PGO.emitCounterIncrement(Builder, S); + PGO.setCurrentStmt(S); + } + + /// Get the profiler's count for the given statement. + uint64_t getProfileCount(const Stmt *S) { + Optional<uint64_t> Count = PGO.getStmtCount(S); + if (!Count.hasValue()) + return 0; + return *Count; + } + + /// Set the profiler's current count. + void setCurrentProfileCount(uint64_t Count) { + PGO.setCurrentRegionCount(Count); + } + + /// Get the profiler's current count. This is generally the count for the most + /// recently incremented counter. + uint64_t getCurrentProfileCount() { + return PGO.getCurrentRegionCount(); } + private: /// SwitchInsn - This is nearest current switch instruction. It is null if @@ -1097,6 +1091,11 @@ public: return getInvokeDestImpl(); } + bool currentFunctionUsesSEHTry() const { + const auto *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl); + return FD && FD->usesSEHTry(); + } + const TargetInfo &getTarget() const { return Target; } llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); } @@ -1248,7 +1247,7 @@ public: void EmitDestructorBody(FunctionArgList &Args); void emitImplicitAssignmentOperatorBody(FunctionArgList &Args); void EmitFunctionBody(FunctionArgList &Args, const Stmt *Body); - void EmitBlockWithFallThrough(llvm::BasicBlock *BB, RegionCounter &Cnt); + void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S); void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator, CallArgList &CallArgs); @@ -1258,15 +1257,18 @@ public: void EmitLambdaStaticInvokeFunction(const CXXMethodDecl *MD); void EmitAsanPrologueOrEpilogue(bool Prologue); - /// EmitReturnBlock - Emit the unified return block, trying to avoid its - /// emission when possible. + /// \brief Emit the unified return block, trying to avoid its emission when + /// possible. + /// \return The debug location of the user written return statement if the + /// return block is is avoided. llvm::DebugLoc EmitReturnBlock(); /// FinishFunction - Complete IR generation of the current function. It is /// legal to call this function even if there is no current insertion point. void FinishFunction(SourceLocation EndLoc=SourceLocation()); - void StartThunk(llvm::Function *Fn, GlobalDecl GD, const CGFunctionInfo &FnInfo); + void StartThunk(llvm::Function *Fn, GlobalDecl GD, + const CGFunctionInfo &FnInfo); void EmitCallAndReturnForThunk(llvm::Value *Callee, const ThunkInfo *Thunk); @@ -1309,6 +1311,19 @@ public: /// to by This. llvm::Value *GetVTablePtr(llvm::Value *This, llvm::Type *Ty); + /// \brief Derived is the presumed address of an object of type T after a + /// cast. If T is a polymorphic class type, emit a check that the virtual + /// table for Derived belongs to a class derived from T. + void EmitVTablePtrCheckForCast(QualType T, llvm::Value *Derived, + bool MayBeNull); + + /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable. + /// If vptr CFI is enabled, emit a check that VTable is valid. + void EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, llvm::Value *VTable); + + /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for + /// RD using llvm.bitset.test. + void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable); /// CanDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given /// expr can be devirtualized. @@ -1527,6 +1542,8 @@ public: void EmitAnyExprToMem(const Expr *E, llvm::Value *Location, Qualifiers Quals, bool IsInitializer); + void EmitAnyExprToExn(const Expr *E, llvm::Value *Addr); + /// EmitExprAsInit - Emits the code necessary to initialize a /// location in memory with the given initializer. void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, @@ -1552,6 +1569,15 @@ public: true); } + void EmitAggregateCopyCtor(llvm::Value *DestPtr, llvm::Value *SrcPtr, + QualType DestTy, QualType SrcTy) { + CharUnits DestTypeAlign = getContext().getTypeAlignInChars(DestTy); + CharUnits SrcTypeAlign = getContext().getTypeAlignInChars(SrcTy); + EmitAggregateCopy(DestPtr, SrcPtr, SrcTy, /*IsVolatile=*/false, + std::min(DestTypeAlign, SrcTypeAlign), + /*IsAssignment=*/false); + } + /// EmitAggregateCopy - Emit an aggregate copy. /// /// \param isVolatile - True iff either the source or the destination is @@ -1723,12 +1749,16 @@ public: llvm::Value *This); void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType, - llvm::Value *NewPtr, llvm::Value *NumElements, + llvm::Type *ElementTy, llvm::Value *NewPtr, + llvm::Value *NumElements, llvm::Value *AllocSizeWithoutCookie); void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType, llvm::Value *Ptr); + llvm::Value *EmitLifetimeStart(uint64_t Size, llvm::Value *Addr); + void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr); + llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E); void EmitCXXDeleteExpr(const CXXDeleteExpr *E); @@ -1889,8 +1919,8 @@ public: llvm::Value *getObjectAddress(CodeGenFunction &CGF) const { if (!IsByRef) return Address; - return CGF.Builder.CreateStructGEP(Address, - CGF.getByRefValueLLVMField(Variable), + auto F = CGF.getByRefValueLLVMField(Variable); + return CGF.Builder.CreateStructGEP(F.first, Address, F.second, Variable->getNameAsString()); } }; @@ -1988,6 +2018,30 @@ public: void EmitCXXTryStmt(const CXXTryStmt &S); void EmitSEHTryStmt(const SEHTryStmt &S); void EmitSEHLeaveStmt(const SEHLeaveStmt &S); + void EnterSEHTryStmt(const SEHTryStmt &S); + void ExitSEHTryStmt(const SEHTryStmt &S); + + void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, StringRef Name, + QualType RetTy, FunctionArgList &Args, + const Stmt *OutlinedStmt); + + llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF, + const SEHExceptStmt &Except); + + llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF, + const SEHFinallyStmt &Finally); + + void EmitSEHExceptionCodeSave(); + llvm::Value *EmitSEHExceptionCode(); + llvm::Value *EmitSEHExceptionInfo(); + llvm::Value *EmitSEHAbnormalTermination(); + + /// Scan the outlined statement for captures from the parent function. For + /// each capture, mark the capture as escaped and emit a call to + /// llvm.framerecover. Insert the framerecover result into the LocalDeclMap. + void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt, + llvm::Value *ParentFP); + void EmitCXXForRangeStmt(const CXXForRangeStmt &S, ArrayRef<const Attr *> Attrs = None); @@ -1997,13 +2051,102 @@ public: llvm::Function *GenerateCapturedStmtFunctionEpilog(const CapturedStmt &S); llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S); llvm::Value *GenerateCapturedStmtArgument(const CapturedStmt &S); - void EmitOMPAggregateAssign(LValue OriginalAddr, llvm::Value *PrivateAddr, - const Expr *AssignExpr, QualType Type, - const VarDecl *VDInit); - void EmitOMPFirstprivateClause(const OMPExecutableDirective &D, + /// \brief Perform element by element copying of arrays with type \a + /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure + /// generated by \a CopyGen. + /// + /// \param DestAddr Address of the destination array. + /// \param SrcAddr Address of the source array. + /// \param OriginalType Type of destination and source arrays. + /// \param CopyGen Copying procedure that copies value of single array element + /// to another single array element. + void EmitOMPAggregateAssign( + llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, + const llvm::function_ref<void(llvm::Value *, llvm::Value *)> &CopyGen); + /// \brief Emit proper copying of data from one variable to another. + /// + /// \param OriginalType Original type of the copied variables. + /// \param DestAddr Destination address. + /// \param SrcAddr Source address. + /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has + /// type of the base array element). + /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of + /// the base array element). + /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a + /// DestVD. + void EmitOMPCopy(CodeGenFunction &CGF, QualType OriginalType, + llvm::Value *DestAddr, llvm::Value *SrcAddr, + const VarDecl *DestVD, const VarDecl *SrcVD, + const Expr *Copy); + /// \brief Emit atomic update code for constructs: \a X = \a X \a BO \a E or + /// \a X = \a E \a BO \a E. + /// + /// \param X Value to be updated. + /// \param E Update value. + /// \param BO Binary operation for update operation. + /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update + /// expression, false otherwise. + /// \param AO Atomic ordering of the generated atomic instructions. + /// \param CommonGen Code generator for complex expressions that cannot be + /// expressed through atomicrmw instruction. + /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was + /// generated, <false, RValue::get(nullptr)> otherwise. + std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr( + LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, + llvm::AtomicOrdering AO, SourceLocation Loc, + const llvm::function_ref<RValue(RValue)> &CommonGen); + bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope); void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope); + /// \brief Emit code for copyin clause in \a D directive. The next code is + /// generated at the start of outlined functions for directives: + /// \code + /// threadprivate_var1 = master_threadprivate_var1; + /// operator=(threadprivate_var2, master_threadprivate_var2); + /// ... + /// __kmpc_barrier(&loc, global_tid); + /// \endcode + /// + /// \param D OpenMP directive possibly with 'copyin' clause(s). + /// \returns true if at least one copyin variable is found, false otherwise. + bool EmitOMPCopyinClause(const OMPExecutableDirective &D); + /// \brief Emit initial code for lastprivate variables. If some variable is + /// not also firstprivate, then the default initialization is used. Otherwise + /// initialization of this variable is performed by EmitOMPFirstprivateClause + /// method. + /// + /// \param D Directive that may have 'lastprivate' directives. + /// \param PrivateScope Private scope for capturing lastprivate variables for + /// proper codegen in internal captured statement. + /// + /// \returns true if there is at least one lastprivate variable, false + /// otherwise. + bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, + OMPPrivateScope &PrivateScope); + /// \brief Emit final copying of lastprivate values to original variables at + /// the end of the worksharing or simd directive. + /// + /// \param D Directive that has at least one 'lastprivate' directives. + /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if + /// it is the last iteration of the loop code in associated directive, or to + /// 'i1 false' otherwise. + void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, + llvm::Value *IsLastIterCond); + /// \brief Emit initial code for reduction variables. Creates reduction copies + /// and initializes them with the values according to OpenMP standard. + /// + /// \param D Directive (possibly) with the 'reduction' clause. + /// \param PrivateScope Private scope for capturing reduction variables for + /// proper codegen in internal captured statement. + /// + void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, + OMPPrivateScope &PrivateScope); + /// \brief Emit final update of reduction values to original variables at + /// the end of the directive. + /// + /// \param D Directive that has at least one 'reduction' directives. + void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D); void EmitOMPParallelDirective(const OMPParallelDirective &S); void EmitOMPSimdDirective(const OMPSimdDirective &S); @@ -2027,15 +2170,37 @@ public: void EmitOMPTargetDirective(const OMPTargetDirective &S); void EmitOMPTeamsDirective(const OMPTeamsDirective &S); + /// \brief Emit inner loop of the worksharing/simd construct. + /// + /// \param S Directive, for which the inner loop must be emitted. + /// \param RequiresCleanup true, if directive has some associated private + /// variables. + /// \param LoopCond Bollean condition for loop continuation. + /// \param IncExpr Increment expression for loop control variable. + /// \param BodyGen Generator for the inner body of the inner loop. + /// \param PostIncGen Genrator for post-increment code (required for ordered + /// loop directvies). + void EmitOMPInnerLoop( + const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, + const Expr *IncExpr, + const llvm::function_ref<void(CodeGenFunction &)> &BodyGen, + const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen); + private: /// Helpers for the OpenMP loop directives. void EmitOMPLoopBody(const OMPLoopDirective &Directive, bool SeparateIter = false); - void EmitOMPInnerLoop(const OMPLoopDirective &S, OMPPrivateScope &LoopScope, - bool SeparateIter = false); void EmitOMPSimdFinal(const OMPLoopDirective &S); - void EmitOMPWorksharingLoop(const OMPLoopDirective &S); + /// \brief Emit code for the worksharing loop-based directive. + /// \return true, if this construct has any lastprivate clause, false - + /// otherwise. + bool EmitOMPWorksharingLoop(const OMPLoopDirective &S); + void EmitOMPForOuterLoop(OpenMPScheduleClauseKind ScheduleKind, + const OMPLoopDirective &S, + OMPPrivateScope &LoopScope, bool Ordered, + llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, + llvm::Value *IL, llvm::Value *Chunk); public: @@ -2085,17 +2250,31 @@ public: void EmitAtomicInit(Expr *E, LValue lvalue); + bool LValueIsSuitableForInlineAtomic(LValue Src); + bool typeIsSuitableForInlineAtomic(QualType Ty, bool IsVolatile) const; + + RValue EmitAtomicLoad(LValue LV, SourceLocation SL, + AggValueSlot Slot = AggValueSlot::ignored()); + RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc, + llvm::AtomicOrdering AO, bool IsVolatile = false, AggValueSlot slot = AggValueSlot::ignored()); void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit); - std::pair<RValue, RValue> EmitAtomicCompareExchange( + void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO, + bool IsVolatile, bool isInit); + + std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange( LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success = llvm::SequentiallyConsistent, llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent, bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored()); + void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, + const llvm::function_ref<RValue(RValue)> &UpdateOp, + bool IsVolatile); + /// EmitToMemory - Change a scalar value from its value /// representation to its in-memory representation. llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty); @@ -2163,8 +2342,8 @@ public: /// Emit an l-value for an assignment (simple or compound) of complex type. LValue EmitComplexAssignmentLValue(const BinaryOperator *E); LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E); - LValue EmitScalarCompooundAssignWithComplex(const CompoundAssignOperator *E, - llvm::Value *&Result); + LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E, + llvm::Value *&Result); // Note: only available for agg return types LValue EmitBinaryOperatorLValue(const BinaryOperator *E); @@ -2208,7 +2387,7 @@ public: return ConstantEmission(C, false); } - LLVM_EXPLICIT operator bool() const { + explicit operator bool() const { return ValueAndIsReference.getOpaqueValue() != nullptr; } @@ -2397,6 +2576,7 @@ public: llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitR600BuiltinExpr(unsigned BuiltinID, const CallExpr *E); + llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E); llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E); @@ -2655,7 +2835,7 @@ public: /// \brief Create a basic block that will call a handler function in a /// sanitizer runtime with the provided arguments, and create a conditional /// branch to it. - void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind>> Checked, + void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked, StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs, ArrayRef<llvm::Value *> DynamicArgs); @@ -2714,7 +2894,7 @@ public: CallExpr::const_arg_iterator ArgBeg, CallExpr::const_arg_iterator ArgEnd, const FunctionDecl *CalleeDecl = nullptr, - unsigned ParamsToSkip = 0, bool ForceColumnInfo = false) { + unsigned ParamsToSkip = 0) { SmallVector<QualType, 16> ArgTypes; CallExpr::const_arg_iterator Arg = ArgBeg; @@ -2747,15 +2927,14 @@ public: for (; Arg != ArgEnd; ++Arg) ArgTypes.push_back(getVarArgType(*Arg)); - EmitCallArgs(Args, ArgTypes, ArgBeg, ArgEnd, CalleeDecl, ParamsToSkip, - ForceColumnInfo); + EmitCallArgs(Args, ArgTypes, ArgBeg, ArgEnd, CalleeDecl, ParamsToSkip); } void EmitCallArgs(CallArgList &Args, ArrayRef<QualType> ArgTypes, CallExpr::const_arg_iterator ArgBeg, CallExpr::const_arg_iterator ArgEnd, const FunctionDecl *CalleeDecl = nullptr, - unsigned ParamsToSkip = 0, bool ForceColumnInfo = false); + unsigned ParamsToSkip = 0); private: QualType getVarArgType(const Expr *Arg); diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp index d653130df3d0..400506129e64 100644 --- a/lib/CodeGen/CodeGenModule.cpp +++ b/lib/CodeGen/CodeGenModule.cpp @@ -141,12 +141,14 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO, RRData = new RREntrypoints(); if (!CodeGenOpts.InstrProfileInput.empty()) { - if (std::error_code EC = llvm::IndexedInstrProfReader::create( - CodeGenOpts.InstrProfileInput, PGOReader)) { + auto ReaderOrErr = + llvm::IndexedInstrProfReader::create(CodeGenOpts.InstrProfileInput); + if (std::error_code EC = ReaderOrErr.getError()) { unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error, "Could not read profile: %0"); getDiags().Report(DiagID) << EC.message(); - } + } else + PGOReader = std::move(ReaderOrErr.get()); } // If coverage mapping generation is enabled, create the @@ -321,6 +323,8 @@ void CodeGenModule::checkAliases() { void CodeGenModule::clear() { DeferredDeclsToEmit.clear(); + if (OpenMPRuntime) + OpenMPRuntime->clear(); } void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags, @@ -346,6 +350,13 @@ void CodeGenModule::Release() { if (ObjCRuntime) if (llvm::Function *ObjCInitFunction = ObjCRuntime->ModuleInitFunction()) AddGlobalCtor(ObjCInitFunction); + if (Context.getLangOpts().CUDA && !Context.getLangOpts().CUDAIsDevice && + CUDARuntime) { + if (llvm::Function *CudaCtorFunction = CUDARuntime->makeModuleCtorFunction()) + AddGlobalCtor(CudaCtorFunction); + if (llvm::Function *CudaDtorFunction = CUDARuntime->makeModuleDtorFunction()) + AddGlobalDtor(CudaDtorFunction); + } if (PGOReader && PGOStats.hasDiagnostics()) PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName); EmitCtorList(GlobalCtors, "llvm.global_ctors"); @@ -741,23 +752,6 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D, else if (LangOpts.getStackProtector() == LangOptions::SSPReq) B.addAttribute(llvm::Attribute::StackProtectReq); - // Add sanitizer attributes if function is not blacklisted. - if (!isInSanitizerBlacklist(F, D->getLocation())) { - // When AddressSanitizer is enabled, set SanitizeAddress attribute - // unless __attribute__((no_sanitize_address)) is used. - if (LangOpts.Sanitize.has(SanitizerKind::Address) && - !D->hasAttr<NoSanitizeAddressAttr>()) - B.addAttribute(llvm::Attribute::SanitizeAddress); - // Same for ThreadSanitizer and __attribute__((no_sanitize_thread)) - if (LangOpts.Sanitize.has(SanitizerKind::Thread) && - !D->hasAttr<NoSanitizeThreadAttr>()) - B.addAttribute(llvm::Attribute::SanitizeThread); - // Same for MemorySanitizer and __attribute__((no_sanitize_memory)) - if (LangOpts.Sanitize.has(SanitizerKind::Memory) && - !D->hasAttr<NoSanitizeMemoryAttr>()) - B.addAttribute(llvm::Attribute::SanitizeMemory); - } - F->addAttributes(llvm::AttributeSet::FunctionIndex, llvm::AttributeSet::get( F->getContext(), llvm::AttributeSet::FunctionIndex, B)); @@ -865,11 +859,10 @@ static void setLinkageAndVisibilityForGV(llvm::GlobalValue *GV, void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, bool IsIncompleteFunction, bool IsThunk) { - if (unsigned IID = F->getIntrinsicID()) { + if (llvm::Intrinsic::ID IID = F->getIntrinsicID()) { // If this is an intrinsic function, set the function's attributes // to the intrinsic's attributes. - F->setAttributes(llvm::Intrinsic::getAttributes(getLLVMContext(), - (llvm::Intrinsic::ID)IID)); + F->setAttributes(llvm::Intrinsic::getAttributes(getLLVMContext(), IID)); return; } @@ -1085,29 +1078,33 @@ void CodeGenModule::EmitDeferred() { // previously unused static decl may become used during the generation of code // for a static function, iterate until no changes are made. - while (true) { - if (!DeferredVTables.empty()) { - EmitDeferredVTables(); + if (!DeferredVTables.empty()) { + EmitDeferredVTables(); - // Emitting a v-table doesn't directly cause more v-tables to - // become deferred, although it can cause functions to be - // emitted that then need those v-tables. - assert(DeferredVTables.empty()); - } + // Emitting a v-table doesn't directly cause more v-tables to + // become deferred, although it can cause functions to be + // emitted that then need those v-tables. + assert(DeferredVTables.empty()); + } + + // Stop if we're out of both deferred v-tables and deferred declarations. + if (DeferredDeclsToEmit.empty()) + return; - // Stop if we're out of both deferred v-tables and deferred declarations. - if (DeferredDeclsToEmit.empty()) break; + // Grab the list of decls to emit. If EmitGlobalDefinition schedules more + // work, it will not interfere with this. + std::vector<DeferredGlobal> CurDeclsToEmit; + CurDeclsToEmit.swap(DeferredDeclsToEmit); - DeferredGlobal &G = DeferredDeclsToEmit.back(); + for (DeferredGlobal &G : CurDeclsToEmit) { GlobalDecl D = G.GD; llvm::GlobalValue *GV = G.GV; - DeferredDeclsToEmit.pop_back(); + G.GV = nullptr; assert(!GV || GV == GetGlobalValue(getMangledName(D))); if (!GV) GV = GetGlobalValue(getMangledName(D)); - // Check to see if we've already emitted this. This is necessary // for a couple of reasons: first, decls can end up in the // deferred-decls queue multiple times, and second, decls can end @@ -1119,6 +1116,14 @@ void CodeGenModule::EmitDeferred() { // Otherwise, emit the definition and move on to the next one. EmitGlobalDefinition(D, GV); + + // If we found out that we need to emit more decls, do that recursively. + // This has the advantage that the decls are emitted in a DFS and related + // ones are close together, which is convenient for testing. + if (!DeferredVTables.empty() || !DeferredDeclsToEmit.empty()) { + EmitDeferred(); + assert(DeferredVTables.empty() && DeferredDeclsToEmit.empty()); + } } } @@ -1275,6 +1280,8 @@ llvm::Constant *CodeGenModule::GetAddrOfUuidDescriptor( auto *GV = new llvm::GlobalVariable( getModule(), Init->getType(), /*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name); + if (supportsCOMDAT()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); return GV; } @@ -1322,7 +1329,7 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) { // If this is CUDA, be selective about which declarations we emit. if (LangOpts.CUDA) { - if (CodeGenOpts.CUDAIsDevice) { + if (LangOpts.CUDAIsDevice) { if (!Global->hasAttr<CUDADeviceAttr>() && !Global->hasAttr<CUDAGlobalAttr>() && !Global->hasAttr<CUDAConstantAttr>() && @@ -1603,13 +1610,6 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName, addDeferredDeclToEmit(F, DDI->second); DeferredDecls.erase(DDI); - // Otherwise, if this is a sized deallocation function, emit a weak - // definition - // for it at the end of the translation unit. - } else if (D && cast<FunctionDecl>(D) - ->getCorrespondingUnsizedGlobalDeallocationFunction()) { - addDeferredDeclToEmit(F, GD); - // Otherwise, there are cases we have to worry about where we're // using a declaration for which we must emit a definition but where // we might not find a top-level definition: @@ -1766,6 +1766,8 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, // handling. GV->setConstant(isTypeConstant(D->getType(), false)); + GV->setAlignment(getContext().getDeclAlign(D).getQuantity()); + setLinkageAndVisibilityForGV(GV, D); if (D->getTLSKind()) { @@ -1829,7 +1831,11 @@ CodeGenModule::CreateOrReplaceCXXRuntimeVariable(StringRef Name, OldGV->eraseFromParent(); } - + + if (supportsCOMDAT() && GV->isWeakForLinker() && + !GV->hasAvailableExternallyLinkage()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); + return GV; } @@ -1884,7 +1890,7 @@ CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const { unsigned CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D, unsigned AddrSpace) { - if (LangOpts.CUDA && CodeGenOpts.CUDAIsDevice) { + if (LangOpts.CUDA && LangOpts.CUDAIsDevice) { if (D->hasAttr<CUDAConstantAttr>()) AddrSpace = getContext().getTargetAddressSpace(LangAS::cuda_constant); else if (D->hasAttr<CUDASharedAttr>()) @@ -1929,6 +1935,38 @@ void CodeGenModule::MaybeHandleStaticInExternC(const SomeDecl *D, R.first->second = nullptr; } +static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) { + if (!CGM.supportsCOMDAT()) + return false; + + if (D.hasAttr<SelectAnyAttr>()) + return true; + + GVALinkage Linkage; + if (auto *VD = dyn_cast<VarDecl>(&D)) + Linkage = CGM.getContext().GetGVALinkageForVariable(VD); + else + Linkage = CGM.getContext().GetGVALinkageForFunction(cast<FunctionDecl>(&D)); + + switch (Linkage) { + case GVA_Internal: + case GVA_AvailableExternally: + case GVA_StrongExternal: + return false; + case GVA_DiscardableODR: + case GVA_StrongODR: + return true; + } + llvm_unreachable("No such linkage"); +} + +void CodeGenModule::maybeSetTrivialComdat(const Decl &D, + llvm::GlobalObject &GO) { + if (!shouldBeInCOMDAT(*this, D)) + return; + GO.setComdat(TheModule.getOrInsertComdat(GO.getName())); +} + void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { llvm::Constant *Init = nullptr; QualType ASTTy = D->getType(); @@ -2072,6 +2110,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { setTLSMode(GV, *D); } + maybeSetTrivialComdat(*D, *GV); + // Emit the initializer function if necessary. if (NeedsGlobalCtor || NeedsGlobalDtor) EmitCXXGlobalVarDeclInitFunc(D, GV, NeedsGlobalCtor); @@ -2085,7 +2125,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) { } static bool isVarDeclStrongDefinition(const ASTContext &Context, - const VarDecl *D, bool NoCommon) { + CodeGenModule &CGM, const VarDecl *D, + bool NoCommon) { // Don't give variables common linkage if -fno-common was specified unless it // was overridden by a NoCommon attribute. if ((NoCommon || D->hasAttr<NoCommonAttr>()) && !D->hasAttr<CommonAttr>()) @@ -2110,11 +2151,31 @@ static bool isVarDeclStrongDefinition(const ASTContext &Context, if (D->hasAttr<WeakImportAttr>()) return true; + // A variable cannot be both common and exist in a comdat. + if (shouldBeInCOMDAT(CGM, *D)) + return true; + // Declarations with a required alignment do not have common linakge in MSVC // mode. - if (Context.getLangOpts().MSVCCompat && - (Context.isAlignmentRequired(D->getType()) || D->hasAttr<AlignedAttr>())) - return true; + if (Context.getLangOpts().MSVCCompat) { + if (D->hasAttr<AlignedAttr>()) + return true; + QualType VarType = D->getType(); + if (Context.isAlignmentRequired(VarType)) + return true; + + if (const auto *RT = VarType->getAs<RecordType>()) { + const RecordDecl *RD = RT->getDecl(); + for (const FieldDecl *FD : RD->fields()) { + if (FD->isBitField()) + continue; + if (FD->hasAttr<AlignedAttr>()) + return true; + if (Context.isAlignmentRequired(FD->getType())) + return true; + } + } + } return false; } @@ -2162,7 +2223,7 @@ llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageForDeclarator( // C++ doesn't have tentative definitions and thus cannot have common // linkage. if (!getLangOpts().CPlusPlus && isa<VarDecl>(D) && - !isVarDeclStrongDefinition(Context, cast<VarDecl>(D), + !isVarDeclStrongDefinition(Context, *this, cast<VarDecl>(D), CodeGenOpts.NoCommon)) return llvm::GlobalVariable::CommonLinkage; @@ -2285,7 +2346,7 @@ static void replaceUsesOfNonProtoConstant(llvm::Constant *old, callSite->replaceAllUsesWith(newCall.getInstruction()); // Copy debug location attached to CI. - if (!callSite->getDebugLoc().isUnknown()) + if (callSite->getDebugLoc()) newCall->setDebugLoc(callSite->getDebugLoc()); callSite->eraseFromParent(); } @@ -2406,6 +2467,8 @@ void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD, MaybeHandleStaticInExternC(D, Fn); + maybeSetTrivialComdat(*D, *Fn); + CodeGenFunction(*this).GenerateCode(D, Fn, FI); setFunctionDefinitionAttributes(D, Fn); @@ -2449,7 +2512,7 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) { // Create the new alias itself, but don't set a name yet. auto *GA = llvm::GlobalAlias::create( - cast<llvm::PointerType>(Aliasee->getType())->getElementType(), 0, + cast<llvm::PointerType>(Aliasee->getType()), llvm::Function::ExternalLinkage, "", Aliasee, &getModule()); if (Entry) { @@ -2497,12 +2560,10 @@ llvm::Function *CodeGenModule::getIntrinsic(unsigned IID, Tys); } -static llvm::StringMapEntry<llvm::Constant*> & -GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map, - const StringLiteral *Literal, - bool TargetIsLSB, - bool &IsUTF16, - unsigned &StringLength) { +static llvm::StringMapEntry<llvm::GlobalVariable *> & +GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, + const StringLiteral *Literal, bool TargetIsLSB, + bool &IsUTF16, unsigned &StringLength) { StringRef String = Literal->getString(); unsigned NumBytes = String.size(); @@ -2534,10 +2595,9 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map, nullptr)).first; } -static llvm::StringMapEntry<llvm::Constant*> & -GetConstantStringEntry(llvm::StringMap<llvm::Constant*> &Map, - const StringLiteral *Literal, - unsigned &StringLength) { +static llvm::StringMapEntry<llvm::GlobalVariable *> & +GetConstantStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map, + const StringLiteral *Literal, unsigned &StringLength) { StringRef String = Literal->getString(); StringLength = String.size(); return *Map.insert(std::make_pair(String, nullptr)).first; @@ -2547,10 +2607,10 @@ llvm::Constant * CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { unsigned StringLength = 0; bool isUTF16 = false; - llvm::StringMapEntry<llvm::Constant*> &Entry = - GetConstantCFStringEntry(CFConstantStringMap, Literal, - getDataLayout().isLittleEndian(), - isUTF16, StringLength); + llvm::StringMapEntry<llvm::GlobalVariable *> &Entry = + GetConstantCFStringEntry(CFConstantStringMap, Literal, + getDataLayout().isLittleEndian(), isUTF16, + StringLength); if (auto *C = Entry.second) return C; @@ -2566,7 +2626,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { llvm::Constant *GV = CreateRuntimeVariable(Ty, "__CFConstantStringClassReference"); // Decay array -> ptr - V = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); + V = llvm::ConstantExpr::getGetElementPtr(Ty, GV, Zeros); CFConstantStringClassRef = V; } else @@ -2619,7 +2679,8 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { } // String. - Fields[2] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); + Fields[2] = + llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); if (isUTF16) // Cast the UTF16 string to the correct type. @@ -2640,11 +2701,11 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) { return GV; } -llvm::Constant * +llvm::GlobalVariable * CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { unsigned StringLength = 0; - llvm::StringMapEntry<llvm::Constant*> &Entry = - GetConstantStringEntry(CFConstantStringMap, Literal, StringLength); + llvm::StringMapEntry<llvm::GlobalVariable *> &Entry = + GetConstantStringEntry(CFConstantStringMap, Literal, StringLength); if (auto *C = Entry.second) return C; @@ -2673,11 +2734,10 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { llvm::Type *PTy = llvm::ArrayType::get(Ty, 0); GV = CreateRuntimeVariable(PTy, str); // Decay array -> ptr - V = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); + V = llvm::ConstantExpr::getGetElementPtr(PTy, GV, Zeros); ConstantStringClassRef = V; } - } - else + } else V = ConstantStringClassRef; if (!NSConstantStringType) { @@ -2733,8 +2793,9 @@ CodeGenModule::GetAddrOfConstantString(const StringLiteral *Literal) { // of the string is via this class initializer. CharUnits Align = getContext().getTypeAlignInChars(getContext().CharTy); GV->setAlignment(Align.getQuantity()); - Fields[1] = llvm::ConstantExpr::getGetElementPtr(GV, Zeros); - + Fields[1] = + llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros); + // String length. llvm::Type *Ty = getTypes().ConvertType(getContext().UnsignedIntTy); Fields[2] = llvm::ConstantInt::get(Ty, StringLength); @@ -2838,12 +2899,18 @@ GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT, if (CGM.getLangOpts().OpenCL) AddrSpace = CGM.getContext().getTargetAddressSpace(LangAS::opencl_constant); + llvm::Module &M = CGM.getModule(); // Create a global variable for this string auto *GV = new llvm::GlobalVariable( - CGM.getModule(), C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, - GlobalName, nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace); + M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName, + nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace); GV->setAlignment(Alignment); GV->setUnnamedAddr(true); + if (GV->isWeakForLinker()) { + assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals"); + GV->setComdat(M.getOrInsertComdat(GV->getName())); + } + return GV; } @@ -3002,10 +3069,19 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( // Create a global variable for this lifetime-extended temporary. llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD, Constant); - // There is no need for this temporary to have global linkage if the global - // variable has external linkage. - if (Linkage == llvm::GlobalVariable::ExternalLinkage) - Linkage = llvm::GlobalVariable::PrivateLinkage; + if (Linkage == llvm::GlobalVariable::ExternalLinkage) { + const VarDecl *InitVD; + if (VD->isStaticDataMember() && VD->getAnyInitializer(InitVD) && + isa<CXXRecordDecl>(InitVD->getLexicalDeclContext())) { + // Temporaries defined inside a class get linkonce_odr linkage because the + // class can be defined in multipe translation units. + Linkage = llvm::GlobalVariable::LinkOnceODRLinkage; + } else { + // There is no need for this temporary to have external linkage if the + // VarDecl has external linkage. + Linkage = llvm::GlobalVariable::InternalLinkage; + } + } unsigned AddrSpace = GetGlobalVarAddressSpace( VD, getContext().getTargetAddressSpace(MaterializedType)); auto *GV = new llvm::GlobalVariable( @@ -3015,6 +3091,8 @@ llvm::Constant *CodeGenModule::GetAddrOfGlobalTemporary( setGlobalVisibility(GV, VD); GV->setAlignment( getContext().getTypeAlignInChars(MaterializedType).getQuantity()); + if (supportsCOMDAT() && GV->isWeakForLinker()) + GV->setComdat(TheModule.getOrInsertComdat(GV->getName())); if (VD->getTLSKind()) setTLSMode(GV, *VD); Slot = GV; @@ -3268,16 +3346,11 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { break; case Decl::FileScopeAsm: { + // File-scope asm is ignored during device-side CUDA compilation. + if (LangOpts.CUDA && LangOpts.CUDAIsDevice) + break; auto *AD = cast<FileScopeAsmDecl>(D); - StringRef AsmString = AD->getAsmString()->getString(); - - const std::string &S = getModule().getModuleInlineAsm(); - if (S.empty()) - getModule().setModuleInlineAsm(AsmString); - else if (S.end()[-1] == '\n') - getModule().setModuleInlineAsm(S + AsmString.str()); - else - getModule().setModuleInlineAsm(S + '\n' + AsmString.str()); + getModule().appendModuleInlineAsm(AD->getAsmString()->getString()); break; } @@ -3285,7 +3358,7 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) { auto *Import = cast<ImportDecl>(D); // Ignore import declarations that come from imported modules. - if (clang::Module *Owner = Import->getOwningModule()) { + if (clang::Module *Owner = Import->getImportedOwningModule()) { if (getLangOpts().CurrentModule.empty() || Owner->getTopLevelModule()->Name == getLangOpts().CurrentModule) break; @@ -3357,7 +3430,7 @@ void CodeGenModule::ClearUnusedCoverageMapping(const Decl *D) { void CodeGenModule::EmitDeferredUnusedCoverageMappings() { std::vector<const Decl *> DeferredDecls; - for (const auto I : DeferredEmptyCoverageMappingDecls) { + for (const auto &I : DeferredEmptyCoverageMappingDecls) { if (!I.second) continue; DeferredDecls.push_back(I.first); @@ -3561,6 +3634,12 @@ llvm::Constant *CodeGenModule::EmitUuidofInitializer(StringRef Uuid) { return llvm::ConstantStruct::getAnon(Fields); } +llvm::Constant * +CodeGenModule::getAddrOfCXXCatchHandlerType(QualType Ty, + QualType CatchHandlerType) { + return getCXXABI().getAddrOfCXXCatchHandlerType(Ty, CatchHandlerType); +} + llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH) { // Return a bogus pointer if RTTI is disabled, unless it's for EH. @@ -3583,11 +3662,8 @@ void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) { VD->getAnyInitializer() && !VD->getAnyInitializer()->isConstantInitializer(getContext(), /*ForRef=*/false); - if (auto InitFunction = - getOpenMPRuntime().EmitOMPThreadPrivateVarDefinition( - VD, GetAddrOfGlobalVar(VD), RefExpr->getLocStart(), - PerformInit)) + if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition( + VD, GetAddrOfGlobalVar(VD), RefExpr->getLocStart(), PerformInit)) CXXGlobalInits.push_back(InitFunction); } } - diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h index 2aafe7e88bfe..feef6c2583ec 100644 --- a/lib/CodeGen/CodeGenModule.h +++ b/lib/CodeGen/CodeGenModule.h @@ -258,8 +258,8 @@ public: /// This class organizes the cross-function state that is used while generating /// LLVM code. class CodeGenModule : public CodeGenTypeCache { - CodeGenModule(const CodeGenModule &) LLVM_DELETED_FUNCTION; - void operator=(const CodeGenModule &) LLVM_DELETED_FUNCTION; + CodeGenModule(const CodeGenModule &) = delete; + void operator=(const CodeGenModule &) = delete; public: struct Structor { @@ -366,7 +366,7 @@ private: /// Map used to get unique annotation strings. llvm::StringMap<llvm::Constant*> AnnotationStrings; - llvm::StringMap<llvm::Constant*> CFConstantStringMap; + llvm::StringMap<llvm::GlobalVariable *> CFConstantStringMap; llvm::DenseMap<llvm::Constant *, llvm::GlobalVariable *> ConstantStringMap; llvm::DenseMap<const Decl*, llvm::Constant *> StaticLocalDeclMap; @@ -400,7 +400,8 @@ private: /// When a C++ decl with an initializer is deferred, null is /// appended to CXXGlobalInits, and the index of that null is placed /// here so that the initializer will be performed in the correct - /// order. + /// order. Once the decl is emitted, the index is replaced with ~0U to ensure + /// that we don't re-emit the initializer. llvm::DenseMap<const Decl*, unsigned> DelayedCXXInitPosition; typedef std::pair<OrderGlobalInits, llvm::Function*> GlobalInitData; @@ -606,6 +607,7 @@ public: const TargetInfo &getTarget() const { return Target; } const llvm::Triple &getTriple() const; bool supportsCOMDAT() const; + void maybeSetTrivialComdat(const Decl &D, llvm::GlobalObject &GO); CGCXXABI &getCXXABI() const { return *ABI; } llvm::LLVMContext &getLLVMContext() { return VMContext; } @@ -718,6 +720,9 @@ public: /// Get the address of the RTTI descriptor for the given type. llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false); + llvm::Constant *getAddrOfCXXCatchHandlerType(QualType Ty, + QualType CatchHandlerType); + /// Get the address of a uuid descriptor . llvm::Constant *GetAddrOfUuidDescriptor(const CXXUuidofExpr* E); @@ -782,7 +787,7 @@ public: /// Return a pointer to a constant NSString object for the given string. Or a /// user defined String object as defined via /// -fconstant-string-class=class_name option. - llvm::Constant *GetAddrOfConstantString(const StringLiteral *Literal); + llvm::GlobalVariable *GetAddrOfConstantString(const StringLiteral *Literal); /// Return a constant array for the given string. llvm::Constant *GetConstantArrayFromStringLiteral(const StringLiteral *E); @@ -993,7 +998,7 @@ public: void EmitTentativeDefinition(const VarDecl *D); - void EmitVTable(CXXRecordDecl *Class, bool DefinitionRequired); + void EmitVTable(CXXRecordDecl *Class); /// Emit the RTTI descriptors for the builtin types. void EmitFundamentalRTTIDescriptors(); @@ -1102,6 +1107,14 @@ public: /// \param D Threadprivate declaration. void EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D); + /// Emit bit set entries for the given vtable using the given layout if + /// vptr CFI is enabled. + void EmitVTableBitSetEntries(llvm::GlobalVariable *VTable, + const VTableLayout &VTLayout); + + /// \breif Get the declaration of std::terminate for the platform. + llvm::Constant *getTerminateFn(); + private: llvm::Constant * GetOrCreateLLVMFunction(StringRef MangledName, llvm::Type *Ty, GlobalDecl D, diff --git a/lib/CodeGen/CodeGenPGO.cpp b/lib/CodeGen/CodeGenPGO.cpp index 24b035d67598..c97244328d37 100644 --- a/lib/CodeGen/CodeGenPGO.cpp +++ b/lib/CodeGen/CodeGenPGO.cpp @@ -58,12 +58,16 @@ void CodeGenPGO::setFuncName(llvm::Function *Fn) { } void CodeGenPGO::createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage) { - // Usually, we want to match the function's linkage, but - // available_externally and extern_weak both have the wrong semantics. + // We generally want to match the function's linkage, but available_externally + // and extern_weak both have the wrong semantics, and anything that doesn't + // need to link across compilation units doesn't need to be visible at all. if (Linkage == llvm::GlobalValue::ExternalWeakLinkage) Linkage = llvm::GlobalValue::LinkOnceAnyLinkage; else if (Linkage == llvm::GlobalValue::AvailableExternallyLinkage) Linkage = llvm::GlobalValue::LinkOnceODRLinkage; + else if (Linkage == llvm::GlobalValue::InternalLinkage || + Linkage == llvm::GlobalValue::ExternalLinkage) + Linkage = llvm::GlobalValue::PrivateLinkage; auto *Value = llvm::ConstantDataArray::getString(CGM.getLLVMContext(), FuncName, false); @@ -138,482 +142,469 @@ const int PGOHash::NumBitsPerType; const unsigned PGOHash::NumTypesPerWord; const unsigned PGOHash::TooBig; - /// A RecursiveASTVisitor that fills a map of statements to PGO counters. - struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { - /// The next counter value to assign. - unsigned NextCounter; - /// The function hash. - PGOHash Hash; - /// The map of statements to counters. - llvm::DenseMap<const Stmt *, unsigned> &CounterMap; - - MapRegionCounters(llvm::DenseMap<const Stmt *, unsigned> &CounterMap) - : NextCounter(0), CounterMap(CounterMap) {} - - // Blocks and lambdas are handled as separate functions, so we need not - // traverse them in the parent context. - bool TraverseBlockExpr(BlockExpr *BE) { return true; } - bool TraverseLambdaBody(LambdaExpr *LE) { return true; } - bool TraverseCapturedStmt(CapturedStmt *CS) { return true; } - - bool VisitDecl(const Decl *D) { - switch (D->getKind()) { - default: - break; - case Decl::Function: - case Decl::CXXMethod: - case Decl::CXXConstructor: - case Decl::CXXDestructor: - case Decl::CXXConversion: - case Decl::ObjCMethod: - case Decl::Block: - case Decl::Captured: - CounterMap[D->getBody()] = NextCounter++; - break; - } - return true; +/// A RecursiveASTVisitor that fills a map of statements to PGO counters. +struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { + /// The next counter value to assign. + unsigned NextCounter; + /// The function hash. + PGOHash Hash; + /// The map of statements to counters. + llvm::DenseMap<const Stmt *, unsigned> &CounterMap; + + MapRegionCounters(llvm::DenseMap<const Stmt *, unsigned> &CounterMap) + : NextCounter(0), CounterMap(CounterMap) {} + + // Blocks and lambdas are handled as separate functions, so we need not + // traverse them in the parent context. + bool TraverseBlockExpr(BlockExpr *BE) { return true; } + bool TraverseLambdaBody(LambdaExpr *LE) { return true; } + bool TraverseCapturedStmt(CapturedStmt *CS) { return true; } + + bool VisitDecl(const Decl *D) { + switch (D->getKind()) { + default: + break; + case Decl::Function: + case Decl::CXXMethod: + case Decl::CXXConstructor: + case Decl::CXXDestructor: + case Decl::CXXConversion: + case Decl::ObjCMethod: + case Decl::Block: + case Decl::Captured: + CounterMap[D->getBody()] = NextCounter++; + break; } + return true; + } - bool VisitStmt(const Stmt *S) { - auto Type = getHashType(S); - if (Type == PGOHash::None) - return true; - - CounterMap[S] = NextCounter++; - Hash.combine(Type); + bool VisitStmt(const Stmt *S) { + auto Type = getHashType(S); + if (Type == PGOHash::None) return true; + + CounterMap[S] = NextCounter++; + Hash.combine(Type); + return true; + } + PGOHash::HashType getHashType(const Stmt *S) { + switch (S->getStmtClass()) { + default: + break; + case Stmt::LabelStmtClass: + return PGOHash::LabelStmt; + case Stmt::WhileStmtClass: + return PGOHash::WhileStmt; + case Stmt::DoStmtClass: + return PGOHash::DoStmt; + case Stmt::ForStmtClass: + return PGOHash::ForStmt; + case Stmt::CXXForRangeStmtClass: + return PGOHash::CXXForRangeStmt; + case Stmt::ObjCForCollectionStmtClass: + return PGOHash::ObjCForCollectionStmt; + case Stmt::SwitchStmtClass: + return PGOHash::SwitchStmt; + case Stmt::CaseStmtClass: + return PGOHash::CaseStmt; + case Stmt::DefaultStmtClass: + return PGOHash::DefaultStmt; + case Stmt::IfStmtClass: + return PGOHash::IfStmt; + case Stmt::CXXTryStmtClass: + return PGOHash::CXXTryStmt; + case Stmt::CXXCatchStmtClass: + return PGOHash::CXXCatchStmt; + case Stmt::ConditionalOperatorClass: + return PGOHash::ConditionalOperator; + case Stmt::BinaryConditionalOperatorClass: + return PGOHash::BinaryConditionalOperator; + case Stmt::BinaryOperatorClass: { + const BinaryOperator *BO = cast<BinaryOperator>(S); + if (BO->getOpcode() == BO_LAnd) + return PGOHash::BinaryOperatorLAnd; + if (BO->getOpcode() == BO_LOr) + return PGOHash::BinaryOperatorLOr; + break; } - PGOHash::HashType getHashType(const Stmt *S) { - switch (S->getStmtClass()) { - default: - break; - case Stmt::LabelStmtClass: - return PGOHash::LabelStmt; - case Stmt::WhileStmtClass: - return PGOHash::WhileStmt; - case Stmt::DoStmtClass: - return PGOHash::DoStmt; - case Stmt::ForStmtClass: - return PGOHash::ForStmt; - case Stmt::CXXForRangeStmtClass: - return PGOHash::CXXForRangeStmt; - case Stmt::ObjCForCollectionStmtClass: - return PGOHash::ObjCForCollectionStmt; - case Stmt::SwitchStmtClass: - return PGOHash::SwitchStmt; - case Stmt::CaseStmtClass: - return PGOHash::CaseStmt; - case Stmt::DefaultStmtClass: - return PGOHash::DefaultStmt; - case Stmt::IfStmtClass: - return PGOHash::IfStmt; - case Stmt::CXXTryStmtClass: - return PGOHash::CXXTryStmt; - case Stmt::CXXCatchStmtClass: - return PGOHash::CXXCatchStmt; - case Stmt::ConditionalOperatorClass: - return PGOHash::ConditionalOperator; - case Stmt::BinaryConditionalOperatorClass: - return PGOHash::BinaryConditionalOperator; - case Stmt::BinaryOperatorClass: { - const BinaryOperator *BO = cast<BinaryOperator>(S); - if (BO->getOpcode() == BO_LAnd) - return PGOHash::BinaryOperatorLAnd; - if (BO->getOpcode() == BO_LOr) - return PGOHash::BinaryOperatorLOr; - break; - } - } - return PGOHash::None; } + return PGOHash::None; + } +}; + +/// A StmtVisitor that propagates the raw counts through the AST and +/// records the count at statements where the value may change. +struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { + /// PGO state. + CodeGenPGO &PGO; + + /// A flag that is set when the current count should be recorded on the + /// next statement, such as at the exit of a loop. + bool RecordNextStmtCount; + + /// The count at the current location in the traversal. + uint64_t CurrentCount; + + /// The map of statements to count values. + llvm::DenseMap<const Stmt *, uint64_t> &CountMap; + + /// BreakContinueStack - Keep counts of breaks and continues inside loops. + struct BreakContinue { + uint64_t BreakCount; + uint64_t ContinueCount; + BreakContinue() : BreakCount(0), ContinueCount(0) {} }; + SmallVector<BreakContinue, 8> BreakContinueStack; - /// A StmtVisitor that propagates the raw counts through the AST and - /// records the count at statements where the value may change. - struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { - /// PGO state. - CodeGenPGO &PGO; - - /// A flag that is set when the current count should be recorded on the - /// next statement, such as at the exit of a loop. - bool RecordNextStmtCount; - - /// The map of statements to count values. - llvm::DenseMap<const Stmt *, uint64_t> &CountMap; - - /// BreakContinueStack - Keep counts of breaks and continues inside loops. - struct BreakContinue { - uint64_t BreakCount; - uint64_t ContinueCount; - BreakContinue() : BreakCount(0), ContinueCount(0) {} - }; - SmallVector<BreakContinue, 8> BreakContinueStack; - - ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &CountMap, - CodeGenPGO &PGO) - : PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {} - - void RecordStmtCount(const Stmt *S) { - if (RecordNextStmtCount) { - CountMap[S] = PGO.getCurrentRegionCount(); - RecordNextStmtCount = false; - } - } + ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &CountMap, + CodeGenPGO &PGO) + : PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {} - void VisitStmt(const Stmt *S) { - RecordStmtCount(S); - for (Stmt::const_child_range I = S->children(); I; ++I) { - if (*I) - this->Visit(*I); - } + void RecordStmtCount(const Stmt *S) { + if (RecordNextStmtCount) { + CountMap[S] = CurrentCount; + RecordNextStmtCount = false; } + } - void VisitFunctionDecl(const FunctionDecl *D) { - // Counter tracks entry to the function body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); - } + /// Set and return the current count. + uint64_t setCount(uint64_t Count) { + CurrentCount = Count; + return Count; + } - // Skip lambda expressions. We visit these as FunctionDecls when we're - // generating them and aren't interested in the body when generating a - // parent context. - void VisitLambdaExpr(const LambdaExpr *LE) {} - - void VisitCapturedDecl(const CapturedDecl *D) { - // Counter tracks entry to the capture body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); + void VisitStmt(const Stmt *S) { + RecordStmtCount(S); + for (Stmt::const_child_range I = S->children(); I; ++I) { + if (*I) + this->Visit(*I); } + } - void VisitObjCMethodDecl(const ObjCMethodDecl *D) { - // Counter tracks entry to the method body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); - } + void VisitFunctionDecl(const FunctionDecl *D) { + // Counter tracks entry to the function body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitBlockDecl(const BlockDecl *D) { - // Counter tracks entry to the block body. - RegionCounter Cnt(PGO, D->getBody()); - Cnt.beginRegion(); - CountMap[D->getBody()] = PGO.getCurrentRegionCount(); - Visit(D->getBody()); - } + // Skip lambda expressions. We visit these as FunctionDecls when we're + // generating them and aren't interested in the body when generating a + // parent context. + void VisitLambdaExpr(const LambdaExpr *LE) {} - void VisitReturnStmt(const ReturnStmt *S) { - RecordStmtCount(S); - if (S->getRetValue()) - Visit(S->getRetValue()); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitCapturedDecl(const CapturedDecl *D) { + // Counter tracks entry to the capture body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitGotoStmt(const GotoStmt *S) { - RecordStmtCount(S); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitObjCMethodDecl(const ObjCMethodDecl *D) { + // Counter tracks entry to the method body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitLabelStmt(const LabelStmt *S) { - RecordNextStmtCount = false; - // Counter tracks the block following the label. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(); - CountMap[S] = PGO.getCurrentRegionCount(); - Visit(S->getSubStmt()); - } + void VisitBlockDecl(const BlockDecl *D) { + // Counter tracks entry to the block body. + uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); + CountMap[D->getBody()] = BodyCount; + Visit(D->getBody()); + } - void VisitBreakStmt(const BreakStmt *S) { - RecordStmtCount(S); - assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); - BreakContinueStack.back().BreakCount += PGO.getCurrentRegionCount(); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitReturnStmt(const ReturnStmt *S) { + RecordStmtCount(S); + if (S->getRetValue()) + Visit(S->getRetValue()); + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitContinueStmt(const ContinueStmt *S) { - RecordStmtCount(S); - assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); - BreakContinueStack.back().ContinueCount += PGO.getCurrentRegionCount(); - PGO.setCurrentRegionUnreachable(); - RecordNextStmtCount = true; - } + void VisitCXXThrowExpr(const CXXThrowExpr *E) { + RecordStmtCount(E); + if (E->getSubExpr()) + Visit(E->getSubExpr()); + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitWhileStmt(const WhileStmt *S) { - RecordStmtCount(S); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first so the break/continue adjustments can be - // included when visiting the condition. - Cnt.beginRegion(); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // ...then go back and propagate counts through the condition. The count - // at the start of the condition is the sum of the incoming edges, - // the backedge from the end of the loop body, and the edges from - // continue statements. - BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitGotoStmt(const GotoStmt *S) { + RecordStmtCount(S); + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitDoStmt(const DoStmt *S) { - RecordStmtCount(S); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - BreakContinue BC = BreakContinueStack.pop_back_val(); - // The count at the start of the condition is equal to the count at the - // end of the body. The adjusted count does not include either the - // fall-through count coming into the loop or the continue count, so add - // both of those separately. This is coincidentally the same equation as - // with while loops but for different reasons. - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitLabelStmt(const LabelStmt *S) { + RecordNextStmtCount = false; + // Counter tracks the block following the label. + uint64_t BlockCount = setCount(PGO.getRegionCount(S)); + CountMap[S] = BlockCount; + Visit(S->getSubStmt()); + } - void VisitForStmt(const ForStmt *S) { - RecordStmtCount(S); - if (S->getInit()) - Visit(S->getInit()); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // The increment is essentially part of the body but it needs to include - // the count for all the continue statements. - if (S->getInc()) { - Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() + - BreakContinueStack.back().ContinueCount); - CountMap[S->getInc()] = PGO.getCurrentRegionCount(); - Visit(S->getInc()); - Cnt.adjustForControlFlow(); - } - - BreakContinue BC = BreakContinueStack.pop_back_val(); - - // ...then go back and propagate counts through the condition. - if (S->getCond()) { - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + - BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitBreakStmt(const BreakStmt *S) { + RecordStmtCount(S); + assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); + BreakContinueStack.back().BreakCount += CurrentCount; + CurrentCount = 0; + RecordNextStmtCount = true; + } - void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { - RecordStmtCount(S); - Visit(S->getRangeStmt()); - Visit(S->getBeginEndStmt()); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - CountMap[S->getLoopVarStmt()] = PGO.getCurrentRegionCount(); - Visit(S->getLoopVarStmt()); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // The increment is essentially part of the body but it needs to include - // the count for all the continue statements. - Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() + - BreakContinueStack.back().ContinueCount); - CountMap[S->getInc()] = PGO.getCurrentRegionCount(); - Visit(S->getInc()); - Cnt.adjustForControlFlow(); + void VisitContinueStmt(const ContinueStmt *S) { + RecordStmtCount(S); + assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); + BreakContinueStack.back().ContinueCount += CurrentCount; + CurrentCount = 0; + RecordNextStmtCount = true; + } - BreakContinue BC = BreakContinueStack.pop_back_val(); + void VisitWhileStmt(const WhileStmt *S) { + RecordStmtCount(S); + uint64_t ParentCount = CurrentCount; + + BreakContinueStack.push_back(BreakContinue()); + // Visit the body region first so the break/continue adjustments can be + // included when visiting the condition. + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = CurrentCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + + // ...then go back and propagate counts through the condition. The count + // at the start of the condition is the sum of the incoming edges, + // the backedge from the end of the loop body, and the edges from + // continue statements. + BreakContinue BC = BreakContinueStack.pop_back_val(); + uint64_t CondCount = + setCount(ParentCount + BackedgeCount + BC.ContinueCount); + CountMap[S->getCond()] = CondCount; + Visit(S->getCond()); + setCount(BC.BreakCount + CondCount - BodyCount); + RecordNextStmtCount = true; + } - // ...then go back and propagate counts through the condition. - Cnt.setCurrentRegionCount(Cnt.getParentCount() + - Cnt.getAdjustedCount() + - BC.ContinueCount); - CountMap[S->getCond()] = PGO.getCurrentRegionCount(); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; - } + void VisitDoStmt(const DoStmt *S) { + RecordStmtCount(S); + uint64_t LoopCount = PGO.getRegionCount(S); + + BreakContinueStack.push_back(BreakContinue()); + // The count doesn't include the fallthrough from the parent scope. Add it. + uint64_t BodyCount = setCount(LoopCount + CurrentCount); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + + BreakContinue BC = BreakContinueStack.pop_back_val(); + // The count at the start of the condition is equal to the count at the + // end of the body, plus any continues. + uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount); + CountMap[S->getCond()] = CondCount; + Visit(S->getCond()); + setCount(BC.BreakCount + CondCount - LoopCount); + RecordNextStmtCount = true; + } - void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { - RecordStmtCount(S); - Visit(S->getElement()); - // Counter tracks the body of the loop. - RegionCounter Cnt(PGO, S); - BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(); - CountMap[S->getBody()] = PGO.getCurrentRegionCount(); - Visit(S->getBody()); - BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount); - RecordNextStmtCount = true; + void VisitForStmt(const ForStmt *S) { + RecordStmtCount(S); + if (S->getInit()) + Visit(S->getInit()); + + uint64_t ParentCount = CurrentCount; + + BreakContinueStack.push_back(BreakContinue()); + // Visit the body region first. (This is basically the same as a while + // loop; see further comments in VisitWhileStmt.) + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + BreakContinue BC = BreakContinueStack.pop_back_val(); + + // The increment is essentially part of the body but it needs to include + // the count for all the continue statements. + if (S->getInc()) { + uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); + CountMap[S->getInc()] = IncCount; + Visit(S->getInc()); } - void VisitSwitchStmt(const SwitchStmt *S) { - RecordStmtCount(S); + // ...then go back and propagate counts through the condition. + uint64_t CondCount = + setCount(ParentCount + BackedgeCount + BC.ContinueCount); + if (S->getCond()) { + CountMap[S->getCond()] = CondCount; Visit(S->getCond()); - PGO.setCurrentRegionUnreachable(); - BreakContinueStack.push_back(BreakContinue()); - Visit(S->getBody()); - // If the switch is inside a loop, add the continue counts. - BreakContinue BC = BreakContinueStack.pop_back_val(); - if (!BreakContinueStack.empty()) - BreakContinueStack.back().ContinueCount += BC.ContinueCount; - // Counter tracks the exit block of the switch. - RegionCounter ExitCnt(PGO, S); - ExitCnt.beginRegion(); - RecordNextStmtCount = true; } + setCount(BC.BreakCount + CondCount - BodyCount); + RecordNextStmtCount = true; + } - void VisitCaseStmt(const CaseStmt *S) { - RecordNextStmtCount = false; - // Counter for this particular case. This counts only jumps from the - // switch header and does not include fallthrough from the case before - // this one. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - CountMap[S] = Cnt.getCount(); - RecordNextStmtCount = true; - Visit(S->getSubStmt()); - } + void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { + RecordStmtCount(S); + Visit(S->getLoopVarStmt()); + Visit(S->getRangeStmt()); + Visit(S->getBeginEndStmt()); + + uint64_t ParentCount = CurrentCount; + BreakContinueStack.push_back(BreakContinue()); + // Visit the body region first. (This is basically the same as a while + // loop; see further comments in VisitWhileStmt.) + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + BreakContinue BC = BreakContinueStack.pop_back_val(); + + // The increment is essentially part of the body but it needs to include + // the count for all the continue statements. + uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); + CountMap[S->getInc()] = IncCount; + Visit(S->getInc()); + + // ...then go back and propagate counts through the condition. + uint64_t CondCount = + setCount(ParentCount + BackedgeCount + BC.ContinueCount); + CountMap[S->getCond()] = CondCount; + Visit(S->getCond()); + setCount(BC.BreakCount + CondCount - BodyCount); + RecordNextStmtCount = true; + } - void VisitDefaultStmt(const DefaultStmt *S) { - RecordNextStmtCount = false; - // Counter for this default case. This does not include fallthrough from - // the previous case. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - CountMap[S] = Cnt.getCount(); - RecordNextStmtCount = true; - Visit(S->getSubStmt()); - } + void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { + RecordStmtCount(S); + Visit(S->getElement()); + uint64_t ParentCount = CurrentCount; + BreakContinueStack.push_back(BreakContinue()); + // Counter tracks the body of the loop. + uint64_t BodyCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getBody()] = BodyCount; + Visit(S->getBody()); + uint64_t BackedgeCount = CurrentCount; + BreakContinue BC = BreakContinueStack.pop_back_val(); + + setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount - + BodyCount); + RecordNextStmtCount = true; + } - void VisitIfStmt(const IfStmt *S) { - RecordStmtCount(S); - // Counter tracks the "then" part of an if statement. The count for - // the "else" part, if it exists, will be calculated from this counter. - RegionCounter Cnt(PGO, S); - Visit(S->getCond()); + void VisitSwitchStmt(const SwitchStmt *S) { + RecordStmtCount(S); + Visit(S->getCond()); + CurrentCount = 0; + BreakContinueStack.push_back(BreakContinue()); + Visit(S->getBody()); + // If the switch is inside a loop, add the continue counts. + BreakContinue BC = BreakContinueStack.pop_back_val(); + if (!BreakContinueStack.empty()) + BreakContinueStack.back().ContinueCount += BC.ContinueCount; + // Counter tracks the exit block of the switch. + setCount(PGO.getRegionCount(S)); + RecordNextStmtCount = true; + } - Cnt.beginRegion(); - CountMap[S->getThen()] = PGO.getCurrentRegionCount(); - Visit(S->getThen()); - Cnt.adjustForControlFlow(); - - if (S->getElse()) { - Cnt.beginElseRegion(); - CountMap[S->getElse()] = PGO.getCurrentRegionCount(); - Visit(S->getElse()); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } + void VisitSwitchCase(const SwitchCase *S) { + RecordNextStmtCount = false; + // Counter for this particular case. This counts only jumps from the + // switch header and does not include fallthrough from the case before + // this one. + uint64_t CaseCount = PGO.getRegionCount(S); + setCount(CurrentCount + CaseCount); + // We need the count without fallthrough in the mapping, so it's more useful + // for branch probabilities. + CountMap[S] = CaseCount; + RecordNextStmtCount = true; + Visit(S->getSubStmt()); + } - void VisitCXXTryStmt(const CXXTryStmt *S) { - RecordStmtCount(S); - Visit(S->getTryBlock()); - for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) - Visit(S->getHandler(I)); - // Counter tracks the continuation block of the try statement. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(); - RecordNextStmtCount = true; - } + void VisitIfStmt(const IfStmt *S) { + RecordStmtCount(S); + uint64_t ParentCount = CurrentCount; + Visit(S->getCond()); + + // Counter tracks the "then" part of an if statement. The count for + // the "else" part, if it exists, will be calculated from this counter. + uint64_t ThenCount = setCount(PGO.getRegionCount(S)); + CountMap[S->getThen()] = ThenCount; + Visit(S->getThen()); + uint64_t OutCount = CurrentCount; + + uint64_t ElseCount = ParentCount - ThenCount; + if (S->getElse()) { + setCount(ElseCount); + CountMap[S->getElse()] = ElseCount; + Visit(S->getElse()); + OutCount += CurrentCount; + } else + OutCount += ElseCount; + setCount(OutCount); + RecordNextStmtCount = true; + } - void VisitCXXCatchStmt(const CXXCatchStmt *S) { - RecordNextStmtCount = false; - // Counter tracks the catch statement's handler block. - RegionCounter Cnt(PGO, S); - Cnt.beginRegion(); - CountMap[S] = PGO.getCurrentRegionCount(); - Visit(S->getHandlerBlock()); - } + void VisitCXXTryStmt(const CXXTryStmt *S) { + RecordStmtCount(S); + Visit(S->getTryBlock()); + for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) + Visit(S->getHandler(I)); + // Counter tracks the continuation block of the try statement. + setCount(PGO.getRegionCount(S)); + RecordNextStmtCount = true; + } - void VisitAbstractConditionalOperator( - const AbstractConditionalOperator *E) { - RecordStmtCount(E); - // Counter tracks the "true" part of a conditional operator. The - // count in the "false" part will be calculated from this counter. - RegionCounter Cnt(PGO, E); - Visit(E->getCond()); - - Cnt.beginRegion(); - CountMap[E->getTrueExpr()] = PGO.getCurrentRegionCount(); - Visit(E->getTrueExpr()); - Cnt.adjustForControlFlow(); - - Cnt.beginElseRegion(); - CountMap[E->getFalseExpr()] = PGO.getCurrentRegionCount(); - Visit(E->getFalseExpr()); - Cnt.adjustForControlFlow(); - - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } + void VisitCXXCatchStmt(const CXXCatchStmt *S) { + RecordNextStmtCount = false; + // Counter tracks the catch statement's handler block. + uint64_t CatchCount = setCount(PGO.getRegionCount(S)); + CountMap[S] = CatchCount; + Visit(S->getHandlerBlock()); + } - void VisitBinLAnd(const BinaryOperator *E) { - RecordStmtCount(E); - // Counter tracks the right hand side of a logical and operator. - RegionCounter Cnt(PGO, E); - Visit(E->getLHS()); - Cnt.beginRegion(); - CountMap[E->getRHS()] = PGO.getCurrentRegionCount(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } + void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { + RecordStmtCount(E); + uint64_t ParentCount = CurrentCount; + Visit(E->getCond()); + + // Counter tracks the "true" part of a conditional operator. The + // count in the "false" part will be calculated from this counter. + uint64_t TrueCount = setCount(PGO.getRegionCount(E)); + CountMap[E->getTrueExpr()] = TrueCount; + Visit(E->getTrueExpr()); + uint64_t OutCount = CurrentCount; + + uint64_t FalseCount = setCount(ParentCount - TrueCount); + CountMap[E->getFalseExpr()] = FalseCount; + Visit(E->getFalseExpr()); + OutCount += CurrentCount; + + setCount(OutCount); + RecordNextStmtCount = true; + } - void VisitBinLOr(const BinaryOperator *E) { - RecordStmtCount(E); - // Counter tracks the right hand side of a logical or operator. - RegionCounter Cnt(PGO, E); - Visit(E->getLHS()); - Cnt.beginRegion(); - CountMap[E->getRHS()] = PGO.getCurrentRegionCount(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(0); - RecordNextStmtCount = true; - } - }; + void VisitBinLAnd(const BinaryOperator *E) { + RecordStmtCount(E); + uint64_t ParentCount = CurrentCount; + Visit(E->getLHS()); + // Counter tracks the right hand side of a logical and operator. + uint64_t RHSCount = setCount(PGO.getRegionCount(E)); + CountMap[E->getRHS()] = RHSCount; + Visit(E->getRHS()); + setCount(ParentCount + RHSCount - CurrentCount); + RecordNextStmtCount = true; + } + + void VisitBinLOr(const BinaryOperator *E) { + RecordStmtCount(E); + uint64_t ParentCount = CurrentCount; + Visit(E->getLHS()); + // Counter tracks the right hand side of a logical or operator. + uint64_t RHSCount = setCount(PGO.getRegionCount(E)); + CountMap[E->getRHS()] = RHSCount; + Visit(E->getRHS()); + setCount(ParentCount + RHSCount - CurrentCount); + RecordNextStmtCount = true; + } +}; } void PGOHash::combine(HashType Type) { @@ -728,12 +719,10 @@ void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { } void -CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef FuncName, +CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef Name, llvm::GlobalValue::LinkageTypes Linkage) { if (SkipCoverageMapping) return; - setFuncName(FuncName, Linkage); - // Don't map the functions inside the system headers auto Loc = D->getBody()->getLocStart(); if (CGM.getContext().getSourceManager().isInSystemHeader(Loc)) @@ -750,6 +739,7 @@ CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef FuncName, if (CoverageMapping.empty()) return; + setFuncName(Name, Linkage); CGM.getCoverageMapping()->addFunctionMappingRecord( FuncNameVar, FuncName, FunctionHash, CoverageMapping); } @@ -785,17 +775,19 @@ CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, Fn->addFnAttr(llvm::Attribute::Cold); } -void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter) { +void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) { if (!CGM.getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap) return; if (!Builder.GetInsertPoint()) return; + + unsigned Counter = (*RegionCounterMap)[S]; auto *I8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext()); - Builder.CreateCall4(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), - llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), + {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FunctionHash), Builder.getInt32(NumRegionCounters), - Builder.getInt32(Counter)); + Builder.getInt32(Counter)}); } void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, @@ -839,8 +831,8 @@ static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale) { return Scaled; } -llvm::MDNode *CodeGenPGO::createBranchWeights(uint64_t TrueCount, - uint64_t FalseCount) { +llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount, + uint64_t FalseCount) { // Check for empty weights. if (!TrueCount && !FalseCount) return nullptr; @@ -853,7 +845,8 @@ llvm::MDNode *CodeGenPGO::createBranchWeights(uint64_t TrueCount, scaleBranchWeight(FalseCount, Scale)); } -llvm::MDNode *CodeGenPGO::createBranchWeights(ArrayRef<uint64_t> Weights) { +llvm::MDNode * +CodeGenFunction::createProfileWeights(ArrayRef<uint64_t> Weights) { // We need at least two elements to create meaningful weights. if (Weights.size() < 2) return nullptr; @@ -875,17 +868,14 @@ llvm::MDNode *CodeGenPGO::createBranchWeights(ArrayRef<uint64_t> Weights) { return MDHelper.createBranchWeights(ScaledWeights); } -llvm::MDNode *CodeGenPGO::createLoopWeights(const Stmt *Cond, - RegionCounter &Cnt) { - if (!haveRegionCounts()) +llvm::MDNode *CodeGenFunction::createProfileWeightsForLoop(const Stmt *Cond, + uint64_t LoopCount) { + if (!PGO.haveRegionCounts()) return nullptr; - uint64_t LoopCount = Cnt.getCount(); - uint64_t CondCount = 0; - bool Found = getStmtCount(Cond, CondCount); - assert(Found && "missing expected loop condition count"); - (void)Found; - if (CondCount == 0) + Optional<uint64_t> CondCount = PGO.getStmtCount(Cond); + assert(CondCount.hasValue() && "missing expected loop condition count"); + if (*CondCount == 0) return nullptr; - return createBranchWeights(LoopCount, - std::max(CondCount, LoopCount) - LoopCount); + return createProfileWeights(LoopCount, + std::max(*CondCount, LoopCount) - LoopCount); } diff --git a/lib/CodeGen/CodeGenPGO.h b/lib/CodeGen/CodeGenPGO.h index 431c850ef81e..de6f369fb351 100644 --- a/lib/CodeGen/CodeGenPGO.h +++ b/lib/CodeGen/CodeGenPGO.h @@ -24,10 +24,8 @@ namespace clang { namespace CodeGen { -class RegionCounter; -/// Per-function PGO state. This class should generally not be used directly, -/// but instead through the CodeGenFunction and RegionCounter types. +/// Per-function PGO state. class CodeGenPGO { private: CodeGenModule &CGM; @@ -62,37 +60,24 @@ public: /// exits. void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } - /// Indicate that the current region is never reached, and thus should have a - /// counter value of zero. This is important so that subsequent regions can - /// correctly track their parent counts. - void setCurrentRegionUnreachable() { setCurrentRegionCount(0); } - /// Check if an execution count is known for a given statement. If so, return /// true and put the value in Count; else return false. - bool getStmtCount(const Stmt *S, uint64_t &Count) { + Optional<uint64_t> getStmtCount(const Stmt *S) { if (!StmtCountMap) - return false; - llvm::DenseMap<const Stmt*, uint64_t>::const_iterator - I = StmtCountMap->find(S); + return None; + auto I = StmtCountMap->find(S); if (I == StmtCountMap->end()) - return false; - Count = I->second; - return true; + return None; + return I->second; } /// If the execution count for the current statement is known, record that /// as the current count. void setCurrentStmt(const Stmt *S) { - uint64_t Count; - if (getStmtCount(S, Count)) - setCurrentRegionCount(Count); + if (auto Count = getStmtCount(S)) + setCurrentRegionCount(*Count); } - /// Calculate branch weights appropriate for PGO data - llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount); - llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights); - llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt); - /// Check if we need to emit coverage mapping for a given declaration void checkGlobalDecl(GlobalDecl GD); /// Assign counters to regions and configure them for PGO of a given @@ -117,110 +102,16 @@ private: void emitCounterVariables(); void emitCounterRegionMapping(const Decl *D); - /// Emit code to increment the counter at the given index - void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter); +public: + void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S); - /// Return the region counter for the given statement. This should only be - /// called on statements that have a dedicated counter. - unsigned getRegionCounter(const Stmt *S) { + /// Return the region count for the counter at the given index. + uint64_t getRegionCount(const Stmt *S) { if (!RegionCounterMap) return 0; - return (*RegionCounterMap)[S]; - } - - /// Return the region count for the counter at the given index. - uint64_t getRegionCount(unsigned Counter) { if (!haveRegionCounts()) return 0; - return RegionCounts[Counter]; - } - - friend class RegionCounter; -}; - -/// A counter for a particular region. This is the primary interface through -/// which clients manage PGO counters and their values. -class RegionCounter { - CodeGenPGO *PGO; - unsigned Counter; - uint64_t Count; - uint64_t ParentCount; - uint64_t RegionCount; - int64_t Adjust; - - RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex) - : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)), - ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} - -public: - RegionCounter(CodeGenPGO &PGO, const Stmt *S) - : PGO(&PGO), Counter(PGO.getRegionCounter(S)), - Count(PGO.getRegionCount(Counter)), - ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} - - /// Get the value of the counter. In most cases this is the number of times - /// the region of the counter was entered, but for switch labels it's the - /// number of direct jumps to that label. - uint64_t getCount() const { return Count; } - - /// Get the value of the counter with adjustments applied. Adjustments occur - /// when control enters or leaves the region abnormally; i.e., if there is a - /// jump to a label within the region, or if the function can return from - /// within the region. The adjusted count, then, is the value of the counter - /// at the end of the region. - uint64_t getAdjustedCount() const { - return Count + Adjust; - } - - /// Get the value of the counter in this region's parent, i.e., the region - /// that was active when this region began. This is useful for deriving - /// counts in implicitly counted regions, like the false case of a condition - /// or the normal exits of a loop. - uint64_t getParentCount() const { return ParentCount; } - - /// Activate the counter by emitting an increment and starting to track - /// adjustments. If AddIncomingFallThrough is true, the current region count - /// will be added to the counter for the purposes of tracking the region. - void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) { - beginRegion(AddIncomingFallThrough); - PGO->emitCounterIncrement(Builder, Counter); - } - void beginRegion(bool AddIncomingFallThrough=false) { - RegionCount = Count; - if (AddIncomingFallThrough) - RegionCount += PGO->getCurrentRegionCount(); - PGO->setCurrentRegionCount(RegionCount); - } - - /// For counters on boolean branches, begins tracking adjustments for the - /// uncounted path. - void beginElseRegion() { - RegionCount = ParentCount - Count; - PGO->setCurrentRegionCount(RegionCount); - } - - /// Reset the current region count. - void setCurrentRegionCount(uint64_t CurrentCount) { - RegionCount = CurrentCount; - PGO->setCurrentRegionCount(RegionCount); - } - - /// Adjust for non-local control flow after emitting a subexpression or - /// substatement. This must be called to account for constructs such as gotos, - /// labels, and returns, so that we can ensure that our region's count is - /// correct in the code that follows. - void adjustForControlFlow() { - Adjust += PGO->getCurrentRegionCount() - RegionCount; - // Reset the region count in case this is called again later. - RegionCount = PGO->getCurrentRegionCount(); - } - - /// Commit all adjustments to the current region. If the region is a loop, - /// the LoopAdjust value should be the count of all the breaks and continues - /// from the loop, to compensate for those counts being deducted from the - /// adjustments for the body of the loop. - void applyAdjustmentsToRegion(uint64_t LoopAdjust) { - PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust); + return RegionCounts[(*RegionCounterMap)[S]]; } }; diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp index 67a9fbec2690..e0f926cabd71 100644 --- a/lib/CodeGen/CodeGenTypes.cpp +++ b/lib/CodeGen/CodeGenTypes.cpp @@ -715,9 +715,16 @@ bool CodeGenTypes::isZeroInitializable(QualType T) { // No need to check for member pointers when not compiling C++. if (!Context.getLangOpts().CPlusPlus) return true; - - T = Context.getBaseElementType(T); - + + if (const auto *AT = Context.getAsArrayType(T)) { + if (isa<IncompleteArrayType>(AT)) + return true; + if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) + if (Context.getConstantArrayElementCount(CAT) == 0) + return true; + T = Context.getBaseElementType(T); + } + // Records are non-zero-initializable if they contain any // non-zero-initializable subobjects. if (const RecordType *RT = T->getAs<RecordType>()) { @@ -733,6 +740,6 @@ bool CodeGenTypes::isZeroInitializable(QualType T) { return true; } -bool CodeGenTypes::isZeroInitializable(const CXXRecordDecl *RD) { +bool CodeGenTypes::isZeroInitializable(const RecordDecl *RD) { return getCGRecordLayout(RD).isZeroInitializable(); } diff --git a/lib/CodeGen/CodeGenTypes.h b/lib/CodeGen/CodeGenTypes.h index 64c5799ccec9..1580e21d11dc 100644 --- a/lib/CodeGen/CodeGenTypes.h +++ b/lib/CodeGen/CodeGenTypes.h @@ -82,6 +82,9 @@ inline StructorType getFromCtorType(CXXCtorType T) { return StructorType::Base; case Ctor_Comdat: llvm_unreachable("not expecting a COMDAT"); + case Ctor_CopyingClosure: + case Ctor_DefaultClosure: + llvm_unreachable("not expecting a closure"); } llvm_unreachable("not a CXXCtorType"); } @@ -112,8 +115,8 @@ inline StructorType getFromDtorType(CXXDtorType T) { llvm_unreachable("not a CXXDtorType"); } -/// CodeGenTypes - This class organizes the cross-module state that is used -/// while lowering AST types to LLVM types. +/// This class organizes the cross-module state that is used while lowering +/// AST types to LLVM types. class CodeGenTypes { CodeGenModule &CGM; // Some of this stuff should probably be left on the CGM. @@ -133,34 +136,32 @@ class CodeGenTypes { /// types are never refined. llvm::DenseMap<const ObjCInterfaceType*, llvm::Type *> InterfaceTypes; - /// CGRecordLayouts - This maps llvm struct type with corresponding - /// record layout info. + /// Maps clang struct type with corresponding record layout info. llvm::DenseMap<const Type*, CGRecordLayout *> CGRecordLayouts; - /// RecordDeclTypes - This contains the LLVM IR type for any converted - /// RecordDecl. + /// Contains the LLVM IR type for any converted RecordDecl. llvm::DenseMap<const Type*, llvm::StructType *> RecordDeclTypes; - /// FunctionInfos - Hold memoized CGFunctionInfo results. + /// Hold memoized CGFunctionInfo results. llvm::FoldingSet<CGFunctionInfo> FunctionInfos; - /// RecordsBeingLaidOut - This set keeps track of records that we're currently - /// converting to an IR type. For example, when converting: + /// This set keeps track of records that we're currently converting + /// to an IR type. For example, when converting: /// struct A { struct B { int x; } } when processing 'x', the 'A' and 'B' /// types will be in this set. llvm::SmallPtrSet<const Type*, 4> RecordsBeingLaidOut; llvm::SmallPtrSet<const CGFunctionInfo*, 4> FunctionsBeingProcessed; - /// SkippedLayout - True if we didn't layout a function due to a being inside + /// True if we didn't layout a function due to a being inside /// a recursive struct conversion, set this to true. bool SkippedLayout; SmallVector<const RecordDecl *, 8> DeferredRecords; private: - /// TypeCache - This map keeps cache of llvm::Types - /// and maps clang::Type to corresponding llvm::Type. + /// This map keeps cache of llvm::Types and maps clang::Type to + /// corresponding llvm::Type. llvm::DenseMap<const Type *, llvm::Type *> TypeCache; public: @@ -261,6 +262,8 @@ public: const FunctionProtoType *type, RequiredArgs required); const CGFunctionInfo &arrangeMSMemberPointerThunk(const CXXMethodDecl *MD); + const CGFunctionInfo &arrangeMSCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT); const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionProtoType> Ty); const CGFunctionInfo &arrangeFreeFunctionType(CanQual<FunctionNoProtoType> Ty); @@ -305,7 +308,7 @@ public: // These are internal details of CGT that shouldn't be used externally. /// IsZeroInitializable - Return whether a record type can be /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer. - bool isZeroInitializable(const CXXRecordDecl *RD); + bool isZeroInitializable(const RecordDecl *RD); bool isRecordLayoutComplete(const Type *Ty) const; bool noRecordsBeingLaidOut() const { diff --git a/lib/CodeGen/CoverageMappingGen.cpp b/lib/CodeGen/CoverageMappingGen.cpp index 6f159d47375e..024a45dba2b7 100644 --- a/lib/CodeGen/CoverageMappingGen.cpp +++ b/lib/CodeGen/CoverageMappingGen.cpp @@ -15,6 +15,7 @@ #include "CodeGenFunction.h" #include "clang/AST/StmtVisitor.h" #include "clang/Lex/Lexer.h" +#include "llvm/ADT/Optional.h" #include "llvm/ProfileData/CoverageMapping.h" #include "llvm/ProfileData/CoverageMappingReader.h" #include "llvm/ProfileData/CoverageMappingWriter.h" @@ -33,87 +34,50 @@ namespace { /// \brief A region of source code that can be mapped to a counter. class SourceMappingRegion { -public: - enum RegionFlags { - /// \brief This region won't be emitted if it wasn't extended. - /// This is useful so that we won't emit source ranges for single tokens - /// that we don't really care that much about, like: - /// the '(' token in #define MACRO ( - IgnoreIfNotExtended = 0x0001, - }; - -private: - FileID File, MacroArgumentFile; - Counter Count; - /// \brief A statement that initiated the count of Zero. - /// - /// This initiator statement is useful to prevent merging of unreachable - /// regions with different statements that caused the counter to become - /// unreachable. - const Stmt *UnreachableInitiator; - - /// \brief A statement that separates certain mapping regions into groups. - /// - /// The group statement is sometimes useful when we are emitting the source - /// regions not in their correct lexical order, e.g. the regions for the - /// incrementation expression in the 'for' construct. By marking the regions - /// in the incrementation expression with the group statement, we avoid the - /// merging of the regions from the incrementation expression and the loop's - /// body. - const Stmt *Group; - /// \brief The region's starting location. - SourceLocation LocStart; + Optional<SourceLocation> LocStart; /// \brief The region's ending location. - SourceLocation LocEnd, AlternativeLocEnd; - unsigned Flags; + Optional<SourceLocation> LocEnd; public: - SourceMappingRegion(FileID File, FileID MacroArgumentFile, Counter Count, - const Stmt *UnreachableInitiator, const Stmt *Group, - SourceLocation LocStart, SourceLocation LocEnd, - unsigned Flags = 0) - : File(File), MacroArgumentFile(MacroArgumentFile), Count(Count), - UnreachableInitiator(UnreachableInitiator), Group(Group), - LocStart(LocStart), LocEnd(LocEnd), AlternativeLocEnd(LocStart), - Flags(Flags) {} + SourceMappingRegion(Counter Count, Optional<SourceLocation> LocStart, + Optional<SourceLocation> LocEnd) + : Count(Count), LocStart(LocStart), LocEnd(LocEnd) {} + + SourceMappingRegion(SourceMappingRegion &&Region) + : Count(std::move(Region.Count)), LocStart(std::move(Region.LocStart)), + LocEnd(std::move(Region.LocEnd)) {} - const FileID &getFile() const { return File; } + SourceMappingRegion &operator=(SourceMappingRegion &&RHS) { + Count = std::move(RHS.Count); + LocStart = std::move(RHS.LocStart); + LocEnd = std::move(RHS.LocEnd); + return *this; + } const Counter &getCounter() const { return Count; } - const SourceLocation &getStartLoc() const { return LocStart; } + void setCounter(Counter C) { Count = C; } - const SourceLocation &getEndLoc(const SourceManager &SM) const { - if (SM.getFileID(LocEnd) != File) - return AlternativeLocEnd; - return LocEnd; - } + bool hasStartLoc() const { return LocStart.hasValue(); } - bool hasFlag(RegionFlags Flag) const { return (Flags & Flag) != 0; } + void setStartLoc(SourceLocation Loc) { LocStart = Loc; } - void setFlag(RegionFlags Flag) { Flags |= Flag; } + const SourceLocation &getStartLoc() const { + assert(LocStart && "Region has no start location"); + return *LocStart; + } - void clearFlag(RegionFlags Flag) { Flags &= ~Flag; } + bool hasEndLoc() const { return LocEnd.hasValue(); } - /// \brief Return true if two regions can be merged together. - bool isMergeable(SourceMappingRegion &R) { - // FIXME: We allow merging regions with a gap in between them. Should we? - return File == R.File && MacroArgumentFile == R.MacroArgumentFile && - Count == R.Count && UnreachableInitiator == R.UnreachableInitiator && - Group == R.Group; - } + void setEndLoc(SourceLocation Loc) { LocEnd = Loc; } - /// \brief A comparison that sorts such that mergeable regions are adjacent. - friend bool operator<(const SourceMappingRegion &LHS, - const SourceMappingRegion &RHS) { - return std::tie(LHS.File, LHS.MacroArgumentFile, LHS.Count, - LHS.UnreachableInitiator, LHS.Group) < - std::tie(RHS.File, RHS.MacroArgumentFile, RHS.Count, - RHS.UnreachableInitiator, RHS.Group); + const SourceLocation &getEndLoc() const { + assert(LocEnd && "Region has no end location"); + return *LocEnd; } }; @@ -126,26 +90,11 @@ public: const LangOptions &LangOpts; private: - struct FileInfo { - /// \brief The file id that will be used by the coverage mapping system. - unsigned CovMappingFileID; - const FileEntry *Entry; - - FileInfo(unsigned CovMappingFileID, const FileEntry *Entry) - : CovMappingFileID(CovMappingFileID), Entry(Entry) {} - }; - - /// \brief This mapping maps clang's FileIDs to file ids used - /// by the coverage mapping system and clang's file entries. - llvm::SmallDenseMap<FileID, FileInfo, 8> FileIDMapping; + /// \brief Map of clang's FileIDs to IDs used for coverage mapping. + llvm::SmallDenseMap<FileID, std::pair<unsigned, SourceLocation>, 8> + FileIDMapping; public: - /// \brief The statement that corresponds to the current source group. - const Stmt *CurrentSourceGroup; - - /// \brief The statement the initiated the current unreachable region. - const Stmt *CurrentUnreachableRegionInitiator; - /// \brief The coverage mapping regions for this function llvm::SmallVector<CounterMappingRegion, 32> MappingRegions; /// \brief The source mapping regions for this function. @@ -153,60 +102,104 @@ public: CoverageMappingBuilder(CoverageMappingModuleGen &CVM, SourceManager &SM, const LangOptions &LangOpts) - : CVM(CVM), SM(SM), LangOpts(LangOpts), - CurrentSourceGroup(nullptr), - CurrentUnreachableRegionInitiator(nullptr) {} + : CVM(CVM), SM(SM), LangOpts(LangOpts) {} /// \brief Return the precise end location for the given token. SourceLocation getPreciseTokenLocEnd(SourceLocation Loc) { - return Lexer::getLocForEndOfToken(SM.getSpellingLoc(Loc), 0, SM, LangOpts); + // We avoid getLocForEndOfToken here, because it doesn't do what we want for + // macro locations, which we just treat as expanded files. + unsigned TokLen = + Lexer::MeasureTokenLength(SM.getSpellingLoc(Loc), SM, LangOpts); + return Loc.getLocWithOffset(TokLen); } - /// \brief Create the mapping that maps from the function's file ids to - /// the indices for the translation unit's filenames. - void createFileIDMapping(SmallVectorImpl<unsigned> &Mapping) { - Mapping.resize(FileIDMapping.size(), 0); - for (const auto &I : FileIDMapping) - Mapping[I.second.CovMappingFileID] = CVM.getFileID(I.second.Entry); + /// \brief Return the start location of an included file or expanded macro. + SourceLocation getStartOfFileOrMacro(SourceLocation Loc) { + if (Loc.isMacroID()) + return Loc.getLocWithOffset(-SM.getFileOffset(Loc)); + return SM.getLocForStartOfFile(SM.getFileID(Loc)); } - /// \brief Get the coverage mapping file id that corresponds to the given - /// clang file id. If such file id doesn't exist, it gets added to the - /// mapping that maps from clang's file ids to coverage mapping file ids. - /// Return true if there was an error getting the coverage mapping file id. - /// An example of an when this function fails is when the region tries - /// to get a coverage file id for a location in a built-in macro. - bool getCoverageFileID(SourceLocation LocStart, FileID File, - FileID SpellingFile, unsigned &Result) { - auto Mapping = FileIDMapping.find(File); - if (Mapping != FileIDMapping.end()) { - Result = Mapping->second.CovMappingFileID; - return false; - } + /// \brief Return the end location of an included file or expanded macro. + SourceLocation getEndOfFileOrMacro(SourceLocation Loc) { + if (Loc.isMacroID()) + return Loc.getLocWithOffset(SM.getFileIDSize(SM.getFileID(Loc)) - + SM.getFileOffset(Loc)); + return SM.getLocForEndOfFile(SM.getFileID(Loc)); + } + + /// \brief Find out where the current file is included or macro is expanded. + SourceLocation getIncludeOrExpansionLoc(SourceLocation Loc) { + return Loc.isMacroID() ? SM.getImmediateExpansionRange(Loc).first + : SM.getIncludeLoc(SM.getFileID(Loc)); + } + + /// \brief Return true if \c Loc is a location in a built-in macro. + bool isInBuiltin(SourceLocation Loc) { + return strcmp(SM.getBufferName(SM.getSpellingLoc(Loc)), "<built-in>") == 0; + } - auto Entry = SM.getFileEntryForID(SpellingFile); - if (!Entry) - return true; + /// \brief Get the start of \c S ignoring macro arguments and builtin macros. + SourceLocation getStart(const Stmt *S) { + SourceLocation Loc = S->getLocStart(); + while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc)) + Loc = SM.getImmediateExpansionRange(Loc).first; + return Loc; + } - Result = FileIDMapping.size(); - FileIDMapping.insert(std::make_pair(File, FileInfo(Result, Entry))); - createFileExpansionRegion(LocStart, File); - return false; + /// \brief Get the end of \c S ignoring macro arguments and builtin macros. + SourceLocation getEnd(const Stmt *S) { + SourceLocation Loc = S->getLocEnd(); + while (SM.isMacroArgExpansion(Loc) || isInBuiltin(Loc)) + Loc = SM.getImmediateExpansionRange(Loc).first; + return getPreciseTokenLocEnd(Loc); } - /// \brief Get the coverage mapping file id that corresponds to the given - /// clang file id. - /// Return true if there was an error getting the coverage mapping file id. - bool getExistingCoverageFileID(FileID File, unsigned &Result) { - // Make sure that the file is valid. - if (File.isInvalid()) - return true; - auto Mapping = FileIDMapping.find(File); - if (Mapping != FileIDMapping.end()) { - Result = Mapping->second.CovMappingFileID; - return false; + /// \brief Find the set of files we have regions for and assign IDs + /// + /// Fills \c Mapping with the virtual file mapping needed to write out + /// coverage and collects the necessary file information to emit source and + /// expansion regions. + void gatherFileIDs(SmallVectorImpl<unsigned> &Mapping) { + FileIDMapping.clear(); + + SmallVector<FileID, 8> Visited; + SmallVector<std::pair<SourceLocation, unsigned>, 8> FileLocs; + for (const auto &Region : SourceRegions) { + SourceLocation Loc = Region.getStartLoc(); + FileID File = SM.getFileID(Loc); + if (std::find(Visited.begin(), Visited.end(), File) != Visited.end()) + continue; + Visited.push_back(File); + + unsigned Depth = 0; + for (SourceLocation Parent = getIncludeOrExpansionLoc(Loc); + !Parent.isInvalid(); Parent = getIncludeOrExpansionLoc(Parent)) + ++Depth; + FileLocs.push_back(std::make_pair(Loc, Depth)); } - return true; + std::stable_sort(FileLocs.begin(), FileLocs.end(), llvm::less_second()); + + for (const auto &FL : FileLocs) { + SourceLocation Loc = FL.first; + FileID SpellingFile = SM.getDecomposedSpellingLoc(Loc).first; + auto Entry = SM.getFileEntryForID(SpellingFile); + if (!Entry) + continue; + + FileIDMapping[SM.getFileID(Loc)] = std::make_pair(Mapping.size(), Loc); + Mapping.push_back(CVM.getFileID(Entry)); + } + } + + /// \brief Get the coverage mapping file ID for \c Loc. + /// + /// If such file id doesn't exist, return None. + Optional<unsigned> getCoverageFileID(SourceLocation Loc) { + auto Mapping = FileIDMapping.find(SM.getFileID(Loc)); + if (Mapping != FileIDMapping.end()) + return Mapping->second.first; + return None; } /// \brief Return true if the given clang's file id has a corresponding @@ -235,154 +228,83 @@ public: for (const auto &I : SkippedRanges) { auto LocStart = I.getBegin(); auto LocEnd = I.getEnd(); - auto FileStart = SM.getFileID(LocStart); - if (!hasExistingCoverageFileID(FileStart)) - continue; - auto ActualFileStart = SM.getDecomposedSpellingLoc(LocStart).first; - if (ActualFileStart != SM.getDecomposedSpellingLoc(LocEnd).first) - // Ignore regions that span across multiple files. - continue; + assert(SM.isWrittenInSameFile(LocStart, LocEnd) && + "region spans multiple files"); - unsigned CovFileID; - if (getCoverageFileID(LocStart, FileStart, ActualFileStart, CovFileID)) + auto CovFileID = getCoverageFileID(LocStart); + if (!CovFileID) continue; unsigned LineStart = SM.getSpellingLineNumber(LocStart); unsigned ColumnStart = SM.getSpellingColumnNumber(LocStart); unsigned LineEnd = SM.getSpellingLineNumber(LocEnd); unsigned ColumnEnd = SM.getSpellingColumnNumber(LocEnd); - CounterMappingRegion Region(Counter(), CovFileID, LineStart, ColumnStart, - LineEnd, ColumnEnd, false, - CounterMappingRegion::SkippedRegion); + auto Region = CounterMappingRegion::makeSkipped( + *CovFileID, LineStart, ColumnStart, LineEnd, ColumnEnd); // Make sure that we only collect the regions that are inside // the souce code of this function. - if (Region.LineStart >= FileLineRanges[CovFileID].first && - Region.LineEnd <= FileLineRanges[CovFileID].second) + if (Region.LineStart >= FileLineRanges[*CovFileID].first && + Region.LineEnd <= FileLineRanges[*CovFileID].second) MappingRegions.push_back(Region); } } - /// \brief Create a mapping region that correponds to an expansion of - /// a macro or an embedded include. - void createFileExpansionRegion(SourceLocation Loc, FileID ExpandedFile) { - SourceLocation LocStart; - if (Loc.isMacroID()) - LocStart = SM.getImmediateExpansionRange(Loc).first; - else { - LocStart = SM.getIncludeLoc(ExpandedFile); - if (LocStart.isInvalid()) - return; // This file has no expansion region. - } - - auto File = SM.getFileID(LocStart); - auto SpellingFile = SM.getDecomposedSpellingLoc(LocStart).first; - unsigned CovFileID, ExpandedFileID; - if (getExistingCoverageFileID(ExpandedFile, ExpandedFileID)) - return; - if (getCoverageFileID(LocStart, File, SpellingFile, CovFileID)) - return; - unsigned LineStart = SM.getSpellingLineNumber(LocStart); - unsigned ColumnStart = SM.getSpellingColumnNumber(LocStart); - unsigned LineEnd = LineStart; - // Compute the end column manually as Lexer::getLocForEndOfToken doesn't - // give the correct result in all cases. - unsigned ColumnEnd = - ColumnStart + - Lexer::MeasureTokenLength(SM.getSpellingLoc(LocStart), SM, LangOpts); - - MappingRegions.push_back(CounterMappingRegion( - Counter(), CovFileID, LineStart, ColumnStart, LineEnd, ColumnEnd, - false, CounterMappingRegion::ExpansionRegion)); - MappingRegions.back().ExpandedFileID = ExpandedFileID; - } - - /// \brief Enter a source region group that is identified by the given - /// statement. - /// It's not possible to enter a group when there is already - /// another group present. - void beginSourceRegionGroup(const Stmt *Group) { - assert(!CurrentSourceGroup); - CurrentSourceGroup = Group; - } - - /// \brief Exit the current source region group. - void endSourceRegionGroup() { CurrentSourceGroup = nullptr; } - - /// \brief Associate a counter with a given source code range. - void mapSourceCodeRange(SourceLocation LocStart, SourceLocation LocEnd, - Counter Count, const Stmt *UnreachableInitiator, - const Stmt *SourceGroup, unsigned Flags = 0, - FileID MacroArgumentFile = FileID()) { - if (SM.isMacroArgExpansion(LocStart)) { - // Map the code range with the macro argument's value. - mapSourceCodeRange(SM.getImmediateSpellingLoc(LocStart), - SM.getImmediateSpellingLoc(LocEnd), Count, - UnreachableInitiator, SourceGroup, Flags, - SM.getFileID(LocStart)); - // Map the code range where the macro argument is referenced. - SourceLocation RefLocStart(SM.getImmediateExpansionRange(LocStart).first); - SourceLocation RefLocEnd(RefLocStart); - if (SM.isMacroArgExpansion(RefLocStart)) - mapSourceCodeRange(RefLocStart, RefLocEnd, Count, UnreachableInitiator, - SourceGroup, 0, SM.getFileID(RefLocStart)); - else - mapSourceCodeRange(RefLocStart, RefLocEnd, Count, UnreachableInitiator, - SourceGroup); - return; - } - auto File = SM.getFileID(LocStart); - // Make sure that the file id is valid. - if (File.isInvalid()) - return; - SourceRegions.emplace_back(File, MacroArgumentFile, Count, - UnreachableInitiator, SourceGroup, LocStart, - LocEnd, Flags); - } - - void mapSourceCodeRange(SourceLocation LocStart, SourceLocation LocEnd, - Counter Count, unsigned Flags = 0) { - mapSourceCodeRange(LocStart, LocEnd, Count, - CurrentUnreachableRegionInitiator, CurrentSourceGroup, - Flags); - } - /// \brief Generate the coverage counter mapping regions from collected /// source regions. void emitSourceRegions() { - std::sort(SourceRegions.begin(), SourceRegions.end()); - - for (auto I = SourceRegions.begin(), E = SourceRegions.end(); I != E; ++I) { - // Keep the original start location of this region. - SourceLocation LocStart = I->getStartLoc(); - SourceLocation LocEnd = I->getEndLoc(SM); - - bool Ignore = I->hasFlag(SourceMappingRegion::IgnoreIfNotExtended); - // We need to handle mergeable regions together. - for (auto Next = I + 1; Next != E && Next->isMergeable(*I); ++Next) { - ++I; - LocStart = std::min(LocStart, I->getStartLoc()); - LocEnd = std::max(LocEnd, I->getEndLoc(SM)); - // FIXME: Should we && together the Ignore flag of multiple regions? - Ignore = false; - } - if (Ignore) + for (const auto &Region : SourceRegions) { + assert(Region.hasEndLoc() && "incomplete region"); + + SourceLocation LocStart = Region.getStartLoc(); + assert(!SM.getFileID(LocStart).isInvalid() && "region in invalid file"); + + auto CovFileID = getCoverageFileID(LocStart); + // Ignore regions that don't have a file, such as builtin macros. + if (!CovFileID) continue; + SourceLocation LocEnd = Region.getEndLoc(); + assert(SM.isWrittenInSameFile(LocStart, LocEnd) && + "region spans multiple files"); + // Find the spilling locations for the mapping region. - LocEnd = getPreciseTokenLocEnd(LocEnd); unsigned LineStart = SM.getSpellingLineNumber(LocStart); unsigned ColumnStart = SM.getSpellingColumnNumber(LocStart); unsigned LineEnd = SM.getSpellingLineNumber(LocEnd); unsigned ColumnEnd = SM.getSpellingColumnNumber(LocEnd); - auto SpellingFile = SM.getDecomposedSpellingLoc(LocStart).first; - unsigned CovFileID; - if (getCoverageFileID(LocStart, I->getFile(), SpellingFile, CovFileID)) + assert(LineStart <= LineEnd && "region start and end out of order"); + MappingRegions.push_back(CounterMappingRegion::makeRegion( + Region.getCounter(), *CovFileID, LineStart, ColumnStart, LineEnd, + ColumnEnd)); + } + } + + /// \brief Generate expansion regions for each virtual file we've seen. + void emitExpansionRegions() { + for (const auto &FM : FileIDMapping) { + SourceLocation ExpandedLoc = FM.second.second; + SourceLocation ParentLoc = getIncludeOrExpansionLoc(ExpandedLoc); + if (ParentLoc.isInvalid()) + continue; + + auto ParentFileID = getCoverageFileID(ParentLoc); + if (!ParentFileID) continue; + auto ExpandedFileID = getCoverageFileID(ExpandedLoc); + assert(ExpandedFileID && "expansion in uncovered file"); + + SourceLocation LocEnd = getPreciseTokenLocEnd(ParentLoc); + assert(SM.isWrittenInSameFile(ParentLoc, LocEnd) && + "region spans multiple files"); - assert(LineStart <= LineEnd); - MappingRegions.push_back(CounterMappingRegion( - I->getCounter(), CovFileID, LineStart, ColumnStart, LineEnd, - ColumnEnd, false, CounterMappingRegion::CodeRegion)); + unsigned LineStart = SM.getSpellingLineNumber(ParentLoc); + unsigned ColumnStart = SM.getSpellingColumnNumber(ParentLoc); + unsigned LineEnd = SM.getSpellingLineNumber(LocEnd); + unsigned ColumnEnd = SM.getSpellingColumnNumber(LocEnd); + + MappingRegions.push_back(CounterMappingRegion::makeExpansion( + *ParentFileID, *ExpandedFileID, LineStart, ColumnStart, LineEnd, + ColumnEnd)); } } }; @@ -398,14 +320,14 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder { if (!D->hasBody()) return; auto Body = D->getBody(); - mapSourceCodeRange(Body->getLocStart(), Body->getLocEnd(), Counter()); + SourceRegions.emplace_back(Counter(), getStart(Body), getEnd(Body)); } /// \brief Write the mapping data to the output stream void write(llvm::raw_ostream &OS) { - emitSourceRegions(); SmallVector<unsigned, 16> FileIDMapping; - createFileIDMapping(FileIDMapping); + gatherFileIDs(FileIDMapping); + emitSourceRegions(); CoverageMappingWriter Writer(FileIDMapping, None, MappingRegions); Writer.write(OS); @@ -420,136 +342,215 @@ struct CounterCoverageMappingBuilder /// \brief The map of statements to count values. llvm::DenseMap<const Stmt *, unsigned> &CounterMap; - Counter CurrentRegionCount; + /// \brief A stack of currently live regions. + std::vector<SourceMappingRegion> RegionStack; CounterExpressionBuilder Builder; - /// \brief Return a counter that represents the - /// expression that subracts rhs from lhs. + /// \brief A location in the most recently visited file or macro. + /// + /// This is used to adjust the active source regions appropriately when + /// expressions cross file or macro boundaries. + SourceLocation MostRecentLocation; + + /// \brief Return a counter for the subtraction of \c RHS from \c LHS Counter subtractCounters(Counter LHS, Counter RHS) { return Builder.subtract(LHS, RHS); } - /// \brief Return a counter that represents the - /// the exression that adds lhs and rhs. + /// \brief Return a counter for the sum of \c LHS and \c RHS. Counter addCounters(Counter LHS, Counter RHS) { return Builder.add(LHS, RHS); } + Counter addCounters(Counter C1, Counter C2, Counter C3) { + return addCounters(addCounters(C1, C2), C3); + } + + Counter addCounters(Counter C1, Counter C2, Counter C3, Counter C4) { + return addCounters(addCounters(C1, C2, C3), C4); + } + /// \brief Return the region counter for the given statement. + /// /// This should only be called on statements that have a dedicated counter. - unsigned getRegionCounter(const Stmt *S) { return CounterMap[S]; } + Counter getRegionCounter(const Stmt *S) { + return Counter::getCounter(CounterMap[S]); + } + + /// \brief Push a region onto the stack. + /// + /// Returns the index on the stack where the region was pushed. This can be + /// used with popRegions to exit a "scope", ending the region that was pushed. + size_t pushRegion(Counter Count, Optional<SourceLocation> StartLoc = None, + Optional<SourceLocation> EndLoc = None) { + if (StartLoc) + MostRecentLocation = *StartLoc; + RegionStack.emplace_back(Count, StartLoc, EndLoc); - /// \brief Return the region count for the counter at the given index. - Counter getRegionCount(unsigned CounterId) { - return Counter::getCounter(CounterId); + return RegionStack.size() - 1; } - /// \brief Return the counter value of the current region. - Counter getCurrentRegionCount() { return CurrentRegionCount; } + /// \brief Pop regions from the stack into the function's list of regions. + /// + /// Adds all regions from \c ParentIndex to the top of the stack to the + /// function's \c SourceRegions. + void popRegions(size_t ParentIndex) { + assert(RegionStack.size() >= ParentIndex && "parent not in stack"); + while (RegionStack.size() > ParentIndex) { + SourceMappingRegion &Region = RegionStack.back(); + if (Region.hasStartLoc()) { + SourceLocation StartLoc = Region.getStartLoc(); + SourceLocation EndLoc = Region.hasEndLoc() + ? Region.getEndLoc() + : RegionStack[ParentIndex].getEndLoc(); + while (!SM.isWrittenInSameFile(StartLoc, EndLoc)) { + // The region ends in a nested file or macro expansion. Create a + // separate region for each expansion. + SourceLocation NestedLoc = getStartOfFileOrMacro(EndLoc); + assert(SM.isWrittenInSameFile(NestedLoc, EndLoc)); + + SourceRegions.emplace_back(Region.getCounter(), NestedLoc, EndLoc); + + EndLoc = getPreciseTokenLocEnd(getIncludeOrExpansionLoc(EndLoc)); + assert(!EndLoc.isInvalid() && + "File exit was not handled before popRegions"); + } + Region.setEndLoc(EndLoc); + + MostRecentLocation = EndLoc; + // If this region happens to span an entire expansion, we need to make + // sure we don't overlap the parent region with it. + if (StartLoc == getStartOfFileOrMacro(StartLoc) && + EndLoc == getEndOfFileOrMacro(EndLoc)) + MostRecentLocation = getIncludeOrExpansionLoc(EndLoc); + + assert(SM.isWrittenInSameFile(Region.getStartLoc(), EndLoc)); + SourceRegions.push_back(std::move(Region)); + } + RegionStack.pop_back(); + } + } - /// \brief Set the counter value for the current region. - /// This is used to keep track of changes to the most recent counter - /// from control flow and non-local exits. - void setCurrentRegionCount(Counter Count) { - CurrentRegionCount = Count; - CurrentUnreachableRegionInitiator = nullptr; + /// \brief Return the currently active region. + SourceMappingRegion &getRegion() { + assert(!RegionStack.empty() && "statement has no region"); + return RegionStack.back(); } - /// \brief Indicate that the current region is never reached, - /// and thus should have a counter value of zero. - /// This is important so that subsequent regions can correctly track - /// their parent counts. - void setCurrentRegionUnreachable(const Stmt *Initiator) { - CurrentRegionCount = Counter::getZero(); - CurrentUnreachableRegionInitiator = Initiator; + /// \brief Propagate counts through the children of \c S. + Counter propagateCounts(Counter TopCount, const Stmt *S) { + size_t Index = pushRegion(TopCount, getStart(S), getEnd(S)); + Visit(S); + Counter ExitCount = getRegion().getCounter(); + popRegions(Index); + return ExitCount; } - /// \brief A counter for a particular region. - /// This is the primary interface through - /// which the coverage mapping builder manages counters and their values. - class RegionMapper { - CounterCoverageMappingBuilder &Mapping; - Counter Count; - Counter ParentCount; - Counter RegionCount; - Counter Adjust; - - public: - RegionMapper(CounterCoverageMappingBuilder *Mapper, const Stmt *S) - : Mapping(*Mapper), - Count(Mapper->getRegionCount(Mapper->getRegionCounter(S))), - ParentCount(Mapper->getCurrentRegionCount()) {} - - /// Get the value of the counter. In most cases this is the number of times - /// the region of the counter was entered, but for switch labels it's the - /// number of direct jumps to that label. - Counter getCount() const { return Count; } - - /// Get the value of the counter with adjustments applied. Adjustments occur - /// when control enters or leaves the region abnormally; i.e., if there is a - /// jump to a label within the region, or if the function can return from - /// within the region. The adjusted count, then, is the value of the counter - /// at the end of the region. - Counter getAdjustedCount() const { - return Mapping.addCounters(Count, Adjust); - } + /// \brief Adjust the most recently visited location to \c EndLoc. + /// + /// This should be used after visiting any statements in non-source order. + void adjustForOutOfOrderTraversal(SourceLocation EndLoc) { + MostRecentLocation = EndLoc; + // Avoid adding duplicate regions if we have a completed region on the top + // of the stack and are adjusting to the end of a virtual file. + if (getRegion().hasEndLoc() && + MostRecentLocation == getEndOfFileOrMacro(MostRecentLocation)) + MostRecentLocation = getIncludeOrExpansionLoc(MostRecentLocation); + } + + /// \brief Check whether \c Loc is included or expanded from \c Parent. + bool isNestedIn(SourceLocation Loc, FileID Parent) { + do { + Loc = getIncludeOrExpansionLoc(Loc); + if (Loc.isInvalid()) + return false; + } while (!SM.isInFileID(Loc, Parent)); + return true; + } - /// Get the value of the counter in this region's parent, i.e., the region - /// that was active when this region began. This is useful for deriving - /// counts in implicitly counted regions, like the false case of a condition - /// or the normal exits of a loop. - Counter getParentCount() const { return ParentCount; } - - /// Activate the counter by emitting an increment and starting to track - /// adjustments. If AddIncomingFallThrough is true, the current region count - /// will be added to the counter for the purposes of tracking the region. - void beginRegion(bool AddIncomingFallThrough = false) { - RegionCount = Count; - if (AddIncomingFallThrough) - RegionCount = - Mapping.addCounters(RegionCount, Mapping.getCurrentRegionCount()); - Mapping.setCurrentRegionCount(RegionCount); - } + /// \brief Adjust regions and state when \c NewLoc exits a file. + /// + /// If moving from our most recently tracked location to \c NewLoc exits any + /// files, this adjusts our current region stack and creates the file regions + /// for the exited file. + void handleFileExit(SourceLocation NewLoc) { + if (SM.isWrittenInSameFile(MostRecentLocation, NewLoc)) + return; - /// For counters on boolean branches, begins tracking adjustments for the - /// uncounted path. - void beginElseRegion() { - RegionCount = Mapping.subtractCounters(ParentCount, Count); - Mapping.setCurrentRegionCount(RegionCount); + // If NewLoc is not in a file that contains MostRecentLocation, walk up to + // find the common ancestor. + SourceLocation LCA = NewLoc; + FileID ParentFile = SM.getFileID(LCA); + while (!isNestedIn(MostRecentLocation, ParentFile)) { + LCA = getIncludeOrExpansionLoc(LCA); + if (LCA.isInvalid() || SM.isWrittenInSameFile(LCA, MostRecentLocation)) { + // Since there isn't a common ancestor, no file was exited. We just need + // to adjust our location to the new file. + MostRecentLocation = NewLoc; + return; + } + ParentFile = SM.getFileID(LCA); } - /// Reset the current region count. - void setCurrentRegionCount(Counter CurrentCount) { - RegionCount = CurrentCount; - Mapping.setCurrentRegionCount(RegionCount); - } + llvm::SmallSet<SourceLocation, 8> StartLocs; + Optional<Counter> ParentCounter; + for (auto I = RegionStack.rbegin(), E = RegionStack.rend(); I != E; ++I) { + if (!I->hasStartLoc()) + continue; + SourceLocation Loc = I->getStartLoc(); + if (!isNestedIn(Loc, ParentFile)) { + ParentCounter = I->getCounter(); + break; + } - /// Adjust for non-local control flow after emitting a subexpression or - /// substatement. This must be called to account for constructs such as - /// gotos, - /// labels, and returns, so that we can ensure that our region's count is - /// correct in the code that follows. - void adjustForControlFlow() { - Adjust = Mapping.addCounters( - Adjust, Mapping.subtractCounters(Mapping.getCurrentRegionCount(), - RegionCount)); - // Reset the region count in case this is called again later. - RegionCount = Mapping.getCurrentRegionCount(); + while (!SM.isInFileID(Loc, ParentFile)) { + // The most nested region for each start location is the one with the + // correct count. We avoid creating redundant regions by stopping once + // we've seen this region. + if (StartLocs.insert(Loc).second) + SourceRegions.emplace_back(I->getCounter(), Loc, + getEndOfFileOrMacro(Loc)); + Loc = getIncludeOrExpansionLoc(Loc); + } + I->setStartLoc(getPreciseTokenLocEnd(Loc)); } - /// Commit all adjustments to the current region. If the region is a loop, - /// the LoopAdjust value should be the count of all the breaks and continues - /// from the loop, to compensate for those counts being deducted from the - /// adjustments for the body of the loop. - void applyAdjustmentsToRegion() { - Mapping.setCurrentRegionCount(Mapping.addCounters(ParentCount, Adjust)); - } - void applyAdjustmentsToRegion(Counter LoopAdjust) { - Mapping.setCurrentRegionCount(Mapping.addCounters( - Mapping.addCounters(ParentCount, Adjust), LoopAdjust)); + if (ParentCounter) { + // If the file is contained completely by another region and doesn't + // immediately start its own region, the whole file gets a region + // corresponding to the parent. + SourceLocation Loc = MostRecentLocation; + while (isNestedIn(Loc, ParentFile)) { + SourceLocation FileStart = getStartOfFileOrMacro(Loc); + if (StartLocs.insert(FileStart).second) + SourceRegions.emplace_back(*ParentCounter, FileStart, + getEndOfFileOrMacro(Loc)); + Loc = getIncludeOrExpansionLoc(Loc); + } } - }; + + MostRecentLocation = NewLoc; + } + + /// \brief Ensure that \c S is included in the current region. + void extendRegion(const Stmt *S) { + SourceMappingRegion &Region = getRegion(); + SourceLocation StartLoc = getStart(S); + + handleFileExit(StartLoc); + if (!Region.hasStartLoc()) + Region.setStartLoc(StartLoc); + } + + /// \brief Mark \c S as a terminator, starting a zero region. + void terminateRegion(const Stmt *S) { + extendRegion(S); + SourceMappingRegion &Region = getRegion(); + if (!Region.hasEndLoc()) + Region.setEndLoc(getEnd(S)); + pushRegion(Counter::getZero()); + } /// \brief Keep counts of breaks and continues inside loops. struct BreakContinue { @@ -566,437 +567,318 @@ struct CounterCoverageMappingBuilder /// \brief Write the mapping data to the output stream void write(llvm::raw_ostream &OS) { - emitSourceRegions(); llvm::SmallVector<unsigned, 8> VirtualFileMapping; - createFileIDMapping(VirtualFileMapping); + gatherFileIDs(VirtualFileMapping); + emitSourceRegions(); + emitExpansionRegions(); gatherSkippedRegions(); - CoverageMappingWriter Writer( - VirtualFileMapping, Builder.getExpressions(), MappingRegions); + CoverageMappingWriter Writer(VirtualFileMapping, Builder.getExpressions(), + MappingRegions); Writer.write(OS); } - /// \brief Associate the source code range with the current region count. - void mapSourceCodeRange(SourceLocation LocStart, SourceLocation LocEnd, - unsigned Flags = 0) { - CoverageMappingBuilder::mapSourceCodeRange(LocStart, LocEnd, - CurrentRegionCount, Flags); - } - - void mapSourceCodeRange(SourceLocation LocStart) { - CoverageMappingBuilder::mapSourceCodeRange(LocStart, LocStart, - CurrentRegionCount); - } - - /// \brief Associate the source range of a token with the current region - /// count. - /// Ignore the source range for this token if it produces a distinct - /// mapping region with no other source ranges. - void mapToken(SourceLocation LocStart) { - CoverageMappingBuilder::mapSourceCodeRange( - LocStart, LocStart, CurrentRegionCount, - SourceMappingRegion::IgnoreIfNotExtended); - } - void VisitStmt(const Stmt *S) { - mapSourceCodeRange(S->getLocStart()); + if (!S->getLocStart().isInvalid()) + extendRegion(S); for (Stmt::const_child_range I = S->children(); I; ++I) { if (*I) this->Visit(*I); } + handleFileExit(getEnd(S)); } void VisitDecl(const Decl *D) { - if (!D->hasBody()) - return; - // Counter tracks entry to the function body. - auto Body = D->getBody(); - RegionMapper Cnt(this, Body); - Cnt.beginRegion(); - Visit(Body); - } - - void VisitDeclStmt(const DeclStmt *S) { - mapSourceCodeRange(S->getLocStart()); - for (Stmt::const_child_range I = static_cast<const Stmt *>(S)->children(); - I; ++I) { - if (*I) - this->Visit(*I); - } - } - - void VisitCompoundStmt(const CompoundStmt *S) { - mapSourceCodeRange(S->getLBracLoc()); - mapSourceCodeRange(S->getRBracLoc()); - for (Stmt::const_child_range I = S->children(); I; ++I) { - if (*I) - this->Visit(*I); - } + Stmt *Body = D->getBody(); + propagateCounts(getRegionCounter(Body), Body); } void VisitReturnStmt(const ReturnStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); if (S->getRetValue()) Visit(S->getRetValue()); - setCurrentRegionUnreachable(S); + terminateRegion(S); } - void VisitGotoStmt(const GotoStmt *S) { - mapSourceCodeRange(S->getLocStart()); - mapToken(S->getLabelLoc()); - setCurrentRegionUnreachable(S); + void VisitCXXThrowExpr(const CXXThrowExpr *E) { + extendRegion(E); + if (E->getSubExpr()) + Visit(E->getSubExpr()); + terminateRegion(E); } + void VisitGotoStmt(const GotoStmt *S) { terminateRegion(S); } + void VisitLabelStmt(const LabelStmt *S) { - // Counter tracks the block following the label. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); - mapSourceCodeRange(S->getLocStart()); - // Can't map the ':' token as its location isn't known. + SourceLocation Start = getStart(S); + // We can't extendRegion here or we risk overlapping with our new region. + handleFileExit(Start); + pushRegion(getRegionCounter(S), Start); Visit(S->getSubStmt()); } void VisitBreakStmt(const BreakStmt *S) { - mapSourceCodeRange(S->getLocStart()); assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); BreakContinueStack.back().BreakCount = addCounters( - BreakContinueStack.back().BreakCount, getCurrentRegionCount()); - setCurrentRegionUnreachable(S); + BreakContinueStack.back().BreakCount, getRegion().getCounter()); + terminateRegion(S); } void VisitContinueStmt(const ContinueStmt *S) { - mapSourceCodeRange(S->getLocStart()); assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); BreakContinueStack.back().ContinueCount = addCounters( - BreakContinueStack.back().ContinueCount, getCurrentRegionCount()); - setCurrentRegionUnreachable(S); + BreakContinueStack.back().ContinueCount, getRegion().getCounter()); + terminateRegion(S); } void VisitWhileStmt(const WhileStmt *S) { - mapSourceCodeRange(S->getLocStart()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + extendRegion(S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + + // Handle the body first so that we can get the backedge count. BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first so the break/continue adjustments can be - // included when visiting the condition. - Cnt.beginRegion(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); - - // ...then go back and propagate counts through the condition. The count - // at the start of the condition is the sum of the incoming edges, - // the backedge from the end of the loop body, and the edges from - // continue statements. + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.setCurrentRegionCount( - addCounters(Cnt.getParentCount(), - addCounters(Cnt.getAdjustedCount(), BC.ContinueCount))); - beginSourceRegionGroup(S->getCond()); - Visit(S->getCond()); - endSourceRegionGroup(); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + // Go back to handle the condition. + Counter CondCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + propagateCounts(CondCount, S->getCond()); + adjustForOutOfOrderTraversal(getEnd(S)); + + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitDoStmt(const DoStmt *S) { - mapSourceCodeRange(S->getLocStart()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); - BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); + extendRegion(S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + BreakContinueStack.push_back(BreakContinue()); + extendRegion(S->getBody()); + Counter BackedgeCount = + propagateCounts(addCounters(ParentCount, BodyCount), S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - // The count at the start of the condition is equal to the count at the - // end of the body. The adjusted count does not include either the - // fall-through count coming into the loop or the continue count, so add - // both of those separately. This is coincidentally the same equation as - // with while loops but for different reasons. - Cnt.setCurrentRegionCount( - addCounters(Cnt.getParentCount(), - addCounters(Cnt.getAdjustedCount(), BC.ContinueCount))); - Visit(S->getCond()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + Counter CondCount = addCounters(BackedgeCount, BC.ContinueCount); + propagateCounts(CondCount, S->getCond()); + + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitForStmt(const ForStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); if (S->getInit()) Visit(S->getInit()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + + // Handle the body first so that we can get the backedge count. BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); + BreakContinue BC = BreakContinueStack.pop_back_val(); // The increment is essentially part of the body but it needs to include // the count for all the continue statements. - if (S->getInc()) { - Cnt.setCurrentRegionCount(addCounters( - getCurrentRegionCount(), BreakContinueStack.back().ContinueCount)); - beginSourceRegionGroup(S->getInc()); - Visit(S->getInc()); - endSourceRegionGroup(); - Cnt.adjustForControlFlow(); + if (const Stmt *Inc = S->getInc()) + propagateCounts(addCounters(BackedgeCount, BC.ContinueCount), Inc); + + // Go back to handle the condition. + Counter CondCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + if (const Expr *Cond = S->getCond()) { + propagateCounts(CondCount, Cond); + adjustForOutOfOrderTraversal(getEnd(S)); } - BreakContinue BC = BreakContinueStack.pop_back_val(); - - // ...then go back and propagate counts through the condition. - if (S->getCond()) { - Cnt.setCurrentRegionCount( - addCounters(addCounters(Cnt.getParentCount(), Cnt.getAdjustedCount()), - BC.ContinueCount)); - beginSourceRegionGroup(S->getCond()); - Visit(S->getCond()); - endSourceRegionGroup(); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(CondCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); + Visit(S->getLoopVarStmt()); Visit(S->getRangeStmt()); - Visit(S->getBeginEndStmt()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + BreakContinueStack.push_back(BreakContinue()); - // Visit the body region first. (This is basically the same as a while - // loop; see further comments in VisitWhileStmt.) - Cnt.beginRegion(); - Visit(S->getBody()); - Cnt.adjustForControlFlow(); + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + Counter LoopCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(LoopCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); Visit(S->getElement()); - // Counter tracks the body of the loop. - RegionMapper Cnt(this, S); + + Counter ParentCount = getRegion().getCounter(); + Counter BodyCount = getRegionCounter(S); + BreakContinueStack.push_back(BreakContinue()); - Cnt.beginRegion(); - Visit(S->getBody()); + extendRegion(S->getBody()); + Counter BackedgeCount = propagateCounts(BodyCount, S->getBody()); BreakContinue BC = BreakContinueStack.pop_back_val(); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(addCounters(BC.BreakCount, BC.ContinueCount)); + + Counter LoopCount = + addCounters(ParentCount, BackedgeCount, BC.ContinueCount); + Counter OutCount = + addCounters(BC.BreakCount, subtractCounters(LoopCount, BodyCount)); + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitSwitchStmt(const SwitchStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); Visit(S->getCond()); + BreakContinueStack.push_back(BreakContinue()); - // Map the '}' for the body to have the same count as the regions after - // the switch. - SourceLocation RBracLoc; - if (const auto *CS = dyn_cast<CompoundStmt>(S->getBody())) { - mapSourceCodeRange(CS->getLBracLoc()); - setCurrentRegionUnreachable(S); - for (Stmt::const_child_range I = CS->children(); I; ++I) { - if (*I) - this->Visit(*I); + + const Stmt *Body = S->getBody(); + extendRegion(Body); + if (const auto *CS = dyn_cast<CompoundStmt>(Body)) { + if (!CS->body_empty()) { + // The body of the switch needs a zero region so that fallthrough counts + // behave correctly, but it would be misleading to include the braces of + // the compound statement in the zeroed area, so we need to handle this + // specially. + size_t Index = + pushRegion(Counter::getZero(), getStart(CS->body_front()), + getEnd(CS->body_back())); + for (const auto *Child : CS->children()) + Visit(Child); + popRegions(Index); } - RBracLoc = CS->getRBracLoc(); - } else { - setCurrentRegionUnreachable(S); - Visit(S->getBody()); - } - // If the switch is inside a loop, add the continue counts. + } else + propagateCounts(Counter::getZero(), Body); BreakContinue BC = BreakContinueStack.pop_back_val(); + if (!BreakContinueStack.empty()) BreakContinueStack.back().ContinueCount = addCounters( BreakContinueStack.back().ContinueCount, BC.ContinueCount); - // Counter tracks the exit block of the switch. - RegionMapper ExitCnt(this, S); - ExitCnt.beginRegion(); - if (RBracLoc.isValid()) - mapSourceCodeRange(RBracLoc); - } - void VisitCaseStmt(const CaseStmt *S) { - // Counter for this particular case. This counts only jumps from the - // switch header and does not include fallthrough from the case before - // this one. - RegionMapper Cnt(this, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - mapSourceCodeRange(S->getLocStart()); - mapToken(S->getColonLoc()); - Visit(S->getSubStmt()); + Counter ExitCount = getRegionCounter(S); + pushRegion(ExitCount); } - void VisitDefaultStmt(const DefaultStmt *S) { - // Counter for this default case. This does not include fallthrough from - // the previous case. - RegionMapper Cnt(this, S); - Cnt.beginRegion(/*AddIncomingFallThrough=*/true); - mapSourceCodeRange(S->getLocStart()); - mapToken(S->getColonLoc()); + void VisitSwitchCase(const SwitchCase *S) { + extendRegion(S); + + SourceMappingRegion &Parent = getRegion(); + + Counter Count = addCounters(Parent.getCounter(), getRegionCounter(S)); + // Reuse the existing region if it starts at our label. This is typical of + // the first case in a switch. + if (Parent.hasStartLoc() && Parent.getStartLoc() == getStart(S)) + Parent.setCounter(Count); + else + pushRegion(Count, getStart(S)); + + if (const CaseStmt *CS = dyn_cast<CaseStmt>(S)) { + Visit(CS->getLHS()); + if (const Expr *RHS = CS->getRHS()) + Visit(RHS); + } Visit(S->getSubStmt()); } void VisitIfStmt(const IfStmt *S) { - mapSourceCodeRange(S->getLocStart()); - Visit(S->getCond()); - mapToken(S->getElseLoc()); - - // Counter tracks the "then" part of an if statement. The count for - // the "else" part, if it exists, will be calculated from this counter. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); - Visit(S->getThen()); - Cnt.adjustForControlFlow(); - - if (S->getElse()) { - Cnt.beginElseRegion(); - Visit(S->getElse()); - Cnt.adjustForControlFlow(); - } - Cnt.applyAdjustmentsToRegion(); + extendRegion(S); + + Counter ParentCount = getRegion().getCounter(); + Counter ThenCount = getRegionCounter(S); + + // Emitting a counter for the condition makes it easier to interpret the + // counter for the body when looking at the coverage. + propagateCounts(ParentCount, S->getCond()); + + extendRegion(S->getThen()); + Counter OutCount = propagateCounts(ThenCount, S->getThen()); + + Counter ElseCount = subtractCounters(ParentCount, ThenCount); + if (const Stmt *Else = S->getElse()) { + extendRegion(S->getElse()); + OutCount = addCounters(OutCount, propagateCounts(ElseCount, Else)); + } else + OutCount = addCounters(OutCount, ElseCount); + + if (OutCount != ParentCount) + pushRegion(OutCount); } void VisitCXXTryStmt(const CXXTryStmt *S) { - mapSourceCodeRange(S->getLocStart()); + extendRegion(S); Visit(S->getTryBlock()); for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) Visit(S->getHandler(I)); - // Counter tracks the continuation block of the try statement. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); + + Counter ExitCount = getRegionCounter(S); + pushRegion(ExitCount); } void VisitCXXCatchStmt(const CXXCatchStmt *S) { - mapSourceCodeRange(S->getLocStart()); - // Counter tracks the catch statement's handler block. - RegionMapper Cnt(this, S); - Cnt.beginRegion(); - Visit(S->getHandlerBlock()); + extendRegion(S); + propagateCounts(getRegionCounter(S), S->getHandlerBlock()); } void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { - Visit(E->getCond()); - mapToken(E->getQuestionLoc()); - mapToken(E->getColonLoc()); + extendRegion(E); - // Counter tracks the "true" part of a conditional operator. The - // count in the "false" part will be calculated from this counter. - RegionMapper Cnt(this, E); - Cnt.beginRegion(); - Visit(E->getTrueExpr()); - Cnt.adjustForControlFlow(); + Counter ParentCount = getRegion().getCounter(); + Counter TrueCount = getRegionCounter(E); - Cnt.beginElseRegion(); - Visit(E->getFalseExpr()); - Cnt.adjustForControlFlow(); + Visit(E->getCond()); - Cnt.applyAdjustmentsToRegion(); + if (!isa<BinaryConditionalOperator>(E)) { + extendRegion(E->getTrueExpr()); + propagateCounts(TrueCount, E->getTrueExpr()); + } + extendRegion(E->getFalseExpr()); + propagateCounts(subtractCounters(ParentCount, TrueCount), + E->getFalseExpr()); } void VisitBinLAnd(const BinaryOperator *E) { + extendRegion(E); Visit(E->getLHS()); - mapToken(E->getOperatorLoc()); - // Counter tracks the right hand side of a logical and operator. - RegionMapper Cnt(this, E); - Cnt.beginRegion(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(); - } - - void VisitBinLOr(const BinaryOperator *E) { - Visit(E->getLHS()); - mapToken(E->getOperatorLoc()); - // Counter tracks the right hand side of a logical or operator. - RegionMapper Cnt(this, E); - Cnt.beginRegion(); - Visit(E->getRHS()); - Cnt.adjustForControlFlow(); - Cnt.applyAdjustmentsToRegion(); - } - void VisitParenExpr(const ParenExpr *E) { - mapToken(E->getLParen()); - Visit(E->getSubExpr()); - mapToken(E->getRParen()); + extendRegion(E->getRHS()); + propagateCounts(getRegionCounter(E), E->getRHS()); } - void VisitBinaryOperator(const BinaryOperator *E) { - Visit(E->getLHS()); - mapToken(E->getOperatorLoc()); - Visit(E->getRHS()); - } - - void VisitUnaryOperator(const UnaryOperator *E) { - bool Postfix = E->isPostfix(); - if (!Postfix) - mapToken(E->getOperatorLoc()); - Visit(E->getSubExpr()); - if (Postfix) - mapToken(E->getOperatorLoc()); - } - - void VisitMemberExpr(const MemberExpr *E) { - Visit(E->getBase()); - mapToken(E->getMemberLoc()); - } - - void VisitCallExpr(const CallExpr *E) { - Visit(E->getCallee()); - for (const auto &Arg : E->arguments()) - Visit(Arg); - mapToken(E->getRParenLoc()); - } - - void VisitArraySubscriptExpr(const ArraySubscriptExpr *E) { + void VisitBinLOr(const BinaryOperator *E) { + extendRegion(E); Visit(E->getLHS()); - Visit(E->getRHS()); - mapToken(E->getRBracketLoc()); - } - - void VisitCStyleCastExpr(const CStyleCastExpr *E) { - mapToken(E->getLParenLoc()); - mapToken(E->getRParenLoc()); - Visit(E->getSubExpr()); - } - - // Map literals as tokens so that the macros like #define PI 3.14 - // won't generate coverage mapping regions. - void VisitIntegerLiteral(const IntegerLiteral *E) { - mapToken(E->getLocStart()); + extendRegion(E->getRHS()); + propagateCounts(getRegionCounter(E), E->getRHS()); } - void VisitFloatingLiteral(const FloatingLiteral *E) { - mapToken(E->getLocStart()); - } - - void VisitCharacterLiteral(const CharacterLiteral *E) { - mapToken(E->getLocStart()); - } - - void VisitStringLiteral(const StringLiteral *E) { - mapToken(E->getLocStart()); - } - - void VisitImaginaryLiteral(const ImaginaryLiteral *E) { - mapToken(E->getLocStart()); - } - - void VisitObjCMessageExpr(const ObjCMessageExpr *E) { - mapToken(E->getLeftLoc()); - for (Stmt::const_child_range I = static_cast<const Stmt*>(E)->children(); I; - ++I) { - if (*I) - this->Visit(*I); - } - mapToken(E->getRightLoc()); + void VisitLambdaExpr(const LambdaExpr *LE) { + // Lambdas are treated as their own functions for now, so we shouldn't + // propagate counts into them. } }; } @@ -1009,10 +891,12 @@ static StringRef getCoverageSection(const CodeGenModule &CGM) { return isMachO(CGM) ? "__DATA,__llvm_covmap" : "__llvm_covmap"; } -static void dump(llvm::raw_ostream &OS, const CoverageMappingRecord &Function) { - OS << Function.FunctionName << ":\n"; - CounterMappingContext Ctx(Function.Expressions); - for (const auto &R : Function.MappingRegions) { +static void dump(llvm::raw_ostream &OS, StringRef FunctionName, + ArrayRef<CounterExpression> Expressions, + ArrayRef<CounterMappingRegion> Regions) { + OS << FunctionName << ":\n"; + CounterMappingContext Ctx(Expressions); + for (const auto &R : Regions) { OS.indent(2); switch (R.Kind) { case CounterMappingRegion::CodeRegion: @@ -1025,15 +909,12 @@ static void dump(llvm::raw_ostream &OS, const CoverageMappingRecord &Function) { break; } - OS << "File " << R.FileID << ", " << R.LineStart << ":" - << R.ColumnStart << " -> " << R.LineEnd << ":" << R.ColumnEnd - << " = "; - Ctx.dump(R.Count); - OS << " (HasCodeBefore = " << R.HasCodeBefore; + OS << "File " << R.FileID << ", " << R.LineStart << ":" << R.ColumnStart + << " -> " << R.LineEnd << ":" << R.ColumnEnd << " = "; + Ctx.dump(R.Count, OS); if (R.Kind == CounterMappingRegion::ExpansionRegion) - OS << ", Expanded file = " << R.ExpandedFileID; - - OS << ")\n"; + OS << " (Expanded file = " << R.ExpandedFileID << ")"; + OS << "\n"; } } @@ -1072,13 +953,11 @@ void CoverageMappingModuleGen::addFunctionMappingRecord( FilenameRefs.resize(FileEntries.size()); for (const auto &Entry : FileEntries) FilenameRefs[Entry.second] = Entry.first->getName(); - RawCoverageMappingReader Reader(FunctionNameValue, CoverageMapping, - FilenameRefs, - Filenames, Expressions, Regions); - CoverageMappingRecord FunctionRecord; - if (Reader.read(FunctionRecord)) + RawCoverageMappingReader Reader(CoverageMapping, FilenameRefs, Filenames, + Expressions, Regions); + if (Reader.read()) return; - dump(llvm::outs(), FunctionRecord); + dump(llvm::outs(), FunctionNameValue, Expressions, Regions); } } @@ -1098,7 +977,7 @@ void CoverageMappingModuleGen::emit() { llvm::sys::fs::make_absolute(Path); auto I = Entry.second; - FilenameStrs[I] = std::move(std::string(Path.begin(), Path.end())); + FilenameStrs[I] = std::string(Path.begin(), Path.end()); FilenameRefs[I] = FilenameStrs[I]; } diff --git a/lib/CodeGen/EHScopeStack.h b/lib/CodeGen/EHScopeStack.h index e69584840990..a7951888c825 100644 --- a/lib/CodeGen/EHScopeStack.h +++ b/lib/CodeGen/EHScopeStack.h @@ -17,6 +17,7 @@ #define LLVM_CLANG_LIB_CODEGEN_EHSCOPESTACK_H #include "clang/Basic/LLVM.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Instructions.h" @@ -75,8 +76,14 @@ template <class T> struct DominatingPointer<T,false> : InvariantValue<T*> {}; template <class T> struct DominatingValue<T*> : DominatingPointer<T> {}; enum CleanupKind : unsigned { + /// Denotes a cleanup that should run when a scope is exited using exceptional + /// control flow (a throw statement leading to stack unwinding, ). EHCleanup = 0x1, + + /// Denotes a cleanup that should run when a scope is exited using normal + /// control flow (falling off the end of the scope, return, goto, ...). NormalCleanup = 0x2, + NormalAndEHCleanup = EHCleanup | NormalCleanup, InactiveCleanup = 0x4, @@ -175,84 +182,28 @@ public: virtual void Emit(CodeGenFunction &CGF, Flags flags) = 0; }; - /// ConditionalCleanupN stores the saved form of its N parameters, + /// ConditionalCleanup stores the saved form of its parameters, /// then restores them and performs the cleanup. - template <class T, class A0> - class ConditionalCleanup1 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - A0_saved a0_saved; - - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - T(a0).Emit(CGF, flags); - } - - public: - ConditionalCleanup1(A0_saved a0) - : a0_saved(a0) {} - }; - - template <class T, class A0, class A1> - class ConditionalCleanup2 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - typedef typename DominatingValue<A1>::saved_type A1_saved; - A0_saved a0_saved; - A1_saved a1_saved; - - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - A1 a1 = DominatingValue<A1>::restore(CGF, a1_saved); - T(a0, a1).Emit(CGF, flags); + template <class T, class... As> class ConditionalCleanup : public Cleanup { + typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple; + SavedTuple Saved; + + template <std::size_t... Is> + T restore(CodeGenFunction &CGF, llvm::index_sequence<Is...>) { + // It's important that the restores are emitted in order. The braced init + // list guarentees that. + return T{DominatingValue<As>::restore(CGF, std::get<Is>(Saved))...}; } - public: - ConditionalCleanup2(A0_saved a0, A1_saved a1) - : a0_saved(a0), a1_saved(a1) {} - }; - - template <class T, class A0, class A1, class A2> - class ConditionalCleanup3 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - typedef typename DominatingValue<A1>::saved_type A1_saved; - typedef typename DominatingValue<A2>::saved_type A2_saved; - A0_saved a0_saved; - A1_saved a1_saved; - A2_saved a2_saved; - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - A1 a1 = DominatingValue<A1>::restore(CGF, a1_saved); - A2 a2 = DominatingValue<A2>::restore(CGF, a2_saved); - T(a0, a1, a2).Emit(CGF, flags); + restore(CGF, llvm::index_sequence_for<As...>()).Emit(CGF, flags); } public: - ConditionalCleanup3(A0_saved a0, A1_saved a1, A2_saved a2) - : a0_saved(a0), a1_saved(a1), a2_saved(a2) {} - }; - - template <class T, class A0, class A1, class A2, class A3> - class ConditionalCleanup4 : public Cleanup { - typedef typename DominatingValue<A0>::saved_type A0_saved; - typedef typename DominatingValue<A1>::saved_type A1_saved; - typedef typename DominatingValue<A2>::saved_type A2_saved; - typedef typename DominatingValue<A3>::saved_type A3_saved; - A0_saved a0_saved; - A1_saved a1_saved; - A2_saved a2_saved; - A3_saved a3_saved; + ConditionalCleanup(typename DominatingValue<As>::saved_type... A) + : Saved(A...) {} - void Emit(CodeGenFunction &CGF, Flags flags) override { - A0 a0 = DominatingValue<A0>::restore(CGF, a0_saved); - A1 a1 = DominatingValue<A1>::restore(CGF, a1_saved); - A2 a2 = DominatingValue<A2>::restore(CGF, a2_saved); - A3 a3 = DominatingValue<A3>::restore(CGF, a3_saved); - T(a0, a1, a2, a3).Emit(CGF, flags); - } - - public: - ConditionalCleanup4(A0_saved a0, A1_saved a1, A2_saved a2, A3_saved a3) - : a0_saved(a0), a1_saved(a1), a2_saved(a2), a3_saved(a3) {} + ConditionalCleanup(SavedTuple Tuple) : Saved(std::move(Tuple)) {} }; private: @@ -306,53 +257,18 @@ public: InnermostEHScope(stable_end()) {} ~EHScopeStack() { delete[] StartOfBuffer; } - // Variadic templates would make this not terrible. - /// Push a lazily-created cleanup on the stack. - template <class T> - void pushCleanup(CleanupKind Kind) { + template <class T, class... As> void pushCleanup(CleanupKind Kind, As... A) { void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(); + Cleanup *Obj = new (Buffer) T(A...); (void) Obj; } - /// Push a lazily-created cleanup on the stack. - template <class T, class A0> - void pushCleanup(CleanupKind Kind, A0 a0) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1, class A2> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2) { + /// Push a lazily-created cleanup on the stack. Tuple version. + template <class T, class... As> + void pushCleanupTuple(CleanupKind Kind, std::tuple<As...> A) { void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1, a2); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1, class A2, class A3> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1, a2, a3); - (void) Obj; - } - - /// Push a lazily-created cleanup on the stack. - template <class T, class A0, class A1, class A2, class A3, class A4> - void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3, A4 a4) { - void *Buffer = pushCleanup(Kind, sizeof(T)); - Cleanup *Obj = new(Buffer) T(a0, a1, a2, a3, a4); + Cleanup *Obj = new (Buffer) T(std::move(A)); (void) Obj; } @@ -369,10 +285,10 @@ public: /// /// The pointer returned from this method is valid until the cleanup /// stack is modified. - template <class T, class A0, class A1, class A2> - T *pushCleanupWithExtra(CleanupKind Kind, size_t N, A0 a0, A1 a1, A2 a2) { + template <class T, class... As> + T *pushCleanupWithExtra(CleanupKind Kind, size_t N, As... A) { void *Buffer = pushCleanup(Kind, sizeof(T) + T::getExtraSize(N)); - return new (Buffer) T(N, a0, a1, a2); + return new (Buffer) T(N, A...); } void pushCopyOfCleanup(CleanupKind Kind, const void *Cleanup, size_t Size) { @@ -403,6 +319,10 @@ public: /// Pops a terminate handler off the stack. void popTerminate(); + // Returns true iff the current scope is either empty or contains only + // lifetime markers, i.e. no real cleanup code + bool containsOnlyLifetimeMarkers(stable_iterator Old) const; + /// Determines whether the exception-scopes stack is empty. bool empty() const { return StartOfData == EndOfBuffer; } diff --git a/lib/CodeGen/ItaniumCXXABI.cpp b/lib/CodeGen/ItaniumCXXABI.cpp index 9e88b1ebbadc..0a1a4ceea3ef 100644 --- a/lib/CodeGen/ItaniumCXXABI.cpp +++ b/lib/CodeGen/ItaniumCXXABI.cpp @@ -19,14 +19,18 @@ //===----------------------------------------------------------------------===// #include "CGCXXABI.h" +#include "CGCleanup.h" #include "CGRecordLayout.h" #include "CGVTables.h" #include "CodeGenFunction.h" #include "CodeGenModule.h" +#include "TargetInfo.h" #include "clang/AST/Mangle.h" #include "clang/AST/Type.h" +#include "clang/AST/StmtCXX.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Value.h" @@ -111,10 +115,22 @@ public: const CXXDestructorDecl *Dtor) override; void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override; + void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override; + + void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override; + + llvm::CallInst * + emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn) override; void EmitFundamentalRTTIDescriptor(QualType Type); void EmitFundamentalRTTIDescriptors(); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; + llvm::Constant * + getAddrOfCXXCatchHandlerType(QualType Ty, + QualType CatchHandlerType) override { + return getAddrOfRTTIDescriptor(Ty); + } bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; @@ -849,7 +865,7 @@ bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const { /// The Itanium ABI requires non-zero initialization only for data /// member pointers, for which '0' is a valid offset. bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) { - return MPT->getPointeeType()->isFunctionType(); + return MPT->isMemberFunctionPointer(); } /// The Itanium ABI always places an offset to the complete object @@ -906,6 +922,59 @@ void ItaniumCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) { CGF.EmitRuntimeCallOrInvoke(Fn); } +static llvm::Constant *getAllocateExceptionFn(CodeGenModule &CGM) { + // void *__cxa_allocate_exception(size_t thrown_size); + + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.Int8PtrTy, CGM.SizeTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_allocate_exception"); +} + +static llvm::Constant *getThrowFn(CodeGenModule &CGM) { + // void __cxa_throw(void *thrown_exception, std::type_info *tinfo, + // void (*dest) (void *)); + + llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.Int8PtrTy, CGM.Int8PtrTy }; + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, Args, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_throw"); +} + +void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { + QualType ThrowType = E->getSubExpr()->getType(); + // Now allocate the exception object. + llvm::Type *SizeTy = CGF.ConvertType(getContext().getSizeType()); + uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity(); + + llvm::Constant *AllocExceptionFn = getAllocateExceptionFn(CGM); + llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall( + AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception"); + + CGF.EmitAnyExprToExn(E->getSubExpr(), ExceptionPtr); + + // Now throw the exception. + llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType, + /*ForEH=*/true); + + // The address of the destructor. If the exception type has a + // trivial destructor (or isn't a record), we just pass null. + llvm::Constant *Dtor = nullptr; + if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) { + CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl()); + if (!Record->hasTrivialDestructor()) { + CXXDestructorDecl *DtorD = Record->getDestructor(); + Dtor = CGM.getAddrOfCXXStructor(DtorD, StructorType::Complete); + Dtor = llvm::ConstantExpr::getBitCast(Dtor, CGM.Int8PtrTy); + } + } + if (!Dtor) Dtor = llvm::Constant::getNullValue(CGM.Int8PtrTy); + + llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor }; + CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(CGM), args); +} + static llvm::Constant *getItaniumDynamicCastFn(CodeGenFunction &CGF) { // void *__dynamic_cast(const void *sub, // const abi::__class_type_info *src, @@ -1259,6 +1328,9 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, // Set the correct linkage. VTable->setLinkage(Linkage); + if (CGM.supportsCOMDAT() && VTable->isWeakForLinker()) + VTable->setComdat(CGM.getModule().getOrInsertComdat(VTable->getName())); + // Set the right visibility. CGM.setGlobalVisibility(VTable, RD); @@ -1278,6 +1350,8 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT, cast<NamespaceDecl>(DC)->getIdentifier()->isStr("__cxxabiv1") && DC->getParent()->isTranslationUnit()) EmitFundamentalRTTIDescriptors(); + + CGM.EmitVTableBitSetEntries(VTable, VTLayout); } llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor( @@ -1314,7 +1388,7 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor( llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( BaseSubobject Base, const CXXRecordDecl *VTableClass) { - llvm::Constant *VTable = getAddrOfVTable(VTableClass, CharUnits()); + auto *VTable = getAddrOfVTable(VTableClass, CharUnits()); // Find the appropriate vtable within the vtable group. uint64_t AddressPoint = CGM.getItaniumVTableContext() @@ -1325,7 +1399,8 @@ llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr( llvm::ConstantInt::get(CGM.Int64Ty, AddressPoint) }; - return llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Indices); + return llvm::ConstantExpr::getInBoundsGetElementPtr(VTable->getValueType(), + VTable, Indices); } llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, @@ -1369,6 +1444,9 @@ llvm::Value *ItaniumCXXABI::getVirtualFunctionPointer(CodeGenFunction &CGF, Ty = Ty->getPointerTo()->getPointerTo(); llvm::Value *VTable = CGF.GetVTablePtr(This, Ty); + if (CGF.SanOpts.has(SanitizerKind::CFIVCall)) + CGF.EmitVTablePtrCheckForCall(cast<CXXMethodDecl>(GD.getDecl()), VTable); + uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); llvm::Value *VFuncPtr = CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); @@ -1591,7 +1669,7 @@ llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF, CGF.Builder.CreateStore(elementSize, cookie); // The second element is the element count. - cookie = CGF.Builder.CreateConstInBoundsGEP1_32(cookie, 1); + cookie = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.SizeTy, cookie, 1); CGF.Builder.CreateStore(numElements, cookie); // Finally, compute a pointer to the actual data buffer by skipping @@ -1714,11 +1792,12 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF, // The ABI says: It is suggested that it be emitted in the same COMDAT group // as the associated data object - if (!D.isLocalVarDecl() && var->isWeakForLinker() && CGM.supportsCOMDAT()) { - llvm::Comdat *C = CGM.getModule().getOrInsertComdat(var->getName()); + llvm::Comdat *C = var->getComdat(); + if (!D.isLocalVarDecl() && C) { guard->setComdat(C); - var->setComdat(C); CGF.CurFn->setComdat(C); + } else if (CGM.supportsCOMDAT() && guard->isWeakForLinker()) { + guard->setComdat(CGM.getModule().getOrInsertComdat(guard->getName())); } CGM.setStaticLocalDeclGuardAddress(&D, guard); @@ -2011,7 +2090,7 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( CGBuilderTy Builder(Entry); if (InitIsInitFunc) { if (Init) - Builder.CreateCall(Init); + Builder.CreateCall(Init, {}); } else { // Don't know whether we have an init function. Call it if it exists. llvm::Value *Have = Builder.CreateIsNotNull(Init); @@ -2020,7 +2099,7 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs( Builder.CreateCondBr(Have, InitBB, ExitBB); Builder.SetInsertPoint(InitBB); - Builder.CreateCall(Init); + Builder.CreateCall(Init, {}); Builder.CreateBr(ExitBB); Builder.SetInsertPoint(ExitBB); @@ -2049,7 +2128,7 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, llvm::Value *Val = CGF.CGM.GetAddrOfGlobalVar(VD, Ty); llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Val); - Val = CGF.Builder.CreateCall(Wrapper); + Val = CGF.Builder.CreateCall(Wrapper, {}); LValue LV; if (VD->getType()->isReferenceType()) @@ -2528,7 +2607,8 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) { // The vtable address point is 2. llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2); - VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, Two); + VTable = + llvm::ConstantExpr::getInBoundsGetElementPtr(CGM.Int8PtrTy, VTable, Two); VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); Fields.push_back(VTable); @@ -2718,9 +2798,13 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) { llvm::Constant *Init = llvm::ConstantStruct::getAnon(Fields); + llvm::Module &M = CGM.getModule(); llvm::GlobalVariable *GV = - new llvm::GlobalVariable(CGM.getModule(), Init->getType(), - /*Constant=*/true, Linkage, Init, Name); + new llvm::GlobalVariable(M, Init->getType(), + /*Constant=*/true, Linkage, Init, Name); + + if (CGM.supportsCOMDAT() && GV->isWeakForLinker()) + GV->setComdat(M.getOrInsertComdat(GV->getName())); // If there's already an old global variable, replace it with the new one. if (OldGV) { @@ -3136,8 +3220,8 @@ static void emitConstructorDestructorAlias(CodeGenModule &CGM, llvm::PointerType *AliasType = Aliasee->getType(); // Create the alias with no name. - auto *Alias = llvm::GlobalAlias::create( - AliasType->getElementType(), 0, Linkage, "", Aliasee, &CGM.getModule()); + auto *Alias = llvm::GlobalAlias::create(AliasType, Linkage, "", Aliasee, + &CGM.getModule()); // Switch any previous uses to the alias. if (Entry) { @@ -3204,5 +3288,352 @@ void ItaniumCXXABI::emitCXXStructor(const CXXMethodDecl *MD, getMangleContext().mangleCXXCtorComdat(CD, Out); llvm::Comdat *C = CGM.getModule().getOrInsertComdat(Out.str()); Fn->setComdat(C); + } else { + CGM.maybeSetTrivialComdat(*MD, *Fn); + } +} + +static llvm::Constant *getBeginCatchFn(CodeGenModule &CGM) { + // void *__cxa_begin_catch(void*); + llvm::FunctionType *FTy = llvm::FunctionType::get( + CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_begin_catch"); +} + +static llvm::Constant *getEndCatchFn(CodeGenModule &CGM) { + // void __cxa_end_catch(); + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_end_catch"); +} + +static llvm::Constant *getGetExceptionPtrFn(CodeGenModule &CGM) { + // void *__cxa_get_exception_ptr(void*); + llvm::FunctionType *FTy = llvm::FunctionType::get( + CGM.Int8PtrTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); + + return CGM.CreateRuntimeFunction(FTy, "__cxa_get_exception_ptr"); +} + +namespace { + /// A cleanup to call __cxa_end_catch. In many cases, the caught + /// exception type lets us state definitively that the thrown exception + /// type does not have a destructor. In particular: + /// - Catch-alls tell us nothing, so we have to conservatively + /// assume that the thrown exception might have a destructor. + /// - Catches by reference behave according to their base types. + /// - Catches of non-record types will only trigger for exceptions + /// of non-record types, which never have destructors. + /// - Catches of record types can trigger for arbitrary subclasses + /// of the caught type, so we have to assume the actual thrown + /// exception type might have a throwing destructor, even if the + /// caught type's destructor is trivial or nothrow. + struct CallEndCatch : EHScopeStack::Cleanup { + CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {} + bool MightThrow; + + void Emit(CodeGenFunction &CGF, Flags flags) override { + if (!MightThrow) { + CGF.EmitNounwindRuntimeCall(getEndCatchFn(CGF.CGM)); + return; + } + + CGF.EmitRuntimeCallOrInvoke(getEndCatchFn(CGF.CGM)); + } + }; +} + +/// Emits a call to __cxa_begin_catch and enters a cleanup to call +/// __cxa_end_catch. +/// +/// \param EndMightThrow - true if __cxa_end_catch might throw +static llvm::Value *CallBeginCatch(CodeGenFunction &CGF, + llvm::Value *Exn, + bool EndMightThrow) { + llvm::CallInst *call = + CGF.EmitNounwindRuntimeCall(getBeginCatchFn(CGF.CGM), Exn); + + CGF.EHStack.pushCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow); + + return call; +} + +/// A "special initializer" callback for initializing a catch +/// parameter during catch initialization. +static void InitCatchParam(CodeGenFunction &CGF, + const VarDecl &CatchParam, + llvm::Value *ParamAddr, + SourceLocation Loc) { + // Load the exception from where the landing pad saved it. + llvm::Value *Exn = CGF.getExceptionFromSlot(); + + CanQualType CatchType = + CGF.CGM.getContext().getCanonicalType(CatchParam.getType()); + llvm::Type *LLVMCatchTy = CGF.ConvertTypeForMem(CatchType); + + // If we're catching by reference, we can just cast the object + // pointer to the appropriate pointer. + if (isa<ReferenceType>(CatchType)) { + QualType CaughtType = cast<ReferenceType>(CatchType)->getPointeeType(); + bool EndCatchMightThrow = CaughtType->isRecordType(); + + // __cxa_begin_catch returns the adjusted object pointer. + llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndCatchMightThrow); + + // We have no way to tell the personality function that we're + // catching by reference, so if we're catching a pointer, + // __cxa_begin_catch will actually return that pointer by value. + if (const PointerType *PT = dyn_cast<PointerType>(CaughtType)) { + QualType PointeeType = PT->getPointeeType(); + + // When catching by reference, generally we should just ignore + // this by-value pointer and use the exception object instead. + if (!PointeeType->isRecordType()) { + + // Exn points to the struct _Unwind_Exception header, which + // we have to skip past in order to reach the exception data. + unsigned HeaderSize = + CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException(); + AdjustedExn = CGF.Builder.CreateConstGEP1_32(Exn, HeaderSize); + + // However, if we're catching a pointer-to-record type that won't + // work, because the personality function might have adjusted + // the pointer. There's actually no way for us to fully satisfy + // the language/ABI contract here: we can't use Exn because it + // might have the wrong adjustment, but we can't use the by-value + // pointer because it's off by a level of abstraction. + // + // The current solution is to dump the adjusted pointer into an + // alloca, which breaks language semantics (because changing the + // pointer doesn't change the exception) but at least works. + // The better solution would be to filter out non-exact matches + // and rethrow them, but this is tricky because the rethrow + // really needs to be catchable by other sites at this landing + // pad. The best solution is to fix the personality function. + } else { + // Pull the pointer for the reference type off. + llvm::Type *PtrTy = + cast<llvm::PointerType>(LLVMCatchTy)->getElementType(); + + // Create the temporary and write the adjusted pointer into it. + llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp"); + llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); + CGF.Builder.CreateStore(Casted, ExnPtrTmp); + + // Bind the reference to the temporary. + AdjustedExn = ExnPtrTmp; + } + } + + llvm::Value *ExnCast = + CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref"); + CGF.Builder.CreateStore(ExnCast, ParamAddr); + return; + } + + // Scalars and complexes. + TypeEvaluationKind TEK = CGF.getEvaluationKind(CatchType); + if (TEK != TEK_Aggregate) { + llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, false); + + // If the catch type is a pointer type, __cxa_begin_catch returns + // the pointer by value. + if (CatchType->hasPointerRepresentation()) { + llvm::Value *CastExn = + CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.casted"); + + switch (CatchType.getQualifiers().getObjCLifetime()) { + case Qualifiers::OCL_Strong: + CastExn = CGF.EmitARCRetainNonBlock(CastExn); + // fallthrough + + case Qualifiers::OCL_None: + case Qualifiers::OCL_ExplicitNone: + case Qualifiers::OCL_Autoreleasing: + CGF.Builder.CreateStore(CastExn, ParamAddr); + return; + + case Qualifiers::OCL_Weak: + CGF.EmitARCInitWeak(ParamAddr, CastExn); + return; + } + llvm_unreachable("bad ownership qualifier!"); + } + + // Otherwise, it returns a pointer into the exception object. + + llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok + llvm::Value *Cast = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy); + + LValue srcLV = CGF.MakeNaturalAlignAddrLValue(Cast, CatchType); + LValue destLV = CGF.MakeAddrLValue(ParamAddr, CatchType, + CGF.getContext().getDeclAlign(&CatchParam)); + switch (TEK) { + case TEK_Complex: + CGF.EmitStoreOfComplex(CGF.EmitLoadOfComplex(srcLV, Loc), destLV, + /*init*/ true); + return; + case TEK_Scalar: { + llvm::Value *ExnLoad = CGF.EmitLoadOfScalar(srcLV, Loc); + CGF.EmitStoreOfScalar(ExnLoad, destLV, /*init*/ true); + return; + } + case TEK_Aggregate: + llvm_unreachable("evaluation kind filtered out!"); + } + llvm_unreachable("bad evaluation kind"); + } + + assert(isa<RecordType>(CatchType) && "unexpected catch type!"); + + llvm::Type *PtrTy = LLVMCatchTy->getPointerTo(0); // addrspace 0 ok + + // Check for a copy expression. If we don't have a copy expression, + // that means a trivial copy is okay. + const Expr *copyExpr = CatchParam.getInit(); + if (!copyExpr) { + llvm::Value *rawAdjustedExn = CallBeginCatch(CGF, Exn, true); + llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); + CGF.EmitAggregateCopy(ParamAddr, adjustedExn, CatchType); + return; + } + + // We have to call __cxa_get_exception_ptr to get the adjusted + // pointer before copying. + llvm::CallInst *rawAdjustedExn = + CGF.EmitNounwindRuntimeCall(getGetExceptionPtrFn(CGF.CGM), Exn); + + // Cast that to the appropriate type. + llvm::Value *adjustedExn = CGF.Builder.CreateBitCast(rawAdjustedExn, PtrTy); + + // The copy expression is defined in terms of an OpaqueValueExpr. + // Find it and map it to the adjusted expression. + CodeGenFunction::OpaqueValueMapping + opaque(CGF, OpaqueValueExpr::findInCopyConstruct(copyExpr), + CGF.MakeAddrLValue(adjustedExn, CatchParam.getType())); + + // Call the copy ctor in a terminate scope. + CGF.EHStack.pushTerminate(); + + // Perform the copy construction. + CharUnits Alignment = CGF.getContext().getDeclAlign(&CatchParam); + CGF.EmitAggExpr(copyExpr, + AggValueSlot::forAddr(ParamAddr, Alignment, Qualifiers(), + AggValueSlot::IsNotDestructed, + AggValueSlot::DoesNotNeedGCBarriers, + AggValueSlot::IsNotAliased)); + + // Leave the terminate scope. + CGF.EHStack.popTerminate(); + + // Undo the opaque value mapping. + opaque.pop(); + + // Finally we can call __cxa_begin_catch. + CallBeginCatch(CGF, Exn, true); +} + +/// Begins a catch statement by initializing the catch variable and +/// calling __cxa_begin_catch. +void ItaniumCXXABI::emitBeginCatch(CodeGenFunction &CGF, + const CXXCatchStmt *S) { + // We have to be very careful with the ordering of cleanups here: + // C++ [except.throw]p4: + // The destruction [of the exception temporary] occurs + // immediately after the destruction of the object declared in + // the exception-declaration in the handler. + // + // So the precise ordering is: + // 1. Construct catch variable. + // 2. __cxa_begin_catch + // 3. Enter __cxa_end_catch cleanup + // 4. Enter dtor cleanup + // + // We do this by using a slightly abnormal initialization process. + // Delegation sequence: + // - ExitCXXTryStmt opens a RunCleanupsScope + // - EmitAutoVarAlloca creates the variable and debug info + // - InitCatchParam initializes the variable from the exception + // - CallBeginCatch calls __cxa_begin_catch + // - CallBeginCatch enters the __cxa_end_catch cleanup + // - EmitAutoVarCleanups enters the variable destructor cleanup + // - EmitCXXTryStmt emits the code for the catch body + // - EmitCXXTryStmt close the RunCleanupsScope + + VarDecl *CatchParam = S->getExceptionDecl(); + if (!CatchParam) { + llvm::Value *Exn = CGF.getExceptionFromSlot(); + CallBeginCatch(CGF, Exn, true); + return; + } + + // Emit the local. + CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); + InitCatchParam(CGF, *CatchParam, var.getObjectAddress(CGF), S->getLocStart()); + CGF.EmitAutoVarCleanups(var); +} + +/// Get or define the following function: +/// void @__clang_call_terminate(i8* %exn) nounwind noreturn +/// This code is used only in C++. +static llvm::Constant *getClangCallTerminateFn(CodeGenModule &CGM) { + llvm::FunctionType *fnTy = + llvm::FunctionType::get(CGM.VoidTy, CGM.Int8PtrTy, /*IsVarArgs=*/false); + llvm::Constant *fnRef = + CGM.CreateRuntimeFunction(fnTy, "__clang_call_terminate"); + + llvm::Function *fn = dyn_cast<llvm::Function>(fnRef); + if (fn && fn->empty()) { + fn->setDoesNotThrow(); + fn->setDoesNotReturn(); + + // What we really want is to massively penalize inlining without + // forbidding it completely. The difference between that and + // 'noinline' is negligible. + fn->addFnAttr(llvm::Attribute::NoInline); + + // Allow this function to be shared across translation units, but + // we don't want it to turn into an exported symbol. + fn->setLinkage(llvm::Function::LinkOnceODRLinkage); + fn->setVisibility(llvm::Function::HiddenVisibility); + if (CGM.supportsCOMDAT()) + fn->setComdat(CGM.getModule().getOrInsertComdat(fn->getName())); + + // Set up the function. + llvm::BasicBlock *entry = + llvm::BasicBlock::Create(CGM.getLLVMContext(), "", fn); + CGBuilderTy builder(entry); + + // Pull the exception pointer out of the parameter list. + llvm::Value *exn = &*fn->arg_begin(); + + // Call __cxa_begin_catch(exn). + llvm::CallInst *catchCall = builder.CreateCall(getBeginCatchFn(CGM), exn); + catchCall->setDoesNotThrow(); + catchCall->setCallingConv(CGM.getRuntimeCC()); + + // Call std::terminate(). + llvm::CallInst *termCall = builder.CreateCall(CGM.getTerminateFn(), {}); + termCall->setDoesNotThrow(); + termCall->setDoesNotReturn(); + termCall->setCallingConv(CGM.getRuntimeCC()); + + // std::terminate cannot return. + builder.CreateUnreachable(); + } + + return fnRef; +} + +llvm::CallInst * +ItaniumCXXABI::emitTerminateForUnexpectedException(CodeGenFunction &CGF, + llvm::Value *Exn) { + // In C++, we want to call __cxa_begin_catch() before terminating. + if (Exn) { + assert(CGF.CGM.getLangOpts().CPlusPlus); + return CGF.EmitNounwindRuntimeCall(getClangCallTerminateFn(CGF.CGM), Exn); } + return CGF.EmitNounwindRuntimeCall(CGF.CGM.getTerminateFn()); } diff --git a/lib/CodeGen/MicrosoftCXXABI.cpp b/lib/CodeGen/MicrosoftCXXABI.cpp index c067fab1e921..de30883c545f 100644 --- a/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/lib/CodeGen/MicrosoftCXXABI.cpp @@ -17,12 +17,16 @@ #include "CGCXXABI.h" #include "CGVTables.h" #include "CodeGenModule.h" +#include "CodeGenTypes.h" +#include "TargetInfo.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclCXX.h" +#include "clang/AST/StmtCXX.h" #include "clang/AST/VTableBuilder.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Intrinsics.h" using namespace clang; using namespace CodeGen; @@ -40,7 +44,8 @@ public: MicrosoftCXXABI(CodeGenModule &CGM) : CGCXXABI(CGM), BaseClassDescriptorType(nullptr), ClassHierarchyDescriptorType(nullptr), - CompleteObjectLocatorType(nullptr) {} + CompleteObjectLocatorType(nullptr), CatchableTypeType(nullptr), + ThrowInfoType(nullptr), CatchHandlerTypeType(nullptr) {} bool HasThisReturn(GlobalDecl GD) const override; bool hasMostDerivedReturn(GlobalDecl GD) const override; @@ -71,11 +76,16 @@ public: const CXXDestructorDecl *Dtor) override; void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override; + void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override; + + void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override; llvm::GlobalVariable *getMSCompleteObjectLocator(const CXXRecordDecl *RD, const VPtrInfo *Info); llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override; + llvm::Constant * + getAddrOfCXXCatchHandlerType(QualType Ty, QualType CatchHandlerType) override; bool shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) override; void EmitBadTypeidCall(CodeGenFunction &CGF) override; @@ -225,7 +235,7 @@ public: assert(GD.getDtorType() == Dtor_Deleting && "Only deleting destructor thunks are available in this ABI"); CallArgs.add(RValue::get(getStructorImplicitParamValue(CGF)), - CGM.getContext().IntTy); + getContext().IntTy); } void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override; @@ -410,6 +420,9 @@ public: if (!isImageRelative()) return PtrVal; + if (PtrVal->isNullValue()) + return llvm::Constant::getNullValue(CGM.IntTy); + llvm::Constant *ImageBaseAsInt = llvm::ConstantExpr::getPtrToInt(getImageBase(), CGM.IntPtrTy); llvm::Constant *PtrValAsInt = @@ -467,6 +480,10 @@ private: return GetVBaseOffsetFromVBPtr(CGF, Base, VBPOffset, VBTOffset, VBPtr); } + std::pair<llvm::Value *, llvm::Value *> + performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, + QualType SrcRecordTy); + /// \brief Performs a full virtual base adjustment. Used to dereference /// pointers to members of virtual bases. llvm::Value *AdjustVirtualBase(CodeGenFunction &CGF, const Expr *E, @@ -479,7 +496,8 @@ private: llvm::Constant *EmitFullMemberPointer(llvm::Constant *FirstField, bool IsMemberFunction, const CXXRecordDecl *RD, - CharUnits NonVirtualBaseAdjustment); + CharUnits NonVirtualBaseAdjustment, + unsigned VBTableIndex); llvm::Constant *BuildMemberPointer(const CXXRecordDecl *RD, const CXXMethodDecl *MD, @@ -556,6 +574,94 @@ public: void emitCXXStructor(const CXXMethodDecl *MD, StructorType Type) override; + llvm::StructType *getCatchHandlerTypeType() { + if (!CatchHandlerTypeType) { + llvm::Type *FieldTypes[] = { + CGM.IntTy, // Flags + CGM.Int8PtrTy, // TypeDescriptor + }; + CatchHandlerTypeType = llvm::StructType::create( + CGM.getLLVMContext(), FieldTypes, "eh.CatchHandlerType"); + } + return CatchHandlerTypeType; + } + + llvm::StructType *getCatchableTypeType() { + if (CatchableTypeType) + return CatchableTypeType; + llvm::Type *FieldTypes[] = { + CGM.IntTy, // Flags + getImageRelativeType(CGM.Int8PtrTy), // TypeDescriptor + CGM.IntTy, // NonVirtualAdjustment + CGM.IntTy, // OffsetToVBPtr + CGM.IntTy, // VBTableIndex + CGM.IntTy, // Size + getImageRelativeType(CGM.Int8PtrTy) // CopyCtor + }; + CatchableTypeType = llvm::StructType::create( + CGM.getLLVMContext(), FieldTypes, "eh.CatchableType"); + return CatchableTypeType; + } + + llvm::StructType *getCatchableTypeArrayType(uint32_t NumEntries) { + llvm::StructType *&CatchableTypeArrayType = + CatchableTypeArrayTypeMap[NumEntries]; + if (CatchableTypeArrayType) + return CatchableTypeArrayType; + + llvm::SmallString<23> CTATypeName("eh.CatchableTypeArray."); + CTATypeName += llvm::utostr(NumEntries); + llvm::Type *CTType = + getImageRelativeType(getCatchableTypeType()->getPointerTo()); + llvm::Type *FieldTypes[] = { + CGM.IntTy, // NumEntries + llvm::ArrayType::get(CTType, NumEntries) // CatchableTypes + }; + CatchableTypeArrayType = + llvm::StructType::create(CGM.getLLVMContext(), FieldTypes, CTATypeName); + return CatchableTypeArrayType; + } + + llvm::StructType *getThrowInfoType() { + if (ThrowInfoType) + return ThrowInfoType; + llvm::Type *FieldTypes[] = { + CGM.IntTy, // Flags + getImageRelativeType(CGM.Int8PtrTy), // CleanupFn + getImageRelativeType(CGM.Int8PtrTy), // ForwardCompat + getImageRelativeType(CGM.Int8PtrTy) // CatchableTypeArray + }; + ThrowInfoType = llvm::StructType::create(CGM.getLLVMContext(), FieldTypes, + "eh.ThrowInfo"); + return ThrowInfoType; + } + + llvm::Constant *getThrowFn() { + // _CxxThrowException is passed an exception object and a ThrowInfo object + // which describes the exception. + llvm::Type *Args[] = {CGM.Int8PtrTy, getThrowInfoType()->getPointerTo()}; + llvm::FunctionType *FTy = + llvm::FunctionType::get(CGM.VoidTy, Args, /*IsVarArgs=*/false); + auto *Fn = cast<llvm::Function>( + CGM.CreateRuntimeFunction(FTy, "_CxxThrowException")); + // _CxxThrowException is stdcall on 32-bit x86 platforms. + if (CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) + Fn->setCallingConv(llvm::CallingConv::X86_StdCall); + return Fn; + } + + llvm::Function *getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT); + + llvm::Constant *getCatchableType(QualType T, + uint32_t NVOffset = 0, + int32_t VBPtrOffset = -1, + uint32_t VBIndex = 0); + + llvm::GlobalVariable *getCatchableTypeArray(QualType T); + + llvm::GlobalVariable *getThrowInfo(QualType T) override; + private: typedef std::pair<const CXXRecordDecl *, CharUnits> VFTableIdTy; typedef llvm::DenseMap<VFTableIdTy, llvm::GlobalVariable *> VTablesMapTy; @@ -582,11 +688,20 @@ private: /// Map from DeclContext to the current guard variable. We assume that the /// AST is visited in source code order. llvm::DenseMap<const DeclContext *, GuardInfo> GuardVariableMap; + llvm::DenseMap<const DeclContext *, GuardInfo> ThreadLocalGuardVariableMap; + llvm::DenseMap<const DeclContext *, unsigned> ThreadSafeGuardNumMap; llvm::DenseMap<size_t, llvm::StructType *> TypeDescriptorTypeMap; llvm::StructType *BaseClassDescriptorType; llvm::StructType *ClassHierarchyDescriptorType; llvm::StructType *CompleteObjectLocatorType; + + llvm::DenseMap<QualType, llvm::GlobalVariable *> CatchableTypeArrays; + + llvm::StructType *CatchableTypeType; + llvm::DenseMap<uint32_t, llvm::StructType *> CatchableTypeArrayTypeMap; + llvm::StructType *ThrowInfoType; + llvm::StructType *CatchHandlerTypeType; }; } @@ -667,55 +782,73 @@ void MicrosoftCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF, CGF.EmitDeleteCall(DE->getOperatorDelete(), MDThis, ElementType); } -static llvm::Function *getRethrowFn(CodeGenModule &CGM) { - // _CxxThrowException takes two pointer width arguments: a value and a context - // object which points to a TypeInfo object. - llvm::Type *ArgTypes[] = {CGM.Int8PtrTy, CGM.Int8PtrTy}; - llvm::FunctionType *FTy = - llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false); - auto *Fn = cast<llvm::Function>( - CGM.CreateRuntimeFunction(FTy, "_CxxThrowException")); - // _CxxThrowException is stdcall on 32-bit x86 platforms. - if (CGM.getTarget().getTriple().getArch() == llvm::Triple::x86) - Fn->setCallingConv(llvm::CallingConv::X86_StdCall); - return Fn; -} - void MicrosoftCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) { - llvm::Value *Args[] = {llvm::ConstantPointerNull::get(CGM.Int8PtrTy), - llvm::ConstantPointerNull::get(CGM.Int8PtrTy)}; - auto *Fn = getRethrowFn(CGM); + llvm::Value *Args[] = { + llvm::ConstantPointerNull::get(CGM.Int8PtrTy), + llvm::ConstantPointerNull::get(getThrowInfoType()->getPointerTo())}; + auto *Fn = getThrowFn(); if (isNoReturn) CGF.EmitNoreturnRuntimeCallOrInvoke(Fn, Args); else CGF.EmitRuntimeCallOrInvoke(Fn, Args); } -/// \brief Gets the offset to the virtual base that contains the vfptr for -/// MS-ABI polymorphic types. -static llvm::Value *getPolymorphicOffset(CodeGenFunction &CGF, - const CXXRecordDecl *RD, - llvm::Value *Value) { - const ASTContext &Context = RD->getASTContext(); - for (const CXXBaseSpecifier &Base : RD->vbases()) - if (Context.getASTRecordLayout(Base.getType()->getAsCXXRecordDecl()) - .hasExtendableVFPtr()) - return CGF.CGM.getCXXABI().GetVirtualBaseClassOffset( - CGF, Value, RD, Base.getType()->getAsCXXRecordDecl()); - llvm_unreachable("One of our vbases should be polymorphic."); +namespace { +struct CallEndCatchMSVC : EHScopeStack::Cleanup { + CallEndCatchMSVC() {} + void Emit(CodeGenFunction &CGF, Flags flags) override { + CGF.EmitNounwindRuntimeCall( + CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_endcatch)); + } +}; } -static std::pair<llvm::Value *, llvm::Value *> -performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, - QualType SrcRecordTy) { +void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF, + const CXXCatchStmt *S) { + // In the MS ABI, the runtime handles the copy, and the catch handler is + // responsible for destruction. + VarDecl *CatchParam = S->getExceptionDecl(); + llvm::Value *Exn = CGF.getExceptionFromSlot(); + llvm::Function *BeginCatch = + CGF.CGM.getIntrinsic(llvm::Intrinsic::eh_begincatch); + + // If this is a catch-all or the catch parameter is unnamed, we don't need to + // emit an alloca to the object. + if (!CatchParam || !CatchParam->getDeclName()) { + llvm::Value *Args[2] = {Exn, llvm::Constant::getNullValue(CGF.Int8PtrTy)}; + CGF.EmitNounwindRuntimeCall(BeginCatch, Args); + CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup); + return; + } + + CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam); + llvm::Value *ParamAddr = + CGF.Builder.CreateBitCast(var.getObjectAddress(CGF), CGF.Int8PtrTy); + llvm::Value *Args[2] = {Exn, ParamAddr}; + CGF.EmitNounwindRuntimeCall(BeginCatch, Args); + CGF.EHStack.pushCleanup<CallEndCatchMSVC>(NormalCleanup); + CGF.EmitAutoVarCleanups(var); +} + +std::pair<llvm::Value *, llvm::Value *> +MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, llvm::Value *Value, + QualType SrcRecordTy) { Value = CGF.Builder.CreateBitCast(Value, CGF.Int8PtrTy); const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); + const ASTContext &Context = getContext(); - if (CGF.getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr()) + if (Context.getASTRecordLayout(SrcDecl).hasExtendableVFPtr()) return std::make_pair(Value, llvm::ConstantInt::get(CGF.Int32Ty, 0)); // Perform a base adjustment. - llvm::Value *Offset = getPolymorphicOffset(CGF, SrcDecl, Value); + const CXXBaseSpecifier *PolymorphicBase = std::find_if( + SrcDecl->vbases_begin(), SrcDecl->vbases_end(), + [&](const CXXBaseSpecifier &Base) { + const CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl(); + return Context.getASTRecordLayout(BaseDecl).hasExtendableVFPtr(); + }); + llvm::Value *Offset = GetVirtualBaseClassOffset( + CGF, Value, SrcDecl, PolymorphicBase->getType()->getAsCXXRecordDecl()); Value = CGF.Builder.CreateInBoundsGEP(Value, Offset); Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty); return std::make_pair(Value, Offset); @@ -725,7 +858,7 @@ bool MicrosoftCXXABI::shouldTypeidBeNullChecked(bool IsDeref, QualType SrcRecordTy) { const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); return IsDeref && - !CGM.getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); + !getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); } static llvm::CallSite emitRTtypeidCall(CodeGenFunction &CGF, @@ -759,7 +892,7 @@ bool MicrosoftCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr, QualType SrcRecordTy) { const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl(); return SrcIsPtr && - !CGM.getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); + !getContext().getASTRecordLayout(SrcDecl).hasExtendableVFPtr(); } llvm::Value *MicrosoftCXXABI::EmitDynamicCastCall( @@ -817,10 +950,11 @@ bool MicrosoftCXXABI::EmitBadCastCall(CodeGenFunction &CGF) { llvm::Value *MicrosoftCXXABI::GetVirtualBaseClassOffset( CodeGenFunction &CGF, llvm::Value *This, const CXXRecordDecl *ClassDecl, const CXXRecordDecl *BaseClassDecl) { + const ASTContext &Context = getContext(); int64_t VBPtrChars = - getContext().getASTRecordLayout(ClassDecl).getVBPtrOffset().getQuantity(); + Context.getASTRecordLayout(ClassDecl).getVBPtrOffset().getQuantity(); llvm::Value *VBPtrOffset = llvm::ConstantInt::get(CGM.PtrDiffTy, VBPtrChars); - CharUnits IntSize = getContext().getTypeSizeInChars(getContext().IntTy); + CharUnits IntSize = Context.getTypeSizeInChars(Context.IntTy); CharUnits VBTableChars = IntSize * CGM.getMicrosoftVTableContext().getVBTableIndex(ClassDecl, BaseClassDecl); @@ -947,30 +1081,52 @@ void MicrosoftCXXABI::initializeHiddenVirtualInheritanceMembers( } } +static bool hasDefaultCXXMethodCC(ASTContext &Context, + const CXXMethodDecl *MD) { + CallingConv ExpectedCallingConv = Context.getDefaultCallingConvention( + /*IsVariadic=*/false, /*IsCXXMethod=*/true); + CallingConv ActualCallingConv = + MD->getType()->getAs<FunctionProtoType>()->getCallConv(); + return ExpectedCallingConv == ActualCallingConv; +} + void MicrosoftCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) { // There's only one constructor type in this ABI. CGM.EmitGlobal(GlobalDecl(D, Ctor_Complete)); + + // Exported default constructors either have a simple call-site where they use + // the typical calling convention and have a single 'this' pointer for an + // argument -or- they get a wrapper function which appropriately thunks to the + // real default constructor. This thunk is the default constructor closure. + if (D->hasAttr<DLLExportAttr>() && D->isDefaultConstructor()) + if (!hasDefaultCXXMethodCC(getContext(), D) || D->getNumParams() != 0) { + llvm::Function *Fn = getAddrOfCXXCtorClosure(D, Ctor_DefaultClosure); + Fn->setLinkage(llvm::GlobalValue::WeakODRLinkage); + Fn->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); + } } void MicrosoftCXXABI::EmitVBPtrStores(CodeGenFunction &CGF, const CXXRecordDecl *RD) { llvm::Value *ThisInt8Ptr = CGF.Builder.CreateBitCast(getThisValue(CGF), CGM.Int8PtrTy, "this.int8"); - const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD); + const ASTContext &Context = getContext(); + const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); const VBTableGlobals &VBGlobals = enumerateVBTables(RD); for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) { const VPtrInfo *VBT = (*VBGlobals.VBTables)[I]; llvm::GlobalVariable *GV = VBGlobals.Globals[I]; const ASTRecordLayout &SubobjectLayout = - CGM.getContext().getASTRecordLayout(VBT->BaseWithVPtr); + Context.getASTRecordLayout(VBT->BaseWithVPtr); CharUnits Offs = VBT->NonVirtualOffset; Offs += SubobjectLayout.getVBPtrOffset(); if (VBT->getVBaseWithVPtr()) Offs += Layout.getVBaseClassOffset(VBT->getVBaseWithVPtr()); llvm::Value *VBPtr = CGF.Builder.CreateConstInBoundsGEP1_64(ThisInt8Ptr, Offs.getQuantity()); - llvm::Value *GVPtr = CGF.Builder.CreateConstInBoundsGEP2_32(GV, 0, 0); + llvm::Value *GVPtr = + CGF.Builder.CreateConstInBoundsGEP2_32(GV->getValueType(), GV, 0, 0); VBPtr = CGF.Builder.CreateBitCast(VBPtr, GVPtr->getType()->getPointerTo(0), "vbptr." + VBT->ReusingBase->getName()); CGF.Builder.CreateStore(GVPtr, VBPtr); @@ -983,7 +1139,7 @@ MicrosoftCXXABI::buildStructorSignature(const CXXMethodDecl *MD, StructorType T, // TODO: 'for base' flag if (T == StructorType::Deleting) { // The scalar deleting destructor takes an implicit int parameter. - ArgTys.push_back(CGM.getContext().IntTy); + ArgTys.push_back(getContext().IntTy); } auto *CD = dyn_cast<CXXConstructorDecl>(MD); if (!CD) @@ -996,9 +1152,9 @@ MicrosoftCXXABI::buildStructorSignature(const CXXMethodDecl *MD, StructorType T, const FunctionProtoType *FPT = CD->getType()->castAs<FunctionProtoType>(); if (Class->getNumVBases()) { if (FPT->isVariadic()) - ArgTys.insert(ArgTys.begin() + 1, CGM.getContext().IntTy); + ArgTys.insert(ArgTys.begin() + 1, getContext().IntTy); else - ArgTys.push_back(CGM.getContext().IntTy); + ArgTys.push_back(getContext().IntTy); } } @@ -1038,7 +1194,7 @@ MicrosoftCXXABI::getVirtualFunctionPrologueThisAdjustment(GlobalDecl GD) { if (ML.VBase) { const ASTRecordLayout &DerivedLayout = - CGM.getContext().getASTRecordLayout(MD->getParent()); + getContext().getASTRecordLayout(MD->getParent()); Adjustment += DerivedLayout.getVBaseClassOffset(ML.VBase); } @@ -1104,7 +1260,7 @@ llvm::Value *MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall( // FIXME: Update the code that emits this adjustment in thunks prologues. This = CGF.Builder.CreateConstGEP1_32(This, StaticOffset.getQuantity()); } else { - This = CGF.Builder.CreateConstInBoundsGEP1_32(This, + This = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, This, StaticOffset.getQuantity()); } } @@ -1159,8 +1315,8 @@ llvm::Value *MicrosoftCXXABI::adjustThisParameterInVirtualFunctionPrologue( This = CGF.Builder.CreateBitCast(This, charPtrTy); assert(Adjustment.isPositive()); - This = - CGF.Builder.CreateConstInBoundsGEP1_32(This, -Adjustment.getQuantity()); + This = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, This, + -Adjustment.getQuantity()); return CGF.Builder.CreateBitCast(This, thisTy); } @@ -1280,7 +1436,7 @@ llvm::Value *MicrosoftCXXABI::getVTableAddressPointInStructor( llvm::GlobalValue *VTableAddressPoint = VFTablesMap[ID]; if (!VTableAddressPoint) { assert(Base.getBase()->getNumVBases() && - !CGM.getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr()); + !getContext().getASTRecordLayout(Base.getBase()).hasOwnVFPtr()); } return VTableAddressPoint; } @@ -1336,98 +1492,96 @@ llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD, #endif } - for (size_t J = 0, F = VFPtrs.size(); J != F; ++J) { - if (VFPtrs[J]->FullOffsetInMDC != VPtrOffset) - continue; - SmallString<256> VFTableName; - mangleVFTableName(getMangleContext(), RD, VFPtrs[J], VFTableName); - StringRef VTableName = VFTableName; - - uint64_t NumVTableSlots = - VTContext.getVFTableLayout(RD, VFPtrs[J]->FullOffsetInMDC) - .getNumVTableComponents(); - llvm::GlobalValue::LinkageTypes VTableLinkage = - llvm::GlobalValue::ExternalLinkage; - llvm::ArrayType *VTableType = - llvm::ArrayType::get(CGM.Int8PtrTy, NumVTableSlots); - if (getContext().getLangOpts().RTTIData) { - VTableLinkage = llvm::GlobalValue::PrivateLinkage; - VTableName = ""; - } - - VTable = CGM.getModule().getNamedGlobal(VFTableName); - if (!VTable) { - // Create a backing variable for the contents of VTable. The VTable may - // or may not include space for a pointer to RTTI data. - llvm::GlobalValue *VFTable = VTable = new llvm::GlobalVariable( - CGM.getModule(), VTableType, /*isConstant=*/true, VTableLinkage, - /*Initializer=*/nullptr, VTableName); - VTable->setUnnamedAddr(true); - - // Only insert a pointer into the VFTable for RTTI data if we are not - // importing it. We never reference the RTTI data directly so there is no - // need to make room for it. - if (getContext().getLangOpts().RTTIData && - !RD->hasAttr<DLLImportAttr>()) { - llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(CGM.IntTy, 0), - llvm::ConstantInt::get(CGM.IntTy, 1)}; - // Create a GEP which points just after the first entry in the VFTable, - // this should be the location of the first virtual method. - llvm::Constant *VTableGEP = - llvm::ConstantExpr::getInBoundsGetElementPtr(VTable, GEPIndices); - // The symbol for the VFTable is an alias to the GEP. It is - // transparent, to other modules, what the nature of this symbol is; all - // that matters is that the alias be the address of the first virtual - // method. - VFTable = llvm::GlobalAlias::create( - cast<llvm::SequentialType>(VTableGEP->getType())->getElementType(), - /*AddressSpace=*/0, llvm::GlobalValue::ExternalLinkage, - VFTableName.str(), VTableGEP, &CGM.getModule()); - } else { - // We don't need a GlobalAlias to be a symbol for the VTable if we won't - // be referencing any RTTI data. The GlobalVariable will end up being - // an appropriate definition of the VFTable. - VTable->setName(VFTableName.str()); - } - - VFTable->setUnnamedAddr(true); - if (RD->hasAttr<DLLImportAttr>()) - VFTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); - else if (RD->hasAttr<DLLExportAttr>()) - VFTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); - - llvm::GlobalValue::LinkageTypes VFTableLinkage = CGM.getVTableLinkage(RD); - if (VFTable != VTable) { - if (llvm::GlobalValue::isAvailableExternallyLinkage(VFTableLinkage)) { - // AvailableExternally implies that we grabbed the data from another - // executable. No need to stick the alias in a Comdat. - } else if (llvm::GlobalValue::isInternalLinkage(VFTableLinkage) || - llvm::GlobalValue::isWeakODRLinkage(VFTableLinkage) || - llvm::GlobalValue::isLinkOnceODRLinkage(VFTableLinkage)) { - // The alias is going to be dropped into a Comdat, no need to make it - // weak. - if (!llvm::GlobalValue::isInternalLinkage(VFTableLinkage)) - VFTableLinkage = llvm::GlobalValue::ExternalLinkage; - llvm::Comdat *C = - CGM.getModule().getOrInsertComdat(VFTable->getName()); - // We must indicate which VFTable is larger to support linking between - // translation units which do and do not have RTTI data. The largest - // VFTable contains the RTTI data; translation units which reference - // the smaller VFTable always reference it relative to the first - // virtual method. - C->setSelectionKind(llvm::Comdat::Largest); - VTable->setComdat(C); - } else { - llvm_unreachable("unexpected linkage for vftable!"); - } - } - VFTable->setLinkage(VFTableLinkage); - CGM.setGlobalVisibility(VFTable, RD); - VFTablesMap[ID] = VFTable; + VPtrInfo *const *VFPtrI = + std::find_if(VFPtrs.begin(), VFPtrs.end(), [&](VPtrInfo *VPI) { + return VPI->FullOffsetInMDC == VPtrOffset; + }); + if (VFPtrI == VFPtrs.end()) { + VFTablesMap[ID] = nullptr; + return nullptr; + } + VPtrInfo *VFPtr = *VFPtrI; + + SmallString<256> VFTableName; + mangleVFTableName(getMangleContext(), RD, VFPtr, VFTableName); + + llvm::GlobalValue::LinkageTypes VFTableLinkage = CGM.getVTableLinkage(RD); + bool VFTableComesFromAnotherTU = + llvm::GlobalValue::isAvailableExternallyLinkage(VFTableLinkage) || + llvm::GlobalValue::isExternalLinkage(VFTableLinkage); + bool VTableAliasIsRequred = + !VFTableComesFromAnotherTU && getContext().getLangOpts().RTTIData; + + if (llvm::GlobalValue *VFTable = + CGM.getModule().getNamedGlobal(VFTableName)) { + VFTablesMap[ID] = VFTable; + return VTableAliasIsRequred + ? cast<llvm::GlobalVariable>( + cast<llvm::GlobalAlias>(VFTable)->getBaseObject()) + : cast<llvm::GlobalVariable>(VFTable); + } + + uint64_t NumVTableSlots = + VTContext.getVFTableLayout(RD, VFPtr->FullOffsetInMDC) + .getNumVTableComponents(); + llvm::GlobalValue::LinkageTypes VTableLinkage = + VTableAliasIsRequred ? llvm::GlobalValue::PrivateLinkage : VFTableLinkage; + + StringRef VTableName = VTableAliasIsRequred ? StringRef() : VFTableName.str(); + + llvm::ArrayType *VTableType = + llvm::ArrayType::get(CGM.Int8PtrTy, NumVTableSlots); + + // Create a backing variable for the contents of VTable. The VTable may + // or may not include space for a pointer to RTTI data. + llvm::GlobalValue *VFTable; + VTable = new llvm::GlobalVariable(CGM.getModule(), VTableType, + /*isConstant=*/true, VTableLinkage, + /*Initializer=*/nullptr, VTableName); + VTable->setUnnamedAddr(true); + + llvm::Comdat *C = nullptr; + if (!VFTableComesFromAnotherTU && + (llvm::GlobalValue::isWeakForLinker(VFTableLinkage) || + (llvm::GlobalValue::isLocalLinkage(VFTableLinkage) && + VTableAliasIsRequred))) + C = CGM.getModule().getOrInsertComdat(VFTableName.str()); + + // Only insert a pointer into the VFTable for RTTI data if we are not + // importing it. We never reference the RTTI data directly so there is no + // need to make room for it. + if (VTableAliasIsRequred) { + llvm::Value *GEPIndices[] = {llvm::ConstantInt::get(CGM.IntTy, 0), + llvm::ConstantInt::get(CGM.IntTy, 1)}; + // Create a GEP which points just after the first entry in the VFTable, + // this should be the location of the first virtual method. + llvm::Constant *VTableGEP = llvm::ConstantExpr::getInBoundsGetElementPtr( + VTable->getValueType(), VTable, GEPIndices); + if (llvm::GlobalValue::isWeakForLinker(VFTableLinkage)) { + VFTableLinkage = llvm::GlobalValue::ExternalLinkage; + if (C) + C->setSelectionKind(llvm::Comdat::Largest); } - break; + VFTable = llvm::GlobalAlias::create( + cast<llvm::PointerType>(VTableGEP->getType()), VFTableLinkage, + VFTableName.str(), VTableGEP, &CGM.getModule()); + VFTable->setUnnamedAddr(true); + } else { + // We don't need a GlobalAlias to be a symbol for the VTable if we won't + // be referencing any RTTI data. + // The GlobalVariable will end up being an appropriate definition of the + // VFTable. + VFTable = VTable; } + if (C) + VTable->setComdat(C); + + if (RD->hasAttr<DLLImportAttr>()) + VFTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass); + else if (RD->hasAttr<DLLExportAttr>()) + VFTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); + VFTablesMap[ID] = VFTable; return VTable; } @@ -1464,7 +1618,7 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall( llvm::Type *Ty = CGF.CGM.getTypes().GetFunctionType(*FInfo); llvm::Value *Callee = getVirtualFunctionPointer(CGF, GD, This, Ty); - ASTContext &Context = CGF.getContext(); + ASTContext &Context = getContext(); llvm::Value *ImplicitParam = llvm::ConstantInt::get( llvm::IntegerType::getInt32Ty(CGF.getLLVMContext()), DtorType == Dtor_Deleting); @@ -1530,6 +1684,8 @@ llvm::Function *MicrosoftCXXABI::EmitVirtualMemPtrThunk( ThunkFn->setLinkage(MD->isExternallyVisible() ? llvm::GlobalValue::LinkOnceODRLinkage : llvm::GlobalValue::InternalLinkage); + if (MD->isExternallyVisible()) + ThunkFn->setComdat(CGM.getModule().getOrInsertComdat(ThunkFn->getName())); CGM.SetLLVMFunctionAttributes(MD, FnInfo, ThunkFn); CGM.SetLLVMFunctionAttributesForDefinition(MD, ThunkFn); @@ -1576,7 +1732,8 @@ void MicrosoftCXXABI::emitVirtualInheritanceTables(const CXXRecordDecl *RD) { for (unsigned I = 0, E = VBGlobals.VBTables->size(); I != E; ++I) { const VPtrInfo *VBT = (*VBGlobals.VBTables)[I]; llvm::GlobalVariable *GV = VBGlobals.Globals[I]; - emitVBTableDefinition(*VBT, RD, GV); + if (GV->isDeclaration()) + emitVBTableDefinition(*VBT, RD, GV); } } @@ -1603,6 +1760,9 @@ MicrosoftCXXABI::getAddrOfVBTable(const VPtrInfo &VBT, const CXXRecordDecl *RD, else if (RD->hasAttr<DLLExportAttr>()) GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass); + if (!GV->hasExternalLinkage()) + emitVBTableDefinition(VBT, RD, GV); + return GV; } @@ -1615,9 +1775,8 @@ void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, "should only emit vbtables for classes with vbtables"); const ASTRecordLayout &BaseLayout = - CGM.getContext().getASTRecordLayout(VBT.BaseWithVPtr); - const ASTRecordLayout &DerivedLayout = - CGM.getContext().getASTRecordLayout(RD); + getContext().getASTRecordLayout(VBT.BaseWithVPtr); + const ASTRecordLayout &DerivedLayout = getContext().getASTRecordLayout(RD); SmallVector<llvm::Constant *, 4> Offsets(1 + ReusingBase->getNumVBases(), nullptr); @@ -1651,9 +1810,6 @@ void MicrosoftCXXABI::emitVBTableDefinition(const VPtrInfo &VBT, llvm::ArrayType::get(CGM.IntTy, Offsets.size()); llvm::Constant *Init = llvm::ConstantArray::get(VBTableType, Offsets); GV->setInitializer(Init); - - // Set the right visibility. - CGM.setGlobalVisibility(GV, RD); } llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, @@ -1709,8 +1865,8 @@ MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, if (RA.Virtual.Microsoft.VBIndex) { assert(RA.Virtual.Microsoft.VBIndex > 0); - int32_t IntSize = - getContext().getTypeSizeInChars(getContext().IntTy).getQuantity(); + const ASTContext &Context = getContext(); + int32_t IntSize = Context.getTypeSizeInChars(Context.IntTy).getQuantity(); llvm::Value *VBPtr; llvm::Value *VBaseOffset = GetVBaseOffsetFromVBPtr(CGF, V, RA.Virtual.Microsoft.VBPtrOffset, @@ -1719,7 +1875,7 @@ MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, llvm::Value *Ret, } if (RA.NonVirtual) - V = CGF.Builder.CreateConstInBoundsGEP1_32(V, RA.NonVirtual); + V = CGF.Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, V, RA.NonVirtual); // Cast back to the original type. return CGF.Builder.CreateBitCast(V, Ret->getType()); @@ -1835,18 +1991,10 @@ void MicrosoftCXXABI::EmitThreadLocalInitFuncs( llvm::Function *F = CXXThreadLocalInits[I]; // If the GV is already in a comdat group, then we have to join it. - llvm::Comdat *C = GV->getComdat(); - - // LinkOnce and Weak linkage are lowered down to a single-member comdat - // group. - // Make an explicit group so we can join it. - if (!C && (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage())) { - C = CGM.getModule().getOrInsertComdat(GV->getName()); - GV->setComdat(C); + if (llvm::Comdat *C = GV->getComdat()) AddToXDU(F)->setComdat(C); - } else { + else NonComdatInits.push_back(F); - } } if (!NonComdatInits.empty()) { @@ -1868,6 +2016,81 @@ LValue MicrosoftCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, return LValue(); } +static llvm::GlobalVariable *getInitThreadEpochPtr(CodeGenModule &CGM) { + StringRef VarName("_Init_thread_epoch"); + if (auto *GV = CGM.getModule().getNamedGlobal(VarName)) + return GV; + auto *GV = new llvm::GlobalVariable( + CGM.getModule(), CGM.IntTy, + /*Constant=*/false, llvm::GlobalVariable::ExternalLinkage, + /*Initializer=*/nullptr, VarName, + /*InsertBefore=*/nullptr, llvm::GlobalVariable::GeneralDynamicTLSModel); + GV->setAlignment(CGM.getTarget().getIntAlign() / 8); + return GV; +} + +static llvm::Constant *getInitThreadHeaderFn(CodeGenModule &CGM) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()), + CGM.IntTy->getPointerTo(), /*isVarArg=*/false); + return CGM.CreateRuntimeFunction( + FTy, "_Init_thread_header", + llvm::AttributeSet::get(CGM.getLLVMContext(), + llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoUnwind)); +} + +static llvm::Constant *getInitThreadFooterFn(CodeGenModule &CGM) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()), + CGM.IntTy->getPointerTo(), /*isVarArg=*/false); + return CGM.CreateRuntimeFunction( + FTy, "_Init_thread_footer", + llvm::AttributeSet::get(CGM.getLLVMContext(), + llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoUnwind)); +} + +static llvm::Constant *getInitThreadAbortFn(CodeGenModule &CGM) { + llvm::FunctionType *FTy = + llvm::FunctionType::get(llvm::Type::getVoidTy(CGM.getLLVMContext()), + CGM.IntTy->getPointerTo(), /*isVarArg=*/false); + return CGM.CreateRuntimeFunction( + FTy, "_Init_thread_abort", + llvm::AttributeSet::get(CGM.getLLVMContext(), + llvm::AttributeSet::FunctionIndex, + llvm::Attribute::NoUnwind)); +} + +namespace { +struct ResetGuardBit : EHScopeStack::Cleanup { + llvm::GlobalVariable *Guard; + unsigned GuardNum; + ResetGuardBit(llvm::GlobalVariable *Guard, unsigned GuardNum) + : Guard(Guard), GuardNum(GuardNum) {} + + void Emit(CodeGenFunction &CGF, Flags flags) override { + // Reset the bit in the mask so that the static variable may be + // reinitialized. + CGBuilderTy &Builder = CGF.Builder; + llvm::LoadInst *LI = Builder.CreateLoad(Guard); + llvm::ConstantInt *Mask = + llvm::ConstantInt::get(CGF.IntTy, ~(1U << GuardNum)); + Builder.CreateStore(Builder.CreateAnd(LI, Mask), Guard); + } +}; + +struct CallInitThreadAbort : EHScopeStack::Cleanup { + llvm::GlobalVariable *Guard; + CallInitThreadAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {} + + void Emit(CodeGenFunction &CGF, Flags flags) override { + // Calling _Init_thread_abort will reset the guard's state. + CGF.EmitNounwindRuntimeCall(getInitThreadAbortFn(CGF.CGM), Guard); + } +}; +} + void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *GV, bool PerformInit) { @@ -1875,91 +2098,161 @@ void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D, if (!D.isStaticLocal()) { assert(GV->hasWeakLinkage() || GV->hasLinkOnceLinkage()); // GlobalOpt is allowed to discard the initializer, so use linkonce_odr. - CGF.CurFn->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage); + llvm::Function *F = CGF.CurFn; + F->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage); + F->setComdat(CGM.getModule().getOrInsertComdat(F->getName())); CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); return; } - // MSVC always uses an i32 bitfield to guard initialization, which is *not* - // threadsafe. Since the user may be linking in inline functions compiled by - // cl.exe, there's no reason to provide a false sense of security by using - // critical sections here. + bool ThreadlocalStatic = D.getTLSKind(); + bool ThreadsafeStatic = getContext().getLangOpts().ThreadsafeStatics; - if (D.getTLSKind()) - CGM.ErrorUnsupported(&D, "dynamic TLS initialization"); + // Thread-safe static variables which aren't thread-specific have a + // per-variable guard. + bool HasPerVariableGuard = ThreadsafeStatic && !ThreadlocalStatic; CGBuilderTy &Builder = CGF.Builder; llvm::IntegerType *GuardTy = CGF.Int32Ty; llvm::ConstantInt *Zero = llvm::ConstantInt::get(GuardTy, 0); // Get the guard variable for this function if we have one already. - GuardInfo *GI = &GuardVariableMap[D.getDeclContext()]; - - unsigned BitIndex; - if (D.isStaticLocal() && D.isExternallyVisible()) { + GuardInfo *GI = nullptr; + if (ThreadlocalStatic) + GI = &ThreadLocalGuardVariableMap[D.getDeclContext()]; + else if (!ThreadsafeStatic) + GI = &GuardVariableMap[D.getDeclContext()]; + + llvm::GlobalVariable *GuardVar = GI ? GI->Guard : nullptr; + unsigned GuardNum; + if (D.isExternallyVisible()) { // Externally visible variables have to be numbered in Sema to properly // handle unreachable VarDecls. - BitIndex = getContext().getStaticLocalNumber(&D); - assert(BitIndex > 0); - BitIndex--; + GuardNum = getContext().getStaticLocalNumber(&D); + assert(GuardNum > 0); + GuardNum--; + } else if (HasPerVariableGuard) { + GuardNum = ThreadSafeGuardNumMap[D.getDeclContext()]++; } else { // Non-externally visible variables are numbered here in CodeGen. - BitIndex = GI->BitIndex++; + GuardNum = GI->BitIndex++; } - if (BitIndex >= 32) { + if (!HasPerVariableGuard && GuardNum >= 32) { if (D.isExternallyVisible()) ErrorUnsupportedABI(CGF, "more than 32 guarded initializations"); - BitIndex %= 32; - GI->Guard = nullptr; + GuardNum %= 32; + GuardVar = nullptr; } - // Lazily create the i32 bitfield for this function. - if (!GI->Guard) { + if (!GuardVar) { // Mangle the name for the guard. SmallString<256> GuardName; { llvm::raw_svector_ostream Out(GuardName); - getMangleContext().mangleStaticGuardVariable(&D, Out); + if (HasPerVariableGuard) + getMangleContext().mangleThreadSafeStaticGuardVariable(&D, GuardNum, + Out); + else + getMangleContext().mangleStaticGuardVariable(&D, Out); Out.flush(); } // Create the guard variable with a zero-initializer. Just absorb linkage, // visibility and dll storage class from the guarded variable. - GI->Guard = - new llvm::GlobalVariable(CGM.getModule(), GuardTy, false, + GuardVar = + new llvm::GlobalVariable(CGM.getModule(), GuardTy, /*isConstant=*/false, GV->getLinkage(), Zero, GuardName.str()); - GI->Guard->setVisibility(GV->getVisibility()); - GI->Guard->setDLLStorageClass(GV->getDLLStorageClass()); - } else { - assert(GI->Guard->getLinkage() == GV->getLinkage() && - "static local from the same function had different linkage"); - } - - // Pseudo code for the test: - // if (!(GuardVar & MyGuardBit)) { - // GuardVar |= MyGuardBit; - // ... initialize the object ...; - // } - - // Test our bit from the guard variable. - llvm::ConstantInt *Bit = llvm::ConstantInt::get(GuardTy, 1U << BitIndex); - llvm::LoadInst *LI = Builder.CreateLoad(GI->Guard); - llvm::Value *IsInitialized = - Builder.CreateICmpNE(Builder.CreateAnd(LI, Bit), Zero); - llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init"); - llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end"); - Builder.CreateCondBr(IsInitialized, EndBlock, InitBlock); + GuardVar->setVisibility(GV->getVisibility()); + GuardVar->setDLLStorageClass(GV->getDLLStorageClass()); + if (GuardVar->isWeakForLinker()) + GuardVar->setComdat( + CGM.getModule().getOrInsertComdat(GuardVar->getName())); + if (D.getTLSKind()) + GuardVar->setThreadLocal(true); + if (GI && !HasPerVariableGuard) + GI->Guard = GuardVar; + } + + assert(GuardVar->getLinkage() == GV->getLinkage() && + "static local from the same function had different linkage"); + + if (!HasPerVariableGuard) { + // Pseudo code for the test: + // if (!(GuardVar & MyGuardBit)) { + // GuardVar |= MyGuardBit; + // ... initialize the object ...; + // } + + // Test our bit from the guard variable. + llvm::ConstantInt *Bit = llvm::ConstantInt::get(GuardTy, 1U << GuardNum); + llvm::LoadInst *LI = Builder.CreateLoad(GuardVar); + llvm::Value *IsInitialized = + Builder.CreateICmpNE(Builder.CreateAnd(LI, Bit), Zero); + llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init"); + llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end"); + Builder.CreateCondBr(IsInitialized, EndBlock, InitBlock); + + // Set our bit in the guard variable and emit the initializer and add a global + // destructor if appropriate. + CGF.EmitBlock(InitBlock); + Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardVar); + CGF.EHStack.pushCleanup<ResetGuardBit>(EHCleanup, GuardVar, GuardNum); + CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); + CGF.PopCleanupBlock(); + Builder.CreateBr(EndBlock); - // Set our bit in the guard variable and emit the initializer and add a global - // destructor if appropriate. - CGF.EmitBlock(InitBlock); - Builder.CreateStore(Builder.CreateOr(LI, Bit), GI->Guard); - CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); - Builder.CreateBr(EndBlock); + // Continue. + CGF.EmitBlock(EndBlock); + } else { + // Pseudo code for the test: + // if (TSS > _Init_thread_epoch) { + // _Init_thread_header(&TSS); + // if (TSS == -1) { + // ... initialize the object ...; + // _Init_thread_footer(&TSS); + // } + // } + // + // The algorithm is almost identical to what can be found in the appendix + // found in N2325. + + unsigned IntAlign = CGM.getTarget().getIntAlign() / 8; + + // This BasicBLock determines whether or not we have any work to do. + llvm::LoadInst *FirstGuardLoad = + Builder.CreateAlignedLoad(GuardVar, IntAlign); + FirstGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered); + llvm::LoadInst *InitThreadEpoch = + Builder.CreateLoad(getInitThreadEpochPtr(CGM)); + llvm::Value *IsUninitialized = + Builder.CreateICmpSGT(FirstGuardLoad, InitThreadEpoch); + llvm::BasicBlock *AttemptInitBlock = CGF.createBasicBlock("init.attempt"); + llvm::BasicBlock *EndBlock = CGF.createBasicBlock("init.end"); + Builder.CreateCondBr(IsUninitialized, AttemptInitBlock, EndBlock); + + // This BasicBlock attempts to determine whether or not this thread is + // responsible for doing the initialization. + CGF.EmitBlock(AttemptInitBlock); + CGF.EmitNounwindRuntimeCall(getInitThreadHeaderFn(CGM), GuardVar); + llvm::LoadInst *SecondGuardLoad = + Builder.CreateAlignedLoad(GuardVar, IntAlign); + SecondGuardLoad->setOrdering(llvm::AtomicOrdering::Unordered); + llvm::Value *ShouldDoInit = + Builder.CreateICmpEQ(SecondGuardLoad, getAllOnesInt()); + llvm::BasicBlock *InitBlock = CGF.createBasicBlock("init"); + Builder.CreateCondBr(ShouldDoInit, InitBlock, EndBlock); + + // Ok, we ended up getting selected as the initializing thread. + CGF.EmitBlock(InitBlock); + CGF.EHStack.pushCleanup<CallInitThreadAbort>(EHCleanup, GuardVar); + CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit); + CGF.PopCleanupBlock(); + CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM), GuardVar); + Builder.CreateBr(EndBlock); - // Continue. - CGF.EmitBlock(EndBlock); + CGF.EmitBlock(EndBlock); + } } bool MicrosoftCXXABI::isZeroInitializable(const MemberPointerType *MPT) { @@ -2040,8 +2333,8 @@ llvm::Constant * MicrosoftCXXABI::EmitFullMemberPointer(llvm::Constant *FirstField, bool IsMemberFunction, const CXXRecordDecl *RD, - CharUnits NonVirtualBaseAdjustment) -{ + CharUnits NonVirtualBaseAdjustment, + unsigned VBTableIndex) { MSInheritanceAttr::Spelling Inheritance = RD->getMSInheritanceModel(); // Single inheritance class member pointer are represented as scalars instead @@ -2065,7 +2358,7 @@ MicrosoftCXXABI::EmitFullMemberPointer(llvm::Constant *FirstField, // The rest of the fields are adjusted by conversions to a more derived class. if (MSInheritanceAttr::hasVBTableOffsetField(Inheritance)) - fields.push_back(getZeroInt()); + fields.push_back(llvm::ConstantInt::get(CGM.IntTy, VBTableIndex)); return llvm::ConstantStruct::getAnon(fields); } @@ -2077,7 +2370,7 @@ MicrosoftCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT, llvm::Constant *FirstField = llvm::ConstantInt::get(CGM.IntTy, offset.getQuantity()); return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/false, RD, - CharUnits::Zero()); + CharUnits::Zero(), /*VBTableIndex=*/0); } llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) { @@ -2113,6 +2406,7 @@ MicrosoftCXXABI::BuildMemberPointer(const CXXRecordDecl *RD, RD = RD->getMostRecentDecl(); CodeGenTypes &Types = CGM.getTypes(); + unsigned VBTableIndex = 0; llvm::Constant *FirstField; const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>(); if (!MD->isVirtual()) { @@ -2129,8 +2423,6 @@ MicrosoftCXXABI::BuildMemberPointer(const CXXRecordDecl *RD, FirstField = CGM.GetAddrOfFunction(MD, Ty); FirstField = llvm::ConstantExpr::getBitCast(FirstField, CGM.VoidPtrTy); } else { - MicrosoftVTableContext::MethodVFTableLocation ML = - CGM.getMicrosoftVTableContext().getMethodVFTableLocation(MD); if (!CGM.getTypes().isFuncTypeConvertible( MD->getType()->castAs<FunctionType>())) { CGM.ErrorUnsupported(MD, "pointer to virtual member function with " @@ -2139,21 +2431,22 @@ MicrosoftCXXABI::BuildMemberPointer(const CXXRecordDecl *RD, } else if (FPT->getCallConv() == CC_X86FastCall) { CGM.ErrorUnsupported(MD, "pointer to fastcall virtual member function"); FirstField = llvm::Constant::getNullValue(CGM.VoidPtrTy); - } else if (ML.VBase) { - CGM.ErrorUnsupported(MD, "pointer to virtual member function overriding " - "member function in virtual base class"); - FirstField = llvm::Constant::getNullValue(CGM.VoidPtrTy); } else { + auto &VTableContext = CGM.getMicrosoftVTableContext(); + MicrosoftVTableContext::MethodVFTableLocation ML = + VTableContext.getMethodVFTableLocation(MD); llvm::Function *Thunk = EmitVirtualMemPtrThunk(MD, ML); FirstField = llvm::ConstantExpr::getBitCast(Thunk, CGM.VoidPtrTy); // Include the vfptr adjustment if the method is in a non-primary vftable. NonVirtualBaseAdjustment += ML.VFPtrOffset; + if (ML.VBase) + VBTableIndex = VTableContext.getVBTableIndex(RD, ML.VBase) * 4; } } // The rest of the fields are common with data member pointers. return EmitFullMemberPointer(FirstField, /*IsMemberFunction=*/true, RD, - NonVirtualBaseAdjustment); + NonVirtualBaseAdjustment, VBTableIndex); } /// Member pointers are the same if they're either bitwise identical *or* both @@ -2867,7 +3160,11 @@ llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() { auto Type = ABI.getClassHierarchyDescriptorType(); auto CHD = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, /*Initializer=*/nullptr, - MangledName.c_str()); + StringRef(MangledName)); + if (CHD->isWeakForLinker()) + CHD->setComdat(CGM.getModule().getOrInsertComdat(CHD->getName())); + + auto *Bases = getBaseClassArray(Classes); // Initialize the base class ClassHierarchyDescriptor. llvm::Constant *Fields[] = { @@ -2875,7 +3172,7 @@ llvm::GlobalVariable *MSRTTIBuilder::getClassHierarchyDescriptor() { llvm::ConstantInt::get(CGM.IntTy, Flags), llvm::ConstantInt::get(CGM.IntTy, Classes.size()), ABI.getImageRelativeConstant(llvm::ConstantExpr::getInBoundsGetElementPtr( - getBaseClassArray(Classes), + Bases->getValueType(), Bases, llvm::ArrayRef<llvm::Value *>(GEPIndices))), }; CHD->setInitializer(llvm::ConstantStruct::get(Type, Fields)); @@ -2898,9 +3195,12 @@ MSRTTIBuilder::getBaseClassArray(SmallVectorImpl<MSRTTIClass> &Classes) { llvm::Type *PtrType = ABI.getImageRelativeType( ABI.getBaseClassDescriptorType()->getPointerTo()); auto *ArrType = llvm::ArrayType::get(PtrType, Classes.size() + 1); - auto *BCA = new llvm::GlobalVariable( - Module, ArrType, - /*Constant=*/true, Linkage, /*Initializer=*/nullptr, MangledName.c_str()); + auto *BCA = + new llvm::GlobalVariable(Module, ArrType, + /*Constant=*/true, Linkage, + /*Initializer=*/nullptr, StringRef(MangledName)); + if (BCA->isWeakForLinker()) + BCA->setComdat(CGM.getModule().getOrInsertComdat(BCA->getName())); // Initialize the BaseClassArray. SmallVector<llvm::Constant *, 8> BaseClassArrayData; @@ -2938,9 +3238,11 @@ MSRTTIBuilder::getBaseClassDescriptor(const MSRTTIClass &Class) { // Forward-declare the base class descriptor. auto Type = ABI.getBaseClassDescriptorType(); - auto BCD = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, - /*Initializer=*/nullptr, - MangledName.c_str()); + auto BCD = + new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, + /*Initializer=*/nullptr, StringRef(MangledName)); + if (BCD->isWeakForLinker()) + BCD->setComdat(CGM.getModule().getOrInsertComdat(BCD->getName())); // Initialize the BaseClassDescriptor. llvm::Constant *Fields[] = { @@ -2984,7 +3286,7 @@ MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo *Info) { // Forward-declare the complete object locator. llvm::StructType *Type = ABI.getCompleteObjectLocatorType(); auto COL = new llvm::GlobalVariable(Module, Type, /*Constant=*/true, Linkage, - /*Initializer=*/nullptr, MangledName.c_str()); + /*Initializer=*/nullptr, StringRef(MangledName)); // Initialize the CompleteObjectLocator. llvm::Constant *Fields[] = { @@ -3000,15 +3302,91 @@ MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo *Info) { if (!ABI.isImageRelative()) FieldsRef = FieldsRef.drop_back(); COL->setInitializer(llvm::ConstantStruct::get(Type, FieldsRef)); + if (COL->isWeakForLinker()) + COL->setComdat(CGM.getModule().getOrInsertComdat(COL->getName())); return COL; } +static QualType decomposeTypeForEH(ASTContext &Context, QualType T, + bool &IsConst, bool &IsVolatile) { + T = Context.getExceptionObjectType(T); + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if [...] + // - the handler is of type cv T or const T& where T is a pointer type and + // E is a pointer type that can be converted to T by [...] + // - a qualification conversion + IsConst = false; + IsVolatile = false; + QualType PointeeType = T->getPointeeType(); + if (!PointeeType.isNull()) { + IsConst = PointeeType.isConstQualified(); + IsVolatile = PointeeType.isVolatileQualified(); + } + + // Member pointer types like "const int A::*" are represented by having RTTI + // for "int A::*" and separately storing the const qualifier. + if (const auto *MPTy = T->getAs<MemberPointerType>()) + T = Context.getMemberPointerType(PointeeType.getUnqualifiedType(), + MPTy->getClass()); + + // Pointer types like "const int * const *" are represented by having RTTI + // for "const int **" and separately storing the const qualifier. + if (T->isPointerType()) + T = Context.getPointerType(PointeeType.getUnqualifiedType()); + + return T; +} + +llvm::Constant * +MicrosoftCXXABI::getAddrOfCXXCatchHandlerType(QualType Type, + QualType CatchHandlerType) { + // TypeDescriptors for exceptions never have qualified pointer types, + // qualifiers are stored seperately in order to support qualification + // conversions. + bool IsConst, IsVolatile; + Type = decomposeTypeForEH(getContext(), Type, IsConst, IsVolatile); + + bool IsReference = CatchHandlerType->isReferenceType(); + + uint32_t Flags = 0; + if (IsConst) + Flags |= 1; + if (IsVolatile) + Flags |= 2; + if (IsReference) + Flags |= 8; + + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXCatchHandlerType(Type, Flags, Out); + } + + if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName)) + return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy); + + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags + getAddrOfRTTIDescriptor(Type), // TypeDescriptor + }; + llvm::StructType *CatchHandlerTypeType = getCatchHandlerTypeType(); + auto *Var = new llvm::GlobalVariable( + CGM.getModule(), CatchHandlerTypeType, /*Constant=*/true, + llvm::GlobalValue::PrivateLinkage, + llvm::ConstantStruct::get(CatchHandlerTypeType, Fields), + StringRef(MangledName)); + Var->setUnnamedAddr(true); + Var->setSection("llvm.metadata"); + return Var; +} + /// \brief Gets a TypeDescriptor. Returns a llvm::Constant * rather than a /// llvm::GlobalVariable * because different type descriptors have different /// types, and need to be abstracted. They are abstracting by casting the /// address to an Int8PtrTy. llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { - SmallString<256> MangledName, TypeInfoString; + SmallString<256> MangledName; { llvm::raw_svector_ostream Out(MangledName); getMangleContext().mangleCXXRTTI(Type, Out); @@ -3019,6 +3397,7 @@ llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { return llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy); // Compute the fields for the TypeDescriptor. + SmallString<256> TypeInfoString; { llvm::raw_svector_ostream Out(TypeInfoString); getMangleContext().mangleCXXRTTIName(Type, Out); @@ -3031,13 +3410,14 @@ llvm::Constant *MicrosoftCXXABI::getAddrOfRTTIDescriptor(QualType Type) { llvm::ConstantDataArray::getString(CGM.getLLVMContext(), TypeInfoString)}; llvm::StructType *TypeDescriptorType = getTypeDescriptorType(TypeInfoString); - return llvm::ConstantExpr::getBitCast( - new llvm::GlobalVariable( - CGM.getModule(), TypeDescriptorType, /*Constant=*/false, - getLinkageForRTTI(Type), - llvm::ConstantStruct::get(TypeDescriptorType, Fields), - MangledName.c_str()), - CGM.Int8PtrTy); + auto *Var = new llvm::GlobalVariable( + CGM.getModule(), TypeDescriptorType, /*Constant=*/false, + getLinkageForRTTI(Type), + llvm::ConstantStruct::get(TypeDescriptorType, Fields), + StringRef(MangledName)); + if (Var->isWeakForLinker()) + Var->setComdat(CGM.getModule().getOrInsertComdat(Var->getName())); + return llvm::ConstantExpr::getBitCast(Var, CGM.Int8PtrTy); } /// \brief Gets or a creates a Microsoft CompleteObjectLocator. @@ -3051,7 +3431,8 @@ static void emitCXXConstructor(CodeGenModule &CGM, const CXXConstructorDecl *ctor, StructorType ctorType) { // There are no constructor variants, always emit the complete destructor. - CGM.codegenCXXStructor(ctor, StructorType::Complete); + llvm::Function *Fn = CGM.codegenCXXStructor(ctor, StructorType::Complete); + CGM.maybeSetTrivialComdat(*ctor, *Fn); } static void emitCXXDestructor(CodeGenModule &CGM, const CXXDestructorDecl *dtor, @@ -3077,7 +3458,9 @@ static void emitCXXDestructor(CodeGenModule &CGM, const CXXDestructorDecl *dtor, if (dtorType == StructorType::Base && !CGM.TryEmitBaseDestructorAsAlias(dtor)) return; - CGM.codegenCXXStructor(dtor, dtorType); + llvm::Function *Fn = CGM.codegenCXXStructor(dtor, dtorType); + if (Fn->isWeakForLinker()) + Fn->setComdat(CGM.getModule().getOrInsertComdat(Fn->getName())); } void MicrosoftCXXABI::emitCXXStructor(const CXXMethodDecl *MD, @@ -3088,3 +3471,400 @@ void MicrosoftCXXABI::emitCXXStructor(const CXXMethodDecl *MD, } emitCXXDestructor(CGM, cast<CXXDestructorDecl>(MD), Type); } + +llvm::Function * +MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD, + CXXCtorType CT) { + assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure); + + // Calculate the mangled name. + SmallString<256> ThunkName; + llvm::raw_svector_ostream Out(ThunkName); + getMangleContext().mangleCXXCtor(CD, CT, Out); + Out.flush(); + + // If the thunk has been generated previously, just return it. + if (llvm::GlobalValue *GV = CGM.getModule().getNamedValue(ThunkName)) + return cast<llvm::Function>(GV); + + // Create the llvm::Function. + const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeMSCtorClosure(CD, CT); + llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(FnInfo); + const CXXRecordDecl *RD = CD->getParent(); + QualType RecordTy = getContext().getRecordType(RD); + llvm::Function *ThunkFn = llvm::Function::Create( + ThunkTy, getLinkageForRTTI(RecordTy), ThunkName.str(), &CGM.getModule()); + ThunkFn->setCallingConv(static_cast<llvm::CallingConv::ID>( + FnInfo.getEffectiveCallingConvention())); + bool IsCopy = CT == Ctor_CopyingClosure; + + // Start codegen. + CodeGenFunction CGF(CGM); + CGF.CurGD = GlobalDecl(CD, Ctor_Complete); + + // Build FunctionArgs. + FunctionArgList FunctionArgs; + + // A constructor always starts with a 'this' pointer as its first argument. + buildThisParam(CGF, FunctionArgs); + + // Following the 'this' pointer is a reference to the source object that we + // are copying from. + ImplicitParamDecl SrcParam( + getContext(), nullptr, SourceLocation(), &getContext().Idents.get("src"), + getContext().getLValueReferenceType(RecordTy, + /*SpelledAsLValue=*/true)); + if (IsCopy) + FunctionArgs.push_back(&SrcParam); + + // Constructors for classes which utilize virtual bases have an additional + // parameter which indicates whether or not it is being delegated to by a more + // derived constructor. + ImplicitParamDecl IsMostDerived(getContext(), nullptr, SourceLocation(), + &getContext().Idents.get("is_most_derived"), + getContext().IntTy); + // Only add the parameter to the list if thie class has virtual bases. + if (RD->getNumVBases() > 0) + FunctionArgs.push_back(&IsMostDerived); + + // Start defining the function. + CGF.StartFunction(GlobalDecl(), FnInfo.getReturnType(), ThunkFn, FnInfo, + FunctionArgs, CD->getLocation(), SourceLocation()); + EmitThisParam(CGF); + llvm::Value *This = getThisValue(CGF); + + llvm::Value *SrcVal = + IsCopy ? CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&SrcParam), "src") + : nullptr; + + CallArgList Args; + + // Push the this ptr. + Args.add(RValue::get(This), CD->getThisType(getContext())); + + // Push the src ptr. + if (SrcVal) + Args.add(RValue::get(SrcVal), SrcParam.getType()); + + // Add the rest of the default arguments. + std::vector<Stmt *> ArgVec; + for (unsigned I = IsCopy ? 1 : 0, E = CD->getNumParams(); I != E; ++I) { + Stmt *DefaultArg = getContext().getDefaultArgExprForConstructor(CD, I); + assert(DefaultArg && "sema forgot to instantiate default args"); + ArgVec.push_back(DefaultArg); + } + + CodeGenFunction::RunCleanupsScope Cleanups(CGF); + + const auto *FPT = CD->getType()->castAs<FunctionProtoType>(); + ConstExprIterator ArgBegin(ArgVec.data()), + ArgEnd(ArgVec.data() + ArgVec.size()); + CGF.EmitCallArgs(Args, FPT, ArgBegin, ArgEnd, CD, IsCopy ? 1 : 0); + + // Insert any ABI-specific implicit constructor arguments. + unsigned ExtraArgs = addImplicitConstructorArgs(CGF, CD, Ctor_Complete, + /*ForVirtualBase=*/false, + /*Delegating=*/false, Args); + + // Call the destructor with our arguments. + llvm::Value *CalleeFn = CGM.getAddrOfCXXStructor(CD, StructorType::Complete); + const CGFunctionInfo &CalleeInfo = CGM.getTypes().arrangeCXXConstructorCall( + Args, CD, Ctor_Complete, ExtraArgs); + CGF.EmitCall(CalleeInfo, CalleeFn, ReturnValueSlot(), Args, CD); + + Cleanups.ForceCleanup(); + + // Emit the ret instruction, remove any temporary instructions created for the + // aid of CodeGen. + CGF.FinishFunction(SourceLocation()); + + return ThunkFn; +} + +llvm::Constant *MicrosoftCXXABI::getCatchableType(QualType T, + uint32_t NVOffset, + int32_t VBPtrOffset, + uint32_t VBIndex) { + assert(!T->isReferenceType()); + + CXXRecordDecl *RD = T->getAsCXXRecordDecl(); + const CXXConstructorDecl *CD = + RD ? CGM.getContext().getCopyConstructorForExceptionObject(RD) : nullptr; + CXXCtorType CT = Ctor_Complete; + if (CD) + if (!hasDefaultCXXMethodCC(getContext(), CD) || CD->getNumParams() != 1) + CT = Ctor_CopyingClosure; + + uint32_t Size = getContext().getTypeSizeInChars(T).getQuantity(); + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXCatchableType(T, CD, CT, Size, NVOffset, + VBPtrOffset, VBIndex, Out); + } + if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName)) + return getImageRelativeConstant(GV); + + // The TypeDescriptor is used by the runtime to determine if a catch handler + // is appropriate for the exception object. + llvm::Constant *TD = getImageRelativeConstant(getAddrOfRTTIDescriptor(T)); + + // The runtime is responsible for calling the copy constructor if the + // exception is caught by value. + llvm::Constant *CopyCtor; + if (CD) { + if (CT == Ctor_CopyingClosure) + CopyCtor = getAddrOfCXXCtorClosure(CD, Ctor_CopyingClosure); + else + CopyCtor = CGM.getAddrOfCXXStructor(CD, StructorType::Complete); + + CopyCtor = llvm::ConstantExpr::getBitCast(CopyCtor, CGM.Int8PtrTy); + } else { + CopyCtor = llvm::Constant::getNullValue(CGM.Int8PtrTy); + } + CopyCtor = getImageRelativeConstant(CopyCtor); + + bool IsScalar = !RD; + bool HasVirtualBases = false; + bool IsStdBadAlloc = false; // std::bad_alloc is special for some reason. + QualType PointeeType = T; + if (T->isPointerType()) + PointeeType = T->getPointeeType(); + if (const CXXRecordDecl *RD = PointeeType->getAsCXXRecordDecl()) { + HasVirtualBases = RD->getNumVBases() > 0; + if (IdentifierInfo *II = RD->getIdentifier()) + IsStdBadAlloc = II->isStr("bad_alloc") && RD->isInStdNamespace(); + } + + // Encode the relevant CatchableType properties into the Flags bitfield. + // FIXME: Figure out how bits 2 or 8 can get set. + uint32_t Flags = 0; + if (IsScalar) + Flags |= 1; + if (HasVirtualBases) + Flags |= 4; + if (IsStdBadAlloc) + Flags |= 16; + + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags + TD, // TypeDescriptor + llvm::ConstantInt::get(CGM.IntTy, NVOffset), // NonVirtualAdjustment + llvm::ConstantInt::get(CGM.IntTy, VBPtrOffset), // OffsetToVBPtr + llvm::ConstantInt::get(CGM.IntTy, VBIndex), // VBTableIndex + llvm::ConstantInt::get(CGM.IntTy, Size), // Size + CopyCtor // CopyCtor + }; + llvm::StructType *CTType = getCatchableTypeType(); + auto *GV = new llvm::GlobalVariable( + CGM.getModule(), CTType, /*Constant=*/true, getLinkageForRTTI(T), + llvm::ConstantStruct::get(CTType, Fields), StringRef(MangledName)); + GV->setUnnamedAddr(true); + GV->setSection(".xdata"); + if (GV->isWeakForLinker()) + GV->setComdat(CGM.getModule().getOrInsertComdat(GV->getName())); + return getImageRelativeConstant(GV); +} + +llvm::GlobalVariable *MicrosoftCXXABI::getCatchableTypeArray(QualType T) { + assert(!T->isReferenceType()); + + // See if we've already generated a CatchableTypeArray for this type before. + llvm::GlobalVariable *&CTA = CatchableTypeArrays[T]; + if (CTA) + return CTA; + + // Ensure that we don't have duplicate entries in our CatchableTypeArray by + // using a SmallSetVector. Duplicates may arise due to virtual bases + // occurring more than once in the hierarchy. + llvm::SmallSetVector<llvm::Constant *, 2> CatchableTypes; + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if [...] + // - the handler is of type cv T or cv T& and T is an unambiguous public + // base class of E, or + // - the handler is of type cv T or const T& where T is a pointer type and + // E is a pointer type that can be converted to T by [...] + // - a standard pointer conversion (4.10) not involving conversions to + // pointers to private or protected or ambiguous classes + const CXXRecordDecl *MostDerivedClass = nullptr; + bool IsPointer = T->isPointerType(); + if (IsPointer) + MostDerivedClass = T->getPointeeType()->getAsCXXRecordDecl(); + else + MostDerivedClass = T->getAsCXXRecordDecl(); + + // Collect all the unambiguous public bases of the MostDerivedClass. + if (MostDerivedClass) { + const ASTContext &Context = getContext(); + const ASTRecordLayout &MostDerivedLayout = + Context.getASTRecordLayout(MostDerivedClass); + MicrosoftVTableContext &VTableContext = CGM.getMicrosoftVTableContext(); + SmallVector<MSRTTIClass, 8> Classes; + serializeClassHierarchy(Classes, MostDerivedClass); + Classes.front().initialize(/*Parent=*/nullptr, /*Specifier=*/nullptr); + detectAmbiguousBases(Classes); + for (const MSRTTIClass &Class : Classes) { + // Skip any ambiguous or private bases. + if (Class.Flags & + (MSRTTIClass::IsPrivateOnPath | MSRTTIClass::IsAmbiguous)) + continue; + // Write down how to convert from a derived pointer to a base pointer. + uint32_t OffsetInVBTable = 0; + int32_t VBPtrOffset = -1; + if (Class.VirtualRoot) { + OffsetInVBTable = + VTableContext.getVBTableIndex(MostDerivedClass, Class.VirtualRoot)*4; + VBPtrOffset = MostDerivedLayout.getVBPtrOffset().getQuantity(); + } + + // Turn our record back into a pointer if the exception object is a + // pointer. + QualType RTTITy = QualType(Class.RD->getTypeForDecl(), 0); + if (IsPointer) + RTTITy = Context.getPointerType(RTTITy); + CatchableTypes.insert(getCatchableType(RTTITy, Class.OffsetInVBase, + VBPtrOffset, OffsetInVBTable)); + } + } + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if + // - The handler is of type cv T or cv T& and E and T are the same type + // (ignoring the top-level cv-qualifiers) + CatchableTypes.insert(getCatchableType(T)); + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if + // - the handler is of type cv T or const T& where T is a pointer type and + // E is a pointer type that can be converted to T by [...] + // - a standard pointer conversion (4.10) not involving conversions to + // pointers to private or protected or ambiguous classes + // + // C++14 [conv.ptr]p2: + // A prvalue of type "pointer to cv T," where T is an object type, can be + // converted to a prvalue of type "pointer to cv void". + if (IsPointer && T->getPointeeType()->isObjectType()) + CatchableTypes.insert(getCatchableType(getContext().VoidPtrTy)); + + // C++14 [except.handle]p3: + // A handler is a match for an exception object of type E if [...] + // - the handler is of type cv T or const T& where T is a pointer or + // pointer to member type and E is std::nullptr_t. + // + // We cannot possibly list all possible pointer types here, making this + // implementation incompatible with the standard. However, MSVC includes an + // entry for pointer-to-void in this case. Let's do the same. + if (T->isNullPtrType()) + CatchableTypes.insert(getCatchableType(getContext().VoidPtrTy)); + + uint32_t NumEntries = CatchableTypes.size(); + llvm::Type *CTType = + getImageRelativeType(getCatchableTypeType()->getPointerTo()); + llvm::ArrayType *AT = llvm::ArrayType::get(CTType, NumEntries); + llvm::StructType *CTAType = getCatchableTypeArrayType(NumEntries); + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, NumEntries), // NumEntries + llvm::ConstantArray::get( + AT, llvm::makeArrayRef(CatchableTypes.begin(), + CatchableTypes.end())) // CatchableTypes + }; + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXCatchableTypeArray(T, NumEntries, Out); + } + CTA = new llvm::GlobalVariable( + CGM.getModule(), CTAType, /*Constant=*/true, getLinkageForRTTI(T), + llvm::ConstantStruct::get(CTAType, Fields), StringRef(MangledName)); + CTA->setUnnamedAddr(true); + CTA->setSection(".xdata"); + if (CTA->isWeakForLinker()) + CTA->setComdat(CGM.getModule().getOrInsertComdat(CTA->getName())); + return CTA; +} + +llvm::GlobalVariable *MicrosoftCXXABI::getThrowInfo(QualType T) { + bool IsConst, IsVolatile; + T = decomposeTypeForEH(getContext(), T, IsConst, IsVolatile); + + // The CatchableTypeArray enumerates the various (CV-unqualified) types that + // the exception object may be caught as. + llvm::GlobalVariable *CTA = getCatchableTypeArray(T); + // The first field in a CatchableTypeArray is the number of CatchableTypes. + // This is used as a component of the mangled name which means that we need to + // know what it is in order to see if we have previously generated the + // ThrowInfo. + uint32_t NumEntries = + cast<llvm::ConstantInt>(CTA->getInitializer()->getAggregateElement(0U)) + ->getLimitedValue(); + + SmallString<256> MangledName; + { + llvm::raw_svector_ostream Out(MangledName); + getMangleContext().mangleCXXThrowInfo(T, IsConst, IsVolatile, NumEntries, + Out); + } + + // Reuse a previously generated ThrowInfo if we have generated an appropriate + // one before. + if (llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(MangledName)) + return GV; + + // The RTTI TypeDescriptor uses an unqualified type but catch clauses must + // be at least as CV qualified. Encode this requirement into the Flags + // bitfield. + uint32_t Flags = 0; + if (IsConst) + Flags |= 1; + if (IsVolatile) + Flags |= 2; + + // The cleanup-function (a destructor) must be called when the exception + // object's lifetime ends. + llvm::Constant *CleanupFn = llvm::Constant::getNullValue(CGM.Int8PtrTy); + if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl()) + if (CXXDestructorDecl *DtorD = RD->getDestructor()) + if (!DtorD->isTrivial()) + CleanupFn = llvm::ConstantExpr::getBitCast( + CGM.getAddrOfCXXStructor(DtorD, StructorType::Complete), + CGM.Int8PtrTy); + // This is unused as far as we can tell, initialize it to null. + llvm::Constant *ForwardCompat = + getImageRelativeConstant(llvm::Constant::getNullValue(CGM.Int8PtrTy)); + llvm::Constant *PointerToCatchableTypes = getImageRelativeConstant( + llvm::ConstantExpr::getBitCast(CTA, CGM.Int8PtrTy)); + llvm::StructType *TIType = getThrowInfoType(); + llvm::Constant *Fields[] = { + llvm::ConstantInt::get(CGM.IntTy, Flags), // Flags + getImageRelativeConstant(CleanupFn), // CleanupFn + ForwardCompat, // ForwardCompat + PointerToCatchableTypes // CatchableTypeArray + }; + auto *GV = new llvm::GlobalVariable( + CGM.getModule(), TIType, /*Constant=*/true, getLinkageForRTTI(T), + llvm::ConstantStruct::get(TIType, Fields), StringRef(MangledName)); + GV->setUnnamedAddr(true); + GV->setSection(".xdata"); + if (GV->isWeakForLinker()) + GV->setComdat(CGM.getModule().getOrInsertComdat(GV->getName())); + return GV; +} + +void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) { + const Expr *SubExpr = E->getSubExpr(); + QualType ThrowType = SubExpr->getType(); + // The exception object lives on the stack and it's address is passed to the + // runtime function. + llvm::AllocaInst *AI = CGF.CreateMemTemp(ThrowType); + CGF.EmitAnyExprToMem(SubExpr, AI, ThrowType.getQualifiers(), + /*IsInit=*/true); + + // The so-called ThrowInfo is used to describe how the exception object may be + // caught. + llvm::GlobalVariable *TI = getThrowInfo(ThrowType); + + // Call into the runtime to throw the exception. + llvm::Value *Args[] = {CGF.Builder.CreateBitCast(AI, CGM.Int8PtrTy), TI}; + CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args); +} diff --git a/lib/CodeGen/ModuleBuilder.cpp b/lib/CodeGen/ModuleBuilder.cpp index 4f1a82e0248d..25e57401fd57 100644 --- a/lib/CodeGen/ModuleBuilder.cpp +++ b/lib/CodeGen/ModuleBuilder.cpp @@ -63,7 +63,7 @@ namespace { CoverageInfo(CoverageInfo), M(new llvm::Module(ModuleName, C)) {} - virtual ~CodeGeneratorImpl() { + ~CodeGeneratorImpl() override { // There should normally not be any leftover inline method definitions. assert(DeferredInlineMethodDefinitions.empty() || Diags.hasErrorOccurred()); @@ -95,7 +95,8 @@ namespace { M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple()); M->setDataLayout(Ctx->getTargetInfo().getTargetDescription()); - TD.reset(new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription())); + TD.reset( + new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription())); Builder.reset(new CodeGen::CodeGenModule(Context, CodeGenOpts, *M, *TD, Diags, CoverageInfo)); @@ -211,11 +212,11 @@ namespace { Builder->EmitTentativeDefinition(D); } - void HandleVTable(CXXRecordDecl *RD, bool DefinitionRequired) override { + void HandleVTable(CXXRecordDecl *RD) override { if (Diags.hasErrorOccurred()) return; - Builder->EmitVTable(RD, DefinitionRequired); + Builder->EmitVTable(RD); } void HandleLinkerOptionPragma(llvm::StringRef Opts) override { @@ -238,7 +239,6 @@ void CodeGenerator::anchor() { } CodeGenerator *clang::CreateLLVMCodeGen(DiagnosticsEngine &Diags, const std::string& ModuleName, const CodeGenOptions &CGO, - const TargetOptions &/*TO*/, llvm::LLVMContext& C, CoverageSourceInfo *CoverageInfo) { return new CodeGeneratorImpl(Diags, ModuleName, CGO, C, CoverageInfo); diff --git a/lib/CodeGen/SanitizerMetadata.h b/lib/CodeGen/SanitizerMetadata.h index d2f0651159a2..166f0e6c9b58 100644 --- a/lib/CodeGen/SanitizerMetadata.h +++ b/lib/CodeGen/SanitizerMetadata.h @@ -31,8 +31,8 @@ namespace CodeGen { class CodeGenModule; class SanitizerMetadata { - SanitizerMetadata(const SanitizerMetadata &) LLVM_DELETED_FUNCTION; - void operator=(const SanitizerMetadata &) LLVM_DELETED_FUNCTION; + SanitizerMetadata(const SanitizerMetadata &) = delete; + void operator=(const SanitizerMetadata &) = delete; CodeGenModule &CGM; public: diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp index c05b23a32e86..53154b513eb7 100644 --- a/lib/CodeGen/TargetInfo.cpp +++ b/lib/CodeGen/TargetInfo.cpp @@ -37,7 +37,8 @@ static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder, unsigned LastIndex) { // Alternatively, we could emit this as a loop in the source. for (unsigned I = FirstIndex; I <= LastIndex; ++I) { - llvm::Value *Cell = Builder.CreateConstInBoundsGEP1_32(Array, I); + llvm::Value *Cell = + Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I); Builder.CreateStore(Value, Cell); } } @@ -107,6 +108,10 @@ bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, return false; } +bool ABIInfo::shouldSignExtUnsignedType(QualType Ty) const { + return false; +} + void ABIArgInfo::dump() const { raw_ostream &OS = llvm::errs(); OS << "(ABIArgInfo Kind="; @@ -238,7 +243,7 @@ static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) { /// \return The field declaration for the single non-empty field, if /// it exists. static const Type *isSingleElementStruct(QualType T, ASTContext &Context) { - const RecordType *RT = T->getAsStructureType(); + const RecordType *RT = T->getAs<RecordType>(); if (!RT) return nullptr; @@ -339,9 +344,15 @@ static bool canExpandIndirectArgument(QualType Ty, ASTContext &Context) { // // FIXME: This needs to be generalized to handle classes as well. const RecordDecl *RD = RT->getDecl(); - if (!RD->isStruct() || isa<CXXRecordDecl>(RD)) + if (!RD->isStruct()) return false; + // We try to expand CLike CXXRecordDecl. + if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { + if (!CXXRD->isCLike()) + return false; + } + uint64_t Size = 0; for (const auto *FD : RD->fields()) { @@ -399,8 +410,16 @@ llvm::Value *DefaultABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, } ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const { - if (isAggregateTypeForABI(Ty)) + Ty = useFirstFieldIfTransparentUnion(Ty); + + if (isAggregateTypeForABI(Ty)) { + // Records with non-trivial destructors/copy-constructors should not be + // passed by value. + if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) + return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); + return ABIArgInfo::getIndirect(0); + } // Treat an enum type as its underlying type. if (const EnumType *EnumTy = Ty->getAs<EnumType>()) @@ -1352,7 +1371,8 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable( } else { // 9 is %eflags, which doesn't get a size on Darwin for some // reason. - Builder.CreateStore(Four8, Builder.CreateConstInBoundsGEP1_32(Address, 9)); + Builder.CreateStore( + Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9)); // 11-16 are st(0..5). Not sure why we stop at 5. // These have size 12, which is sizeof(long double) on @@ -1475,14 +1495,13 @@ class X86_64ABIInfo : public ABIInfo { return !getTarget().getTriple().isOSDarwin(); } - bool HasAVX; // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on // 64-bit hardware. bool Has64BitPointers; public: - X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, bool hasavx) : - ABIInfo(CGT), HasAVX(hasavx), + X86_64ABIInfo(CodeGen::CodeGenTypes &CGT) : + ABIInfo(CGT), Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) { } @@ -1503,6 +1522,14 @@ public: llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const override; + + bool has64BitPointers() const { + return Has64BitPointers; + } + + bool hasAVX() const { + return getTarget().getABI() == "avx"; + } }; /// WinX86_64ABIInfo - The Windows X86_64 ABI information. @@ -1532,10 +1559,9 @@ public: }; class X86_64TargetCodeGenInfo : public TargetCodeGenInfo { - bool HasAVX; public: - X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new X86_64ABIInfo(CGT, HasAVX)), HasAVX(HasAVX) {} + X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) + : TargetCodeGenInfo(new X86_64ABIInfo(CGT)) {} const X86_64ABIInfo &getABIInfo() const { return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo()); @@ -1588,24 +1614,47 @@ public: llvm::Constant * getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override { - unsigned Sig = (0xeb << 0) | // jmp rel8 - (0x0a << 8) | // .+0x0c - ('F' << 16) | - ('T' << 24); + unsigned Sig; + if (getABIInfo().has64BitPointers()) + Sig = (0xeb << 0) | // jmp rel8 + (0x0a << 8) | // .+0x0c + ('F' << 16) | + ('T' << 24); + else + Sig = (0xeb << 0) | // jmp rel8 + (0x06 << 8) | // .+0x08 + ('F' << 16) | + ('T' << 24); return llvm::ConstantInt::get(CGM.Int32Ty, Sig); } unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return HasAVX ? 32 : 16; + return getABIInfo().hasAVX() ? 32 : 16; + } +}; + +class PS4TargetCodeGenInfo : public X86_64TargetCodeGenInfo { +public: + PS4TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) + : X86_64TargetCodeGenInfo(CGT) {} + + void getDependentLibraryOption(llvm::StringRef Lib, + llvm::SmallString<24> &Opt) const override { + Opt = "\01"; + Opt += Lib; } }; static std::string qualifyWindowsLibrary(llvm::StringRef Lib) { - // If the argument does not end in .lib, automatically add the suffix. This - // matches the behavior of MSVC. - std::string ArgStr = Lib; + // If the argument does not end in .lib, automatically add the suffix. + // If the argument contains a space, enclose it in quotes. + // This matches the behavior of MSVC. + bool Quote = (Lib.find(" ") != StringRef::npos); + std::string ArgStr = Quote ? "\"" : ""; + ArgStr += Lib; if (!Lib.endswith_lower(".lib")) ArgStr += ".lib"; + ArgStr += Quote ? "\"" : ""; return ArgStr; } @@ -1615,6 +1664,9 @@ public: bool d, bool p, bool w, unsigned RegParms) : X86_32TargetCodeGenInfo(CGT, d, p, w, RegParms) {} + void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const override; + void getDependentLibraryOption(llvm::StringRef Lib, llvm::SmallString<24> &Opt) const override { Opt = "/DEFAULTLIB:"; @@ -1628,11 +1680,35 @@ public: } }; +static void addStackProbeSizeTargetAttribute(const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) { + if (isa<FunctionDecl>(D)) { + if (CGM.getCodeGenOpts().StackProbeSize != 4096) { + llvm::Function *Fn = cast<llvm::Function>(GV); + + Fn->addFnAttr("stack-probe-size", llvm::utostr(CGM.getCodeGenOpts().StackProbeSize)); + } + } +} + +void WinX86_32TargetCodeGenInfo::SetTargetAttributes(const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const { + X86_32TargetCodeGenInfo::SetTargetAttributes(D, GV, CGM); + + addStackProbeSizeTargetAttribute(D, GV, CGM); +} + class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo { - bool HasAVX; + bool hasAVX() const { return getABIInfo().getTarget().getABI() == "avx"; } + public: - WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)), HasAVX(HasAVX) {} + WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT) + : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)) {} + + void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const override; int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override { return 7; @@ -1661,10 +1737,17 @@ public: } unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return HasAVX ? 32 : 16; + return hasAVX() ? 32 : 16; } }; +void WinX86_64TargetCodeGenInfo::SetTargetAttributes(const Decl *D, + llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const { + TargetCodeGenInfo::SetTargetAttributes(D, GV, CGM); + + addStackProbeSizeTargetAttribute(D, GV, CGM); +} } void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo, @@ -1843,7 +1926,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, // split. if (OffsetBase && OffsetBase != 64) Hi = Lo; - } else if (Size == 128 || (HasAVX && isNamedArg && Size == 256)) { + } else if (Size == 128 || (hasAVX() && isNamedArg && Size == 256)) { // Arguments of 256-bits are split into four eightbyte chunks. The // least significant one belongs to class SSE and all the others to class // SSEUP. The original Lo and Hi design considers that types can't be @@ -2065,7 +2148,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const { bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const { if (const VectorType *VecTy = Ty->getAs<VectorType>()) { uint64_t Size = getContext().getTypeSize(VecTy); - unsigned LargestVector = HasAVX ? 256 : 128; + unsigned LargestVector = hasAVX() ? 256 : 128; if (Size <= 64 || Size > LargestVector) return true; } @@ -2142,20 +2225,9 @@ llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const { Ty = QualType(InnerTy, 0); llvm::Type *IRType = CGT.ConvertType(Ty); - - // If the preferred type is a 16-byte vector, prefer to pass it. - if (llvm::VectorType *VT = dyn_cast<llvm::VectorType>(IRType)){ - llvm::Type *EltTy = VT->getElementType(); - unsigned BitWidth = VT->getBitWidth(); - if ((BitWidth >= 128 && BitWidth <= 256) && - (EltTy->isFloatTy() || EltTy->isDoubleTy() || - EltTy->isIntegerTy(8) || EltTy->isIntegerTy(16) || - EltTy->isIntegerTy(32) || EltTy->isIntegerTy(64) || - EltTy->isIntegerTy(128))) - return VT; - } - - return llvm::VectorType::get(llvm::Type::getDoubleTy(getVMContext()), 2); + assert(isa<llvm::VectorType>(IRType) && + "Trying to return a non-vector type in a vector register!"); + return IRType; } /// BitsContainNoUserData - Return true if the specified [start,end) bit range @@ -2717,8 +2789,8 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) { - llvm::Value *overflow_arg_area_p = - CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p"); + llvm::Value *overflow_arg_area_p = CGF.Builder.CreateStructGEP( + nullptr, VAListAddr, 2, "overflow_arg_area_p"); llvm::Value *overflow_arg_area = CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area"); @@ -2798,14 +2870,16 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *gp_offset_p = nullptr, *gp_offset = nullptr; llvm::Value *fp_offset_p = nullptr, *fp_offset = nullptr; if (neededInt) { - gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p"); + gp_offset_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "gp_offset_p"); gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset"); InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8); InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp"); } if (neededSSE) { - fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p"); + fp_offset_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 1, "fp_offset_p"); fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset"); llvm::Value *FitsInFP = llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16); @@ -2833,9 +2907,8 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // simple assembling of a structure from scattered addresses has many more // loads than necessary. Can we clean this up? llvm::Type *LTy = CGF.ConvertTypeForMem(Ty); - llvm::Value *RegAddr = - CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(VAListAddr, 3), - "reg_save_area"); + llvm::Value *RegAddr = CGF.Builder.CreateLoad( + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3), "reg_save_area"); if (neededInt && neededSSE) { // FIXME: Cleanup. assert(AI.isDirect() && "Unexpected ABI info for mixed regs"); @@ -2855,9 +2928,9 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr; llvm::Value *V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegLoAddr, PTyLo)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0)); V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegHiAddr, PTyHi)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1)); RegAddr = CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(LTy)); @@ -2894,10 +2967,10 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, Tmp = CGF.Builder.CreateBitCast(Tmp, ST->getPointerTo()); V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrLo, DblPtrTy)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 0)); V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrHi, DblPtrTy)); - CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1)); + CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(ST, Tmp, 1)); RegAddr = CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(LTy)); } @@ -2984,7 +3057,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs, return ABIArgInfo::getDirect(); } - if (RT || Ty->isMemberPointerType()) { + if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) { // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is // not 1, 2, 4, or 8 bytes, must be passed by reference." if (Width > 64 || !llvm::isPowerOf2_64(Width)) @@ -3040,48 +3113,6 @@ llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, return AddrTyped; } -namespace { - -class NaClX86_64ABIInfo : public ABIInfo { - public: - NaClX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : ABIInfo(CGT), PInfo(CGT), NInfo(CGT, HasAVX) {} - void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; - private: - PNaClABIInfo PInfo; // Used for generating calls with pnaclcall callingconv. - X86_64ABIInfo NInfo; // Used for everything else. -}; - -class NaClX86_64TargetCodeGenInfo : public TargetCodeGenInfo { - bool HasAVX; - public: - NaClX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX) - : TargetCodeGenInfo(new NaClX86_64ABIInfo(CGT, HasAVX)), HasAVX(HasAVX) { - } - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { - return HasAVX ? 32 : 16; - } -}; - -} - -void NaClX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const { - if (FI.getASTCallingConvention() == CC_PnaclCall) - PInfo.computeInfo(FI); - else - NInfo.computeInfo(FI); -} - -llvm::Value *NaClX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - // Always use the native convention; calling pnacl-style varargs functions - // is unuspported. - return NInfo.EmitVAArg(VAListAddr, Ty, CGF); -} - - // PowerPC-32 namespace { /// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information. @@ -3257,13 +3288,42 @@ public: private: static const unsigned GPRBits = 64; ABIKind Kind; + bool HasQPX; + + // A vector of float or double will be promoted to <4 x f32> or <4 x f64> and + // will be passed in a QPX register. + bool IsQPXVectorTy(const Type *Ty) const { + if (!HasQPX) + return false; + + if (const VectorType *VT = Ty->getAs<VectorType>()) { + unsigned NumElements = VT->getNumElements(); + if (NumElements == 1) + return false; + + if (VT->getElementType()->isSpecificBuiltinType(BuiltinType::Double)) { + if (getContext().getTypeSize(Ty) <= 256) + return true; + } else if (VT->getElementType()-> + isSpecificBuiltinType(BuiltinType::Float)) { + if (getContext().getTypeSize(Ty) <= 128) + return true; + } + } + + return false; + } + + bool IsQPXVectorTy(QualType Ty) const { + return IsQPXVectorTy(Ty.getTypePtr()); + } public: - PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind) - : DefaultABIInfo(CGT), Kind(Kind) {} + PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind, bool HasQPX) + : DefaultABIInfo(CGT), Kind(Kind), HasQPX(HasQPX) {} bool isPromotableTypeForABI(QualType Ty) const; - bool isAlignedParamType(QualType Ty) const; + bool isAlignedParamType(QualType Ty, bool &Align32) const; ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType Ty) const; @@ -3288,7 +3348,8 @@ public: const Type *T = isSingleElementStruct(I.type, getContext()); if (T) { const BuiltinType *BT = T->getAs<BuiltinType>(); - if ((T->isVectorType() && getContext().getTypeSize(T) == 128) || + if (IsQPXVectorTy(T) || + (T->isVectorType() && getContext().getTypeSize(T) == 128) || (BT && BT->isFloatingPoint())) { QualType QT(T, 0); I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT)); @@ -3304,10 +3365,13 @@ public: }; class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo { + bool HasQPX; + public: PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT, - PPC64_SVR4_ABIInfo::ABIKind Kind) - : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT, Kind)) {} + PPC64_SVR4_ABIInfo::ABIKind Kind, bool HasQPX) + : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT, Kind, HasQPX)), + HasQPX(HasQPX) {} int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { // This is recovered from gcc output. @@ -3317,7 +3381,12 @@ public: bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const override; - unsigned getOpenMPSimdDefaultAlignment(QualType) const override { + unsigned getOpenMPSimdDefaultAlignment(QualType QT) const override { + if (HasQPX) + if (const PointerType *PT = QT->getAs<PointerType>()) + if (PT->getPointeeType()->isSpecificBuiltinType(BuiltinType::Double)) + return 32; // Natural alignment for QPX doubles. + return 16; // Natural alignment for Altivec and VSX vectors. } }; @@ -3370,15 +3439,23 @@ PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const { /// isAlignedParamType - Determine whether a type requires 16-byte /// alignment in the parameter area. bool -PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty) const { +PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty, bool &Align32) const { + Align32 = false; + // Complex types are passed just like their elements. if (const ComplexType *CTy = Ty->getAs<ComplexType>()) Ty = CTy->getElementType(); // Only vector types of size 16 bytes need alignment (larger types are // passed via reference, smaller types are not aligned). - if (Ty->isVectorType()) + if (IsQPXVectorTy(Ty)) { + if (getContext().getTypeSize(Ty) > 128) + Align32 = true; + + return true; + } else if (Ty->isVectorType()) { return getContext().getTypeSize(Ty) == 128; + } // For single-element float/vector structs, we consider the whole type // to have the same alignment requirements as its single element. @@ -3386,7 +3463,7 @@ PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty) const { const Type *EltType = isSingleElementStruct(Ty, getContext()); if (EltType) { const BuiltinType *BT = EltType->getAs<BuiltinType>(); - if ((EltType->isVectorType() && + if (IsQPXVectorTy(EltType) || (EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) || (BT && BT->isFloatingPoint())) AlignAsType = EltType; @@ -3400,13 +3477,22 @@ PPC64_SVR4_ABIInfo::isAlignedParamType(QualType Ty) const { AlignAsType = Base; // With special case aggregates, only vector base types need alignment. - if (AlignAsType) + if (AlignAsType && IsQPXVectorTy(AlignAsType)) { + if (getContext().getTypeSize(AlignAsType) > 128) + Align32 = true; + + return true; + } else if (AlignAsType) { return AlignAsType->isVectorType(); + } // Otherwise, we only need alignment for any aggregate type that // has an alignment requirement of >= 16 bytes. - if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) + if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) { + if (HasQPX && getContext().getTypeAlign(Ty) >= 256) + Align32 = true; return true; + } return false; } @@ -3512,7 +3598,7 @@ bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { return true; } if (const VectorType *VT = Ty->getAs<VectorType>()) { - if (getContext().getTypeSize(VT) == 128) + if (getContext().getTypeSize(VT) == 128 || IsQPXVectorTy(Ty)) return true; } return false; @@ -3538,7 +3624,7 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes) // or via reference (larger than 16 bytes). - if (Ty->isVectorType()) { + if (Ty->isVectorType() && !IsQPXVectorTy(Ty)) { uint64_t Size = getContext().getTypeSize(Ty); if (Size > 128) return ABIArgInfo::getIndirect(0, /*ByVal=*/false); @@ -3552,7 +3638,9 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const { if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); - uint64_t ABIAlign = isAlignedParamType(Ty)? 16 : 8; + bool Align32; + uint64_t ABIAlign = isAlignedParamType(Ty, Align32) ? + (Align32 ? 32 : 16) : 8; uint64_t TyAlign = getContext().getTypeAlign(Ty) / 8; // ELFv2 homogeneous aggregates are passed as array types. @@ -3609,7 +3697,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const { // Non-Altivec vector types are returned in GPRs (smaller than 16 bytes) // or via reference (larger than 16 bytes). - if (RetTy->isVectorType()) { + if (RetTy->isVectorType() && !IsQPXVectorTy(RetTy)) { uint64_t Size = getContext().getTypeSize(RetTy); if (Size > 128) return ABIArgInfo::getIndirect(0); @@ -3666,10 +3754,13 @@ llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur"); // Handle types that require 16-byte alignment in the parameter save area. - if (isAlignedParamType(Ty)) { + bool Align32; + if (isAlignedParamType(Ty, Align32)) { llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty); - AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt64(15)); - AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt64(-16)); + AddrAsInt = Builder.CreateAdd(AddrAsInt, + Builder.getInt64(Align32 ? 31 : 15)); + AddrAsInt = Builder.CreateAnd(AddrAsInt, + Builder.getInt64(Align32 ? -32 : -16)); Addr = Builder.CreateIntToPtr(AddrAsInt, BP, "ap.align"); } @@ -3714,10 +3805,12 @@ llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr, ImagAddr = Builder.CreateIntToPtr(ImagAddr, PBaseTy); llvm::Value *Real = Builder.CreateLoad(RealAddr, false, ".vareal"); llvm::Value *Imag = Builder.CreateLoad(ImagAddr, false, ".vaimag"); - llvm::Value *Ptr = CGF.CreateTempAlloca(CGT.ConvertTypeForMem(Ty), - "vacplx"); - llvm::Value *RealPtr = Builder.CreateStructGEP(Ptr, 0, ".real"); - llvm::Value *ImagPtr = Builder.CreateStructGEP(Ptr, 1, ".imag"); + llvm::AllocaInst *Ptr = + CGF.CreateTempAlloca(CGT.ConvertTypeForMem(Ty), "vacplx"); + llvm::Value *RealPtr = + Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 0, ".real"); + llvm::Value *ImagPtr = + Builder.CreateStructGEP(Ptr->getAllocatedType(), Ptr, 1, ".imag"); Builder.CreateStore(Real, RealPtr, false); Builder.CreateStore(Imag, ImagPtr, false); return Ptr; @@ -3837,8 +3930,8 @@ private: llvm::Value *EmitAAPCSVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const; - virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override { + llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, + CodeGenFunction &CGF) const override { return isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF) : EmitAAPCSVAArg(VAListAddr, Ty, CGF); } @@ -3849,13 +3942,15 @@ public: AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIInfo::ABIKind Kind) : TargetCodeGenInfo(new AArch64ABIInfo(CGT, Kind)) {} - StringRef getARCRetainAutoreleasedReturnValueMarker() const { + StringRef getARCRetainAutoreleasedReturnValueMarker() const override { return "mov\tfp, fp\t\t; marker for objc_retainAutoreleaseReturnValue"; } - int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const { return 31; } + int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { + return 31; + } - virtual bool doesReturnSlotInterfereWithArgs() const { return false; } + bool doesReturnSlotInterfereWithArgs() const override { return false; } }; } @@ -3964,7 +4059,15 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy) const { // Aggregates <= 16 bytes are returned directly in registers or on the stack. uint64_t Size = getContext().getTypeSize(RetTy); if (Size <= 128) { + unsigned Alignment = getContext().getTypeAlign(RetTy); Size = 64 * ((Size + 63) / 64); // round up to multiple of 8 bytes + + // We use a pair of i64 for 16-byte aggregate with 8-byte alignment. + // For aggregates with 16-byte alignment, we use i128. + if (Alignment < 128 && Size == 128) { + llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext()); + return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64)); + } return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); } @@ -4047,13 +4150,15 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, int RegSize = IsIndirect ? 8 : getContext().getTypeSize(Ty) / 8; if (!IsFPR) { // 3 is the field number of __gr_offs - reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p"); + reg_offs_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "gr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs"); reg_top_index = 1; // field number for __gr_top RegSize = llvm::RoundUpToAlignment(RegSize, 8); } else { // 4 is the field number of __vr_offs. - reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p"); + reg_offs_p = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 4, "vr_offs_p"); reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs"); reg_top_index = 2; // field number for __vr_top RegSize = 16 * NumRegs; @@ -4114,8 +4219,8 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, CGF.EmitBlock(InRegBlock); llvm::Value *reg_top_p = nullptr, *reg_top = nullptr; - reg_top_p = - CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p"); + reg_top_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, reg_top_index, + "reg_top_p"); reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top"); llvm::Value *BaseAddr = CGF.Builder.CreateGEP(reg_top, reg_offs); llvm::Value *RegAddr = nullptr; @@ -4138,7 +4243,7 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, assert(!IsIndirect && "Homogeneous aggregates should be passed directly"); llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0)); llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers); - llvm::Value *Tmp = CGF.CreateTempAlloca(HFATy); + llvm::AllocaInst *Tmp = CGF.CreateTempAlloca(HFATy); int Offset = 0; if (CGF.CGM.getDataLayout().isBigEndian() && Ctx.getTypeSize(Base) < 128) @@ -4149,7 +4254,8 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, llvm::Value *LoadAddr = CGF.Builder.CreateGEP(BaseAddr, BaseOffset); LoadAddr = CGF.Builder.CreateBitCast( LoadAddr, llvm::PointerType::getUnqual(BaseTy)); - llvm::Value *StoreAddr = CGF.Builder.CreateStructGEP(Tmp, i); + llvm::Value *StoreAddr = + CGF.Builder.CreateStructGEP(Tmp->getAllocatedType(), Tmp, i); llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr); CGF.Builder.CreateStore(Elem, StoreAddr); @@ -4182,7 +4288,7 @@ llvm::Value *AArch64ABIInfo::EmitAAPCSVAArg(llvm::Value *VAListAddr, CGF.EmitBlock(OnStackBlock); llvm::Value *stack_p = nullptr, *OnStackAddr = nullptr; - stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p"); + stack_p = CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 0, "stack_p"); OnStackAddr = CGF.Builder.CreateLoad(stack_p, "stack"); // Again, stack arguments may need realigmnent. In this case both integer and @@ -4324,17 +4430,10 @@ public: private: ABIKind Kind; - mutable int VFPRegs[16]; - const unsigned NumVFPs; - const unsigned NumGPRs; - mutable unsigned AllocatedGPRs; - mutable unsigned AllocatedVFPs; public: - ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : ABIInfo(CGT), Kind(_Kind), - NumVFPs(16), NumGPRs(4) { + ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : ABIInfo(CGT), Kind(_Kind) { setCCs(); - resetAllocatedRegs(); } bool isEABI() const { @@ -4364,8 +4463,7 @@ public: private: ABIArgInfo classifyReturnType(QualType RetTy, bool isVariadic) const; - ABIArgInfo classifyArgumentType(QualType RetTy, bool isVariadic, - bool &IsCPRC) const; + ABIArgInfo classifyArgumentType(QualType RetTy, bool isVariadic) const; bool isIllegalVectorType(QualType Ty) const; bool isHomogeneousAggregateBaseType(QualType Ty) const override; @@ -4380,10 +4478,6 @@ private: llvm::CallingConv::ID getLLVMDefaultCC() const; llvm::CallingConv::ID getABIDefaultCC() const; void setCCs(); - - void markAllocatedGPRs(unsigned Alignment, unsigned NumRequired) const; - void markAllocatedVFPs(unsigned Alignment, unsigned NumRequired) const; - void resetAllocatedRegs(void) const; }; class ARMTargetCodeGenInfo : public TargetCodeGenInfo { @@ -4456,55 +4550,43 @@ public: } }; +class WindowsARMTargetCodeGenInfo : public ARMTargetCodeGenInfo { + void addStackProbeSizeTargetAttribute(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const; + +public: + WindowsARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K) + : ARMTargetCodeGenInfo(CGT, K) {} + + void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, + CodeGen::CodeGenModule &CGM) const override; +}; + +void WindowsARMTargetCodeGenInfo::addStackProbeSizeTargetAttribute( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const { + if (!isa<FunctionDecl>(D)) + return; + if (CGM.getCodeGenOpts().StackProbeSize == 4096) + return; + + llvm::Function *F = cast<llvm::Function>(GV); + F->addFnAttr("stack-probe-size", + llvm::utostr(CGM.getCodeGenOpts().StackProbeSize)); +} + +void WindowsARMTargetCodeGenInfo::SetTargetAttributes( + const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const { + ARMTargetCodeGenInfo::SetTargetAttributes(D, GV, CGM); + addStackProbeSizeTargetAttribute(D, GV, CGM); +} } void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const { - // To correctly handle Homogeneous Aggregate, we need to keep track of the - // VFP registers allocated so far. - // C.1.vfp If the argument is a VFP CPRC and there are sufficient consecutive - // VFP registers of the appropriate type unallocated then the argument is - // allocated to the lowest-numbered sequence of such registers. - // C.2.vfp If the argument is a VFP CPRC then any VFP registers that are - // unallocated are marked as unavailable. - resetAllocatedRegs(); - - if (getCXXABI().classifyReturnType(FI)) { - if (FI.getReturnInfo().isIndirect()) - markAllocatedGPRs(1, 1); - } else { + if (!getCXXABI().classifyReturnType(FI)) FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), FI.isVariadic()); - } - for (auto &I : FI.arguments()) { - unsigned PreAllocationVFPs = AllocatedVFPs; - unsigned PreAllocationGPRs = AllocatedGPRs; - bool IsCPRC = false; - // 6.1.2.3 There is one VFP co-processor register class using registers - // s0-s15 (d0-d7) for passing arguments. - I.info = classifyArgumentType(I.type, FI.isVariadic(), IsCPRC); - - // If we have allocated some arguments onto the stack (due to running - // out of VFP registers), we cannot split an argument between GPRs and - // the stack. If this situation occurs, we add padding to prevent the - // GPRs from being used. In this situation, the current argument could - // only be allocated by rule C.8, so rule C.6 would mark these GPRs as - // unusable anyway. - // We do not have to do this if the argument is being passed ByVal, as the - // backend can handle that situation correctly. - const bool StackUsed = PreAllocationGPRs > NumGPRs || PreAllocationVFPs > NumVFPs; - const bool IsByVal = I.info.isIndirect() && I.info.getIndirectByVal(); - if (!IsCPRC && PreAllocationGPRs < NumGPRs && AllocatedGPRs > NumGPRs && - StackUsed && !IsByVal) { - llvm::Type *PaddingTy = llvm::ArrayType::get( - llvm::Type::getInt32Ty(getVMContext()), NumGPRs - PreAllocationGPRs); - if (I.info.canHaveCoerceToType()) { - I.info = ABIArgInfo::getDirect(I.info.getCoerceToType() /* type */, - 0 /* offset */, PaddingTy, true); - } else { - I.info = ABIArgInfo::getDirect(nullptr /* type */, 0 /* offset */, - PaddingTy, true); - } - } - } + + for (auto &I : FI.arguments()) + I.info = classifyArgumentType(I.type, FI.isVariadic()); // Always honor user-specified calling convention. if (FI.getCallingConvention() != llvm::CallingConv::C) @@ -4512,7 +4594,7 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const { llvm::CallingConv::ID cc = getRuntimeCC(); if (cc != llvm::CallingConv::C) - FI.setEffectiveCallingConvention(cc); + FI.setEffectiveCallingConvention(cc); } /// Return the default calling convention that LLVM will use. @@ -4550,64 +4632,8 @@ void ARMABIInfo::setCCs() { llvm::CallingConv::ARM_APCS : llvm::CallingConv::ARM_AAPCS); } -/// markAllocatedVFPs - update VFPRegs according to the alignment and -/// number of VFP registers (unit is S register) requested. -void ARMABIInfo::markAllocatedVFPs(unsigned Alignment, - unsigned NumRequired) const { - // Early Exit. - if (AllocatedVFPs >= 16) { - // We use AllocatedVFP > 16 to signal that some CPRCs were allocated on - // the stack. - AllocatedVFPs = 17; - return; - } - // C.1.vfp If the argument is a VFP CPRC and there are sufficient consecutive - // VFP registers of the appropriate type unallocated then the argument is - // allocated to the lowest-numbered sequence of such registers. - for (unsigned I = 0; I < 16; I += Alignment) { - bool FoundSlot = true; - for (unsigned J = I, JEnd = I + NumRequired; J < JEnd; J++) - if (J >= 16 || VFPRegs[J]) { - FoundSlot = false; - break; - } - if (FoundSlot) { - for (unsigned J = I, JEnd = I + NumRequired; J < JEnd; J++) - VFPRegs[J] = 1; - AllocatedVFPs += NumRequired; - return; - } - } - // C.2.vfp If the argument is a VFP CPRC then any VFP registers that are - // unallocated are marked as unavailable. - for (unsigned I = 0; I < 16; I++) - VFPRegs[I] = 1; - AllocatedVFPs = 17; // We do not have enough VFP registers. -} - -/// Update AllocatedGPRs to record the number of general purpose registers -/// which have been allocated. It is valid for AllocatedGPRs to go above 4, -/// this represents arguments being stored on the stack. -void ARMABIInfo::markAllocatedGPRs(unsigned Alignment, - unsigned NumRequired) const { - assert((Alignment == 1 || Alignment == 2) && "Alignment must be 4 or 8 bytes"); - - if (Alignment == 2 && AllocatedGPRs & 0x1) - AllocatedGPRs += 1; - - AllocatedGPRs += NumRequired; -} - -void ARMABIInfo::resetAllocatedRegs(void) const { - AllocatedGPRs = 0; - AllocatedVFPs = 0; - for (unsigned i = 0; i < NumVFPs; ++i) - VFPRegs[i] = 0; -} - -ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, - bool &IsCPRC) const { - // We update number of allocated VFPs according to +ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, + bool isVariadic) const { // 6.1.2.1 The following argument types are VFP CPRCs: // A single-precision floating-point type (including promoted // half-precision types); A double-precision floating-point type; @@ -4625,58 +4651,20 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (Size <= 32) { llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext()); - markAllocatedGPRs(1, 1); return ABIArgInfo::getDirect(ResType); } if (Size == 64) { llvm::Type *ResType = llvm::VectorType::get( llvm::Type::getInt32Ty(getVMContext()), 2); - if (getABIKind() == ARMABIInfo::AAPCS || isVariadic){ - markAllocatedGPRs(2, 2); - } else { - markAllocatedVFPs(2, 2); - IsCPRC = true; - } return ABIArgInfo::getDirect(ResType); } if (Size == 128) { llvm::Type *ResType = llvm::VectorType::get( llvm::Type::getInt32Ty(getVMContext()), 4); - if (getABIKind() == ARMABIInfo::AAPCS || isVariadic) { - markAllocatedGPRs(2, 4); - } else { - markAllocatedVFPs(4, 4); - IsCPRC = true; - } return ABIArgInfo::getDirect(ResType); } - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0, /*ByVal=*/false); } - // Update VFPRegs for legal vector types. - if (getABIKind() == ARMABIInfo::AAPCS_VFP && !isVariadic) { - if (const VectorType *VT = Ty->getAs<VectorType>()) { - uint64_t Size = getContext().getTypeSize(VT); - // Size of a legal vector should be power of 2 and above 64. - markAllocatedVFPs(Size >= 128 ? 4 : 2, Size / 32); - IsCPRC = true; - } - } - // Update VFPRegs for floating point types. - if (getABIKind() == ARMABIInfo::AAPCS_VFP && !isVariadic) { - if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { - if (BT->getKind() == BuiltinType::Half || - BT->getKind() == BuiltinType::Float) { - markAllocatedVFPs(1, 1); - IsCPRC = true; - } - if (BT->getKind() == BuiltinType::Double || - BT->getKind() == BuiltinType::LongDouble) { - markAllocatedVFPs(2, 2); - IsCPRC = true; - } - } - } if (!isAggregateTypeForABI(Ty)) { // Treat an enum type as its underlying type. @@ -4684,15 +4672,11 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, Ty = EnumTy->getDecl()->getIntegerType(); } - unsigned Size = getContext().getTypeSize(Ty); - if (!IsCPRC) - markAllocatedGPRs(Size > 32 ? 2 : 1, (Size + 31) / 32); return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend() : ABIArgInfo::getDirect()); } if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0, RAA == CGCXXABI::RAA_DirectInMemory); } @@ -4708,19 +4692,6 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (isHomogeneousAggregate(Ty, Base, Members)) { assert(Base && "Base class should be set for homogeneous aggregate"); // Base can be a floating-point or a vector. - if (Base->isVectorType()) { - // ElementSize is in number of floats. - unsigned ElementSize = getContext().getTypeSize(Base) == 64 ? 2 : 4; - markAllocatedVFPs(ElementSize, - Members * ElementSize); - } else if (Base->isSpecificBuiltinType(BuiltinType::Float)) - markAllocatedVFPs(1, Members); - else { - assert(Base->isSpecificBuiltinType(BuiltinType::Double) || - Base->isSpecificBuiltinType(BuiltinType::LongDouble)); - markAllocatedVFPs(2, Members * 2); - } - IsCPRC = true; return ABIArgInfo::getDirect(nullptr, 0, nullptr, false); } } @@ -4732,15 +4703,11 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, uint64_t ABIAlign = 4; uint64_t TyAlign = getContext().getTypeAlign(Ty) / 8; if (getABIKind() == ARMABIInfo::AAPCS_VFP || - getABIKind() == ARMABIInfo::AAPCS) + getABIKind() == ARMABIInfo::AAPCS) ABIAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8); + if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64)) { - // Update Allocated GPRs. Since this is only used when the size of the - // argument is greater than 64 bytes, this will always use up any available - // registers (of which there are 4). We also don't care about getting the - // alignment right, because general-purpose registers cannot be back-filled. - markAllocatedGPRs(1, 4); - return ABIArgInfo::getIndirect(TyAlign, /*ByVal=*/true, + return ABIArgInfo::getIndirect(ABIAlign, /*ByVal=*/true, /*Realign=*/TyAlign > ABIAlign); } @@ -4752,11 +4719,9 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic, if (getContext().getTypeAlign(Ty) <= 32) { ElemTy = llvm::Type::getInt32Ty(getVMContext()); SizeRegs = (getContext().getTypeSize(Ty) + 31) / 32; - markAllocatedGPRs(1, SizeRegs); } else { ElemTy = llvm::Type::getInt64Ty(getVMContext()); SizeRegs = (getContext().getTypeSize(Ty) + 63) / 64; - markAllocatedGPRs(2, SizeRegs * 2); } return ABIArgInfo::getDirect(llvm::ArrayType::get(ElemTy, SizeRegs)); @@ -4856,7 +4821,6 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, // Large vector types should be returned via memory. if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128) { - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0); } @@ -4894,7 +4858,6 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, } // Otherwise return in memory. - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0); } @@ -4930,7 +4893,6 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext())); } - markAllocatedGPRs(1, 1); return ABIArgInfo::getIndirect(0); } @@ -5046,42 +5008,6 @@ llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, return AddrTyped; } -namespace { - -class NaClARMABIInfo : public ABIInfo { - public: - NaClARMABIInfo(CodeGen::CodeGenTypes &CGT, ARMABIInfo::ABIKind Kind) - : ABIInfo(CGT), PInfo(CGT), NInfo(CGT, Kind) {} - void computeInfo(CGFunctionInfo &FI) const override; - llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const override; - private: - PNaClABIInfo PInfo; // Used for generating calls with pnaclcall callingconv. - ARMABIInfo NInfo; // Used for everything else. -}; - -class NaClARMTargetCodeGenInfo : public TargetCodeGenInfo { - public: - NaClARMTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, ARMABIInfo::ABIKind Kind) - : TargetCodeGenInfo(new NaClARMABIInfo(CGT, Kind)) {} -}; - -} - -void NaClARMABIInfo::computeInfo(CGFunctionInfo &FI) const { - if (FI.getASTCallingConvention() == CC_PnaclCall) - PInfo.computeInfo(FI); - else - static_cast<const ABIInfo&>(NInfo).computeInfo(FI); -} - -llvm::Value *NaClARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, - CodeGenFunction &CGF) const { - // Always use the native convention; calling pnacl-style varargs functions - // is unsupported. - return static_cast<const ABIInfo&>(NInfo).EmitVAArg(VAListAddr, Ty, CGF); -} - //===----------------------------------------------------------------------===// // NVPTX ABI Implementation //===----------------------------------------------------------------------===// @@ -5190,18 +5116,22 @@ SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV, // Create !{<func-ref>, metadata !"kernel", i32 1} node addNVVMMetadata(F, "kernel", 1); } - if (FD->hasAttr<CUDALaunchBoundsAttr>()) { + if (CUDALaunchBoundsAttr *Attr = FD->getAttr<CUDALaunchBoundsAttr>()) { // Create !{<func-ref>, metadata !"maxntidx", i32 <val>} node - addNVVMMetadata(F, "maxntidx", - FD->getAttr<CUDALaunchBoundsAttr>()->getMaxThreads()); - // min blocks is a default argument for CUDALaunchBoundsAttr, so getting a - // zero value from getMinBlocks either means it was not specified in - // __launch_bounds__ or the user specified a 0 value. In both cases, we - // don't have to add a PTX directive. - int MinCTASM = FD->getAttr<CUDALaunchBoundsAttr>()->getMinBlocks(); - if (MinCTASM > 0) { - // Create !{<func-ref>, metadata !"minctasm", i32 <val>} node - addNVVMMetadata(F, "minctasm", MinCTASM); + llvm::APSInt MaxThreads(32); + MaxThreads = Attr->getMaxThreads()->EvaluateKnownConstInt(M.getContext()); + if (MaxThreads > 0) + addNVVMMetadata(F, "maxntidx", MaxThreads.getExtValue()); + + // min blocks is an optional argument for CUDALaunchBoundsAttr. If it was + // not specified in __launch_bounds__ or if the user specified a 0 value, + // we don't have to add a PTX directive. + if (Attr->getMinBlocks()) { + llvm::APSInt MinBlocks(32); + MinBlocks = Attr->getMinBlocks()->EvaluateKnownConstInt(M.getContext()); + if (MinBlocks > 0) + // Create !{<func-ref>, metadata !"minctasm", i32 <val>} node + addNVVMMetadata(F, "minctasm", MinBlocks.getExtValue()); } } } @@ -5231,12 +5161,17 @@ void NVPTXTargetCodeGenInfo::addNVVMMetadata(llvm::Function *F, StringRef Name, namespace { class SystemZABIInfo : public ABIInfo { + bool HasVector; + public: - SystemZABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {} + SystemZABIInfo(CodeGenTypes &CGT, bool HV) + : ABIInfo(CGT), HasVector(HV) {} bool isPromotableIntegerType(QualType Ty) const; bool isCompoundType(QualType Ty) const; + bool isVectorArgumentType(QualType Ty) const; bool isFPArgumentType(QualType Ty) const; + QualType GetSingleElementType(QualType Ty) const; ABIArgInfo classifyReturnType(QualType RetTy) const; ABIArgInfo classifyArgumentType(QualType ArgTy) const; @@ -5254,8 +5189,8 @@ public: class SystemZTargetCodeGenInfo : public TargetCodeGenInfo { public: - SystemZTargetCodeGenInfo(CodeGenTypes &CGT) - : TargetCodeGenInfo(new SystemZABIInfo(CGT)) {} + SystemZTargetCodeGenInfo(CodeGenTypes &CGT, bool HasVector) + : TargetCodeGenInfo(new SystemZABIInfo(CGT, HasVector)) {} }; } @@ -5282,7 +5217,15 @@ bool SystemZABIInfo::isPromotableIntegerType(QualType Ty) const { } bool SystemZABIInfo::isCompoundType(QualType Ty) const { - return Ty->isAnyComplexType() || isAggregateTypeForABI(Ty); + return (Ty->isAnyComplexType() || + Ty->isVectorType() || + isAggregateTypeForABI(Ty)); +} + +bool SystemZABIInfo::isVectorArgumentType(QualType Ty) const { + return (HasVector && + Ty->isVectorType() && + getContext().getTypeSize(Ty) <= 128); } bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { @@ -5295,9 +5238,13 @@ bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { return false; } + return false; +} + +QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const { if (const RecordType *RT = Ty->getAsStructureType()) { const RecordDecl *RD = RT->getDecl(); - bool Found = false; + QualType Found; // If this is a C++ record, check the bases first. if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) @@ -5308,36 +5255,34 @@ bool SystemZABIInfo::isFPArgumentType(QualType Ty) const { if (isEmptyRecord(getContext(), Base, true)) continue; - if (Found) - return false; - Found = isFPArgumentType(Base); - if (!Found) - return false; + if (!Found.isNull()) + return Ty; + Found = GetSingleElementType(Base); } // Check the fields. for (const auto *FD : RD->fields()) { - // Empty bitfields don't affect things either way. + // For compatibility with GCC, ignore empty bitfields in C++ mode. // Unlike isSingleElementStruct(), empty structure and array fields // do count. So do anonymous bitfields that aren't zero-sized. - if (FD->isBitField() && FD->getBitWidthValue(getContext()) == 0) - return true; + if (getContext().getLangOpts().CPlusPlus && + FD->isBitField() && FD->getBitWidthValue(getContext()) == 0) + continue; // Unlike isSingleElementStruct(), arrays do not count. - // Nested isFPArgumentType structures still do though. - if (Found) - return false; - Found = isFPArgumentType(FD->getType()); - if (!Found) - return false; + // Nested structures still do though. + if (!Found.isNull()) + return Ty; + Found = GetSingleElementType(FD->getType()); } // Unlike isSingleElementStruct(), trailing padding is allowed. // An 8-byte aligned struct s { float f; } is passed as a double. - return Found; + if (!Found.isNull()) + return Found; } - return false; + return Ty; } llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, @@ -5350,26 +5295,56 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, // i8 *__reg_save_area; // }; - // Every argument occupies 8 bytes and is passed by preference in either - // GPRs or FPRs. + // Every non-vector argument occupies 8 bytes and is passed by preference + // in either GPRs or FPRs. Vector arguments occupy 8 or 16 bytes and are + // always passed on the stack. Ty = CGF.getContext().getCanonicalType(Ty); + llvm::Type *ArgTy = CGF.ConvertTypeForMem(Ty); + llvm::Type *APTy = llvm::PointerType::getUnqual(ArgTy); ABIArgInfo AI = classifyArgumentType(Ty); - bool InFPRs = isFPArgumentType(Ty); - - llvm::Type *APTy = llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty)); bool IsIndirect = AI.isIndirect(); + bool InFPRs = false; + bool IsVector = false; unsigned UnpaddedBitSize; if (IsIndirect) { APTy = llvm::PointerType::getUnqual(APTy); UnpaddedBitSize = 64; - } else + } else { + if (AI.getCoerceToType()) + ArgTy = AI.getCoerceToType(); + InFPRs = ArgTy->isFloatTy() || ArgTy->isDoubleTy(); + IsVector = ArgTy->isVectorTy(); UnpaddedBitSize = getContext().getTypeSize(Ty); - unsigned PaddedBitSize = 64; + } + unsigned PaddedBitSize = (IsVector && UnpaddedBitSize > 64) ? 128 : 64; assert((UnpaddedBitSize <= PaddedBitSize) && "Invalid argument size."); unsigned PaddedSize = PaddedBitSize / 8; unsigned Padding = (PaddedBitSize - UnpaddedBitSize) / 8; + llvm::Type *IndexTy = CGF.Int64Ty; + llvm::Value *PaddedSizeV = llvm::ConstantInt::get(IndexTy, PaddedSize); + + if (IsVector) { + // Work out the address of a vector argument on the stack. + // Vector arguments are always passed in the high bits of a + // single (8 byte) or double (16 byte) stack slot. + llvm::Value *OverflowArgAreaPtr = + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 2, + "overflow_arg_area_ptr"); + llvm::Value *OverflowArgArea = + CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"); + llvm::Value *MemAddr = + CGF.Builder.CreateBitCast(OverflowArgArea, APTy, "mem_addr"); + + // Update overflow_arg_area_ptr pointer + llvm::Value *NewOverflowArgArea = + CGF.Builder.CreateGEP(OverflowArgArea, PaddedSizeV, "overflow_arg_area"); + CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr); + + return MemAddr; + } + unsigned MaxRegs, RegCountField, RegSaveIndex, RegPadding; if (InFPRs) { MaxRegs = 4; // Maximum of 4 FPR arguments @@ -5383,10 +5358,9 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, RegPadding = Padding; // values are passed in the low bits of a GPR } - llvm::Value *RegCountPtr = - CGF.Builder.CreateStructGEP(VAListAddr, RegCountField, "reg_count_ptr"); + llvm::Value *RegCountPtr = CGF.Builder.CreateStructGEP( + nullptr, VAListAddr, RegCountField, "reg_count_ptr"); llvm::Value *RegCount = CGF.Builder.CreateLoad(RegCountPtr, "reg_count"); - llvm::Type *IndexTy = RegCount->getType(); llvm::Value *MaxRegsV = llvm::ConstantInt::get(IndexTy, MaxRegs); llvm::Value *InRegs = CGF.Builder.CreateICmpULT(RegCount, MaxRegsV, "fits_in_regs"); @@ -5400,7 +5374,6 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CGF.EmitBlock(InRegBlock); // Work out the address of an argument register. - llvm::Value *PaddedSizeV = llvm::ConstantInt::get(IndexTy, PaddedSize); llvm::Value *ScaledRegCount = CGF.Builder.CreateMul(RegCount, PaddedSizeV, "scaled_reg_count"); llvm::Value *RegBase = @@ -5408,7 +5381,7 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, llvm::Value *RegOffset = CGF.Builder.CreateAdd(ScaledRegCount, RegBase, "reg_offset"); llvm::Value *RegSaveAreaPtr = - CGF.Builder.CreateStructGEP(VAListAddr, 3, "reg_save_area_ptr"); + CGF.Builder.CreateStructGEP(nullptr, VAListAddr, 3, "reg_save_area_ptr"); llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(RegSaveAreaPtr, "reg_save_area"); llvm::Value *RawRegAddr = @@ -5427,8 +5400,8 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CGF.EmitBlock(InMemBlock); // Work out the address of a stack argument. - llvm::Value *OverflowArgAreaPtr = - CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_ptr"); + llvm::Value *OverflowArgAreaPtr = CGF.Builder.CreateStructGEP( + nullptr, VAListAddr, 2, "overflow_arg_area_ptr"); llvm::Value *OverflowArgArea = CGF.Builder.CreateLoad(OverflowArgAreaPtr, "overflow_arg_area"); llvm::Value *PaddingV = llvm::ConstantInt::get(IndexTy, Padding); @@ -5458,6 +5431,8 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const { if (RetTy->isVoidType()) return ABIArgInfo::getIgnore(); + if (isVectorArgumentType(RetTy)) + return ABIArgInfo::getDirect(); if (isCompoundType(RetTy) || getContext().getTypeSize(RetTy) > 64) return ABIArgInfo::getIndirect(0); return (isPromotableIntegerType(RetTy) ? @@ -5473,8 +5448,16 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { if (isPromotableIntegerType(Ty)) return ABIArgInfo::getExtend(); - // Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly. + // Handle vector types and vector-like structure types. Note that + // as opposed to float-like structure types, we do not allow any + // padding for vector-like structures, so verify the sizes match. uint64_t Size = getContext().getTypeSize(Ty); + QualType SingleElementTy = GetSingleElementType(Ty); + if (isVectorArgumentType(SingleElementTy) && + getContext().getTypeSize(SingleElementTy) == Size) + return ABIArgInfo::getDirect(CGT.ConvertType(SingleElementTy)); + + // Values that are not 1, 2, 4 or 8 bytes in size are passed indirectly. if (Size != 8 && Size != 16 && Size != 32 && Size != 64) return ABIArgInfo::getIndirect(0, /*ByVal=*/false); @@ -5488,7 +5471,7 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const { // The structure is passed as an unextended integer, a float, or a double. llvm::Type *PassTy; - if (isFPArgumentType(Ty)) { + if (isFPArgumentType(SingleElementTy)) { assert(Size == 32 || Size == 64); if (Size == 32) PassTy = llvm::Type::getFloatTy(getVMContext()); @@ -5568,6 +5551,7 @@ public: void computeInfo(CGFunctionInfo &FI) const override; llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty, CodeGenFunction &CGF) const override; + bool shouldSignExtUnsignedType(QualType Ty) const override; }; class MIPSTargetCodeGenInfo : public TargetCodeGenInfo { @@ -5870,6 +5854,16 @@ llvm::Value* MipsABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty, return AddrTyped; } +bool MipsABIInfo::shouldSignExtUnsignedType(QualType Ty) const { + int TySize = getContext().getTypeSize(Ty); + + // MIPS64 ABI requires unsigned 32 bit integers to be sign extended. + if (Ty->isUnsignedIntegerOrEnumerationType() && TySize == 32) + return true; + + return false; +} + bool MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const { @@ -6264,12 +6258,7 @@ private: // Check if Ty is a usable substitute for the coercion type. bool isUsableType(llvm::StructType *Ty) const { - if (Ty->getNumElements() != Elems.size()) - return false; - for (unsigned i = 0, e = Elems.size(); i != e; ++i) - if (Elems[i] != Ty->getElementType(i)) - return false; - return true; + return llvm::makeArrayRef(Elems) == Ty->elements(); } // Get the coercion type as a literal struct type. @@ -7102,6 +7091,12 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { case llvm::Triple::thumb: case llvm::Triple::thumbeb: { + if (Triple.getOS() == llvm::Triple::Win32) { + TheTargetCodeGenInfo = + new WindowsARMTargetCodeGenInfo(Types, ARMABIInfo::AAPCS_VFP); + return *TheTargetCodeGenInfo; + } + ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS; if (getTarget().getABI() == "apcs-gnu") Kind = ARMABIInfo::APCS; @@ -7110,14 +7105,7 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { Triple.getEnvironment() == llvm::Triple::GNUEABIHF)) Kind = ARMABIInfo::AAPCS_VFP; - switch (Triple.getOS()) { - case llvm::Triple::NaCl: - return *(TheTargetCodeGenInfo = - new NaClARMTargetCodeGenInfo(Types, Kind)); - default: - return *(TheTargetCodeGenInfo = - new ARMTargetCodeGenInfo(Types, Kind)); - } + return *(TheTargetCodeGenInfo = new ARMTargetCodeGenInfo(Types, Kind)); } case llvm::Triple::ppc: @@ -7127,19 +7115,21 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1; if (getTarget().getABI() == "elfv2") Kind = PPC64_SVR4_ABIInfo::ELFv2; + bool HasQPX = getTarget().getABI() == "elfv1-qpx"; return *(TheTargetCodeGenInfo = - new PPC64_SVR4_TargetCodeGenInfo(Types, Kind)); + new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, HasQPX)); } else return *(TheTargetCodeGenInfo = new PPC64TargetCodeGenInfo(Types)); case llvm::Triple::ppc64le: { assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!"); PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv2; - if (getTarget().getABI() == "elfv1") + if (getTarget().getABI() == "elfv1" || getTarget().getABI() == "elfv1-qpx") Kind = PPC64_SVR4_ABIInfo::ELFv1; + bool HasQPX = getTarget().getABI() == "elfv1-qpx"; return *(TheTargetCodeGenInfo = - new PPC64_SVR4_TargetCodeGenInfo(Types, Kind)); + new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, HasQPX)); } case llvm::Triple::nvptx: @@ -7149,8 +7139,11 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { case llvm::Triple::msp430: return *(TheTargetCodeGenInfo = new MSP430TargetCodeGenInfo(Types)); - case llvm::Triple::systemz: - return *(TheTargetCodeGenInfo = new SystemZTargetCodeGenInfo(Types)); + case llvm::Triple::systemz: { + bool HasVector = getTarget().getABI() == "vector"; + return *(TheTargetCodeGenInfo = new SystemZTargetCodeGenInfo(Types, + HasVector)); + } case llvm::Triple::tce: return *(TheTargetCodeGenInfo = new TCETargetCodeGenInfo(Types)); @@ -7177,18 +7170,13 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() { } case llvm::Triple::x86_64: { - bool HasAVX = getTarget().getABI() == "avx"; - switch (Triple.getOS()) { case llvm::Triple::Win32: - return *(TheTargetCodeGenInfo = - new WinX86_64TargetCodeGenInfo(Types, HasAVX)); - case llvm::Triple::NaCl: - return *(TheTargetCodeGenInfo = - new NaClX86_64TargetCodeGenInfo(Types, HasAVX)); + return *(TheTargetCodeGenInfo = new WinX86_64TargetCodeGenInfo(Types)); + case llvm::Triple::PS4: + return *(TheTargetCodeGenInfo = new PS4TargetCodeGenInfo(Types)); default: - return *(TheTargetCodeGenInfo = - new X86_64TargetCodeGenInfo(Types, HasAVX)); + return *(TheTargetCodeGenInfo = new X86_64TargetCodeGenInfo(Types)); } } case llvm::Triple::hexagon: |