diff options
Diffstat (limited to 'contrib/llvm-project/llvm/include/llvm')
22 files changed, 230 insertions, 55 deletions
diff --git a/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h b/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h index 0ef63dc68e1c..c4602d3449c0 100644 --- a/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h +++ b/contrib/llvm-project/llvm/include/llvm/Analysis/AssumptionCache.h @@ -45,7 +45,7 @@ public: enum : unsigned { ExprResultIdx = std::numeric_limits<unsigned>::max() }; struct ResultElem { - WeakTrackingVH Assume; + WeakVH Assume; /// contains either ExprResultIdx or the index of the operand bundle /// containing the knowledge. diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h index 81c1d6aad49a..26bf4ab2618c 100644 --- a/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h +++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/FastISel.h @@ -490,7 +490,10 @@ protected: /// - \c Add has a constant operand. bool canFoldAddIntoGEP(const User *GEP, const Value *Add); - /// Test whether the given value has exactly one use. + /// Test whether the register associated with this value has exactly one use, + /// in which case that single use is killing. Note that multiple IR values + /// may map onto the same register, in which case this is not the same as + /// checking that an IR value has one use. bool hasTrivialKill(const Value *V); /// Create a machine mem operand from the given instruction. diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h index 6bbe2d03f9e5..f8d97c2c07a6 100644 --- a/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/MachineInstr.h @@ -1156,6 +1156,10 @@ public: return getOpcode() == TargetOpcode::CFI_INSTRUCTION; } + bool isPseudoProbe() const { + return getOpcode() == TargetOpcode::PSEUDO_PROBE; + } + // True if the instruction represents a position in the function. bool isPosition() const { return isLabel() || isCFIInstruction(); } @@ -1165,6 +1169,9 @@ public: bool isDebugInstr() const { return isDebugValue() || isDebugLabel() || isDebugRef(); } + bool isDebugOrPseudoInstr() const { + return isDebugInstr() || isPseudoProbe(); + } bool isDebugOffsetImm() const { return getDebugOffset().isImm(); } diff --git a/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h b/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h index c3221aac8eea..40115fbd2f15 100644 --- a/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/contrib/llvm-project/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2785,6 +2785,10 @@ public: return false; } + /// Does this target require the clearing of high-order bits in a register + /// passed to the fp16 to fp conversion library function. + virtual bool shouldKeepZExtForFP16Conv() const { return false; } + //===--------------------------------------------------------------------===// // Runtime Library hooks // diff --git a/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h b/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h index 6bfc02d15379..e5fca98f9271 100644 --- a/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h +++ b/contrib/llvm-project/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -96,7 +96,6 @@ X(InitListExpr) \ X(FoldExpr) \ X(ThrowExpr) \ - X(UUIDOfExpr) \ X(BoolExpr) \ X(StringLiteral) \ X(LambdaExpr) \ @@ -2035,21 +2034,6 @@ public: } }; -// MSVC __uuidof extension, generated by clang in -fms-extensions mode. -class UUIDOfExpr : public Node { - Node *Operand; -public: - UUIDOfExpr(Node *Operand_) : Node(KUUIDOfExpr), Operand(Operand_) {} - - template<typename Fn> void match(Fn F) const { F(Operand); } - - void printLeft(OutputStream &S) const override { - S << "__uuidof("; - Operand->print(S); - S << ")"; - } -}; - class BoolExpr : public Node { bool Value; @@ -5013,6 +4997,43 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { } } return nullptr; + case 'u': { + ++First; + Node *Name = getDerived().parseSourceName(/*NameState=*/nullptr); + if (!Name) + return nullptr; + // Special case legacy __uuidof mangling. The 't' and 'z' appear where the + // standard encoding expects a <template-arg>, and would be otherwise be + // interpreted as <type> node 'short' or 'ellipsis'. However, neither + // __uuidof(short) nor __uuidof(...) can actually appear, so there is no + // actual conflict here. + if (Name->getBaseName() == "__uuidof") { + if (numLeft() < 2) + return nullptr; + if (*First == 't') { + ++First; + Node *Ty = getDerived().parseType(); + if (!Ty) + return nullptr; + return make<CallExpr>(Name, makeNodeArray(&Ty, &Ty + 1)); + } + if (*First == 'z') { + ++First; + Node *Ex = getDerived().parseExpr(); + if (!Ex) + return nullptr; + return make<CallExpr>(Name, makeNodeArray(&Ex, &Ex + 1)); + } + } + size_t ExprsBegin = Names.size(); + while (!consumeIf('E')) { + Node *E = getDerived().parseTemplateArg(); + if (E == nullptr) + return E; + Names.push_back(E); + } + return make<CallExpr>(Name, popTrailingNodeArray(ExprsBegin)); + } case '1': case '2': case '3': @@ -5024,21 +5045,6 @@ Node *AbstractManglingParser<Derived, Alloc>::parseExpr() { case '9': return getDerived().parseUnresolvedName(); } - - if (consumeIf("u8__uuidoft")) { - Node *Ty = getDerived().parseType(); - if (!Ty) - return nullptr; - return make<UUIDOfExpr>(Ty); - } - - if (consumeIf("u8__uuidofz")) { - Node *Ex = getDerived().parseExpr(); - if (!Ex) - return nullptr; - return make<UUIDOfExpr>(Ex); - } - return nullptr; } diff --git a/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def index 844046167975..75d360bf4237 100644 --- a/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def +++ b/contrib/llvm-project/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def @@ -375,7 +375,7 @@ __OMP_RTL(__kmpc_init_allocator, false, /* omp_allocator_handle_t */ VoidPtr, __OMP_RTL(__kmpc_destroy_allocator, false, Void, /* Int */ Int32, /* omp_allocator_handle_t */ VoidPtr) -__OMP_RTL(__kmpc_push_target_tripcount, false, Void, IdentPtr, Int64, Int64) +__OMP_RTL(__kmpc_push_target_tripcount_mapper, false, Void, IdentPtr, Int64, Int64) __OMP_RTL(__tgt_target_mapper, false, Int32, IdentPtr, Int64, VoidPtr, Int32, VoidPtrPtr, VoidPtrPtr, Int64Ptr, Int64Ptr, VoidPtrPtr, VoidPtrPtr) __OMP_RTL(__tgt_target_nowait_mapper, false, Int32, IdentPtr, Int64, VoidPtr, Int32, @@ -844,7 +844,7 @@ __OMP_RTL_ATTRS(__kmpc_free, AllocAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__kmpc_init_allocator, DefaultAttrs, ReturnPtrAttrs, {}) __OMP_RTL_ATTRS(__kmpc_destroy_allocator, AllocAttrs, AttributeSet(), {}) -__OMP_RTL_ATTRS(__kmpc_push_target_tripcount, SetterAttrs, AttributeSet(), {}) +__OMP_RTL_ATTRS(__kmpc_push_target_tripcount_mapper, SetterAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__tgt_target_mapper, ForkAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__tgt_target_nowait_mapper, ForkAttrs, AttributeSet(), {}) __OMP_RTL_ATTRS(__tgt_target_teams_mapper, ForkAttrs, AttributeSet(), {}) diff --git a/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h b/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h index f42ef48de6b3..955ac8e537fe 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/InstrTypes.h @@ -1757,9 +1757,6 @@ public: return doesNotAccessMemory() || hasFnAttr(Attribute::ReadOnly); } - /// Returns true if this function is guaranteed to return. - bool willReturn() const { return hasFnAttr(Attribute::WillReturn); } - void setOnlyReadsMemory() { addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly); } diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h b/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h index d2a55f89fac9..b99dc62bbb9d 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/Instruction.h @@ -633,6 +633,10 @@ public: /// generated program. bool isSafeToRemove() const; + /// Return true if the instruction will return (unwinding is considered as + /// a form of returning control flow here). + bool willReturn() const; + /// Return true if the instruction is a variety of EH-block. bool isEHPad() const { switch (getOpcode()) { @@ -650,6 +654,9 @@ public: /// llvm.lifetime.end marker. bool isLifetimeStartOrEnd() const; + /// Return true if the instruction is a DbgInfoIntrinsic or PseudoProbeInst. + bool isDebugOrPseudoInst() const; + /// Return a pointer to the next non-debug instruction in the same basic /// block as 'this', or nullptr if no such instruction exists. Skip any pseudo /// operations if \c SkipPseudoOp is true. diff --git a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h index 9d68f3fdde6c..df3a1d568756 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicInst.h @@ -981,12 +981,16 @@ public: return cast<ConstantInt>(const_cast<Value *>(getArgOperand(0))); } + ConstantInt *getIndex() const { + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1))); + } + ConstantInt *getAttributes() const { return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2))); } - ConstantInt *getIndex() const { - return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1))); + ConstantInt *getFactor() const { + return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3))); } }; diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td b/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td index b2bfc6e6f9e6..21307ed1bd91 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td +++ b/contrib/llvm-project/llvm/include/llvm/IR/Intrinsics.td @@ -1298,7 +1298,7 @@ def int_sideeffect : DefaultAttrsIntrinsic<[], [], [IntrInaccessibleMemOnly, Int // Like the sideeffect intrinsic defined above, this intrinsic is treated by the // optimizer as having opaque side effects so that it won't be get rid of or moved // out of the block it probes. -def int_pseudoprobe : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty], +def int_pseudoprobe : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty, llvm_i64_ty], [IntrInaccessibleMemOnly, IntrWillReturn]>; // Intrinsics to support half precision floating point format diff --git a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td index ab5b09b72ac3..c4056895f68e 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -790,6 +790,9 @@ let TargetPrefix = "riscv" in { defm vsoxei : RISCVIStore; defm vsuxei : RISCVIStore; + def int_riscv_vle1 : RISCVUSLoad; + def int_riscv_vse1 : RISCVUSStore; + defm vamoswap : RISCVAMO; defm vamoadd : RISCVAMO; defm vamoxor : RISCVAMO; @@ -940,8 +943,8 @@ let TargetPrefix = "riscv" in { defm vfwnmsac : RISCVTernaryWide; defm vfsqrt : RISCVUnaryAA; - defm vfrsqrte7 : RISCVUnaryAA; - defm vfrece7 : RISCVUnaryAA; + defm vfrsqrt7 : RISCVUnaryAA; + defm vfrec7 : RISCVUnaryAA; defm vfmin : RISCVBinaryAAX; defm vfmax : RISCVBinaryAAX; diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h b/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h index 0b87416befe9..9a4480b75a30 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/Metadata.h @@ -667,6 +667,12 @@ struct AAMDNodes { /// The tag specifying the noalias scope. MDNode *NoAlias = nullptr; + // Shift tbaa Metadata node to start off bytes later + static MDNode *ShiftTBAA(MDNode *M, size_t off); + + // Shift tbaa.struct Metadata node to start off bytes later + static MDNode *ShiftTBAAStruct(MDNode *M, size_t off); + /// Given two sets of AAMDNodes that apply to the same pointer, /// give the best AAMDNodes that are compatible with both (i.e. a set of /// nodes whose allowable aliasing conclusions are a subset of those @@ -680,6 +686,18 @@ struct AAMDNodes { Result.NoAlias = Other.NoAlias == NoAlias ? NoAlias : nullptr; return Result; } + + /// Create a new AAMDNode that describes this AAMDNode after applying a + /// constant offset to the start of the pointer + AAMDNodes shift(size_t Offset) { + AAMDNodes Result; + Result.TBAA = TBAA ? ShiftTBAA(TBAA, Offset) : nullptr; + Result.TBAAStruct = + TBAAStruct ? ShiftTBAAStruct(TBAAStruct, Offset) : nullptr; + Result.Scope = Scope; + Result.NoAlias = NoAlias; + return Result; + } }; // Specialize DenseMapInfo for AAMDNodes. diff --git a/contrib/llvm-project/llvm/include/llvm/IR/Operator.h b/contrib/llvm-project/llvm/include/llvm/IR/Operator.h index acfacbd6c74e..945f7e46e142 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/Operator.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/Operator.h @@ -568,6 +568,11 @@ public: bool accumulateConstantOffset( const DataLayout &DL, APInt &Offset, function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr) const; + + static bool accumulateConstantOffset( + Type *SourceType, ArrayRef<const Value *> Index, const DataLayout &DL, + APInt &Offset, + function_ref<bool(Value &, APInt &)> ExternalAnalysis = nullptr); }; class PtrToIntOperator diff --git a/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h b/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h index e0370c264102..5165e80caa2d 100644 --- a/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h +++ b/contrib/llvm-project/llvm/include/llvm/IR/PseudoProbe.h @@ -16,28 +16,39 @@ #include "llvm/ADT/Optional.h" #include <cassert> #include <cstdint> +#include <limits> namespace llvm { class Instruction; +class BasicBlock; constexpr const char *PseudoProbeDescMetadataName = "llvm.pseudo_probe_desc"; enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall }; +// The saturated distrution factor representing 100% for block probes. +constexpr static uint64_t PseudoProbeFullDistributionFactor = + std::numeric_limits<uint64_t>::max(); + struct PseudoProbeDwarfDiscriminator { +public: // The following APIs encodes/decodes per-probe information to/from a // 32-bit integer which is organized as: // [2:0] - 0x7, this is reserved for regular discriminator, // see DWARF discriminator encoding rule // [18:3] - probe id - // [25:19] - reserved + // [25:19] - probe distribution factor // [28:26] - probe type, see PseudoProbeType // [31:29] - reserved for probe attributes - static uint32_t packProbeData(uint32_t Index, uint32_t Type) { + static uint32_t packProbeData(uint32_t Index, uint32_t Type, uint32_t Flags, + uint32_t Factor) { assert(Index <= 0xFFFF && "Probe index too big to encode, exceeding 2^16"); assert(Type <= 0x7 && "Probe type too big to encode, exceeding 7"); - return (Index << 3) | (Type << 26) | 0x7; + assert(Flags <= 0x7); + assert(Factor <= 100 && + "Probe distribution factor too big to encode, exceeding 100"); + return (Index << 3) | (Factor << 19) | (Type << 26) | 0x7; } static uint32_t extractProbeIndex(uint32_t Value) { @@ -51,16 +62,26 @@ struct PseudoProbeDwarfDiscriminator { static uint32_t extractProbeAttributes(uint32_t Value) { return (Value >> 29) & 0x7; } + + static uint32_t extractProbeFactor(uint32_t Value) { + return (Value >> 19) & 0x7F; + } + + // The saturated distrution factor representing 100% for callsites. + constexpr static uint8_t FullDistributionFactor = 100; }; struct PseudoProbe { uint32_t Id; uint32_t Type; uint32_t Attr; + float Factor; }; Optional<PseudoProbe> extractProbe(const Instruction &Inst); +void setProbeDistributionFactor(Instruction &Inst, float Factor); + } // end namespace llvm #endif // LLVM_IR_PSEUDOPROBE_H diff --git a/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h b/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h index 795a980878e2..61c86b0468f2 100644 --- a/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h +++ b/contrib/llvm-project/llvm/include/llvm/Passes/StandardInstrumentations.h @@ -22,6 +22,7 @@ #include "llvm/IR/PassTimingInfo.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Transforms/IPO/SampleProfileProbe.h" #include <string> #include <utility> @@ -273,6 +274,7 @@ class StandardInstrumentations { OptBisectInstrumentation OptBisect; PreservedCFGCheckerInstrumentation PreservedCFGChecker; IRChangedPrinter PrintChangedIR; + PseudoProbeVerifier PseudoProbeVerification; VerifyInstrumentation Verify; bool VerifyEach; diff --git a/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h b/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h index 6bb5825339ae..55b94b2e690d 100644 --- a/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h +++ b/contrib/llvm-project/llvm/include/llvm/ProfileData/ProfileCommon.h @@ -17,6 +17,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Error.h" #include <algorithm> #include <cstdint> @@ -89,6 +90,8 @@ public: void addRecord(const sampleprof::FunctionSamples &FS, bool isCallsiteSample = false); + std::unique_ptr<ProfileSummary> computeSummaryForProfiles( + const StringMap<sampleprof::FunctionSamples> &Profiles); std::unique_ptr<ProfileSummary> getSummary(); }; diff --git a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h index c45ace9e68c1..25d5b2376c11 100644 --- a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h +++ b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProf.h @@ -347,6 +347,16 @@ public: return SortedTargets; } + /// Prorate call targets by a distribution factor. + static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets, + float DistributionFactor) { + CallTargetMap AdjustedTargets; + for (const auto &I : Targets) { + AdjustedTargets[I.first()] = I.second * DistributionFactor; + } + return AdjustedTargets; + } + /// Merge the samples in \p Other into this record. /// Optionally scale sample counts by \p Weight. sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1) { @@ -439,9 +449,11 @@ public: void clearState(ContextStateMask S) { State &= (uint32_t)~S; } bool hasContext() const { return State != UnknownContext; } bool isBaseContext() const { return CallingContext.empty(); } - StringRef getName() const { return Name; } + StringRef getNameWithoutContext() const { return Name; } StringRef getCallingContext() const { return CallingContext; } - StringRef getNameWithContext() const { return FullContext; } + StringRef getNameWithContext(bool WithBracket = false) const { + return WithBracket ? InputContext : FullContext; + } private: // Give a context string, decode and populate internal states like @@ -449,6 +461,7 @@ private: // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` void setContext(StringRef ContextStr, ContextStateMask CState) { assert(!ContextStr.empty()); + InputContext = ContextStr; // Note that `[]` wrapped input indicates a full context string, otherwise // it's treated as context-less function name only. bool HasContext = ContextStr.startswith("["); @@ -480,6 +493,9 @@ private: } } + // Input context string including bracketed calling context and leaf function + // name + StringRef InputContext; // Full context string including calling context and leaf function name StringRef FullContext; // Function name for the associated sample profile @@ -676,7 +692,8 @@ public: Name = Other.getName(); if (!GUIDToFuncNameMap) GUIDToFuncNameMap = Other.GUIDToFuncNameMap; - + if (Context.getNameWithContext(true).empty()) + Context = Other.getContext(); if (FunctionHash == 0) { // Set the function hash code for the target profile. FunctionHash = Other.getFunctionHash(); @@ -743,8 +760,10 @@ public: StringRef getName() const { return Name; } /// Return function name with context. - StringRef getNameWithContext() const { - return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name; + StringRef getNameWithContext(bool WithBracket = false) const { + return FunctionSamples::ProfileIsCS + ? Context.getNameWithContext(WithBracket) + : Name; } /// Return the original function name. diff --git a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h index 3f52a2f6163b..999e75eddffa 100644 --- a/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/contrib/llvm-project/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -488,8 +488,12 @@ protected: /// \brief Whether samples are collected based on pseudo probes. bool ProfileIsProbeBased = false; + /// Whether function profiles are context-sensitive. bool ProfileIsCS = false; + /// Number of context-sensitive profiles. + uint32_t CSProfileCount = 0; + /// \brief The format of sample. SampleProfileFormat Format = SPF_None; }; diff --git a/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h b/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h index 38f3e188be55..0706aa226c0e 100644 --- a/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h +++ b/contrib/llvm-project/llvm/include/llvm/Support/CommandLine.h @@ -369,9 +369,22 @@ public: virtual void setDefault() = 0; + // Prints the help string for an option. + // + // This maintains the Indent for multi-line descriptions. + // FirstLineIndentedBy is the count of chars of the first line + // i.e. the one containing the --<option name>. static void printHelpStr(StringRef HelpStr, size_t Indent, size_t FirstLineIndentedBy); + // Prints the help string for an enum value. + // + // This maintains the Indent for multi-line descriptions. + // FirstLineIndentedBy is the count of chars of the first line + // i.e. the one containing the =<value>. + static void printEnumValHelpStr(StringRef HelpStr, size_t Indent, + size_t FirstLineIndentedBy); + virtual void getExtraOptionNames(SmallVectorImpl<StringRef> &) {} // addOccurrence - Wrapper around handleOccurrence that enforces Flags. diff --git a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h index 5b2600144fa3..da0bdae0eaee 100644 --- a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h +++ b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h @@ -18,11 +18,13 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/CallGraph.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/Instructions.h" #include "llvm/ProfileData/SampleProf.h" #include <list> #include <map> +#include <vector> using namespace llvm; using namespace sampleprof; @@ -42,7 +44,7 @@ public: CallSiteLoc(CallLoc){}; ContextTrieNode *getChildContext(const LineLocation &CallSite, StringRef CalleeName); - ContextTrieNode *getChildContext(const LineLocation &CallSite); + ContextTrieNode *getHottestChildContext(const LineLocation &CallSite); ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate = true); @@ -89,16 +91,24 @@ private: // calling context and the context is identified by path from root to the node. class SampleContextTracker { public: + using ContextSamplesTy = SmallSet<FunctionSamples *, 16>; + SampleContextTracker(StringMap<FunctionSamples> &Profiles); // Query context profile for a specific callee with given name at a given // call-site. The full context is identified by location of call instruction. FunctionSamples *getCalleeContextSamplesFor(const CallBase &Inst, StringRef CalleeName); + // Get samples for indirect call targets for call site at given location. + std::vector<const FunctionSamples *> + getIndirectCalleeContextSamplesFor(const DILocation *DIL); // Query context profile for a given location. The full context // is identified by input DILocation. FunctionSamples *getContextSamplesFor(const DILocation *DIL); // Query context profile for a given sample contxt of a function. FunctionSamples *getContextSamplesFor(const SampleContext &Context); + // Get all context profile for given function. + ContextSamplesTy &getAllContextSamplesFor(const Function &Func); + ContextSamplesTy &getAllContextSamplesFor(StringRef Name); // Query base profile for a given function. A base profile is a merged view // of all context profiles for contexts that are not inlined. FunctionSamples *getBaseSamplesFor(const Function &Func, @@ -109,6 +119,9 @@ public: // This makes sure that inlined context profile will be excluded in // function's base profile. void markContextSamplesInlined(const FunctionSamples *InlinedSamples); + void promoteMergeContextSamplesTree(const Instruction &Inst, + StringRef CalleeName); + void addCallGraphEdges(CallGraph &CG, StringMap<Function *> &SymbolMap); // Dump the internal context profile trie. void dump(); @@ -122,8 +135,6 @@ private: ContextTrieNode *getTopLevelContextNode(StringRef FName); ContextTrieNode &addTopLevelContextNode(StringRef FName); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo); - void promoteMergeContextSamplesTree(const Instruction &Inst, - StringRef CalleeName); void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode, StringRef ContextStrToRemove); ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &FromNode, @@ -131,7 +142,7 @@ private: StringRef ContextStrToRemove); // Map from function name to context profiles (excluding base profile) - StringMap<SmallSet<FunctionSamples *, 16>> FuncToCtxtProfileSet; + StringMap<ContextSamplesTy> FuncToCtxtProfileSet; // Root node for context trie tree ContextTrieNode RootContext; diff --git a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h index 78117fd4a9c2..0fd79d8ff7f3 100644 --- a/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h +++ b/contrib/llvm-project/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -16,6 +16,10 @@ #define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H #include "llvm/ADT/DenseMap.h" +#include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/PseudoProbe.h" #include "llvm/ProfileData/SampleProf.h" @@ -29,6 +33,8 @@ class Module; using namespace sampleprof; using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>; using InstructionIdMap = std::unordered_map<Instruction *, uint32_t>; +using ProbeFactorMap = std::unordered_map<uint64_t, float>; +using FuncProbeFactorMap = StringMap<ProbeFactorMap>; enum class PseudoProbeReservedId { Invalid = 0, Last = Invalid }; @@ -43,6 +49,33 @@ public: uint64_t getFunctionHash() const { return FunctionHash; } }; +// A pseudo probe verifier that can be run after each IR passes to detect the +// violation of updating probe factors. In principle, the sum of distribution +// factor for a probe should be identical before and after a pass. For a +// function pass, the factor sum for a probe would be typically 100%. +class PseudoProbeVerifier { +public: + void registerCallbacks(PassInstrumentationCallbacks &PIC); + + // Implementation of pass instrumentation callbacks for new pass manager. + void runAfterPass(StringRef PassID, Any IR); + +private: + // Allow a little bias due the rounding to integral factors. + constexpr static float DistributionFactorVariance = 0.02f; + // Distribution factors from last pass. + FuncProbeFactorMap FunctionProbeFactors; + + void collectProbeFactors(const BasicBlock *BB, ProbeFactorMap &ProbeFactors); + void runAfterPass(const Module *M); + void runAfterPass(const LazyCallGraph::SCC *C); + void runAfterPass(const Function *F); + void runAfterPass(const Loop *L); + bool shouldVerifyFunction(const Function *F); + void verifyProbeFactors(const Function *F, + const ProbeFactorMap &ProbeFactors); +}; + // This class serves sample counts correlation for SampleProfileLoader by // analyzing pseudo probes and their function descriptors injected by // SampleProfileProber. @@ -102,5 +135,13 @@ public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; +class PseudoProbeUpdatePass : public PassInfoMixin<PseudoProbeUpdatePass> { + void runOnFunction(Function &F, FunctionAnalysisManager &FAM); + +public: + PseudoProbeUpdatePass() {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + } // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H diff --git a/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h b/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h index 56aaa5d48e2a..aa960c625630 100644 --- a/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/contrib/llvm-project/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -274,6 +274,13 @@ void updateProfileCallee( void identifyNoAliasScopesToClone( ArrayRef<BasicBlock *> BBs, SmallVectorImpl<MDNode *> &NoAliasDeclScopes); +/// Find the 'llvm.experimental.noalias.scope.decl' intrinsics in the specified +/// instruction range and extract their scope. These are candidates for +/// duplication when cloning. +void identifyNoAliasScopesToClone( + BasicBlock::iterator Start, BasicBlock::iterator End, + SmallVectorImpl<MDNode *> &NoAliasDeclScopes); + /// Duplicate the specified list of noalias decl scopes. /// The 'Ext' string is added as an extension to the name. /// Afterwards, the ClonedScopes contains the mapping of the original scope |