diff options
Diffstat (limited to 'include/llvm/IR')
75 files changed, 5377 insertions, 3937 deletions
diff --git a/include/llvm/IR/Argument.h b/include/llvm/IR/Argument.h index 0092f49e49ad..d8b280a66f18 100644 --- a/include/llvm/IR/Argument.h +++ b/include/llvm/IR/Argument.h @@ -73,6 +73,12 @@ public: /// containing function. bool hasByValAttr() const; + /// \brief Return true if this argument has the swiftself attribute. + bool hasSwiftSelfAttr() const; + + /// \brief Return true if this argument has the swifterror attribute. + bool hasSwiftErrorAttr() const; + /// \brief Return true if this argument has the byval attribute or inalloca /// attribute on it in its containing function. These attributes both /// represent arguments being passed by value. @@ -120,9 +126,20 @@ public: /// \brief Add a Attribute to an argument. void addAttr(AttributeSet AS); + void addAttr(Attribute::AttrKind Kind) { + addAttr(AttributeSet::get(getContext(), getArgNo() + 1, Kind)); + } + /// \brief Remove a Attribute from an argument. void removeAttr(AttributeSet AS); + void removeAttr(Attribute::AttrKind Kind) { + removeAttr(AttributeSet::get(getContext(), getArgNo() + 1, Kind)); + } + + /// \brief Checks if an argument has a given attribute. + bool hasAttribute(Attribute::AttrKind Kind) const; + /// \brief Method for support type inquiry through isa, cast, and /// dyn_cast. static inline bool classof(const Value *V) { diff --git a/include/llvm/IR/Attributes.h b/include/llvm/IR/Attributes.h index 0e3373165407..af1bf0a354ec 100644 --- a/include/llvm/IR/Attributes.h +++ b/include/llvm/IR/Attributes.h @@ -18,8 +18,10 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/Optional.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm-c/Types.h" #include <bitset> #include <cassert> #include <map> @@ -94,6 +96,9 @@ public: uint64_t Bytes); static Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes); + static Attribute getWithAllocSizeArgs(LLVMContext &Context, + unsigned ElemSizeArg, + const Optional<unsigned> &NumElemsArg); //===--------------------------------------------------------------------===// // Attribute Accessors @@ -116,11 +121,11 @@ public: bool hasAttribute(StringRef Val) const; /// \brief Return the attribute's kind as an enum (Attribute::AttrKind). This - /// requires the attribute to be an enum or alignment attribute. + /// requires the attribute to be an enum or integer attribute. Attribute::AttrKind getKindAsEnum() const; /// \brief Return the attribute's value as an integer. This requires that the - /// attribute be an alignment attribute. + /// attribute be an integer attribute. uint64_t getValueAsInt() const; /// \brief Return the attribute's kind as a string. This requires the @@ -147,6 +152,10 @@ public: /// dereferenceable_or_null attribute. uint64_t getDereferenceableOrNullBytes() const; + /// Returns the argument numbers for the allocsize attribute (or pair(0, 0) + /// if not known). + std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const; + /// \brief The Attribute is converted to a string of equivalent mnemonic. This /// is, presumably, for writing out the mnemonics for the assembly writer. std::string getAsString(bool InAttrGrp = false) const; @@ -161,8 +170,28 @@ public: void Profile(FoldingSetNodeID &ID) const { ID.AddPointer(pImpl); } + + /// \brief Return a raw pointer that uniquely identifies this attribute. + void *getRawPointer() const { + return pImpl; + } + + /// \brief Get an attribute from a raw pointer created by getRawPointer. + static Attribute fromRawPointer(void *RawPtr) { + return Attribute(reinterpret_cast<AttributeImpl*>(RawPtr)); + } }; +// Specialized opaque value conversions. +inline LLVMAttributeRef wrap(Attribute Attr) { + return reinterpret_cast<LLVMAttributeRef>(Attr.getRawPointer()); +} + +// Specialized opaque value conversions. +inline Attribute unwrap(LLVMAttributeRef Attr) { + return Attribute::fromRawPointer(Attr); +} + //===----------------------------------------------------------------------===// /// \class /// \brief This class holds the attributes for a function, its return value, and @@ -213,20 +242,20 @@ public: /// \brief Return an AttributeSet with the specified parameters in it. static AttributeSet get(LLVMContext &C, ArrayRef<AttributeSet> Attrs); static AttributeSet get(LLVMContext &C, unsigned Index, - ArrayRef<Attribute::AttrKind> Kind); + ArrayRef<Attribute::AttrKind> Kinds); + static AttributeSet get(LLVMContext &C, unsigned Index, + ArrayRef<StringRef> Kind); static AttributeSet get(LLVMContext &C, unsigned Index, const AttrBuilder &B); /// \brief Add an attribute to the attribute set at the given index. Because /// attribute sets are immutable, this returns a new set. AttributeSet addAttribute(LLVMContext &C, unsigned Index, - Attribute::AttrKind Attr) const; + Attribute::AttrKind Kind) const; /// \brief Add an attribute to the attribute set at the given index. Because /// attribute sets are immutable, this returns a new set. - AttributeSet addAttribute(LLVMContext &C, unsigned Index, - StringRef Kind) const; - AttributeSet addAttribute(LLVMContext &C, unsigned Index, - StringRef Kind, StringRef Value) const; + AttributeSet addAttribute(LLVMContext &C, unsigned Index, StringRef Kind, + StringRef Value = StringRef()) const; /// Add an attribute to the attribute set at the given indices. Because /// attribute sets are immutable, this returns a new set. @@ -242,7 +271,13 @@ public: /// attribute list. Because attribute lists are immutable, this returns the /// new list. AttributeSet removeAttribute(LLVMContext &C, unsigned Index, - Attribute::AttrKind Attr) const; + Attribute::AttrKind Kind) const; + + /// \brief Remove the specified attribute at the specified index from this + /// attribute list. Because attribute lists are immutable, this returns the + /// new list. + AttributeSet removeAttribute(LLVMContext &C, unsigned Index, + StringRef Kind) const; /// \brief Remove the specified attributes at the specified index from this /// attribute list. Because attribute lists are immutable, this returns the @@ -267,6 +302,12 @@ public: AttributeSet addDereferenceableOrNullAttr(LLVMContext &C, unsigned Index, uint64_t Bytes) const; + /// Add the allocsize attribute to the attribute set at the given index. + /// Because attribute sets are immutable, this returns a new set. + AttributeSet addAllocSizeAttr(LLVMContext &C, unsigned Index, + unsigned ElemSizeArg, + const Optional<unsigned> &NumElemsArg); + //===--------------------------------------------------------------------===// // AttributeSet Accessors //===--------------------------------------------------------------------===// @@ -292,9 +333,15 @@ public: /// \brief Return true if attribute exists at the given index. bool hasAttributes(unsigned Index) const; + /// \brief Equivalent to hasAttribute(AttributeSet::FunctionIndex, Kind) but + /// may be faster. + bool hasFnAttribute(Attribute::AttrKind Kind) const; + /// \brief Return true if the specified attribute is set for at least one - /// parameter or for the return value. - bool hasAttrSomewhere(Attribute::AttrKind Attr) const; + /// parameter or for the return value. If Index is not nullptr, the index + /// of a parameter with the specified attribute is provided. + bool hasAttrSomewhere(Attribute::AttrKind Kind, + unsigned *Index = nullptr) const; /// \brief Return the attribute object that exists at the given index. Attribute getAttribute(unsigned Index, Attribute::AttrKind Kind) const; @@ -315,6 +362,10 @@ public: /// unknown). uint64_t getDereferenceableOrNullBytes(unsigned Index) const; + /// Get the allocsize argument numbers (or pair(0, 0) if unknown). + std::pair<unsigned, Optional<unsigned>> + getAllocSizeArgs(unsigned Index) const; + /// \brief Return the attributes at the index as a string. std::string getAsString(unsigned Index, bool InAttrGrp = false) const; @@ -396,19 +447,20 @@ class AttrBuilder { uint64_t StackAlignment; uint64_t DerefBytes; uint64_t DerefOrNullBytes; + uint64_t AllocSizeArgs; public: AttrBuilder() : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0), - DerefOrNullBytes(0) {} + DerefOrNullBytes(0), AllocSizeArgs(0) {} explicit AttrBuilder(uint64_t Val) : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0), - DerefOrNullBytes(0) { + DerefOrNullBytes(0), AllocSizeArgs(0) { addRawValue(Val); } AttrBuilder(const Attribute &A) : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0), - DerefOrNullBytes(0) { + DerefOrNullBytes(0), AllocSizeArgs(0) { addAttribute(A); } AttrBuilder(AttributeSet AS, unsigned Idx); @@ -477,6 +529,10 @@ public: /// dereferenceable_or_null attribute exists (zero is returned otherwise). uint64_t getDereferenceableOrNullBytes() const { return DerefOrNullBytes; } + /// Retrieve the allocsize args, if the allocsize attribute exists. If it + /// doesn't exist, pair(0, 0) is returned. + std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const; + /// \brief This turns an int alignment (which must be a power of 2) into the /// form used internally in Attribute. AttrBuilder &addAlignmentAttr(unsigned Align); @@ -493,6 +549,14 @@ public: /// form used internally in Attribute. AttrBuilder &addDereferenceableOrNullAttr(uint64_t Bytes); + /// This turns one (or two) ints into the form used internally in Attribute. + AttrBuilder &addAllocSizeAttr(unsigned ElemSizeArg, + const Optional<unsigned> &NumElemsArg); + + /// Add an allocsize attribute, using the representation returned by + /// Attribute.getIntValue(). + AttrBuilder &addAllocSizeAttrFromRawRepr(uint64_t RawAllocSizeRepr); + /// \brief Return true if the builder contains no target-independent /// attributes. bool empty() const { return Attrs.none(); } diff --git a/include/llvm/IR/Attributes.td b/include/llvm/IR/Attributes.td index 30249bbd8fab..7b63638a3f6a 100644 --- a/include/llvm/IR/Attributes.td +++ b/include/llvm/IR/Attributes.td @@ -16,6 +16,10 @@ class StrBoolAttr<string S> : Attr<S>; /// 0 means unaligned (different from align(1)). def Alignment : EnumAttr<"align">; +/// The result of the function is guaranteed to point to a number of bytes that +/// we can determine if we know the value of the function's arguments. +def AllocSize : EnumAttr<"allocsize">; + /// inline=always. def AlwaysInline : EnumAttr<"alwaysinline">; @@ -154,9 +158,18 @@ def SanitizeThread : EnumAttr<"sanitize_thread">; /// MemorySanitizer is on. def SanitizeMemory : EnumAttr<"sanitize_memory">; +/// Argument is swift error. +def SwiftError : EnumAttr<"swifterror">; + +/// Argument is swift self/context. +def SwiftSelf : EnumAttr<"swiftself">; + /// Function must be in a unwind table. def UWTable : EnumAttr<"uwtable">; +/// Function only writes to memory. +def WriteOnly : EnumAttr<"writeonly">; + /// Zero extended before/after call. def ZExt : EnumAttr<"zeroext">; @@ -165,6 +178,7 @@ def LessPreciseFPMAD : StrBoolAttr<"less-precise-fpmad">; def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">; def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">; def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">; +def NoJumpTables : StrBoolAttr<"no-jump-tables">; class CompatRule<string F> { // The name of the function called to check the attribute of the caller and @@ -179,6 +193,7 @@ class CompatRule<string F> { def : CompatRule<"isEqual<SanitizeAddressAttr>">; def : CompatRule<"isEqual<SanitizeThreadAttr>">; def : CompatRule<"isEqual<SanitizeMemoryAttr>">; +def : CompatRule<"isEqual<SafeStackAttr>">; class MergeRule<string F> { // The name of the function called to merge the attributes of the caller and @@ -194,4 +209,5 @@ def : MergeRule<"setAND<NoInfsFPMathAttr>">; def : MergeRule<"setAND<NoNansFPMathAttr>">; def : MergeRule<"setAND<UnsafeFPMathAttr>">; def : MergeRule<"setOR<NoImplicitFloatAttr>">; +def : MergeRule<"setOR<NoJumpTablesAttr>">; def : MergeRule<"adjustCallerSSPLevel">; diff --git a/include/llvm/IR/AutoUpgrade.h b/include/llvm/IR/AutoUpgrade.h index a4b3c410c4f6..9eb358682c65 100644 --- a/include/llvm/IR/AutoUpgrade.h +++ b/include/llvm/IR/AutoUpgrade.h @@ -14,13 +14,14 @@ #ifndef LLVM_IR_AUTOUPGRADE_H #define LLVM_IR_AUTOUPGRADE_H -#include <string> +#include "llvm/ADT/StringRef.h" namespace llvm { class CallInst; class Constant; class Function; class Instruction; + class MDNode; class Module; class GlobalVariable; class Type; @@ -46,6 +47,10 @@ namespace llvm { /// if it requires upgrading. bool UpgradeGlobalVariable(GlobalVariable *GV); + /// This checks for module flags which should be upgraded. It returns true if + /// module is modified. + bool UpgradeModuleFlags(Module &M); + /// If the TBAA tag for the given instruction uses the scalar TBAA format, /// we upgrade it to the struct-path aware TBAA format. void UpgradeInstWithTBAATag(Instruction *I); @@ -64,8 +69,14 @@ namespace llvm { /// info. Return true if module is modified. bool UpgradeDebugInfo(Module &M); - /// Upgrade a metadata string constant in place. - void UpgradeMDStringConstant(std::string &String); + /// Check whether a string looks like an old loop attachment tag. + inline bool mayBeOldLoopAttachmentTag(StringRef Name) { + return Name.startswith("llvm.vectorizer."); + } + + /// Upgrade the loop attachment metadata node. + MDNode *upgradeInstructionLoopAttachment(MDNode &N); + } // End llvm namespace #endif diff --git a/include/llvm/IR/BasicBlock.h b/include/llvm/IR/BasicBlock.h index c6b54d308ce6..e7daf6ee238e 100644 --- a/include/llvm/IR/BasicBlock.h +++ b/include/llvm/IR/BasicBlock.h @@ -111,6 +111,14 @@ public: TerminatorInst *getTerminator(); const TerminatorInst *getTerminator() const; + /// \brief Returns the call instruction calling @llvm.experimental.deoptimize + /// prior to the terminating return instruction of this basic block, if such a + /// call is present. Otherwise, returns null. + CallInst *getTerminatingDeoptimizeCall(); + const CallInst *getTerminatingDeoptimizeCall() const { + return const_cast<BasicBlock *>(this)->getTerminatingDeoptimizeCall(); + } + /// \brief Returns the call instruction marked 'musttail' prior to the /// terminating return instruction of this basic block, if such a call is /// present. Otherwise, returns null. diff --git a/include/llvm/IR/CallSite.h b/include/llvm/IR/CallSite.h index f7bfb47a5b44..9c977aef941a 100644 --- a/include/llvm/IR/CallSite.h +++ b/include/llvm/IR/CallSite.h @@ -273,6 +273,10 @@ public: CALLSITE_DELEGATE_GETTER(getArgOperand(i)); } + ValTy *getReturnedArgOperand() const { + CALLSITE_DELEGATE_GETTER(getReturnedArgOperand()); + } + bool isInlineAsm() const { if (isCall()) return cast<CallInst>(getInstruction())->isInlineAsm(); @@ -305,19 +309,51 @@ public: CALLSITE_DELEGATE_SETTER(setAttributes(PAL)); } + void addAttribute(unsigned i, Attribute::AttrKind Kind) { + CALLSITE_DELEGATE_SETTER(addAttribute(i, Kind)); + } + + void addAttribute(unsigned i, StringRef Kind, StringRef Value) { + CALLSITE_DELEGATE_SETTER(addAttribute(i, Kind, Value)); + } + + void addAttribute(unsigned i, Attribute Attr) { + CALLSITE_DELEGATE_SETTER(addAttribute(i, Attr)); + } + + void removeAttribute(unsigned i, Attribute::AttrKind Kind) { + CALLSITE_DELEGATE_SETTER(removeAttribute(i, Kind)); + } + + void removeAttribute(unsigned i, StringRef Kind) { + CALLSITE_DELEGATE_SETTER(removeAttribute(i, Kind)); + } + + void removeAttribute(unsigned i, Attribute Attr) { + CALLSITE_DELEGATE_SETTER(removeAttribute(i, Attr)); + } + /// \brief Return true if this function has the given attribute. - bool hasFnAttr(Attribute::AttrKind A) const { - CALLSITE_DELEGATE_GETTER(hasFnAttr(A)); + bool hasFnAttr(Attribute::AttrKind Kind) const { + CALLSITE_DELEGATE_GETTER(hasFnAttr(Kind)); } /// \brief Return true if this function has the given attribute. - bool hasFnAttr(StringRef A) const { - CALLSITE_DELEGATE_GETTER(hasFnAttr(A)); + bool hasFnAttr(StringRef Kind) const { + CALLSITE_DELEGATE_GETTER(hasFnAttr(Kind)); } /// \brief Return true if the call or the callee has the given attribute. - bool paramHasAttr(unsigned i, Attribute::AttrKind A) const { - CALLSITE_DELEGATE_GETTER(paramHasAttr(i, A)); + bool paramHasAttr(unsigned i, Attribute::AttrKind Kind) const { + CALLSITE_DELEGATE_GETTER(paramHasAttr(i, Kind)); + } + + Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const { + CALLSITE_DELEGATE_GETTER(getAttribute(i, Kind)); + } + + Attribute getAttribute(unsigned i, StringRef Kind) const { + CALLSITE_DELEGATE_GETTER(getAttribute(i, Kind)); } /// \brief Return true if the data operand at index \p i directly or @@ -327,8 +363,8 @@ public: /// in the attribute set attached to this instruction, while operand bundle /// operands may have some attributes implied by the type of its containing /// operand bundle. - bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind A) const { - CALLSITE_DELEGATE_GETTER(dataOperandHasImpliedAttr(i, A)); + bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const { + CALLSITE_DELEGATE_GETTER(dataOperandHasImpliedAttr(i, Kind)); } /// @brief Extract the alignment for a call or parameter (0=unknown). @@ -385,6 +421,14 @@ public: CALLSITE_DELEGATE_SETTER(setOnlyReadsMemory()); } + /// @brief Determine if the call does not access or only writes memory. + bool doesNotReadMemory() const { + CALLSITE_DELEGATE_GETTER(doesNotReadMemory()); + } + void setDoesNotReadMemory() { + CALLSITE_DELEGATE_SETTER(setDoesNotReadMemory()); + } + /// @brief Determine if the call can access memmory only using pointers based /// on its arguments. bool onlyAccessesArgMemory() const { @@ -410,6 +454,25 @@ public: CALLSITE_DELEGATE_SETTER(setDoesNotThrow()); } + /// @brief Determine if the call can be duplicated. + bool cannotDuplicate() const { + CALLSITE_DELEGATE_GETTER(cannotDuplicate()); + } + void setCannotDuplicate() { + CALLSITE_DELEGATE_GETTER(setCannotDuplicate()); + } + + /// @brief Determine if the call is convergent. + bool isConvergent() const { + CALLSITE_DELEGATE_GETTER(isConvergent()); + } + void setConvergent() { + CALLSITE_DELEGATE_SETTER(setConvergent()); + } + void setNotConvergent() { + CALLSITE_DELEGATE_SETTER(setNotConvergent()); + } + unsigned getNumOperandBundles() const { CALLSITE_DELEGATE_GETTER(getNumOperandBundles()); } @@ -442,6 +505,10 @@ public: CALLSITE_DELEGATE_GETTER(getOperandBundle(ID)); } + unsigned countOperandBundlesOfType(uint32_t ID) const { + CALLSITE_DELEGATE_GETTER(countOperandBundlesOfType(ID)); + } + IterTy arg_begin() const { CALLSITE_DELEGATE_GETTER(arg_begin()); } diff --git a/include/llvm/IR/CallingConv.h b/include/llvm/IR/CallingConv.h index bc050928266e..4987b7e943f2 100644 --- a/include/llvm/IR/CallingConv.h +++ b/include/llvm/IR/CallingConv.h @@ -168,6 +168,31 @@ namespace CallingConv { /// subtargets. X86_INTR = 83, + /// Used for AVR interrupt routines. + AVR_INTR = 84, + + /// Calling convention used for AVR signal routines. + AVR_SIGNAL = 85, + + /// Calling convention used for special AVR rtlib functions + /// which have an "optimized" convention to preserve registers. + AVR_BUILTIN = 86, + + /// Calling convention used for Mesa vertex shaders. + AMDGPU_VS = 87, + + /// Calling convention used for Mesa geometry shaders. + AMDGPU_GS = 88, + + /// Calling convention used for Mesa pixel shaders. + AMDGPU_PS = 89, + + /// Calling convention used for Mesa compute shaders. + AMDGPU_CS = 90, + + /// Calling convention for AMDGPU code object kernels. + AMDGPU_KERNEL = 91, + /// The highest possible calling convention ID. Must be some 2^k - 1. MaxID = 1023 }; diff --git a/include/llvm/IR/Comdat.h b/include/llvm/IR/Comdat.h index fb79e13af3a5..577247f27e20 100644 --- a/include/llvm/IR/Comdat.h +++ b/include/llvm/IR/Comdat.h @@ -16,12 +16,10 @@ #ifndef LLVM_IR_COMDAT_H #define LLVM_IR_COMDAT_H -#include "llvm/ADT/StringRef.h" -#include "llvm/Support/Compiler.h" - namespace llvm { class raw_ostream; +class StringRef; template <typename ValueTy> class StringMapEntry; // This is a Name X SelectionKind pair. The reason for having this be an @@ -48,7 +46,6 @@ public: private: friend class Module; Comdat(); - Comdat(SelectionKind SK, StringMapEntry<Comdat> *Name); Comdat(const Comdat &) = delete; // Points to the map in Module. diff --git a/include/llvm/IR/Constant.h b/include/llvm/IR/Constant.h index bb88905aa57a..3c5fe556d50f 100644 --- a/include/llvm/IR/Constant.h +++ b/include/llvm/IR/Constant.h @@ -48,42 +48,41 @@ protected: : User(ty, vty, Ops, NumOps) {} public: - /// isNullValue - Return true if this is the value that would be returned by - /// getNullValue. + /// Return true if this is the value that would be returned by getNullValue. bool isNullValue() const; - /// \brief Returns true if the value is one. + /// Returns true if the value is one. bool isOneValue() const; - /// isAllOnesValue - Return true if this is the value that would be returned by + /// Return true if this is the value that would be returned by /// getAllOnesValue. bool isAllOnesValue() const; - /// isNegativeZeroValue - Return true if the value is what would be returned - /// by getZeroValueForNegation. + /// Return true if the value is what would be returned by + /// getZeroValueForNegation. bool isNegativeZeroValue() const; /// Return true if the value is negative zero or null value. bool isZeroValue() const; - /// \brief Return true if the value is not the smallest signed value. + /// Return true if the value is not the smallest signed value. bool isNotMinSignedValue() const; - /// \brief Return true if the value is the smallest signed value. + /// Return true if the value is the smallest signed value. bool isMinSignedValue() const; - /// canTrap - Return true if evaluation of this constant could trap. This is - /// true for things like constant expressions that could divide by zero. + /// Return true if evaluation of this constant could trap. This is true for + /// things like constant expressions that could divide by zero. bool canTrap() const; - /// isThreadDependent - Return true if the value can vary between threads. + /// Return true if the value can vary between threads. bool isThreadDependent() const; /// Return true if the value is dependent on a dllimport variable. bool isDLLImportDependent() const; - /// isConstantUsed - Return true if the constant has users other than constant - /// exprs and other dangling things. + /// Return true if the constant has users other than constant expressions and + /// other dangling things. bool isConstantUsed() const; /// This method classifies the entry according to whether or not it may @@ -93,15 +92,14 @@ public: /// FIXME: This really should not be in IR. bool needsRelocation() const; - /// getAggregateElement - For aggregates (struct/array/vector) return the - /// constant that corresponds to the specified element if possible, or null if - /// not. This can return null if the element index is a ConstantExpr, or if - /// 'this' is a constant expr. + /// For aggregates (struct/array/vector) return the constant that corresponds + /// to the specified element if possible, or null if not. This can return null + /// if the element index is a ConstantExpr, or if 'this' is a constant expr. Constant *getAggregateElement(unsigned Elt) const; Constant *getAggregateElement(Constant *Elt) const; - /// getSplatValue - If this is a splat vector constant, meaning that all of - /// the elements have the same value, return that value. Otherwise return 0. + /// If this is a splat vector constant, meaning that all of the elements have + /// the same value, return that value. Otherwise return 0. Constant *getSplatValue() const; /// If C is a constant integer then return its value, otherwise C must be a @@ -133,7 +131,7 @@ public: /// use Value::replaceAllUsesWith, which automatically dispatches to this /// method as needed. /// - void handleOperandChange(Value *, Value *, Use *); + void handleOperandChange(Value *, Value *); static Constant *getNullValue(Type* Ty); @@ -142,14 +140,14 @@ public: /// @brief Get the all ones value static Constant *getAllOnesValue(Type* Ty); - /// getIntegerValue - Return the value for an integer or pointer constant, - /// or a vector thereof, with the given scalar value. + /// Return the value for an integer or pointer constant, or a vector thereof, + /// with the given scalar value. static Constant *getIntegerValue(Type *Ty, const APInt &V); - /// removeDeadConstantUsers - If there are any dead constant users dangling - /// off of this constant, remove them. This method is useful for clients - /// that want to check to see if a global is unused, but don't want to deal - /// with potentially dead constants hanging off of the globals. + /// If there are any dead constant users dangling off of this constant, remove + /// them. This method is useful for clients that want to check to see if a + /// global is unused, but don't want to deal with potentially dead constants + /// hanging off of the globals. void removeDeadConstantUsers() const; Constant *stripPointerCasts() { diff --git a/include/llvm/IR/ConstantRange.h b/include/llvm/IR/ConstantRange.h index fb596a3bf16e..9458fa9f5c86 100644 --- a/include/llvm/IR/ConstantRange.h +++ b/include/llvm/IR/ConstantRange.h @@ -82,16 +82,42 @@ public: static ConstantRange makeSatisfyingICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other); - /// Return the largest range containing all X such that "X BinOpC C" does not - /// wrap (overflow). + /// Produce the exact range such that all values in the returned range satisfy + /// the given predicate with any value contained within Other. Formally, this + /// returns the exact answer when the superset of 'union over all y in Other + /// is exactly same as the subset of intersection over all y in Other. + /// { x : icmp op x y is true}'. /// - /// Example: + /// Example: Pred = ult and Other = i8 3 returns [0, 3) + static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, + const APInt &Other); + + /// Return the largest range containing all X such that "X BinOpC Y" is + /// guaranteed not to wrap (overflow) for all Y in Other. + /// + /// NB! The returned set does *not* contain **all** possible values of X for + /// which "X BinOpC Y" does not wrap -- some viable values of X may be + /// missing, so you cannot use this to contrain X's range. E.g. in the last + /// example, "(-2) + 1" is both nsw and nuw (so the "X" could be -2), but (-2) + /// is not in the set returned. + /// + /// Examples: /// typedef OverflowingBinaryOperator OBO; - /// makeNoWrapRegion(Add, i8 1, OBO::NoSignedWrap) == [-128, 127) - /// makeNoWrapRegion(Add, i8 1, OBO::NoUnsignedWrap) == [0, -1) - /// makeNoWrapRegion(Add, i8 0, OBO::NoUnsignedWrap) == Full Set - static ConstantRange makeNoWrapRegion(Instruction::BinaryOps BinOp, - const APInt &C, unsigned NoWrapKind); + /// #define MGNR makeGuaranteedNoWrapRegion + /// MGNR(Add, [i8 1, 2), OBO::NoSignedWrap) == [-128, 127) + /// MGNR(Add, [i8 1, 2), OBO::NoUnsignedWrap) == [0, -1) + /// MGNR(Add, [i8 0, 1), OBO::NoUnsignedWrap) == Full Set + /// MGNR(Add, [i8 1, 2), OBO::NoUnsignedWrap | OBO::NoSignedWrap) + /// == [0,INT_MAX) + /// MGNR(Add, [i8 -1, 6), OBO::NoSignedWrap) == [INT_MIN+1, INT_MAX-4) + static ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, + const ConstantRange &Other, + unsigned NoWrapKind); + + /// Set up \p Pred and \p RHS such that + /// ConstantRange::makeExactICmpRegion(Pred, RHS) == *this. Return true if + /// successful. + bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const; /// Return the lower value for this range. /// @@ -245,6 +271,14 @@ public: ConstantRange umax(const ConstantRange &Other) const; /// Return a new range representing the possible values resulting + /// from a signed minimum of a value in this range and a value in \p Other. + ConstantRange smin(const ConstantRange &Other) const; + + /// Return a new range representing the possible values resulting + /// from an unsigned minimum of a value in this range and a value in \p Other. + ConstantRange umin(const ConstantRange &Other) const; + + /// Return a new range representing the possible values resulting /// from an unsigned division of a value in this range and a value in /// \p Other. ConstantRange udiv(const ConstantRange &Other) const; diff --git a/include/llvm/IR/Constants.h b/include/llvm/IR/Constants.h index a5a20c9c5701..2a5d14d94646 100644 --- a/include/llvm/IR/Constants.h +++ b/include/llvm/IR/Constants.h @@ -40,26 +40,47 @@ class SequentialType; struct ConstantExprKeyType; template <class ConstantClass> struct ConstantAggrKeyType; +/// Base class for constants with no operands. +/// +/// These constants have no operands; they represent their data directly. +/// Since they can be in use by unrelated modules (and are never based on +/// GlobalValues), it never makes sense to RAUW them. +class ConstantData : public Constant { + void anchor() override; + void *operator new(size_t, unsigned) = delete; + ConstantData() = delete; + ConstantData(const ConstantData &) = delete; + + friend class Constant; + Value *handleOperandChangeImpl(Value *From, Value *To) { + llvm_unreachable("Constant data does not have operands!"); + } + +protected: + explicit ConstantData(Type *Ty, ValueTy VT) : Constant(Ty, VT, nullptr, 0) {} + void *operator new(size_t s) { return User::operator new(s, 0); } + +public: + /// Methods to support type inquiry through isa, cast, and dyn_cast. + static bool classof(const Value *V) { + return V->getValueID() >= ConstantDataFirstVal && + V->getValueID() <= ConstantDataLastVal; + } +}; + //===----------------------------------------------------------------------===// /// This is the shared class of boolean and integer constants. This class /// represents both boolean and integral constants. /// @brief Class for constant integers. -class ConstantInt : public Constant { +class ConstantInt final : public ConstantData { void anchor() override; - void *operator new(size_t, unsigned) = delete; ConstantInt(const ConstantInt &) = delete; ConstantInt(IntegerType *Ty, const APInt& V); APInt Val; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); -protected: - // allocate space for exactly zero operands - void *operator new(size_t s) { - return User::operator new(s, 0); - } public: static ConstantInt *getTrue(LLVMContext &Context); static ConstantInt *getFalse(LLVMContext &Context); @@ -230,34 +251,26 @@ public: //===----------------------------------------------------------------------===// /// ConstantFP - Floating Point Values [float, double] /// -class ConstantFP : public Constant { +class ConstantFP final : public ConstantData { APFloat Val; void anchor() override; - void *operator new(size_t, unsigned) = delete; ConstantFP(const ConstantFP &) = delete; - friend class LLVMContextImpl; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); -protected: ConstantFP(Type *Ty, const APFloat& V); -protected: - // allocate space for exactly zero operands - void *operator new(size_t s) { - return User::operator new(s, 0); - } + public: /// Floating point negation must be implemented with f(x) = -0.0 - x. This /// method returns the negative zero constant for floating point or vector /// floating point types; for all other types, it returns the null value. static Constant *getZeroValueForNegation(Type *Ty); - /// get() - This returns a ConstantFP, or a vector containing a splat of a - /// ConstantFP, for the specified value in the specified type. This should - /// only be used for simple constant values like 2.0/1.0 etc, that are - /// known-valid both as host double and as the target format. + /// This returns a ConstantFP, or a vector containing a splat of a ConstantFP, + /// for the specified value in the specified type. This should only be used + /// for simple constant values like 2.0/1.0 etc, that are known-valid both as + /// host double and as the target format. static Constant *get(Type* Ty, double V); static Constant *get(Type* Ty, StringRef Str); static ConstantFP *get(LLVMContext &Context, const APFloat &V); @@ -265,24 +278,24 @@ public: static Constant *getNegativeZero(Type *Ty); static Constant *getInfinity(Type *Ty, bool Negative = false); - /// isValueValidForType - return true if Ty is big enough to represent V. + /// Return true if Ty is big enough to represent V. static bool isValueValidForType(Type *Ty, const APFloat &V); inline const APFloat &getValueAPF() const { return Val; } - /// isZero - Return true if the value is positive or negative zero. + /// Return true if the value is positive or negative zero. bool isZero() const { return Val.isZero(); } - /// isNegative - Return true if the sign bit is set. + /// Return true if the sign bit is set. bool isNegative() const { return Val.isNegative(); } - /// isInfinity - Return true if the value is infinity + /// Return true if the value is infinity bool isInfinity() const { return Val.isInfinity(); } - /// isNaN - Return true if the value is a NaN. + /// Return true if the value is a NaN. bool isNaN() const { return Val.isNaN(); } - /// isExactlyValue - We don't rely on operator== working on double values, as - /// it returns true for things that are clearly not equal, like -0.0 and 0.0. + /// We don't rely on operator== working on double values, as it returns true + /// for things that are clearly not equal, like -0.0 and 0.0. /// As such, this method can be used to do an exact bit-for-bit comparison of /// two floating point values. The version with a double operand is retained /// because it's so convenient to write isExactlyValue(2.0), but please use @@ -302,44 +315,36 @@ public: }; //===----------------------------------------------------------------------===// -/// ConstantAggregateZero - All zero aggregate value +/// All zero aggregate value /// -class ConstantAggregateZero : public Constant { - void *operator new(size_t, unsigned) = delete; +class ConstantAggregateZero final : public ConstantData { ConstantAggregateZero(const ConstantAggregateZero &) = delete; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); -protected: - explicit ConstantAggregateZero(Type *ty) - : Constant(ty, ConstantAggregateZeroVal, nullptr, 0) {} -protected: - // allocate space for exactly zero operands - void *operator new(size_t s) { - return User::operator new(s, 0); - } + explicit ConstantAggregateZero(Type *Ty) + : ConstantData(Ty, ConstantAggregateZeroVal) {} + public: static ConstantAggregateZero *get(Type *Ty); - /// getSequentialElement - If this CAZ has array or vector type, return a zero - /// with the right element type. + /// If this CAZ has array or vector type, return a zero with the right element + /// type. Constant *getSequentialElement() const; - /// getStructElement - If this CAZ has struct type, return a zero with the - /// right element type for the specified element. + /// If this CAZ has struct type, return a zero with the right element type for + /// the specified element. Constant *getStructElement(unsigned Elt) const; - /// getElementValue - Return a zero of the right value for the specified GEP - /// index. + /// Return a zero of the right value for the specified GEP index if we can, + /// otherwise return null (e.g. if C is a ConstantExpr). Constant *getElementValue(Constant *C) const; - /// getElementValue - Return a zero of the right value for the specified GEP - /// index. + /// Return a zero of the right value for the specified GEP index. Constant *getElementValue(unsigned Idx) const; - /// \brief Return the number of elements in the array, vector, or struct. + /// Return the number of elements in the array, vector, or struct. unsigned getNumElements() const; /// Methods for support type inquiry through isa, cast, and dyn_cast: @@ -349,20 +354,49 @@ public: } }; +/// Base class for aggregate constants (with operands). +/// +/// These constants are aggregates of other constants, which are stored as +/// operands. +/// +/// Subclasses are \a ConstantStruct, \a ConstantArray, and \a +/// ConstantVector. +/// +/// \note Some subclasses of \a ConstantData are semantically aggregates -- +/// such as \a ConstantDataArray -- but are not subclasses of this because they +/// use operands. +class ConstantAggregate : public Constant { +protected: + ConstantAggregate(CompositeType *T, ValueTy VT, ArrayRef<Constant *> V); + +public: + /// Transparently provide more efficient getOperand methods. + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); + + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Value *V) { + return V->getValueID() >= ConstantAggregateFirstVal && + V->getValueID() <= ConstantAggregateLastVal; + } +}; + +template <> +struct OperandTraits<ConstantAggregate> + : public VariadicOperandTraits<ConstantAggregate> {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantAggregate, Constant) //===----------------------------------------------------------------------===// /// ConstantArray - Constant Array Declarations /// -class ConstantArray : public Constant { +class ConstantArray final : public ConstantAggregate { friend struct ConstantAggrKeyType<ConstantArray>; - ConstantArray(const ConstantArray &) = delete; - friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); + Value *handleOperandChangeImpl(Value *From, Value *To); -protected: ConstantArray(ArrayType *T, ArrayRef<Constant *> Val); + public: // ConstantArray accessors static Constant *get(ArrayType *T, ArrayRef<Constant*> V); @@ -371,12 +405,8 @@ private: static Constant *getImpl(ArrayType *T, ArrayRef<Constant *> V); public: - /// Transparently provide more efficient getOperand methods. - DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); - - /// getType - Specialize the getType() method to always return an ArrayType, + /// Specialize the getType() method to always return an ArrayType, /// which reduces the amount of casting needed in parts of the compiler. - /// inline ArrayType *getType() const { return cast<ArrayType>(Value::getType()); } @@ -387,34 +417,24 @@ public: } }; -template <> -struct OperandTraits<ConstantArray> : - public VariadicOperandTraits<ConstantArray> { -}; - -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantArray, Constant) - //===----------------------------------------------------------------------===// -// ConstantStruct - Constant Struct Declarations +// Constant Struct Declarations // -class ConstantStruct : public Constant { +class ConstantStruct final : public ConstantAggregate { friend struct ConstantAggrKeyType<ConstantStruct>; - ConstantStruct(const ConstantStruct &) = delete; - friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); + Value *handleOperandChangeImpl(Value *From, Value *To); -protected: ConstantStruct(StructType *T, ArrayRef<Constant *> Val); + public: // ConstantStruct accessors static Constant *get(StructType *T, ArrayRef<Constant*> V); static Constant *get(StructType *T, ...) LLVM_END_WITH_NULL; - /// getAnon - Return an anonymous struct that has the specified - /// elements. If the struct is possibly empty, then you must specify a - /// context. + /// Return an anonymous struct that has the specified elements. + /// If the struct is possibly empty, then you must specify a context. static Constant *getAnon(ArrayRef<Constant*> V, bool Packed = false) { return get(getTypeForElements(V, Packed), V); } @@ -423,20 +443,16 @@ public: return get(getTypeForElements(Ctx, V, Packed), V); } - /// getTypeForElements - Return an anonymous struct type to use for a constant - /// with the specified set of elements. The list must not be empty. + /// Return an anonymous struct type to use for a constant with the specified + /// set of elements. The list must not be empty. static StructType *getTypeForElements(ArrayRef<Constant*> V, bool Packed = false); - /// getTypeForElements - This version of the method allows an empty list. + /// This version of the method allows an empty list. static StructType *getTypeForElements(LLVMContext &Ctx, ArrayRef<Constant*> V, bool Packed = false); - /// Transparently provide more efficient getOperand methods. - DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); - - /// getType() specialization - Reduce amount of casting... - /// + /// Specialization - reduce amount of casting. inline StructType *getType() const { return cast<StructType>(Value::getType()); } @@ -447,27 +463,18 @@ public: } }; -template <> -struct OperandTraits<ConstantStruct> : - public VariadicOperandTraits<ConstantStruct> { -}; - -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantStruct, Constant) - //===----------------------------------------------------------------------===// -/// ConstantVector - Constant Vector Declarations +/// Constant Vector Declarations /// -class ConstantVector : public Constant { +class ConstantVector final : public ConstantAggregate { friend struct ConstantAggrKeyType<ConstantVector>; - ConstantVector(const ConstantVector &) = delete; - friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); + Value *handleOperandChangeImpl(Value *From, Value *To); -protected: ConstantVector(VectorType *T, ArrayRef<Constant *> Val); + public: // ConstantVector accessors static Constant *get(ArrayRef<Constant*> V); @@ -476,22 +483,17 @@ private: static Constant *getImpl(ArrayRef<Constant *> V); public: - /// getSplat - Return a ConstantVector with the specified constant in each - /// element. + /// Return a ConstantVector with the specified constant in each element. static Constant *getSplat(unsigned NumElts, Constant *Elt); - /// Transparently provide more efficient getOperand methods. - DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); - - /// getType - Specialize the getType() method to always return a VectorType, + /// Specialize the getType() method to always return a VectorType, /// which reduces the amount of casting needed in parts of the compiler. - /// inline VectorType *getType() const { return cast<VectorType>(Value::getType()); } - /// getSplatValue - If this is a splat constant, meaning that all of the - /// elements have the same value, return that value. Otherwise return NULL. + /// If this is a splat constant, meaning that all of the elements have the + /// same value, return that value. Otherwise return NULL. Constant *getSplatValue() const; /// Methods for support type inquiry through isa, cast, and dyn_cast: @@ -500,41 +502,24 @@ public: } }; -template <> -struct OperandTraits<ConstantVector> : - public VariadicOperandTraits<ConstantVector> { -}; - -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantVector, Constant) - //===----------------------------------------------------------------------===// -/// ConstantPointerNull - a constant pointer value that points to null +/// A constant pointer value that points to null /// -class ConstantPointerNull : public Constant { - void *operator new(size_t, unsigned) = delete; +class ConstantPointerNull final : public ConstantData { ConstantPointerNull(const ConstantPointerNull &) = delete; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); -protected: explicit ConstantPointerNull(PointerType *T) - : Constant(T, - Value::ConstantPointerNullVal, nullptr, 0) {} + : ConstantData(T, Value::ConstantPointerNullVal) {} -protected: - // allocate space for exactly zero operands - void *operator new(size_t s) { - return User::operator new(s, 0); - } public: - /// get() - Static factory methods - Return objects of the specified value + /// Static factory methods - Return objects of the specified value static ConstantPointerNull *get(PointerType *T); - /// getType - Specialize the getType() method to always return an PointerType, + /// Specialize the getType() method to always return an PointerType, /// which reduces the amount of casting needed in parts of the compiler. - /// inline PointerType *getType() const { return cast<PointerType>(Value::getType()); } @@ -554,116 +539,101 @@ public: /// /// This is the common base class of ConstantDataArray and ConstantDataVector. /// -class ConstantDataSequential : public Constant { +class ConstantDataSequential : public ConstantData { friend class LLVMContextImpl; - /// DataElements - A pointer to the bytes underlying this constant (which is - /// owned by the uniquing StringMap). + /// A pointer to the bytes underlying this constant (which is owned by the + /// uniquing StringMap). const char *DataElements; - /// Next - This forms a link list of ConstantDataSequential nodes that have + /// This forms a link list of ConstantDataSequential nodes that have /// the same value but different type. For example, 0,0,0,1 could be a 4 /// element array of i8, or a 1-element array of i32. They'll both end up in /// the same StringMap bucket, linked up. ConstantDataSequential *Next; - void *operator new(size_t, unsigned) = delete; ConstantDataSequential(const ConstantDataSequential &) = delete; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); protected: explicit ConstantDataSequential(Type *ty, ValueTy VT, const char *Data) - : Constant(ty, VT, nullptr, 0), DataElements(Data), Next(nullptr) {} + : ConstantData(ty, VT), DataElements(Data), Next(nullptr) {} ~ConstantDataSequential() override { delete Next; } static Constant *getImpl(StringRef Bytes, Type *Ty); -protected: - // allocate space for exactly zero operands. - void *operator new(size_t s) { - return User::operator new(s, 0); - } public: - - /// isElementTypeCompatible - Return true if a ConstantDataSequential can be - /// formed with a vector or array of the specified element type. + /// Return true if a ConstantDataSequential can be formed with a vector or + /// array of the specified element type. /// ConstantDataArray only works with normal float and int types that are /// stored densely in memory, not with things like i42 or x86_f80. static bool isElementTypeCompatible(Type *Ty); - /// getElementAsInteger - If this is a sequential container of integers (of - /// any size), return the specified element in the low bits of a uint64_t. + /// If this is a sequential container of integers (of any size), return the + /// specified element in the low bits of a uint64_t. uint64_t getElementAsInteger(unsigned i) const; - /// getElementAsAPFloat - If this is a sequential container of floating point - /// type, return the specified element as an APFloat. + /// If this is a sequential container of floating point type, return the + /// specified element as an APFloat. APFloat getElementAsAPFloat(unsigned i) const; - /// getElementAsFloat - If this is an sequential container of floats, return - /// the specified element as a float. + /// If this is an sequential container of floats, return the specified element + /// as a float. float getElementAsFloat(unsigned i) const; - /// getElementAsDouble - If this is an sequential container of doubles, return - /// the specified element as a double. + /// If this is an sequential container of doubles, return the specified + /// element as a double. double getElementAsDouble(unsigned i) const; - /// getElementAsConstant - Return a Constant for a specified index's element. + /// Return a Constant for a specified index's element. /// Note that this has to compute a new constant to return, so it isn't as /// efficient as getElementAsInteger/Float/Double. Constant *getElementAsConstant(unsigned i) const; - /// getType - Specialize the getType() method to always return a - /// SequentialType, which reduces the amount of casting needed in parts of the - /// compiler. + /// Specialize the getType() method to always return a SequentialType, which + /// reduces the amount of casting needed in parts of the compiler. inline SequentialType *getType() const { return cast<SequentialType>(Value::getType()); } - /// getElementType - Return the element type of the array/vector. + /// Return the element type of the array/vector. Type *getElementType() const; - /// getNumElements - Return the number of elements in the array or vector. + /// Return the number of elements in the array or vector. unsigned getNumElements() const; - /// getElementByteSize - Return the size (in bytes) of each element in the - /// array/vector. The size of the elements is known to be a multiple of one - /// byte. + /// Return the size (in bytes) of each element in the array/vector. + /// The size of the elements is known to be a multiple of one byte. uint64_t getElementByteSize() const; - - /// isString - This method returns true if this is an array of i8. + /// This method returns true if this is an array of i8. bool isString() const; - /// isCString - This method returns true if the array "isString", ends with a - /// nul byte, and does not contains any other nul bytes. + /// This method returns true if the array "isString", ends with a null byte, + /// and does not contains any other null bytes. bool isCString() const; - /// getAsString - If this array is isString(), then this method returns the - /// array as a StringRef. Otherwise, it asserts out. - /// + /// If this array is isString(), then this method returns the array as a + /// StringRef. Otherwise, it asserts out. StringRef getAsString() const { assert(isString() && "Not a string"); return getRawDataValues(); } - /// getAsCString - If this array is isCString(), then this method returns the - /// array (without the trailing null byte) as a StringRef. Otherwise, it - /// asserts out. - /// + /// If this array is isCString(), then this method returns the array (without + /// the trailing null byte) as a StringRef. Otherwise, it asserts out. StringRef getAsCString() const { assert(isCString() && "Isn't a C string"); StringRef Str = getAsString(); return Str.substr(0, Str.size()-1); } - /// getRawDataValues - Return the raw, underlying, bytes of this data. Note - /// that this is an extremely tricky thing to work with, as it exposes the - /// host endianness of the data elements. + /// Return the raw, underlying, bytes of this data. Note that this is an + /// extremely tricky thing to work with, as it exposes the host endianness of + /// the data elements. StringRef getRawDataValues() const; /// Methods for support type inquiry through isa, cast, and dyn_cast: - /// static bool classof(const Value *V) { return V->getValueID() == ConstantDataArrayVal || V->getValueID() == ConstantDataVectorVal; @@ -673,25 +643,24 @@ private: }; //===----------------------------------------------------------------------===// -/// ConstantDataArray - An array constant whose element type is a simple -/// 1/2/4/8-byte integer or float/double, and whose elements are just simple -/// data values (i.e. ConstantInt/ConstantFP). This Constant node has no -/// operands because it stores all of the elements of the constant as densely -/// packed data, instead of as Value*'s. -class ConstantDataArray : public ConstantDataSequential { +/// An array constant whose element type is a simple 1/2/4/8-byte integer or +/// float/double, and whose elements are just simple data values +/// (i.e. ConstantInt/ConstantFP). This Constant node has no operands because it +/// stores all of the elements of the constant as densely packed data, instead +/// of as Value*'s. +class ConstantDataArray final : public ConstantDataSequential { void *operator new(size_t, unsigned) = delete; ConstantDataArray(const ConstantDataArray &) = delete; void anchor() override; friend class ConstantDataSequential; explicit ConstantDataArray(Type *ty, const char *Data) - : ConstantDataSequential(ty, ConstantDataArrayVal, Data) {} -protected: - // allocate space for exactly zero operands. + : ConstantDataSequential(ty, ConstantDataArrayVal, Data) {} + /// Allocate space for exactly zero operands. void *operator new(size_t s) { return User::operator new(s, 0); } -public: +public: /// get() constructors - Return a constant with array type with an element /// count and element type matching the ArrayRef passed in. Note that this /// can return a ConstantAggregateZero object. @@ -711,48 +680,45 @@ public: static Constant *getFP(LLVMContext &Context, ArrayRef<uint32_t> Elts); static Constant *getFP(LLVMContext &Context, ArrayRef<uint64_t> Elts); - /// getString - This method constructs a CDS and initializes it with a text - /// string. The default behavior (AddNull==true) causes a null terminator to + /// This method constructs a CDS and initializes it with a text string. + /// The default behavior (AddNull==true) causes a null terminator to /// be placed at the end of the array (increasing the length of the string by /// one more than the StringRef would normally indicate. Pass AddNull=false /// to disable this behavior. static Constant *getString(LLVMContext &Context, StringRef Initializer, bool AddNull = true); - /// getType - Specialize the getType() method to always return an ArrayType, + /// Specialize the getType() method to always return an ArrayType, /// which reduces the amount of casting needed in parts of the compiler. - /// inline ArrayType *getType() const { return cast<ArrayType>(Value::getType()); } /// Methods for support type inquiry through isa, cast, and dyn_cast: - /// static bool classof(const Value *V) { return V->getValueID() == ConstantDataArrayVal; } }; //===----------------------------------------------------------------------===// -/// ConstantDataVector - A vector constant whose element type is a simple -/// 1/2/4/8-byte integer or float/double, and whose elements are just simple -/// data values (i.e. ConstantInt/ConstantFP). This Constant node has no -/// operands because it stores all of the elements of the constant as densely -/// packed data, instead of as Value*'s. -class ConstantDataVector : public ConstantDataSequential { +/// A vector constant whose element type is a simple 1/2/4/8-byte integer or +/// float/double, and whose elements are just simple data values +/// (i.e. ConstantInt/ConstantFP). This Constant node has no operands because it +/// stores all of the elements of the constant as densely packed data, instead +/// of as Value*'s. +class ConstantDataVector final : public ConstantDataSequential { void *operator new(size_t, unsigned) = delete; ConstantDataVector(const ConstantDataVector &) = delete; void anchor() override; friend class ConstantDataSequential; explicit ConstantDataVector(Type *ty, const char *Data) - : ConstantDataSequential(ty, ConstantDataVectorVal, Data) {} -protected: + : ConstantDataSequential(ty, ConstantDataVectorVal, Data) {} // allocate space for exactly zero operands. void *operator new(size_t s) { return User::operator new(s, 0); } -public: +public: /// get() constructors - Return a constant with vector type with an element /// count and element type matching the ArrayRef passed in. Note that this /// can return a ConstantAggregateZero object. @@ -772,45 +738,38 @@ public: static Constant *getFP(LLVMContext &Context, ArrayRef<uint32_t> Elts); static Constant *getFP(LLVMContext &Context, ArrayRef<uint64_t> Elts); - /// getSplat - Return a ConstantVector with the specified constant in each - /// element. The specified constant has to be a of a compatible type (i8/i16/ + /// Return a ConstantVector with the specified constant in each element. + /// The specified constant has to be a of a compatible type (i8/i16/ /// i32/i64/float/double) and must be a ConstantFP or ConstantInt. static Constant *getSplat(unsigned NumElts, Constant *Elt); - /// getSplatValue - If this is a splat constant, meaning that all of the - /// elements have the same value, return that value. Otherwise return NULL. + /// If this is a splat constant, meaning that all of the elements have the + /// same value, return that value. Otherwise return NULL. Constant *getSplatValue() const; - /// getType - Specialize the getType() method to always return a VectorType, + /// Specialize the getType() method to always return a VectorType, /// which reduces the amount of casting needed in parts of the compiler. - /// inline VectorType *getType() const { return cast<VectorType>(Value::getType()); } /// Methods for support type inquiry through isa, cast, and dyn_cast: - /// static bool classof(const Value *V) { return V->getValueID() == ConstantDataVectorVal; } }; //===----------------------------------------------------------------------===// -/// ConstantTokenNone - a constant token which is empty +/// A constant token which is empty /// -class ConstantTokenNone : public Constant { - void *operator new(size_t, unsigned) = delete; +class ConstantTokenNone final : public ConstantData { ConstantTokenNone(const ConstantTokenNone &) = delete; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); -protected: explicit ConstantTokenNone(LLVMContext &Context) - : Constant(Type::getTokenTy(Context), ConstantTokenNoneVal, nullptr, 0) {} - // allocate space for exactly zero operands - void *operator new(size_t s) { return User::operator new(s, 0); } + : ConstantData(Type::getTokenTy(Context), ConstantTokenNoneVal) {} public: /// Return the ConstantTokenNone. @@ -822,27 +781,26 @@ public: } }; -/// BlockAddress - The address of a basic block. +/// The address of a basic block. /// -class BlockAddress : public Constant { +class BlockAddress final : public Constant { void *operator new(size_t, unsigned) = delete; void *operator new(size_t s) { return User::operator new(s, 2); } BlockAddress(Function *F, BasicBlock *BB); friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); + Value *handleOperandChangeImpl(Value *From, Value *To); public: - /// get - Return a BlockAddress for the specified function and basic block. + /// Return a BlockAddress for the specified function and basic block. static BlockAddress *get(Function *F, BasicBlock *BB); - /// get - Return a BlockAddress for the specified basic block. The basic + /// Return a BlockAddress for the specified basic block. The basic /// block must be embedded into a function. static BlockAddress *get(BasicBlock *BB); - /// \brief Lookup an existing \c BlockAddress constant for the given - /// BasicBlock. + /// Lookup an existing \c BlockAddress constant for the given BasicBlock. /// /// \returns 0 if \c !BB->hasAddressTaken(), otherwise the \c BlockAddress. static BlockAddress *lookup(const BasicBlock *BB); @@ -868,7 +826,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BlockAddress, Value) //===----------------------------------------------------------------------===// -/// ConstantExpr - a constant value that is initialized with an expression using +/// A constant value that is initialized with an expression using /// other constant values. /// /// This class uses the standard Instruction opcodes to define the various @@ -879,11 +837,11 @@ class ConstantExpr : public Constant { friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); + Value *handleOperandChangeImpl(Value *From, Value *To); protected: ConstantExpr(Type *ty, unsigned Opcode, Use *Ops, unsigned NumOps) - : Constant(ty, ConstantExprVal, Ops, NumOps) { + : Constant(ty, ConstantExprVal, Ops, NumOps) { // Operation type (an Instruction opcode) is stored as the SubclassData. setValueSubclassData(Opcode); } @@ -998,12 +956,12 @@ public: return getLShr(C1, C2, true); } - /// getBinOpIdentity - Return the identity for the given binary operation, + /// Return the identity for the given binary operation, /// i.e. a constant C such that X op C = X and C op X = X for every X. It /// returns null if the operator doesn't have an identity. static Constant *getBinOpIdentity(unsigned Opcode, Type *Ty); - /// getBinOpAbsorber - Return the absorbing element for the given binary + /// Return the absorbing element for the given binary /// operation, i.e. a constant C such that X op C = C and C op X = C for /// every X. For example, this returns zero for integer multiplication. /// It returns null if the operator doesn't have an absorbing element. @@ -1165,32 +1123,32 @@ public: ArrayRef<unsigned> Idxs, Type *OnlyIfReducedTy = nullptr); - /// getOpcode - Return the opcode at the root of this constant expression + /// Return the opcode at the root of this constant expression unsigned getOpcode() const { return getSubclassDataFromValue(); } - /// getPredicate - Return the ICMP or FCMP predicate value. Assert if this is - /// not an ICMP or FCMP constant expression. + /// Return the ICMP or FCMP predicate value. Assert if this is not an ICMP or + /// FCMP constant expression. unsigned getPredicate() const; - /// getIndices - Assert that this is an insertvalue or exactvalue + /// Assert that this is an insertvalue or exactvalue /// expression and return the list of indices. ArrayRef<unsigned> getIndices() const; - /// getOpcodeName - Return a string representation for an opcode. + /// Return a string representation for an opcode. const char *getOpcodeName() const; - /// getWithOperandReplaced - Return a constant expression identical to this - /// one, but with the specified operand set to the specified value. + /// Return a constant expression identical to this one, but with the specified + /// operand set to the specified value. Constant *getWithOperandReplaced(unsigned OpNo, Constant *Op) const; - /// getWithOperands - This returns the current constant expression with the - /// operands replaced with the specified values. The specified array must - /// have the same number of operands as our current one. + /// This returns the current constant expression with the operands replaced + /// with the specified values. The specified array must have the same number + /// of operands as our current one. Constant *getWithOperands(ArrayRef<Constant*> Ops) const { return getWithOperands(Ops, getType()); } - /// \brief Get the current expression with the operands replaced. + /// Get the current expression with the operands replaced. /// /// Return the current constant expression with the operands replaced with \c /// Ops and the type with \c Ty. The new operands must have the same number @@ -1203,9 +1161,8 @@ public: bool OnlyIfReduced = false, Type *SrcTy = nullptr) const; - /// getAsInstruction - Returns an Instruction which implements the same - /// operation as this ConstantExpr. The instruction is not linked to any basic - /// block. + /// Returns an Instruction which implements the same operation as this + /// ConstantExpr. The instruction is not linked to any basic block. /// /// A better approach to this could be to have a constructor for Instruction /// which would take a ConstantExpr parameter, but that would have spread @@ -1234,7 +1191,7 @@ struct OperandTraits<ConstantExpr> : DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantExpr, Constant) //===----------------------------------------------------------------------===// -/// UndefValue - 'undef' values are things that do not have specified contents. +/// 'undef' values are things that do not have specified contents. /// These are used for a variety of purposes, including global variable /// initializers and operands to instructions. 'undef' values can occur with /// any first-class type. @@ -1243,44 +1200,34 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantExpr, Constant) /// can appear to have different bit patterns at each use. See /// LangRef.html#undefvalues for details. /// -class UndefValue : public Constant { - void *operator new(size_t, unsigned) = delete; +class UndefValue final : public ConstantData { UndefValue(const UndefValue &) = delete; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); -protected: - explicit UndefValue(Type *T) : Constant(T, UndefValueVal, nullptr, 0) {} -protected: - // allocate space for exactly zero operands - void *operator new(size_t s) { - return User::operator new(s, 0); - } + explicit UndefValue(Type *T) : ConstantData(T, UndefValueVal) {} + public: - /// get() - Static factory methods - Return an 'undef' object of the specified - /// type. - /// + /// Static factory methods - Return an 'undef' object of the specified type. static UndefValue *get(Type *T); - /// getSequentialElement - If this Undef has array or vector type, return a - /// undef with the right element type. + /// If this Undef has array or vector type, return a undef with the right + /// element type. UndefValue *getSequentialElement() const; - /// getStructElement - If this undef has struct type, return a undef with the - /// right element type for the specified element. + /// If this undef has struct type, return a undef with the right element type + /// for the specified element. UndefValue *getStructElement(unsigned Elt) const; - /// getElementValue - Return an undef of the right value for the specified GEP - /// index. + /// Return an undef of the right value for the specified GEP index if we can, + /// otherwise return null (e.g. if C is a ConstantExpr). UndefValue *getElementValue(Constant *C) const; - /// getElementValue - Return an undef of the right value for the specified GEP - /// index. + /// Return an undef of the right value for the specified GEP index. UndefValue *getElementValue(unsigned Idx) const; - /// \brief Return the number of elements in the array, vector, or struct. + /// Return the number of elements in the array, vector, or struct. unsigned getNumElements() const; /// Methods for support type inquiry through isa, cast, and dyn_cast: diff --git a/include/llvm/IR/DIBuilder.h b/include/llvm/IR/DIBuilder.h index aeec39541154..0f2f67f5feaf 100644 --- a/include/llvm/IR/DIBuilder.h +++ b/include/llvm/IR/DIBuilder.h @@ -15,8 +15,6 @@ #ifndef LLVM_IR_DIBUILDER_H #define LLVM_IR_DIBUILDER_H -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringRef.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/TrackingMDRef.h" #include "llvm/IR/ValueHandle.h" @@ -31,6 +29,7 @@ namespace llvm { class Constant; class LLVMContext; class StringRef; + template <typename T> class ArrayRef; class DIBuilder { Module &M; @@ -52,7 +51,11 @@ namespace llvm { bool AllowUnresolvedNodes; /// Each subprogram's preserved local variables. - DenseMap<MDNode *, std::vector<TrackingMDNodeRef>> PreservedVariables; + /// + /// Do not use a std::vector. Some versions of libc++ apparently copy + /// instead of move on grow operations, and TrackingMDRef is expensive to + /// copy. + DenseMap<MDNode *, SmallVector<TrackingMDNodeRef, 1>> PreservedVariables; DIBuilder(const DIBuilder &) = delete; void operator=(const DIBuilder &) = delete; @@ -68,7 +71,6 @@ namespace llvm { /// If \c AllowUnresolved, collect unresolved nodes attached to the module /// in order to resolve cycles during \a finalize(). explicit DIBuilder(Module &M, bool AllowUnresolved = true); - enum DebugEmissionKind { FullDebug=1, LineTablesOnly }; /// Construct any deferred debug info descriptors. void finalize(); @@ -93,22 +95,13 @@ namespace llvm { /// out into. /// \param Kind The kind of debug information to generate. /// \param DWOId The DWOId if this is a split skeleton compile unit. - /// \param EmitDebugInfo A boolean flag which indicates whether - /// debug information should be written to - /// the final output or not. When this is - /// false, debug information annotations will - /// be present in the IL but they are not - /// written to the final assembly or object - /// file. This supports tracking source - /// location information in the back end - /// without actually changing the output - /// (e.g., when using optimization remarks). DICompileUnit * createCompileUnit(unsigned Lang, StringRef File, StringRef Dir, StringRef Producer, bool isOptimized, StringRef Flags, unsigned RV, StringRef SplitName = StringRef(), - DebugEmissionKind Kind = FullDebug, uint64_t DWOId = 0, - bool EmitDebugInfo = true); + DICompileUnit::DebugEmissionKind Kind = + DICompileUnit::DebugEmissionKind::FullDebug, + uint64_t DWOId = 0); /// Create a file descriptor to hold debugging information /// for a file. @@ -154,7 +147,8 @@ namespace llvm { /// \param Class Type for which this pointer points to members of. DIDerivedType *createMemberPointerType(DIType *PointeeTy, DIType *Class, uint64_t SizeInBits, - uint64_t AlignInBits = 0); + uint64_t AlignInBits = 0, + unsigned Flags = 0); /// Create debugging information entry for a c++ /// style reference or rvalue reference type. @@ -200,6 +194,22 @@ namespace llvm { uint64_t OffsetInBits, unsigned Flags, DIType *Ty); + /// Create debugging information entry for a bit field member. + /// \param Scope Member scope. + /// \param Name Member name. + /// \param File File where this member is defined. + /// \param LineNo Line number. + /// \param SizeInBits Member size. + /// \param AlignInBits Member alignment. + /// \param OffsetInBits Member offset. + /// \param StorageOffsetInBits Member storage offset. + /// \param Flags Flags to encode member attribute. + /// \param Ty Parent type. + DIDerivedType *createBitFieldMemberType( + DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, + uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, + uint64_t StorageOffsetInBits, unsigned Flags, DIType *Ty); + /// Create debugging information entry for a /// C++ static data member. /// \param Scope Member scope. @@ -381,8 +391,9 @@ namespace llvm { /// includes return type at 0th index. /// \param Flags E.g.: LValueReference. /// These flags are used to emit dwarf attributes. + /// \param CC Calling convention, e.g. dwarf::DW_CC_normal DISubroutineType *createSubroutineType(DITypeRefArray ParameterTypes, - unsigned Flags = 0); + unsigned Flags = 0, unsigned CC = 0); /// Create an external type reference. /// \param Tag Dwarf TAG. @@ -413,9 +424,9 @@ namespace llvm { uint64_t AlignInBits = 0, unsigned Flags = DINode::FlagFwdDecl, StringRef UniqueIdentifier = ""); - /// Retain DIType* in a module even if it is not referenced + /// Retain DIScope* in a module even if it is not referenced /// through debug info anchors. - void retainType(DIType *T); + void retainType(DIScope *T); /// Create unspecified parameter type /// for a subroutine type. @@ -535,17 +546,6 @@ namespace llvm { bool isOptimized = false, DITemplateParameterArray TParams = nullptr, DISubprogram *Decl = nullptr); - /// FIXME: this is added for dragonegg. Once we update dragonegg - /// to call resolve function, this will be removed. - DISubprogram *createFunction(DIScopeRef Scope, StringRef Name, - StringRef LinkageName, DIFile *File, - unsigned LineNo, DISubroutineType *Ty, - bool isLocalToUnit, bool isDefinition, - unsigned ScopeLine, unsigned Flags = 0, - bool isOptimized = false, - DITemplateParameterArray TParams = nullptr, - DISubprogram *Decl = nullptr); - /// Create a new descriptor for the specified C++ method. /// See comments in \a DISubprogram* for descriptions of these fields. /// \param Scope Function scope. @@ -558,7 +558,11 @@ namespace llvm { /// \param isDefinition True if this is a function definition. /// \param Virtuality Attributes describing virtualness. e.g. pure /// virtual function. - /// \param VTableIndex Index no of this method in virtual table. + /// \param VTableIndex Index no of this method in virtual table, or -1u if + /// unrepresentable. + /// \param ThisAdjustment + /// MS ABI-specific adjustment of 'this' that occurs + /// in the prologue. /// \param VTableHolder Type that holds vtable. /// \param Flags e.g. is this function prototyped or not. /// This flags are used to emit dwarf attributes. @@ -568,8 +572,9 @@ namespace llvm { createMethod(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, bool isDefinition, unsigned Virtuality = 0, - unsigned VTableIndex = 0, DIType *VTableHolder = nullptr, - unsigned Flags = 0, bool isOptimized = false, + unsigned VTableIndex = 0, int ThisAdjustment = 0, + DIType *VTableHolder = nullptr, unsigned Flags = 0, + bool isOptimized = false, DITemplateParameterArray TParams = nullptr); /// This creates new descriptor for a namespace with the specified diff --git a/include/llvm/IR/DataLayout.h b/include/llvm/IR/DataLayout.h index 19a3a6661feb..173121b72ffd 100644 --- a/include/llvm/IR/DataLayout.h +++ b/include/llvm/IR/DataLayout.h @@ -20,7 +20,6 @@ #ifndef LLVM_IR_DATALAYOUT_H #define LLVM_IR_DATALAYOUT_H -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Type.h" @@ -34,8 +33,6 @@ typedef struct LLVMOpaqueTargetData *LLVMTargetDataRef; namespace llvm { class Value; -class Type; -class IntegerType; class StructType; class StructLayout; class Triple; @@ -236,14 +233,14 @@ public: /// on any known one. This returns false if the integer width is not legal. /// /// The width is specified in bits. - bool isLegalInteger(unsigned Width) const { + bool isLegalInteger(uint64_t Width) const { for (unsigned LegalIntWidth : LegalIntWidths) if (LegalIntWidth == Width) return true; return false; } - bool isIllegalInteger(unsigned Width) const { return !isLegalInteger(Width); } + bool isIllegalInteger(uint64_t Width) const { return !isLegalInteger(Width); } /// Returns true if the given alignment exceeds the natural stack alignment. bool exceedsNaturalStackAlignment(unsigned Align) const { @@ -387,7 +384,7 @@ public: /// returns 12 or 16 for x86_fp80, depending on alignment. uint64_t getTypeAllocSize(Type *Ty) const { // Round up to the next alignment boundary. - return RoundUpToAlignment(getTypeStoreSize(Ty), getABITypeAlignment(Ty)); + return alignTo(getTypeStoreSize(Ty), getABITypeAlignment(Ty)); } /// \brief Returns the offset in bits between successive objects of the @@ -430,19 +427,20 @@ public: /// \brief Returns the largest legal integer type, or null if none are set. Type *getLargestLegalIntType(LLVMContext &C) const { - unsigned LargestSize = getLargestLegalIntTypeSize(); + unsigned LargestSize = getLargestLegalIntTypeSizeInBits(); return (LargestSize == 0) ? nullptr : Type::getIntNTy(C, LargestSize); } /// \brief Returns the size of largest legal integer type size, or 0 if none /// are set. - unsigned getLargestLegalIntTypeSize() const; + unsigned getLargestLegalIntTypeSizeInBits() const; /// \brief Returns the offset from the beginning of the type for the specified /// indices. /// + /// Note that this takes the element type, not the pointer type. /// This is used to implement getelementptr. - uint64_t getIndexedOffset(Type *Ty, ArrayRef<Value *> Indices) const; + int64_t getIndexedOffsetInType(Type *ElemTy, ArrayRef<Value *> Indices) const; /// \brief Returns a StructLayout object, indicating the alignment of the /// struct, its size, and the offsets of its fields. @@ -475,7 +473,7 @@ inline LLVMTargetDataRef wrap(const DataLayout *P) { class StructLayout { uint64_t StructSize; unsigned StructAlignment; - bool IsPadded : 1; + unsigned IsPadded : 1; unsigned NumElements : 31; uint64_t MemberOffsets[1]; // variable sized array! public: diff --git a/include/llvm/IR/DebugInfo.h b/include/llvm/IR/DebugInfo.h index 4caceacbb58e..972042432b7b 100644 --- a/include/llvm/IR/DebugInfo.h +++ b/include/llvm/IR/DebugInfo.h @@ -17,10 +17,8 @@ #ifndef LLVM_IR_DEBUGINFO_H #define LLVM_IR_DEBUGINFO_H -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/Support/Casting.h" @@ -32,21 +30,12 @@ namespace llvm { class Module; class DbgDeclareInst; class DbgValueInst; - -/// \brief Maps from type identifier to the actual MDNode. -typedef DenseMap<const MDString *, DIType *> DITypeIdentifierMap; +template <typename K, typename V, typename KeyInfoT, typename BucketT> +class DenseMap; /// \brief Find subprogram that is enclosing this scope. DISubprogram *getDISubprogram(const MDNode *Scope); -/// \brief Find debug info for a given function. -/// -/// \returns a valid subprogram, if found. Otherwise, return \c nullptr. -DISubprogram *getDISubprogram(const Function *F); - -/// \brief Generate map by visiting all retained types. -DITypeIdentifierMap generateDITypeIdentifierMap(const NamedMDNode *CU_Nodes); - /// \brief Strip debug info in the module if it exists. /// /// To do this, we remove all calls to the debugger intrinsics and any named @@ -68,8 +57,6 @@ unsigned getDebugMetadataVersionFromModule(const Module &M); /// used by the CUs. class DebugInfoFinder { public: - DebugInfoFinder() : TypeMapInitialized(false) {} - /// \brief Process entire module and collect debug info anchors. void processModule(const Module &M); @@ -136,11 +123,7 @@ private: SmallVector<DIGlobalVariable *, 8> GVs; SmallVector<DIType *, 8> TYs; SmallVector<DIScope *, 8> Scopes; - SmallPtrSet<const MDNode *, 64> NodesSeen; - DITypeIdentifierMap TypeIdentifierMap; - - /// \brief Specify if TypeIdentifierMap is initialized. - bool TypeMapInitialized; + SmallPtrSet<const MDNode *, 32> NodesSeen; }; } // end namespace llvm diff --git a/include/llvm/IR/DebugInfoFlags.def b/include/llvm/IR/DebugInfoFlags.def index 9756c12264b4..26238c349e7e 100644 --- a/include/llvm/IR/DebugInfoFlags.def +++ b/include/llvm/IR/DebugInfoFlags.def @@ -33,5 +33,10 @@ HANDLE_DI_FLAG((1 << 12), StaticMember) HANDLE_DI_FLAG((1 << 13), LValueReference) HANDLE_DI_FLAG((1 << 14), RValueReference) HANDLE_DI_FLAG((1 << 15), ExternalTypeRef) +HANDLE_DI_FLAG((1 << 16), SingleInheritance) +HANDLE_DI_FLAG((2 << 16), MultipleInheritance) +HANDLE_DI_FLAG((3 << 16), VirtualInheritance) +HANDLE_DI_FLAG((1 << 18), IntroducedVirtual) +HANDLE_DI_FLAG((1 << 19), BitField) #undef HANDLE_DI_FLAG diff --git a/include/llvm/IR/DebugInfoMetadata.h b/include/llvm/IR/DebugInfoMetadata.h index 456313a70e83..853a94afd9d9 100644 --- a/include/llvm/IR/DebugInfoMetadata.h +++ b/include/llvm/IR/DebugInfoMetadata.h @@ -43,21 +43,25 @@ namespace llvm { -/// \brief Pointer union between a subclass of DINode and MDString. +template <typename T> class Optional; + +/// Holds a subclass of DINode. /// -/// \a DICompositeType can be referenced via an \a MDString unique identifier. -/// This class allows some type safety in the face of that, requiring either a -/// node of a particular type or an \a MDString. +/// FIXME: This class doesn't currently make much sense. Previously it was a +/// union beteen MDString (for ODR-uniqued types) and things like DIType. To +/// support CodeView work, it wasn't deleted outright when MDString-based type +/// references were deleted; we'll soon need a similar concept for CodeView +/// DITypeIndex. template <class T> class TypedDINodeRef { const Metadata *MD = nullptr; public: TypedDINodeRef() = default; TypedDINodeRef(std::nullptr_t) {} + TypedDINodeRef(const T *MD) : MD(MD) {} - /// \brief Construct from a raw pointer. explicit TypedDINodeRef(const Metadata *MD) : MD(MD) { - assert((!MD || isa<MDString>(MD) || isa<T>(MD)) && "Expected valid ref"); + assert((!MD || isa<T>(MD)) && "Expected valid type ref"); } template <class U> @@ -69,26 +73,10 @@ public: operator Metadata *() const { return const_cast<Metadata *>(MD); } + T *resolve() const { return const_cast<T *>(cast_or_null<T>(MD)); } + bool operator==(const TypedDINodeRef<T> &X) const { return MD == X.MD; } bool operator!=(const TypedDINodeRef<T> &X) const { return MD != X.MD; } - - /// \brief Create a reference. - /// - /// Get a reference to \c N, using an \a MDString reference if available. - static TypedDINodeRef get(const T *N); - - template <class MapTy> T *resolve(const MapTy &Map) const { - if (!MD) - return nullptr; - - if (auto *Typed = dyn_cast<T>(MD)) - return const_cast<T *>(Typed); - - auto *S = cast<MDString>(MD); - auto I = Map.find(S); - assert(I != Map.end() && "Missing identifier in type map"); - return cast<T>(I->second); - } }; typedef TypedDINodeRef<DINode> DINodeRef; @@ -173,6 +161,9 @@ protected: return MDString::get(Context, S); } + /// Allow subclasses to mutate the tag. + void setTag(unsigned Tag) { SubclassData16 = Tag; } + public: unsigned getTag() const { return SubclassData16; } @@ -183,7 +174,9 @@ public: enum DIFlags { #define HANDLE_DI_FLAG(ID, NAME) Flag##NAME = ID, #include "llvm/IR/DebugInfoFlags.def" - FlagAccessibility = FlagPrivate | FlagProtected | FlagPublic + FlagAccessibility = FlagPrivate | FlagProtected | FlagPublic, + FlagPtrToMemberRep = FlagSingleInheritance | FlagMultipleInheritance | + FlagVirtualInheritance, }; static unsigned getFlag(StringRef Flag); @@ -196,8 +189,6 @@ public: static unsigned splitFlags(unsigned Flags, SmallVectorImpl<unsigned> &SplitFlags); - DINodeRef getRef() const { return DINodeRef::get(this); } - static bool classof(const Metadata *MD) { switch (MD->getMetadataID()) { default: @@ -291,6 +282,7 @@ public: unsigned getTag() const { return SubclassData16; } StringRef getHeader() const { return getStringOperand(0); } + MDString *getRawHeader() const { return getOperandAs<MDString>(0); } op_iterator dwarf_op_begin() const { return op_begin() + 1; } op_iterator dwarf_op_end() const { return op_end(); } @@ -431,8 +423,6 @@ public: : static_cast<Metadata *>(getOperand(0)); } - DIScopeRef getRef() const { return DIScopeRef::get(this); } - static bool classof(const Metadata *MD) { switch (MD->getMetadataID()) { default: @@ -527,11 +517,28 @@ protected: DIType(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag, unsigned Line, uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, ArrayRef<Metadata *> Ops) - : DIScope(C, ID, Storage, Tag, Ops), Line(Line), Flags(Flags), - SizeInBits(SizeInBits), AlignInBits(AlignInBits), - OffsetInBits(OffsetInBits) {} + : DIScope(C, ID, Storage, Tag, Ops) { + init(Line, SizeInBits, AlignInBits, OffsetInBits, Flags); + } ~DIType() = default; + void init(unsigned Line, uint64_t SizeInBits, uint64_t AlignInBits, + uint64_t OffsetInBits, unsigned Flags) { + this->Line = Line; + this->Flags = Flags; + this->SizeInBits = SizeInBits; + this->AlignInBits = AlignInBits; + this->OffsetInBits = OffsetInBits; + } + + /// Change fields in place. + void mutate(unsigned Tag, unsigned Line, uint64_t SizeInBits, + uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags) { + assert(isDistinct() && "Only distinct nodes can mutate"); + setTag(Tag); + init(Line, SizeInBits, AlignInBits, OffsetInBits, Flags); + } + public: TempDIType clone() const { return TempDIType(cast<DIType>(MDNode::clone().release())); @@ -574,13 +581,12 @@ public: return getFlags() & FlagObjcClassComplete; } bool isVector() const { return getFlags() & FlagVector; } + bool isBitField() const { return getFlags() & FlagBitField; } bool isStaticMember() const { return getFlags() & FlagStaticMember; } bool isLValueReference() const { return getFlags() & FlagLValueReference; } bool isRValueReference() const { return getFlags() & FlagRValueReference; } bool isExternalTypeRef() const { return getFlags() & FlagExternalTypeRef; } - DITypeRef getRef() const { return DITypeRef::get(this); } - static bool classof(const Metadata *MD) { switch (MD->getMetadataID()) { default: @@ -735,6 +741,12 @@ public: DIObjCProperty *getObjCProperty() const { return dyn_cast_or_null<DIObjCProperty>(getExtraData()); } + Constant *getStorageOffsetInBits() const { + assert(getTag() == dwarf::DW_TAG_member && isBitField()); + if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) + return C->getValue(); + return nullptr; + } Constant *getConstant() const { assert(getTag() == dwarf::DW_TAG_member && isStaticMember()); if (auto *C = cast_or_null<ConstantAsMetadata>(getExtraData())) @@ -767,6 +779,16 @@ class DICompositeType : public DIType { RuntimeLang(RuntimeLang) {} ~DICompositeType() = default; + /// Change fields in place. + void mutate(unsigned Tag, unsigned Line, unsigned RuntimeLang, + uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, + unsigned Flags) { + assert(isDistinct() && "Only distinct nodes can mutate"); + assert(getRawIdentifier() && "Only ODR-uniqued nodes should mutate"); + this->RuntimeLang = RuntimeLang; + DIType::mutate(Tag, Line, SizeInBits, AlignInBits, OffsetInBits, Flags); + } + static DICompositeType * getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, Metadata *File, unsigned Line, DIScopeRef Scope, DITypeRef BaseType, @@ -822,6 +844,40 @@ public: TempDICompositeType clone() const { return cloneImpl(); } + /// Get a DICompositeType with the given ODR identifier. + /// + /// If \a LLVMContext::isODRUniquingDebugTypes(), gets the mapped + /// DICompositeType for the given ODR \c Identifier. If none exists, creates + /// a new node. + /// + /// Else, returns \c nullptr. + static DICompositeType * + getODRType(LLVMContext &Context, MDString &Identifier, unsigned Tag, + MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, + Metadata *BaseType, uint64_t SizeInBits, uint64_t AlignInBits, + uint64_t OffsetInBits, unsigned Flags, Metadata *Elements, + unsigned RuntimeLang, Metadata *VTableHolder, + Metadata *TemplateParams); + static DICompositeType *getODRTypeIfExists(LLVMContext &Context, + MDString &Identifier); + + /// Build a DICompositeType with the given ODR identifier. + /// + /// Looks up the mapped DICompositeType for the given ODR \c Identifier. If + /// it doesn't exist, creates a new one. If it does exist and \a + /// isForwardDecl(), and the new arguments would be a definition, mutates the + /// the type in place. In either case, returns the type. + /// + /// If not \a LLVMContext::isODRUniquingDebugTypes(), this function returns + /// nullptr. + static DICompositeType * + buildODRType(LLVMContext &Context, MDString &Identifier, unsigned Tag, + MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, + Metadata *BaseType, uint64_t SizeInBits, uint64_t AlignInBits, + uint64_t OffsetInBits, unsigned Flags, Metadata *Elements, + unsigned RuntimeLang, Metadata *VTableHolder, + Metadata *TemplateParams); + DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); } DINodeArray getElements() const { return cast_or_null<MDTuple>(getRawElements()); @@ -866,14 +922,6 @@ public: } }; -template <class T> TypedDINodeRef<T> TypedDINodeRef<T>::get(const T *N) { - if (N) - if (auto *Composite = dyn_cast<DICompositeType>(N)) - if (auto *S = Composite->getRawIdentifier()) - return TypedDINodeRef<T>(S); - return TypedDINodeRef<T>(N); -} - /// \brief Type array for a subprogram. /// /// TODO: Fold the array of types in directly as operands. @@ -881,35 +929,44 @@ class DISubroutineType : public DIType { friend class LLVMContextImpl; friend class MDNode; + /// The calling convention used with DW_AT_calling_convention. Actually of + /// type dwarf::CallingConvention. + uint8_t CC; + DISubroutineType(LLVMContext &C, StorageType Storage, unsigned Flags, - ArrayRef<Metadata *> Ops) + uint8_t CC, ArrayRef<Metadata *> Ops) : DIType(C, DISubroutineTypeKind, Storage, dwarf::DW_TAG_subroutine_type, - 0, 0, 0, 0, Flags, Ops) {} + 0, 0, 0, 0, Flags, Ops), + CC(CC) {} ~DISubroutineType() = default; static DISubroutineType *getImpl(LLVMContext &Context, unsigned Flags, - DITypeRefArray TypeArray, + uint8_t CC, DITypeRefArray TypeArray, StorageType Storage, bool ShouldCreate = true) { - return getImpl(Context, Flags, TypeArray.get(), Storage, ShouldCreate); + return getImpl(Context, Flags, CC, TypeArray.get(), Storage, ShouldCreate); } static DISubroutineType *getImpl(LLVMContext &Context, unsigned Flags, - Metadata *TypeArray, StorageType Storage, + uint8_t CC, Metadata *TypeArray, + StorageType Storage, bool ShouldCreate = true); TempDISubroutineType cloneImpl() const { - return getTemporary(getContext(), getFlags(), getTypeArray()); + return getTemporary(getContext(), getFlags(), getCC(), getTypeArray()); } public: DEFINE_MDNODE_GET(DISubroutineType, - (unsigned Flags, DITypeRefArray TypeArray), - (Flags, TypeArray)) - DEFINE_MDNODE_GET(DISubroutineType, (unsigned Flags, Metadata *TypeArray), - (Flags, TypeArray)) + (unsigned Flags, uint8_t CC, DITypeRefArray TypeArray), + (Flags, CC, TypeArray)) + DEFINE_MDNODE_GET(DISubroutineType, + (unsigned Flags, uint8_t CC, Metadata *TypeArray), + (Flags, CC, TypeArray)) TempDISubroutineType clone() const { return cloneImpl(); } + uint8_t getCC() const { return CC; } + DITypeRefArray getTypeArray() const { return cast_or_null<MDTuple>(getRawTypeArray()); } @@ -924,7 +981,17 @@ public: class DICompileUnit : public DIScope { friend class LLVMContextImpl; friend class MDNode; +public: + enum DebugEmissionKind : unsigned { + NoDebug = 0, + FullDebug, + LineTablesOnly, + LastEmissionKind = LineTablesOnly + }; + static Optional<DebugEmissionKind> getEmissionKind(StringRef Str); + static const char *EmissionKindString(DebugEmissionKind EK); +private: unsigned SourceLanguage; bool IsOptimized; unsigned RuntimeVersion; @@ -947,33 +1014,30 @@ class DICompileUnit : public DIScope { StringRef Producer, bool IsOptimized, StringRef Flags, unsigned RuntimeVersion, StringRef SplitDebugFilename, unsigned EmissionKind, DICompositeTypeArray EnumTypes, - DITypeArray RetainedTypes, DISubprogramArray Subprograms, - DIGlobalVariableArray GlobalVariables, + DIScopeArray RetainedTypes, DIGlobalVariableArray GlobalVariables, DIImportedEntityArray ImportedEntities, DIMacroNodeArray Macros, uint64_t DWOId, StorageType Storage, bool ShouldCreate = true) { - return getImpl(Context, SourceLanguage, File, - getCanonicalMDString(Context, Producer), IsOptimized, - getCanonicalMDString(Context, Flags), RuntimeVersion, - getCanonicalMDString(Context, SplitDebugFilename), - EmissionKind, EnumTypes.get(), RetainedTypes.get(), - Subprograms.get(), GlobalVariables.get(), - ImportedEntities.get(), Macros.get(), DWOId, Storage, - ShouldCreate); + return getImpl( + Context, SourceLanguage, File, getCanonicalMDString(Context, Producer), + IsOptimized, getCanonicalMDString(Context, Flags), RuntimeVersion, + getCanonicalMDString(Context, SplitDebugFilename), EmissionKind, + EnumTypes.get(), RetainedTypes.get(), GlobalVariables.get(), + ImportedEntities.get(), Macros.get(), DWOId, Storage, ShouldCreate); } static DICompileUnit * getImpl(LLVMContext &Context, unsigned SourceLanguage, Metadata *File, MDString *Producer, bool IsOptimized, MDString *Flags, unsigned RuntimeVersion, MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, - Metadata *Subprograms, Metadata *GlobalVariables, - Metadata *ImportedEntities, Metadata *Macros, uint64_t DWOId, - StorageType Storage, bool ShouldCreate = true); + Metadata *GlobalVariables, Metadata *ImportedEntities, + Metadata *Macros, uint64_t DWOId, StorageType Storage, + bool ShouldCreate = true); TempDICompileUnit cloneImpl() const { return getTemporary( getContext(), getSourceLanguage(), getFile(), getProducer(), isOptimized(), getFlags(), getRuntimeVersion(), getSplitDebugFilename(), - getEmissionKind(), getEnumTypes(), getRetainedTypes(), getSubprograms(), + getEmissionKind(), getEnumTypes(), getRetainedTypes(), getGlobalVariables(), getImportedEntities(), getMacros(), DWOId); } @@ -985,24 +1049,23 @@ public: DICompileUnit, (unsigned SourceLanguage, DIFile *File, StringRef Producer, bool IsOptimized, StringRef Flags, unsigned RuntimeVersion, - StringRef SplitDebugFilename, unsigned EmissionKind, - DICompositeTypeArray EnumTypes, DITypeArray RetainedTypes, - DISubprogramArray Subprograms, DIGlobalVariableArray GlobalVariables, + StringRef SplitDebugFilename, DebugEmissionKind EmissionKind, + DICompositeTypeArray EnumTypes, DIScopeArray RetainedTypes, + DIGlobalVariableArray GlobalVariables, DIImportedEntityArray ImportedEntities, DIMacroNodeArray Macros, uint64_t DWOId), (SourceLanguage, File, Producer, IsOptimized, Flags, RuntimeVersion, - SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, Subprograms, + SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, GlobalVariables, ImportedEntities, Macros, DWOId)) DEFINE_MDNODE_GET_DISTINCT_TEMPORARY( DICompileUnit, (unsigned SourceLanguage, Metadata *File, MDString *Producer, bool IsOptimized, MDString *Flags, unsigned RuntimeVersion, MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes, - Metadata *RetainedTypes, Metadata *Subprograms, - Metadata *GlobalVariables, Metadata *ImportedEntities, Metadata *Macros, - uint64_t DWOId), + Metadata *RetainedTypes, Metadata *GlobalVariables, + Metadata *ImportedEntities, Metadata *Macros, uint64_t DWOId), (SourceLanguage, File, Producer, IsOptimized, Flags, RuntimeVersion, - SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, Subprograms, + SplitDebugFilename, EmissionKind, EnumTypes, RetainedTypes, GlobalVariables, ImportedEntities, Macros, DWOId)) TempDICompileUnit clone() const { return cloneImpl(); } @@ -1010,19 +1073,18 @@ public: unsigned getSourceLanguage() const { return SourceLanguage; } bool isOptimized() const { return IsOptimized; } unsigned getRuntimeVersion() const { return RuntimeVersion; } - unsigned getEmissionKind() const { return EmissionKind; } + DebugEmissionKind getEmissionKind() const { + return (DebugEmissionKind)EmissionKind; + } StringRef getProducer() const { return getStringOperand(1); } StringRef getFlags() const { return getStringOperand(2); } StringRef getSplitDebugFilename() const { return getStringOperand(3); } DICompositeTypeArray getEnumTypes() const { return cast_or_null<MDTuple>(getRawEnumTypes()); } - DITypeArray getRetainedTypes() const { + DIScopeArray getRetainedTypes() const { return cast_or_null<MDTuple>(getRawRetainedTypes()); } - DISubprogramArray getSubprograms() const { - return cast_or_null<MDTuple>(getRawSubprograms()); - } DIGlobalVariableArray getGlobalVariables() const { return cast_or_null<MDTuple>(getRawGlobalVariables()); } @@ -1042,10 +1104,9 @@ public: } Metadata *getRawEnumTypes() const { return getOperand(4); } Metadata *getRawRetainedTypes() const { return getOperand(5); } - Metadata *getRawSubprograms() const { return getOperand(6); } - Metadata *getRawGlobalVariables() const { return getOperand(7); } - Metadata *getRawImportedEntities() const { return getOperand(8); } - Metadata *getRawMacros() const { return getOperand(9); } + Metadata *getRawGlobalVariables() const { return getOperand(6); } + Metadata *getRawImportedEntities() const { return getOperand(7); } + Metadata *getRawMacros() const { return getOperand(8); } /// \brief Replace arrays. /// @@ -1059,16 +1120,13 @@ public: void replaceRetainedTypes(DITypeArray N) { replaceOperandWith(5, N.get()); } - void replaceSubprograms(DISubprogramArray N) { - replaceOperandWith(6, N.get()); - } void replaceGlobalVariables(DIGlobalVariableArray N) { - replaceOperandWith(7, N.get()); + replaceOperandWith(6, N.get()); } void replaceImportedEntities(DIImportedEntityArray N) { - replaceOperandWith(8, N.get()); + replaceOperandWith(7, N.get()); } - void replaceMacros(DIMacroNodeArray N) { replaceOperandWith(9, N.get()); } + void replaceMacros(DIMacroNodeArray N) { replaceOperandWith(8, N.get()); } /// @} static bool classof(const Metadata *MD) { @@ -1095,6 +1153,12 @@ public: /// chain. DISubprogram *getSubprogram() const; + /// Get the first non DILexicalBlockFile scope of this scope. + /// + /// Return this if it's not a \a DILexicalBlockFIle; otherwise, look up the + /// scope chain. + DILocalScope *getNonLexicalBlockFileScope() const; + static bool classof(const Metadata *MD) { return MD->getMetadataID() == DISubprogramKind || MD->getMetadataID() == DILexicalBlockKind || @@ -1192,15 +1256,6 @@ public: /// instructions that are on different basic blocks. inline unsigned getDiscriminator() const; - /// \brief Compute new discriminator in the given context. - /// - /// This modifies the \a LLVMContext that \c this is in to increment the next - /// discriminator for \c this's line/filename combination. - /// - /// FIXME: Delete this. See comments in implementation and at the only call - /// site in \a AddDiscriminators::runOnFunction(). - unsigned computeNewDiscriminator() const; - Metadata *getRawScope() const { return getOperand(0); } Metadata *getRawInlinedAt() const { if (getNumOperands() == 2) @@ -1223,22 +1278,41 @@ class DISubprogram : public DILocalScope { unsigned Line; unsigned ScopeLine; - unsigned Virtuality; unsigned VirtualIndex; - unsigned Flags; - bool IsLocalToUnit; - bool IsDefinition; - bool IsOptimized; + + /// In the MS ABI, the implicit 'this' parameter is adjusted in the prologue + /// of method overrides from secondary bases by this amount. It may be + /// negative. + int ThisAdjustment; + + // Virtuality can only assume three values, so we can pack + // in 2 bits (none/pure/pure_virtual). + unsigned Virtuality : 2; + + unsigned Flags : 27; + + // These are boolean flags so one bit is enough. + // MSVC starts a new container field every time the base + // type changes so we can't use 'bool' to ensure these bits + // are packed. + unsigned IsLocalToUnit : 1; + unsigned IsDefinition : 1; + unsigned IsOptimized : 1; DISubprogram(LLVMContext &C, StorageType Storage, unsigned Line, unsigned ScopeLine, unsigned Virtuality, unsigned VirtualIndex, - unsigned Flags, bool IsLocalToUnit, bool IsDefinition, - bool IsOptimized, ArrayRef<Metadata *> Ops) + int ThisAdjustment, unsigned Flags, bool IsLocalToUnit, + bool IsDefinition, bool IsOptimized, ArrayRef<Metadata *> Ops) : DILocalScope(C, DISubprogramKind, Storage, dwarf::DW_TAG_subprogram, Ops), - Line(Line), ScopeLine(ScopeLine), Virtuality(Virtuality), - VirtualIndex(VirtualIndex), Flags(Flags), IsLocalToUnit(IsLocalToUnit), - IsDefinition(IsDefinition), IsOptimized(IsOptimized) {} + Line(Line), ScopeLine(ScopeLine), VirtualIndex(VirtualIndex), + ThisAdjustment(ThisAdjustment), Virtuality(Virtuality), Flags(Flags), + IsLocalToUnit(IsLocalToUnit), IsDefinition(IsDefinition), + IsOptimized(IsOptimized) { + static_assert(dwarf::DW_VIRTUALITY_max < 4, "Virtuality out of range"); + assert(Virtuality < 4 && "Virtuality out of range"); + assert((Flags < (1 << 27)) && "Flags out of range"); + } ~DISubprogram() = default; static DISubprogram * @@ -1246,32 +1320,34 @@ class DISubprogram : public DILocalScope { StringRef LinkageName, DIFile *File, unsigned Line, DISubroutineType *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, DITypeRef ContainingType, unsigned Virtuality, - unsigned VirtualIndex, unsigned Flags, bool IsOptimized, + unsigned VirtualIndex, int ThisAdjustment, unsigned Flags, + bool IsOptimized, DICompileUnit *Unit, DITemplateParameterArray TemplateParams, DISubprogram *Declaration, DILocalVariableArray Variables, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, Scope, getCanonicalMDString(Context, Name), getCanonicalMDString(Context, LinkageName), File, Line, Type, IsLocalToUnit, IsDefinition, ScopeLine, ContainingType, - Virtuality, VirtualIndex, Flags, IsOptimized, - TemplateParams.get(), Declaration, Variables.get(), Storage, - ShouldCreate); + Virtuality, VirtualIndex, ThisAdjustment, Flags, IsOptimized, + Unit, TemplateParams.get(), Declaration, Variables.get(), + Storage, ShouldCreate); } static DISubprogram * getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality, unsigned VirtualIndex, - unsigned Flags, bool IsOptimized, Metadata *TemplateParams, - Metadata *Declaration, Metadata *Variables, StorageType Storage, - bool ShouldCreate = true); + int ThisAdjustment, unsigned Flags, bool IsOptimized, Metadata *Unit, + Metadata *TemplateParams, Metadata *Declaration, Metadata *Variables, + StorageType Storage, bool ShouldCreate = true); TempDISubprogram cloneImpl() const { return getTemporary( getContext(), getScope(), getName(), getLinkageName(), getFile(), getLine(), getType(), isLocalToUnit(), isDefinition(), getScopeLine(), - getContainingType(), getVirtuality(), getVirtualIndex(), getFlags(), - isOptimized(), getTemplateParams(), getDeclaration(), getVariables()); + getContainingType(), getVirtuality(), getVirtualIndex(), + getThisAdjustment(), getFlags(), isOptimized(), getUnit(), + getTemplateParams(), getDeclaration(), getVariables()); } public: @@ -1280,25 +1356,26 @@ public: DIFile *File, unsigned Line, DISubroutineType *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, DITypeRef ContainingType, unsigned Virtuality, - unsigned VirtualIndex, unsigned Flags, bool IsOptimized, + unsigned VirtualIndex, int ThisAdjustment, unsigned Flags, + bool IsOptimized, DICompileUnit *Unit, DITemplateParameterArray TemplateParams = nullptr, DISubprogram *Declaration = nullptr, DILocalVariableArray Variables = nullptr), (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit, IsDefinition, ScopeLine, ContainingType, Virtuality, - VirtualIndex, Flags, IsOptimized, TemplateParams, - Declaration, Variables)) + VirtualIndex, ThisAdjustment, Flags, IsOptimized, Unit, + TemplateParams, Declaration, Variables)) DEFINE_MDNODE_GET( DISubprogram, (Metadata * Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality, - unsigned VirtualIndex, unsigned Flags, bool IsOptimized, - Metadata *TemplateParams = nullptr, Metadata *Declaration = nullptr, - Metadata *Variables = nullptr), + unsigned VirtualIndex, int ThisAdjustment, unsigned Flags, + bool IsOptimized, Metadata *Unit, Metadata *TemplateParams = nullptr, + Metadata *Declaration = nullptr, Metadata *Variables = nullptr), (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit, IsDefinition, - ScopeLine, ContainingType, Virtuality, VirtualIndex, Flags, IsOptimized, - TemplateParams, Declaration, Variables)) + ScopeLine, ContainingType, Virtuality, VirtualIndex, ThisAdjustment, + Flags, IsOptimized, Unit, TemplateParams, Declaration, Variables)) TempDISubprogram clone() const { return cloneImpl(); } @@ -1306,6 +1383,7 @@ public: unsigned getLine() const { return Line; } unsigned getVirtuality() const { return Virtuality; } unsigned getVirtualIndex() const { return VirtualIndex; } + int getThisAdjustment() const { return ThisAdjustment; } unsigned getScopeLine() const { return ScopeLine; } unsigned getFlags() const { return Flags; } bool isLocalToUnit() const { return IsLocalToUnit; } @@ -1357,6 +1435,12 @@ public: return DITypeRef(getRawContainingType()); } + DICompileUnit *getUnit() const { + return cast_or_null<DICompileUnit>(getRawUnit()); + } + void replaceUnit(DICompileUnit *CU) { + replaceOperandWith(7, CU); + } DITemplateParameterArray getTemplateParams() const { return cast_or_null<MDTuple>(getRawTemplateParams()); } @@ -1370,9 +1454,10 @@ public: Metadata *getRawScope() const { return getOperand(1); } Metadata *getRawType() const { return getOperand(5); } Metadata *getRawContainingType() const { return getOperand(6); } - Metadata *getRawTemplateParams() const { return getOperand(7); } - Metadata *getRawDeclaration() const { return getOperand(8); } - Metadata *getRawVariables() const { return getOperand(9); } + Metadata *getRawUnit() const { return getOperand(7); } + Metadata *getRawTemplateParams() const { return getOperand(8); } + Metadata *getRawDeclaration() const { return getOperand(9); } + Metadata *getRawVariables() const { return getOperand(10); } /// \brief Check if this subprogram describes the given function. /// @@ -1852,13 +1937,16 @@ class DILocalVariable : public DIVariable { friend class LLVMContextImpl; friend class MDNode; - unsigned Arg; - unsigned Flags; + unsigned Arg : 16; + unsigned Flags : 16; DILocalVariable(LLVMContext &C, StorageType Storage, unsigned Line, unsigned Arg, unsigned Flags, ArrayRef<Metadata *> Ops) : DIVariable(C, DILocalVariableKind, Storage, Line, Ops), Arg(Arg), - Flags(Flags) {} + Flags(Flags) { + assert(Flags < (1 << 16) && "DILocalVariable: Flags out of range"); + assert(Arg < (1 << 16) && "DILocalVariable: Arg out of range"); + } ~DILocalVariable() = default; static DILocalVariable *getImpl(LLVMContext &Context, DIScope *Scope, diff --git a/include/llvm/IR/DerivedTypes.h b/include/llvm/IR/DerivedTypes.h index 071e69b1e808..efd0d07366ee 100644 --- a/include/llvm/IR/DerivedTypes.h +++ b/include/llvm/IR/DerivedTypes.h @@ -61,15 +61,14 @@ public: /// @brief Get the number of bits in this IntegerType unsigned getBitWidth() const { return getSubclassData(); } - /// getBitMask - Return a bitmask with ones set for all of the bits - /// that can be set by an unsigned version of this type. This is 0xFF for - /// i8, 0xFFFF for i16, etc. + /// Return a bitmask with ones set for all of the bits that can be set by an + /// unsigned version of this type. This is 0xFF for i8, 0xFFFF for i16, etc. uint64_t getBitMask() const { return ~uint64_t(0UL) >> (64-getBitWidth()); } - /// getSignBit - Return a uint64_t with just the most significant bit set (the - /// sign bit, if the value is treated as a signed number). + /// Return a uint64_t with just the most significant bit set (the sign bit, if + /// the value is treated as a signed number). uint64_t getSignBit() const { return 1ULL << (getBitWidth()-1); } @@ -95,7 +94,7 @@ unsigned Type::getIntegerBitWidth() const { return cast<IntegerType>(this)->getBitWidth(); } -/// FunctionType - Class to represent function types +/// Class to represent function types /// class FunctionType : public Type { FunctionType(const FunctionType &) = delete; @@ -103,22 +102,17 @@ class FunctionType : public Type { FunctionType(Type *Result, ArrayRef<Type*> Params, bool IsVarArgs); public: - /// FunctionType::get - This static method is the primary way of constructing - /// a FunctionType. - /// + /// This static method is the primary way of constructing a FunctionType. static FunctionType *get(Type *Result, ArrayRef<Type*> Params, bool isVarArg); - /// FunctionType::get - Create a FunctionType taking no parameters. - /// + /// Create a FunctionType taking no parameters. static FunctionType *get(Type *Result, bool isVarArg); - /// isValidReturnType - Return true if the specified type is valid as a return - /// type. + /// Return true if the specified type is valid as a return type. static bool isValidReturnType(Type *RetTy); - /// isValidArgumentType - Return true if the specified type is valid as an - /// argument type. + /// Return true if the specified type is valid as an argument type. static bool isValidArgumentType(Type *ArgTy); bool isVarArg() const { return getSubclassData()!=0; } @@ -134,9 +128,8 @@ public: /// Parameter type accessors. Type *getParamType(unsigned i) const { return ContainedTys[i+1]; } - /// getNumParams - Return the number of fixed parameters this function type - /// requires. This does not consider varargs. - /// + /// Return the number of fixed parameters this function type requires. + /// This does not consider varargs. unsigned getNumParams() const { return NumContainedTys - 1; } /// Methods for support type inquiry through isa, cast, and dyn_cast. @@ -159,16 +152,13 @@ unsigned Type::getFunctionNumParams() const { return cast<FunctionType>(this)->getNumParams(); } -/// CompositeType - Common super class of ArrayType, StructType, PointerType -/// and VectorType. +/// Common super class of ArrayType, StructType, PointerType and VectorType. class CompositeType : public Type { protected: explicit CompositeType(LLVMContext &C, TypeID tid) : Type(C, tid) {} public: - /// getTypeAtIndex - Given an index value into the type, return the type of - /// the element. - /// + /// Given an index value into the type, return the type of the element. Type *getTypeAtIndex(const Value *V) const; Type *getTypeAtIndex(unsigned Idx) const; bool indexValid(const Value *V) const; @@ -183,8 +173,8 @@ public: } }; -/// StructType - Class to represent struct types. There are two different kinds -/// of struct types: Literal structs and Identified structs. +/// Class to represent struct types. There are two different kinds of struct +/// types: Literal structs and Identified structs. /// /// Literal struct types (e.g. { i32, i32 }) are uniqued structurally, and must /// always have a body when created. You can get one of these by using one of @@ -216,15 +206,14 @@ class StructType : public CompositeType { SCDB_IsSized = 8 }; - /// SymbolTableEntry - For a named struct that actually has a name, this is a - /// pointer to the symbol table entry (maintained by LLVMContext) for the - /// struct. This is null if the type is an literal struct or if it is - /// a identified type that has an empty name. - /// + /// For a named struct that actually has a name, this is a pointer to the + /// symbol table entry (maintained by LLVMContext) for the struct. + /// This is null if the type is an literal struct or if it is a identified + /// type that has an empty name. void *SymbolTableEntry; public: - /// StructType::create - This creates an identified struct. + /// This creates an identified struct. static StructType *create(LLVMContext &Context, StringRef Name); static StructType *create(LLVMContext &Context); @@ -236,53 +225,48 @@ public: static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements); static StructType *create(StringRef Name, Type *elt1, ...) LLVM_END_WITH_NULL; - /// StructType::get - This static method is the primary way to create a - /// literal StructType. + /// This static method is the primary way to create a literal StructType. static StructType *get(LLVMContext &Context, ArrayRef<Type*> Elements, bool isPacked = false); - /// StructType::get - Create an empty structure type. - /// + /// Create an empty structure type. static StructType *get(LLVMContext &Context, bool isPacked = false); - /// StructType::get - This static method is a convenience method for creating - /// structure types by specifying the elements as arguments. Note that this - /// method always returns a non-packed struct, and requires at least one - /// element type. + /// This static method is a convenience method for creating structure types by + /// specifying the elements as arguments. Note that this method always returns + /// a non-packed struct, and requires at least one element type. static StructType *get(Type *elt1, ...) LLVM_END_WITH_NULL; bool isPacked() const { return (getSubclassData() & SCDB_Packed) != 0; } - /// isLiteral - Return true if this type is uniqued by structural - /// equivalence, false if it is a struct definition. + /// Return true if this type is uniqued by structural equivalence, false if it + /// is a struct definition. bool isLiteral() const { return (getSubclassData() & SCDB_IsLiteral) != 0; } - /// isOpaque - Return true if this is a type with an identity that has no body - /// specified yet. These prints as 'opaque' in .ll files. + /// Return true if this is a type with an identity that has no body specified + /// yet. These prints as 'opaque' in .ll files. bool isOpaque() const { return (getSubclassData() & SCDB_HasBody) == 0; } /// isSized - Return true if this is a sized type. bool isSized(SmallPtrSetImpl<Type *> *Visited = nullptr) const; - /// hasName - Return true if this is a named struct that has a non-empty name. + /// Return true if this is a named struct that has a non-empty name. bool hasName() const { return SymbolTableEntry != nullptr; } - /// getName - Return the name for this struct type if it has an identity. + /// Return the name for this struct type if it has an identity. /// This may return an empty string for an unnamed struct type. Do not call /// this on an literal type. StringRef getName() const; - /// setName - Change the name of this type to the specified name, or to a name - /// with a suffix if there is a collision. Do not call this on an literal - /// type. + /// Change the name of this type to the specified name, or to a name with a + /// suffix if there is a collision. Do not call this on an literal type. void setName(StringRef Name); - /// setBody - Specify a body for an opaque identified type. + /// Specify a body for an opaque identified type. void setBody(ArrayRef<Type*> Elements, bool isPacked = false); void setBody(Type *elt1, ...) LLVM_END_WITH_NULL; - /// isValidElementType - Return true if the specified type is valid as a - /// element type. + /// Return true if the specified type is valid as a element type. static bool isValidElementType(Type *ElemTy); // Iterator access to the elements. @@ -293,8 +277,7 @@ public: return makeArrayRef(element_begin(), element_end()); } - /// isLayoutIdentical - Return true if this is layout identical to the - /// specified struct. + /// Return true if this is layout identical to the specified struct. bool isLayoutIdentical(StructType *Other) const; /// Random access to the elements @@ -322,14 +305,12 @@ Type *Type::getStructElementType(unsigned N) const { return cast<StructType>(this)->getElementType(N); } -/// SequentialType - This is the superclass of the array, pointer and vector -/// type classes. All of these represent "arrays" in memory. The array type -/// represents a specifically sized array, pointer types are unsized/unknown -/// size arrays, vector types represent specifically sized arrays that -/// allow for use of SIMD instructions. SequentialType holds the common -/// features of all, which stem from the fact that all three lay their -/// components out in memory identically. -/// +/// This is the superclass of the array, pointer and vector type classes. +/// All of these represent "arrays" in memory. The array type represents a +/// specifically sized array, pointer types are unsized/unknown size arrays, +/// vector types represent specifically sized arrays that allow for use of SIMD +/// instructions. SequentialType holds the common features of all, which stem +/// from the fact that all three lay their components out in memory identically. class SequentialType : public CompositeType { Type *ContainedType; ///< Storage for the single contained type. SequentialType(const SequentialType &) = delete; @@ -343,7 +324,7 @@ protected: } public: - Type *getElementType() const { return ContainedTys[0]; } + Type *getElementType() const { return getSequentialElementType(); } /// Methods for support type inquiry through isa, cast, and dyn_cast. static inline bool classof(const Type *T) { @@ -353,12 +334,7 @@ public: } }; -Type *Type::getSequentialElementType() const { - return cast<SequentialType>(this)->getElementType(); -} - -/// ArrayType - Class to represent array types. -/// +/// Class to represent array types. class ArrayType : public SequentialType { uint64_t NumElements; @@ -367,13 +343,10 @@ class ArrayType : public SequentialType { ArrayType(Type *ElType, uint64_t NumEl); public: - /// ArrayType::get - This static method is the primary way to construct an - /// ArrayType - /// + /// This static method is the primary way to construct an ArrayType static ArrayType *get(Type *ElementType, uint64_t NumElements); - /// isValidElementType - Return true if the specified type is valid as a - /// element type. + /// Return true if the specified type is valid as a element type. static bool isValidElementType(Type *ElemTy); uint64_t getNumElements() const { return NumElements; } @@ -388,8 +361,7 @@ uint64_t Type::getArrayNumElements() const { return cast<ArrayType>(this)->getNumElements(); } -/// VectorType - Class to represent vector types. -/// +/// Class to represent vector types. class VectorType : public SequentialType { unsigned NumElements; @@ -398,15 +370,12 @@ class VectorType : public SequentialType { VectorType(Type *ElType, unsigned NumEl); public: - /// VectorType::get - This static method is the primary way to construct an - /// VectorType. - /// + /// This static method is the primary way to construct an VectorType. static VectorType *get(Type *ElementType, unsigned NumElements); - /// VectorType::getInteger - This static method gets a VectorType with the - /// same number of elements as the input type, and the element type is an - /// integer type of the same width as the input element type. - /// + /// This static method gets a VectorType with the same number of elements as + /// the input type, and the element type is an integer type of the same width + /// as the input element type. static VectorType *getInteger(VectorType *VTy) { unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); assert(EltBits && "Element size must be of a non-zero size"); @@ -414,20 +383,16 @@ public: return VectorType::get(EltTy, VTy->getNumElements()); } - /// VectorType::getExtendedElementVectorType - This static method is like - /// getInteger except that the element types are twice as wide as the - /// elements in the input type. - /// + /// This static method is like getInteger except that the element types are + /// twice as wide as the elements in the input type. static VectorType *getExtendedElementVectorType(VectorType *VTy) { unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); Type *EltTy = IntegerType::get(VTy->getContext(), EltBits * 2); return VectorType::get(EltTy, VTy->getNumElements()); } - /// VectorType::getTruncatedElementVectorType - This static method is like - /// getInteger except that the element types are half as wide as the - /// elements in the input type. - /// + /// This static method is like getInteger except that the element types are + /// half as wide as the elements in the input type. static VectorType *getTruncatedElementVectorType(VectorType *VTy) { unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); assert((EltBits & 1) == 0 && @@ -436,10 +401,8 @@ public: return VectorType::get(EltTy, VTy->getNumElements()); } - /// VectorType::getHalfElementsVectorType - This static method returns - /// a VectorType with half as many elements as the input type and the - /// same element type. - /// + /// This static method returns a VectorType with half as many elements as the + /// input type and the same element type. static VectorType *getHalfElementsVectorType(VectorType *VTy) { unsigned NumElts = VTy->getNumElements(); assert ((NumElts & 1) == 0 && @@ -447,23 +410,20 @@ public: return VectorType::get(VTy->getElementType(), NumElts/2); } - /// VectorType::getDoubleElementsVectorType - This static method returns - /// a VectorType with twice as many elements as the input type and the - /// same element type. - /// + /// This static method returns a VectorType with twice as many elements as the + /// input type and the same element type. static VectorType *getDoubleElementsVectorType(VectorType *VTy) { unsigned NumElts = VTy->getNumElements(); return VectorType::get(VTy->getElementType(), NumElts*2); } - /// isValidElementType - Return true if the specified type is valid as a - /// element type. + /// Return true if the specified type is valid as a element type. static bool isValidElementType(Type *ElemTy); - /// @brief Return the number of elements in the Vector type. + /// Return the number of elements in the Vector type. unsigned getNumElements() const { return NumElements; } - /// @brief Return the number of bits in the Vector type. + /// Return the number of bits in the Vector type. /// Returns zero when the vector is a vector of pointers. unsigned getBitWidth() const { return NumElements * getElementType()->getPrimitiveSizeInBits(); @@ -479,32 +439,30 @@ unsigned Type::getVectorNumElements() const { return cast<VectorType>(this)->getNumElements(); } -/// PointerType - Class to represent pointers. -/// +/// Class to represent pointers. class PointerType : public SequentialType { PointerType(const PointerType &) = delete; const PointerType &operator=(const PointerType &) = delete; explicit PointerType(Type *ElType, unsigned AddrSpace); public: - /// PointerType::get - This constructs a pointer to an object of the specified - /// type in a numbered address space. + /// This constructs a pointer to an object of the specified type in a numbered + /// address space. static PointerType *get(Type *ElementType, unsigned AddressSpace); - /// PointerType::getUnqual - This constructs a pointer to an object of the - /// specified type in the generic address space (address space zero). + /// This constructs a pointer to an object of the specified type in the + /// generic address space (address space zero). static PointerType *getUnqual(Type *ElementType) { return PointerType::get(ElementType, 0); } - /// isValidElementType - Return true if the specified type is valid as a - /// element type. + /// Return true if the specified type is valid as a element type. static bool isValidElementType(Type *ElemTy); /// Return true if we can load or store from a pointer to this type. static bool isLoadableOrStorableType(Type *ElemTy); - /// @brief Return the address space of the Pointer type. + /// Return the address space of the Pointer type. inline unsigned getAddressSpace() const { return getSubclassData(); } /// Implement support type inquiry through isa, cast, and dyn_cast. diff --git a/include/llvm/IR/DiagnosticInfo.h b/include/llvm/IR/DiagnosticInfo.h index f69955e5ed48..1c78684da64d 100644 --- a/include/llvm/IR/DiagnosticInfo.h +++ b/include/llvm/IR/DiagnosticInfo.h @@ -1,4 +1,4 @@ -//===- llvm/Support/DiagnosticInfo.h - Diagnostic Declaration ---*- C++ -*-===// +//===- llvm/IR/DiagnosticInfo.h - Diagnostic Declaration --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,11 +15,14 @@ #ifndef LLVM_IR_DIAGNOSTICINFO_H #define LLVM_IR_DIAGNOSTICINFO_H -#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" #include "llvm/IR/DebugLoc.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/Casting.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm-c/Types.h" #include <functional> +#include <string> namespace llvm { @@ -27,14 +30,12 @@ namespace llvm { class DiagnosticPrinter; class Function; class Instruction; -class LLVMContextImpl; -class Twine; -class Value; -class DebugLoc; +class LLVMContext; +class Module; class SMDiagnostic; /// \brief Defines the different supported severity of a diagnostic. -enum DiagnosticSeverity { +enum DiagnosticSeverity : char { DS_Error, DS_Warning, DS_Remark, @@ -48,9 +49,11 @@ enum DiagnosticSeverity { enum DiagnosticKind { DK_Bitcode, DK_InlineAsm, + DK_ResourceLimit, DK_StackSize, DK_Linker, DK_DebugMetadataVersion, + DK_DebugMetadataInvalid, DK_SampleProfile, DK_OptimizationRemark, DK_OptimizationRemarkMissed, @@ -58,8 +61,11 @@ enum DiagnosticKind { DK_OptimizationRemarkAnalysisFPCommute, DK_OptimizationRemarkAnalysisAliasing, DK_OptimizationFailure, + DK_FirstRemark = DK_OptimizationRemark, + DK_LastRemark = DK_OptimizationFailure, DK_MIRParser, DK_PGOProfile, + DK_Unsupported, DK_FirstPluginKind }; @@ -101,8 +107,6 @@ public: /// The printed message must not end with '.' nor start with a severity /// keyword. virtual void print(DiagnosticPrinter &DP) const = 0; - - static const char *AlwaysPrint; }; typedef std::function<void(const DiagnosticInfo &)> DiagnosticHandlerFunction; @@ -156,29 +160,64 @@ public: } }; -/// Diagnostic information for stack size reporting. +/// Diagnostic information for stack size etc. reporting. /// This is basically a function and a size. -class DiagnosticInfoStackSize : public DiagnosticInfo { +class DiagnosticInfoResourceLimit : public DiagnosticInfo { private: - /// The function that is concerned by this stack size diagnostic. + /// The function that is concerned by this resource limit diagnostic. const Function &Fn; - /// The computed stack size. - unsigned StackSize; + + /// Description of the resource type (e.g. stack size) + const char *ResourceName; + + /// The computed size usage + uint64_t ResourceSize; + + // Threshould passed + uint64_t ResourceLimit; public: /// \p The function that is concerned by this stack size diagnostic. /// \p The computed stack size. - DiagnosticInfoStackSize(const Function &Fn, unsigned StackSize, - DiagnosticSeverity Severity = DS_Warning) - : DiagnosticInfo(DK_StackSize, Severity), Fn(Fn), StackSize(StackSize) {} + DiagnosticInfoResourceLimit(const Function &Fn, + const char *ResourceName, + uint64_t ResourceSize, + DiagnosticSeverity Severity = DS_Warning, + DiagnosticKind Kind = DK_ResourceLimit, + uint64_t ResourceLimit = 0) + : DiagnosticInfo(Kind, Severity), + Fn(Fn), + ResourceName(ResourceName), + ResourceSize(ResourceSize), + ResourceLimit(ResourceLimit) {} const Function &getFunction() const { return Fn; } - unsigned getStackSize() const { return StackSize; } + const char *getResourceName() const { return ResourceName; } + uint64_t getResourceSize() const { return ResourceSize; } + uint64_t getResourceLimit() const { return ResourceLimit; } /// \see DiagnosticInfo::print. void print(DiagnosticPrinter &DP) const override; static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_ResourceLimit || + DI->getKind() == DK_StackSize; + } +}; + +class DiagnosticInfoStackSize : public DiagnosticInfoResourceLimit { +public: + DiagnosticInfoStackSize(const Function &Fn, + uint64_t StackSize, + DiagnosticSeverity Severity = DS_Warning, + uint64_t StackLimit = 0) + : DiagnosticInfoResourceLimit(Fn, "stack size", StackSize, + Severity, DK_StackSize, StackLimit) {} + + uint64_t getStackSize() const { return getResourceSize(); } + uint64_t getStackLimit() const { return getResourceLimit(); } + + static bool classof(const DiagnosticInfo *DI) { return DI->getKind() == DK_StackSize; } }; @@ -211,6 +250,29 @@ public: } }; +/// Diagnostic information for stripping invalid debug metadata. +class DiagnosticInfoIgnoringInvalidDebugMetadata : public DiagnosticInfo { +private: + /// The module that is concerned by this debug metadata version diagnostic. + const Module &M; + +public: + /// \p The module that is concerned by this debug metadata version diagnostic. + DiagnosticInfoIgnoringInvalidDebugMetadata( + const Module &M, DiagnosticSeverity Severity = DS_Warning) + : DiagnosticInfo(DK_DebugMetadataVersion, Severity), M(M) {} + + const Module &getModule() const { return M; } + + /// \see DiagnosticInfo::print. + void print(DiagnosticPrinter &DP) const override; + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_DebugMetadataInvalid; + } +}; + + /// Diagnostic information for the sample profiler. class DiagnosticInfoSampleProfile : public DiagnosticInfo { public: @@ -275,8 +337,42 @@ private: const Twine &Msg; }; +/// Common features for diagnostics with an associated DebugLoc +class DiagnosticInfoWithDebugLocBase : public DiagnosticInfo { +public: + /// \p Fn is the function where the diagnostic is being emitted. \p DLoc is + /// the location information to use in the diagnostic. + DiagnosticInfoWithDebugLocBase(enum DiagnosticKind Kind, + enum DiagnosticSeverity Severity, + const Function &Fn, + const DebugLoc &DLoc) + : DiagnosticInfo(Kind, Severity), Fn(Fn), DLoc(DLoc) {} + + /// Return true if location information is available for this diagnostic. + bool isLocationAvailable() const; + + /// Return a string with the location information for this diagnostic + /// in the format "file:line:col". If location information is not available, + /// it returns "<unknown>:0:0". + const std::string getLocationStr() const; + + /// Return location information for this diagnostic in three parts: + /// the source file name, line number and column. + void getLocation(StringRef *Filename, unsigned *Line, unsigned *Column) const; + + const Function &getFunction() const { return Fn; } + const DebugLoc &getDebugLoc() const { return DLoc; } + +private: + /// Function where this diagnostic is triggered. + const Function &Fn; + + /// Debug location where this diagnostic is triggered. + DebugLoc DLoc; +}; + /// Common features for diagnostics dealing with optimization remarks. -class DiagnosticInfoOptimizationBase : public DiagnosticInfo { +class DiagnosticInfoOptimizationBase : public DiagnosticInfoWithDebugLocBase { public: /// \p PassName is the name of the pass emitting this diagnostic. /// \p Fn is the function where the diagnostic is being emitted. \p DLoc is @@ -288,9 +384,10 @@ public: DiagnosticInfoOptimizationBase(enum DiagnosticKind Kind, enum DiagnosticSeverity Severity, const char *PassName, const Function &Fn, - const DebugLoc &DLoc, const Twine &Msg) - : DiagnosticInfo(Kind, Severity), PassName(PassName), Fn(Fn), DLoc(DLoc), - Msg(Msg) {} + const DebugLoc &DLoc, const Twine &Msg, + Optional<uint64_t> Hotness = None) + : DiagnosticInfoWithDebugLocBase(Kind, Severity, Fn, DLoc), + PassName(PassName), Msg(Msg), Hotness(Hotness) {} /// \see DiagnosticInfo::print. void print(DiagnosticPrinter &DP) const override; @@ -302,37 +399,26 @@ public: /// in BackendConsumer::OptimizationRemarkHandler). virtual bool isEnabled() const = 0; - /// Return true if location information is available for this diagnostic. - bool isLocationAvailable() const; - - /// Return a string with the location information for this diagnostic - /// in the format "file:line:col". If location information is not available, - /// it returns "<unknown>:0:0". - const std::string getLocationStr() const; - - /// Return location information for this diagnostic in three parts: - /// the source file name, line number and column. - void getLocation(StringRef *Filename, unsigned *Line, unsigned *Column) const; - const char *getPassName() const { return PassName; } - const Function &getFunction() const { return Fn; } - const DebugLoc &getDebugLoc() const { return DLoc; } const Twine &getMsg() const { return Msg; } + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() >= DK_FirstRemark && + DI->getKind() <= DK_LastRemark; + } + private: /// Name of the pass that triggers this report. If this matches the /// regular expression given in -Rpass=regexp, then the remark will /// be emitted. const char *PassName; - /// Function where this diagnostic is triggered. - const Function &Fn; - - /// Debug location where this diagnostic is triggered. - DebugLoc DLoc; - /// Message to report. const Twine &Msg; + + /// If profile information is available, this is the number of times the + /// corresponding code was executed in a profile instrumentation run. + Optional<uint64_t> Hotness; }; /// Diagnostic information for applied optimization remarks. @@ -373,9 +459,10 @@ public: /// must be valid for the whole life time of the diagnostic. DiagnosticInfoOptimizationRemarkMissed(const char *PassName, const Function &Fn, - const DebugLoc &DLoc, const Twine &Msg) + const DebugLoc &DLoc, const Twine &Msg, + Optional<uint64_t> Hotness = None) : DiagnosticInfoOptimizationBase(DK_OptimizationRemarkMissed, DS_Remark, - PassName, Fn, DLoc, Msg) {} + PassName, Fn, DLoc, Msg, Hotness) {} static bool classof(const DiagnosticInfo *DI) { return DI->getKind() == DK_OptimizationRemarkMissed; @@ -411,6 +498,10 @@ public: /// \see DiagnosticInfoOptimizationBase::isEnabled. bool isEnabled() const override; + static const char *AlwaysPrint; + + bool shouldAlwaysPrint() const { return getPassName() == AlwaysPrint; } + protected: DiagnosticInfoOptimizationRemarkAnalysis(enum DiagnosticKind Kind, const char *PassName, @@ -572,6 +663,34 @@ public: bool isEnabled() const override; }; +/// Diagnostic information for unsupported feature in backend. +class DiagnosticInfoUnsupported + : public DiagnosticInfoWithDebugLocBase { +private: + Twine Msg; + +public: + /// \p Fn is the function where the diagnostic is being emitted. \p DLoc is + /// the location information to use in the diagnostic. If line table + /// information is available, the diagnostic will include the source code + /// location. \p Msg is the message to show. Note that this class does not + /// copy this message, so this reference must be valid for the whole life time + /// of the diagnostic. + DiagnosticInfoUnsupported(const Function &Fn, const Twine &Msg, + DebugLoc DLoc = DebugLoc(), + DiagnosticSeverity Severity = DS_Error) + : DiagnosticInfoWithDebugLocBase(DK_Unsupported, Severity, Fn, DLoc), + Msg(Msg) {} + + static bool classof(const DiagnosticInfo *DI) { + return DI->getKind() == DK_Unsupported; + } + + const Twine &getMessage() const { return Msg; } + + void print(DiagnosticPrinter &DP) const override; +}; + /// Emit a warning when loop vectorization is specified but fails. \p Fn is the /// function triggering the warning, \p DLoc is the debug location where the /// diagnostic is generated. \p Msg is the message string to use. @@ -584,6 +703,6 @@ void emitLoopVectorizeWarning(LLVMContext &Ctx, const Function &Fn, void emitLoopInterleaveWarning(LLVMContext &Ctx, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg); -} // End namespace llvm +} // end namespace llvm -#endif +#endif // LLVM_IR_DIAGNOSTICINFO_H diff --git a/include/llvm/IR/Dominators.h b/include/llvm/IR/Dominators.h index 37447c353b19..f445a49b67b8 100644 --- a/include/llvm/IR/Dominators.h +++ b/include/llvm/IR/Dominators.h @@ -15,26 +15,19 @@ #ifndef LLVM_IR_DOMINATORS_H #define LLVM_IR_DOMINATORS_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" -#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" #include "llvm/Pass.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/GenericDomTree.h" -#include "llvm/Support/raw_ostream.h" -#include <algorithm> namespace llvm { -// FIXME: Replace this brittle forward declaration with the include of the new -// PassManager.h when doing so doesn't break the PassManagerBuilder. -template <typename IRUnitT> class AnalysisManager; -class PreservedAnalyses; +class Function; +class BasicBlock; +class raw_ostream; extern template class DomTreeNodeBase<BasicBlock>; extern template class DominatorTreeBase<BasicBlock>; @@ -62,6 +55,26 @@ public: bool isSingleEdge() const; }; +template <> struct DenseMapInfo<BasicBlockEdge> { + static unsigned getHashValue(const BasicBlockEdge *V); + typedef DenseMapInfo<const BasicBlock *> BBInfo; + static inline BasicBlockEdge getEmptyKey() { + return BasicBlockEdge(BBInfo::getEmptyKey(), BBInfo::getEmptyKey()); + } + static inline BasicBlockEdge getTombstoneKey() { + return BasicBlockEdge(BBInfo::getTombstoneKey(), BBInfo::getTombstoneKey()); + } + + static unsigned getHashValue(const BasicBlockEdge &Edge) { + return hash_combine(BBInfo::getHashValue(Edge.getStart()), + BBInfo::getHashValue(Edge.getEnd())); + } + static bool isEqual(const BasicBlockEdge &LHS, const BasicBlockEdge &RHS) { + return BBInfo::isEqual(LHS.getStart(), RHS.getStart()) && + BBInfo::isEqual(LHS.getEnd(), RHS.getEnd()); + } +}; + /// \brief Concrete subclass of DominatorTreeBase that is used to compute a /// normal dominator tree. /// @@ -186,40 +199,31 @@ template <> struct GraphTraits<DominatorTree*> }; /// \brief Analysis pass which computes a \c DominatorTree. -class DominatorTreeAnalysis { +class DominatorTreeAnalysis : public AnalysisInfoMixin<DominatorTreeAnalysis> { + friend AnalysisInfoMixin<DominatorTreeAnalysis>; + static char PassID; + public: /// \brief Provide the result typedef for this analysis pass. typedef DominatorTree Result; - /// \brief Opaque, unique identifier for this analysis pass. - static void *ID() { return (void *)&PassID; } - /// \brief Run the analysis pass over a function and produce a dominator tree. - DominatorTree run(Function &F); - - /// \brief Provide access to a name for this pass for debugging purposes. - static StringRef name() { return "DominatorTreeAnalysis"; } - -private: - static char PassID; + DominatorTree run(Function &F, AnalysisManager<Function> &); }; /// \brief Printer pass for the \c DominatorTree. -class DominatorTreePrinterPass { +class DominatorTreePrinterPass + : public PassInfoMixin<DominatorTreePrinterPass> { raw_ostream &OS; public: explicit DominatorTreePrinterPass(raw_ostream &OS); - PreservedAnalyses run(Function &F, AnalysisManager<Function> *AM); - - static StringRef name() { return "DominatorTreePrinterPass"; } + PreservedAnalyses run(Function &F, AnalysisManager<Function> &AM); }; /// \brief Verifier pass for the \c DominatorTree. -struct DominatorTreeVerifierPass { - PreservedAnalyses run(Function &F, AnalysisManager<Function> *AM); - - static StringRef name() { return "DominatorTreeVerifierPass"; } +struct DominatorTreeVerifierPass : PassInfoMixin<DominatorTreeVerifierPass> { + PreservedAnalyses run(Function &F, AnalysisManager<Function> &AM); }; /// \brief Legacy analysis pass which computes a \c DominatorTree. diff --git a/include/llvm/IR/Function.h b/include/llvm/IR/Function.h index 4f64caeade20..d7d27e7585c1 100644 --- a/include/llvm/IR/Function.h +++ b/include/llvm/IR/Function.h @@ -19,7 +19,6 @@ #define LLVM_IR_FUNCTION_H #include "llvm/ADT/iterator_range.h" -#include "llvm/ADT/Optional.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" @@ -30,6 +29,7 @@ namespace llvm { +template <typename T> class Optional; class FunctionType; class LLVMContext; class DISubprogram; @@ -56,7 +56,6 @@ private: mutable ArgumentListType ArgumentList; ///< The formal arguments ValueSymbolTable *SymTab; ///< Symbol table of args/instructions AttributeSet AttributeSets; ///< Parameter attributes - FunctionType *Ty; /* * Value::SubclassData @@ -73,13 +72,8 @@ private: /// Bits from GlobalObject::GlobalObjectSubclassData. enum { /// Whether this function is materializable. - IsMaterializableBit = 1 << 0, - HasMetadataHashEntryBit = 1 << 1 + IsMaterializableBit = 0, }; - void setGlobalObjectBit(unsigned Mask, bool Value) { - setGlobalObjectSubClassData((~Mask & getGlobalObjectSubClassData()) | - (Value ? Mask : 0u)); - } friend class SymbolTableListTraits<Function>; @@ -89,9 +83,12 @@ private: /// built on demand, so that the list isn't allocated until the first client /// needs it. The hasLazyArguments predicate returns true if the arg list /// hasn't been set up yet. +public: bool hasLazyArguments() const { return getSubclassDataFromValue() & (1<<0); } + +private: void CheckLazyArguments() const { if (hasLazyArguments()) BuildLazyArguments(); @@ -166,7 +163,7 @@ public: AttributeSet getAttributes() const { return AttributeSets; } /// @brief Set the attribute list for this Function. - void setAttributes(AttributeSet attrs) { AttributeSets = attrs; } + void setAttributes(AttributeSet Attrs) { AttributeSets = Attrs; } /// @brief Add function attributes to this function. void addFnAttr(Attribute::AttrKind N) { @@ -175,9 +172,9 @@ public: } /// @brief Remove function attributes from this function. - void removeFnAttr(Attribute::AttrKind N) { + void removeFnAttr(Attribute::AttrKind Kind) { setAttributes(AttributeSets.removeAttribute( - getContext(), AttributeSet::FunctionIndex, N)); + getContext(), AttributeSet::FunctionIndex, Kind)); } /// @brief Add function attributes to this function. @@ -200,7 +197,7 @@ public: /// @brief Return true if the function has the attribute. bool hasFnAttribute(Attribute::AttrKind Kind) const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, Kind); + return AttributeSets.hasFnAttribute(Kind); } bool hasFnAttribute(StringRef Kind) const { return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, Kind); @@ -208,14 +205,16 @@ public: /// @brief Return the attribute for the given attribute kind. Attribute getFnAttribute(Attribute::AttrKind Kind) const { - return AttributeSets.getAttribute(AttributeSet::FunctionIndex, Kind); + return getAttribute(AttributeSet::FunctionIndex, Kind); } Attribute getFnAttribute(StringRef Kind) const { - return AttributeSets.getAttribute(AttributeSet::FunctionIndex, Kind); + return getAttribute(AttributeSet::FunctionIndex, Kind); } /// \brief Return the stack alignment for the function. unsigned getFnStackAlignment() const { + if (!hasFnAttribute(Attribute::StackAlignment)) + return 0; return AttributeSets.getStackAlignment(AttributeSet::FunctionIndex); } @@ -225,17 +224,39 @@ public: return getSubclassDataFromValue() & (1<<14); } const std::string &getGC() const; - void setGC(const std::string Str); + void setGC(std::string Str); void clearGC(); /// @brief adds the attribute to the list of attributes. - void addAttribute(unsigned i, Attribute::AttrKind attr); + void addAttribute(unsigned i, Attribute::AttrKind Kind); + + /// @brief adds the attribute to the list of attributes. + void addAttribute(unsigned i, Attribute Attr); /// @brief adds the attributes to the list of attributes. - void addAttributes(unsigned i, AttributeSet attrs); + void addAttributes(unsigned i, AttributeSet Attrs); + + /// @brief removes the attribute from the list of attributes. + void removeAttribute(unsigned i, Attribute::AttrKind Kind); + + /// @brief removes the attribute from the list of attributes. + void removeAttribute(unsigned i, StringRef Kind); /// @brief removes the attributes from the list of attributes. - void removeAttributes(unsigned i, AttributeSet attr); + void removeAttributes(unsigned i, AttributeSet Attrs); + + /// @brief check if an attributes is in the list of attributes. + bool hasAttribute(unsigned i, Attribute::AttrKind Kind) const { + return getAttributes().hasAttribute(i, Kind); + } + + Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const { + return AttributeSets.getAttribute(i, Kind); + } + + Attribute getAttribute(unsigned i, StringRef Kind) const { + return AttributeSets.getAttribute(i, Kind); + } /// @brief adds the dereferenceable attribute to the list of attributes. void addDereferenceableAttr(unsigned i, uint64_t Bytes); @@ -263,8 +284,7 @@ public: /// @brief Determine if the function does not access memory. bool doesNotAccessMemory() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::ReadNone); + return hasFnAttribute(Attribute::ReadNone); } void setDoesNotAccessMemory() { addFnAttr(Attribute::ReadNone); @@ -272,27 +292,31 @@ public: /// @brief Determine if the function does not access or only reads memory. bool onlyReadsMemory() const { - return doesNotAccessMemory() || - AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::ReadOnly); + return doesNotAccessMemory() || hasFnAttribute(Attribute::ReadOnly); } void setOnlyReadsMemory() { addFnAttr(Attribute::ReadOnly); } + /// @brief Determine if the function does not access or only writes memory. + bool doesNotReadMemory() const { + return doesNotAccessMemory() || hasFnAttribute(Attribute::WriteOnly); + } + void setDoesNotReadMemory() { + addFnAttr(Attribute::WriteOnly); + } + /// @brief Determine if the call can access memmory only using pointers based /// on its arguments. bool onlyAccessesArgMemory() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::ArgMemOnly); + return hasFnAttribute(Attribute::ArgMemOnly); } void setOnlyAccessesArgMemory() { addFnAttr(Attribute::ArgMemOnly); } /// @brief Determine if the function may only access memory that is /// inaccessible from the IR. bool onlyAccessesInaccessibleMemory() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::InaccessibleMemOnly); + return hasFnAttribute(Attribute::InaccessibleMemOnly); } void setOnlyAccessesInaccessibleMemory() { addFnAttr(Attribute::InaccessibleMemOnly); @@ -301,8 +325,7 @@ public: /// @brief Determine if the function may only access memory that is // either inaccessible from the IR or pointed to by its arguments. bool onlyAccessesInaccessibleMemOrArgMem() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::InaccessibleMemOrArgMemOnly); + return hasFnAttribute(Attribute::InaccessibleMemOrArgMemOnly); } void setOnlyAccessesInaccessibleMemOrArgMem() { addFnAttr(Attribute::InaccessibleMemOrArgMemOnly); @@ -310,8 +333,7 @@ public: /// @brief Determine if the function cannot return. bool doesNotReturn() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::NoReturn); + return hasFnAttribute(Attribute::NoReturn); } void setDoesNotReturn() { addFnAttr(Attribute::NoReturn); @@ -319,8 +341,7 @@ public: /// @brief Determine if the function cannot unwind. bool doesNotThrow() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::NoUnwind); + return hasFnAttribute(Attribute::NoUnwind); } void setDoesNotThrow() { addFnAttr(Attribute::NoUnwind); @@ -328,8 +349,7 @@ public: /// @brief Determine if the call cannot be duplicated. bool cannotDuplicate() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::NoDuplicate); + return hasFnAttribute(Attribute::NoDuplicate); } void setCannotDuplicate() { addFnAttr(Attribute::NoDuplicate); @@ -337,18 +357,19 @@ public: /// @brief Determine if the call is convergent. bool isConvergent() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::Convergent); + return hasFnAttribute(Attribute::Convergent); } void setConvergent() { addFnAttr(Attribute::Convergent); } + void setNotConvergent() { + removeFnAttr(Attribute::Convergent); + } /// Determine if the function is known not to recurse, directly or /// indirectly. bool doesNotRecurse() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::NoRecurse); + return hasFnAttribute(Attribute::NoRecurse); } void setDoesNotRecurse() { addFnAttr(Attribute::NoRecurse); @@ -357,8 +378,7 @@ public: /// @brief True if the ABI mandates (or the user requested) that this /// function be in a unwind table. bool hasUWTable() const { - return AttributeSets.hasAttribute(AttributeSet::FunctionIndex, - Attribute::UWTable); + return hasFnAttribute(Attribute::UWTable); } void setHasUWTable() { addFnAttr(Attribute::UWTable); @@ -440,6 +460,12 @@ public: /// void eraseFromParent() override; + /// Steal arguments from another function. + /// + /// Drop this function's arguments and splice in the ones from \c Src. + /// Requires that this has no function body. + void stealArgumentListFrom(Function &Src); + /// Get the underlying elements of the Function... the basic block list is /// empty for external functions. /// @@ -547,6 +573,12 @@ public: Constant *getPrologueData() const; void setPrologueData(Constant *PrologueData); + /// Print the function to an output stream with an optional + /// AssemblyAnnotationWriter. + void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW = nullptr, + bool ShouldPreserveUseListOrder = false, + bool IsForDebug = false) const; + /// viewCFG - This function is meant for use from the debugger. You can just /// say 'call F->viewCFG()' and a ghostview window should pop up from the /// program, displaying the CFG of the current function with the code for each @@ -597,35 +629,6 @@ public: /// setjmp or other function that gcc recognizes as "returning twice". bool callsFunctionThatReturnsTwice() const; - /// \brief Check if this has any metadata. - bool hasMetadata() const { return hasMetadataHashEntry(); } - - /// \brief Get the current metadata attachment, if any. - /// - /// Returns \c nullptr if such an attachment is missing. - /// @{ - MDNode *getMetadata(unsigned KindID) const; - MDNode *getMetadata(StringRef Kind) const; - /// @} - - /// \brief Set a particular kind of metadata attachment. - /// - /// Sets the given attachment to \c MD, erasing it if \c MD is \c nullptr or - /// replacing it if it already exists. - /// @{ - void setMetadata(unsigned KindID, MDNode *MD); - void setMetadata(StringRef Kind, MDNode *MD); - /// @} - - /// \brief Get all current metadata attachments. - void - getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const; - - /// \brief Drop metadata not in the given list. - /// - /// Drop all metadata from \c this not included in \c KnownIDs. - void dropUnknownMetadata(ArrayRef<unsigned> KnownIDs); - /// \brief Set the attached subprogram. /// /// Calls \a setMetadata() with \a LLVMContext::MD_dbg. @@ -647,15 +650,6 @@ private: Value::setValueSubclassData(D); } void setValueSubclassDataBit(unsigned Bit, bool On); - - bool hasMetadataHashEntry() const { - return getGlobalObjectSubClassData() & HasMetadataHashEntryBit; - } - void setHasMetadataHashEntry(bool HasEntry) { - setGlobalObjectBit(HasMetadataHashEntryBit, HasEntry); - } - - void clearMetadata(); }; template <> diff --git a/include/llvm/IR/FunctionInfo.h b/include/llvm/IR/FunctionInfo.h deleted file mode 100644 index eba088a61bc0..000000000000 --- a/include/llvm/IR/FunctionInfo.h +++ /dev/null @@ -1,241 +0,0 @@ -//===-- llvm/FunctionInfo.h - Function Info Index ---------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -/// @file -/// FunctionInfo.h This file contains the declarations the classes that hold -/// the function info index and summary. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_IR_FUNCTIONINFO_H -#define LLVM_IR_FUNCTIONINFO_H - -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" - -namespace llvm { - -/// \brief Function summary information to aid decisions and implementation of -/// importing. -/// -/// This is a separate class from FunctionInfo to enable lazy reading of this -/// function summary information from the combined index file during imporing. -class FunctionSummary { -private: - /// \brief Path of module containing function IR, used to locate module when - /// importing this function. - /// - /// This is only used during parsing of the combined function index, or when - /// parsing the per-module index for creation of the combined function index, - /// not during writing of the per-module index which doesn't contain a - /// module path string table. - StringRef ModulePath; - - /// \brief Used to flag functions that have local linkage types and need to - /// have module identifier appended before placing into the combined - /// index, to disambiguate from other functions with the same name. - /// - /// This is only used in the per-module function index, as it is consumed - /// while creating the combined index. - bool IsLocalFunction; - - // The rest of the information is used to help decide whether importing - // is likely to be profitable. - // Other information will be added as the importing is tuned, such - // as hotness (when profile available), and other function characteristics. - - /// Number of instructions (ignoring debug instructions, e.g.) computed - /// during the initial compile step when the function index is first built. - unsigned InstCount; - -public: - /// Construct a summary object from summary data expected for all - /// summary records. - FunctionSummary(unsigned NumInsts) : InstCount(NumInsts) {} - - /// Set the path to the module containing this function, for use in - /// the combined index. - void setModulePath(StringRef ModPath) { ModulePath = ModPath; } - - /// Get the path to the module containing this function. - StringRef modulePath() const { return ModulePath; } - - /// Record whether this is a local function in the per-module index. - void setLocalFunction(bool IsLocal) { IsLocalFunction = IsLocal; } - - /// Check whether this was a local function, for use in creating - /// the combined index. - bool isLocalFunction() const { return IsLocalFunction; } - - /// Get the instruction count recorded for this function. - unsigned instCount() const { return InstCount; } -}; - -/// \brief Class to hold pointer to function summary and information required -/// for parsing it. -/// -/// For the per-module index, this holds the bitcode offset -/// of the corresponding function block. For the combined index, -/// after parsing of the \a ValueSymbolTable, this initially -/// holds the offset of the corresponding function summary bitcode -/// record. After parsing the associated summary information from the summary -/// block the \a FunctionSummary is populated and stored here. -class FunctionInfo { -private: - /// Function summary information used to help make ThinLTO importing - /// decisions. - std::unique_ptr<FunctionSummary> Summary; - - /// \brief The bitcode offset corresponding to either the associated - /// function's function body record, or its function summary record, - /// depending on whether this is a per-module or combined index. - /// - /// This bitcode offset is written to or read from the associated - /// \a ValueSymbolTable entry for the function. - /// For the per-module index this holds the bitcode offset of the - /// function's body record within bitcode module block in its module, - /// which is used during lazy function parsing or ThinLTO importing. - /// For the combined index this holds the offset of the corresponding - /// function summary record, to enable associating the combined index - /// VST records with the summary records. - uint64_t BitcodeIndex; - -public: - /// Constructor used during parsing of VST entries. - FunctionInfo(uint64_t FuncOffset) - : Summary(nullptr), BitcodeIndex(FuncOffset) {} - - /// Constructor used for per-module index bitcode writing. - FunctionInfo(uint64_t FuncOffset, - std::unique_ptr<FunctionSummary> FuncSummary) - : Summary(std::move(FuncSummary)), BitcodeIndex(FuncOffset) {} - - /// Record the function summary information parsed out of the function - /// summary block during parsing or combined index creation. - void setFunctionSummary(std::unique_ptr<FunctionSummary> FuncSummary) { - Summary = std::move(FuncSummary); - } - - /// Get the function summary recorded for this function. - FunctionSummary *functionSummary() const { return Summary.get(); } - - /// Get the bitcode index recorded for this function, depending on - /// the index type. - uint64_t bitcodeIndex() const { return BitcodeIndex; } - - /// Record the bitcode index for this function, depending on - /// the index type. - void setBitcodeIndex(uint64_t FuncOffset) { BitcodeIndex = FuncOffset; } -}; - -/// List of function info structures for a particular function name held -/// in the FunctionMap. Requires a vector in the case of multiple -/// COMDAT functions of the same name. -typedef std::vector<std::unique_ptr<FunctionInfo>> FunctionInfoList; - -/// Map from function name to corresponding function info structures. -typedef StringMap<FunctionInfoList> FunctionInfoMapTy; - -/// Type used for iterating through the function info map. -typedef FunctionInfoMapTy::const_iterator const_funcinfo_iterator; -typedef FunctionInfoMapTy::iterator funcinfo_iterator; - -/// String table to hold/own module path strings, which additionally holds the -/// module ID assigned to each module during the plugin step. The StringMap -/// makes a copy of and owns inserted strings. -typedef StringMap<uint64_t> ModulePathStringTableTy; - -/// Class to hold module path string table and function map, -/// and encapsulate methods for operating on them. -class FunctionInfoIndex { -private: - /// Map from function name to list of function information instances - /// for functions of that name (may be duplicates in the COMDAT case, e.g.). - FunctionInfoMapTy FunctionMap; - - /// Holds strings for combined index, mapping to the corresponding module ID. - ModulePathStringTableTy ModulePathStringTable; - -public: - FunctionInfoIndex() = default; - - // Disable the copy constructor and assignment operators, so - // no unexpected copying/moving occurs. - FunctionInfoIndex(const FunctionInfoIndex &) = delete; - void operator=(const FunctionInfoIndex &) = delete; - - funcinfo_iterator begin() { return FunctionMap.begin(); } - const_funcinfo_iterator begin() const { return FunctionMap.begin(); } - funcinfo_iterator end() { return FunctionMap.end(); } - const_funcinfo_iterator end() const { return FunctionMap.end(); } - - /// Get the list of function info objects for a given function. - const FunctionInfoList &getFunctionInfoList(StringRef FuncName) { - return FunctionMap[FuncName]; - } - - /// Get the list of function info objects for a given function. - const const_funcinfo_iterator findFunctionInfoList(StringRef FuncName) const { - return FunctionMap.find(FuncName); - } - - /// Add a function info for a function of the given name. - void addFunctionInfo(StringRef FuncName, std::unique_ptr<FunctionInfo> Info) { - FunctionMap[FuncName].push_back(std::move(Info)); - } - - /// Iterator to allow writer to walk through table during emission. - iterator_range<StringMap<uint64_t>::const_iterator> - modPathStringEntries() const { - return llvm::make_range(ModulePathStringTable.begin(), - ModulePathStringTable.end()); - } - - /// Get the module ID recorded for the given module path. - uint64_t getModuleId(const StringRef ModPath) const { - return ModulePathStringTable.lookup(ModPath); - } - - /// Add the given per-module index into this function index/summary, - /// assigning it the given module ID. Each module merged in should have - /// a unique ID, necessary for consistent renaming of promoted - /// static (local) variables. - void mergeFrom(std::unique_ptr<FunctionInfoIndex> Other, - uint64_t NextModuleId); - - /// Convenience method for creating a promoted global name - /// for the given value name of a local, and its original module's ID. - static std::string getGlobalNameForLocal(StringRef Name, uint64_t ModId) { - SmallString<256> NewName(Name); - NewName += ".llvm."; - raw_svector_ostream(NewName) << ModId; - return NewName.str(); - } - - /// Add a new module path, mapped to the given module Id, and return StringRef - /// owned by string table map. - StringRef addModulePath(StringRef ModPath, uint64_t ModId) { - return ModulePathStringTable.insert(std::make_pair(ModPath, ModId)) - .first->first(); - } - - /// Check if the given Module has any functions available for exporting - /// in the index. We consider any module present in the ModulePathStringTable - /// to have exported functions. - bool hasExportedFunctions(const Module &M) const { - return ModulePathStringTable.count(M.getModuleIdentifier()); - } -}; - -} // End llvm namespace - -#endif diff --git a/include/llvm/IR/GVMaterializer.h b/include/llvm/IR/GVMaterializer.h index 6cb593c7a3da..9e47722c892b 100644 --- a/include/llvm/IR/GVMaterializer.h +++ b/include/llvm/IR/GVMaterializer.h @@ -18,14 +18,12 @@ #ifndef LLVM_IR_GVMATERIALIZER_H #define LLVM_IR_GVMATERIALIZER_H -#include "llvm/ADT/DenseMap.h" #include <system_error> #include <vector> namespace llvm { class Function; class GlobalValue; -class Metadata; class Module; class StructType; @@ -47,14 +45,6 @@ public: virtual std::error_code materializeMetadata() = 0; virtual void setStripDebugInfo() = 0; - /// Client should define this interface if the mapping between metadata - /// values and value ids needs to be preserved, e.g. across materializer - /// instantiations. If OnlyTempMD is true, only those that have remained - /// temporary metadata are recorded in the map. - virtual void - saveMetadataList(DenseMap<const Metadata *, unsigned> &MetadataToIDs, - bool OnlyTempMD) {} - virtual std::vector<StructType *> getIdentifiedStructTypes() const = 0; }; diff --git a/include/llvm/IR/GetElementPtrTypeIterator.h b/include/llvm/IR/GetElementPtrTypeIterator.h index 7cb13fa33aa6..4953aebbe8aa 100644 --- a/include/llvm/IR/GetElementPtrTypeIterator.h +++ b/include/llvm/IR/GetElementPtrTypeIterator.h @@ -33,12 +33,6 @@ namespace llvm { generic_gep_type_iterator() {} public: - static generic_gep_type_iterator begin(Type *Ty, ItTy It) { - generic_gep_type_iterator I; - I.CurTy.setPointer(Ty); - I.OpIt = It; - return I; - } static generic_gep_type_iterator begin(Type *Ty, unsigned AddrSpace, ItTy It) { generic_gep_type_iterator I; @@ -125,13 +119,13 @@ namespace llvm { template<typename T> inline generic_gep_type_iterator<const T *> - gep_type_begin(Type *Op0, ArrayRef<T> A) { - return generic_gep_type_iterator<const T *>::begin(Op0, A.begin()); + gep_type_begin(Type *Op0, unsigned AS, ArrayRef<T> A) { + return generic_gep_type_iterator<const T *>::begin(Op0, AS, A.begin()); } template<typename T> inline generic_gep_type_iterator<const T *> - gep_type_end(Type * /*Op0*/, ArrayRef<T> A) { + gep_type_end(Type * /*Op0*/, unsigned /*AS*/, ArrayRef<T> A) { return generic_gep_type_iterator<const T *>::end(A.end()); } } // end namespace llvm diff --git a/include/llvm/IR/GlobalAlias.h b/include/llvm/IR/GlobalAlias.h index b0772143309f..3ae3e4a001e1 100644 --- a/include/llvm/IR/GlobalAlias.h +++ b/include/llvm/IR/GlobalAlias.h @@ -15,17 +15,17 @@ #ifndef LLVM_IR_GLOBALALIAS_H #define LLVM_IR_GLOBALALIAS_H -#include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist_node.h" -#include "llvm/IR/GlobalValue.h" -#include "llvm/IR/OperandTraits.h" +#include "llvm/IR/GlobalIndirectSymbol.h" namespace llvm { +class Twine; class Module; template <typename ValueSubClass> class SymbolTableListTraits; -class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> { +class GlobalAlias : public GlobalIndirectSymbol, + public ilist_node<GlobalAlias> { friend class SymbolTableListTraits<GlobalAlias>; void operator=(const GlobalAlias &) = delete; GlobalAlias(const GlobalAlias &) = delete; @@ -36,11 +36,6 @@ class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> { const Twine &Name, Constant *Aliasee, Module *Parent); public: - // allocate space for exactly one operand - void *operator new(size_t s) { - return User::operator new(s, 1); - } - /// If a parent module is specified, the alias is automatically inserted into /// the end of the specified module's alias list. static GlobalAlias *create(Type *Ty, unsigned AddressSpace, @@ -64,9 +59,6 @@ public: // Linkage, Type, Parent and AddressSpace taken from the Aliasee. static GlobalAlias *create(const Twine &Name, GlobalValue *Aliasee); - /// Provide fast operand accessors - DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); - /// removeFromParent - This method unlinks 'this' from the containing module, /// but does not delete it. /// @@ -77,28 +69,13 @@ public: /// void eraseFromParent() override; - /// These methods retrive and set alias target. + /// These methods retrieve and set alias target. void setAliasee(Constant *Aliasee); const Constant *getAliasee() const { - return const_cast<GlobalAlias *>(this)->getAliasee(); + return getIndirectSymbol(); } Constant *getAliasee() { - return getOperand(0); - } - - const GlobalObject *getBaseObject() const { - return const_cast<GlobalAlias *>(this)->getBaseObject(); - } - GlobalObject *getBaseObject() { - return dyn_cast<GlobalObject>(getAliasee()->stripInBoundsOffsets()); - } - - const GlobalObject *getBaseObject(const DataLayout &DL, APInt &Offset) const { - return const_cast<GlobalAlias *>(this)->getBaseObject(DL, Offset); - } - GlobalObject *getBaseObject(const DataLayout &DL, APInt &Offset) { - return dyn_cast<GlobalObject>( - getAliasee()->stripAndAccumulateInBoundsConstantOffsets(DL, Offset)); + return getIndirectSymbol(); } static bool isValidLinkage(LinkageTypes L) { @@ -112,13 +89,6 @@ public: } }; -template <> -struct OperandTraits<GlobalAlias> : - public FixedNumOperandTraits<GlobalAlias, 1> { -}; - -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GlobalAlias, Constant) - } // End llvm namespace #endif diff --git a/include/llvm/IR/GlobalIFunc.h b/include/llvm/IR/GlobalIFunc.h new file mode 100644 index 000000000000..0cbe882c58d8 --- /dev/null +++ b/include/llvm/IR/GlobalIFunc.h @@ -0,0 +1,76 @@ +//===-------- llvm/GlobalIFunc.h - GlobalIFunc class ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \brief +/// This file contains the declaration of the GlobalIFunc class, which +/// represents a single indirect function in the IR. Indirect function uses +/// ELF symbol type extension to mark that the address of a declaration should +/// be resolved at runtime by calling a resolver function. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_GLOBALIFUNC_H +#define LLVM_IR_GLOBALIFUNC_H + +#include "llvm/ADT/ilist_node.h" +#include "llvm/IR/GlobalIndirectSymbol.h" + +namespace llvm { + +class Twine; +class Module; + +// Traits class for using GlobalIFunc in symbol table in Module. +template <typename ValueSubClass> class SymbolTableListTraits; + +class GlobalIFunc final : public GlobalIndirectSymbol, + public ilist_node<GlobalIFunc> { + friend class SymbolTableListTraits<GlobalIFunc>; + void operator=(const GlobalIFunc &) = delete; + GlobalIFunc(const GlobalIFunc &) = delete; + + void setParent(Module *parent); + + GlobalIFunc(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, + const Twine &Name, Constant *Resolver, Module *Parent); + +public: + /// If a parent module is specified, the ifunc is automatically inserted into + /// the end of the specified module's ifunc list. + static GlobalIFunc *create(Type *Ty, unsigned AddressSpace, + LinkageTypes Linkage, const Twine &Name, + Constant *Resolver, Module *Parent); + + /// This method unlinks 'this' from the containing module, but does not + /// delete it. + void removeFromParent() final; + + /// This method unlinks 'this' from the containing module and deletes it. + void eraseFromParent() final; + + /// These methods retrieve and set ifunc resolver function. + void setResolver(Constant *Resolver) { + setIndirectSymbol(Resolver); + } + const Constant *getResolver() const { + return getIndirectSymbol(); + } + Constant *getResolver() { + return getIndirectSymbol(); + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Value *V) { + return V->getValueID() == Value::GlobalIFuncVal; + } +}; + +} // End llvm namespace + +#endif diff --git a/include/llvm/IR/GlobalIndirectSymbol.h b/include/llvm/IR/GlobalIndirectSymbol.h new file mode 100644 index 000000000000..8edb3d1dbf4b --- /dev/null +++ b/include/llvm/IR/GlobalIndirectSymbol.h @@ -0,0 +1,84 @@ +//===- llvm/GlobalIndirectSymbol.h - GlobalIndirectSymbol class -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains the declaration of the GlobalIndirectSymbol class, which +// is a base class for GlobalAlias and GlobalIFunc. It contains all common code +// for aliases and ifuncs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_GLOBALINDIRECTSYMBOL_H +#define LLVM_IR_GLOBALINDIRECTSYMBOL_H + +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/OperandTraits.h" + +namespace llvm { + +class GlobalIndirectSymbol : public GlobalValue { + void operator=(const GlobalIndirectSymbol &) = delete; + GlobalIndirectSymbol(const GlobalIndirectSymbol &) = delete; + +protected: + GlobalIndirectSymbol(Type *Ty, ValueTy VTy, unsigned AddressSpace, + LinkageTypes Linkage, const Twine &Name, Constant *Symbol); + +public: + // allocate space for exactly one operand + void *operator new(size_t s) { + return User::operator new(s, 1); + } + + /// Provide fast operand accessors + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Constant); + + /// These methods set and retrieve indirect symbol. + void setIndirectSymbol(Constant *Symbol) { + setOperand(0, Symbol); + } + const Constant *getIndirectSymbol() const { + return const_cast<GlobalIndirectSymbol *>(this)->getIndirectSymbol(); + } + Constant *getIndirectSymbol() { + return getOperand(0); + } + + const GlobalObject *getBaseObject() const { + return const_cast<GlobalIndirectSymbol *>(this)->getBaseObject(); + } + GlobalObject *getBaseObject() { + return dyn_cast<GlobalObject>(getIndirectSymbol()->stripInBoundsOffsets()); + } + + const GlobalObject *getBaseObject(const DataLayout &DL, APInt &Offset) const { + return const_cast<GlobalIndirectSymbol *>(this)->getBaseObject(DL, Offset); + } + GlobalObject *getBaseObject(const DataLayout &DL, APInt &Offset) { + return dyn_cast<GlobalObject>( + getIndirectSymbol()->stripAndAccumulateInBoundsConstantOffsets(DL, + Offset)); + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Value *V) { + return V->getValueID() == Value::GlobalAliasVal || + V->getValueID() == Value::GlobalIFuncVal; + } +}; + +template <> +struct OperandTraits<GlobalIndirectSymbol> : + public FixedNumOperandTraits<GlobalIndirectSymbol, 1> { +}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GlobalIndirectSymbol, Constant) + +} // End llvm namespace + +#endif diff --git a/include/llvm/IR/GlobalObject.h b/include/llvm/IR/GlobalObject.h index ee111a046d73..04737a045ae5 100644 --- a/include/llvm/IR/GlobalObject.h +++ b/include/llvm/IR/GlobalObject.h @@ -15,12 +15,13 @@ #ifndef LLVM_IR_GLOBALOBJECT_H #define LLVM_IR_GLOBALOBJECT_H -#include "llvm/IR/Constant.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalValue.h" namespace llvm { class Comdat; +class MDNode; +class Metadata; class Module; class GlobalObject : public GlobalValue { @@ -37,12 +38,19 @@ protected: std::string Section; // Section to emit this into, empty means default Comdat *ObjComdat; - static const unsigned AlignmentBits = 5; + enum { + LastAlignmentBit = 4, + HasMetadataHashEntryBit, + + GlobalObjectBits, + }; static const unsigned GlobalObjectSubClassDataBits = - GlobalValueSubClassDataBits - AlignmentBits; + GlobalValueSubClassDataBits - GlobalObjectBits; private: + static const unsigned AlignmentBits = LastAlignmentBit + 1; static const unsigned AlignmentMask = (1 << AlignmentBits) - 1; + static const unsigned GlobalObjectMask = (1 << GlobalObjectBits) - 1; public: unsigned getAlignment() const { @@ -55,8 +63,8 @@ public: unsigned getGlobalObjectSubClassData() const; void setGlobalObjectSubClassData(unsigned Val); - bool hasSection() const { return !StringRef(getSection()).empty(); } - const char *getSection() const { return Section.c_str(); } + bool hasSection() const { return !getSection().empty(); } + StringRef getSection() const { return Section; } void setSection(StringRef S); bool hasComdat() const { return getComdat() != nullptr; } @@ -64,6 +72,54 @@ public: Comdat *getComdat() { return ObjComdat; } void setComdat(Comdat *C) { ObjComdat = C; } + /// Check if this has any metadata. + bool hasMetadata() const { return hasMetadataHashEntry(); } + + /// Get the current metadata attachments for the given kind, if any. + /// + /// These functions require that the function have at most a single attachment + /// of the given kind, and return \c nullptr if such an attachment is missing. + /// @{ + MDNode *getMetadata(unsigned KindID) const; + MDNode *getMetadata(StringRef Kind) const; + /// @} + + /// Appends all attachments with the given ID to \c MDs in insertion order. + /// If the global has no attachments with the given ID, or if ID is invalid, + /// leaves MDs unchanged. + /// @{ + void getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const; + void getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const; + /// @} + + /// Set a particular kind of metadata attachment. + /// + /// Sets the given attachment to \c MD, erasing it if \c MD is \c nullptr or + /// replacing it if it already exists. + /// @{ + void setMetadata(unsigned KindID, MDNode *MD); + void setMetadata(StringRef Kind, MDNode *MD); + /// @} + + /// Add a metadata attachment. + /// @{ + void addMetadata(unsigned KindID, MDNode &MD); + void addMetadata(StringRef Kind, MDNode &MD); + /// @} + + /// Appends all attachments for the global to \c MDs, sorting by attachment + /// ID. Attachments with the same ID appear in insertion order. + void + getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const; + + /// Erase all metadata attachments with the given kind. + void eraseMetadata(unsigned KindID); + + /// Copy metadata from Src, adjusting offsets by Offset. + void copyMetadata(const GlobalObject *Src, unsigned Offset); + + void addTypeMetadata(unsigned Offset, Metadata *TypeID); + void copyAttributesFrom(const GlobalValue *Src) override; // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -71,6 +127,18 @@ public: return V->getValueID() == Value::FunctionVal || V->getValueID() == Value::GlobalVariableVal; } + + void clearMetadata(); + +private: + bool hasMetadataHashEntry() const { + return getGlobalValueSubClassData() & (1 << HasMetadataHashEntryBit); + } + void setHasMetadataHashEntry(bool HasEntry) { + unsigned Mask = 1 << HasMetadataHashEntryBit; + setGlobalValueSubClassData((~Mask & getGlobalValueSubClassData()) | + (HasEntry ? Mask : 0u)); + } }; } // End llvm namespace diff --git a/include/llvm/IR/GlobalValue.h b/include/llvm/IR/GlobalValue.h index fa6469aa0ade..09682f7aa349 100644 --- a/include/llvm/IR/GlobalValue.h +++ b/include/llvm/IR/GlobalValue.h @@ -20,6 +20,7 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/Support/MD5.h" #include <system_error> namespace llvm { @@ -69,17 +70,18 @@ protected: LinkageTypes Linkage, const Twine &Name, unsigned AddressSpace) : Constant(PointerType::get(Ty, AddressSpace), VTy, Ops, NumOps), ValueType(Ty), Linkage(Linkage), Visibility(DefaultVisibility), - UnnamedAddr(0), DllStorageClass(DefaultStorageClass), - ThreadLocal(NotThreadLocal), IntID((Intrinsic::ID)0U), Parent(nullptr) { + UnnamedAddrVal(unsigned(UnnamedAddr::None)), + DllStorageClass(DefaultStorageClass), ThreadLocal(NotThreadLocal), + IntID((Intrinsic::ID)0U), Parent(nullptr) { setName(Name); } Type *ValueType; - // Note: VC++ treats enums as signed, so an extra bit is required to prevent - // Linkage and Visibility from turning into negative values. - LinkageTypes Linkage : 5; // The linkage of this global + // All bitfields use unsigned as the underlying type so that MSVC will pack + // them. + unsigned Linkage : 4; // The linkage of this global unsigned Visibility : 2; // The visibility style of this global - unsigned UnnamedAddr : 1; // This value's address is not significant + unsigned UnnamedAddrVal : 2; // This value's address is not significant unsigned DllStorageClass : 2; // DLL storage class unsigned ThreadLocal : 3; // Is this symbol "Thread Local", if so, what is @@ -88,12 +90,36 @@ protected: private: // Give subclasses access to what otherwise would be wasted padding. - // (19 + 3 + 2 + 1 + 2 + 5) == 32. + // (19 + 4 + 2 + 2 + 2 + 3) == 32. unsigned SubClassData : GlobalValueSubClassDataBits; friend class Constant; void destroyConstantImpl(); - Value *handleOperandChangeImpl(Value *From, Value *To, Use *U); + Value *handleOperandChangeImpl(Value *From, Value *To); + + /// Returns true if the definition of this global may be replaced by a + /// differently optimized variant of the same source level function at link + /// time. + bool mayBeDerefined() const { + switch (getLinkage()) { + case WeakODRLinkage: + case LinkOnceODRLinkage: + case AvailableExternallyLinkage: + return true; + + case WeakAnyLinkage: + case LinkOnceAnyLinkage: + case CommonLinkage: + case ExternalWeakLinkage: + case ExternalLinkage: + case AppendingLinkage: + case InternalLinkage: + case PrivateLinkage: + return isInterposable(); + } + + llvm_unreachable("Fully covered switch above!"); + } protected: /// \brief The intrinsic ID for this subclass (which must be a Function). @@ -128,8 +154,37 @@ public: unsigned getAlignment() const; - bool hasUnnamedAddr() const { return UnnamedAddr; } - void setUnnamedAddr(bool Val) { UnnamedAddr = Val; } + enum class UnnamedAddr { + None, + Local, + Global, + }; + + bool hasGlobalUnnamedAddr() const { + return getUnnamedAddr() == UnnamedAddr::Global; + } + + /// Returns true if this value's address is not significant in this module. + /// This attribute is intended to be used only by the code generator and LTO + /// to allow the linker to decide whether the global needs to be in the symbol + /// table. It should probably not be used in optimizations, as the value may + /// have uses outside the module; use hasGlobalUnnamedAddr() instead. + bool hasAtLeastLocalUnnamedAddr() const { + return getUnnamedAddr() != UnnamedAddr::None; + } + + UnnamedAddr getUnnamedAddr() const { + return UnnamedAddr(UnnamedAddrVal); + } + void setUnnamedAddr(UnnamedAddr Val) { UnnamedAddrVal = unsigned(Val); } + + static UnnamedAddr getMinUnnamedAddr(UnnamedAddr A, UnnamedAddr B) { + if (A == UnnamedAddr::None || B == UnnamedAddr::None) + return UnnamedAddr::None; + if (A == UnnamedAddr::Local || B == UnnamedAddr::Local) + return UnnamedAddr::Local; + return UnnamedAddr::Global; + } bool hasComdat() const { return getComdat() != nullptr; } Comdat *getComdat(); @@ -173,14 +228,8 @@ public: } void setDLLStorageClass(DLLStorageClassTypes C) { DllStorageClass = C; } - bool hasSection() const { return !StringRef(getSection()).empty(); } - // It is unfortunate that we have to use "char *" in here since this is - // always non NULL, but: - // * The C API expects a null terminated string, so we cannot use StringRef. - // * The C API expects us to own it, so we cannot use a std:string. - // * For GlobalAliases we can fail to find the section and we have to - // return "", so we cannot use a "const std::string &". - const char *getSection() const; + bool hasSection() const { return !getSection().empty(); } + StringRef getSection() const; /// Global values are always pointers. PointerType *getType() const { return cast<PointerType>(User::getType()); } @@ -233,6 +282,34 @@ public: static bool isCommonLinkage(LinkageTypes Linkage) { return Linkage == CommonLinkage; } + static bool isValidDeclarationLinkage(LinkageTypes Linkage) { + return isExternalWeakLinkage(Linkage) || isExternalLinkage(Linkage); + } + + /// Whether the definition of this global may be replaced by something + /// non-equivalent at link time. For example, if a function has weak linkage + /// then the code defining it may be replaced by different code. + static bool isInterposableLinkage(LinkageTypes Linkage) { + switch (Linkage) { + case WeakAnyLinkage: + case LinkOnceAnyLinkage: + case CommonLinkage: + case ExternalWeakLinkage: + return true; + + case AvailableExternallyLinkage: + case LinkOnceODRLinkage: + case WeakODRLinkage: + // The above three cannot be overridden but can be de-refined. + + case ExternalLinkage: + case AppendingLinkage: + case InternalLinkage: + case PrivateLinkage: + return false; + } + llvm_unreachable("Fully covered switch above!"); + } /// Whether the definition of this global may be discarded if it is not used /// in its compilation unit. @@ -241,17 +318,9 @@ public: isAvailableExternallyLinkage(Linkage); } - /// Whether the definition of this global may be replaced by something - /// non-equivalent at link time. For example, if a function has weak linkage - /// then the code defining it may be replaced by different code. - static bool mayBeOverridden(LinkageTypes Linkage) { - return Linkage == WeakAnyLinkage || Linkage == LinkOnceAnyLinkage || - Linkage == CommonLinkage || Linkage == ExternalWeakLinkage; - } - /// Whether the definition of this global may be replaced at link time. NB: /// Using this method outside of the code generators is almost always a - /// mistake: when working at the IR level use mayBeOverridden instead as it + /// mistake: when working at the IR level use isInterposable instead as it /// knows about ODR semantics. static bool isWeakForLinker(LinkageTypes Linkage) { return Linkage == WeakAnyLinkage || Linkage == WeakODRLinkage || @@ -259,44 +328,87 @@ public: Linkage == CommonLinkage || Linkage == ExternalWeakLinkage; } - bool hasExternalLinkage() const { return isExternalLinkage(Linkage); } + /// Return true if the currently visible definition of this global (if any) is + /// exactly the definition we will see at runtime. + /// + /// Non-exact linkage types inhibits most non-inlining IPO, since a + /// differently optimized variant of the same function can have different + /// observable or undefined behavior than in the variant currently visible. + /// For instance, we could have started with + /// + /// void foo(int *v) { + /// int t = 5 / v[0]; + /// (void) t; + /// } + /// + /// and "refined" it to + /// + /// void foo(int *v) { } + /// + /// However, we cannot infer readnone for `foo`, since that would justify + /// DSE'ing a store to `v[0]` across a call to `foo`, which can cause + /// undefined behavior if the linker replaces the actual call destination with + /// the unoptimized `foo`. + /// + /// Inlining is okay across non-exact linkage types as long as they're not + /// interposable (see \c isInterposable), since in such cases the currently + /// visible variant is *a* correct implementation of the original source + /// function; it just isn't the *only* correct implementation. + bool isDefinitionExact() const { + return !mayBeDerefined(); + } + + /// Return true if this global has an exact defintion. + bool hasExactDefinition() const { + // While this computes exactly the same thing as + // isStrongDefinitionForLinker, the intended uses are different. This + // function is intended to help decide if specific inter-procedural + // transforms are correct, while isStrongDefinitionForLinker's intended use + // is in low level code generation. + return !isDeclaration() && isDefinitionExact(); + } + + /// Return true if this global's definition can be substituted with an + /// *arbitrary* definition at link time. We cannot do any IPO or inlinining + /// across interposable call edges, since the callee can be replaced with + /// something arbitrary at link time. + bool isInterposable() const { return isInterposableLinkage(getLinkage()); } + + bool hasExternalLinkage() const { return isExternalLinkage(getLinkage()); } bool hasAvailableExternallyLinkage() const { - return isAvailableExternallyLinkage(Linkage); - } - bool hasLinkOnceLinkage() const { - return isLinkOnceLinkage(Linkage); + return isAvailableExternallyLinkage(getLinkage()); } - bool hasLinkOnceODRLinkage() const { return isLinkOnceODRLinkage(Linkage); } - bool hasWeakLinkage() const { - return isWeakLinkage(Linkage); + bool hasLinkOnceLinkage() const { return isLinkOnceLinkage(getLinkage()); } + bool hasLinkOnceODRLinkage() const { + return isLinkOnceODRLinkage(getLinkage()); } - bool hasWeakAnyLinkage() const { - return isWeakAnyLinkage(Linkage); + bool hasWeakLinkage() const { return isWeakLinkage(getLinkage()); } + bool hasWeakAnyLinkage() const { return isWeakAnyLinkage(getLinkage()); } + bool hasWeakODRLinkage() const { return isWeakODRLinkage(getLinkage()); } + bool hasAppendingLinkage() const { return isAppendingLinkage(getLinkage()); } + bool hasInternalLinkage() const { return isInternalLinkage(getLinkage()); } + bool hasPrivateLinkage() const { return isPrivateLinkage(getLinkage()); } + bool hasLocalLinkage() const { return isLocalLinkage(getLinkage()); } + bool hasExternalWeakLinkage() const { + return isExternalWeakLinkage(getLinkage()); } - bool hasWeakODRLinkage() const { - return isWeakODRLinkage(Linkage); + bool hasCommonLinkage() const { return isCommonLinkage(getLinkage()); } + bool hasValidDeclarationLinkage() const { + return isValidDeclarationLinkage(getLinkage()); } - bool hasAppendingLinkage() const { return isAppendingLinkage(Linkage); } - bool hasInternalLinkage() const { return isInternalLinkage(Linkage); } - bool hasPrivateLinkage() const { return isPrivateLinkage(Linkage); } - bool hasLocalLinkage() const { return isLocalLinkage(Linkage); } - bool hasExternalWeakLinkage() const { return isExternalWeakLinkage(Linkage); } - bool hasCommonLinkage() const { return isCommonLinkage(Linkage); } void setLinkage(LinkageTypes LT) { if (isLocalLinkage(LT)) Visibility = DefaultVisibility; Linkage = LT; } - LinkageTypes getLinkage() const { return Linkage; } + LinkageTypes getLinkage() const { return LinkageTypes(Linkage); } bool isDiscardableIfUnused() const { - return isDiscardableIfUnused(Linkage); + return isDiscardableIfUnused(getLinkage()); } - bool mayBeOverridden() const { return mayBeOverridden(Linkage); } - - bool isWeakForLinker() const { return isWeakForLinker(Linkage); } + bool isWeakForLinker() const { return isWeakForLinker(getLinkage()); } /// Copy all additional attributes (those not needed to create a GlobalValue) /// from the GlobalValue Src to this one. @@ -311,11 +423,37 @@ public: return Name; } -/// @name Materialization -/// Materialization is used to construct functions only as they're needed. This -/// is useful to reduce memory usage in LLVM or parsing work done by the -/// BitcodeReader to load the Module. -/// @{ + /// Return the modified name for a global value suitable to be + /// used as the key for a global lookup (e.g. profile or ThinLTO). + /// The value's original name is \c Name and has linkage of type + /// \c Linkage. The value is defined in module \c FileName. + static std::string getGlobalIdentifier(StringRef Name, + GlobalValue::LinkageTypes Linkage, + StringRef FileName); + + /// Return the modified name for this global value suitable to be + /// used as the key for a global lookup (e.g. profile or ThinLTO). + std::string getGlobalIdentifier() const; + + /// Declare a type to represent a global unique identifier for a global value. + /// This is a 64 bits hash that is used by PGO and ThinLTO to have a compact + /// unique way to identify a symbol. + using GUID = uint64_t; + + /// Return a 64-bit global unique ID constructed from global value name + /// (i.e. returned by getGlobalIdentifier()). + static GUID getGUID(StringRef GlobalName) { return MD5Hash(GlobalName); } + + /// Return a 64-bit global unique ID constructed from global value name + /// (i.e. returned by getGlobalIdentifier()). + GUID getGUID() const { return getGUID(getGlobalIdentifier()); } + + /// @name Materialization + /// Materialization is used to construct functions only as they're needed. + /// This + /// is useful to reduce memory usage in LLVM or parsing work done by the + /// BitcodeReader to load the Module. + /// @{ /// If this function's Module is being lazily streamed in functions from disk /// or some other source, this method can be used to check to see if the @@ -342,6 +480,10 @@ public: /// Returns true if this global's definition will be the one chosen by the /// linker. + /// + /// NB! Ideally this should not be used at the IR level at all. If you're + /// interested in optimization constraints implied by the linker's ability to + /// choose an implementation, prefer using \c hasExactDefinition. bool isStrongDefinitionForLinker() const { return !(isDeclarationForLinker() || isWeakForLinker()); } @@ -365,7 +507,8 @@ public: static bool classof(const Value *V) { return V->getValueID() == Value::FunctionVal || V->getValueID() == Value::GlobalVariableVal || - V->getValueID() == Value::GlobalAliasVal; + V->getValueID() == Value::GlobalAliasVal || + V->getValueID() == Value::GlobalIFuncVal; } }; diff --git a/include/llvm/IR/GlobalVariable.h b/include/llvm/IR/GlobalVariable.h index 342bdc01bfbd..ebeb635468d0 100644 --- a/include/llvm/IR/GlobalVariable.h +++ b/include/llvm/IR/GlobalVariable.h @@ -65,6 +65,8 @@ public: bool isExternallyInitialized = false); ~GlobalVariable() override { + dropAllReferences(); + // FIXME: needed by operator delete setGlobalVariableNumOperands(1); } @@ -94,9 +96,9 @@ public: /// unique. inline bool hasDefinitiveInitializer() const { return hasInitializer() && - // The initializer of a global variable with weak linkage may change at - // link time. - !mayBeOverridden() && + // The initializer of a global variable may change to something arbitrary + // at link time. + !isInterposable() && // The initializer of a global variable with the externally_initialized // marker may change at runtime before C++ initializers are evaluated. !isExternallyInitialized(); @@ -159,6 +161,10 @@ public: /// void eraseFromParent() override; + /// Drop all references in preparation to destroy the GlobalVariable. This + /// drops not only the reference to the initializer but also to any metadata. + void dropAllReferences(); + // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Value *V) { return V->getValueID() == Value::GlobalVariableVal; diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index 2f8c3c499295..016e9e1d2c50 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -16,36 +16,54 @@ #define LLVM_IR_IRBUILDER_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/ConstantFolder.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/Casting.h" +#include "llvm-c/Types.h" +#include <cassert> +#include <cstddef> +#include <cstdint> namespace llvm { + +class APInt; class MDNode; +class Module; +class Use; /// \brief This provides the default implementation of the IRBuilder /// 'InsertHelper' method that is called whenever an instruction is created by /// IRBuilder and needs to be inserted. /// /// By default, this inserts the instruction at the insertion point. -template <bool preserveNames = true> class IRBuilderDefaultInserter { protected: void InsertHelper(Instruction *I, const Twine &Name, BasicBlock *BB, BasicBlock::iterator InsertPt) const { if (BB) BB->getInstList().insert(InsertPt, I); - if (preserveNames) - I->setName(Name); + I->setName(Name); } }; @@ -436,6 +454,16 @@ public: CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align, Value *Mask); + /// \brief Create a call to Masked Gather intrinsic + CallInst *CreateMaskedGather(Value *Ptrs, unsigned Align, + Value *Mask = nullptr, + Value *PassThru = nullptr, + const Twine& Name = ""); + + /// \brief Create a call to Masked Scatter intrinsic + CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, unsigned Align, + Value *Mask = nullptr); + /// \brief Create an assume intrinsic call that allows the optimizer to /// assume that the provided condition will be true. CallInst *CreateAssumption(Value *Cond); @@ -512,9 +540,9 @@ public: private: /// \brief Create a call to a masked intrinsic with given Id. - /// Masked intrinsic has only one overloaded type - data type. CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops, - Type *DataTy, const Twine &Name = ""); + ArrayRef<Type *> OverloadedTypes, + const Twine &Name = ""); Value *getCastedInt8PtrValue(Value *Ptr); }; @@ -529,14 +557,12 @@ private: /// created. Convenience state exists to specify fast-math flags and fp-math /// tags. /// -/// The first template argument handles whether or not to preserve names in the -/// final instruction output. This defaults to on. The second template argument -/// specifies a class to use for creating constants. This defaults to creating -/// minimally folded constants. The third template argument allows clients to -/// specify custom insertion hooks that are called on every newly created -/// insertion. -template<bool preserveNames = true, typename T = ConstantFolder, - typename Inserter = IRBuilderDefaultInserter<preserveNames> > +/// The first template argument specifies a class to use for creating constants. +/// This defaults to creating minimally folded constants. The second template +/// argument allows clients to specify custom insertion hooks that are called on +/// every newly created insertion. +template <typename T = ConstantFolder, + typename Inserter = IRBuilderDefaultInserter> class IRBuilder : public IRBuilderBase, public Inserter { T Folder; @@ -586,10 +612,6 @@ public: /// \brief Get the constant folder being used. const T &getFolder() { return Folder; } - /// \brief Return true if this builder is configured to actually add the - /// requested names to IR created through it. - bool isNamePreserving() const { return preserveNames; } - /// \brief Insert and return the specified instruction. template<typename InstTy> InstTy *Insert(InstTy *I, const Twine &Name = "") const { @@ -676,28 +698,10 @@ public: return Insert(IndirectBrInst::Create(Addr, NumDests)); } - InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, - BasicBlock *UnwindDest, const Twine &Name = "") { - return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, None), - Name); - } - InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, - BasicBlock *UnwindDest, Value *Arg1, - const Twine &Name = "") { - return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, Arg1), - Name); - } - InvokeInst *CreateInvoke3(Value *Callee, BasicBlock *NormalDest, - BasicBlock *UnwindDest, Value *Arg1, - Value *Arg2, Value *Arg3, - const Twine &Name = "") { - Value *Args[] = { Arg1, Arg2, Arg3 }; - return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, Args), - Name); - } /// \brief Create an invoke instruction. InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, - BasicBlock *UnwindDest, ArrayRef<Value *> Args, + BasicBlock *UnwindDest, + ArrayRef<Value *> Args = None, const Twine &Name = "") { return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, Args), Name); @@ -1569,12 +1573,19 @@ public: } Value *CreateSelect(Value *C, Value *True, Value *False, - const Twine &Name = "") { + const Twine &Name = "", Instruction *MDFrom = nullptr) { if (Constant *CC = dyn_cast<Constant>(C)) if (Constant *TC = dyn_cast<Constant>(True)) if (Constant *FC = dyn_cast<Constant>(False)) return Insert(Folder.CreateSelect(CC, TC, FC), Name); - return Insert(SelectInst::Create(C, True, False), Name); + + SelectInst *Sel = SelectInst::Create(C, True, False); + if (MDFrom) { + MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); + MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); + Sel = addBranchMetadata(Sel, Prof, Unpred); + } + return Insert(Sel, Name); } VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") { @@ -1617,13 +1628,9 @@ public: return Insert(new ShuffleVectorInst(V1, V2, Mask), Name); } - Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> IntMask, + Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask, const Twine &Name = "") { - size_t MaskSize = IntMask.size(); - SmallVector<Constant*, 8> MaskVec(MaskSize); - for (size_t i = 0; i != MaskSize; ++i) - MaskVec[i] = getInt32(IntMask[i]); - Value *Mask = ConstantVector::get(MaskVec); + Value *Mask = ConstantDataVector::get(Context, IntMask); return CreateShuffleVector(V1, V2, Mask, Name); } diff --git a/include/llvm/IR/IRPrintingPasses.h b/include/llvm/IR/IRPrintingPasses.h index 88b18e826daf..bc6de19a6c3a 100644 --- a/include/llvm/IR/IRPrintingPasses.h +++ b/include/llvm/IR/IRPrintingPasses.h @@ -30,6 +30,7 @@ class Module; class ModulePass; class PreservedAnalyses; class raw_ostream; +template <typename IRUnitT> class AnalysisManager; /// \brief Create and return a pass that writes the module to the specified /// \c raw_ostream. @@ -67,7 +68,7 @@ public: PrintModulePass(raw_ostream &OS, const std::string &Banner = "", bool ShouldPreserveUseListOrder = false); - PreservedAnalyses run(Module &M); + PreservedAnalyses run(Module &M, AnalysisManager<Module> &); static StringRef name() { return "PrintModulePass"; } }; @@ -84,7 +85,7 @@ public: PrintFunctionPass(); PrintFunctionPass(raw_ostream &OS, const std::string &Banner = ""); - PreservedAnalyses run(Function &F); + PreservedAnalyses run(Function &F, AnalysisManager<Function> &); static StringRef name() { return "PrintFunctionPass"; } }; diff --git a/include/llvm/IR/InlineAsm.h b/include/llvm/IR/InlineAsm.h index d2e9e48539ce..40ba830b8819 100644 --- a/include/llvm/IR/InlineAsm.h +++ b/include/llvm/IR/InlineAsm.h @@ -223,6 +223,7 @@ public: Extra_AsmDialect = 4, Extra_MayLoad = 8, Extra_MayStore = 16, + Extra_IsConvergent = 32, // Inline asm operands map to multiple SDNode / MachineInstr operands. // The first operand is an immediate describing the asm operand, the low @@ -271,6 +272,16 @@ public: return Kind | (NumOps << 3); } + static bool isRegDefKind(unsigned Flag){ return getKind(Flag) == Kind_RegDef;} + static bool isImmKind(unsigned Flag) { return getKind(Flag) == Kind_Imm; } + static bool isMemKind(unsigned Flag) { return getKind(Flag) == Kind_Mem; } + static bool isRegDefEarlyClobberKind(unsigned Flag) { + return getKind(Flag) == Kind_RegDefEarlyClobber; + } + static bool isClobberKind(unsigned Flag) { + return getKind(Flag) == Kind_Clobber; + } + /// getFlagWordForMatchingOp - Augment an existing flag word returned by /// getFlagWord with information indicating that this input operand is tied /// to a previous output operand. @@ -289,6 +300,8 @@ public: static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC) { // Store RC + 1, reserve the value 0 to mean 'no register class'. ++RC; + assert(!isImmKind(InputFlag) && "Immediates cannot have a register class"); + assert(!isMemKind(InputFlag) && "Memory operand cannot have a register class"); assert(RC <= 0x7fff && "Too large register class ID"); assert((InputFlag & ~0xffff) == 0 && "High bits already contain data"); return InputFlag | (RC << 16); @@ -297,6 +310,7 @@ public: /// Augment an existing flag word returned by getFlagWord with the constraint /// code for a memory constraint. static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint) { + assert(isMemKind(InputFlag) && "InputFlag is not a memory constraint!"); assert(Constraint <= 0x7fff && "Too large a memory constraint ID"); assert(Constraint <= Constraints_Max && "Unknown constraint ID"); assert((InputFlag & ~0xffff) == 0 && "High bits already contain data"); @@ -312,16 +326,6 @@ public: return Flags & 7; } - static bool isRegDefKind(unsigned Flag){ return getKind(Flag) == Kind_RegDef;} - static bool isImmKind(unsigned Flag) { return getKind(Flag) == Kind_Imm; } - static bool isMemKind(unsigned Flag) { return getKind(Flag) == Kind_Mem; } - static bool isRegDefEarlyClobberKind(unsigned Flag) { - return getKind(Flag) == Kind_RegDefEarlyClobber; - } - static bool isClobberKind(unsigned Flag) { - return getKind(Flag) == Kind_Clobber; - } - static unsigned getMemoryConstraintID(unsigned Flag) { assert(isMemKind(Flag)); return (Flag >> Constraints_ShiftAmount) & 0x7fff; diff --git a/include/llvm/IR/InstrTypes.h b/include/llvm/IR/InstrTypes.h index 5091bb407833..39514c5675a7 100644 --- a/include/llvm/IR/InstrTypes.h +++ b/include/llvm/IR/InstrTypes.h @@ -386,6 +386,15 @@ public: } #include "llvm/IR/Instruction.def" + static BinaryOperator *CreateWithCopiedFlags(BinaryOps Opc, + Value *V1, Value *V2, + BinaryOperator *CopyBO, + const Twine &Name = "") { + BinaryOperator *BO = Create(Opc, V1, V2, Name); + BO->copyIRFlags(CopyBO); + return BO; + } + static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name = "") { BinaryOperator *BO = Create(Opc, V1, V2, Name); @@ -526,35 +535,6 @@ public: /// bool swapOperands(); - /// Set or clear the nsw flag on this instruction, which must be an operator - /// which supports this flag. See LangRef.html for the meaning of this flag. - void setHasNoUnsignedWrap(bool b = true); - - /// Set or clear the nsw flag on this instruction, which must be an operator - /// which supports this flag. See LangRef.html for the meaning of this flag. - void setHasNoSignedWrap(bool b = true); - - /// Set or clear the exact flag on this instruction, which must be an operator - /// which supports this flag. See LangRef.html for the meaning of this flag. - void setIsExact(bool b = true); - - /// Determine whether the no unsigned wrap flag is set. - bool hasNoUnsignedWrap() const; - - /// Determine whether the no signed wrap flag is set. - bool hasNoSignedWrap() const; - - /// Determine whether the exact flag is set. - bool isExact() const; - - /// Convenience method to copy supported wrapping, exact, and fast-math flags - /// from V to this instruction. - void copyIRFlags(const Value *V); - - /// Logical 'and' of any supported wrapping, exact, and fast-math flags of - /// V and this instruction. - void andIRFlags(const Value *V); - // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return I->isBinaryOp(); @@ -879,6 +859,10 @@ public: /// Values in the range 0-31 are reserved for FCmpInst, while values in the /// range 32-64 are reserved for ICmpInst. This is necessary to ensure the /// predicate values are not overlapping between the classes. + /// + /// Some passes (e.g. InstCombine) depend on the bit-wise characteristics of + /// FCMP_* values. Changing the bit patterns requires a potential change to + /// those passes. enum Predicate { // Opcode U L G E Intuitive operation FCMP_FALSE = 0, ///< 0 0 0 0 Always false (always folded) @@ -1058,6 +1042,18 @@ public: return isFalseWhenEqual(getPredicate()); } + /// @brief Determine if Pred1 implies Pred2 is true when two compares have + /// matching operands. + bool isImpliedTrueByMatchingCmp(Predicate Pred2) { + return isImpliedTrueByMatchingCmp(getPredicate(), Pred2); + } + + /// @brief Determine if Pred1 implies Pred2 is false when two compares have + /// matching operands. + bool isImpliedFalseByMatchingCmp(Predicate Pred2) { + return isImpliedFalseByMatchingCmp(getPredicate(), Pred2); + } + /// @returns true if the predicate is unsigned, false otherwise. /// @brief Determine if the predicate is an unsigned operation. static bool isUnsigned(Predicate predicate); @@ -1078,6 +1074,14 @@ public: /// Determine if the predicate is false when comparing a value with itself. static bool isFalseWhenEqual(Predicate predicate); + /// Determine if Pred1 implies Pred2 is true when two compares have matching + /// operands. + static bool isImpliedTrueByMatchingCmp(Predicate Pred1, Predicate Pred2); + + /// Determine if Pred1 implies Pred2 is false when two compares have matching + /// operands. + static bool isImpliedFalseByMatchingCmp(Predicate Pred1, Predicate Pred2); + /// @brief Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return I->getOpcode() == Instruction::ICmp || @@ -1469,6 +1473,17 @@ public: Other.bundle_op_info_begin()); }; + /// \brief Return true if this operand bundle user contains operand bundles + /// with tags other than those specified in \p IDs. + bool hasOperandBundlesOtherThan(ArrayRef<uint32_t> IDs) const { + for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) { + uint32_t ID = getOperandBundleAt(i).getTagID(); + if (std::find(IDs.begin(), IDs.end(), ID) == IDs.end()) + return true; + } + return false; + } + protected: /// \brief Is the function attribute S disallowed by some operand bundle on /// this operand bundle user? diff --git a/include/llvm/IR/Instruction.h b/include/llvm/IR/Instruction.h index 03c45497fa95..df4f8df78b12 100644 --- a/include/llvm/IR/Instruction.h +++ b/include/llvm/IR/Instruction.h @@ -42,23 +42,23 @@ class Instruction : public User, DebugLoc DbgLoc; // 'dbg' Metadata cache. enum { - /// HasMetadataBit - This is a bit stored in the SubClassData field which - /// indicates whether this instruction has metadata attached to it or not. + /// This is a bit stored in the SubClassData field which indicates whether + /// this instruction has metadata attached to it or not. HasMetadataBit = 1 << 15 }; public: // Out of line virtual method, so the vtable, etc has a home. ~Instruction() override; - /// user_back - Specialize the methods defined in Value, as we know that an - /// instruction can only be used by other instructions. + /// Specialize the methods defined in Value, as we know that an instruction + /// can only be used by other instructions. Instruction *user_back() { return cast<Instruction>(*user_begin());} const Instruction *user_back() const { return cast<Instruction>(*user_begin());} inline const BasicBlock *getParent() const { return Parent; } inline BasicBlock *getParent() { return Parent; } - /// \brief Return the module owning the function this instruction belongs to + /// Return the module owning the function this instruction belongs to /// or nullptr it the function does not have a module. /// /// Note: this is undefined behavior if the instruction does not have a @@ -66,20 +66,18 @@ public: const Module *getModule() const; Module *getModule(); - /// \brief Return the function this instruction belongs to. + /// Return the function this instruction belongs to. /// /// Note: it is undefined behavior to call this on an instruction not /// currently inserted into a function. const Function *getFunction() const; Function *getFunction(); - /// removeFromParent - This method unlinks 'this' from the containing basic - /// block, but does not delete it. - /// + /// This method unlinks 'this' from the containing basic block, but does not + /// delete it. void removeFromParent(); - /// eraseFromParent - This method unlinks 'this' from the containing basic - /// block and deletes it. + /// This method unlinks 'this' from the containing basic block and deletes it. /// /// \returns an iterator pointing to the element after the erased one SymbolTableList<Instruction>::iterator eraseFromParent(); @@ -92,16 +90,15 @@ public: /// specified instruction. void insertAfter(Instruction *InsertPos); - /// moveBefore - Unlink this instruction from its current basic block and - /// insert it into the basic block that MovePos lives in, right before - /// MovePos. + /// Unlink this instruction from its current basic block and insert it into + /// the basic block that MovePos lives in, right before MovePos. void moveBefore(Instruction *MovePos); //===--------------------------------------------------------------------===// // Subclass classification. //===--------------------------------------------------------------------===// - /// getOpcode() returns a member of one of the enums like Instruction::Add. + /// Returns a member of one of the enums like Instruction::Add. unsigned getOpcode() const { return getValueID() - InstructionVal; } const char *getOpcodeName() const { return getOpcodeName(getOpcode()); } @@ -121,28 +118,27 @@ public: return Opcode >= BinaryOpsBegin && Opcode < BinaryOpsEnd; } - /// @brief Determine if the Opcode is one of the shift instructions. + /// Determine if the Opcode is one of the shift instructions. static inline bool isShift(unsigned Opcode) { return Opcode >= Shl && Opcode <= AShr; } - /// isLogicalShift - Return true if this is a logical shift left or a logical - /// shift right. + /// Return true if this is a logical shift left or a logical shift right. inline bool isLogicalShift() const { return getOpcode() == Shl || getOpcode() == LShr; } - /// isArithmeticShift - Return true if this is an arithmetic shift right. + /// Return true if this is an arithmetic shift right. inline bool isArithmeticShift() const { return getOpcode() == AShr; } - /// @brief Determine if the OpCode is one of the CastInst instructions. + /// Determine if the OpCode is one of the CastInst instructions. static inline bool isCast(unsigned OpCode) { return OpCode >= CastOpsBegin && OpCode < CastOpsEnd; } - /// @brief Determine if the OpCode is one of the FuncletPadInst instructions. + /// Determine if the OpCode is one of the FuncletPadInst instructions. static inline bool isFuncletPad(unsigned OpCode) { return OpCode >= FuncletPadOpsBegin && OpCode < FuncletPadOpsEnd; } @@ -151,55 +147,53 @@ public: // Metadata manipulation. //===--------------------------------------------------------------------===// - /// hasMetadata() - Return true if this instruction has any metadata attached - /// to it. + /// Return true if this instruction has any metadata attached to it. bool hasMetadata() const { return DbgLoc || hasMetadataHashEntry(); } - /// hasMetadataOtherThanDebugLoc - Return true if this instruction has - /// metadata attached to it other than a debug location. + /// Return true if this instruction has metadata attached to it other than a + /// debug location. bool hasMetadataOtherThanDebugLoc() const { return hasMetadataHashEntry(); } - /// getMetadata - Get the metadata of given kind attached to this Instruction. + /// Get the metadata of given kind attached to this Instruction. /// If the metadata is not found then return null. MDNode *getMetadata(unsigned KindID) const { if (!hasMetadata()) return nullptr; return getMetadataImpl(KindID); } - /// getMetadata - Get the metadata of given kind attached to this Instruction. + /// Get the metadata of given kind attached to this Instruction. /// If the metadata is not found then return null. MDNode *getMetadata(StringRef Kind) const { if (!hasMetadata()) return nullptr; return getMetadataImpl(Kind); } - /// getAllMetadata - Get all metadata attached to this Instruction. The first - /// element of each pair returned is the KindID, the second element is the - /// metadata value. This list is returned sorted by the KindID. + /// Get all metadata attached to this Instruction. The first element of each + /// pair returned is the KindID, the second element is the metadata value. + /// This list is returned sorted by the KindID. void getAllMetadata(SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const { if (hasMetadata()) getAllMetadataImpl(MDs); } - /// getAllMetadataOtherThanDebugLoc - This does the same thing as - /// getAllMetadata, except that it filters out the debug location. + /// This does the same thing as getAllMetadata, except that it filters out the + /// debug location. void getAllMetadataOtherThanDebugLoc( SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const { if (hasMetadataOtherThanDebugLoc()) getAllMetadataOtherThanDebugLocImpl(MDs); } - /// getAAMetadata - Fills the AAMDNodes structure with AA metadata from - /// this instruction. When Merge is true, the existing AA metadata is - /// merged with that from this instruction providing the most-general result. + /// Fills the AAMDNodes structure with AA metadata from this instruction. + /// When Merge is true, the existing AA metadata is merged with that from this + /// instruction providing the most-general result. void getAAMetadata(AAMDNodes &N, bool Merge = false) const; - /// setMetadata - Set the metadata of the specified kind to the specified - /// node. This updates/replaces metadata if already present, or removes it if - /// Node is null. + /// Set the metadata of the specified kind to the specified node. This updates + /// or replaces metadata if already present, or removes it if Node is null. void setMetadata(unsigned KindID, MDNode *Node); void setMetadata(StringRef Kind, MDNode *Node); @@ -220,16 +214,46 @@ public: } /// @} - /// setAAMetadata - Sets the metadata on this instruction from the - /// AAMDNodes structure. + /// Sets the metadata on this instruction from the AAMDNodes structure. void setAAMetadata(const AAMDNodes &N); - /// setDebugLoc - Set the debug location information for this instruction. + /// Retrieve the raw weight values of a conditional branch or select. + /// Returns true on success with profile weights filled in. + /// Returns false if no metadata or invalid metadata was found. + bool extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal); + + /// Retrieve total raw weight values of a branch. + /// Returns true on success with profile total weights filled in. + /// Returns false if no metadata was found. + bool extractProfTotalWeight(uint64_t &TotalVal); + + /// Set the debug location information for this instruction. void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); } - /// getDebugLoc - Return the debug location for this node as a DebugLoc. + /// Return the debug location for this node as a DebugLoc. const DebugLoc &getDebugLoc() const { return DbgLoc; } + /// Set or clear the nsw flag on this instruction, which must be an operator + /// which supports this flag. See LangRef.html for the meaning of this flag. + void setHasNoUnsignedWrap(bool b = true); + + /// Set or clear the nsw flag on this instruction, which must be an operator + /// which supports this flag. See LangRef.html for the meaning of this flag. + void setHasNoSignedWrap(bool b = true); + + /// Set or clear the exact flag on this instruction, which must be an operator + /// which supports this flag. See LangRef.html for the meaning of this flag. + void setIsExact(bool b = true); + + /// Determine whether the no unsigned wrap flag is set. + bool hasNoUnsignedWrap() const; + + /// Determine whether the no signed wrap flag is set. + bool hasNoSignedWrap() const; + + /// Determine whether the exact flag is set. + bool isExact() const; + /// Set or clear the unsafe-algebra flag on this instruction, which must be an /// operator which supports this flag. See LangRef.html for the meaning of /// this flag. @@ -288,9 +312,16 @@ public: /// Copy I's fast-math flags void copyFastMathFlags(const Instruction *I); + /// Convenience method to copy supported wrapping, exact, and fast-math flags + /// from V to this instruction. + void copyIRFlags(const Value *V); + + /// Logical 'and' of any supported wrapping, exact, and fast-math flags of + /// V and this instruction. + void andIRFlags(const Value *V); + private: - /// hasMetadataHashEntry - Return true if we have an entry in the on-the-side - /// metadata hash. + /// Return true if we have an entry in the on-the-side metadata hash. bool hasMetadataHashEntry() const { return (getSubclassDataFromValue() & HasMetadataBit) != 0; } @@ -302,6 +333,7 @@ private: getAllMetadataImpl(SmallVectorImpl<std::pair<unsigned, MDNode *>> &) const; void getAllMetadataOtherThanDebugLocImpl( SmallVectorImpl<std::pair<unsigned, MDNode *>> &) const; + /// Clear all hashtable-based metadata from this instruction. void clearMetadataHashEntries(); public: //===--------------------------------------------------------------------===// @@ -309,7 +341,7 @@ public: //===--------------------------------------------------------------------===// - /// isAssociative - Return true if the instruction is associative: + /// Return true if the instruction is associative: /// /// Associative operators satisfy: x op (y op z) === (x op y) op z /// @@ -318,7 +350,7 @@ public: bool isAssociative() const; static bool isAssociative(unsigned op); - /// isCommutative - Return true if the instruction is commutative: + /// Return true if the instruction is commutative: /// /// Commutative operators satisfy: (x op y) === (y op x) /// @@ -328,7 +360,7 @@ public: bool isCommutative() const { return isCommutative(getOpcode()); } static bool isCommutative(unsigned op); - /// isIdempotent - Return true if the instruction is idempotent: + /// Return true if the instruction is idempotent: /// /// Idempotent operators satisfy: x op x === x /// @@ -337,7 +369,7 @@ public: bool isIdempotent() const { return isIdempotent(getOpcode()); } static bool isIdempotent(unsigned op); - /// isNilpotent - Return true if the instruction is nilpotent: + /// Return true if the instruction is nilpotent: /// /// Nilpotent operators satisfy: x op x === Id, /// @@ -349,47 +381,50 @@ public: bool isNilpotent() const { return isNilpotent(getOpcode()); } static bool isNilpotent(unsigned op); - /// mayWriteToMemory - Return true if this instruction may modify memory. - /// + /// Return true if this instruction may modify memory. bool mayWriteToMemory() const; - /// mayReadFromMemory - Return true if this instruction may read memory. - /// + /// Return true if this instruction may read memory. bool mayReadFromMemory() const; - /// mayReadOrWriteMemory - Return true if this instruction may read or - /// write memory. - /// + /// Return true if this instruction may read or write memory. bool mayReadOrWriteMemory() const { return mayReadFromMemory() || mayWriteToMemory(); } - /// isAtomic - Return true if this instruction has an - /// AtomicOrdering of unordered or higher. - /// + /// Return true if this instruction has an AtomicOrdering of unordered or + /// higher. bool isAtomic() const; - /// mayThrow - Return true if this instruction may throw an exception. - /// + /// Return true if this instruction may throw an exception. bool mayThrow() const; - /// mayReturn - Return true if this is a function that may return. - /// this is true for all normal instructions. The only exception - /// is functions that are marked with the 'noreturn' attribute. - /// - bool mayReturn() const; + /// Return true if this instruction behaves like a memory fence: it can load + /// or store to memory location without being given a memory location. + bool isFenceLike() const { + switch (getOpcode()) { + default: + return false; + // This list should be kept in sync with the list in mayWriteToMemory for + // all opcodes which don't have a memory location. + case Instruction::Fence: + case Instruction::CatchPad: + case Instruction::CatchRet: + case Instruction::Call: + case Instruction::Invoke: + return true; + } + } - /// mayHaveSideEffects - Return true if the instruction may have side effects. + /// Return true if the instruction may have side effects. /// /// Note that this does not consider malloc and alloca to have side /// effects because the newly allocated memory is completely invisible to /// instructions which don't use the returned value. For cases where this /// matters, isSafeToSpeculativelyExecute may be more appropriate. - bool mayHaveSideEffects() const { - return mayWriteToMemory() || mayThrow() || !mayReturn(); - } + bool mayHaveSideEffects() const { return mayWriteToMemory() || mayThrow(); } - /// \brief Return true if the instruction is a variety of EH-block. + /// Return true if the instruction is a variety of EH-block. bool isEHPad() const { switch (getOpcode()) { case Instruction::CatchSwitch: @@ -402,21 +437,21 @@ public: } } - /// clone() - Create a copy of 'this' instruction that is identical in all - /// ways except the following: + /// Create a copy of 'this' instruction that is identical in all ways except + /// the following: /// * The instruction has no parent /// * The instruction has no name /// Instruction *clone() const; - /// isIdenticalTo - Return true if the specified instruction is exactly - /// identical to the current one. This means that all operands match and any - /// extra information (e.g. load is volatile) agree. + /// Return true if the specified instruction is exactly identical to the + /// current one. This means that all operands match and any extra information + /// (e.g. load is volatile) agree. bool isIdenticalTo(const Instruction *I) const; - /// isIdenticalToWhenDefined - This is like isIdenticalTo, except that it - /// ignores the SubclassOptionalData flags, which specify conditions - /// under which the instruction's result is undefined. + /// This is like isIdenticalTo, except that it ignores the + /// SubclassOptionalData flags, which specify conditions under which the + /// instruction's result is undefined. bool isIdenticalToWhenDefined(const Instruction *I) const; /// When checking for operation equivalence (using isSameOperationAs) it is @@ -439,10 +474,9 @@ public: /// @brief Determine if one instruction is the same operation as another. bool isSameOperationAs(const Instruction *I, unsigned flags = 0) const; - /// isUsedOutsideOfBlock - Return true if there are any uses of this - /// instruction in blocks other than the specified block. Note that PHI nodes - /// are considered to evaluate their operands in the corresponding predecessor - /// block. + /// Return true if there are any uses of this instruction in blocks other than + /// the specified block. Note that PHI nodes are considered to evaluate their + /// operands in the corresponding predecessor block. bool isUsedOutsideOfBlock(const BasicBlock *BB) const; diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h index 28e1fd90fdf6..be077725f7bc 100644 --- a/include/llvm/IR/Instructions.h +++ b/include/llvm/IR/Instructions.h @@ -25,6 +25,7 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" +#include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/ErrorHandling.h" #include <iterator> @@ -36,38 +37,11 @@ class ConstantRange; class DataLayout; class LLVMContext; -enum AtomicOrdering { - NotAtomic = 0, - Unordered = 1, - Monotonic = 2, - // Consume = 3, // Not specified yet. - Acquire = 4, - Release = 5, - AcquireRelease = 6, - SequentiallyConsistent = 7 -}; - enum SynchronizationScope { SingleThread = 0, CrossThread = 1 }; -/// Returns true if the ordering is at least as strong as acquire -/// (i.e. acquire, acq_rel or seq_cst) -inline bool isAtLeastAcquire(AtomicOrdering Ord) { - return (Ord == Acquire || - Ord == AcquireRelease || - Ord == SequentiallyConsistent); -} - -/// Returns true if the ordering is at least as strong as release -/// (i.e. release, acq_rel or seq_cst) -inline bool isAtLeastRelease(AtomicOrdering Ord) { -return (Ord == Release || - Ord == AcquireRelease || - Ord == SequentiallyConsistent); -} - //===----------------------------------------------------------------------===// // AllocaInst Class //===----------------------------------------------------------------------===// @@ -152,6 +126,18 @@ public: (V ? 32 : 0)); } + /// \brief Return true if this alloca is used as a swifterror argument to a + /// call. + bool isSwiftError() const { + return getSubclassDataFromInstruction() & 64; + } + + /// \brief Specify whether this alloca is used to represent a swifterror. + void setSwiftError(bool V) { + setInstructionSubclassData((getSubclassDataFromInstruction() & ~64) | + (V ? 64 : 0)); + } + // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return (I->getOpcode() == Instruction::Alloca); @@ -257,7 +243,7 @@ public: /// AcquireRelease. void setOrdering(AtomicOrdering Ordering) { setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) | - (Ordering << 7)); + ((unsigned)Ordering << 7)); } SynchronizationScope getSynchScope() const { @@ -280,7 +266,9 @@ public: bool isSimple() const { return !isAtomic() && !isVolatile(); } bool isUnordered() const { - return getOrdering() <= Unordered && !isVolatile(); + return (getOrdering() == AtomicOrdering::NotAtomic || + getOrdering() == AtomicOrdering::Unordered) && + !isVolatile(); } Value *getPointerOperand() { return getOperand(0); } @@ -378,7 +366,7 @@ public: /// AcquireRelease. void setOrdering(AtomicOrdering Ordering) { setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 7)) | - (Ordering << 7)); + ((unsigned)Ordering << 7)); } SynchronizationScope getSynchScope() const { @@ -401,7 +389,9 @@ public: bool isSimple() const { return !isAtomic() && !isVolatile(); } bool isUnordered() const { - return getOrdering() <= Unordered && !isVolatile(); + return (getOrdering() == AtomicOrdering::NotAtomic || + getOrdering() == AtomicOrdering::Unordered) && + !isVolatile(); } Value *getValueOperand() { return getOperand(0); } @@ -477,7 +467,7 @@ public: /// AcquireRelease, or SequentiallyConsistent. void setOrdering(AtomicOrdering Ordering) { setInstructionSubclassData((getSubclassDataFromInstruction() & 1) | - (Ordering << 1)); + ((unsigned)Ordering << 1)); } SynchronizationScope getSynchScope() const { @@ -572,17 +562,17 @@ public: /// Set the ordering constraint on this cmpxchg. void setSuccessOrdering(AtomicOrdering Ordering) { - assert(Ordering != NotAtomic && + assert(Ordering != AtomicOrdering::NotAtomic && "CmpXchg instructions can only be atomic."); setInstructionSubclassData((getSubclassDataFromInstruction() & ~0x1c) | - (Ordering << 2)); + ((unsigned)Ordering << 2)); } void setFailureOrdering(AtomicOrdering Ordering) { - assert(Ordering != NotAtomic && + assert(Ordering != AtomicOrdering::NotAtomic && "CmpXchg instructions can only be atomic."); setInstructionSubclassData((getSubclassDataFromInstruction() & ~0xe0) | - (Ordering << 5)); + ((unsigned)Ordering << 5)); } /// Specify whether this cmpxchg is atomic and orders other operations with @@ -634,15 +624,16 @@ public: static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) { switch (SuccessOrdering) { - default: llvm_unreachable("invalid cmpxchg success ordering"); - case Release: - case Monotonic: - return Monotonic; - case AcquireRelease: - case Acquire: - return Acquire; - case SequentiallyConsistent: - return SequentiallyConsistent; + default: + llvm_unreachable("invalid cmpxchg success ordering"); + case AtomicOrdering::Release: + case AtomicOrdering::Monotonic: + return AtomicOrdering::Monotonic; + case AtomicOrdering::AcquireRelease: + case AtomicOrdering::Acquire: + return AtomicOrdering::Acquire; + case AtomicOrdering::SequentiallyConsistent: + return AtomicOrdering::SequentiallyConsistent; } } @@ -758,10 +749,10 @@ public: /// Set the ordering constraint on this RMW. void setOrdering(AtomicOrdering Ordering) { - assert(Ordering != NotAtomic && + assert(Ordering != AtomicOrdering::NotAtomic && "atomicrmw instructions can only be atomic."); setInstructionSubclassData((getSubclassDataFromInstruction() & ~(7 << 2)) | - (Ordering << 2)); + ((unsigned)Ordering << 2)); } /// Specify whether this RMW orders other operations with respect to all @@ -1490,9 +1481,29 @@ public: Value *AllocSize, Value *ArraySize = nullptr, Function* MallocF = nullptr, const Twine &Name = ""); + static Instruction *CreateMalloc(Instruction *InsertBefore, + Type *IntPtrTy, Type *AllocTy, + Value *AllocSize, Value *ArraySize = nullptr, + ArrayRef<OperandBundleDef> Bundles = None, + Function* MallocF = nullptr, + const Twine &Name = ""); + static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, + Type *IntPtrTy, Type *AllocTy, + Value *AllocSize, Value *ArraySize = nullptr, + ArrayRef<OperandBundleDef> Bundles = None, + Function* MallocF = nullptr, + const Twine &Name = ""); /// CreateFree - Generate the IR for a call to the builtin free function. - static Instruction* CreateFree(Value* Source, Instruction *InsertBefore); - static Instruction* CreateFree(Value* Source, BasicBlock *InsertAtEnd); + static Instruction *CreateFree(Value *Source, + Instruction *InsertBefore); + static Instruction *CreateFree(Value *Source, + BasicBlock *InsertAtEnd); + static Instruction *CreateFree(Value *Source, + ArrayRef<OperandBundleDef> Bundles, + Instruction *InsertBefore); + static Instruction *CreateFree(Value *Source, + ArrayRef<OperandBundleDef> Bundles, + BasicBlock *InsertAtEnd); ~CallInst() override; @@ -1586,6 +1597,10 @@ public: return getOperandUse(i); } + /// If one of the arguments has the 'returned' attribute, return its + /// operand value. Otherwise, return nullptr. + Value *getReturnedArgOperand() const; + /// getCallingConv/setCallingConv - Get or set the calling convention of this /// function call. CallingConv::ID getCallingConv() const { @@ -1607,13 +1622,22 @@ public: void setAttributes(const AttributeSet &Attrs) { AttributeList = Attrs; } /// addAttribute - adds the attribute to the list of attributes. - void addAttribute(unsigned i, Attribute::AttrKind attr); + void addAttribute(unsigned i, Attribute::AttrKind Kind); /// addAttribute - adds the attribute to the list of attributes. void addAttribute(unsigned i, StringRef Kind, StringRef Value); + /// addAttribute - adds the attribute to the list of attributes. + void addAttribute(unsigned i, Attribute Attr); + + /// removeAttribute - removes the attribute from the list of attributes. + void removeAttribute(unsigned i, Attribute::AttrKind Kind); + + /// removeAttribute - removes the attribute from the list of attributes. + void removeAttribute(unsigned i, StringRef Kind); + /// removeAttribute - removes the attribute from the list of attributes. - void removeAttribute(unsigned i, Attribute attr); + void removeAttribute(unsigned i, Attribute Attr); /// \brief adds the dereferenceable attribute to the list of attributes. void addDereferenceableAttr(unsigned i, uint64_t Bytes); @@ -1623,19 +1647,25 @@ public: void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes); /// \brief Determine whether this call has the given attribute. - bool hasFnAttr(Attribute::AttrKind A) const { - assert(A != Attribute::NoBuiltin && + bool hasFnAttr(Attribute::AttrKind Kind) const { + assert(Kind != Attribute::NoBuiltin && "Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin"); - return hasFnAttrImpl(A); + return hasFnAttrImpl(Kind); } /// \brief Determine whether this call has the given attribute. - bool hasFnAttr(StringRef A) const { - return hasFnAttrImpl(A); + bool hasFnAttr(StringRef Kind) const { + return hasFnAttrImpl(Kind); } /// \brief Determine whether the call or the callee has the given attributes. - bool paramHasAttr(unsigned i, Attribute::AttrKind A) const; + bool paramHasAttr(unsigned i, Attribute::AttrKind Kind) const; + + /// \brief Get the attribute of a given kind at a position. + Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const; + + /// \brief Get the attribute of a given kind at a position. + Attribute getAttribute(unsigned i, StringRef Kind) const; /// \brief Return true if the data operand at index \p i has the attribute \p /// A. @@ -1650,7 +1680,7 @@ public: /// \p i in [1, arg_size + 1) -> argument number (\p i - 1) /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index /// (\p i - 1) in the operand list. - bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind A) const; + bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const; /// \brief Extract the alignment for a call or parameter (0=unknown). unsigned getParamAlignment(unsigned i) const { @@ -1713,6 +1743,14 @@ public: addAttribute(AttributeSet::FunctionIndex, Attribute::ReadOnly); } + /// \brief Determine if the call does not access or only writes memory. + bool doesNotReadMemory() const { + return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly); + } + void setDoesNotReadMemory() { + addAttribute(AttributeSet::FunctionIndex, Attribute::WriteOnly); + } + /// @brief Determine if the call can access memmory only using pointers based /// on its arguments. bool onlyAccessesArgMemory() const { @@ -1745,6 +1783,10 @@ public: void setConvergent() { addAttribute(AttributeSet::FunctionIndex, Attribute::Convergent); } + void setNotConvergent() { + removeAttribute(AttributeSet::FunctionIndex, + Attribute::get(getContext(), Attribute::Convergent)); + } /// \brief Determine if the call returns a structure through first /// pointer argument. @@ -1906,6 +1948,10 @@ public: Value *getTrueValue() { return Op<1>(); } Value *getFalseValue() { return Op<2>(); } + void setCondition(Value *V) { Op<0>() = V; } + void setTrueValue(Value *V) { Op<1>() = V; } + void setFalseValue(Value *V) { Op<2>() = V; } + /// areInvalidOperands - Return a string if the specified operands are invalid /// for a select operation, otherwise return null. static const char *areInvalidOperands(Value *Cond, Value *True, Value *False); @@ -2619,6 +2665,11 @@ public: /// same value, return the value, otherwise return null. Value *hasConstantValue() const; + /// hasConstantOrUndefValue - Whether the specified PHI node always merges + /// together the same value, assuming undefs are equal to a unique + /// non-undef value. + bool hasConstantOrUndefValue() const; + /// Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const Instruction *I) { return I->getOpcode() == Instruction::PHI; @@ -2928,7 +2979,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BranchInst, Value) //===----------------------------------------------------------------------===// //===--------------------------------------------------------------------------- -/// SwitchInst - Multiway switch +/// Multiway switch /// class SwitchInst : public TerminatorInst { void *operator new(size_t, unsigned) = delete; @@ -2944,17 +2995,17 @@ class SwitchInst : public TerminatorInst { void *operator new(size_t s) { return User::operator new(s); } - /// SwitchInst ctor - Create a new switch instruction, specifying a value to - /// switch on and a default destination. The number of additional cases can - /// be specified here to make memory allocation more efficient. This - /// constructor can also autoinsert before another instruction. + /// Create a new switch instruction, specifying a value to switch on and a + /// default destination. The number of additional cases can be specified here + /// to make memory allocation more efficient. This constructor can also + /// auto-insert before another instruction. SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases, Instruction *InsertBefore); - /// SwitchInst ctor - Create a new switch instruction, specifying a value to - /// switch on and a default destination. The number of additional cases can - /// be specified here to make memory allocation more efficient. This - /// constructor also autoinserts at the end of the specified BasicBlock. + /// Create a new switch instruction, specifying a value to switch on and a + /// default destination. The number of additional cases can be specified here + /// to make memory allocation more efficient. This constructor also + /// auto-inserts at the end of the specified BasicBlock. SwitchInst(Value *Value, BasicBlock *Default, unsigned NumCases, BasicBlock *InsertAtEnd); @@ -3104,40 +3155,40 @@ public: setOperand(1, reinterpret_cast<Value*>(DefaultCase)); } - /// getNumCases - return the number of 'cases' in this switch instruction, - /// except the default case + /// Return the number of 'cases' in this switch instruction, excluding the + /// default case. unsigned getNumCases() const { return getNumOperands()/2 - 1; } - /// Returns a read/write iterator that points to the first - /// case in SwitchInst. + /// Returns a read/write iterator that points to the first case in the + /// SwitchInst. CaseIt case_begin() { return CaseIt(this, 0); } - /// Returns a read-only iterator that points to the first - /// case in the SwitchInst. + /// Returns a read-only iterator that points to the first case in the + /// SwitchInst. ConstCaseIt case_begin() const { return ConstCaseIt(this, 0); } - /// Returns a read/write iterator that points one past the last - /// in the SwitchInst. + /// Returns a read/write iterator that points one past the last in the + /// SwitchInst. CaseIt case_end() { return CaseIt(this, getNumCases()); } - /// Returns a read-only iterator that points one past the last - /// in the SwitchInst. + /// Returns a read-only iterator that points one past the last in the + /// SwitchInst. ConstCaseIt case_end() const { return ConstCaseIt(this, getNumCases()); } - /// cases - iteration adapter for range-for loops. + /// Iteration adapter for range-for loops. iterator_range<CaseIt> cases() { return make_range(case_begin(), case_end()); } - /// cases - iteration adapter for range-for loops. + /// Constant iteration adapter for range-for loops. iterator_range<ConstCaseIt> cases() const { return make_range(case_begin(), case_end()); } @@ -3154,10 +3205,10 @@ public: return ConstCaseIt(this, DefaultPseudoIndex); } - /// findCaseValue - Search all of the case values for the specified constant. - /// If it is explicitly handled, return the case iterator of it, otherwise - /// return default case iterator to indicate - /// that it is handled by the default handler. + /// Search all of the case values for the specified constant. If it is + /// explicitly handled, return the case iterator of it, otherwise return + /// default case iterator to indicate that it is handled by the default + /// handler. CaseIt findCaseValue(const ConstantInt *C) { for (CaseIt i = case_begin(), e = case_end(); i != e; ++i) if (i.getCaseValue() == C) @@ -3171,8 +3222,8 @@ public: return case_default(); } - /// findCaseDest - Finds the unique case value for a given successor. Returns - /// null if the successor is not found, not unique, or is the default case. + /// Finds the unique case value for a given successor. Returns null if the + /// successor is not found, not unique, or is the default case. ConstantInt *findCaseDest(BasicBlock *BB) { if (BB == getDefaultDest()) return nullptr; @@ -3186,15 +3237,15 @@ public: return CI; } - /// addCase - Add an entry to the switch instruction... + /// Add an entry to the switch instruction. /// Note: /// This action invalidates case_end(). Old case_end() iterator will /// point to the added case. void addCase(ConstantInt *OnVal, BasicBlock *Dest); - /// removeCase - This method removes the specified case and its successor - /// from the switch instruction. Note that this operation may reorder the - /// remaining cases at index idx and above. + /// This method removes the specified case and its successor from the switch + /// instruction. Note that this operation may reorder the remaining cases at + /// index idx and above. /// Note: /// This action invalidates iterators for all cases following the one removed, /// including the case_end() iterator. @@ -3519,6 +3570,10 @@ public: return getOperandUse(i); } + /// If one of the arguments has the 'returned' attribute, return its + /// operand value. Otherwise, return nullptr. + Value *getReturnedArgOperand() const; + /// getCallingConv/setCallingConv - Get or set the calling convention of this /// function call. CallingConv::ID getCallingConv() const { @@ -3539,10 +3594,19 @@ public: void setAttributes(const AttributeSet &Attrs) { AttributeList = Attrs; } /// addAttribute - adds the attribute to the list of attributes. - void addAttribute(unsigned i, Attribute::AttrKind attr); + void addAttribute(unsigned i, Attribute::AttrKind Kind); + + /// addAttribute - adds the attribute to the list of attributes. + void addAttribute(unsigned i, Attribute Attr); + + /// removeAttribute - removes the attribute from the list of attributes. + void removeAttribute(unsigned i, Attribute::AttrKind Kind); + + /// removeAttribute - removes the attribute from the list of attributes. + void removeAttribute(unsigned i, StringRef Kind); /// removeAttribute - removes the attribute from the list of attributes. - void removeAttribute(unsigned i, Attribute attr); + void removeAttribute(unsigned i, Attribute Attr); /// \brief adds the dereferenceable attribute to the list of attributes. void addDereferenceableAttr(unsigned i, uint64_t Bytes); @@ -3552,19 +3616,25 @@ public: void addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes); /// \brief Determine whether this call has the given attribute. - bool hasFnAttr(Attribute::AttrKind A) const { - assert(A != Attribute::NoBuiltin && + bool hasFnAttr(Attribute::AttrKind Kind) const { + assert(Kind != Attribute::NoBuiltin && "Use CallInst::isNoBuiltin() to check for Attribute::NoBuiltin"); - return hasFnAttrImpl(A); + return hasFnAttrImpl(Kind); } /// \brief Determine whether this call has the given attribute. - bool hasFnAttr(StringRef A) const { - return hasFnAttrImpl(A); + bool hasFnAttr(StringRef Kind) const { + return hasFnAttrImpl(Kind); } /// \brief Determine whether the call or the callee has the given attributes. - bool paramHasAttr(unsigned i, Attribute::AttrKind A) const; + bool paramHasAttr(unsigned i, Attribute::AttrKind Kind) const; + + /// \brief Get the attribute of a given kind at a position. + Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const; + + /// \brief Get the attribute of a given kind at a position. + Attribute getAttribute(unsigned i, StringRef Kind) const; /// \brief Return true if the data operand at index \p i has the attribute \p /// A. @@ -3580,7 +3650,7 @@ public: /// \p i in [1, arg_size + 1) -> argument number (\p i - 1) /// \p i in [arg_size + 1, data_operand_size + 1) -> bundle operand at index /// (\p i - 1) in the operand list. - bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind A) const; + bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const; /// \brief Extract the alignment for a call or parameter (0=unknown). unsigned getParamAlignment(unsigned i) const { @@ -3637,6 +3707,14 @@ public: addAttribute(AttributeSet::FunctionIndex, Attribute::ReadOnly); } + /// \brief Determine if the call does not access or only writes memory. + bool doesNotReadMemory() const { + return doesNotAccessMemory() || hasFnAttr(Attribute::WriteOnly); + } + void setDoesNotReadMemory() { + addAttribute(AttributeSet::FunctionIndex, Attribute::WriteOnly); + } + /// @brief Determine if the call access memmory only using it's pointer /// arguments. bool onlyAccessesArgMemory() const { @@ -3664,6 +3742,16 @@ public: addAttribute(AttributeSet::FunctionIndex, Attribute::NoDuplicate); } + /// \brief Determine if the invoke is convergent + bool isConvergent() const { return hasFnAttr(Attribute::Convergent); } + void setConvergent() { + addAttribute(AttributeSet::FunctionIndex, Attribute::Convergent); + } + void setNotConvergent() { + removeAttribute(AttributeSet::FunctionIndex, + Attribute::get(getContext(), Attribute::Convergent)); + } + /// \brief Determine if the call returns a structure through first /// pointer argument. bool hasStructRetAttr() const { @@ -4160,7 +4248,9 @@ public: } unsigned getNumSuccessors() const { return 1; } - Value *getParentPad() const { + /// Get the parentPad of this catchret's catchpad's catchswitch. + /// The successor block is implicitly a member of this funclet. + Value *getCatchSwitchParentPad() const { return getCatchPad()->getCatchSwitch()->getParentPad(); } @@ -4826,6 +4916,31 @@ public: static inline bool classof(const Value *V) { return isa<Instruction>(V) && classof(cast<Instruction>(V)); } + + /// \brief Gets the pointer operand. + Value *getPointerOperand() { + return getOperand(0); + } + + /// \brief Gets the pointer operand. + const Value *getPointerOperand() const { + return getOperand(0); + } + + /// \brief Gets the operand index of the pointer operand. + static unsigned getPointerOperandIndex() { + return 0U; + } + + /// \brief Returns the address space of the pointer operand. + unsigned getSrcAddressSpace() const { + return getPointerOperand()->getType()->getPointerAddressSpace(); + } + + /// \brief Returns the address space of the result. + unsigned getDestAddressSpace() const { + return getType()->getPointerAddressSpace(); + } }; } // End llvm namespace diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h index 169bcc021984..52044e0a0cc8 100644 --- a/include/llvm/IR/IntrinsicInst.h +++ b/include/llvm/IR/IntrinsicInst.h @@ -31,16 +31,15 @@ #include "llvm/IR/Metadata.h" namespace llvm { - /// IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic - /// functions. This allows the standard isa/dyncast/cast functionality to - /// work with calls to intrinsic functions. + /// A wrapper class for inspecting calls to intrinsic functions. + /// This allows the standard isa/dyncast/cast functionality to work with calls + /// to intrinsic functions. class IntrinsicInst : public CallInst { IntrinsicInst() = delete; IntrinsicInst(const IntrinsicInst&) = delete; void operator=(const IntrinsicInst&) = delete; public: - /// getIntrinsicID - Return the intrinsic ID of this intrinsic. - /// + /// Return the intrinsic ID of this intrinsic. Intrinsic::ID getIntrinsicID() const { return getCalledFunction()->getIntrinsicID(); } @@ -56,10 +55,13 @@ namespace llvm { } }; - /// DbgInfoIntrinsic - This is the common base class for debug info intrinsics - /// + /// This is the common base class for debug info intrinsics. class DbgInfoIntrinsic : public IntrinsicInst { public: + /// Get the location corresponding to the variable referenced by the debug + /// info intrinsic. Depending on the intrinsic, this could be the + /// variable's value or its address. + Value *getVariableLocation(bool AllowNullOp = true) const; // Methods for support type inquiry through isa, cast, and dyn_cast: static inline bool classof(const IntrinsicInst *I) { @@ -73,15 +75,12 @@ namespace llvm { static inline bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); } - - static Value *StripCast(Value *C); }; - /// DbgDeclareInst - This represents the llvm.dbg.declare instruction. - /// + /// This represents the llvm.dbg.declare instruction. class DbgDeclareInst : public DbgInfoIntrinsic { public: - Value *getAddress() const; + Value *getAddress() const { return getVariableLocation(); } DILocalVariable *getVariable() const { return cast<DILocalVariable>(getRawVariable()); } @@ -105,12 +104,12 @@ namespace llvm { } }; - /// DbgValueInst - This represents the llvm.dbg.value instruction. - /// + /// This represents the llvm.dbg.value instruction. class DbgValueInst : public DbgInfoIntrinsic { public: - const Value *getValue() const; - Value *getValue(); + Value *getValue() const { + return getVariableLocation(/* AllowNullOp = */ false); + } uint64_t getOffset() const { return cast<ConstantInt>( const_cast<Value*>(getArgOperand(1)))->getZExtValue(); @@ -138,8 +137,7 @@ namespace llvm { } }; - /// MemIntrinsic - This is the common base class for memset/memcpy/memmove. - /// + /// This is the common base class for memset/memcpy/memmove. class MemIntrinsic : public IntrinsicInst { public: Value *getRawDest() const { return const_cast<Value*>(getArgOperand(0)); } @@ -169,13 +167,12 @@ namespace llvm { return cast<PointerType>(getRawDest()->getType())->getAddressSpace(); } - /// getDest - This is just like getRawDest, but it strips off any cast + /// This is just like getRawDest, but it strips off any cast /// instructions that feed it, giving the original input. The returned /// value is guaranteed to be a pointer. Value *getDest() const { return getRawDest()->stripPointerCasts(); } - /// set* - Set the specified arguments of the instruction. - /// + /// Set the specified arguments of the instruction. void setDest(Value *Ptr) { assert(getRawDest()->getType() == Ptr->getType() && "setDest called with pointer of wrong type!"); @@ -215,12 +212,10 @@ namespace llvm { } }; - /// MemSetInst - This class wraps the llvm.memset intrinsic. - /// + /// This class wraps the llvm.memset intrinsic. class MemSetInst : public MemIntrinsic { public: - /// get* - Return the arguments to the instruction. - /// + /// Return the arguments to the instruction. Value *getValue() const { return const_cast<Value*>(getArgOperand(1)); } const Use &getValueUse() const { return getArgOperandUse(1); } Use &getValueUse() { return getArgOperandUse(1); } @@ -240,17 +235,15 @@ namespace llvm { } }; - /// MemTransferInst - This class wraps the llvm.memcpy/memmove intrinsics. - /// + /// This class wraps the llvm.memcpy/memmove intrinsics. class MemTransferInst : public MemIntrinsic { public: - /// get* - Return the arguments to the instruction. - /// + /// Return the arguments to the instruction. Value *getRawSource() const { return const_cast<Value*>(getArgOperand(1)); } const Use &getRawSourceUse() const { return getArgOperandUse(1); } Use &getRawSourceUse() { return getArgOperandUse(1); } - /// getSource - This is just like getRawSource, but it strips off any cast + /// This is just like getRawSource, but it strips off any cast /// instructions that feed it, giving the original input. The returned /// value is guaranteed to be a pointer. Value *getSource() const { return getRawSource()->stripPointerCasts(); } @@ -276,8 +269,7 @@ namespace llvm { }; - /// MemCpyInst - This class wraps the llvm.memcpy intrinsic. - /// + /// This class wraps the llvm.memcpy intrinsic. class MemCpyInst : public MemTransferInst { public: // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -289,8 +281,7 @@ namespace llvm { } }; - /// MemMoveInst - This class wraps the llvm.memmove intrinsic. - /// + /// This class wraps the llvm.memmove intrinsic. class MemMoveInst : public MemTransferInst { public: // Methods for support type inquiry through isa, cast, and dyn_cast: @@ -302,8 +293,7 @@ namespace llvm { } }; - /// VAStartInst - This represents the llvm.va_start intrinsic. - /// + /// This represents the llvm.va_start intrinsic. class VAStartInst : public IntrinsicInst { public: static inline bool classof(const IntrinsicInst *I) { @@ -316,8 +306,7 @@ namespace llvm { Value *getArgList() const { return const_cast<Value*>(getArgOperand(0)); } }; - /// VAEndInst - This represents the llvm.va_end intrinsic. - /// + /// This represents the llvm.va_end intrinsic. class VAEndInst : public IntrinsicInst { public: static inline bool classof(const IntrinsicInst *I) { @@ -330,8 +319,7 @@ namespace llvm { Value *getArgList() const { return const_cast<Value*>(getArgOperand(0)); } }; - /// VACopyInst - This represents the llvm.va_copy intrinsic. - /// + /// This represents the llvm.va_copy intrinsic. class VACopyInst : public IntrinsicInst { public: static inline bool classof(const IntrinsicInst *I) { diff --git a/include/llvm/IR/Intrinsics.h b/include/llvm/IR/Intrinsics.h index 314e2aaecf4b..7a87c2167710 100644 --- a/include/llvm/IR/Intrinsics.h +++ b/include/llvm/IR/Intrinsics.h @@ -17,6 +17,8 @@ #define LLVM_IR_INTRINSICS_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" #include <string> namespace llvm { @@ -69,6 +71,13 @@ namespace Intrinsic { /// the intrinsic. Function *getDeclaration(Module *M, ID id, ArrayRef<Type*> Tys = None); + /// Looks up Name in NameTable via binary search. NameTable must be sorted + /// and all entries must start with "llvm.". If NameTable contains an exact + /// match for Name or a prefix of Name followed by a dot, its index in + /// NameTable is returned. Otherwise, -1 is returned. + int lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable, + StringRef Name); + /// Map a GCC builtin name to an intrinsic ID. ID getIntrinsicForGCCBuiltin(const char *Prefix, const char *BuiltinName); @@ -126,6 +135,25 @@ namespace Intrinsic { /// of IITDescriptors. void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl<IITDescriptor> &T); + /// Match the specified type (which comes from an intrinsic argument or return + /// value) with the type constraints specified by the .td file. If the given + /// type is an overloaded type it is pushed to the ArgTys vector. + /// + /// Returns false if the given type matches with the constraints, true + /// otherwise. + bool matchIntrinsicType(Type *Ty, ArrayRef<IITDescriptor> &Infos, + SmallVectorImpl<Type*> &ArgTys); + + /// Verify if the intrinsic has variable arguments. This method is intended to + /// be called after all the fixed arguments have been matched first. + /// + /// This method returns true on error. + bool matchIntrinsicVarArg(bool isVarArg, ArrayRef<IITDescriptor> &Infos); + + // Checks if the intrinsic name matches with its signature and if not + // returns the declaration with the same signature and remangled name. + llvm::Optional<Function*> remangleIntrinsicFunction(Function *F); + } // End Intrinsic namespace } // End llvm namespace diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td index f67029ab56e3..5ece731fa143 100644 --- a/include/llvm/IR/Intrinsics.td +++ b/include/llvm/IR/Intrinsics.td @@ -19,9 +19,7 @@ include "llvm/CodeGen/ValueTypes.td" class IntrinsicProperty; -// Intr*Mem - Memory properties. An intrinsic is allowed to have at most one of -// these properties set. They are listed from the most aggressive (best to use -// if correct) to the least aggressive. If no property is set, the worst case +// Intr*Mem - Memory properties. If no property is set, the worst case // is assumed (it may read and write any memory it can get access to and it may // have other side effects). @@ -29,20 +27,21 @@ class IntrinsicProperty; // effects. It may be CSE'd deleted if dead, etc. def IntrNoMem : IntrinsicProperty; -// IntrReadArgMem - This intrinsic reads only from memory that one of its -// pointer-typed arguments points to, but may read an unspecified amount. -def IntrReadArgMem : IntrinsicProperty; - -// IntrReadMem - This intrinsic reads from unspecified memory, so it cannot be -// moved across stores. However, it can be reordered otherwise and can be -// deleted if dead. +// IntrReadMem - This intrinsic only reads from memory. It does not write to +// memory and has no other side effects. Therefore, it cannot be moved across +// potentially aliasing stores. However, it can be reordered otherwise and can +// be deleted if dead. def IntrReadMem : IntrinsicProperty; -// IntrReadWriteArgMem - This intrinsic reads and writes only from memory that -// one of its arguments points to, but may access an unspecified amount. The -// reads and writes may be volatile, but except for this it has no other side -// effects. -def IntrReadWriteArgMem : IntrinsicProperty; +// IntrWriteMem - This intrinsic only writes to memory, but does not read from +// memory, and has no other side effects. This means dead stores before calls +// to this intrinsics may be removed. +def IntrWriteMem : IntrinsicProperty; + +// IntrArgMemOnly - This intrinsic only accesses memory that its pointer-typed +// argument(s) points to, but may access an unspecified amount. Other than +// reads from and (possibly volatile) writes to memory, it has no side effects. +def IntrArgMemOnly : IntrinsicProperty; // Commutative - This intrinsic is commutative: X op Y == Y op X. def Commutative : IntrinsicProperty; @@ -55,12 +54,24 @@ class NoCapture<int argNo> : IntrinsicProperty { int ArgNo = argNo; } +// Returned - The specified argument is always the return value of the +// intrinsic. +class Returned<int argNo> : IntrinsicProperty { + int ArgNo = argNo; +} + // ReadOnly - The specified argument pointer is not written to through the // pointer by the intrinsic. class ReadOnly<int argNo> : IntrinsicProperty { int ArgNo = argNo; } +// WriteOnly - The intrinsic does not read memory through the specified +// argument pointer. +class WriteOnly<int argNo> : IntrinsicProperty { + int ArgNo = argNo; +} + // ReadNone - The specified argument pointer is not dereferenced by the // intrinsic. class ReadNone<int argNo> : IntrinsicProperty { @@ -240,7 +251,7 @@ class Intrinsic<list<LLVMType> ret_types, string TargetPrefix = ""; // Set to a prefix for target-specific intrinsics. list<LLVMType> RetTypes = ret_types; list<LLVMType> ParamTypes = param_types; - list<IntrinsicProperty> Properties = properties; + list<IntrinsicProperty> IntrProperties = properties; bit isTarget = 0; } @@ -271,10 +282,10 @@ def int_gcroot : Intrinsic<[], [llvm_ptrptr_ty, llvm_ptr_ty]>; def int_gcread : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptrptr_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_gcwrite : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_ptrptr_ty], - [IntrReadWriteArgMem, NoCapture<1>, NoCapture<2>]>; + [IntrArgMemOnly, NoCapture<1>, NoCapture<2>]>; //===--------------------- Code Generator Intrinsics ----------------------===// // @@ -306,13 +317,16 @@ def int_stackrestore : Intrinsic<[], [llvm_ptr_ty]>, def int_get_dynamic_area_offset : Intrinsic<[llvm_anyint_ty]>; -// IntrReadWriteArgMem is more pessimistic than strictly necessary for prefetch, +def int_thread_pointer : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>, + GCCBuiltin<"__builtin_thread_pointer">; + +// IntrArgMemOnly is more pessimistic than strictly necessary for prefetch, // however it does conveniently prevent the prefetch from being reordered // with respect to nearby accesses to the same memory. def int_prefetch : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; def int_pcmarker : Intrinsic<[], [llvm_i32_ty]>; def int_readcyclecounter : Intrinsic<[llvm_i64_ty]>; @@ -324,8 +338,7 @@ def int_assume : Intrinsic<[], [llvm_i1_ty], []>; // Stack Protector Intrinsic - The stackprotector intrinsic writes the stack // guard to the correct place on the stack frame. def int_stackprotector : Intrinsic<[], [llvm_ptr_ty, llvm_ptrptr_ty], []>; -def int_stackprotectorcheck : Intrinsic<[], [llvm_ptrptr_ty], - [IntrReadWriteArgMem]>; +def int_stackguard : Intrinsic<[llvm_ptr_ty], [], []>; // A counter increment for instrumentation based profiling. def int_instrprof_increment : Intrinsic<[], @@ -347,19 +360,19 @@ def int_instrprof_value_profile : Intrinsic<[], def int_memcpy : Intrinsic<[], [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty, llvm_i1_ty], - [IntrReadWriteArgMem, NoCapture<0>, NoCapture<1>, - ReadOnly<1>]>; + [IntrArgMemOnly, NoCapture<0>, NoCapture<1>, + WriteOnly<0>, ReadOnly<1>]>; def int_memmove : Intrinsic<[], [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i32_ty, llvm_i1_ty], - [IntrReadWriteArgMem, NoCapture<0>, NoCapture<1>, + [IntrArgMemOnly, NoCapture<0>, NoCapture<1>, ReadOnly<1>]>; def int_memset : Intrinsic<[], [llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i32_ty, llvm_i1_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>; -let Properties = [IntrNoMem] in { +let IntrProperties = [IntrNoMem] in { def int_fma : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; @@ -382,8 +395,6 @@ let Properties = [IntrNoMem] in { def int_exp : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; def int_exp2 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; def int_fabs : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; - def int_minnum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; - def int_maxnum : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; def int_copysign : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; def int_floor : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; @@ -396,6 +407,13 @@ let Properties = [IntrNoMem] in { [IntrNoMem]>; } +def int_minnum : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, Commutative] +>; +def int_maxnum : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, Commutative] +>; + // NOTE: these are internal interfaces. def int_setjmp : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>; def int_longjmp : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [IntrNoReturn]>; @@ -416,7 +434,7 @@ def int_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, // // None of these intrinsics accesses memory at all. -let Properties = [IntrNoMem] in { +let IntrProperties = [IntrNoMem] in { def int_bswap: Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; def int_ctpop: Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; def int_ctlz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; @@ -430,7 +448,7 @@ let Properties = [IntrNoMem] in { // None of these intrinsics accesses memory at all...but that doesn't mean the // optimizers can change them aggressively. Special handling needed in a few // places. -let Properties = [IntrNoMem] in { +let IntrProperties = [IntrNoMem] in { def int_dbg_declare : Intrinsic<[], [llvm_metadata_ty, llvm_metadata_ty, @@ -467,7 +485,7 @@ def int_eh_unwind_init: Intrinsic<[]>, def int_eh_dwarf_cfa : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty]>; -let Properties = [IntrNoMem] in { +let IntrProperties = [IntrNoMem] in { def int_eh_sjlj_lsda : Intrinsic<[llvm_ptr_ty]>; def int_eh_sjlj_callsite : Intrinsic<[], [llvm_i32_ty]>; } @@ -495,11 +513,11 @@ def int_annotation : Intrinsic<[llvm_anyint_ty], // def int_init_trampoline : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty], - [IntrReadWriteArgMem, NoCapture<0>]>, + [IntrArgMemOnly, NoCapture<0>]>, GCCBuiltin<"__builtin_init_trampoline">; def int_adjust_trampoline : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], - [IntrReadArgMem]>, + [IntrReadMem, IntrArgMemOnly]>, GCCBuiltin<"__builtin_adjust_trampoline">; //===------------------------ Overflow Intrinsics -------------------------===// @@ -531,17 +549,17 @@ def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty], // def int_lifetime_start : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], - [IntrReadWriteArgMem, NoCapture<1>]>; + [IntrArgMemOnly, NoCapture<1>]>; def int_lifetime_end : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], - [IntrReadWriteArgMem, NoCapture<1>]>; + [IntrArgMemOnly, NoCapture<1>]>; def int_invariant_start : Intrinsic<[llvm_descriptor_ty], [llvm_i64_ty, llvm_ptr_ty], - [IntrReadWriteArgMem, NoCapture<1>]>; + [IntrArgMemOnly, NoCapture<1>]>; def int_invariant_end : Intrinsic<[], [llvm_descriptor_ty, llvm_i64_ty, llvm_ptr_ty], - [IntrReadWriteArgMem, NoCapture<2>]>; + [IntrArgMemOnly, NoCapture<2>]>; def int_invariant_group_barrier : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], @@ -588,11 +606,19 @@ def int_trap : Intrinsic<[], [], [IntrNoReturn]>, def int_debugtrap : Intrinsic<[]>, GCCBuiltin<"__builtin_debugtrap">; +// Support for dynamic deoptimization (or de-specialization) +def int_experimental_deoptimize : Intrinsic<[llvm_any_ty], [llvm_vararg_ty], + [Throws]>; + +// Support for speculative runtime guards +def int_experimental_guard : Intrinsic<[], [llvm_i1_ty, llvm_vararg_ty], + [Throws]>; + // NOP: calls/invokes to this intrinsic are removed by codegen def int_donothing : Intrinsic<[], [], [IntrNoMem]>; // Intrisics to support half precision floating point format -let Properties = [IntrNoMem] in { +let IntrProperties = [IntrNoMem] in { def int_convert_to_fp16 : Intrinsic<[llvm_i16_ty], [llvm_anyfloat_ty]>; def int_convert_from_fp16 : Intrinsic<[llvm_anyfloat_ty], [llvm_i16_ty]>; } @@ -627,31 +653,40 @@ def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], //===-------------------------- Masked Intrinsics -------------------------===// // -def int_masked_store : Intrinsic<[], [llvm_anyvector_ty, LLVMPointerTo<0>, +def int_masked_store : Intrinsic<[], [llvm_anyvector_ty, + LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty, LLVMVectorSameWidth<0, llvm_i1_ty>], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_masked_load : Intrinsic<[llvm_anyvector_ty], - [LLVMPointerTo<0>, llvm_i32_ty, + [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty, LLVMVectorSameWidth<0, llvm_i1_ty>, LLVMMatchType<0>], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_masked_gather: Intrinsic<[llvm_anyvector_ty], [LLVMVectorOfPointersToElt<0>, llvm_i32_ty, LLVMVectorSameWidth<0, llvm_i1_ty>, LLVMMatchType<0>], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_masked_scatter: Intrinsic<[], [llvm_anyvector_ty, LLVMVectorOfPointersToElt<0>, llvm_i32_ty, LLVMVectorSameWidth<0, llvm_i1_ty>], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; + +// Test whether a pointer is associated with a type metadata identifier. +def int_type_test : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_metadata_ty], + [IntrNoMem]>; + +// Safely loads a function pointer from a virtual table pointer using type metadata. +def int_type_checked_load : Intrinsic<[llvm_ptr_ty, llvm_i1_ty], + [llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty], + [IntrNoMem]>; -// Intrinsics to support bit sets. -def int_bitset_test : Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_metadata_ty], - [IntrNoMem]>; +def int_load_relative: Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty], + [IntrReadMem, IntrArgMemOnly]>; //===----------------------------------------------------------------------===// // Target-specific intrinsics diff --git a/include/llvm/IR/IntrinsicsAArch64.td b/include/llvm/IR/IntrinsicsAArch64.td index 578f259aae14..d1e331775b7b 100644 --- a/include/llvm/IR/IntrinsicsAArch64.td +++ b/include/llvm/IR/IntrinsicsAArch64.td @@ -13,9 +13,6 @@ let TargetPrefix = "aarch64" in { -def int_aarch64_thread_pointer : GCCBuiltin<"__builtin_thread_pointer">, - Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; - def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>; @@ -159,7 +156,7 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". // Arithmetic ops -let Properties = [IntrNoMem] in { +let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { // Vector Add Across Lanes def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; @@ -212,7 +209,7 @@ let Properties = [IntrNoMem] in { // Vector Extending Multiply def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic { - let Properties = [IntrNoMem, Commutative]; + let IntrProperties = [IntrNoMem, Commutative]; } // Vector Saturating Doubling Long Multiply @@ -436,70 +433,70 @@ def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic; let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". class AdvSIMD_1Vec_Load_Intrinsic : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_1Vec_Store_Lane_Intrinsic : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], - [IntrReadWriteArgMem, NoCapture<2>]>; + [IntrArgMemOnly, NoCapture<2>]>; class AdvSIMD_2Vec_Load_Intrinsic : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [LLVMAnyPointerType<LLVMMatchType<0>>], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_2Vec_Load_Lane_Intrinsic : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i64_ty, llvm_anyptr_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_2Vec_Store_Intrinsic : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>], - [IntrReadWriteArgMem, NoCapture<2>]>; + [IntrArgMemOnly, NoCapture<2>]>; class AdvSIMD_2Vec_Store_Lane_Intrinsic : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, llvm_i64_ty, llvm_anyptr_ty], - [IntrReadWriteArgMem, NoCapture<3>]>; + [IntrArgMemOnly, NoCapture<3>]>; class AdvSIMD_3Vec_Load_Intrinsic : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], [LLVMAnyPointerType<LLVMMatchType<0>>], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_3Vec_Load_Lane_Intrinsic : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i64_ty, llvm_anyptr_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_3Vec_Store_Intrinsic : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>], - [IntrReadWriteArgMem, NoCapture<3>]>; + [IntrArgMemOnly, NoCapture<3>]>; class AdvSIMD_3Vec_Store_Lane_Intrinsic : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i64_ty, llvm_anyptr_ty], - [IntrReadWriteArgMem, NoCapture<4>]>; + [IntrArgMemOnly, NoCapture<4>]>; class AdvSIMD_4Vec_Load_Intrinsic : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [LLVMAnyPointerType<LLVMMatchType<0>>], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_4Vec_Load_Lane_Intrinsic : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i64_ty, llvm_anyptr_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_4Vec_Store_Intrinsic : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>], - [IntrReadWriteArgMem, NoCapture<4>]>; + [IntrArgMemOnly, NoCapture<4>]>; class AdvSIMD_4Vec_Store_Lane_Intrinsic : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i64_ty, llvm_anyptr_ty], - [IntrReadWriteArgMem, NoCapture<5>]>; + [IntrArgMemOnly, NoCapture<5>]>; } // Memory ops diff --git a/include/llvm/IR/IntrinsicsAMDGPU.td b/include/llvm/IR/IntrinsicsAMDGPU.td index 84582e8b9925..9bf2a4dd5a1d 100644 --- a/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/include/llvm/IR/IntrinsicsAMDGPU.td @@ -11,28 +11,45 @@ // //===----------------------------------------------------------------------===// +class AMDGPUReadPreloadRegisterIntrinsic + : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; + +class AMDGPUReadPreloadRegisterIntrinsicNamed<string name> + : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, GCCBuiltin<name>; + let TargetPrefix = "r600" in { -class R600ReadPreloadRegisterIntrinsic<string name> - : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<name>; +multiclass AMDGPUReadPreloadRegisterIntrinsic_xyz { + def _x : AMDGPUReadPreloadRegisterIntrinsic; + def _y : AMDGPUReadPreloadRegisterIntrinsic; + def _z : AMDGPUReadPreloadRegisterIntrinsic; +} -multiclass R600ReadPreloadRegisterIntrinsic_xyz<string prefix> { - def _x : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_x")>; - def _y : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_y")>; - def _z : R600ReadPreloadRegisterIntrinsic<!strconcat(prefix, "_z")>; +multiclass AMDGPUReadPreloadRegisterIntrinsic_xyz_named<string prefix> { + def _x : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_x")>; + def _y : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_y")>; + def _z : AMDGPUReadPreloadRegisterIntrinsicNamed<!strconcat(prefix, "_z")>; } -defm int_r600_read_global_size : R600ReadPreloadRegisterIntrinsic_xyz < - "__builtin_r600_read_global_size">; -defm int_r600_read_local_size : R600ReadPreloadRegisterIntrinsic_xyz < - "__builtin_r600_read_local_size">; -defm int_r600_read_ngroups : R600ReadPreloadRegisterIntrinsic_xyz < - "__builtin_r600_read_ngroups">; -defm int_r600_read_tgid : R600ReadPreloadRegisterIntrinsic_xyz < - "__builtin_r600_read_tgid">; -defm int_r600_read_tidig : R600ReadPreloadRegisterIntrinsic_xyz < - "__builtin_r600_read_tidig">; +defm int_r600_read_global_size : AMDGPUReadPreloadRegisterIntrinsic_xyz_named + <"__builtin_r600_read_global_size">; +defm int_r600_read_ngroups : AMDGPUReadPreloadRegisterIntrinsic_xyz_named + <"__builtin_r600_read_ngroups">; +defm int_r600_read_tgid : AMDGPUReadPreloadRegisterIntrinsic_xyz_named + <"__builtin_r600_read_tgid">; + +defm int_r600_read_local_size : AMDGPUReadPreloadRegisterIntrinsic_xyz; +defm int_r600_read_tidig : AMDGPUReadPreloadRegisterIntrinsic_xyz; + +def int_r600_read_workdim : AMDGPUReadPreloadRegisterIntrinsic; + +def int_r600_group_barrier : GCCBuiltin<"__builtin_r600_group_barrier">, + Intrinsic<[], [], [IntrConvergent]>; + +// AS 7 is PARAM_I_ADDRESS, used for kernel arguments +def int_r600_implicitarg_ptr : + GCCBuiltin<"__builtin_r600_implicitarg_ptr">, + Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 7>], [], [IntrNoMem]>; def int_r600_rat_store_typed : // 1st parameter: Data @@ -41,69 +58,253 @@ def int_r600_rat_store_typed : Intrinsic<[], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], []>, GCCBuiltin<"__builtin_r600_rat_store_typed">; +def int_r600_recipsqrt_ieee : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_r600_recipsqrt_clamped : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + } // End TargetPrefix = "r600" -let TargetPrefix = "AMDGPU" in { +let TargetPrefix = "amdgcn" in { -class AMDGPUReadPreloadRegisterIntrinsic<string name> - : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<name>; +defm int_amdgcn_workitem_id : AMDGPUReadPreloadRegisterIntrinsic_xyz; +defm int_amdgcn_workgroup_id : AMDGPUReadPreloadRegisterIntrinsic_xyz_named + <"__builtin_amdgcn_workgroup_id">; -def int_AMDGPU_div_scale : GCCBuiltin<"__builtin_amdgpu_div_scale">, +def int_amdgcn_s_barrier : GCCBuiltin<"__builtin_amdgcn_s_barrier">, + Intrinsic<[], [], [IntrConvergent]>; + +def int_amdgcn_s_waitcnt : Intrinsic<[], [llvm_i32_ty], []>; + +def int_amdgcn_div_scale : Intrinsic< // 1st parameter: Numerator // 2nd parameter: Denominator // 3rd parameter: Constant to select select between first and // second. (0 = first, 1 = second). - Intrinsic<[llvm_anyfloat_ty, llvm_i1_ty], - [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty], - [IntrNoMem]>; - -def int_AMDGPU_div_fmas : GCCBuiltin<"__builtin_amdgpu_div_fmas">, - Intrinsic<[llvm_anyfloat_ty], - [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty], - [IntrNoMem]>; - -def int_AMDGPU_div_fixup : GCCBuiltin<"__builtin_amdgpu_div_fixup">, - Intrinsic<[llvm_anyfloat_ty], - [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], - [IntrNoMem]>; - -def int_AMDGPU_trig_preop : GCCBuiltin<"__builtin_amdgpu_trig_preop">, - Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem]>; - -def int_AMDGPU_rcp : GCCBuiltin<"__builtin_amdgpu_rcp">, - Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; - -def int_AMDGPU_rsq : GCCBuiltin<"__builtin_amdgpu_rsq">, - Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; - -def int_AMDGPU_rsq_clamped : GCCBuiltin<"__builtin_amdgpu_rsq_clamped">, - Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; - -def int_AMDGPU_ldexp : GCCBuiltin<"__builtin_amdgpu_ldexp">, - Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem]>; + [llvm_anyfloat_ty, llvm_i1_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty], + [IntrNoMem] +>; + +def int_amdgcn_div_fmas : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty], + [IntrNoMem] +>; + +def int_amdgcn_div_fixup : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem] +>; + +def int_amdgcn_trig_preop : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem] +>; + +def int_amdgcn_sin : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_amdgcn_cos : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_amdgcn_log_clamp : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_amdgcn_rcp : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_amdgcn_rsq : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_amdgcn_rsq_legacy : GCCBuiltin<"__builtin_amdgcn_rsq_legacy">, + Intrinsic< + [llvm_float_ty], [llvm_float_ty], [IntrNoMem] +>; + +def int_amdgcn_rsq_clamp : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; + +def int_amdgcn_ldexp : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem] +>; + +def int_amdgcn_frexp_mant : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_amdgcn_frexp_exp : Intrinsic< + [llvm_i32_ty], [llvm_anyfloat_ty], [IntrNoMem] +>; + +// v_fract is buggy on SI/CI. It mishandles infinities, may return 1.0 +// and always uses rtz, so is not suitable for implementing the OpenCL +// fract function. It should be ok on VI. +def int_amdgcn_fract : Intrinsic< + [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem] +>; + +def int_amdgcn_class : Intrinsic< + [llvm_i1_ty], [llvm_anyfloat_ty, llvm_i32_ty], [IntrNoMem] +>; + +def int_amdgcn_cubeid : GCCBuiltin<"__builtin_amdgcn_cubeid">, + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem] +>; -def int_AMDGPU_class : GCCBuiltin<"__builtin_amdgpu_class">, - Intrinsic<[llvm_i1_ty], [llvm_anyfloat_ty, llvm_i32_ty], [IntrNoMem]>; +def int_amdgcn_cubema : GCCBuiltin<"__builtin_amdgcn_cubema">, + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem] +>; -def int_AMDGPU_read_workdim : AMDGPUReadPreloadRegisterIntrinsic < - "__builtin_amdgpu_read_workdim">; +def int_amdgcn_cubesc : GCCBuiltin<"__builtin_amdgcn_cubesc">, + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem] +>; -} // End TargetPrefix = "AMDGPU" +def int_amdgcn_cubetc : GCCBuiltin<"__builtin_amdgcn_cubetc">, + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_float_ty, llvm_float_ty], [IntrNoMem] +>; + +// TODO: Do we want an ordering for these? +def int_amdgcn_atomic_inc : Intrinsic<[llvm_anyint_ty], + [llvm_anyptr_ty, LLVMMatchType<0>], + [IntrArgMemOnly, NoCapture<0>] +>; + +def int_amdgcn_atomic_dec : Intrinsic<[llvm_anyint_ty], + [llvm_anyptr_ty, LLVMMatchType<0>], + [IntrArgMemOnly, NoCapture<0>] +>; + +class AMDGPUImageLoad : Intrinsic < + [llvm_v4f32_ty], // vdata(VGPR) + [llvm_anyint_ty, // vaddr(VGPR) + llvm_v8i32_ty, // rsrc(SGPR) + llvm_i32_ty, // dmask(imm) + llvm_i1_ty, // r128(imm) + llvm_i1_ty, // da(imm) + llvm_i1_ty, // glc(imm) + llvm_i1_ty], // slc(imm) + [IntrReadMem]>; + +def int_amdgcn_image_load : AMDGPUImageLoad; +def int_amdgcn_image_load_mip : AMDGPUImageLoad; + +class AMDGPUImageStore : Intrinsic < + [], + [llvm_v4f32_ty, // vdata(VGPR) + llvm_anyint_ty, // vaddr(VGPR) + llvm_v8i32_ty, // rsrc(SGPR) + llvm_i32_ty, // dmask(imm) + llvm_i1_ty, // r128(imm) + llvm_i1_ty, // da(imm) + llvm_i1_ty, // glc(imm) + llvm_i1_ty], // slc(imm) + []>; + +def int_amdgcn_image_store : AMDGPUImageStore; +def int_amdgcn_image_store_mip : AMDGPUImageStore; + +class AMDGPUImageAtomic : Intrinsic < + [llvm_i32_ty], + [llvm_i32_ty, // vdata(VGPR) + llvm_anyint_ty, // vaddr(VGPR) + llvm_v8i32_ty, // rsrc(SGPR) + llvm_i1_ty, // r128(imm) + llvm_i1_ty, // da(imm) + llvm_i1_ty], // slc(imm) + []>; + +def int_amdgcn_image_atomic_swap : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_add : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_sub : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_smin : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_umin : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_smax : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_umax : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_and : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_or : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_xor : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_inc : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_dec : AMDGPUImageAtomic; +def int_amdgcn_image_atomic_cmpswap : Intrinsic < + [llvm_i32_ty], + [llvm_i32_ty, // src(VGPR) + llvm_i32_ty, // cmp(VGPR) + llvm_anyint_ty, // vaddr(VGPR) + llvm_v8i32_ty, // rsrc(SGPR) + llvm_i1_ty, // r128(imm) + llvm_i1_ty, // da(imm) + llvm_i1_ty], // slc(imm) + []>; + +class AMDGPUBufferLoad : Intrinsic < + [llvm_anyfloat_ty], + [llvm_v4i32_ty, // rsrc(SGPR) + llvm_i32_ty, // vindex(VGPR) + llvm_i32_ty, // offset(SGPR/VGPR/imm) + llvm_i1_ty, // glc(imm) + llvm_i1_ty], // slc(imm) + [IntrReadMem]>; +def int_amdgcn_buffer_load_format : AMDGPUBufferLoad; +def int_amdgcn_buffer_load : AMDGPUBufferLoad; + +class AMDGPUBufferStore : Intrinsic < + [], + [llvm_anyfloat_ty, // vdata(VGPR) -- can currently only select f32, v2f32, v4f32 + llvm_v4i32_ty, // rsrc(SGPR) + llvm_i32_ty, // vindex(VGPR) + llvm_i32_ty, // offset(SGPR/VGPR/imm) + llvm_i1_ty, // glc(imm) + llvm_i1_ty], // slc(imm) + [IntrWriteMem]>; +def int_amdgcn_buffer_store_format : AMDGPUBufferStore; +def int_amdgcn_buffer_store : AMDGPUBufferStore; + +class AMDGPUBufferAtomic : Intrinsic < + [llvm_i32_ty], + [llvm_i32_ty, // vdata(VGPR) + llvm_v4i32_ty, // rsrc(SGPR) + llvm_i32_ty, // vindex(VGPR) + llvm_i32_ty, // offset(SGPR/VGPR/imm) + llvm_i1_ty], // slc(imm) + []>; +def int_amdgcn_buffer_atomic_swap : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_add : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_sub : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_smin : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_umin : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_smax : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_umax : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_and : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_or : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_xor : AMDGPUBufferAtomic; +def int_amdgcn_buffer_atomic_cmpswap : Intrinsic< + [llvm_i32_ty], + [llvm_i32_ty, // src(VGPR) + llvm_i32_ty, // cmp(VGPR) + llvm_v4i32_ty, // rsrc(SGPR) + llvm_i32_ty, // vindex(VGPR) + llvm_i32_ty, // offset(SGPR/VGPR/imm) + llvm_i1_ty], // slc(imm) + []>; + +def int_amdgcn_read_workdim : AMDGPUReadPreloadRegisterIntrinsic; -let TargetPrefix = "amdgcn" in { -// SI only def int_amdgcn_buffer_wbinvl1_sc : GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1_sc">, Intrinsic<[], [], []>; -// On CI+ -def int_amdgcn_buffer_wbinvl1_vol : - GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1_vol">, - Intrinsic<[], [], []>; - def int_amdgcn_buffer_wbinvl1 : GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1">, Intrinsic<[], [], []>; @@ -112,25 +313,39 @@ def int_amdgcn_s_dcache_inv : GCCBuiltin<"__builtin_amdgcn_s_dcache_inv">, Intrinsic<[], [], []>; -// CI+ -def int_amdgcn_s_dcache_inv_vol : - GCCBuiltin<"__builtin_amdgcn_s_dcache_inv_vol">, - Intrinsic<[], [], []>; +def int_amdgcn_s_memtime : + GCCBuiltin<"__builtin_amdgcn_s_memtime">, + Intrinsic<[llvm_i64_ty], [], []>; -// VI -def int_amdgcn_s_dcache_wb : - GCCBuiltin<"__builtin_amdgcn_s_dcache_wb">, - Intrinsic<[], [], []>; +def int_amdgcn_s_sleep : + GCCBuiltin<"__builtin_amdgcn_s_sleep">, + Intrinsic<[], [llvm_i32_ty], []> { +} -// VI -def int_amdgcn_s_dcache_wb_vol : - GCCBuiltin<"__builtin_amdgcn_s_dcache_wb_vol">, - Intrinsic<[], [], []>; +def int_amdgcn_s_getreg : + GCCBuiltin<"__builtin_amdgcn_s_getreg">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem]>; + +def int_amdgcn_groupstaticsize : + GCCBuiltin<"__builtin_amdgcn_groupstaticsize">, + Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; def int_amdgcn_dispatch_ptr : GCCBuiltin<"__builtin_amdgcn_dispatch_ptr">, Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 2>], [], [IntrNoMem]>; +def int_amdgcn_queue_ptr : + GCCBuiltin<"__builtin_amdgcn_queue_ptr">, + Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 2>], [], [IntrNoMem]>; + +def int_amdgcn_kernarg_segment_ptr : + GCCBuiltin<"__builtin_amdgcn_kernarg_segment_ptr">, + Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 2>], [], [IntrNoMem]>; + +def int_amdgcn_implicitarg_ptr : + GCCBuiltin<"__builtin_amdgcn_implicitarg_ptr">, + Intrinsic<[LLVMQualPointerType<llvm_i8_ty, 2>], [], [IntrNoMem]>; + // __builtin_amdgcn_interp_p1 <i>, <attr_chan>, <attr>, <m0> def int_amdgcn_interp_p1 : GCCBuiltin<"__builtin_amdgcn_interp_p1">, @@ -147,6 +362,13 @@ def int_amdgcn_interp_p2 : [IntrNoMem]>; // See int_amdgcn_v_interp_p1 for why this is // IntrNoMem. +// Pixel shaders only: whether the current pixel is live (i.e. not a helper +// invocation for derivative computation). +def int_amdgcn_ps_live : Intrinsic < + [llvm_i1_ty], + [], + [IntrNoMem]>; + def int_amdgcn_mbcnt_lo : GCCBuiltin<"__builtin_amdgcn_mbcnt_lo">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; @@ -154,4 +376,57 @@ def int_amdgcn_mbcnt_lo : def int_amdgcn_mbcnt_hi : GCCBuiltin<"__builtin_amdgcn_mbcnt_hi">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + +// llvm.amdgcn.ds.swizzle src offset +def int_amdgcn_ds_swizzle : + GCCBuiltin<"__builtin_amdgcn_ds_swizzle">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent]>; + +// llvm.amdgcn.lerp +def int_amdgcn_lerp : + GCCBuiltin<"__builtin_amdgcn_lerp">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + +//===----------------------------------------------------------------------===// +// CI+ Intrinsics +//===----------------------------------------------------------------------===// + +def int_amdgcn_s_dcache_inv_vol : + GCCBuiltin<"__builtin_amdgcn_s_dcache_inv_vol">, + Intrinsic<[], [], []>; + +def int_amdgcn_buffer_wbinvl1_vol : + GCCBuiltin<"__builtin_amdgcn_buffer_wbinvl1_vol">, + Intrinsic<[], [], []>; + +//===----------------------------------------------------------------------===// +// VI Intrinsics +//===----------------------------------------------------------------------===// + +// llvm.amdgcn.mov.dpp.i32 <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl> +def int_amdgcn_mov_dpp : + Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i1_ty], [IntrNoMem, IntrConvergent]>; + +def int_amdgcn_s_dcache_wb : + GCCBuiltin<"__builtin_amdgcn_s_dcache_wb">, + Intrinsic<[], [], []>; + +def int_amdgcn_s_dcache_wb_vol : + GCCBuiltin<"__builtin_amdgcn_s_dcache_wb_vol">, + Intrinsic<[], [], []>; + +def int_amdgcn_s_memrealtime : + GCCBuiltin<"__builtin_amdgcn_s_memrealtime">, + Intrinsic<[llvm_i64_ty], [], []>; + +// llvm.amdgcn.ds.permute <index> <src> +def int_amdgcn_ds_permute : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent]>; + +// llvm.amdgcn.ds.bpermute <index> <src> +def int_amdgcn_ds_bpermute : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent]>; + } diff --git a/include/llvm/IR/IntrinsicsARM.td b/include/llvm/IR/IntrinsicsARM.td index c1d911cefee2..099598596885 100644 --- a/include/llvm/IR/IntrinsicsARM.td +++ b/include/llvm/IR/IntrinsicsARM.td @@ -17,9 +17,6 @@ let TargetPrefix = "arm" in { // All intrinsics start with "llvm.arm.". -def int_arm_thread_pointer : GCCBuiltin<"__builtin_thread_pointer">, - Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; - // A space-consuming intrinsic primarily for testing ARMConstantIslands. The // first argument is the number of bytes this "instruction" takes up, the second // and return value are essentially chains, used to force ordering during ISel. @@ -81,6 +78,24 @@ def int_arm_vcvtru : Intrinsic<[llvm_float_ty], [llvm_anyfloat_ty], //===----------------------------------------------------------------------===// // Coprocessor +def int_arm_ldc : GCCBuiltin<"__builtin_arm_ldc">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; +def int_arm_ldcl : GCCBuiltin<"__builtin_arm_ldcl">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; +def int_arm_ldc2 : GCCBuiltin<"__builtin_arm_ldc2">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; +def int_arm_ldc2l : GCCBuiltin<"__builtin_arm_ldc2l">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + +def int_arm_stc : GCCBuiltin<"__builtin_arm_stc">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; +def int_arm_stcl : GCCBuiltin<"__builtin_arm_stcl">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; +def int_arm_stc2 : GCCBuiltin<"__builtin_arm_stc2">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; +def int_arm_stc2l : GCCBuiltin<"__builtin_arm_stc2l">, + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + // Move to coprocessor def int_arm_mcr : GCCBuiltin<"__builtin_arm_mcr">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, @@ -108,12 +123,15 @@ def int_arm_cdp2 : GCCBuiltin<"__builtin_arm_cdp2">, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; // Move from two registers to coprocessor -def int_arm_mcrr : GCCBuiltin<"__builtin_arm_mcrr">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; -def int_arm_mcrr2 : GCCBuiltin<"__builtin_arm_mcrr2">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; +def int_arm_mcrr : Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], []>; +def int_arm_mcrr2 : Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], []>; + +def int_arm_mrrc : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], []>; +def int_arm_mrrc2 : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty], []>; //===----------------------------------------------------------------------===// // CRC32 @@ -207,7 +225,7 @@ class Neon_Tbl6Arg_Intrinsic // Arithmetic ops -let Properties = [IntrNoMem, Commutative] in { +let IntrProperties = [IntrNoMem, Commutative] in { // Vector Add. def int_arm_neon_vhadds : Neon_2Arg_Intrinsic; @@ -406,18 +424,18 @@ def int_arm_neon_vrintp : Neon_1Arg_Intrinsic; // Source operands are the address and alignment. def int_arm_neon_vld1 : Intrinsic<[llvm_anyvector_ty], [llvm_anyptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_arm_neon_vld2 : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_anyptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_arm_neon_vld3 : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], [llvm_anyptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_arm_neon_vld4 : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [llvm_anyptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; // Vector load N-element structure to one lane. // Source operands are: the address, the N input vectors (since only one @@ -425,38 +443,38 @@ def int_arm_neon_vld4 : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, def int_arm_neon_vld2lane : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_anyptr_ty, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, - llvm_i32_ty], [IntrReadArgMem]>; + llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; def int_arm_neon_vld3lane : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], [llvm_anyptr_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_arm_neon_vld4lane : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [llvm_anyptr_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, - llvm_i32_ty], [IntrReadArgMem]>; + llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; // Interleaving vector stores from N-element structures. // Source operands are: the address, the N vectors, and the alignment. def int_arm_neon_vst1 : Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, - llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty], [IntrArgMemOnly]>; def int_arm_neon_vst2 : Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_arm_neon_vst3 : Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, - llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty], [IntrArgMemOnly]>; def int_arm_neon_vst4 : Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; // Vector store N-element structure from one lane. // Source operands are: the address, the N vectors, the lane number, and @@ -464,17 +482,17 @@ def int_arm_neon_vst4 : Intrinsic<[], def int_arm_neon_vst2lane : Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty, - llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty], [IntrArgMemOnly]>; def int_arm_neon_vst3lane : Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_arm_neon_vst4lane : Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty, - llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty], [IntrArgMemOnly]>; // Vector bitwise select. def int_arm_neon_vbsl : Intrinsic<[llvm_anyvector_ty], diff --git a/include/llvm/IR/IntrinsicsHexagon.td b/include/llvm/IR/IntrinsicsHexagon.td index ca6fcbd44337..6519f051deeb 100644 --- a/include/llvm/IR/IntrinsicsHexagon.td +++ b/include/llvm/IR/IntrinsicsHexagon.td @@ -428,42 +428,42 @@ class Hexagon_mem_memmemsi_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; class Hexagon_mem_memsisi_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_ptr_ty], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; class Hexagon_mem_memdisi_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; class Hexagon_mem_memmemsisi_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; class Hexagon_mem_memsisisi_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_ptr_ty], [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; class Hexagon_mem_memdisisi_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; class Hexagon_v256_v256v256_Intrinsic<string GCCIntSuffix> : Hexagon_Intrinsic<GCCIntSuffix, [llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; // // Hexagon_sf_df_Intrinsic<string GCCIntSuffix> @@ -2998,7 +2998,7 @@ Hexagon_di_di_Intrinsic<"HEXAGON_A2_tfrp">; // BUILTIN_INFO(HEXAGON.A2_tfrpi,DI_ftype_SI,1) // def int_hexagon_A2_tfrpi : -Hexagon_di_di_Intrinsic<"HEXAGON_A2_tfrpi">; +Hexagon_di_si_Intrinsic<"HEXAGON_A2_tfrpi">; // // BUILTIN_INFO(HEXAGON.A2_zxtb,SI_ftype_SI,1) // @@ -4971,17 +4971,17 @@ def llvm_ptr64_ty : LLVMPointerType<llvm_i64_ty>; // Mark locked loads as read/write to prevent any accidental reordering. def int_hexagon_L2_loadw_locked : Hexagon_Intrinsic<"HEXAGON_L2_loadw_locked", [llvm_i32_ty], [llvm_ptr32_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; def int_hexagon_L4_loadd_locked : Hexagon_Intrinsic<"HEXAGON_L4_loadd_locked", [llvm_i64_ty], [llvm_ptr64_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; def int_hexagon_S2_storew_locked : Hexagon_Intrinsic<"HEXAGON_S2_storew_locked", [llvm_i32_ty], - [llvm_ptr32_ty, llvm_i32_ty], [IntrReadWriteArgMem, NoCapture<0>]>; + [llvm_ptr32_ty, llvm_i32_ty], [IntrArgMemOnly, NoCapture<0>]>; def int_hexagon_S4_stored_locked : Hexagon_Intrinsic<"HEXAGON_S4_stored_locked", [llvm_i32_ty], - [llvm_ptr64_ty, llvm_i64_ty], [IntrReadWriteArgMem, NoCapture<0>]>; + [llvm_ptr64_ty, llvm_i64_ty], [IntrArgMemOnly, NoCapture<0>]>; // V60 diff --git a/include/llvm/IR/IntrinsicsMips.td b/include/llvm/IR/IntrinsicsMips.td index 34557612cb96..421a79be4ebc 100644 --- a/include/llvm/IR/IntrinsicsMips.td +++ b/include/llvm/IR/IntrinsicsMips.td @@ -264,11 +264,11 @@ def int_mips_bposge32: GCCBuiltin<"__builtin_mips_bposge32">, Intrinsic<[llvm_i32_ty], [], [IntrReadMem]>; def int_mips_lbux: GCCBuiltin<"__builtin_mips_lbux">, - Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; def int_mips_lhx: GCCBuiltin<"__builtin_mips_lhx">, - Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; def int_mips_lwx: GCCBuiltin<"__builtin_mips_lwx">, - Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>; //===----------------------------------------------------------------------===// // MIPS DSP Rev 2 @@ -1261,16 +1261,16 @@ def int_mips_insve_d : GCCBuiltin<"__builtin_msa_insve_d">, def int_mips_ld_b : GCCBuiltin<"__builtin_msa_ld_b">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ld_h : GCCBuiltin<"__builtin_msa_ld_h">, Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ld_w : GCCBuiltin<"__builtin_msa_ld_w">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ld_d : GCCBuiltin<"__builtin_msa_ld_d">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_mips_ldi_b : GCCBuiltin<"__builtin_msa_ldi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty], [IntrNoMem]>; @@ -1685,16 +1685,16 @@ def int_mips_srlri_d : GCCBuiltin<"__builtin_msa_srlri_d">, def int_mips_st_b : GCCBuiltin<"__builtin_msa_st_b">, Intrinsic<[], [llvm_v16i8_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_mips_st_h : GCCBuiltin<"__builtin_msa_st_h">, Intrinsic<[], [llvm_v8i16_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_mips_st_w : GCCBuiltin<"__builtin_msa_st_w">, Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_mips_st_d : GCCBuiltin<"__builtin_msa_st_d">, Intrinsic<[], [llvm_v2i64_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_mips_subs_s_b : GCCBuiltin<"__builtin_msa_subs_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; diff --git a/include/llvm/IR/IntrinsicsNVVM.td b/include/llvm/IR/IntrinsicsNVVM.td index 9deed414b50a..6919ec47eb9a 100644 --- a/include/llvm/IR/IntrinsicsNVVM.td +++ b/include/llvm/IR/IntrinsicsNVVM.td @@ -17,6 +17,7 @@ def llvm_anyi64ptr_ty : LLVMAnyPointerType<llvm_i64_ty>; // (space)i64* // MISC // +let TargetPrefix = "nvvm" in { def int_nvvm_clz_i : GCCBuiltin<"__nvvm_clz_i">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; def int_nvvm_clz_ll : GCCBuiltin<"__nvvm_clz_ll">, @@ -720,25 +721,30 @@ def llvm_anyi64ptr_ty : LLVMAnyPointerType<llvm_i64_ty>; // (space)i64* // Atomic not available as an llvm intrinsic. def int_nvvm_atomic_load_add_f32 : Intrinsic<[llvm_float_ty], [LLVMAnyPointerType<llvm_float_ty>, llvm_float_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; def int_nvvm_atomic_load_inc_32 : Intrinsic<[llvm_i32_ty], [LLVMAnyPointerType<llvm_i32_ty>, llvm_i32_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; def int_nvvm_atomic_load_dec_32 : Intrinsic<[llvm_i32_ty], [LLVMAnyPointerType<llvm_i32_ty>, llvm_i32_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; // Bar.Sync - def int_cuda_syncthreads : GCCBuiltin<"__syncthreads">, - Intrinsic<[], [], [IntrNoDuplicate]>; - def int_nvvm_barrier0 : GCCBuiltin<"__nvvm_bar0">, - Intrinsic<[], [], [IntrNoDuplicate]>; + + // The builtin for "bar.sync 0" is called __syncthreads. Unlike most of the + // intrinsics in this file, this one is a user-facing API. + def int_nvvm_barrier0 : GCCBuiltin<"__syncthreads">, + Intrinsic<[], [], [IntrConvergent]>; def int_nvvm_barrier0_popc : GCCBuiltin<"__nvvm_bar0_popc">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoDuplicate]>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>; def int_nvvm_barrier0_and : GCCBuiltin<"__nvvm_bar0_and">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoDuplicate]>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>; def int_nvvm_barrier0_or : GCCBuiltin<"__nvvm_bar0_or">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoDuplicate]>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>; + + def int_nvvm_bar_sync : + Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, + GCCBuiltin<"__nvvm_bar_sync">; // Membar def int_nvvm_membar_cta : GCCBuiltin<"__nvvm_membar_cta">, @@ -748,79 +754,34 @@ def llvm_anyi64ptr_ty : LLVMAnyPointerType<llvm_i64_ty>; // (space)i64* def int_nvvm_membar_sys : GCCBuiltin<"__nvvm_membar_sys">, Intrinsic<[], [], []>; - -// Accessing special registers - def int_nvvm_read_ptx_sreg_tid_x : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_tid_x">; - def int_nvvm_read_ptx_sreg_tid_y : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_tid_y">; - def int_nvvm_read_ptx_sreg_tid_z : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_tid_z">; - - def int_nvvm_read_ptx_sreg_ntid_x : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_ntid_x">; - def int_nvvm_read_ptx_sreg_ntid_y : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_ntid_y">; - def int_nvvm_read_ptx_sreg_ntid_z : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_ntid_z">; - - def int_nvvm_read_ptx_sreg_ctaid_x : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_ctaid_x">; - def int_nvvm_read_ptx_sreg_ctaid_y : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_ctaid_y">; - def int_nvvm_read_ptx_sreg_ctaid_z : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_ctaid_z">; - - def int_nvvm_read_ptx_sreg_nctaid_x : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_nctaid_x">; - def int_nvvm_read_ptx_sreg_nctaid_y : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_nctaid_y">; - def int_nvvm_read_ptx_sreg_nctaid_z : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_nctaid_z">; - - def int_nvvm_read_ptx_sreg_warpsize : - Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<"__nvvm_read_ptx_sreg_warpsize">; - - -// Generated within nvvm. Use for ldu on sm_20 or later +// Generated within nvvm. Use for ldu on sm_20 or later. Second arg is the +// pointer's alignment. def int_nvvm_ldu_global_i : Intrinsic<[llvm_anyint_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, NoCapture<0>], + [IntrReadMem, IntrArgMemOnly, NoCapture<0>], "llvm.nvvm.ldu.global.i">; def int_nvvm_ldu_global_f : Intrinsic<[llvm_anyfloat_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, NoCapture<0>], + [IntrReadMem, IntrArgMemOnly, NoCapture<0>], "llvm.nvvm.ldu.global.f">; def int_nvvm_ldu_global_p : Intrinsic<[llvm_anyptr_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, NoCapture<0>], + [IntrReadMem, IntrArgMemOnly, NoCapture<0>], "llvm.nvvm.ldu.global.p">; -// Generated within nvvm. Use for ldg on sm_35 or later +// Generated within nvvm. Use for ldg on sm_35 or later. Second arg is the +// pointer's alignment. def int_nvvm_ldg_global_i : Intrinsic<[llvm_anyint_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, NoCapture<0>], + [IntrReadMem, IntrArgMemOnly, NoCapture<0>], "llvm.nvvm.ldg.global.i">; def int_nvvm_ldg_global_f : Intrinsic<[llvm_anyfloat_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, NoCapture<0>], + [IntrReadMem, IntrArgMemOnly, NoCapture<0>], "llvm.nvvm.ldg.global.f">; def int_nvvm_ldg_global_p : Intrinsic<[llvm_anyptr_ty], [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty], - [IntrReadMem, NoCapture<0>], + [IntrReadMem, IntrArgMemOnly, NoCapture<0>], "llvm.nvvm.ldg.global.p">; // Use for generic pointers @@ -3666,9 +3627,8 @@ def int_nvvm_swap_lo_hi_b64 GCCBuiltin<"__nvvm_swap_lo_hi_b64">; -// Old PTX back-end intrinsics retained here for backwards-compatibility - -multiclass PTXReadSpecialRegisterIntrinsic_v4i32<string prefix> { +// Accessing special registers. +multiclass PTXReadSRegIntrinsic_v4i32<string regname> { // FIXME: Do we need the 128-bit integer type version? // def _r64 : Intrinsic<[llvm_i128_ty], [], [IntrNoMem]>; @@ -3676,71 +3636,99 @@ multiclass PTXReadSpecialRegisterIntrinsic_v4i32<string prefix> { // def _v4i16 : Intrinsic<[llvm_v4i32_ty], [], [IntrNoMem]>; def _x : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<!strconcat(prefix, "_x")>; + GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_x">; def _y : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<!strconcat(prefix, "_y")>; + GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_y">; def _z : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<!strconcat(prefix, "_z")>; + GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_z">; def _w : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<!strconcat(prefix, "_w")>; + GCCBuiltin<"__nvvm_read_ptx_sreg_" # regname # "_w">; } -class PTXReadSpecialRegisterIntrinsic_r32<string name> +class PTXReadSRegIntrinsic_r32<string name> : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, - GCCBuiltin<name>; + GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; -class PTXReadSpecialRegisterIntrinsic_r64<string name> +class PTXReadSRegIntrinsic_r64<string name> : Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>, - GCCBuiltin<name>; - -defm int_ptx_read_tid : PTXReadSpecialRegisterIntrinsic_v4i32 - <"__builtin_ptx_read_tid">; -defm int_ptx_read_ntid : PTXReadSpecialRegisterIntrinsic_v4i32 - <"__builtin_ptx_read_ntid">; - -def int_ptx_read_laneid : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_laneid">; -def int_ptx_read_warpid : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_warpid">; -def int_ptx_read_nwarpid : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_nwarpid">; - -defm int_ptx_read_ctaid : PTXReadSpecialRegisterIntrinsic_v4i32 - <"__builtin_ptx_read_ctaid">; -defm int_ptx_read_nctaid : PTXReadSpecialRegisterIntrinsic_v4i32 - <"__builtin_ptx_read_nctaid">; - -def int_ptx_read_smid : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_smid">; -def int_ptx_read_nsmid : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_nsmid">; -def int_ptx_read_gridid : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_gridid">; - -def int_ptx_read_lanemask_eq : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_lanemask_eq">; -def int_ptx_read_lanemask_le : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_lanemask_le">; -def int_ptx_read_lanemask_lt : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_lanemask_lt">; -def int_ptx_read_lanemask_ge : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_lanemask_ge">; -def int_ptx_read_lanemask_gt : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_lanemask_gt">; - -def int_ptx_read_clock : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_clock">; -def int_ptx_read_clock64 : PTXReadSpecialRegisterIntrinsic_r64 - <"__builtin_ptx_read_clock64">; - -def int_ptx_read_pm0 : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_pm0">; -def int_ptx_read_pm1 : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_pm1">; -def int_ptx_read_pm2 : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_pm2">; -def int_ptx_read_pm3 : PTXReadSpecialRegisterIntrinsic_r32 - <"__builtin_ptx_read_pm3">; - -def int_ptx_bar_sync : Intrinsic<[], [llvm_i32_ty], []>, - GCCBuiltin<"__builtin_ptx_bar_sync">; + GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; + +defm int_nvvm_read_ptx_sreg_tid : PTXReadSRegIntrinsic_v4i32<"tid">; +defm int_nvvm_read_ptx_sreg_ntid : PTXReadSRegIntrinsic_v4i32<"ntid">; + +def int_nvvm_read_ptx_sreg_laneid : PTXReadSRegIntrinsic_r32<"laneid">; +def int_nvvm_read_ptx_sreg_warpid : PTXReadSRegIntrinsic_r32<"warpid">; +def int_nvvm_read_ptx_sreg_nwarpid : PTXReadSRegIntrinsic_r32<"nwarpid">; + +defm int_nvvm_read_ptx_sreg_ctaid : PTXReadSRegIntrinsic_v4i32<"ctaid">; +defm int_nvvm_read_ptx_sreg_nctaid : PTXReadSRegIntrinsic_v4i32<"nctaid">; + +def int_nvvm_read_ptx_sreg_smid : PTXReadSRegIntrinsic_r32<"smid">; +def int_nvvm_read_ptx_sreg_nsmid : PTXReadSRegIntrinsic_r32<"nsmid">; +def int_nvvm_read_ptx_sreg_gridid : PTXReadSRegIntrinsic_r32<"gridid">; + +def int_nvvm_read_ptx_sreg_lanemask_eq : + PTXReadSRegIntrinsic_r32<"lanemask_eq">; +def int_nvvm_read_ptx_sreg_lanemask_le : + PTXReadSRegIntrinsic_r32<"lanemask_le">; +def int_nvvm_read_ptx_sreg_lanemask_lt : + PTXReadSRegIntrinsic_r32<"lanemask_lt">; +def int_nvvm_read_ptx_sreg_lanemask_ge : + PTXReadSRegIntrinsic_r32<"lanemask_ge">; +def int_nvvm_read_ptx_sreg_lanemask_gt : + PTXReadSRegIntrinsic_r32<"lanemask_gt">; + +def int_nvvm_read_ptx_sreg_clock : PTXReadSRegIntrinsic_r32<"clock">; +def int_nvvm_read_ptx_sreg_clock64 : PTXReadSRegIntrinsic_r64<"clock64">; + +def int_nvvm_read_ptx_sreg_pm0 : PTXReadSRegIntrinsic_r32<"pm0">; +def int_nvvm_read_ptx_sreg_pm1 : PTXReadSRegIntrinsic_r32<"pm1">; +def int_nvvm_read_ptx_sreg_pm2 : PTXReadSRegIntrinsic_r32<"pm2">; +def int_nvvm_read_ptx_sreg_pm3 : PTXReadSRegIntrinsic_r32<"pm3">; + +def int_nvvm_read_ptx_sreg_warpsize : PTXReadSRegIntrinsic_r32<"warpsize">; + +// +// SHUFFLE +// + +// shfl.down.b32 dest, val, offset, mask_and_clamp +def int_nvvm_shfl_down_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.down.i32">, + GCCBuiltin<"__nvvm_shfl_down_i32">; +def int_nvvm_shfl_down_f32 : + Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.down.f32">, + GCCBuiltin<"__nvvm_shfl_down_f32">; + +// shfl.up.b32 dest, val, offset, mask_and_clamp +def int_nvvm_shfl_up_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.up.i32">, + GCCBuiltin<"__nvvm_shfl_up_i32">; +def int_nvvm_shfl_up_f32 : + Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.up.f32">, + GCCBuiltin<"__nvvm_shfl_up_f32">; + +// shfl.bfly.b32 dest, val, offset, mask_and_clamp +def int_nvvm_shfl_bfly_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.bfly.i32">, + GCCBuiltin<"__nvvm_shfl_bfly_i32">; +def int_nvvm_shfl_bfly_f32 : + Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.bfly.f32">, + GCCBuiltin<"__nvvm_shfl_bfly_f32">; + +// shfl.idx.b32 dest, val, lane, mask_and_clamp +def int_nvvm_shfl_idx_i32 : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.idx.i32">, + GCCBuiltin<"__nvvm_shfl_idx_i32">; +def int_nvvm_shfl_idx_f32 : + Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.idx.f32">, + GCCBuiltin<"__nvvm_shfl_idx_f32">; +} diff --git a/include/llvm/IR/IntrinsicsPowerPC.td b/include/llvm/IR/IntrinsicsPowerPC.td index 5512b1063fb0..e195c0ebac3a 100644 --- a/include/llvm/IR/IntrinsicsPowerPC.td +++ b/include/llvm/IR/IntrinsicsPowerPC.td @@ -23,9 +23,9 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". def int_ppc_dcbi : Intrinsic<[], [llvm_ptr_ty], []>; def int_ppc_dcbst : Intrinsic<[], [llvm_ptr_ty], []>; def int_ppc_dcbt : Intrinsic<[], [llvm_ptr_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; def int_ppc_dcbtst: Intrinsic<[], [llvm_ptr_ty], - [IntrReadWriteArgMem, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>]>; def int_ppc_dcbz : Intrinsic<[], [llvm_ptr_ty], []>; def int_ppc_dcbzl : Intrinsic<[], [llvm_ptr_ty], []>; @@ -189,33 +189,33 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". // Loads. These don't map directly to GCC builtins because they represent the // source address with a single pointer. def int_ppc_altivec_lvx : - Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_ppc_altivec_lvxl : - Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_ppc_altivec_lvebx : - Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_ppc_altivec_lvehx : - Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_ppc_altivec_lvewx : - Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; // Stores. These don't map directly to GCC builtins because they represent the // source address with a single pointer. def int_ppc_altivec_stvx : Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_ppc_altivec_stvxl : Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_ppc_altivec_stvebx : Intrinsic<[], [llvm_v16i8_ty, llvm_ptr_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_ppc_altivec_stvehx : Intrinsic<[], [llvm_v8i16_ty, llvm_ptr_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_ppc_altivec_stvewx : Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; // Comparisons setting a vector. def int_ppc_altivec_vcmpbfp : GCCBuiltin<"__builtin_altivec_vcmpbfp">, @@ -664,15 +664,15 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". // Vector load. def int_ppc_vsx_lxvw4x : - Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_ppc_vsx_lxvd2x : - Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; // Vector store. def int_ppc_vsx_stxvw4x : - Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty], [IntrReadWriteArgMem]>; + Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty], [IntrArgMemOnly]>; def int_ppc_vsx_stxvd2x : - Intrinsic<[], [llvm_v2f64_ty, llvm_ptr_ty], [IntrReadWriteArgMem]>; + Intrinsic<[], [llvm_v2f64_ty, llvm_ptr_ty], [IntrArgMemOnly]>; // Vector and scalar maximum. def int_ppc_vsx_xvmaxdp : PowerPC_VSX_Vec_DDD_Intrinsic<"xvmaxdp">; @@ -790,7 +790,7 @@ class PowerPC_QPX_FFFF_Intrinsic<string GCCIntSuffix> /// and returns a v4f64. class PowerPC_QPX_Load_Intrinsic<string GCCIntSuffix> : PowerPC_QPX_Intrinsic<GCCIntSuffix, - [llvm_v4f64_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + [llvm_v4f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; /// PowerPC_QPX_LoadPerm_Intrinsic - A PowerPC intrinsic that takes a pointer /// and returns a v4f64 permutation. @@ -803,7 +803,7 @@ class PowerPC_QPX_LoadPerm_Intrinsic<string GCCIntSuffix> class PowerPC_QPX_Store_Intrinsic<string GCCIntSuffix> : PowerPC_QPX_Intrinsic<GCCIntSuffix, [], [llvm_v4f64_ty, llvm_ptr_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; //===----------------------------------------------------------------------===// // PowerPC QPX Intrinsic Definitions. diff --git a/include/llvm/IR/IntrinsicsSystemZ.td b/include/llvm/IR/IntrinsicsSystemZ.td index 96e7ca525696..bfc15b9bc09e 100644 --- a/include/llvm/IR/IntrinsicsSystemZ.td +++ b/include/llvm/IR/IntrinsicsSystemZ.td @@ -217,7 +217,7 @@ let TargetPrefix = "s390" in { Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>; def int_s390_ntstg : Intrinsic<[], [llvm_i64_ty, llvm_ptr64_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_s390_ppa_txassist : GCCBuiltin<"__builtin_tx_assist">, Intrinsic<[], [llvm_i32_ty]>; @@ -236,11 +236,11 @@ let TargetPrefix = "s390" in { def int_s390_vlbb : GCCBuiltin<"__builtin_s390_vlbb">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_s390_vll : GCCBuiltin<"__builtin_s390_vll">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_s390_vpdi : GCCBuiltin<"__builtin_s390_vpdi">, Intrinsic<[llvm_v2i64_ty], @@ -262,7 +262,7 @@ let TargetPrefix = "s390" in { Intrinsic<[], [llvm_v16i8_ty, llvm_i32_ty, llvm_ptr_ty], // In fact write-only but there's no property // for that. - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; defm int_s390_vupl : SystemZUnaryExtBHWF<"vupl">; defm int_s390_vupll : SystemZUnaryExtBHF<"vupll">; @@ -374,3 +374,14 @@ let TargetPrefix = "s390" in { [llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; } + +//===----------------------------------------------------------------------===// +// +// Misc intrinsics +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "s390" in { + def int_s390_tdc : Intrinsic<[llvm_i32_ty], [llvm_anyfloat_ty, llvm_i64_ty], + [IntrNoMem]>; +} diff --git a/include/llvm/IR/IntrinsicsWebAssembly.td b/include/llvm/IR/IntrinsicsWebAssembly.td index 3953aef43dad..4234c466d973 100644 --- a/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/include/llvm/IR/IntrinsicsWebAssembly.td @@ -14,9 +14,9 @@ let TargetPrefix = "wasm" in { // All intrinsics start with "llvm.wasm.". -// Note that memory_size is not IntrNoMem because it must be sequenced with +// Note that current_memory is not IntrNoMem because it must be sequenced with // respect to grow_memory calls. -def int_wasm_memory_size : Intrinsic<[llvm_anyint_ty], [], [IntrReadMem]>; +def int_wasm_current_memory : Intrinsic<[llvm_anyint_ty], [], [IntrReadMem]>; def int_wasm_grow_memory : Intrinsic<[], [llvm_anyint_ty], []>; } diff --git a/include/llvm/IR/IntrinsicsX86.td b/include/llvm/IR/IntrinsicsX86.td index 8023a9f6e8e9..74c971552bb3 100644 --- a/include/llvm/IR/IntrinsicsX86.td +++ b/include/llvm/IR/IntrinsicsX86.td @@ -25,6 +25,9 @@ let TargetPrefix = "x86" in { // Marks the EH registration node created in LLVM IR prior to code generation. def int_x86_seh_ehregnode : Intrinsic<[], [llvm_ptr_ty], []>; + // Marks the EH guard slot node created in LLVM IR prior to code generation. + def int_x86_seh_ehguard : Intrinsic<[], [llvm_ptr_ty], []>; + // Given a pointer to the end of an EH registration object, returns the true // parent frame address that can be used with llvm.localrecover. def int_x86_seh_recoverfp : Intrinsic<[llvm_ptr_ty], @@ -51,7 +54,7 @@ let TargetPrefix = "x86" in { def int_x86_rdtsc : GCCBuiltin<"__builtin_ia32_rdtsc">, Intrinsic<[llvm_i64_ty], [], []>; def int_x86_rdtscp : GCCBuiltin<"__builtin_ia32_rdtscp">, - Intrinsic<[llvm_i64_ty], [llvm_ptr_ty], [IntrReadWriteArgMem]>; + Intrinsic<[llvm_i64_ty], [llvm_ptr_ty], [IntrArgMemOnly]>; } // Read Performance-Monitoring Counter. @@ -142,16 +145,16 @@ let TargetPrefix = "x86" in { // Arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse_add_ss : GCCBuiltin<"__builtin_ia32_addss">, + def int_x86_sse_add_ss : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_sub_ss : GCCBuiltin<"__builtin_ia32_subss">, + def int_x86_sse_sub_ss : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_mul_ss : GCCBuiltin<"__builtin_ia32_mulss">, + def int_x86_sse_mul_ss : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse_div_ss : GCCBuiltin<"__builtin_ia32_divss">, + def int_x86_sse_div_ss : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; def int_x86_sse_sqrt_ss : GCCBuiltin<"__builtin_ia32_sqrtss">, @@ -191,7 +194,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse_cmp_ss : GCCBuiltin<"__builtin_ia32_cmpss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_sse_cmp_ps : GCCBuiltin<"__builtin_ia32_cmpps">, + def int_x86_sse_cmp_ps : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_sse_comieq_ss : GCCBuiltin<"__builtin_ia32_comieq">, @@ -259,13 +262,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_x86mmx_ty], [IntrNoMem]>; } -// SIMD store ops -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse_storeu_ps : GCCBuiltin<"__builtin_ia32_storeups">, - Intrinsic<[], [llvm_ptr_ty, - llvm_v4f32_ty], [IntrReadWriteArgMem]>; -} - // Cacheability support ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse_sfence : GCCBuiltin<"__builtin_ia32_sfence">, @@ -291,16 +287,16 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // FP arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_add_sd : GCCBuiltin<"__builtin_ia32_addsd">, + def int_x86_sse2_add_sd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_sub_sd : GCCBuiltin<"__builtin_ia32_subsd">, + def int_x86_sse2_sub_sd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_mul_sd : GCCBuiltin<"__builtin_ia32_mulsd">, + def int_x86_sse2_mul_sd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_sse2_div_sd : GCCBuiltin<"__builtin_ia32_divsd">, + def int_x86_sse2_div_sd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; def int_x86_sse2_sqrt_sd : GCCBuiltin<"__builtin_ia32_sqrtsd">, @@ -328,7 +324,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse2_cmp_sd : GCCBuiltin<"__builtin_ia32_cmpsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_sse2_cmp_pd : GCCBuiltin<"__builtin_ia32_cmppd">, + def int_x86_sse2_cmp_pd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_sse2_comieq_sd : GCCBuiltin<"__builtin_ia32_comisdeq">, @@ -413,18 +409,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pmaxu_b : GCCBuiltin<"__builtin_ia32_pmaxub128">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, - llvm_v16i8_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pmaxs_w : GCCBuiltin<"__builtin_ia32_pmaxsw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, - llvm_v8i16_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pminu_b : GCCBuiltin<"__builtin_ia32_pminub128">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, - llvm_v16i8_ty], [IntrNoMem, Commutative]>; - def int_x86_sse2_pmins_w : GCCBuiltin<"__builtin_ia32_pminsw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, - llvm_v8i16_ty], [IntrNoMem, Commutative]>; def int_x86_sse2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw128">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem, Commutative]>; @@ -485,8 +469,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Conversion ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_cvtdq2pd : GCCBuiltin<"__builtin_ia32_cvtdq2pd">, - Intrinsic<[llvm_v2f64_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_sse2_cvtdq2ps : GCCBuiltin<"__builtin_ia32_cvtdq2ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_sse2_cvtpd2dq : GCCBuiltin<"__builtin_ia32_cvtpd2dq">, @@ -497,10 +479,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f32_ty], [llvm_v2f64_ty], [IntrNoMem]>; def int_x86_sse2_cvtps2dq : GCCBuiltin<"__builtin_ia32_cvtps2dq">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse2_cvttps2dq : GCCBuiltin<"__builtin_ia32_cvttps2dq">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_sse2_cvtps2pd : GCCBuiltin<"__builtin_ia32_cvtps2pd">, - Intrinsic<[llvm_v2f64_ty], [llvm_v4f32_ty], [IntrNoMem]>; def int_x86_sse2_cvtsd2si : GCCBuiltin<"__builtin_ia32_cvtsd2si">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; def int_x86_sse2_cvtsd2si64 : GCCBuiltin<"__builtin_ia32_cvtsd2si64">, @@ -529,19 +507,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v2f64_ty], [llvm_x86mmx_ty], [IntrNoMem]>; } -// SIMD store ops -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse2_storeu_pd : GCCBuiltin<"__builtin_ia32_storeupd">, - Intrinsic<[], [llvm_ptr_ty, - llvm_v2f64_ty], [IntrReadWriteArgMem]>; - def int_x86_sse2_storeu_dq : GCCBuiltin<"__builtin_ia32_storedqu">, - Intrinsic<[], [llvm_ptr_ty, - llvm_v16i8_ty], [IntrReadWriteArgMem]>; - def int_x86_sse2_storel_dq : GCCBuiltin<"__builtin_ia32_storelv4si">, - Intrinsic<[], [llvm_ptr_ty, - llvm_v4i32_ty], [IntrReadWriteArgMem]>; -} - // Misc. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse2_packsswb_128 : GCCBuiltin<"__builtin_ia32_packsswb128">, @@ -688,15 +653,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_ssse3_pshuf_b_128 : GCCBuiltin<"__builtin_ia32_pshufb128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; - def int_x86_sse2_pshuf_d : GCCBuiltin<"__builtin_ia32_pshufd">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_sse2_pshufl_w : GCCBuiltin<"__builtin_ia32_pshuflw">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_sse2_pshufh_w : GCCBuiltin<"__builtin_ia32_pshufhw">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; def int_x86_sse_pshuf_w : GCCBuiltin<"__builtin_ia32_pshufw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i8_ty], [IntrNoMem]>; @@ -763,46 +719,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_i32_ty], [IntrNoMem]>; } -// Vector sign and zero extend -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_pmovsxbd : GCCBuiltin<"__builtin_ia32_pmovsxbd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovsxbq : GCCBuiltin<"__builtin_ia32_pmovsxbq128">, - Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovsxbw : GCCBuiltin<"__builtin_ia32_pmovsxbw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovsxdq : GCCBuiltin<"__builtin_ia32_pmovsxdq128">, - Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovsxwd : GCCBuiltin<"__builtin_ia32_pmovsxwd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovsxwq : GCCBuiltin<"__builtin_ia32_pmovsxwq128">, - Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovzxbd : GCCBuiltin<"__builtin_ia32_pmovzxbd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovzxbq : GCCBuiltin<"__builtin_ia32_pmovzxbq128">, - Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovzxbw : GCCBuiltin<"__builtin_ia32_pmovzxbw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovzxdq : GCCBuiltin<"__builtin_ia32_pmovzxdq128">, - Intrinsic<[llvm_v2i64_ty], [llvm_v4i32_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovzxwd : GCCBuiltin<"__builtin_ia32_pmovzxwd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty], - [IntrNoMem]>; - def int_x86_sse41_pmovzxwq : GCCBuiltin<"__builtin_ia32_pmovzxwq128">, - Intrinsic<[llvm_v2i64_ty], [llvm_v8i16_ty], - [IntrNoMem]>; -} - // Vector min element let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_phminposuw : GCCBuiltin<"__builtin_ia32_phminposuw128">, @@ -810,34 +726,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem]>; } -// Vector compare, min, max -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_pmaxsb : GCCBuiltin<"__builtin_ia32_pmaxsb128">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], - [IntrNoMem, Commutative]>; - def int_x86_sse41_pmaxsd : GCCBuiltin<"__builtin_ia32_pmaxsd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], - [IntrNoMem, Commutative]>; - def int_x86_sse41_pmaxud : GCCBuiltin<"__builtin_ia32_pmaxud128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], - [IntrNoMem, Commutative]>; - def int_x86_sse41_pmaxuw : GCCBuiltin<"__builtin_ia32_pmaxuw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], - [IntrNoMem, Commutative]>; - def int_x86_sse41_pminsb : GCCBuiltin<"__builtin_ia32_pminsb128">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], - [IntrNoMem, Commutative]>; - def int_x86_sse41_pminsd : GCCBuiltin<"__builtin_ia32_pminsd128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], - [IntrNoMem, Commutative]>; - def int_x86_sse41_pminud : GCCBuiltin<"__builtin_ia32_pminud128">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], - [IntrNoMem, Commutative]>; - def int_x86_sse41_pminuw : GCCBuiltin<"__builtin_ia32_pminuw128">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], - [IntrNoMem, Commutative]>; -} - // Advanced Encryption Standard (AES) Instructions let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_aesni_aesimc : GCCBuiltin<"__builtin_ia32_aesimc128">, @@ -882,22 +770,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem, Commutative]>; } -// Vector extract -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_sse41_pextrb : - Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_sse41_pextrd : - Intrinsic<[llvm_i32_ty], [llvm_v4i32_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_sse41_pextrq : - Intrinsic<[llvm_i64_ty], [llvm_v2i64_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_sse41_extractps : GCCBuiltin<"__builtin_ia32_extractps128">, - Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], - [IntrNoMem]>; -} - // Vector insert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_insertps : GCCBuiltin<"__builtin_ia32_insertps128">, @@ -1056,11 +928,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_i8_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_sse4a_insertq : GCCBuiltin<"__builtin_ia32_insertq">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; - - def int_x86_sse4a_movnt_ss : GCCBuiltin<"__builtin_ia32_movntss">, - Intrinsic<[], [llvm_ptr_ty, llvm_v4f32_ty], []>; - def int_x86_sse4a_movnt_sd : GCCBuiltin<"__builtin_ia32_movntsd">, - Intrinsic<[], [llvm_ptr_ty, llvm_v2f64_ty], []>; } //===----------------------------------------------------------------------===// @@ -1151,91 +1018,91 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_d_128 : + def int_x86_avx512_mask_vpermi2var_d_128 : GCCBuiltin<"__builtin_ia32_vpermi2vard128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_d_256 : + def int_x86_avx512_mask_vpermi2var_d_256 : GCCBuiltin<"__builtin_ia32_vpermi2vard256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_d_512 : + def int_x86_avx512_mask_vpermi2var_d_512 : GCCBuiltin<"__builtin_ia32_vpermi2vard512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_hi_128 : + def int_x86_avx512_mask_vpermi2var_hi_128 : GCCBuiltin<"__builtin_ia32_vpermi2varhi128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_hi_256 : + def int_x86_avx512_mask_vpermi2var_hi_256 : GCCBuiltin<"__builtin_ia32_vpermi2varhi256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_hi_512 : + def int_x86_avx512_mask_vpermi2var_hi_512 : GCCBuiltin<"__builtin_ia32_vpermi2varhi512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_pd_128 : + def int_x86_avx512_mask_vpermi2var_pd_128 : GCCBuiltin<"__builtin_ia32_vpermi2varpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_pd_256 : + def int_x86_avx512_mask_vpermi2var_pd_256 : GCCBuiltin<"__builtin_ia32_vpermi2varpd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_pd_512 : + def int_x86_avx512_mask_vpermi2var_pd_512 : GCCBuiltin<"__builtin_ia32_vpermi2varpd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_ps_128 : + def int_x86_avx512_mask_vpermi2var_ps_128 : GCCBuiltin<"__builtin_ia32_vpermi2varps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_ps_256 : + def int_x86_avx512_mask_vpermi2var_ps_256 : GCCBuiltin<"__builtin_ia32_vpermi2varps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_ps_512 : + def int_x86_avx512_mask_vpermi2var_ps_512 : GCCBuiltin<"__builtin_ia32_vpermi2varps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_q_128 : + def int_x86_avx512_mask_vpermi2var_q_128 : GCCBuiltin<"__builtin_ia32_vpermi2varq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_q_256 : + def int_x86_avx512_mask_vpermi2var_q_256 : GCCBuiltin<"__builtin_ia32_vpermi2varq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermi2var_q_512 : + def int_x86_avx512_mask_vpermi2var_q_512 : GCCBuiltin<"__builtin_ia32_vpermi2varq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], @@ -1261,196 +1128,214 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f64_ty], [llvm_v8i64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_d_128 : + def int_x86_avx512_mask_vpermt2var_d_128 : GCCBuiltin<"__builtin_ia32_vpermt2vard128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_d_128 : + def int_x86_avx512_maskz_vpermt2var_d_128 : GCCBuiltin<"__builtin_ia32_vpermt2vard128_maskz">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_d_256 : + def int_x86_avx512_mask_vpermt2var_d_256 : GCCBuiltin<"__builtin_ia32_vpermt2vard256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_d_256 : + def int_x86_avx512_maskz_vpermt2var_d_256 : GCCBuiltin<"__builtin_ia32_vpermt2vard256_maskz">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_d_512 : + def int_x86_avx512_maskz_vpermt2var_d_512 : GCCBuiltin<"__builtin_ia32_vpermt2vard512_maskz">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_hi_128 : + def int_x86_avx512_mask_vpermt2var_hi_128 : GCCBuiltin<"__builtin_ia32_vpermt2varhi128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_hi_128 : + def int_x86_avx512_maskz_vpermt2var_hi_128 : GCCBuiltin<"__builtin_ia32_vpermt2varhi128_maskz">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_hi_256 : + def int_x86_avx512_mask_vpermt2var_hi_256 : GCCBuiltin<"__builtin_ia32_vpermt2varhi256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_hi_256 : + def int_x86_avx512_maskz_vpermt2var_hi_256 : GCCBuiltin<"__builtin_ia32_vpermt2varhi256_maskz">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_hi_512 : + def int_x86_avx512_mask_vpermt2var_hi_512 : GCCBuiltin<"__builtin_ia32_vpermt2varhi512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_hi_512 : + def int_x86_avx512_maskz_vpermt2var_hi_512 : GCCBuiltin<"__builtin_ia32_vpermt2varhi512_maskz">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_pd_128 : + def int_x86_avx512_mask_vpermt2var_pd_128 : GCCBuiltin<"__builtin_ia32_vpermt2varpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2i64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_pd_128 : + def int_x86_avx512_maskz_vpermt2var_pd_128 : GCCBuiltin<"__builtin_ia32_vpermt2varpd128_maskz">, Intrinsic<[llvm_v2f64_ty], [llvm_v2i64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_pd_256 : + def int_x86_avx512_mask_vpermt2var_pd_256 : GCCBuiltin<"__builtin_ia32_vpermt2varpd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4i64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_pd_256 : + def int_x86_avx512_maskz_vpermt2var_pd_256 : GCCBuiltin<"__builtin_ia32_vpermt2varpd256_maskz">, Intrinsic<[llvm_v4f64_ty], [llvm_v4i64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_pd_512 : + def int_x86_avx512_maskz_vpermt2var_pd_512 : GCCBuiltin<"__builtin_ia32_vpermt2varpd512_maskz">, Intrinsic<[llvm_v8f64_ty], [llvm_v8i64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_ps_128 : + def int_x86_avx512_mask_vpermt2var_ps_128 : GCCBuiltin<"__builtin_ia32_vpermt2varps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_ps_128 : + def int_x86_avx512_maskz_vpermt2var_ps_128 : GCCBuiltin<"__builtin_ia32_vpermt2varps128_maskz">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_ps_256 : + def int_x86_avx512_mask_vpermt2var_ps_256 : GCCBuiltin<"__builtin_ia32_vpermt2varps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_ps_256 : + def int_x86_avx512_maskz_vpermt2var_ps_256 : GCCBuiltin<"__builtin_ia32_vpermt2varps256_maskz">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_ps_512 : + def int_x86_avx512_maskz_vpermt2var_ps_512 : GCCBuiltin<"__builtin_ia32_vpermt2varps512_maskz">, Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_q_128 : + def int_x86_avx512_mask_vpermt2var_q_128 : GCCBuiltin<"__builtin_ia32_vpermt2varq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_q_128 : + def int_x86_avx512_maskz_vpermt2var_q_128 : GCCBuiltin<"__builtin_ia32_vpermt2varq128_maskz">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermt2var_q_256 : + def int_x86_avx512_mask_vpermt2var_q_256 : GCCBuiltin<"__builtin_ia32_vpermt2varq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_q_256 : + def int_x86_avx512_maskz_vpermt2var_q_256 : GCCBuiltin<"__builtin_ia32_vpermt2varq256_maskz">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_maskz_vpermt2var_q_512 : + def int_x86_avx512_maskz_vpermt2var_q_512 : GCCBuiltin<"__builtin_ia32_vpermt2varq512_maskz">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermil_pd_128 : - GCCBuiltin<"__builtin_ia32_vpermilpd_mask">, - Intrinsic<[llvm_v2f64_ty], - [llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], + def int_x86_avx512_mask_vpermi2var_qi_128 : + GCCBuiltin<"__builtin_ia32_vpermi2varqi128_mask">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, + llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermil_pd_256 : - GCCBuiltin<"__builtin_ia32_vpermilpd256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], + def int_x86_avx512_mask_vpermt2var_qi_128 : + GCCBuiltin<"__builtin_ia32_vpermt2varqi128_mask">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, + llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermil_pd_512 : - GCCBuiltin<"__builtin_ia32_vpermilpd512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty], + def int_x86_avx512_maskz_vpermt2var_qi_128 : + GCCBuiltin<"__builtin_ia32_vpermt2varqi128_maskz">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, + llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermil_ps_128 : - GCCBuiltin<"__builtin_ia32_vpermilps_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], + def int_x86_avx512_mask_vpermi2var_qi_256 : + GCCBuiltin<"__builtin_ia32_vpermi2varqi256_mask">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, + llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermil_ps_256 : - GCCBuiltin<"__builtin_ia32_vpermilps256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], + def int_x86_avx512_mask_vpermt2var_qi_256 : + GCCBuiltin<"__builtin_ia32_vpermt2varqi256_mask">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, + llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_vpermil_ps_512 : - GCCBuiltin<"__builtin_ia32_vpermilps512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], + def int_x86_avx512_maskz_vpermt2var_qi_256 : + GCCBuiltin<"__builtin_ia32_vpermt2varqi256_maskz">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, + llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermi2var_qi_512 : + GCCBuiltin<"__builtin_ia32_vpermi2varqi512_mask">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, + llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; + + def int_x86_avx512_mask_vpermt2var_qi_512 : + GCCBuiltin<"__builtin_ia32_vpermt2varqi512_mask">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, + llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], + [IntrNoMem]>; + + def int_x86_avx512_maskz_vpermt2var_qi_512 : + GCCBuiltin<"__builtin_ia32_vpermt2varqi512_maskz">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, + llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_vpermilvar_pd_256 : @@ -1489,78 +1374,24 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pshuf_b_128 : + def int_x86_avx512_mask_pshuf_b_128 : GCCBuiltin<"__builtin_ia32_pshufb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pshuf_b_256 : + def int_x86_avx512_mask_pshuf_b_256 : GCCBuiltin<"__builtin_ia32_pshufb256_mask">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pshuf_b_512 : + def int_x86_avx512_mask_pshuf_b_512 : GCCBuiltin<"__builtin_ia32_pshufb512_mask">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pshuf_d_128 : - GCCBuiltin<"__builtin_ia32_pshufd128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_i16_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshuf_d_256 : - GCCBuiltin<"__builtin_ia32_pshufd256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_i16_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshuf_d_512 : - GCCBuiltin<"__builtin_ia32_pshufd512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshufh_w_128 : - GCCBuiltin<"__builtin_ia32_pshufhw128_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_v8i16_ty, llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshufh_w_256 : - GCCBuiltin<"__builtin_ia32_pshufhw256_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_v16i16_ty, llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshufh_w_512 : - GCCBuiltin<"__builtin_ia32_pshufhw512_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_v32i16_ty, llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshufl_w_128 : - GCCBuiltin<"__builtin_ia32_pshuflw128_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_v8i16_ty, llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshufl_w_256 : - GCCBuiltin<"__builtin_ia32_pshuflw256_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_v16i16_ty, llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pshufl_w_512 : - GCCBuiltin<"__builtin_ia32_pshuflw512_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_v32i16_ty, llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_shuf_f32x4_256 : GCCBuiltin<"__builtin_ia32_shuf_f32x4_256_mask">, Intrinsic<[llvm_v8f32_ty], @@ -1644,60 +1475,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_movshdup_128 : - GCCBuiltin<"__builtin_ia32_movshdup128_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movshdup_256 : - GCCBuiltin<"__builtin_ia32_movshdup256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movshdup_512 : - GCCBuiltin<"__builtin_ia32_movshdup512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movsldup_128 : - GCCBuiltin<"__builtin_ia32_movsldup128_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movsldup_256 : - GCCBuiltin<"__builtin_ia32_movsldup256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movsldup_512 : - GCCBuiltin<"__builtin_ia32_movsldup512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movddup_128 : - GCCBuiltin<"__builtin_ia32_movddup128_mask">, - Intrinsic<[llvm_v2f64_ty], - [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movddup_256 : - GCCBuiltin<"__builtin_ia32_movddup256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_movddup_512 : - GCCBuiltin<"__builtin_ia32_movddup512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrNoMem]>; } // Vector blend @@ -1719,32 +1496,24 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector compare let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_cmp_pd_256 : GCCBuiltin<"__builtin_ia32_cmppd256">, + def int_x86_avx_cmp_pd_256 : Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx_cmp_ps_256 : GCCBuiltin<"__builtin_ia32_cmpps256">, + def int_x86_avx_cmp_ps_256 : Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; } // Vector convert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_cvtdq2_pd_256 : GCCBuiltin<"__builtin_ia32_cvtdq2pd256">, - Intrinsic<[llvm_v4f64_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx_cvtdq2_ps_256 : GCCBuiltin<"__builtin_ia32_cvtdq2ps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx_cvt_pd2_ps_256 : GCCBuiltin<"__builtin_ia32_cvtpd2ps256">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f64_ty], [IntrNoMem]>; def int_x86_avx_cvt_ps2dq_256 : GCCBuiltin<"__builtin_ia32_cvtps2dq256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_avx_cvt_ps2_pd_256 : GCCBuiltin<"__builtin_ia32_cvtps2pd256">, - Intrinsic<[llvm_v4f64_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx_cvtt_pd2dq_256 : GCCBuiltin<"__builtin_ia32_cvttpd2dq256">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty], [IntrNoMem]>; def int_x86_avx_cvt_pd2dq_256 : GCCBuiltin<"__builtin_ia32_cvtpd2dq256">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty], [IntrNoMem]>; - def int_x86_avx_cvtt_ps2dq_256 : GCCBuiltin<"__builtin_ia32_cvttps2dq256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty], [IntrNoMem]>; } // Vector bit test @@ -1794,42 +1563,111 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_ptestnzc_256 : GCCBuiltin<"__builtin_ia32_ptestnzc256">, Intrinsic<[llvm_i32_ty], [llvm_v4i64_ty, llvm_v4i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ptestm_d_512 : GCCBuiltin<"__builtin_ia32_ptestmd512">, + def int_x86_avx512_ptestm_d_512 : GCCBuiltin<"__builtin_ia32_ptestmd512">, Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ptestm_q_512 : GCCBuiltin<"__builtin_ia32_ptestmq512">, + def int_x86_avx512_ptestm_q_512 : GCCBuiltin<"__builtin_ia32_ptestmq512">, Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_pd_128 : + + def int_x86_avx512_ptestm_b_128 : GCCBuiltin<"__builtin_ia32_ptestmb128">, + Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, + llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_b_256 : GCCBuiltin<"__builtin_ia32_ptestmb256">, + Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, + llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_b_512 : GCCBuiltin<"__builtin_ia32_ptestmb512">, + Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, + llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_d_128 : GCCBuiltin<"__builtin_ia32_ptestmd128">, + Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_d_256 : GCCBuiltin<"__builtin_ia32_ptestmd256">, + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_q_128 : GCCBuiltin<"__builtin_ia32_ptestmq128">, + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_q_256 : GCCBuiltin<"__builtin_ia32_ptestmq256">, + Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_w_128 : GCCBuiltin<"__builtin_ia32_ptestmw128">, + Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_w_256 : GCCBuiltin<"__builtin_ia32_ptestmw256">, + Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_ptestm_w_512 : GCCBuiltin<"__builtin_ia32_ptestmw512">, + Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_ptestnm_b_128 : GCCBuiltin<"__builtin_ia32_ptestnmb128">, + Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, + llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_b_256 : GCCBuiltin<"__builtin_ia32_ptestnmb256">, + Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, + llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_b_512 : GCCBuiltin<"__builtin_ia32_ptestnmb512">, + Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, + llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_d_128 : GCCBuiltin<"__builtin_ia32_ptestnmd128">, + Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_d_256 : GCCBuiltin<"__builtin_ia32_ptestnmd256">, + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_d_512 : GCCBuiltin<"__builtin_ia32_ptestnmd512">, + Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, + llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_q_128 : GCCBuiltin<"__builtin_ia32_ptestnmq128">, + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_q_256 : GCCBuiltin<"__builtin_ia32_ptestnmq256">, + Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_q_512 : GCCBuiltin<"__builtin_ia32_ptestnmq512">, + Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_w_128 : GCCBuiltin<"__builtin_ia32_ptestnmw128">, + Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, + llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_w_256 : GCCBuiltin<"__builtin_ia32_ptestnmw256">, + Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, + llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_ptestnm_w_512 : GCCBuiltin<"__builtin_ia32_ptestnmw512">, + Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, + llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_fpclass_pd_128 : GCCBuiltin<"__builtin_ia32_fpclasspd128_mask">, Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_pd_256 : + def int_x86_avx512_mask_fpclass_pd_256 : GCCBuiltin<"__builtin_ia32_fpclasspd256_mask">, Intrinsic<[llvm_i8_ty], [llvm_v4f64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_pd_512 : + def int_x86_avx512_mask_fpclass_pd_512 : GCCBuiltin<"__builtin_ia32_fpclasspd512_mask">, Intrinsic<[llvm_i8_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_ps_128 : + def int_x86_avx512_mask_fpclass_ps_128 : GCCBuiltin<"__builtin_ia32_fpclassps128_mask">, Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_ps_256 : + def int_x86_avx512_mask_fpclass_ps_256 : GCCBuiltin<"__builtin_ia32_fpclassps256_mask">, Intrinsic<[llvm_i8_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_ps_512 : + def int_x86_avx512_mask_fpclass_ps_512 : GCCBuiltin<"__builtin_ia32_fpclassps512_mask">, Intrinsic<[llvm_i16_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_sd : - GCCBuiltin<"__builtin_ia32_fpclasssd">, + def int_x86_avx512_mask_fpclass_sd : + GCCBuiltin<"__builtin_ia32_fpclasssd_mask">, Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_fpclass_ss : - GCCBuiltin<"__builtin_ia32_fpclassss">, + def int_x86_avx512_mask_fpclass_ss : + GCCBuiltin<"__builtin_ia32_fpclassss_mask">, Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; } @@ -1854,10 +1692,10 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_vbroadcastf128_pd_256 : GCCBuiltin<"__builtin_ia32_vbroadcastf128_pd256">, - Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx_vbroadcastf128_ps_256 : GCCBuiltin<"__builtin_ia32_vbroadcastf128_ps256">, - Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; + Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; } // SIMD load ops @@ -1866,91 +1704,32 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v32i8_ty], [llvm_ptr_ty], [IntrReadMem]>; } -// SIMD store ops -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx_storeu_pd_256 : GCCBuiltin<"__builtin_ia32_storeupd256">, - Intrinsic<[], [llvm_ptr_ty, llvm_v4f64_ty], [IntrReadWriteArgMem]>; - def int_x86_avx_storeu_ps_256 : GCCBuiltin<"__builtin_ia32_storeups256">, - Intrinsic<[], [llvm_ptr_ty, llvm_v8f32_ty], [IntrReadWriteArgMem]>; - def int_x86_avx_storeu_dq_256 : GCCBuiltin<"__builtin_ia32_storedqu256">, - Intrinsic<[], [llvm_ptr_ty, llvm_v32i8_ty], [IntrReadWriteArgMem]>; -} - // Conditional load ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_maskload_pd : GCCBuiltin<"__builtin_ia32_maskloadpd">, Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty, llvm_v2i64_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx_maskload_ps : GCCBuiltin<"__builtin_ia32_maskloadps">, Intrinsic<[llvm_v4f32_ty], [llvm_ptr_ty, llvm_v4i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx_maskload_pd_256 : GCCBuiltin<"__builtin_ia32_maskloadpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty, llvm_v4i64_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx_maskload_ps_256 : GCCBuiltin<"__builtin_ia32_maskloadps256">, Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; +} - def int_x86_avx512_mask_loadu_ps_128 : - GCCBuiltin<"__builtin_ia32_loadups128_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_ptr_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_ps_256 : - GCCBuiltin<"__builtin_ia32_loadups256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_ptr_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_ps_512 : - GCCBuiltin<"__builtin_ia32_loadups512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrReadArgMem]>; - - def int_x86_avx512_mask_loadu_pd_128 : - GCCBuiltin<"__builtin_ia32_loadupd128_mask">, - Intrinsic<[llvm_v2f64_ty], - [llvm_ptr_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_pd_256 : - GCCBuiltin<"__builtin_ia32_loadupd256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_ptr_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_pd_512 : - GCCBuiltin<"__builtin_ia32_loadupd512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrReadArgMem]>; - - def int_x86_avx512_mask_load_ps_128 : - GCCBuiltin<"__builtin_ia32_loadaps128_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_ptr_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_load_ps_256 : - GCCBuiltin<"__builtin_ia32_loadaps256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_ptr_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_load_ps_512 : - GCCBuiltin<"__builtin_ia32_loadaps512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrReadArgMem]>; - - def int_x86_avx512_mask_load_pd_128 : - GCCBuiltin<"__builtin_ia32_loadapd128_mask">, - Intrinsic<[llvm_v2f64_ty], - [llvm_ptr_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_load_pd_256 : - GCCBuiltin<"__builtin_ia32_loadapd256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_ptr_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_load_pd_512 : - GCCBuiltin<"__builtin_ia32_loadapd512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrReadArgMem]>; - - def int_x86_avx512_mask_move_ss : +// Conditional move ops +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_avx512_mask_move_ss : GCCBuiltin<"__builtin_ia32_movss_mask">, - Intrinsic<[llvm_v4f32_ty], + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_move_sd : + def int_x86_avx512_mask_move_sd : GCCBuiltin<"__builtin_ia32_movsd_mask">, - Intrinsic<[llvm_v2f64_ty], + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; } @@ -1959,38 +1738,23 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_maskstore_pd : GCCBuiltin<"__builtin_ia32_maskstorepd">, Intrinsic<[], [llvm_ptr_ty, - llvm_v2i64_ty, llvm_v2f64_ty], [IntrReadWriteArgMem]>; + llvm_v2i64_ty, llvm_v2f64_ty], [IntrArgMemOnly]>; def int_x86_avx_maskstore_ps : GCCBuiltin<"__builtin_ia32_maskstoreps">, Intrinsic<[], [llvm_ptr_ty, - llvm_v4i32_ty, llvm_v4f32_ty], [IntrReadWriteArgMem]>; + llvm_v4i32_ty, llvm_v4f32_ty], [IntrArgMemOnly]>; def int_x86_avx_maskstore_pd_256 : GCCBuiltin<"__builtin_ia32_maskstorepd256">, Intrinsic<[], [llvm_ptr_ty, - llvm_v4i64_ty, llvm_v4f64_ty], [IntrReadWriteArgMem]>; + llvm_v4i64_ty, llvm_v4f64_ty], [IntrArgMemOnly]>; def int_x86_avx_maskstore_ps_256 : GCCBuiltin<"__builtin_ia32_maskstoreps256">, Intrinsic<[], [llvm_ptr_ty, - llvm_v8i32_ty, llvm_v8f32_ty], [IntrReadWriteArgMem]>; - def int_x86_avx512_mask_storeu_ps_512 : - GCCBuiltin<"__builtin_ia32_storeups512_mask">, - Intrinsic<[], [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; - def int_x86_avx512_mask_storeu_pd_512 : - GCCBuiltin<"__builtin_ia32_storeupd512_mask">, - Intrinsic<[], [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; - def int_x86_avx512_mask_store_ps_512 : - GCCBuiltin<"__builtin_ia32_storeaps512_mask">, - Intrinsic<[], [llvm_ptr_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; - def int_x86_avx512_mask_store_pd_512 : - GCCBuiltin<"__builtin_ia32_storeapd512_mask">, - Intrinsic<[], [llvm_ptr_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + llvm_v8i32_ty, llvm_v8f32_ty], [IntrArgMemOnly]>; + def int_x86_avx512_mask_store_ss : GCCBuiltin<"__builtin_ia32_storess_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; } //===----------------------------------------------------------------------===// @@ -2050,83 +1814,47 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector min, max let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_pmaxu_b : GCCBuiltin<"__builtin_ia32_pmaxub256">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, - llvm_v32i8_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmaxu_w : GCCBuiltin<"__builtin_ia32_pmaxuw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmaxu_d : GCCBuiltin<"__builtin_ia32_pmaxud256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_v8i32_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmaxs_b : GCCBuiltin<"__builtin_ia32_pmaxsb256">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, - llvm_v32i8_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmaxs_w : GCCBuiltin<"__builtin_ia32_pmaxsw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmaxs_d : GCCBuiltin<"__builtin_ia32_pmaxsd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_v8i32_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pminu_b : GCCBuiltin<"__builtin_ia32_pminub256">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, - llvm_v32i8_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pminu_w : GCCBuiltin<"__builtin_ia32_pminuw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pminu_d : GCCBuiltin<"__builtin_ia32_pminud256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_v8i32_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmins_b : GCCBuiltin<"__builtin_ia32_pminsb256">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, - llvm_v32i8_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmins_w : GCCBuiltin<"__builtin_ia32_pminsw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_v16i16_ty], [IntrNoMem, Commutative]>; - def int_x86_avx2_pmins_d : GCCBuiltin<"__builtin_ia32_pminsd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_v8i32_ty], [IntrNoMem, Commutative]>; def int_x86_avx512_mask_pmaxs_b_128 : GCCBuiltin<"__builtin_ia32_pmaxsb128_mask">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxs_b_256 : GCCBuiltin<"__builtin_ia32_pmaxsb256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxs_b_512 : GCCBuiltin<"__builtin_ia32_pmaxsb512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxu_b_128 : GCCBuiltin<"__builtin_ia32_pmaxub128_mask">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxu_b_256 : GCCBuiltin<"__builtin_ia32_pmaxub256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxu_b_512 : GCCBuiltin<"__builtin_ia32_pmaxub512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxs_w_128 : GCCBuiltin<"__builtin_ia32_pmaxsw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxs_w_256 : GCCBuiltin<"__builtin_ia32_pmaxsw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxs_w_512 : GCCBuiltin<"__builtin_ia32_pmaxsw512_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty],[IntrNoMem]>; def int_x86_avx512_mask_pmaxu_w_128 : GCCBuiltin<"__builtin_ia32_pmaxuw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxu_w_256 : GCCBuiltin<"__builtin_ia32_pmaxuw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxu_w_512 : GCCBuiltin<"__builtin_ia32_pmaxuw512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty],[IntrNoMem]>; def int_x86_avx512_mask_pmins_b_128 : GCCBuiltin<"__builtin_ia32_pminsb128_mask">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmins_b_256 : GCCBuiltin<"__builtin_ia32_pminsb256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmins_b_512 : GCCBuiltin<"__builtin_ia32_pminsb512_mask">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, @@ -2135,28 +1863,28 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pminu_b_256 : GCCBuiltin<"__builtin_ia32_pminub256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pminu_b_512 : GCCBuiltin<"__builtin_ia32_pminub512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmins_w_128 : GCCBuiltin<"__builtin_ia32_pminsw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmins_w_256 : GCCBuiltin<"__builtin_ia32_pminsw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmins_w_512 : GCCBuiltin<"__builtin_ia32_pminsw512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty],[IntrNoMem]>; def int_x86_avx512_mask_pminu_w_128 : GCCBuiltin<"__builtin_ia32_pminuw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pminu_w_256 : GCCBuiltin<"__builtin_ia32_pminuw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pminu_w_512 : GCCBuiltin<"__builtin_ia32_pminuw512_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaxu_d_512 : GCCBuiltin<"__builtin_ia32_pmaxud512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, @@ -2284,25 +2012,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pslli_d : GCCBuiltin<"__builtin_ia32_pslldi512">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pslli_q : GCCBuiltin<"__builtin_ia32_psllqi512">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_psrli_d : GCCBuiltin<"__builtin_ia32_psrldi512">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_psrli_q : GCCBuiltin<"__builtin_ia32_psrlqi512">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_psrai_d : GCCBuiltin<"__builtin_ia32_psradi512">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_psrai_q : GCCBuiltin<"__builtin_ia32_psraqi512">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_psrl_w_128 : GCCBuiltin<"__builtin_ia32_psrlw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; @@ -2314,13 +2023,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_wi_128 : GCCBuiltin<"__builtin_ia32_psrlwi128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, - llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_wi_256 : GCCBuiltin<"__builtin_ia32_psrlwi256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_wi_512 : GCCBuiltin<"__builtin_ia32_psrlwi512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, - llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_w_128 : GCCBuiltin<"__builtin_ia32_psraw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, @@ -2333,13 +2042,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_wi_128 : GCCBuiltin<"__builtin_ia32_psrawi128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, - llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_wi_256 : GCCBuiltin<"__builtin_ia32_psrawi256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_wi_512 : GCCBuiltin<"__builtin_ia32_psrawi512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, - llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_d : GCCBuiltin<"__builtin_ia32_pslld512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, @@ -2371,13 +2080,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_wi_128 : GCCBuiltin<"__builtin_ia32_psllwi128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, - llvm_i8_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_wi_256 : GCCBuiltin<"__builtin_ia32_psllwi256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, - llvm_i8_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_wi_512 : GCCBuiltin<"__builtin_ia32_psllwi512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, - llvm_i8_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psllv16_hi : GCCBuiltin<"__builtin_ia32_psllv16hi_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; @@ -2404,64 +2113,76 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_d_256 : GCCBuiltin<"__builtin_ia32_psrad256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_di_128 : GCCBuiltin<"__builtin_ia32_psradi128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, - llvm_i8_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_di_256 : GCCBuiltin<"__builtin_ia32_psradi256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_i8_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_di_512 : GCCBuiltin<"__builtin_ia32_psradi512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_q_128 : GCCBuiltin<"__builtin_ia32_psraq128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_q_256 : GCCBuiltin<"__builtin_ia32_psraq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_qi_128 : GCCBuiltin<"__builtin_ia32_psraqi128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, - llvm_i8_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + llvm_i32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_qi_256 : GCCBuiltin<"__builtin_ia32_psraqi256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, - llvm_i8_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psra_qi_512 : GCCBuiltin<"__builtin_ia32_psraqi512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_d_128: GCCBuiltin<"__builtin_ia32_psrld128_mask">, Intrinsic<[llvm_v4i32_ty], [ llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty ], [IntrNoMem]>; def int_x86_avx512_mask_psrl_d_256: GCCBuiltin<"__builtin_ia32_psrld256_mask">, - Intrinsic<[llvm_v8i32_ty], [ llvm_v8i32_ty, + Intrinsic<[llvm_v8i32_ty], [ llvm_v8i32_ty, llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty ], [IntrNoMem]>; def int_x86_avx512_mask_psrl_di_128: GCCBuiltin<"__builtin_ia32_psrldi128_mask">, - Intrinsic<[llvm_v4i32_ty], [ llvm_v4i32_ty, - llvm_i8_ty, llvm_v4i32_ty, llvm_i8_ty ], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [ llvm_v4i32_ty, + llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty ], [IntrNoMem]>; def int_x86_avx512_mask_psrl_di_256: GCCBuiltin<"__builtin_ia32_psrldi256_mask">, - Intrinsic<[llvm_v8i32_ty], [ llvm_v8i32_ty, - llvm_i8_ty, llvm_v8i32_ty, llvm_i8_ty ], [IntrNoMem]>; + Intrinsic<[llvm_v8i32_ty], [ llvm_v8i32_ty, + llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty ], [IntrNoMem]>; def int_x86_avx512_mask_psrl_di_512: GCCBuiltin<"__builtin_ia32_psrldi512_mask">, - Intrinsic<[llvm_v16i32_ty], [ llvm_v16i32_ty, - llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty ], [IntrNoMem]>; + Intrinsic<[llvm_v16i32_ty], [ llvm_v16i32_ty, + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty ], [IntrNoMem]>; def int_x86_avx512_mask_psrl_q_128: GCCBuiltin<"__builtin_ia32_psrlq128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_q_256: GCCBuiltin<"__builtin_ia32_psrlq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_qi_128: GCCBuiltin<"__builtin_ia32_psrlqi128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, - llvm_i8_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_qi_256: GCCBuiltin<"__builtin_ia32_psrlqi256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, - llvm_i8_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrl_qi_512: GCCBuiltin<"__builtin_ia32_psrlqi512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmultishift_qb_128: + GCCBuiltin<"__builtin_ia32_vpmultishiftqb128_mask">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, + llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmultishift_qb_256: + GCCBuiltin<"__builtin_ia32_vpmultishiftqb256_mask">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, + llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pmultishift_qb_512: + GCCBuiltin<"__builtin_ia32_vpmultishiftqb512_mask">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, + llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; } // Pack ops. @@ -2489,73 +2210,73 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_pabs_d : GCCBuiltin<"__builtin_ia32_pabsd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_b_128 : + def int_x86_avx512_mask_pabs_b_128 : GCCBuiltin<"__builtin_ia32_pabsb128_mask">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_b_256 : + def int_x86_avx512_mask_pabs_b_256 : GCCBuiltin<"__builtin_ia32_pabsb256_mask">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_b_512 : + def int_x86_avx512_mask_pabs_b_512 : GCCBuiltin<"__builtin_ia32_pabsb512_mask">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_d_128 : + def int_x86_avx512_mask_pabs_d_128 : GCCBuiltin<"__builtin_ia32_pabsd128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_d_256 : + def int_x86_avx512_mask_pabs_d_256 : GCCBuiltin<"__builtin_ia32_pabsd256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_d_512 : + def int_x86_avx512_mask_pabs_d_512 : GCCBuiltin<"__builtin_ia32_pabsd512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_q_128 : + def int_x86_avx512_mask_pabs_q_128 : GCCBuiltin<"__builtin_ia32_pabsq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_q_256 : + def int_x86_avx512_mask_pabs_q_256 : GCCBuiltin<"__builtin_ia32_pabsq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_q_512 : + def int_x86_avx512_mask_pabs_q_512 : GCCBuiltin<"__builtin_ia32_pabsq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_w_128 : + def int_x86_avx512_mask_pabs_w_128 : GCCBuiltin<"__builtin_ia32_pabsw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_w_256 : + def int_x86_avx512_mask_pabs_w_256 : GCCBuiltin<"__builtin_ia32_pabsw256_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pabs_w_512 : + def int_x86_avx512_mask_pabs_w_512 : GCCBuiltin<"__builtin_ia32_pabsw512_mask">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], @@ -2606,56 +2327,16 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; def int_x86_avx512_mask_pmul_hr_sw_128 : GCCBuiltin<"__builtin_ia32_pmulhrsw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmul_hr_sw_256 : GCCBuiltin<"__builtin_ia32_pmulhrsw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmul_hr_sw_512 : GCCBuiltin<"__builtin_ia32_pmulhrsw512_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; } -// Vector sign and zero extend -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx2_pmovsxbd : GCCBuiltin<"__builtin_ia32_pmovsxbd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovsxbq : GCCBuiltin<"__builtin_ia32_pmovsxbq256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovsxbw : GCCBuiltin<"__builtin_ia32_pmovsxbw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovsxdq : GCCBuiltin<"__builtin_ia32_pmovsxdq256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i32_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovsxwd : GCCBuiltin<"__builtin_ia32_pmovsxwd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i16_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovsxwq : GCCBuiltin<"__builtin_ia32_pmovsxwq256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v8i16_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovzxbd : GCCBuiltin<"__builtin_ia32_pmovzxbd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovzxbq : GCCBuiltin<"__builtin_ia32_pmovzxbq256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovzxbw : GCCBuiltin<"__builtin_ia32_pmovzxbw256">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i8_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovzxdq : GCCBuiltin<"__builtin_ia32_pmovzxdq256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i32_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovzxwd : GCCBuiltin<"__builtin_ia32_pmovzxwd256">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i16_ty], - [IntrNoMem]>; - def int_x86_avx2_pmovzxwq : GCCBuiltin<"__builtin_ia32_pmovzxwq256">, - Intrinsic<[llvm_v4i64_ty], [llvm_v8i16_ty], - [IntrNoMem]>; -} - // Vector blend let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_pblendvb : GCCBuiltin<"__builtin_ia32_pblendvb256">, @@ -2665,18 +2346,62 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector load with broadcast let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_avx512_mask_pbroadcast_b_gpr_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastb128_gpr_mask">, + Intrinsic<[llvm_v16i8_ty], + [llvm_i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pbroadcast_b_gpr_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastb256_gpr_mask">, + Intrinsic<[llvm_v32i8_ty], + [llvm_i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pbroadcast_b_gpr_512 : + GCCBuiltin<"__builtin_ia32_pbroadcastb512_gpr_mask">, + Intrinsic<[llvm_v64i8_ty], + [llvm_i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pbroadcast_w_gpr_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastw128_gpr_mask">, + Intrinsic<[llvm_v8i16_ty], + [llvm_i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pbroadcast_w_gpr_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastw256_gpr_mask">, + Intrinsic<[llvm_v16i16_ty], + [llvm_i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pbroadcast_w_gpr_512 : + GCCBuiltin<"__builtin_ia32_pbroadcastw512_gpr_mask">, + Intrinsic<[llvm_v32i16_ty], + [llvm_i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pbroadcast_d_gpr_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastd128_gpr_mask">, + Intrinsic<[llvm_v4i32_ty], + [llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pbroadcast_d_gpr_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastd256_gpr_mask">, + Intrinsic<[llvm_v8i32_ty], + [llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pbroadcast_d_gpr_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastd512_gpr_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_i32_ty, llvm_v16i32_ty, - llvm_i16_ty], [IntrNoMem]>; + GCCBuiltin<"__builtin_ia32_pbroadcastd512_gpr_mask">, + Intrinsic<[llvm_v16i32_ty], + [llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_pbroadcast_q_gpr_128 : + GCCBuiltin<"__builtin_ia32_pbroadcastq128_gpr_mask">, + Intrinsic<[llvm_v2i64_ty], + [llvm_i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pbroadcast_q_gpr_256 : + GCCBuiltin<"__builtin_ia32_pbroadcastq256_gpr_mask">, + Intrinsic<[llvm_v4i64_ty], + [llvm_i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pbroadcast_q_gpr_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastq512_gpr_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_i64_ty, llvm_v8i64_ty, - llvm_i8_ty], [IntrNoMem]>; + GCCBuiltin<"__builtin_ia32_pbroadcastq512_gpr_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_pbroadcast_q_mem_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastq512_mem_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_i64_ty, llvm_v8i64_ty, - llvm_i8_ty], [IntrNoMem]>; + GCCBuiltin<"__builtin_ia32_pbroadcastq512_mem_mask">, + Intrinsic<[llvm_v8i64_ty], + [llvm_i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } // Vector permutation @@ -2750,9 +2475,9 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem]>; def int_x86_avx512_mask_insertf32x4_512 : - GCCBuiltin<"__builtin_ia32_insertf32x4_512_mask">, + GCCBuiltin<"__builtin_ia32_insertf32x4_mask">, Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i8_ty], + [llvm_v16f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_insertf32x8_512 : @@ -2786,9 +2511,9 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem]>; def int_x86_avx512_mask_inserti32x4_512 : - GCCBuiltin<"__builtin_ia32_inserti32x4_512_mask">, + GCCBuiltin<"__builtin_ia32_inserti32x4_mask">, Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v16i32_ty, llvm_i8_ty], + [llvm_v16i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_inserti32x8_512 : @@ -2813,55 +2538,41 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_inserti64x4_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem]>; } // Conditional load ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_maskload_d : GCCBuiltin<"__builtin_ia32_maskloadd">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_v4i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_maskload_q : GCCBuiltin<"__builtin_ia32_maskloadq">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_v2i64_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_maskload_d_256 : GCCBuiltin<"__builtin_ia32_maskloadd256">, Intrinsic<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_v8i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_maskload_q_256 : GCCBuiltin<"__builtin_ia32_maskloadq256">, Intrinsic<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_v4i64_ty], - [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_d_512 : GCCBuiltin<"__builtin_ia32_loaddqusi512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadArgMem]>; - def int_x86_avx512_mask_loadu_q_512 : GCCBuiltin<"__builtin_ia32_loaddqudi512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; } // Conditional store ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_maskstore_d : GCCBuiltin<"__builtin_ia32_maskstored">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx2_maskstore_q : GCCBuiltin<"__builtin_ia32_maskstoreq">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i64_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx2_maskstore_d_256 : GCCBuiltin<"__builtin_ia32_maskstored256">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx2_maskstore_q_256 : GCCBuiltin<"__builtin_ia32_maskstoreq256">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i64_ty], - [IntrReadWriteArgMem]>; - def int_x86_avx512_mask_storeu_d_512 : - GCCBuiltin<"__builtin_ia32_storedqusi512_mask">, - Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; - def int_x86_avx512_mask_storeu_q_512 : - GCCBuiltin<"__builtin_ia32_storedqudi512_mask">, - Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; } // Variable bit shift ops @@ -2905,7 +2616,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem]>; def int_x86_avx512_mask_psllv_q : GCCBuiltin<"__builtin_ia32_psllv8di_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav_d : GCCBuiltin<"__builtin_ia32_psrav16si_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, @@ -2921,66 +2632,60 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem]>; def int_x86_avx512_mask_psrlv_q : GCCBuiltin<"__builtin_ia32_psrlv8di_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_psll_dq_512 : GCCBuiltin<"__builtin_ia32_pslldq512">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty], + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_psrl_dq_512 : GCCBuiltin<"__builtin_ia32_psrldq512">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_i32_ty], - [IntrNoMem]>; def int_x86_avx512_mask_psll_d_128 : GCCBuiltin<"__builtin_ia32_pslld128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_d_256 : GCCBuiltin<"__builtin_ia32_pslld256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_di_128 : GCCBuiltin<"__builtin_ia32_pslldi128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, - llvm_i8_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_di_256 : GCCBuiltin<"__builtin_ia32_pslldi256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_i8_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_di_512 : GCCBuiltin<"__builtin_ia32_pslldi512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_q_128 : GCCBuiltin<"__builtin_ia32_psllq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_q_256 : GCCBuiltin<"__builtin_ia32_psllq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_qi_128 : GCCBuiltin<"__builtin_ia32_psllqi128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, - llvm_i8_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + llvm_i32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_qi_256 : GCCBuiltin<"__builtin_ia32_psllqi256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, - llvm_i8_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psll_qi_512 : GCCBuiltin<"__builtin_ia32_psllqi512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav16_hi : GCCBuiltin<"__builtin_ia32_psrav16hi_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav32_hi : GCCBuiltin<"__builtin_ia32_psrav32hi_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav4_si : GCCBuiltin<"__builtin_ia32_psrav4si_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav8_hi : GCCBuiltin<"__builtin_ia32_psrav8hi_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav8_si : GCCBuiltin<"__builtin_ia32_psrav8si_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav_q_128 : GCCBuiltin<"__builtin_ia32_psravq128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrav_q_256 : GCCBuiltin<"__builtin_ia32_psravq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrlv16_hi : GCCBuiltin<"__builtin_ia32_psrlv16hi_mask">, @@ -2990,19 +2695,19 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrlv32hi : GCCBuiltin<"__builtin_ia32_psrlv32hi_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrlv4_di : GCCBuiltin<"__builtin_ia32_psrlv4di_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrlv4_si : GCCBuiltin<"__builtin_ia32_psrlv4si_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrlv8_hi : GCCBuiltin<"__builtin_ia32_psrlv8hi_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_psrlv8_si : GCCBuiltin<"__builtin_ia32_psrlv8si_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_prorv_d_128 : GCCBuiltin<"__builtin_ia32_prorvd128_mask">, @@ -3026,22 +2731,22 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_mask_prol_d_128 : GCCBuiltin<"__builtin_ia32_prold128_mask">, Intrinsic<[llvm_v4i32_ty] , [llvm_v4i32_ty, - llvm_i8_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_prol_d_256 : GCCBuiltin<"__builtin_ia32_prold256_mask">, Intrinsic<[llvm_v8i32_ty] , [llvm_v8i32_ty, - llvm_i8_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_prol_d_512 : GCCBuiltin<"__builtin_ia32_prold512_mask">, Intrinsic<[llvm_v16i32_ty] , [llvm_v16i32_ty, - llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_prol_q_128 : GCCBuiltin<"__builtin_ia32_prolq128_mask">, Intrinsic<[llvm_v2i64_ty] , [llvm_v2i64_ty, - llvm_i8_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_prol_q_256 : GCCBuiltin<"__builtin_ia32_prolq256_mask">, Intrinsic<[llvm_v4i64_ty] , [llvm_v4i64_ty, - llvm_i8_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_prol_q_512 : GCCBuiltin<"__builtin_ia32_prolq512_mask">, Intrinsic<[llvm_v8i64_ty] , [llvm_v8i64_ty, - llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_prolv_d_128 : GCCBuiltin<"__builtin_ia32_prolvd128_mask">, @@ -3064,22 +2769,22 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pror_d_128 : GCCBuiltin<"__builtin_ia32_prord128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, - llvm_i8_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pror_d_256 : GCCBuiltin<"__builtin_ia32_prord256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, - llvm_i8_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pror_d_512 : GCCBuiltin<"__builtin_ia32_prord512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, - llvm_i8_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pror_q_128 : GCCBuiltin<"__builtin_ia32_prorq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, - llvm_i8_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pror_q_256 : GCCBuiltin<"__builtin_ia32_prorq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, - llvm_i8_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_i32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pror_q_512 : GCCBuiltin<"__builtin_ia32_prorq512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, - llvm_i8_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_i32_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } @@ -3088,68 +2793,68 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_gather_d_pd : GCCBuiltin<"__builtin_ia32_gatherd_pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_d_pd_256 : GCCBuiltin<"__builtin_ia32_gatherd_pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_pd : GCCBuiltin<"__builtin_ia32_gatherq_pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_pd_256 : GCCBuiltin<"__builtin_ia32_gatherq_pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_d_ps : GCCBuiltin<"__builtin_ia32_gatherd_ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_d_ps_256 : GCCBuiltin<"__builtin_ia32_gatherd_ps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_ps : GCCBuiltin<"__builtin_ia32_gatherq_ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_ps_256 : GCCBuiltin<"__builtin_ia32_gatherq_ps256">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_d_q : GCCBuiltin<"__builtin_ia32_gatherd_q">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_d_q_256 : GCCBuiltin<"__builtin_ia32_gatherd_q256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_q : GCCBuiltin<"__builtin_ia32_gatherq_q">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_q_256 : GCCBuiltin<"__builtin_ia32_gatherq_q256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_d_d : GCCBuiltin<"__builtin_ia32_gatherd_d">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_d_d_256 : GCCBuiltin<"__builtin_ia32_gatherd_d256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_d : GCCBuiltin<"__builtin_ia32_gatherq_d">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx2_gather_q_d_256 : GCCBuiltin<"__builtin_ia32_gatherq_d256">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; } // Misc. @@ -3520,6 +3225,43 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vfmadd_sd : + GCCBuiltin<"__builtin_ia32_vfmaddsd3_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask_vfmadd_ss : + GCCBuiltin<"__builtin_ia32_vfmaddss3_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_vfmadd_sd : + GCCBuiltin<"__builtin_ia32_vfmaddsd3_maskz">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_maskz_vfmadd_ss : + GCCBuiltin<"__builtin_ia32_vfmaddss3_maskz">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask3_vfmadd_sd : + GCCBuiltin<"__builtin_ia32_vfmaddsd3_mask3">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + + def int_x86_avx512_mask3_vfmadd_ss : + GCCBuiltin<"__builtin_ia32_vfmaddss3_mask3">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask3_vfmsub_pd_128 : GCCBuiltin<"__builtin_ia32_vfmsubpd128_mask3">, Intrinsic<[llvm_v2f64_ty], @@ -3700,30 +3442,79 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpmadd52h_uq_128 : + GCCBuiltin<"__builtin_ia32_vpmadd52huq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpmadd52h_uq_128 : + GCCBuiltin<"__builtin_ia32_vpmadd52huq128_maskz">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpmadd52l_uq_128 : + GCCBuiltin<"__builtin_ia32_vpmadd52luq128_mask">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpmadd52l_uq_128 : + GCCBuiltin<"__builtin_ia32_vpmadd52luq128_maskz">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpmadd52h_uq_256 : + GCCBuiltin<"__builtin_ia32_vpmadd52huq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpmadd52h_uq_256 : + GCCBuiltin<"__builtin_ia32_vpmadd52huq256_maskz">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpmadd52l_uq_256 : + GCCBuiltin<"__builtin_ia32_vpmadd52luq256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpmadd52l_uq_256 : + GCCBuiltin<"__builtin_ia32_vpmadd52luq256_maskz">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpmadd52h_uq_512 : + GCCBuiltin<"__builtin_ia32_vpmadd52huq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpmadd52h_uq_512 : + GCCBuiltin<"__builtin_ia32_vpmadd52huq512_maskz">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_vpmadd52l_uq_512 : + GCCBuiltin<"__builtin_ia32_vpmadd52luq512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_vpmadd52l_uq_512 : + GCCBuiltin<"__builtin_ia32_vpmadd52luq512_maskz">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, + llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } //===----------------------------------------------------------------------===// // XOP +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_xop_vpermil2pd : GCCBuiltin<"__builtin_ia32_vpermil2pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty], + llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_xop_vpermil2pd_256 : GCCBuiltin<"__builtin_ia32_vpermil2pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, - llvm_v4f64_ty, llvm_i8_ty], + llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_xop_vpermil2ps : GCCBuiltin<"__builtin_ia32_vpermil2ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty], + llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_xop_vpermil2ps_256 : GCCBuiltin<"__builtin_ia32_vpermil2ps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, - llvm_v8f32_ty, llvm_i8_ty], + llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_xop_vfrcz_pd : GCCBuiltin<"__builtin_ia32_vfrczpd">, @@ -3943,6 +3734,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_vpshlw">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem]>; +} //===----------------------------------------------------------------------===// // MMX @@ -4131,7 +3923,51 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; } - +// Permute +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_avx512_mask_permvar_df_256 : GCCBuiltin<"__builtin_ia32_permvardf256_mask">, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, + llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_df_512 : GCCBuiltin<"__builtin_ia32_permvardf512_mask">, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, + llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_di_256 : GCCBuiltin<"__builtin_ia32_permvardi256_mask">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, + llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_di_512 : GCCBuiltin<"__builtin_ia32_permvardi512_mask">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, + llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_hi_128 : GCCBuiltin<"__builtin_ia32_permvarhi128_mask">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_hi_256 : GCCBuiltin<"__builtin_ia32_permvarhi256_mask">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_hi_512 : GCCBuiltin<"__builtin_ia32_permvarhi512_mask">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, + llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_qi_128 : GCCBuiltin<"__builtin_ia32_permvarqi128_mask">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, + llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_qi_256 : GCCBuiltin<"__builtin_ia32_permvarqi256_mask">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, + llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_qi_512 : GCCBuiltin<"__builtin_ia32_permvarqi512_mask">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, + llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_sf_256 : GCCBuiltin<"__builtin_ia32_permvarsf256_mask">, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, + llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_sf_512 : GCCBuiltin<"__builtin_ia32_permvarsf512_mask">, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, + llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_si_256 : GCCBuiltin<"__builtin_ia32_permvarsi256_mask">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, + llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_permvar_si_512 : GCCBuiltin<"__builtin_ia32_permvarsi512_mask">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, + llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; +} // Pack ops. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_mmx_packsswb : GCCBuiltin<"__builtin_ia32_packsswb">, @@ -4301,6 +4137,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". } //===----------------------------------------------------------------------===// +// CLFLUSHOPT +let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + def int_x86_clflushopt : GCCBuiltin<"__builtin_ia32_clflushopt">, + Intrinsic<[], [llvm_ptr_ty], []>; +} + +//===----------------------------------------------------------------------===// // Support protection key let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_rdpkru : GCCBuiltin <"__builtin_ia32_rdpkru">, @@ -4374,22 +4217,22 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_addcarryx_u32: GCCBuiltin<"__builtin_ia32_addcarryx_u32">, Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty, - llvm_ptr_ty], [IntrReadWriteArgMem]>; + llvm_ptr_ty], [IntrArgMemOnly]>; def int_x86_addcarryx_u64: GCCBuiltin<"__builtin_ia32_addcarryx_u64">, Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty, - llvm_ptr_ty], [IntrReadWriteArgMem]>; + llvm_ptr_ty], [IntrArgMemOnly]>; def int_x86_addcarry_u32: GCCBuiltin<"__builtin_ia32_addcarry_u32">, Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty, - llvm_ptr_ty], [IntrReadWriteArgMem]>; + llvm_ptr_ty], [IntrArgMemOnly]>; def int_x86_addcarry_u64: GCCBuiltin<"__builtin_ia32_addcarry_u64">, Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty, - llvm_ptr_ty], [IntrReadWriteArgMem]>; + llvm_ptr_ty], [IntrArgMemOnly]>; def int_x86_subborrow_u32: GCCBuiltin<"__builtin_ia32_subborrow_u32">, Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i32_ty, llvm_i32_ty, - llvm_ptr_ty], [IntrReadWriteArgMem]>; + llvm_ptr_ty], [IntrArgMemOnly]>; def int_x86_subborrow_u64: GCCBuiltin<"__builtin_ia32_subborrow_u64">, Intrinsic<[llvm_i8_ty], [llvm_i8_ty, llvm_i64_ty, llvm_i64_ty, - llvm_ptr_ty], [IntrReadWriteArgMem]>; + llvm_ptr_ty], [IntrArgMemOnly]>; } //===----------------------------------------------------------------------===// @@ -4401,7 +4244,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_xend : GCCBuiltin<"__builtin_ia32_xend">, Intrinsic<[], [], []>; def int_x86_xabort : GCCBuiltin<"__builtin_ia32_xabort">, - Intrinsic<[], [llvm_i8_ty], [IntrNoReturn]>; + Intrinsic<[], [llvm_i8_ty], []>; def int_x86_xtest : GCCBuiltin<"__builtin_ia32_xtest">, Intrinsic<[llvm_i32_ty], [], []>; } @@ -4445,7 +4288,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_kortestc_w : GCCBuiltin<"__builtin_ia32_kortestchi">, Intrinsic<[llvm_i32_ty], [llvm_i16_ty, llvm_i16_ty], [IntrNoMem]>; - + def int_x86_avx512_mask_pmovsxb_d_128 : GCCBuiltin<"__builtin_ia32_pmovsxbd128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; @@ -4504,10 +4347,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Conversion ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_cvtss2usi : GCCBuiltin<"__builtin_ia32_cvtss2usi">, - Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty], [IntrNoMem]>; - def int_x86_avx512_cvtss2usi64 : GCCBuiltin<"__builtin_ia32_cvtss2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty], [IntrNoMem]>; def int_x86_avx512_cvttss2si : GCCBuiltin<"__builtin_ia32_vcvttss2si32">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvttss2si64 : GCCBuiltin<"__builtin_ia32_vcvttss2si64">, @@ -4522,11 +4361,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_cvtusi642ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss64">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; - - def int_x86_avx512_cvtsd2usi : GCCBuiltin<"__builtin_ia32_cvtsd2usi">, - Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty], [IntrNoMem]>; - def int_x86_avx512_cvtsd2usi64 : GCCBuiltin<"__builtin_ia32_cvtsd2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty], [IntrNoMem]>; def int_x86_avx512_cvttsd2si : GCCBuiltin<"__builtin_ia32_vcvttsd2si32">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvttsd2si64 : GCCBuiltin<"__builtin_ia32_vcvttsd2si64">, @@ -4541,7 +4375,22 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_cvtusi642sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd64">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; - + def int_x86_avx512_vcvtss2usi32 : GCCBuiltin<"__builtin_ia32_vcvtss2usi32">, + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcvtss2usi64 : GCCBuiltin<"__builtin_ia32_vcvtss2usi64">, + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcvtss2si32 : GCCBuiltin<"__builtin_ia32_vcvtss2si32">, + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcvtss2si64 : GCCBuiltin<"__builtin_ia32_vcvtss2si64">, + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcvtsd2usi32 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi32">, + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcvtsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi64">, + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcvtsd2si32 : GCCBuiltin<"__builtin_ia32_vcvtsd2si32">, + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_vcvtsd2si64 : GCCBuiltin<"__builtin_ia32_vcvtsd2si64">, + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtsi2ss32 : GCCBuiltin<"__builtin_ia32_cvtsi2ss32">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; @@ -4553,7 +4402,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtsi2sd64 : GCCBuiltin<"__builtin_ia32_cvtsi2sd64">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtb2mask_128 : GCCBuiltin<"__builtin_ia32_cvtb2mask128">, Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty], [IntrNoMem]>; @@ -4561,23 +4410,23 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx512_cvtb2mask_512 : GCCBuiltin<"__builtin_ia32_cvtb2mask512">, Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty], [IntrNoMem]>; - + def int_x86_avx512_cvtw2mask_128 : GCCBuiltin<"__builtin_ia32_cvtw2mask128">, Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_avx512_cvtw2mask_256 : GCCBuiltin<"__builtin_ia32_cvtw2mask256">, Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty], [IntrNoMem]>; def int_x86_avx512_cvtw2mask_512 : GCCBuiltin<"__builtin_ia32_cvtw2mask512">, Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty], [IntrNoMem]>; - + def int_x86_avx512_cvtd2mask_128 : GCCBuiltin<"__builtin_ia32_cvtd2mask128">, Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtd2mask_256 : GCCBuiltin<"__builtin_ia32_cvtd2mask256">, - Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtd2mask_512 : GCCBuiltin<"__builtin_ia32_cvtd2mask512">, Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtq2mask_128 : GCCBuiltin<"__builtin_ia32_cvtq2mask128">, - Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty], [IntrNoMem]>; + Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty], [IntrNoMem]>; def int_x86_avx512_cvtq2mask_256 : GCCBuiltin<"__builtin_ia32_cvtq2mask256">, Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty], [IntrNoMem]>; def int_x86_avx512_cvtq2mask_512 : GCCBuiltin<"__builtin_ia32_cvtq2mask512">, @@ -4589,28 +4438,28 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v32i8_ty], [llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_cvtmask2b_512 : GCCBuiltin<"__builtin_ia32_cvtmask2b512">, Intrinsic<[llvm_v64i8_ty], [llvm_i64_ty], [IntrNoMem]>; - + def int_x86_avx512_cvtmask2w_128 : GCCBuiltin<"__builtin_ia32_cvtmask2w128">, Intrinsic<[llvm_v8i16_ty], [llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_cvtmask2w_256 : GCCBuiltin<"__builtin_ia32_cvtmask2w256">, Intrinsic<[llvm_v16i16_ty], [llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_cvtmask2w_512 : GCCBuiltin<"__builtin_ia32_cvtmask2w512">, Intrinsic<[llvm_v32i16_ty], [llvm_i32_ty], [IntrNoMem]>; - + def int_x86_avx512_cvtmask2d_128 : GCCBuiltin<"__builtin_ia32_cvtmask2d128">, Intrinsic<[llvm_v4i32_ty], [llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_cvtmask2d_256 : GCCBuiltin<"__builtin_ia32_cvtmask2d256">, - Intrinsic<[llvm_v8i32_ty], [llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i32_ty], [llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_cvtmask2d_512 : GCCBuiltin<"__builtin_ia32_cvtmask2d512">, Intrinsic<[llvm_v16i32_ty], [llvm_i16_ty], [IntrNoMem]>; - + def int_x86_avx512_cvtmask2q_128 : GCCBuiltin<"__builtin_ia32_cvtmask2q128">, - Intrinsic<[llvm_v2i64_ty], [llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_cvtmask2q_256 : GCCBuiltin<"__builtin_ia32_cvtmask2q256">, Intrinsic<[llvm_v4i64_ty], [llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_cvtmask2q_512 : GCCBuiltin<"__builtin_ia32_cvtmask2q512">, Intrinsic<[llvm_v8i64_ty], [llvm_i8_ty], [IntrNoMem]>; - + } // Pack ops. @@ -4622,7 +4471,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty,llvm_v16i16_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_packsswb_512 : GCCBuiltin<"__builtin_ia32_packsswb512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty, + Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_packssdw_128 : GCCBuiltin<"__builtin_ia32_packssdw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty, @@ -4640,7 +4489,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v32i8_ty], [llvm_v16i16_ty,llvm_v16i16_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_packuswb_512 : GCCBuiltin<"__builtin_ia32_packuswb512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty, + Intrinsic<[llvm_v64i8_ty], [llvm_v32i16_ty,llvm_v32i16_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_packusdw_128 : GCCBuiltin<"__builtin_ia32_packusdw128_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4i32_ty, llvm_v4i32_ty, @@ -4653,702 +4502,483 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; } -// Unpack ops. -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_mask_unpckh_pd_128 : - GCCBuiltin<"__builtin_ia32_unpckhpd128_mask">, - Intrinsic<[llvm_v2f64_ty], - [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckh_pd_256 : - GCCBuiltin<"__builtin_ia32_unpckhpd256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckh_pd_512 : - GCCBuiltin<"__builtin_ia32_unpckhpd512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckh_ps_128 : - GCCBuiltin<"__builtin_ia32_unpckhps128_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckh_ps_256 : - GCCBuiltin<"__builtin_ia32_unpckhps256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckh_ps_512 : - GCCBuiltin<"__builtin_ia32_unpckhps512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckl_pd_128 : - GCCBuiltin<"__builtin_ia32_unpcklpd128_mask">, - Intrinsic<[llvm_v2f64_ty], - [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckl_pd_256 : - GCCBuiltin<"__builtin_ia32_unpcklpd256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckl_pd_512 : - GCCBuiltin<"__builtin_ia32_unpcklpd512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckl_ps_128 : - GCCBuiltin<"__builtin_ia32_unpcklps128_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckl_ps_256 : - GCCBuiltin<"__builtin_ia32_unpcklps256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_unpckl_ps_512 : - GCCBuiltin<"__builtin_ia32_unpcklps512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhb_w_128 : - GCCBuiltin<"__builtin_ia32_punpckhbw128_mask">, - Intrinsic<[llvm_v16i8_ty], - [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhb_w_256 : - GCCBuiltin<"__builtin_ia32_punpckhbw256_mask">, - Intrinsic<[llvm_v32i8_ty], - [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhb_w_512 : - GCCBuiltin<"__builtin_ia32_punpckhbw512_mask">, - Intrinsic<[llvm_v64i8_ty], - [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhd_q_128 : - GCCBuiltin<"__builtin_ia32_punpckhdq128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhd_q_256 : - GCCBuiltin<"__builtin_ia32_punpckhdq256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhd_q_512 : - GCCBuiltin<"__builtin_ia32_punpckhdq512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhqd_q_128 : - GCCBuiltin<"__builtin_ia32_punpckhqdq128_mask">, - Intrinsic<[llvm_v2i64_ty], - [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhqd_q_256 : - GCCBuiltin<"__builtin_ia32_punpckhqdq256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhqd_q_512 : - GCCBuiltin<"__builtin_ia32_punpckhqdq512_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhw_d_128 : - GCCBuiltin<"__builtin_ia32_punpckhwd128_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhw_d_256 : - GCCBuiltin<"__builtin_ia32_punpckhwd256_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckhw_d_512 : - GCCBuiltin<"__builtin_ia32_punpckhwd512_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklb_w_128 : - GCCBuiltin<"__builtin_ia32_punpcklbw128_mask">, - Intrinsic<[llvm_v16i8_ty], - [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklb_w_256 : - GCCBuiltin<"__builtin_ia32_punpcklbw256_mask">, - Intrinsic<[llvm_v32i8_ty], - [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklb_w_512 : - GCCBuiltin<"__builtin_ia32_punpcklbw512_mask">, - Intrinsic<[llvm_v64i8_ty], - [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckld_q_128 : - GCCBuiltin<"__builtin_ia32_punpckldq128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckld_q_256 : - GCCBuiltin<"__builtin_ia32_punpckldq256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpckld_q_512 : - GCCBuiltin<"__builtin_ia32_punpckldq512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklqd_q_128 : - GCCBuiltin<"__builtin_ia32_punpcklqdq128_mask">, - Intrinsic<[llvm_v2i64_ty], - [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklqd_q_256 : - GCCBuiltin<"__builtin_ia32_punpcklqdq256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklqd_q_512 : - GCCBuiltin<"__builtin_ia32_punpcklqdq512_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklw_d_128 : - GCCBuiltin<"__builtin_ia32_punpcklwd128_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklw_d_256 : - GCCBuiltin<"__builtin_ia32_punpcklwd256_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_punpcklw_d_512 : - GCCBuiltin<"__builtin_ia32_punpcklwd512_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; -} - // Vector convert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_mask_cvtdq2pd_128 : + def int_x86_avx512_mask_cvtdq2pd_128 : GCCBuiltin<"__builtin_ia32_cvtdq2pd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtdq2pd_256 : + def int_x86_avx512_mask_cvtdq2pd_256 : GCCBuiltin<"__builtin_ia32_cvtdq2pd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtdq2pd_512 : + def int_x86_avx512_mask_cvtdq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtdq2pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8i32_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtdq2ps_128 : + def int_x86_avx512_mask_cvtdq2ps_128 : GCCBuiltin<"__builtin_ia32_cvtdq2ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtdq2ps_256 : + def int_x86_avx512_mask_cvtdq2ps_256 : GCCBuiltin<"__builtin_ia32_cvtdq2ps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtdq2ps_512 : + def int_x86_avx512_mask_cvtdq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtdq2ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2dq_128 : + def int_x86_avx512_mask_cvtpd2dq_128 : GCCBuiltin<"__builtin_ia32_cvtpd2dq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2dq_256 : + def int_x86_avx512_mask_cvtpd2dq_256 : GCCBuiltin<"__builtin_ia32_cvtpd2dq256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2dq_512 : + def int_x86_avx512_mask_cvtpd2dq_512 : GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2ps_256 : + def int_x86_avx512_mask_cvtpd2ps_256 : GCCBuiltin<"__builtin_ia32_cvtpd2ps256_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2ps_512 : + def int_x86_avx512_mask_cvtpd2ps_512 : GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtsd2ss_round : - GCCBuiltin<"__builtin_ia32_cvtsd2ss_round">, + def int_x86_avx512_mask_cvtsd2ss_round : + GCCBuiltin<"__builtin_ia32_cvtsd2ss_round_mask">, Intrinsic<[llvm_v4f32_ty], - [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], + [llvm_v4f32_ty, llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtss2sd_round : - GCCBuiltin<"__builtin_ia32_cvtss2sd_round">, + def int_x86_avx512_mask_cvtss2sd_round : + GCCBuiltin<"__builtin_ia32_cvtss2sd_round_mask">, Intrinsic<[llvm_v2f64_ty], - [ llvm_v4f32_ty, llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], + [llvm_v2f64_ty, llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2ps : + def int_x86_avx512_mask_cvtpd2ps : GCCBuiltin<"__builtin_ia32_cvtpd2ps_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2qq_128 : + def int_x86_avx512_mask_cvtpd2qq_128 : GCCBuiltin<"__builtin_ia32_cvtpd2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2qq_256 : + def int_x86_avx512_mask_cvtpd2qq_256 : GCCBuiltin<"__builtin_ia32_cvtpd2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2qq_512 : + def int_x86_avx512_mask_cvtpd2qq_512 : GCCBuiltin<"__builtin_ia32_cvtpd2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2udq_128 : + def int_x86_avx512_mask_cvtpd2udq_128 : GCCBuiltin<"__builtin_ia32_cvtpd2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2udq_256 : + def int_x86_avx512_mask_cvtpd2udq_256 : GCCBuiltin<"__builtin_ia32_cvtpd2udq256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2udq_512 : + def int_x86_avx512_mask_cvtpd2udq_512 : GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2uqq_128 : + def int_x86_avx512_mask_cvtpd2uqq_128 : GCCBuiltin<"__builtin_ia32_cvtpd2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2uqq_256 : + def int_x86_avx512_mask_cvtpd2uqq_256 : GCCBuiltin<"__builtin_ia32_cvtpd2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtpd2uqq_512 : + def int_x86_avx512_mask_cvtpd2uqq_512 : GCCBuiltin<"__builtin_ia32_cvtpd2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2dq_128 : + def int_x86_avx512_mask_cvtps2dq_128 : GCCBuiltin<"__builtin_ia32_cvtps2dq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2dq_256 : + def int_x86_avx512_mask_cvtps2dq_256 : GCCBuiltin<"__builtin_ia32_cvtps2dq256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2dq_512 : + def int_x86_avx512_mask_cvtps2dq_512 : GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2pd_128 : + def int_x86_avx512_mask_cvtps2pd_128 : GCCBuiltin<"__builtin_ia32_cvtps2pd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2pd_256 : + def int_x86_avx512_mask_cvtps2pd_256 : GCCBuiltin<"__builtin_ia32_cvtps2pd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2pd_512 : + def int_x86_avx512_mask_cvtps2pd_512 : GCCBuiltin<"__builtin_ia32_cvtps2pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2qq_128 : + def int_x86_avx512_mask_cvtps2qq_128 : GCCBuiltin<"__builtin_ia32_cvtps2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2qq_256 : + def int_x86_avx512_mask_cvtps2qq_256 : GCCBuiltin<"__builtin_ia32_cvtps2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2qq_512 : + def int_x86_avx512_mask_cvtps2qq_512 : GCCBuiltin<"__builtin_ia32_cvtps2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2udq_128 : + def int_x86_avx512_mask_cvtps2udq_128 : GCCBuiltin<"__builtin_ia32_cvtps2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2udq_256 : + def int_x86_avx512_mask_cvtps2udq_256 : GCCBuiltin<"__builtin_ia32_cvtps2udq256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2udq_512 : + def int_x86_avx512_mask_cvtps2udq_512 : GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2uqq_128 : + def int_x86_avx512_mask_cvtps2uqq_128 : GCCBuiltin<"__builtin_ia32_cvtps2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2uqq_256 : + def int_x86_avx512_mask_cvtps2uqq_256 : GCCBuiltin<"__builtin_ia32_cvtps2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtps2uqq_512 : + def int_x86_avx512_mask_cvtps2uqq_512 : GCCBuiltin<"__builtin_ia32_cvtps2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtqq2pd_128 : + def int_x86_avx512_mask_cvtqq2pd_128 : GCCBuiltin<"__builtin_ia32_cvtqq2pd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtqq2pd_256 : + def int_x86_avx512_mask_cvtqq2pd_256 : GCCBuiltin<"__builtin_ia32_cvtqq2pd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtqq2pd_512 : + def int_x86_avx512_mask_cvtqq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtqq2pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtqq2ps_128 : + def int_x86_avx512_mask_cvtqq2ps_128 : GCCBuiltin<"__builtin_ia32_cvtqq2ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtqq2ps_256 : + def int_x86_avx512_mask_cvtqq2ps_256 : GCCBuiltin<"__builtin_ia32_cvtqq2ps256_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtqq2ps_512 : + def int_x86_avx512_mask_cvtqq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtqq2ps512_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2dq_128 : + def int_x86_avx512_mask_cvttpd2dq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2dq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2dq_256 : + def int_x86_avx512_mask_cvttpd2dq_256 : GCCBuiltin<"__builtin_ia32_cvttpd2dq256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2dq_512 : + def int_x86_avx512_mask_cvttpd2dq_512 : GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2qq_128 : + def int_x86_avx512_mask_cvttpd2qq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2qq_256 : + def int_x86_avx512_mask_cvttpd2qq_256 : GCCBuiltin<"__builtin_ia32_cvttpd2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2qq_512 : + def int_x86_avx512_mask_cvttpd2qq_512 : GCCBuiltin<"__builtin_ia32_cvttpd2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2udq_128 : + def int_x86_avx512_mask_cvttpd2udq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v2f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2udq_256 : + def int_x86_avx512_mask_cvttpd2udq_256 : GCCBuiltin<"__builtin_ia32_cvttpd2udq256_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f64_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2udq_512 : + def int_x86_avx512_mask_cvttpd2udq_512 : GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2uqq_128 : + def int_x86_avx512_mask_cvttpd2uqq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2uqq_256 : + def int_x86_avx512_mask_cvttpd2uqq_256 : GCCBuiltin<"__builtin_ia32_cvttpd2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2uqq_512 : + def int_x86_avx512_mask_cvttpd2uqq_512 : GCCBuiltin<"__builtin_ia32_cvttpd2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2dq_128 : + def int_x86_avx512_mask_cvttps2dq_128 : GCCBuiltin<"__builtin_ia32_cvttps2dq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2dq_256 : + def int_x86_avx512_mask_cvttps2dq_256 : GCCBuiltin<"__builtin_ia32_cvttps2dq256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2dq_512 : + def int_x86_avx512_mask_cvttps2dq_512 : GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2qq_128 : + def int_x86_avx512_mask_cvttps2qq_128 : GCCBuiltin<"__builtin_ia32_cvttps2qq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2qq_256 : + def int_x86_avx512_mask_cvttps2qq_256 : GCCBuiltin<"__builtin_ia32_cvttps2qq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2qq_512 : + def int_x86_avx512_mask_cvttps2qq_512 : GCCBuiltin<"__builtin_ia32_cvttps2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2udq_128 : + def int_x86_avx512_mask_cvttps2udq_128 : GCCBuiltin<"__builtin_ia32_cvttps2udq128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2udq_256 : + def int_x86_avx512_mask_cvttps2udq_256 : GCCBuiltin<"__builtin_ia32_cvttps2udq256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2udq_512 : + def int_x86_avx512_mask_cvttps2udq_512 : GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2uqq_128 : + def int_x86_avx512_mask_cvttps2uqq_128 : GCCBuiltin<"__builtin_ia32_cvttps2uqq128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v4f32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2uqq_256 : + def int_x86_avx512_mask_cvttps2uqq_256 : GCCBuiltin<"__builtin_ia32_cvttps2uqq256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4f32_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvttps2uqq_512 : + def int_x86_avx512_mask_cvttps2uqq_512 : GCCBuiltin<"__builtin_ia32_cvttps2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2pd_128 : + def int_x86_avx512_mask_cvtudq2pd_128 : GCCBuiltin<"__builtin_ia32_cvtudq2pd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2pd_256 : + def int_x86_avx512_mask_cvtudq2pd_256 : GCCBuiltin<"__builtin_ia32_cvtudq2pd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2pd_512 : + def int_x86_avx512_mask_cvtudq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtudq2pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8i32_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2ps_128 : + def int_x86_avx512_mask_cvtudq2ps_128 : GCCBuiltin<"__builtin_ia32_cvtudq2ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2ps_256 : + def int_x86_avx512_mask_cvtudq2ps_256 : GCCBuiltin<"__builtin_ia32_cvtudq2ps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtudq2ps_512 : + def int_x86_avx512_mask_cvtudq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtudq2ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtuqq2pd_128 : + def int_x86_avx512_mask_cvtuqq2pd_128 : GCCBuiltin<"__builtin_ia32_cvtuqq2pd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtuqq2pd_256 : + def int_x86_avx512_mask_cvtuqq2pd_256 : GCCBuiltin<"__builtin_ia32_cvtuqq2pd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtuqq2pd_512 : + def int_x86_avx512_mask_cvtuqq2pd_512 : GCCBuiltin<"__builtin_ia32_cvtuqq2pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtuqq2ps_128 : + def int_x86_avx512_mask_cvtuqq2ps_128 : GCCBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtuqq2ps_256 : + def int_x86_avx512_mask_cvtuqq2ps_256 : GCCBuiltin<"__builtin_ia32_cvtuqq2ps256_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtuqq2ps_512 : + def int_x86_avx512_mask_cvtuqq2ps_512 : GCCBuiltin<"__builtin_ia32_cvtuqq2ps512_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], @@ -5358,7 +4988,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_rndscale_pd_256 : GCCBuiltin<"__builtin_ia32_rndscalepd_256_mask">, - Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_rndscale_pd_512 : GCCBuiltin<"__builtin_ia32_rndscalepd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, @@ -5367,13 +4997,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_rndscale_ps_256 : GCCBuiltin<"__builtin_ia32_rndscaleps_256_mask">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_rndscale_ps_512 : GCCBuiltin<"__builtin_ia32_rndscaleps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_reduce_pd_128 : GCCBuiltin<"__builtin_ia32_reducepd128_mask">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_reduce_pd_256 : GCCBuiltin<"__builtin_ia32_reducepd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, @@ -5382,7 +5012,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_reduce_ps_128 : GCCBuiltin<"__builtin_ia32_reduceps128_mask">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_reduce_ps_256 : GCCBuiltin<"__builtin_ia32_reduceps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, @@ -5391,7 +5021,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_range_pd_128 : GCCBuiltin<"__builtin_ia32_rangepd128_mask">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_range_pd_256 : GCCBuiltin<"__builtin_ia32_rangepd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty, @@ -5403,7 +5033,7 @@ def int_x86_avx512_mask_range_ps_128 : GCCBuiltin<"__builtin_ia32_rangeps128_mas Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_range_ps_256 : GCCBuiltin<"__builtin_ia32_rangeps256_mask">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, @@ -5414,75 +5044,11 @@ def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mas let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_vbroadcast_ss_512 : GCCBuiltin<"__builtin_ia32_vbroadcastss512">, - Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_broadcast_ss_ps_512 : - GCCBuiltin<"__builtin_ia32_broadcastss512">, - Intrinsic<[llvm_v16f32_ty], [llvm_v4f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_broadcast_ss_ps_256 : - GCCBuiltin<"__builtin_ia32_broadcastss256_mask">, - Intrinsic<[llvm_v8f32_ty], [llvm_v4f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_broadcast_ss_ps_128 : - GCCBuiltin<"__builtin_ia32_broadcastss128_mask">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_vbroadcast_sd_512 : GCCBuiltin<"__builtin_ia32_vbroadcastsd512">, - Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty], [IntrReadArgMem]>; - def int_x86_avx512_mask_broadcast_sd_pd_512 : - GCCBuiltin<"__builtin_ia32_broadcastsd512">, - Intrinsic<[llvm_v8f64_ty], [llvm_v2f64_ty, llvm_v8f64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_broadcast_sd_pd_256 : - GCCBuiltin<"__builtin_ia32_broadcastsd256_mask">, - Intrinsic<[llvm_v4f64_ty], [llvm_v2f64_ty, llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_pbroadcastb_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastb128_mask">, - Intrinsic<[llvm_v16i8_ty], - [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastb_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastb256_mask">, - Intrinsic<[llvm_v32i8_ty], - [llvm_v16i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastb_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastb512_mask">, - Intrinsic<[llvm_v64i8_ty], - [llvm_v16i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastw_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastw128_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastw_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastw256_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_v8i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastw_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastw512_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_v8i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastd_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastd128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastd_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastd256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v4i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastd_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastd512">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v4i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastq_128 : - GCCBuiltin<"__builtin_ia32_pbroadcastq128_mask">, - Intrinsic<[llvm_v2i64_ty], - [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastq_256 : - GCCBuiltin<"__builtin_ia32_pbroadcastq256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v2i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastq_512 : - GCCBuiltin<"__builtin_ia32_pbroadcastq512">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v2i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_broadcastf32x2_256 : GCCBuiltin<"__builtin_ia32_broadcastf32x2_256_mask">, @@ -5569,11 +5135,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i64_ty], [llvm_v4i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_pbroadcastd_i32_512 : - Intrinsic<[llvm_v16i32_ty], [llvm_i32_ty], [IntrNoMem]>; - - def int_x86_avx512_pbroadcastq_i64_512 : - Intrinsic<[llvm_v8i64_ty], [llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_broadcastmw_512 : GCCBuiltin<"__builtin_ia32_broadcastmw512">, Intrinsic<[llvm_v16i32_ty], [llvm_i16_ty], [IntrNoMem]>; @@ -5668,81 +5229,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } -//Bitwise Ops -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_mask_pand_d_128 : GCCBuiltin<"__builtin_ia32_pandd128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pand_d_256 : GCCBuiltin<"__builtin_ia32_pandd256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, - llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pand_d_512 : GCCBuiltin<"__builtin_ia32_pandd512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, - llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pand_q_128 : GCCBuiltin<"__builtin_ia32_pandq128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pand_q_256 : GCCBuiltin<"__builtin_ia32_pandq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pand_q_512 : GCCBuiltin<"__builtin_ia32_pandq512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, - llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pandn_d_128 : GCCBuiltin<"__builtin_ia32_pandnd128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pandn_d_256 : GCCBuiltin<"__builtin_ia32_pandnd256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, - llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pandn_d_512 : GCCBuiltin<"__builtin_ia32_pandnd512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, - llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pandn_q_128 : GCCBuiltin<"__builtin_ia32_pandnq128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pandn_q_256 : GCCBuiltin<"__builtin_ia32_pandnq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pandn_q_512 : GCCBuiltin<"__builtin_ia32_pandnq512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, - llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_por_d_128 : GCCBuiltin<"__builtin_ia32_pord128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_por_d_256 : GCCBuiltin<"__builtin_ia32_pord256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, - llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_por_d_512 : GCCBuiltin<"__builtin_ia32_pord512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, - llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_por_q_128 : GCCBuiltin<"__builtin_ia32_porq128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_por_q_256 : GCCBuiltin<"__builtin_ia32_porq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_por_q_512 : GCCBuiltin<"__builtin_ia32_porq512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, - llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pxor_d_128 : GCCBuiltin<"__builtin_ia32_pxord128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pxor_d_256 : GCCBuiltin<"__builtin_ia32_pxord256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, - llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pxor_d_512 : GCCBuiltin<"__builtin_ia32_pxord512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, - llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pxor_q_128 : GCCBuiltin<"__builtin_ia32_pxorq128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pxor_q_256 : GCCBuiltin<"__builtin_ia32_pxorq256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, - llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pxor_q_512 : GCCBuiltin<"__builtin_ia32_pxorq512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, - llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; -} + // Arithmetic ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". @@ -5855,96 +5342,96 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_add_ss_round : GCCBuiltin<"__builtin_ia32_addss_round">, + def int_x86_avx512_mask_add_ss_round : GCCBuiltin<"__builtin_ia32_addss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_div_ss_round : GCCBuiltin<"__builtin_ia32_divss_round">, + def int_x86_avx512_mask_div_ss_round : GCCBuiltin<"__builtin_ia32_divss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_mul_ss_round : GCCBuiltin<"__builtin_ia32_mulss_round">, + def int_x86_avx512_mask_mul_ss_round : GCCBuiltin<"__builtin_ia32_mulss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_sub_ss_round : GCCBuiltin<"__builtin_ia32_subss_round">, + def int_x86_avx512_mask_sub_ss_round : GCCBuiltin<"__builtin_ia32_subss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_max_ss_round : GCCBuiltin<"__builtin_ia32_maxss_round">, + def int_x86_avx512_mask_max_ss_round : GCCBuiltin<"__builtin_ia32_maxss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_min_ss_round : GCCBuiltin<"__builtin_ia32_minss_round">, + def int_x86_avx512_mask_min_ss_round : GCCBuiltin<"__builtin_ia32_minss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_add_sd_round : GCCBuiltin<"__builtin_ia32_addsd_round">, + def int_x86_avx512_mask_add_sd_round : GCCBuiltin<"__builtin_ia32_addsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_div_sd_round : GCCBuiltin<"__builtin_ia32_divsd_round">, + def int_x86_avx512_mask_div_sd_round : GCCBuiltin<"__builtin_ia32_divsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_mul_sd_round : GCCBuiltin<"__builtin_ia32_mulsd_round">, + def int_x86_avx512_mask_mul_sd_round : GCCBuiltin<"__builtin_ia32_mulsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_sub_sd_round : GCCBuiltin<"__builtin_ia32_subsd_round">, + def int_x86_avx512_mask_sub_sd_round : GCCBuiltin<"__builtin_ia32_subsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_max_sd_round : GCCBuiltin<"__builtin_ia32_maxsd_round">, + def int_x86_avx512_mask_max_sd_round : GCCBuiltin<"__builtin_ia32_maxsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_min_sd_round : GCCBuiltin<"__builtin_ia32_minsd_round">, + def int_x86_avx512_mask_min_sd_round : GCCBuiltin<"__builtin_ia32_minsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_round">, + def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_round">, + def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess128_round">, + def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess128_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd128_round">, + def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd128_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_reduce_ss : GCCBuiltin<"__builtin_ia32_reducess">, + def int_x86_avx512_mask_reduce_ss : GCCBuiltin<"__builtin_ia32_reducess_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_reduce_sd : GCCBuiltin<"__builtin_ia32_reducesd">, + def int_x86_avx512_mask_reduce_sd : GCCBuiltin<"__builtin_ia32_reducesd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_scalef_sd : GCCBuiltin<"__builtin_ia32_scalefsd_round">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + def int_x86_avx512_mask_scalef_sd : GCCBuiltin<"__builtin_ia32_scalefsd_round_mask">, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_scalef_ss : GCCBuiltin<"__builtin_ia32_scalefss_round">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + def int_x86_avx512_mask_scalef_ss : GCCBuiltin<"__builtin_ia32_scalefss_round_mask">, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_scalef_pd_128 : GCCBuiltin<"__builtin_ia32_scalefpd128_mask">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, + Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_scalef_pd_256 : GCCBuiltin<"__builtin_ia32_scalefpd256_mask">, - Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, + Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty],[IntrNoMem]>; def int_x86_avx512_mask_scalef_pd_512 : GCCBuiltin<"__builtin_ia32_scalefpd512_mask">, - Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_scalef_ps_128 : GCCBuiltin<"__builtin_ia32_scalefps128_mask">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, + Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_scalef_ps_256 : GCCBuiltin<"__builtin_ia32_scalefps256_mask">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, + Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_scalef_ps_512 : GCCBuiltin<"__builtin_ia32_scalefps512_mask">, - Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_sqrt_ss : GCCBuiltin<"__builtin_ia32_sqrtss_round">, + def int_x86_avx512_mask_sqrt_ss : GCCBuiltin<"__builtin_ia32_sqrtss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_sqrt_sd : GCCBuiltin<"__builtin_ia32_sqrtsd_round">, + def int_x86_avx512_mask_sqrt_sd : GCCBuiltin<"__builtin_ia32_sqrtsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -5966,6 +5453,86 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_mask_sqrt_ps_512 : GCCBuiltin<"__builtin_ia32_sqrtps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_pd_128 : + GCCBuiltin<"__builtin_ia32_fixupimmpd128_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_pd_128 : + GCCBuiltin<"__builtin_ia32_fixupimmpd128_maskz">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_pd_256 : + GCCBuiltin<"__builtin_ia32_fixupimmpd256_mask">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_pd_256 : + GCCBuiltin<"__builtin_ia32_fixupimmpd256_maskz">, + Intrinsic<[llvm_v4f64_ty], + [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_pd_512 : + GCCBuiltin<"__builtin_ia32_fixupimmpd512_mask">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_pd_512 : + GCCBuiltin<"__builtin_ia32_fixupimmpd512_maskz">, + Intrinsic<[llvm_v8f64_ty], + [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_ps_128 : + GCCBuiltin<"__builtin_ia32_fixupimmps128_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_ps_128 : + GCCBuiltin<"__builtin_ia32_fixupimmps128_maskz">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_ps_256 : + GCCBuiltin<"__builtin_ia32_fixupimmps256_mask">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_ps_256 : + GCCBuiltin<"__builtin_ia32_fixupimmps256_maskz">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], + [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_ps_512 : + GCCBuiltin<"__builtin_ia32_fixupimmps512_mask">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_ps_512 : + GCCBuiltin<"__builtin_ia32_fixupimmps512_maskz">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty, + llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_sd : + GCCBuiltin<"__builtin_ia32_fixupimmsd_mask">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_sd : + GCCBuiltin<"__builtin_ia32_fixupimmsd_maskz">, + Intrinsic<[llvm_v2f64_ty], + [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_mask_fixupimm_ss : + GCCBuiltin<"__builtin_ia32_fixupimmss_mask">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; + def int_x86_avx512_maskz_fixupimm_ss : + GCCBuiltin<"__builtin_ia32_fixupimmss_maskz">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty, + llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_getexp_pd_128 : GCCBuiltin<"__builtin_ia32_getexppd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; @@ -5985,10 +5552,10 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_getexp_ss : GCCBuiltin<"__builtin_ia32_getexpss_mask">, + def int_x86_avx512_mask_getexp_ss : GCCBuiltin<"__builtin_ia32_getexpss128_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_getexp_sd : GCCBuiltin<"__builtin_ia32_getexpsd_mask">, + def int_x86_avx512_mask_getexp_sd : GCCBuiltin<"__builtin_ia32_getexpsd128_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -6029,21 +5596,21 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [IntrNoMem]>; def int_x86_avx512_mask_getmant_ss : - GCCBuiltin<"__builtin_ia32_getmantss_round">, + GCCBuiltin<"__builtin_ia32_getmantss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_getmant_sd : - GCCBuiltin<"__builtin_ia32_getmantsd_round">, + GCCBuiltin<"__builtin_ia32_getmantsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss">, + def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt14_sd : GCCBuiltin<"__builtin_ia32_rsqrt14sd">, + def int_x86_avx512_rsqrt14_sd : GCCBuiltin<"__builtin_ia32_rsqrt14sd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; @@ -6065,10 +5632,10 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_rsqrt14_ps_512 : GCCBuiltin<"__builtin_ia32_rsqrt14ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_ss : GCCBuiltin<"__builtin_ia32_rcp14ss">, + def int_x86_avx512_rcp14_ss : GCCBuiltin<"__builtin_ia32_rcp14ss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_rcp14_sd : GCCBuiltin<"__builtin_ia32_rcp14sd">, + def int_x86_avx512_rcp14_sd : GCCBuiltin<"__builtin_ia32_rcp14sd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; @@ -6104,11 +5671,11 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_round">, + def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rcp28_sd : GCCBuiltin<"__builtin_ia32_rcp28sd_round">, + def int_x86_avx512_rcp28_sd : GCCBuiltin<"__builtin_ia32_rcp28sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -6120,11 +5687,11 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt28_ss : GCCBuiltin<"__builtin_ia32_rsqrt28ss_round">, + def int_x86_avx512_rsqrt28_ss : GCCBuiltin<"__builtin_ia32_rsqrt28ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_rsqrt28_sd : GCCBuiltin<"__builtin_ia32_rsqrt28sd_round">, + def int_x86_avx512_rsqrt28_sd : GCCBuiltin<"__builtin_ia32_rsqrt28sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -6188,7 +5755,7 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_or_ps_512 : GCCBuiltin<"__builtin_ia32_orps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; - + def int_x86_avx512_mask_xor_pd_128 : GCCBuiltin<"__builtin_ia32_xorpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; @@ -6206,7 +5773,7 @@ let TargetPrefix = "x86" in { llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_xor_ps_512 : GCCBuiltin<"__builtin_ia32_xorps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_v16f32_ty, llvm_i16_ty], [IntrNoMem]>; } // Integer arithmetic ops let TargetPrefix = "x86" in { @@ -6406,34 +5973,34 @@ let TargetPrefix = "x86" in { Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmulhu_w_128 : GCCBuiltin<"__builtin_ia32_pmulhuw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmulhu_w_256 : GCCBuiltin<"__builtin_ia32_pmulhuw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmulh_w_128 : GCCBuiltin<"__builtin_ia32_pmulhw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmulh_w_256 : GCCBuiltin<"__builtin_ia32_pmulhw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pavg_b_512 : GCCBuiltin<"__builtin_ia32_pavgb512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>; def int_x86_avx512_mask_pavg_w_512 : GCCBuiltin<"__builtin_ia32_pavgw512_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pavg_b_128 : GCCBuiltin<"__builtin_ia32_pavgb128_mask">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pavg_b_256 : GCCBuiltin<"__builtin_ia32_pavgb256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; def int_x86_avx512_mask_pavg_w_128 : GCCBuiltin<"__builtin_ia32_pavgw128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_pavg_w_256 : GCCBuiltin<"__builtin_ia32_pavgw256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_pmaddw_d_128 : GCCBuiltin<"__builtin_ia32_pmaddwd128_mask">, @@ -6490,293 +6057,293 @@ let TargetPrefix = "x86" in { def int_x86_avx512_gather_dpd_512 : GCCBuiltin<"__builtin_ia32_gathersiv8df">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_gather_dps_512 : GCCBuiltin<"__builtin_ia32_gathersiv16sf">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_gather_qpd_512 : GCCBuiltin<"__builtin_ia32_gatherdiv8df">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_gather_qps_512 : GCCBuiltin<"__builtin_ia32_gatherdiv16sf">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_gather_dpq_512 : GCCBuiltin<"__builtin_ia32_gathersiv8di">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_gather_dpi_512 : GCCBuiltin<"__builtin_ia32_gathersiv16si">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_gather_qpq_512 : GCCBuiltin<"__builtin_ia32_gatherdiv8di">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_gather_qpi_512 : GCCBuiltin<"__builtin_ia32_gatherdiv16si">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div2_df : + def int_x86_avx512_gather3div2_df : GCCBuiltin<"__builtin_ia32_gather3div2df">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div2_di : + def int_x86_avx512_gather3div2_di : GCCBuiltin<"__builtin_ia32_gather3div2di">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div4_df : + def int_x86_avx512_gather3div4_df : GCCBuiltin<"__builtin_ia32_gather3div4df">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div4_di : + def int_x86_avx512_gather3div4_di : GCCBuiltin<"__builtin_ia32_gather3div4di">, Intrinsic<[llvm_v8i32_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div4_sf : + def int_x86_avx512_gather3div4_sf : GCCBuiltin<"__builtin_ia32_gather3div4sf">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div4_si : + def int_x86_avx512_gather3div4_si : GCCBuiltin<"__builtin_ia32_gather3div4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div8_sf : + def int_x86_avx512_gather3div8_sf : GCCBuiltin<"__builtin_ia32_gather3div8sf">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3div8_si : + def int_x86_avx512_gather3div8_si : GCCBuiltin<"__builtin_ia32_gather3div8si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv2_df : + def int_x86_avx512_gather3siv2_df : GCCBuiltin<"__builtin_ia32_gather3siv2df">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv2_di : + def int_x86_avx512_gather3siv2_di : GCCBuiltin<"__builtin_ia32_gather3siv2di">, Intrinsic<[llvm_v4i32_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv4_df : + def int_x86_avx512_gather3siv4_df : GCCBuiltin<"__builtin_ia32_gather3siv4df">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv4_di : + def int_x86_avx512_gather3siv4_di : GCCBuiltin<"__builtin_ia32_gather3siv4di">, Intrinsic<[llvm_v8i32_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv4_sf : + def int_x86_avx512_gather3siv4_sf : GCCBuiltin<"__builtin_ia32_gather3siv4sf">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv4_si : + def int_x86_avx512_gather3siv4_si : GCCBuiltin<"__builtin_ia32_gather3siv4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv8_sf : + def int_x86_avx512_gather3siv8_sf : GCCBuiltin<"__builtin_ia32_gather3siv8sf">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather3siv8_si : + def int_x86_avx512_gather3siv8_si : GCCBuiltin<"__builtin_ia32_gather3siv8si">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadArgMem]>; + [IntrReadMem, IntrArgMemOnly]>; // scatter def int_x86_avx512_scatter_dpd_512 : GCCBuiltin<"__builtin_ia32_scattersiv8df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_scatter_dps_512 : GCCBuiltin<"__builtin_ia32_scattersiv16sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_v16f32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_scatter_qpd_512 : GCCBuiltin<"__builtin_ia32_scatterdiv8df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_scatter_qps_512 : GCCBuiltin<"__builtin_ia32_scatterdiv16sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8f32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_scatter_dpq_512 : GCCBuiltin<"__builtin_ia32_scattersiv8di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8i64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_scatter_dpi_512 : GCCBuiltin<"__builtin_ia32_scattersiv16si">, Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_scatter_qpq_512 : GCCBuiltin<"__builtin_ia32_scatterdiv8di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_scatter_qpi_512 : GCCBuiltin<"__builtin_ia32_scatterdiv16si">, - Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8i32_ty, + Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv2_df : + def int_x86_avx512_scatterdiv2_df : GCCBuiltin<"__builtin_ia32_scatterdiv2df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv2_di : + def int_x86_avx512_scatterdiv2_di : GCCBuiltin<"__builtin_ia32_scatterdiv2di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv4_df : + def int_x86_avx512_scatterdiv4_df : GCCBuiltin<"__builtin_ia32_scatterdiv4df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv4_di : + def int_x86_avx512_scatterdiv4_di : GCCBuiltin<"__builtin_ia32_scatterdiv4di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv4_sf : + def int_x86_avx512_scatterdiv4_sf : GCCBuiltin<"__builtin_ia32_scatterdiv4sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv4_si : + def int_x86_avx512_scatterdiv4_si : GCCBuiltin<"__builtin_ia32_scatterdiv4si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv8_sf : + def int_x86_avx512_scatterdiv8_sf : GCCBuiltin<"__builtin_ia32_scatterdiv8sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scatterdiv8_si : + def int_x86_avx512_scatterdiv8_si : GCCBuiltin<"__builtin_ia32_scatterdiv8si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv2_df : + def int_x86_avx512_scattersiv2_df : GCCBuiltin<"__builtin_ia32_scattersiv2df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv2_di : + def int_x86_avx512_scattersiv2_di : GCCBuiltin<"__builtin_ia32_scattersiv2di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv4_df : + def int_x86_avx512_scattersiv4_df : GCCBuiltin<"__builtin_ia32_scattersiv4df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv4_di : + def int_x86_avx512_scattersiv4_di : GCCBuiltin<"__builtin_ia32_scattersiv4di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv4_sf : + def int_x86_avx512_scattersiv4_sf : GCCBuiltin<"__builtin_ia32_scattersiv4sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv4_si : + def int_x86_avx512_scattersiv4_si : GCCBuiltin<"__builtin_ia32_scattersiv4si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv8_sf : + def int_x86_avx512_scattersiv8_sf : GCCBuiltin<"__builtin_ia32_scattersiv8sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; - def int_x86_avx512_scattersiv8_si : + def int_x86_avx512_scattersiv8_si : GCCBuiltin<"__builtin_ia32_scattersiv8si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; // gather prefetch def int_x86_avx512_gatherpf_dpd_512 : GCCBuiltin<"__builtin_ia32_gatherpfdpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_gatherpf_dps_512 : GCCBuiltin<"__builtin_ia32_gatherpfdps">, Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_gatherpf_qpd_512 : GCCBuiltin<"__builtin_ia32_gatherpfqpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_gatherpf_qps_512 : GCCBuiltin<"__builtin_ia32_gatherpfqps">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; // scatter prefetch def int_x86_avx512_scatterpf_dpd_512 : GCCBuiltin<"__builtin_ia32_scatterpfdpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_scatterpf_dps_512 : GCCBuiltin<"__builtin_ia32_scatterpfdps">, Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_scatterpf_qpd_512 : GCCBuiltin<"__builtin_ia32_scatterpfqpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; def int_x86_avx512_scatterpf_qps_512 : GCCBuiltin<"__builtin_ia32_scatterpfqps">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrReadWriteArgMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; } // AVX-512 conflict detection instruction @@ -6815,117 +6382,32 @@ let TargetPrefix = "x86" in { [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_d_128 : - GCCBuiltin<"__builtin_ia32_vplzcntd_128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_d_256 : - GCCBuiltin<"__builtin_ia32_vplzcntd_256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_d_512 : - GCCBuiltin<"__builtin_ia32_vplzcntd_512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_q_128 : - GCCBuiltin<"__builtin_ia32_vplzcntq_128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_q_256 : - GCCBuiltin<"__builtin_ia32_vplzcntq_256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_lzcnt_q_512 : - GCCBuiltin<"__builtin_ia32_vplzcntq_512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>; } -// Vector blend -let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_mask_blend_ps_512 : GCCBuiltin<"__builtin_ia32_blendmps_512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_ps_256 : GCCBuiltin<"__builtin_ia32_blendmps_256_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_ps_128 : GCCBuiltin<"__builtin_ia32_blendmps_128_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_pd_512 : GCCBuiltin<"__builtin_ia32_blendmpd_512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_pd_256 : GCCBuiltin<"__builtin_ia32_blendmpd_256_mask">, - Intrinsic<[llvm_v4f64_ty], - [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_pd_128 : GCCBuiltin<"__builtin_ia32_blendmpd_128_mask">, - Intrinsic<[llvm_v2f64_ty], - [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_blend_d_512 : GCCBuiltin<"__builtin_ia32_blendmd_512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_q_512 : GCCBuiltin<"__builtin_ia32_blendmq_512_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_d_256 : GCCBuiltin<"__builtin_ia32_blendmd_256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_q_256 : GCCBuiltin<"__builtin_ia32_blendmq_256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_d_128 : GCCBuiltin<"__builtin_ia32_blendmd_128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_q_128 : GCCBuiltin<"__builtin_ia32_blendmq_128_mask">, - Intrinsic<[llvm_v2i64_ty], - [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_blend_w_512 : GCCBuiltin<"__builtin_ia32_blendmw_512_mask">, - Intrinsic<[llvm_v32i16_ty], - [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_w_256 : GCCBuiltin<"__builtin_ia32_blendmw_256_mask">, - Intrinsic<[llvm_v16i16_ty], - [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_w_128 : GCCBuiltin<"__builtin_ia32_blendmw_128_mask">, - Intrinsic<[llvm_v8i16_ty], - [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_b_512 : GCCBuiltin<"__builtin_ia32_blendmb_512_mask">, - Intrinsic<[llvm_v64i8_ty], - [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_b_256 : GCCBuiltin<"__builtin_ia32_blendmb_256_mask">, - Intrinsic<[llvm_v32i8_ty], - [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_blend_b_128 : GCCBuiltin<"__builtin_ia32_blendmb_128_mask">, - Intrinsic<[llvm_v16i8_ty], - [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], - [IntrNoMem]>; - -} - let TargetPrefix = "x86" in { def int_x86_avx512_mask_valign_q_512 : GCCBuiltin<"__builtin_ia32_alignq512_mask">, @@ -6948,13 +6430,13 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_valign_d_256 : GCCBuiltin<"__builtin_ia32_alignd256_mask">, Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v8i32_ty, + [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_valign_q_128 : GCCBuiltin<"__builtin_ia32_alignq128_mask">, Intrinsic<[llvm_v2i64_ty], - [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_v2i64_ty, + [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_valign_d_128 : @@ -6962,24 +6444,6 @@ let TargetPrefix = "x86" in { Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_palignr_128 : - GCCBuiltin<"__builtin_ia32_palignr128_mask">, - Intrinsic<[llvm_v16i8_ty], - [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, - llvm_i16_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_palignr_256 : - GCCBuiltin<"__builtin_ia32_palignr256_mask">, - Intrinsic<[llvm_v32i8_ty], - [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty, llvm_v32i8_ty, - llvm_i32_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_palignr_512 : - GCCBuiltin<"__builtin_ia32_palignr512_mask">, - Intrinsic<[llvm_v64i8_ty], - [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty, llvm_v64i8_ty, - llvm_i64_ty], [IntrNoMem]>; } // Compares @@ -6991,161 +6455,84 @@ let TargetPrefix = "x86" in { def int_x86_avx512_vcomi_ss : GCCBuiltin<"__builtin_ia32_vcomiss">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_b_512 : GCCBuiltin<"__builtin_ia32_pcmpeqb512_mask">, - Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_w_512 : GCCBuiltin<"__builtin_ia32_pcmpeqw512_mask">, - Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_d_512 : GCCBuiltin<"__builtin_ia32_pcmpeqd512_mask">, - Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_q_512 : GCCBuiltin<"__builtin_ia32_pcmpeqq512_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_b_512: GCCBuiltin<"__builtin_ia32_pcmpgtb512_mask">, - Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_w_512: GCCBuiltin<"__builtin_ia32_pcmpgtw512_mask">, - Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_d_512: GCCBuiltin<"__builtin_ia32_pcmpgtd512_mask">, - Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_q_512: GCCBuiltin<"__builtin_ia32_pcmpgtq512_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cmp_b_512: GCCBuiltin<"__builtin_ia32_cmpb512_mask">, + def int_x86_avx512_mask_cmp_b_512: Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_w_512: GCCBuiltin<"__builtin_ia32_cmpw512_mask">, + def int_x86_avx512_mask_cmp_w_512: Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_d_512: GCCBuiltin<"__builtin_ia32_cmpd512_mask">, + def int_x86_avx512_mask_cmp_d_512: Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_i16_ty], [IntrNoMem ]>; - def int_x86_avx512_mask_cmp_q_512: GCCBuiltin<"__builtin_ia32_cmpq512_mask">, + def int_x86_avx512_mask_cmp_q_512: Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_b_512: GCCBuiltin<"__builtin_ia32_ucmpb512_mask">, + def int_x86_avx512_mask_ucmp_b_512: Intrinsic<[llvm_i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_w_512: GCCBuiltin<"__builtin_ia32_ucmpw512_mask">, + def int_x86_avx512_mask_ucmp_w_512: Intrinsic<[llvm_i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_d_512: GCCBuiltin<"__builtin_ia32_ucmpd512_mask">, + def int_x86_avx512_mask_ucmp_d_512: Intrinsic<[llvm_i16_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_q_512: GCCBuiltin<"__builtin_ia32_ucmpq512_mask">, + def int_x86_avx512_mask_ucmp_q_512: Intrinsic<[llvm_i8_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; // 256-bit - def int_x86_avx512_mask_pcmpeq_b_256 : GCCBuiltin<"__builtin_ia32_pcmpeqb256_mask">, - Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_w_256 : GCCBuiltin<"__builtin_ia32_pcmpeqw256_mask">, - Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_d_256 : GCCBuiltin<"__builtin_ia32_pcmpeqd256_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_q_256 : GCCBuiltin<"__builtin_ia32_pcmpeqq256_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pcmpgt_b_256: GCCBuiltin<"__builtin_ia32_pcmpgtb256_mask">, - Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_w_256: GCCBuiltin<"__builtin_ia32_pcmpgtw256_mask">, - Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_d_256: GCCBuiltin<"__builtin_ia32_pcmpgtd256_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_q_256: GCCBuiltin<"__builtin_ia32_pcmpgtq256_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cmp_b_256: GCCBuiltin<"__builtin_ia32_cmpb256_mask">, + def int_x86_avx512_mask_cmp_b_256: Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_w_256: GCCBuiltin<"__builtin_ia32_cmpw256_mask">, + def int_x86_avx512_mask_cmp_w_256: Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_d_256: GCCBuiltin<"__builtin_ia32_cmpd256_mask">, + def int_x86_avx512_mask_cmp_d_256: Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_q_256: GCCBuiltin<"__builtin_ia32_cmpq256_mask">, + def int_x86_avx512_mask_cmp_q_256: Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_b_256: GCCBuiltin<"__builtin_ia32_ucmpb256_mask">, + def int_x86_avx512_mask_ucmp_b_256: Intrinsic<[llvm_i32_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_w_256: GCCBuiltin<"__builtin_ia32_ucmpw256_mask">, + def int_x86_avx512_mask_ucmp_w_256: Intrinsic<[llvm_i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_d_256: GCCBuiltin<"__builtin_ia32_ucmpd256_mask">, + def int_x86_avx512_mask_ucmp_d_256: Intrinsic<[llvm_i8_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_q_256: GCCBuiltin<"__builtin_ia32_ucmpq256_mask">, + def int_x86_avx512_mask_ucmp_q_256: Intrinsic<[llvm_i8_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; // 128-bit - def int_x86_avx512_mask_pcmpeq_b_128 : GCCBuiltin<"__builtin_ia32_pcmpeqb128_mask">, - Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_w_128 : GCCBuiltin<"__builtin_ia32_pcmpeqw128_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_d_128 : GCCBuiltin<"__builtin_ia32_pcmpeqd128_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpeq_q_128 : GCCBuiltin<"__builtin_ia32_pcmpeqq128_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_pcmpgt_b_128: GCCBuiltin<"__builtin_ia32_pcmpgtb128_mask">, - Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_w_128: GCCBuiltin<"__builtin_ia32_pcmpgtw128_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_d_128: GCCBuiltin<"__builtin_ia32_pcmpgtd128_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_pcmpgt_q_128: GCCBuiltin<"__builtin_ia32_pcmpgtq128_mask">, - Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cmp_b_128: GCCBuiltin<"__builtin_ia32_cmpb128_mask">, + def int_x86_avx512_mask_cmp_b_128: Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_w_128: GCCBuiltin<"__builtin_ia32_cmpw128_mask">, + def int_x86_avx512_mask_cmp_w_128: Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_d_128: GCCBuiltin<"__builtin_ia32_cmpd128_mask">, + def int_x86_avx512_mask_cmp_d_128: Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cmp_q_128: GCCBuiltin<"__builtin_ia32_cmpq128_mask">, + def int_x86_avx512_mask_cmp_q_128: Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_b_128: GCCBuiltin<"__builtin_ia32_ucmpb128_mask">, + def int_x86_avx512_mask_ucmp_b_128: Intrinsic<[llvm_i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_w_128: GCCBuiltin<"__builtin_ia32_ucmpw128_mask">, + def int_x86_avx512_mask_ucmp_w_128: Intrinsic<[llvm_i8_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_d_128: GCCBuiltin<"__builtin_ia32_ucmpd128_mask">, + def int_x86_avx512_mask_ucmp_d_128: Intrinsic<[llvm_i8_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_ucmp_q_128: GCCBuiltin<"__builtin_ia32_ucmpq128_mask">, + def int_x86_avx512_mask_ucmp_q_128: Intrinsic<[llvm_i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], [IntrNoMem]>; } @@ -7180,27 +6567,27 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_compress_store_ps_512 : GCCBuiltin<"__builtin_ia32_compressstoresf512_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16f32_ty, - llvm_i16_ty], [IntrReadWriteArgMem]>; + llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_pd_512 : GCCBuiltin<"__builtin_ia32_compressstoredf512_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8f64_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_ps_256 : GCCBuiltin<"__builtin_ia32_compressstoresf256_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8f32_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_pd_256 : GCCBuiltin<"__builtin_ia32_compressstoredf256_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4f64_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_ps_128 : GCCBuiltin<"__builtin_ia32_compressstoresf128_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4f32_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_pd_128 : GCCBuiltin<"__builtin_ia32_compressstoredf128_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2f64_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_d_512 : GCCBuiltin<"__builtin_ia32_compresssi512_mask">, @@ -7230,27 +6617,27 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_compress_store_d_512 : GCCBuiltin<"__builtin_ia32_compressstoresi512_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, - llvm_i16_ty], [IntrReadWriteArgMem]>; + llvm_i16_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_q_512 : GCCBuiltin<"__builtin_ia32_compressstoredi512_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_d_256 : GCCBuiltin<"__builtin_ia32_compressstoresi256_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_q_256 : GCCBuiltin<"__builtin_ia32_compressstoredi256_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_d_128 : GCCBuiltin<"__builtin_ia32_compressstoresi128_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; def int_x86_avx512_mask_compress_store_q_128 : GCCBuiltin<"__builtin_ia32_compressstoredi128_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, - llvm_i8_ty], [IntrReadWriteArgMem]>; + llvm_i8_ty], [IntrArgMemOnly]>; // expand def int_x86_avx512_mask_expand_ps_512 : @@ -7281,27 +6668,27 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_expand_load_ps_512 : GCCBuiltin<"__builtin_ia32_expandloadsf512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_ptr_ty, llvm_v16f32_ty, - llvm_i16_ty], [IntrReadArgMem]>; + llvm_i16_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_pd_512 : GCCBuiltin<"__builtin_ia32_expandloaddf512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_ptr_ty, llvm_v8f64_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_ps_256 : GCCBuiltin<"__builtin_ia32_expandloadsf256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_ptr_ty, llvm_v8f32_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_pd_256 : GCCBuiltin<"__builtin_ia32_expandloaddf256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_ptr_ty, llvm_v4f64_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_ps_128 : GCCBuiltin<"__builtin_ia32_expandloadsf128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_ptr_ty, llvm_v4f32_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_pd_128 : GCCBuiltin<"__builtin_ia32_expandloaddf128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_ptr_ty, llvm_v2f64_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_d_512 : GCCBuiltin<"__builtin_ia32_expandsi512_mask">, @@ -7331,27 +6718,27 @@ let TargetPrefix = "x86" in { def int_x86_avx512_mask_expand_load_d_512 : GCCBuiltin<"__builtin_ia32_expandloadsi512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_ptr_ty, llvm_v16i32_ty, - llvm_i16_ty], [IntrReadArgMem]>; + llvm_i16_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_q_512 : GCCBuiltin<"__builtin_ia32_expandloaddi512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_ptr_ty, llvm_v8i64_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_d_256 : GCCBuiltin<"__builtin_ia32_expandloadsi256_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_v8i32_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_q_256 : GCCBuiltin<"__builtin_ia32_expandloaddi256_mask">, Intrinsic<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_v4i64_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_d_128 : GCCBuiltin<"__builtin_ia32_expandloadsi128_mask">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_v4i32_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; def int_x86_avx512_mask_expand_load_q_128 : GCCBuiltin<"__builtin_ia32_expandloaddi128_mask">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_v2i64_ty, - llvm_i8_ty], [IntrReadArgMem]>; + llvm_i8_ty], [IntrReadMem, IntrArgMemOnly]>; } @@ -7366,7 +6753,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qb_128 : GCCBuiltin<"__builtin_ia32_pmovsqb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7376,7 +6763,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qb_128 : GCCBuiltin<"__builtin_ia32_pmovusqb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7386,7 +6773,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qb_256 : GCCBuiltin<"__builtin_ia32_pmovqb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7396,7 +6783,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qb_256 : GCCBuiltin<"__builtin_ia32_pmovsqb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7406,7 +6793,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qb_256 : GCCBuiltin<"__builtin_ia32_pmovusqb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7416,7 +6803,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qb_512 : GCCBuiltin<"__builtin_ia32_pmovqb512_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7426,7 +6813,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qb_512 : GCCBuiltin<"__builtin_ia32_pmovsqb512_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7436,7 +6823,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qb_512 : GCCBuiltin<"__builtin_ia32_pmovusqb512_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7446,7 +6833,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qw_128 : GCCBuiltin<"__builtin_ia32_pmovqw128_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7456,7 +6843,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qw_128 : GCCBuiltin<"__builtin_ia32_pmovsqw128_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7466,7 +6853,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qw_128 : GCCBuiltin<"__builtin_ia32_pmovusqw128_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7476,7 +6863,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qw_256 : GCCBuiltin<"__builtin_ia32_pmovqw256_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7486,7 +6873,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qw_256 : GCCBuiltin<"__builtin_ia32_pmovsqw256_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7496,7 +6883,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qw_256 : GCCBuiltin<"__builtin_ia32_pmovusqw256_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7506,7 +6893,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qw_512 : GCCBuiltin<"__builtin_ia32_pmovqw512_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7516,7 +6903,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qw_512 : GCCBuiltin<"__builtin_ia32_pmovsqw512_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7526,7 +6913,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qw_512 : GCCBuiltin<"__builtin_ia32_pmovusqw512_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7536,7 +6923,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qd_128 : GCCBuiltin<"__builtin_ia32_pmovqd128_mask">, Intrinsic<[llvm_v4i32_ty], @@ -7546,7 +6933,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqd128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qd_128 : GCCBuiltin<"__builtin_ia32_pmovsqd128_mask">, Intrinsic<[llvm_v4i32_ty], @@ -7556,7 +6943,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqd128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qd_128 : GCCBuiltin<"__builtin_ia32_pmovusqd128_mask">, Intrinsic<[llvm_v4i32_ty], @@ -7566,7 +6953,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqd128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qd_256 : GCCBuiltin<"__builtin_ia32_pmovqd256_mask">, Intrinsic<[llvm_v4i32_ty], @@ -7576,7 +6963,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqd256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qd_256 : GCCBuiltin<"__builtin_ia32_pmovsqd256_mask">, Intrinsic<[llvm_v4i32_ty], @@ -7586,7 +6973,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqd256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qd_256 : GCCBuiltin<"__builtin_ia32_pmovusqd256_mask">, Intrinsic<[llvm_v4i32_ty], @@ -7596,7 +6983,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqd256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_qd_512 : GCCBuiltin<"__builtin_ia32_pmovqd512_mask">, Intrinsic<[llvm_v8i32_ty], @@ -7606,7 +6993,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovqd512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_qd_512 : GCCBuiltin<"__builtin_ia32_pmovsqd512_mask">, Intrinsic<[llvm_v8i32_ty], @@ -7616,7 +7003,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsqd512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_qd_512 : GCCBuiltin<"__builtin_ia32_pmovusqd512_mask">, Intrinsic<[llvm_v8i32_ty], @@ -7626,7 +7013,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusqd512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_db_128 : GCCBuiltin<"__builtin_ia32_pmovdb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7636,7 +7023,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovdb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_db_128 : GCCBuiltin<"__builtin_ia32_pmovsdb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7646,7 +7033,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsdb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_db_128 : GCCBuiltin<"__builtin_ia32_pmovusdb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7656,7 +7043,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusdb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_db_256 : GCCBuiltin<"__builtin_ia32_pmovdb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7666,7 +7053,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovdb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_db_256 : GCCBuiltin<"__builtin_ia32_pmovsdb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7676,7 +7063,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsdb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_db_256 : GCCBuiltin<"__builtin_ia32_pmovusdb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7686,7 +7073,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusdb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_db_512 : GCCBuiltin<"__builtin_ia32_pmovdb512_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7696,7 +7083,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovdb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_db_512 : GCCBuiltin<"__builtin_ia32_pmovsdb512_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7706,7 +7093,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsdb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_db_512 : GCCBuiltin<"__builtin_ia32_pmovusdb512_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7716,7 +7103,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusdb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_dw_128 : GCCBuiltin<"__builtin_ia32_pmovdw128_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7726,7 +7113,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovdw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_dw_128 : GCCBuiltin<"__builtin_ia32_pmovsdw128_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7736,7 +7123,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsdw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_dw_128 : GCCBuiltin<"__builtin_ia32_pmovusdw128_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7746,7 +7133,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusdw128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_dw_256 : GCCBuiltin<"__builtin_ia32_pmovdw256_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7756,7 +7143,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovdw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_dw_256 : GCCBuiltin<"__builtin_ia32_pmovsdw256_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7766,7 +7153,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsdw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_dw_256 : GCCBuiltin<"__builtin_ia32_pmovusdw256_mask">, Intrinsic<[llvm_v8i16_ty], @@ -7776,7 +7163,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusdw256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_dw_512 : GCCBuiltin<"__builtin_ia32_pmovdw512_mask">, Intrinsic<[llvm_v16i16_ty], @@ -7786,7 +7173,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovdw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_dw_512 : GCCBuiltin<"__builtin_ia32_pmovsdw512_mask">, Intrinsic<[llvm_v16i16_ty], @@ -7796,7 +7183,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovsdw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_dw_512 : GCCBuiltin<"__builtin_ia32_pmovusdw512_mask">, Intrinsic<[llvm_v16i16_ty], @@ -7806,7 +7193,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovusdw512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_wb_128 : GCCBuiltin<"__builtin_ia32_pmovwb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7816,7 +7203,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovwb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_wb_128 : GCCBuiltin<"__builtin_ia32_pmovswb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7826,7 +7213,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovswb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_wb_128 : GCCBuiltin<"__builtin_ia32_pmovuswb128_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7836,7 +7223,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovuswb128mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_wb_256 : GCCBuiltin<"__builtin_ia32_pmovwb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7846,7 +7233,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovwb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_wb_256 : GCCBuiltin<"__builtin_ia32_pmovswb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7856,7 +7243,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovswb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_wb_256 : GCCBuiltin<"__builtin_ia32_pmovuswb256_mask">, Intrinsic<[llvm_v16i8_ty], @@ -7866,7 +7253,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovuswb256mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmov_wb_512 : GCCBuiltin<"__builtin_ia32_pmovwb512_mask">, Intrinsic<[llvm_v32i8_ty], @@ -7876,7 +7263,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovwb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovs_wb_512 : GCCBuiltin<"__builtin_ia32_pmovswb512_mask">, Intrinsic<[llvm_v32i8_ty], @@ -7886,7 +7273,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovswb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; def int_x86_avx512_mask_pmovus_wb_512 : GCCBuiltin<"__builtin_ia32_pmovuswb512_mask">, Intrinsic<[llvm_v32i8_ty], @@ -7896,7 +7283,7 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pmovuswb512mem_mask">, Intrinsic<[], [llvm_ptr_ty, llvm_v32i16_ty, llvm_i32_ty], - [IntrReadWriteArgMem]>; + [IntrArgMemOnly]>; } // Bitwise ternary logic @@ -8034,3 +7421,14 @@ let TargetPrefix = "x86" in { def int_x86_sha256msg2 : GCCBuiltin<"__builtin_ia32_sha256msg2">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; } + +//===----------------------------------------------------------------------===// +// Thread synchronization ops with timer. +let TargetPrefix = "x86" in { + def int_x86_monitorx + : GCCBuiltin<"__builtin_ia32_monitorx">, + Intrinsic<[], [ llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty ], []>; + def int_x86_mwaitx + : GCCBuiltin<"__builtin_ia32_mwaitx">, + Intrinsic<[], [ llvm_i32_ty, llvm_i32_ty, llvm_i32_ty ], []>; +} diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h index 56aa3010d925..dbf2b4562332 100644 --- a/include/llvm/IR/LLVMContext.h +++ b/include/llvm/IR/LLVMContext.h @@ -16,7 +16,6 @@ #define LLVM_IR_LLVMCONTEXT_H #include "llvm/Support/CBindingWrapping.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Options.h" namespace llvm { @@ -26,11 +25,15 @@ class StringRef; class Twine; class Instruction; class Module; +class MDString; +class DICompositeType; class SMDiagnostic; class DiagnosticInfo; +enum DiagnosticSeverity : char; template <typename T> class SmallVectorImpl; class Function; class DebugLoc; +class OptBisect; /// This is an important class for using LLVM in a threaded context. It /// (opaquely) owns and manages the core "global" data of LLVM's core @@ -46,24 +49,26 @@ public: // Pinned metadata names, which always have the same value. This is a // compile-time performance optimization, not a correctness optimization. enum { - MD_dbg = 0, // "dbg" - MD_tbaa = 1, // "tbaa" - MD_prof = 2, // "prof" - MD_fpmath = 3, // "fpmath" - MD_range = 4, // "range" - MD_tbaa_struct = 5, // "tbaa.struct" - MD_invariant_load = 6, // "invariant.load" - MD_alias_scope = 7, // "alias.scope" - MD_noalias = 8, // "noalias", - MD_nontemporal = 9, // "nontemporal" + MD_dbg = 0, // "dbg" + MD_tbaa = 1, // "tbaa" + MD_prof = 2, // "prof" + MD_fpmath = 3, // "fpmath" + MD_range = 4, // "range" + MD_tbaa_struct = 5, // "tbaa.struct" + MD_invariant_load = 6, // "invariant.load" + MD_alias_scope = 7, // "alias.scope" + MD_noalias = 8, // "noalias", + MD_nontemporal = 9, // "nontemporal" MD_mem_parallel_loop_access = 10, // "llvm.mem.parallel_loop_access" - MD_nonnull = 11, // "nonnull" - MD_dereferenceable = 12, // "dereferenceable" - MD_dereferenceable_or_null = 13, // "dereferenceable_or_null" - MD_make_implicit = 14, // "make.implicit" - MD_unpredictable = 15, // "unpredictable" - MD_invariant_group = 16, // "invariant.group" - MD_align = 17 // "align" + MD_nonnull = 11, // "nonnull" + MD_dereferenceable = 12, // "dereferenceable" + MD_dereferenceable_or_null = 13, // "dereferenceable_or_null" + MD_make_implicit = 14, // "make.implicit" + MD_unpredictable = 15, // "unpredictable" + MD_invariant_group = 16, // "invariant.group" + MD_align = 17, // "align" + MD_loop = 18, // "llvm.loop" + MD_type = 19, // "type" }; /// Known operand bundle tag IDs, which always have the same value. All @@ -71,8 +76,9 @@ public: /// Additionally, this scheme allows LLVM to efficiently check for specific /// operand bundle tags without comparing strings. enum { - OB_deopt = 0, // "deopt" - OB_funclet = 1, // "funclet" + OB_deopt = 0, // "deopt" + OB_funclet = 1, // "funclet" + OB_gc_transition = 2, // "gc-transition" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. @@ -93,7 +99,6 @@ public: /// tag registered with an LLVMContext has an unique ID. uint32_t getOperandBundleTagID(StringRef Tag) const; - /// Define the GC for a function void setGC(const Function &Fn, std::string GCName); @@ -103,6 +108,21 @@ public: /// Remove the GC for a function void deleteGC(const Function &Fn); + /// Return true if the Context runtime configuration is set to discard all + /// value names. When true, only GlobalValue names will be available in the + /// IR. + bool shouldDiscardValueNames() const; + + /// Set the Context runtime configuration to discard all value name (but + /// GlobalValue). Clients can use this flag to save memory and runtime, + /// especially in release mode. + void setDiscardValueNames(bool Discard); + + /// Whether there is a string map for uniquing debug info + /// identifiers across the context. Off by default. + bool isODRUniquingDebugTypes() const; + void enableDebugTypeODRUniquing(); + void disableDebugTypeODRUniquing(); typedef void (*InlineAsmDiagHandlerTy)(const SMDiagnostic&, void *Context, unsigned LocCookie); @@ -154,6 +174,17 @@ public: /// setDiagnosticContext. void *getDiagnosticContext() const; + /// \brief Return if a code hotness metric should be included in optimization + /// diagnostics. + bool getDiagnosticHotnessRequested() const; + /// \brief Set if a code hotness metric should be included in optimization + /// diagnostics. + void setDiagnosticHotnessRequested(bool Requested); + + /// \brief Get the prefix that should be printed in front of a diagnostic of + /// the given \p Severity + static const char *getDiagnosticMessagePrefix(DiagnosticSeverity Severity); + /// \brief Report a message to the currently installed diagnostic handler. /// /// This function returns, in particular in the case of error reporting @@ -209,6 +240,9 @@ public: return OptionRegistry::instance().template get<ValT, Base, Mem>(); } + /// \brief Access the object which manages optimization bisection for failure + /// analysis. + OptBisect &getOptBisect(); private: LLVMContext(LLVMContext&) = delete; void operator=(LLVMContext&) = delete; @@ -224,10 +258,6 @@ private: friend class Module; }; -/// getGlobalContext - Returns a global context. This is for LLVM clients that -/// only care about operating on a single thread. -extern LLVMContext &getGlobalContext(); - // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLVMContext, LLVMContextRef) diff --git a/include/llvm/IR/LegacyPassManagers.h b/include/llvm/IR/LegacyPassManagers.h index b8e33478d6a9..530fd7166498 100644 --- a/include/llvm/IR/LegacyPassManagers.h +++ b/include/llvm/IR/LegacyPassManagers.h @@ -14,13 +14,11 @@ #ifndef LLVM_IR_LEGACYPASSMANAGERS_H #define LLVM_IR_LEGACYPASSMANAGERS_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Pass.h" -#include <map> #include <vector> //===----------------------------------------------------------------------===// @@ -93,12 +91,13 @@ #include "llvm/Support/PrettyStackTrace.h" namespace llvm { - class Module; - class Pass; - class StringRef; - class Value; - class Timer; - class PMDataManager; +template <typename T> class ArrayRef; +class Module; +class Pass; +class StringRef; +class Value; +class Timer; +class PMDataManager; // enums for debugging strings enum PassDebuggingString { diff --git a/include/llvm/IR/Mangler.h b/include/llvm/IR/Mangler.h index ea2f0c3f09f3..349218e33817 100644 --- a/include/llvm/IR/Mangler.h +++ b/include/llvm/IR/Mangler.h @@ -16,13 +16,13 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/IR/GlobalValue.h" -#include "llvm/Support/raw_ostream.h" namespace llvm { class DataLayout; template <typename T> class SmallVectorImpl; class Twine; +class raw_ostream; class Mangler { /// We need to give global values the same name every time they are mangled. diff --git a/include/llvm/IR/Metadata.def b/include/llvm/IR/Metadata.def index b1d22178e262..607f5ef125c9 100644 --- a/include/llvm/IR/Metadata.def +++ b/include/llvm/IR/Metadata.def @@ -77,6 +77,7 @@ HANDLE_METADATA_LEAF(MDString) HANDLE_METADATA_BRANCH(ValueAsMetadata) HANDLE_METADATA_LEAF(ConstantAsMetadata) HANDLE_METADATA_LEAF(LocalAsMetadata) +HANDLE_METADATA_LEAF(DistinctMDOperandPlaceholder) HANDLE_MDNODE_BRANCH(MDNode) HANDLE_MDNODE_LEAF_UNIQUABLE(MDTuple) HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DILocation) diff --git a/include/llvm/IR/Metadata.h b/include/llvm/IR/Metadata.h index df8ce354bb7f..91f43d342d27 100644 --- a/include/llvm/IR/Metadata.h +++ b/include/llvm/IR/Metadata.h @@ -19,6 +19,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Constant.h" @@ -51,7 +52,7 @@ protected: enum StorageType { Uniqued, Distinct, Temporary }; /// \brief Storage flag for non-uniqued, otherwise unowned, metadata. - unsigned Storage : 2; + unsigned char Storage; // TODO: expose remaining bits to subclasses. unsigned short SubclassData16; @@ -59,39 +60,14 @@ protected: public: enum MetadataKind { - MDTupleKind, - DILocationKind, - GenericDINodeKind, - DISubrangeKind, - DIEnumeratorKind, - DIBasicTypeKind, - DIDerivedTypeKind, - DICompositeTypeKind, - DISubroutineTypeKind, - DIFileKind, - DICompileUnitKind, - DISubprogramKind, - DILexicalBlockKind, - DILexicalBlockFileKind, - DINamespaceKind, - DIModuleKind, - DITemplateTypeParameterKind, - DITemplateValueParameterKind, - DIGlobalVariableKind, - DILocalVariableKind, - DIExpressionKind, - DIObjCPropertyKind, - DIImportedEntityKind, - ConstantAsMetadataKind, - LocalAsMetadataKind, - MDStringKind, - DIMacroKind, - DIMacroFileKind +#define HANDLE_METADATA_LEAF(CLASS) CLASS##Kind, +#include "llvm/IR/Metadata.def" }; protected: Metadata(unsigned ID, StorageType Storage) : SubclassID(ID), Storage(Storage), SubclassData16(0), SubclassData32(0) { + static_assert(sizeof(*this) == 8, "Metdata fields poorly packed"); } ~Metadata() = default; @@ -283,20 +259,14 @@ private: LLVMContext &Context; uint64_t NextIndex; SmallDenseMap<void *, std::pair<OwnerTy, uint64_t>, 4> UseMap; - /// Flag that can be set to false if this metadata should not be - /// RAUW'ed, e.g. if it is used as the key of a map. - bool CanReplace; public: ReplaceableMetadataImpl(LLVMContext &Context) - : Context(Context), NextIndex(0), CanReplace(true) {} + : Context(Context), NextIndex(0) {} ~ReplaceableMetadataImpl() { assert(UseMap.empty() && "Cannot destroy in-use replaceable metadata"); } - /// Set the CanReplace flag to the given value. - void setCanReplace(bool Replaceable) { CanReplace = Replaceable; } - LLVMContext &getContext() const { return Context; } /// \brief Replace all uses of this with MD. @@ -316,7 +286,19 @@ private: void dropRef(void *Ref); void moveRef(void *Ref, void *New, const Metadata &MD); - static ReplaceableMetadataImpl *get(Metadata &MD); + /// Lazily construct RAUW support on MD. + /// + /// If this is an unresolved MDNode, RAUW support will be created on-demand. + /// ValueAsMetadata always has RAUW support. + static ReplaceableMetadataImpl *getOrCreate(Metadata &MD); + + /// Get RAUW support on MD, if it exists. + static ReplaceableMetadataImpl *getIfExists(Metadata &MD); + + /// Check whether this node will support RAUW. + /// + /// Returns \c true unless getOrCreate() would return null. + static bool isReplaceable(const Metadata &MD); }; /// \brief Value wrapper in the Metadata hierarchy. @@ -592,7 +574,6 @@ class MDString : public Metadata { StringMapEntry<MDString> *Entry; MDString() : Metadata(MDStringKind, Uniqued), Entry(nullptr) {} - MDString(MDString &&) : Metadata(MDStringKind, Uniqued) {} public: static MDString *get(LLVMContext &Context, StringRef Str); @@ -767,6 +748,13 @@ public: return nullptr; } + /// Ensure that this has RAUW support, and then return it. + ReplaceableMetadataImpl *getOrCreateReplaceableUses() { + if (!hasReplaceableUses()) + makeReplaceable(llvm::make_unique<ReplaceableMetadataImpl>(getContext())); + return getReplaceableUses(); + } + /// \brief Assign RAUW support to this. /// /// Make this replaceable, taking ownership of \c ReplaceableUses (which must @@ -828,9 +816,9 @@ class MDNode : public Metadata { unsigned NumOperands; unsigned NumUnresolved; -protected: ContextAndReplaceableUses Context; +protected: void *operator new(size_t Size, unsigned NumOps); void operator delete(void *Mem); @@ -892,7 +880,7 @@ public: /// As forward declarations are resolved, their containers should get /// resolved automatically. However, if this (or one of its operands) is /// involved in a cycle, \a resolveCycles() needs to be called explicitly. - bool isResolved() const { return !Context.hasReplaceableUses(); } + bool isResolved() const { return !isTemporary() && !NumUnresolved; } bool isUniqued() const { return Storage == Uniqued; } bool isDistinct() const { return Storage == Distinct; } @@ -903,33 +891,17 @@ public: /// \pre \a isTemporary() must be \c true. void replaceAllUsesWith(Metadata *MD) { assert(isTemporary() && "Expected temporary node"); - assert(!isResolved() && "Expected RAUW support"); - Context.getReplaceableUses()->replaceAllUsesWith(MD); - } - - /// Set the CanReplace flag to the given value. - void setCanReplace(bool Replaceable) { - Context.getReplaceableUses()->setCanReplace(Replaceable); + if (Context.hasReplaceableUses()) + Context.getReplaceableUses()->replaceAllUsesWith(MD); } /// \brief Resolve cycles. /// /// Once all forward declarations have been resolved, force cycles to be - /// resolved. This interface is used when there are no more temporaries, - /// and thus unresolved nodes are part of cycles and no longer need RAUW - /// support. + /// resolved. /// /// \pre No operands (or operands' operands, etc.) have \a isTemporary(). - void resolveCycles() { resolveRecursivelyImpl(/* AllowTemps */ false); } - - /// \brief Resolve cycles while ignoring temporaries. - /// - /// This drops RAUW support for any temporaries, which can no longer - /// be uniqued. - /// - void resolveNonTemporaries() { - resolveRecursivelyImpl(/* AllowTemps */ true); - } + void resolveCycles(); /// \brief Replace a temporary node with a permanent one. /// @@ -982,15 +954,15 @@ protected: private: void handleChangedOperand(void *Ref, Metadata *New); + /// Resolve a unique, unresolved node. void resolve(); + + /// Drop RAUW support, if any. + void dropReplaceableUses(); + void resolveAfterOperandChange(Metadata *Old, Metadata *New); void decrementUnresolvedOperandCount(); - unsigned countUnresolvedOperands(); - - /// Resolve cycles recursively. If \p AllowTemps is true, then any temporary - /// metadata is ignored, otherwise it asserts when encountering temporary - /// metadata. - void resolveRecursivelyImpl(bool AllowTemps); + void countUnresolvedOperands(); /// \brief Mutate this to be "uniqued". /// @@ -1221,6 +1193,52 @@ public: typedef MDTupleTypedArrayWrapper<CLASS> CLASS##Array; #include "llvm/IR/Metadata.def" +/// Placeholder metadata for operands of distinct MDNodes. +/// +/// This is a lightweight placeholder for an operand of a distinct node. It's +/// purpose is to help track forward references when creating a distinct node. +/// This allows distinct nodes involved in a cycle to be constructed before +/// their operands without requiring a heavyweight temporary node with +/// full-blown RAUW support. +/// +/// Each placeholder supports only a single MDNode user. Clients should pass +/// an ID, retrieved via \a getID(), to indicate the "real" operand that this +/// should be replaced with. +/// +/// While it would be possible to implement move operators, they would be +/// fairly expensive. Leave them unimplemented to discourage their use +/// (clients can use std::deque, std::list, BumpPtrAllocator, etc.). +class DistinctMDOperandPlaceholder : public Metadata { + friend class MetadataTracking; + + Metadata **Use = nullptr; + + DistinctMDOperandPlaceholder() = delete; + DistinctMDOperandPlaceholder(DistinctMDOperandPlaceholder &&) = delete; + DistinctMDOperandPlaceholder(const DistinctMDOperandPlaceholder &) = delete; + +public: + explicit DistinctMDOperandPlaceholder(unsigned ID) + : Metadata(DistinctMDOperandPlaceholderKind, Distinct) { + SubclassData32 = ID; + } + + ~DistinctMDOperandPlaceholder() { + if (Use) + *Use = nullptr; + } + + unsigned getID() const { return SubclassData32; } + + /// Replace the use of this with MD. + void replaceUseWith(Metadata *MD) { + if (!Use) + return; + *Use = MD; + Use = nullptr; + } +}; + //===----------------------------------------------------------------------===// /// \brief A tuple of MDNodes. /// @@ -1297,6 +1315,8 @@ public: void setOperand(unsigned I, MDNode *New); StringRef getName() const; void print(raw_ostream &ROS, bool IsForDebug = false) const; + void print(raw_ostream &ROS, ModuleSlotTracker &MST, + bool IsForDebug = false) const; void dump() const; // --------------------------------------------------------------------------- diff --git a/include/llvm/IR/Module.h b/include/llvm/IR/Module.h index 942f68543cb6..632b27e2d0dd 100644 --- a/include/llvm/IR/Module.h +++ b/include/llvm/IR/Module.h @@ -15,12 +15,12 @@ #ifndef LLVM_IR_MODULE_H #define LLVM_IR_MODULE_H -#include "llvm/ADT/Optional.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalIFunc.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/CBindingWrapping.h" @@ -29,11 +29,13 @@ #include <system_error> namespace llvm { +template <typename T> class Optional; class FunctionType; class GVMaterializer; class LLVMContext; class RandomNumberGenerator; class StructType; +template <class PtrType> class SmallPtrSetImpl; template<> struct ilist_traits<NamedMDNode> : public ilist_default_traits<NamedMDNode> { @@ -75,6 +77,8 @@ public: typedef SymbolTableList<Function> FunctionListType; /// The type for the list of aliases. typedef SymbolTableList<GlobalAlias> AliasListType; + /// The type for the list of ifuncs. + typedef SymbolTableList<GlobalIFunc> IFuncListType; /// The type for the list of named metadata. typedef ilist<NamedMDNode> NamedMDListType; /// The type of the comdat "symbol" table. @@ -100,6 +104,11 @@ public: /// The Global Alias constant iterator typedef AliasListType::const_iterator const_alias_iterator; + /// The Global IFunc iterators. + typedef IFuncListType::iterator ifunc_iterator; + /// The Global IFunc constant iterator + typedef IFuncListType::const_iterator const_ifunc_iterator; + /// The named metadata iterators. typedef NamedMDListType::iterator named_metadata_iterator; /// The named metadata constant iterators. @@ -163,6 +172,7 @@ private: GlobalListType GlobalList; ///< The Global Variables in the module FunctionListType FunctionList; ///< The Functions in the module AliasListType AliasList; ///< The Aliases in the module + IFuncListType IFuncList; ///< The IFuncs in the module NamedMDListType NamedMDList; ///< The named metadata in the module std::string GlobalScopeAsm; ///< Inline Asm at global scope. ValueSymbolTable *ValSymTab; ///< Symbol table for values @@ -170,6 +180,8 @@ private: std::unique_ptr<GVMaterializer> Materializer; ///< Used to materialize GlobalValues std::string ModuleID; ///< Human readable identifier for the module + std::string SourceFileName; ///< Original source file name for module, + ///< recorded in bitcode. std::string TargetTriple; ///< Platform target triple Module compiled on ///< Format: (arch)(sub)-(vendor)-(sys0-(abi) void *NamedMDSymTab; ///< NamedMDNode names. @@ -195,6 +207,12 @@ public: /// @returns the module identifier as a string const std::string &getModuleIdentifier() const { return ModuleID; } + /// Get the module's original source file name. When compiling from + /// bitcode, this is taken from a bitcode record where it was recorded. + /// For other compiles it is the same as the ModuleID, which would + /// contain the source file name. + const std::string &getSourceFileName() const { return SourceFileName; } + /// \brief Get a short "name" for the module. /// /// This is useful for debugging or logging. It is essentially a convenience @@ -240,6 +258,9 @@ public: /// Set the module identifier. void setModuleIdentifier(StringRef ID) { ModuleID = ID; } + /// Set the module's original source file name. + void setSourceFileName(StringRef Name) { SourceFileName = Name; } + /// Set the data layout void setDataLayout(StringRef Desc); void setDataLayout(const DataLayout &Other); @@ -251,8 +272,7 @@ public: /// A trailing newline is added if the input doesn't have one. void setModuleInlineAsm(StringRef Asm) { GlobalScopeAsm = Asm; - if (!GlobalScopeAsm.empty() && - GlobalScopeAsm[GlobalScopeAsm.size()-1] != '\n') + if (!GlobalScopeAsm.empty() && GlobalScopeAsm.back() != '\n') GlobalScopeAsm += '\n'; } @@ -260,8 +280,7 @@ public: /// A trailing newline is added if the input doesn't have one. void appendModuleInlineAsm(StringRef Asm) { GlobalScopeAsm += Asm; - if (!GlobalScopeAsm.empty() && - GlobalScopeAsm[GlobalScopeAsm.size()-1] != '\n') + if (!GlobalScopeAsm.empty() && GlobalScopeAsm.back() != '\n') GlobalScopeAsm += '\n'; } @@ -375,6 +394,15 @@ public: GlobalAlias *getNamedAlias(StringRef Name) const; /// @} +/// @name Global IFunc Accessors +/// @{ + + /// Return the global ifunc in the module with the specified name, of + /// arbitrary type. This method returns null if a global with the specified + /// name is not found. + GlobalIFunc *getNamedIFunc(StringRef Name) const; + +/// @} /// @name Named Metadata Accessors /// @{ @@ -477,6 +505,13 @@ public: static AliasListType Module::*getSublistAccess(GlobalAlias*) { return &Module::AliasList; } + /// Get the Module's list of ifuncs (constant). + const IFuncListType &getIFuncList() const { return IFuncList; } + /// Get the Module's list of ifuncs. + IFuncListType &getIFuncList() { return IFuncList; } + static IFuncListType Module::*getSublistAccess(GlobalIFunc*) { + return &Module::IFuncList; + } /// Get the Module's list of named metadata (constant). const NamedMDListType &getNamedMDList() const { return NamedMDList; } /// Get the Module's list of named metadata. @@ -551,9 +586,96 @@ public: } /// @} -/// @name Named Metadata Iteration +/// @name IFunc Iteration /// @{ + ifunc_iterator ifunc_begin() { return IFuncList.begin(); } + const_ifunc_iterator ifunc_begin() const { return IFuncList.begin(); } + ifunc_iterator ifunc_end () { return IFuncList.end(); } + const_ifunc_iterator ifunc_end () const { return IFuncList.end(); } + size_t ifunc_size () const { return IFuncList.size(); } + bool ifunc_empty() const { return IFuncList.empty(); } + + iterator_range<ifunc_iterator> ifuncs() { + return make_range(ifunc_begin(), ifunc_end()); + } + iterator_range<const_ifunc_iterator> ifuncs() const { + return make_range(ifunc_begin(), ifunc_end()); + } + +/// @} +/// @name Convenience iterators +/// @{ + + template <bool IsConst> class global_object_iterator_t { + friend Module; + + typename std::conditional<IsConst, const_iterator, iterator>::type + function_i, + function_e; + typename std::conditional<IsConst, const_global_iterator, + global_iterator>::type global_i; + + typedef + typename std::conditional<IsConst, const Module, Module>::type ModuleTy; + + global_object_iterator_t(ModuleTy &M) + : function_i(M.begin()), function_e(M.end()), + global_i(M.global_begin()) {} + global_object_iterator_t(ModuleTy &M, int) + : function_i(M.end()), function_e(M.end()), global_i(M.global_end()) {} + + public: + global_object_iterator_t &operator++() { + if (function_i != function_e) + ++function_i; + else + ++global_i; + return *this; + } + + typename std::conditional<IsConst, const GlobalObject, GlobalObject>::type & + operator*() const { + if (function_i != function_e) + return *function_i; + else + return *global_i; + } + + bool operator!=(const global_object_iterator_t &other) const { + return function_i != other.function_i || global_i != other.global_i; + } + }; + + typedef global_object_iterator_t</*IsConst=*/false> global_object_iterator; + typedef global_object_iterator_t</*IsConst=*/true> + const_global_object_iterator; + + global_object_iterator global_object_begin() { + return global_object_iterator(*this); + } + global_object_iterator global_object_end() { + return global_object_iterator(*this, 0); + } + + const_global_object_iterator global_object_begin() const { + return const_global_object_iterator(*this); + } + const_global_object_iterator global_object_end() const { + return const_global_object_iterator(*this, 0); + } + + iterator_range<global_object_iterator> global_objects() { + return make_range(global_object_begin(), global_object_end()); + } + iterator_range<const_global_object_iterator> global_objects() const { + return make_range(global_object_begin(), global_object_end()); + } + + /// @} + /// @name Named Metadata Iteration + /// @{ + named_metadata_iterator named_metadata_begin() { return NamedMDList.begin(); } const_named_metadata_iterator named_metadata_begin() const { return NamedMDList.begin(); @@ -574,6 +696,58 @@ public: return make_range(named_metadata_begin(), named_metadata_end()); } + /// An iterator for DICompileUnits that skips those marked NoDebug. + class debug_compile_units_iterator + : public std::iterator<std::input_iterator_tag, DICompileUnit *> { + NamedMDNode *CUs; + unsigned Idx; + void SkipNoDebugCUs(); + public: + explicit debug_compile_units_iterator(NamedMDNode *CUs, unsigned Idx) + : CUs(CUs), Idx(Idx) { + SkipNoDebugCUs(); + } + debug_compile_units_iterator &operator++() { + ++Idx; + SkipNoDebugCUs(); + return *this; + } + debug_compile_units_iterator operator++(int) { + debug_compile_units_iterator T(*this); + ++Idx; + return T; + } + bool operator==(const debug_compile_units_iterator &I) const { + return Idx == I.Idx; + } + bool operator!=(const debug_compile_units_iterator &I) const { + return Idx != I.Idx; + } + DICompileUnit *operator*() const; + DICompileUnit *operator->() const; + }; + + debug_compile_units_iterator debug_compile_units_begin() const { + auto *CUs = getNamedMetadata("llvm.dbg.cu"); + return debug_compile_units_iterator(CUs, 0); + } + + debug_compile_units_iterator debug_compile_units_end() const { + auto *CUs = getNamedMetadata("llvm.dbg.cu"); + return debug_compile_units_iterator(CUs, CUs ? CUs->getNumOperands() : 0); + } + + /// Return an iterator for all DICompileUnits listed in this Module's + /// llvm.dbg.cu named metadata node and aren't explicitly marked as + /// NoDebug. + iterator_range<debug_compile_units_iterator> debug_compile_units() const { + auto *CUs = getNamedMetadata("llvm.dbg.cu"); + return make_range( + debug_compile_units_iterator(CUs, 0), + debug_compile_units_iterator(CUs, CUs ? CUs->getNumOperands() : 0)); + } +/// @} + /// Destroy ConstantArrays in LLVMContext if they are not used. /// ConstantArrays constructed during linking can cause quadratic memory /// explosion. Releasing all unused constants can cause a 20% LTO compile-time @@ -583,7 +757,6 @@ public: /// be called where all uses of the LLVMContext are understood. void dropTriviallyDeadConstantArrays(); -/// @} /// @name Utility functions for printing and dumping Module objects /// @{ @@ -628,17 +801,34 @@ public: void setPICLevel(PICLevel::Level PL); /// @} - /// @name Utility functions for querying and setting PGO counts +/// @} +/// @name Utility functions for querying and setting PIE level +/// @{ + + /// \brief Returns the PIE level (small or large model) + PIELevel::Level getPIELevel() const; + + /// \brief Set the PIE level (small or large model) + void setPIELevel(PIELevel::Level PL); +/// @} + + /// @name Utility functions for querying and setting PGO summary /// @{ - /// \brief Set maximum function count in PGO mode - void setMaximumFunctionCount(uint64_t); + /// \brief Attach profile summary metadata to this module. + void setProfileSummary(Metadata *M); - /// \brief Returns maximum function count in PGO mode - Optional<uint64_t> getMaximumFunctionCount(); + /// \brief Returns profile summary metadata + Metadata *getProfileSummary(); /// @} }; +/// \brief Given "llvm.used" or "llvm.compiler.used" as a global name, collect +/// the initializer elements of that global in Set and return the global itself. +GlobalVariable *collectUsedGlobalVariables(const Module &M, + SmallPtrSetImpl<GlobalValue *> &Set, + bool CompilerUsed); + /// An raw_ostream inserter for modules. inline raw_ostream &operator<<(raw_ostream &O, const Module &M) { M.print(O, nullptr); diff --git a/include/llvm/IR/ModuleSlotTracker.h b/include/llvm/IR/ModuleSlotTracker.h index 49730a66bdf6..eb26fba906ea 100644 --- a/include/llvm/IR/ModuleSlotTracker.h +++ b/include/llvm/IR/ModuleSlotTracker.h @@ -30,6 +30,8 @@ class Value; class ModuleSlotTracker { /// Storage for a slot tracker. std::unique_ptr<SlotTracker> MachineStorage; + bool ShouldCreateStorage = false; + bool ShouldInitializeAllMetadata = false; const Module *M = nullptr; const Function *F = nullptr; @@ -53,7 +55,9 @@ public: /// Destructor to clean up storage. ~ModuleSlotTracker(); - SlotTracker *getMachine() const { return Machine; } + /// Lazily creates a slot tracker. + SlotTracker *getMachine(); + const Module *getModule() const { return M; } const Function *getCurrentFunction() const { return F; } diff --git a/include/llvm/IR/ModuleSummaryIndex.h b/include/llvm/IR/ModuleSummaryIndex.h new file mode 100644 index 000000000000..45d9bf7af706 --- /dev/null +++ b/include/llvm/IR/ModuleSummaryIndex.h @@ -0,0 +1,508 @@ +//===-- llvm/ModuleSummaryIndex.h - Module Summary Index --------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// @file +/// ModuleSummaryIndex.h This file contains the declarations the classes that +/// hold the module index and summary for function importing. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_MODULESUMMARYINDEX_H +#define LLVM_IR_MODULESUMMARYINDEX_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/IR/Module.h" + +#include <array> + +namespace llvm { + +/// \brief Class to accumulate and hold information about a callee. +struct CalleeInfo { + /// The static number of callsites calling corresponding function. + unsigned CallsiteCount; + /// The cumulative profile count of calls to corresponding function + /// (if using PGO, otherwise 0). + uint64_t ProfileCount; + CalleeInfo() : CallsiteCount(0), ProfileCount(0) {} + CalleeInfo(unsigned CallsiteCount, uint64_t ProfileCount) + : CallsiteCount(CallsiteCount), ProfileCount(ProfileCount) {} + CalleeInfo &operator+=(uint64_t RHSProfileCount) { + CallsiteCount++; + ProfileCount += RHSProfileCount; + return *this; + } +}; + +/// Struct to hold value either by GUID or Value*, depending on whether this +/// is a combined or per-module index, respectively. +struct ValueInfo { + /// The value representation used in this instance. + enum ValueInfoKind { + VI_GUID, + VI_Value, + }; + + /// Union of the two possible value types. + union ValueUnion { + GlobalValue::GUID Id; + const Value *V; + ValueUnion(GlobalValue::GUID Id) : Id(Id) {} + ValueUnion(const Value *V) : V(V) {} + }; + + /// The value being represented. + ValueUnion TheValue; + /// The value representation. + ValueInfoKind Kind; + /// Constructor for a GUID value + ValueInfo(GlobalValue::GUID Id = 0) : TheValue(Id), Kind(VI_GUID) {} + /// Constructor for a Value* value + ValueInfo(const Value *V) : TheValue(V), Kind(VI_Value) {} + /// Accessor for GUID value + GlobalValue::GUID getGUID() const { + assert(Kind == VI_GUID && "Not a GUID type"); + return TheValue.Id; + } + /// Accessor for Value* value + const Value *getValue() const { + assert(Kind == VI_Value && "Not a Value type"); + return TheValue.V; + } + bool isGUID() const { return Kind == VI_GUID; } +}; + +/// \brief Function and variable summary information to aid decisions and +/// implementation of importing. +class GlobalValueSummary { +public: + /// \brief Sububclass discriminator (for dyn_cast<> et al.) + enum SummaryKind { AliasKind, FunctionKind, GlobalVarKind }; + + /// Group flags (Linkage, hasSection, isOptSize, etc.) as a bitfield. + struct GVFlags { + /// \brief The linkage type of the associated global value. + /// + /// One use is to flag values that have local linkage types and need to + /// have module identifier appended before placing into the combined + /// index, to disambiguate from other values with the same name. + /// In the future this will be used to update and optimize linkage + /// types based on global summary-based analysis. + unsigned Linkage : 4; + + /// Indicate if the global value is located in a specific section. + unsigned HasSection : 1; + + /// Convenience Constructors + explicit GVFlags(GlobalValue::LinkageTypes Linkage, bool HasSection) + : Linkage(Linkage), HasSection(HasSection) {} + GVFlags(const GlobalValue &GV) + : Linkage(GV.getLinkage()), HasSection(GV.hasSection()) {} + }; + +private: + /// Kind of summary for use in dyn_cast<> et al. + SummaryKind Kind; + + /// This is the hash of the name of the symbol in the original file. It is + /// identical to the GUID for global symbols, but differs for local since the + /// GUID includes the module level id in the hash. + GlobalValue::GUID OriginalName; + + /// \brief Path of module IR containing value's definition, used to locate + /// module during importing. + /// + /// This is only used during parsing of the combined index, or when + /// parsing the per-module index for creation of the combined summary index, + /// not during writing of the per-module index which doesn't contain a + /// module path string table. + StringRef ModulePath; + + GVFlags Flags; + + /// List of values referenced by this global value's definition + /// (either by the initializer of a global variable, or referenced + /// from within a function). This does not include functions called, which + /// are listed in the derived FunctionSummary object. + std::vector<ValueInfo> RefEdgeList; + +protected: + /// GlobalValueSummary constructor. + GlobalValueSummary(SummaryKind K, GVFlags Flags) : Kind(K), Flags(Flags) {} + +public: + virtual ~GlobalValueSummary() = default; + + /// Returns the hash of the original name, it is identical to the GUID for + /// externally visible symbols, but not for local ones. + GlobalValue::GUID getOriginalName() { return OriginalName; } + + /// Initialize the original name hash in this summary. + void setOriginalName(GlobalValue::GUID Name) { OriginalName = Name; } + + /// Which kind of summary subclass this is. + SummaryKind getSummaryKind() const { return Kind; } + + /// Set the path to the module containing this function, for use in + /// the combined index. + void setModulePath(StringRef ModPath) { ModulePath = ModPath; } + + /// Get the path to the module containing this function. + StringRef modulePath() const { return ModulePath; } + + /// Get the flags for this GlobalValue (see \p struct GVFlags). + GVFlags flags() { return Flags; } + + /// Return linkage type recorded for this global value. + GlobalValue::LinkageTypes linkage() const { + return static_cast<GlobalValue::LinkageTypes>(Flags.Linkage); + } + + /// Sets the linkage to the value determined by global summary-based + /// optimization. Will be applied in the ThinLTO backends. + void setLinkage(GlobalValue::LinkageTypes Linkage) { + Flags.Linkage = Linkage; + } + + /// Return true if this summary is for a GlobalValue that needs promotion + /// to be referenced from another module. + bool needsRenaming() const { return GlobalValue::isLocalLinkage(linkage()); } + + /// Return true if this global value is located in a specific section. + bool hasSection() const { return Flags.HasSection; } + + /// Record a reference from this global value to the global value identified + /// by \p RefGUID. + void addRefEdge(GlobalValue::GUID RefGUID) { RefEdgeList.push_back(RefGUID); } + + /// Record a reference from this global value to the global value identified + /// by \p RefV. + void addRefEdge(const Value *RefV) { RefEdgeList.push_back(RefV); } + + /// Record a reference from this global value to each global value identified + /// in \p RefEdges. + void addRefEdges(DenseSet<const Value *> &RefEdges) { + for (auto &RI : RefEdges) + addRefEdge(RI); + } + + /// Return the list of values referenced by this global value definition. + std::vector<ValueInfo> &refs() { return RefEdgeList; } + const std::vector<ValueInfo> &refs() const { return RefEdgeList; } +}; + +/// \brief Alias summary information. +class AliasSummary : public GlobalValueSummary { + GlobalValueSummary *AliaseeSummary; + +public: + /// Summary constructors. + AliasSummary(GVFlags Flags) : GlobalValueSummary(AliasKind, Flags) {} + + /// Check if this is an alias summary. + static bool classof(const GlobalValueSummary *GVS) { + return GVS->getSummaryKind() == AliasKind; + } + + void setAliasee(GlobalValueSummary *Aliasee) { AliaseeSummary = Aliasee; } + + const GlobalValueSummary &getAliasee() const { + return const_cast<AliasSummary *>(this)->getAliasee(); + } + + GlobalValueSummary &getAliasee() { + assert(AliaseeSummary && "Unexpected missing aliasee summary"); + return *AliaseeSummary; + } +}; + +/// \brief Function summary information to aid decisions and implementation of +/// importing. +class FunctionSummary : public GlobalValueSummary { +public: + /// <CalleeValueInfo, CalleeInfo> call edge pair. + typedef std::pair<ValueInfo, CalleeInfo> EdgeTy; + +private: + /// Number of instructions (ignoring debug instructions, e.g.) computed + /// during the initial compile step when the summary index is first built. + unsigned InstCount; + + /// List of <CalleeValueInfo, CalleeInfo> call edge pairs from this function. + std::vector<EdgeTy> CallGraphEdgeList; + +public: + /// Summary constructors. + FunctionSummary(GVFlags Flags, unsigned NumInsts) + : GlobalValueSummary(FunctionKind, Flags), InstCount(NumInsts) {} + + /// Check if this is a function summary. + static bool classof(const GlobalValueSummary *GVS) { + return GVS->getSummaryKind() == FunctionKind; + } + + /// Get the instruction count recorded for this function. + unsigned instCount() const { return InstCount; } + + /// Record a call graph edge from this function to the function identified + /// by \p CalleeGUID, with \p CalleeInfo including the cumulative profile + /// count (across all calls from this function) or 0 if no PGO. + void addCallGraphEdge(GlobalValue::GUID CalleeGUID, CalleeInfo Info) { + CallGraphEdgeList.push_back(std::make_pair(CalleeGUID, Info)); + } + + /// Record a call graph edge from this function to each function GUID recorded + /// in \p CallGraphEdges. + void + addCallGraphEdges(DenseMap<GlobalValue::GUID, CalleeInfo> &CallGraphEdges) { + for (auto &EI : CallGraphEdges) + addCallGraphEdge(EI.first, EI.second); + } + + /// Record a call graph edge from this function to the function identified + /// by \p CalleeV, with \p CalleeInfo including the cumulative profile + /// count (across all calls from this function) or 0 if no PGO. + void addCallGraphEdge(const Value *CalleeV, CalleeInfo Info) { + CallGraphEdgeList.push_back(std::make_pair(CalleeV, Info)); + } + + /// Record a call graph edge from this function to each function recorded + /// in \p CallGraphEdges. + void addCallGraphEdges(DenseMap<const Value *, CalleeInfo> &CallGraphEdges) { + for (auto &EI : CallGraphEdges) + addCallGraphEdge(EI.first, EI.second); + } + + /// Return the list of <CalleeValueInfo, CalleeInfo> pairs. + std::vector<EdgeTy> &calls() { return CallGraphEdgeList; } + const std::vector<EdgeTy> &calls() const { return CallGraphEdgeList; } +}; + +/// \brief Global variable summary information to aid decisions and +/// implementation of importing. +/// +/// Currently this doesn't add anything to the base \p GlobalValueSummary, +/// but is a placeholder as additional info may be added to the summary +/// for variables. +class GlobalVarSummary : public GlobalValueSummary { + +public: + /// Summary constructors. + GlobalVarSummary(GVFlags Flags) : GlobalValueSummary(GlobalVarKind, Flags) {} + + /// Check if this is a global variable summary. + static bool classof(const GlobalValueSummary *GVS) { + return GVS->getSummaryKind() == GlobalVarKind; + } +}; + +/// 160 bits SHA1 +typedef std::array<uint32_t, 5> ModuleHash; + +/// List of global value summary structures for a particular value held +/// in the GlobalValueMap. Requires a vector in the case of multiple +/// COMDAT values of the same name. +typedef std::vector<std::unique_ptr<GlobalValueSummary>> GlobalValueSummaryList; + +/// Map from global value GUID to corresponding summary structures. +/// Use a std::map rather than a DenseMap since it will likely incur +/// less overhead, as the value type is not very small and the size +/// of the map is unknown, resulting in inefficiencies due to repeated +/// insertions and resizing. +typedef std::map<GlobalValue::GUID, GlobalValueSummaryList> + GlobalValueSummaryMapTy; + +/// Type used for iterating through the global value summary map. +typedef GlobalValueSummaryMapTy::const_iterator const_gvsummary_iterator; +typedef GlobalValueSummaryMapTy::iterator gvsummary_iterator; + +/// String table to hold/own module path strings, which additionally holds the +/// module ID assigned to each module during the plugin step, as well as a hash +/// of the module. The StringMap makes a copy of and owns inserted strings. +typedef StringMap<std::pair<uint64_t, ModuleHash>> ModulePathStringTableTy; + +/// Map of global value GUID to its summary, used to identify values defined in +/// a particular module, and provide efficient access to their summary. +typedef std::map<GlobalValue::GUID, GlobalValueSummary *> GVSummaryMapTy; + +/// Class to hold module path string table and global value map, +/// and encapsulate methods for operating on them. +class ModuleSummaryIndex { +private: + /// Map from value name to list of summary instances for values of that + /// name (may be duplicates in the COMDAT case, e.g.). + GlobalValueSummaryMapTy GlobalValueMap; + + /// Holds strings for combined index, mapping to the corresponding module ID. + ModulePathStringTableTy ModulePathStringTable; + +public: + ModuleSummaryIndex() = default; + + // Disable the copy constructor and assignment operators, so + // no unexpected copying/moving occurs. + ModuleSummaryIndex(const ModuleSummaryIndex &) = delete; + void operator=(const ModuleSummaryIndex &) = delete; + + gvsummary_iterator begin() { return GlobalValueMap.begin(); } + const_gvsummary_iterator begin() const { return GlobalValueMap.begin(); } + gvsummary_iterator end() { return GlobalValueMap.end(); } + const_gvsummary_iterator end() const { return GlobalValueMap.end(); } + + /// Get the list of global value summary objects for a given value name. + const GlobalValueSummaryList &getGlobalValueSummaryList(StringRef ValueName) { + return GlobalValueMap[GlobalValue::getGUID(ValueName)]; + } + + /// Get the list of global value summary objects for a given value name. + const const_gvsummary_iterator + findGlobalValueSummaryList(StringRef ValueName) const { + return GlobalValueMap.find(GlobalValue::getGUID(ValueName)); + } + + /// Get the list of global value summary objects for a given value GUID. + const const_gvsummary_iterator + findGlobalValueSummaryList(GlobalValue::GUID ValueGUID) const { + return GlobalValueMap.find(ValueGUID); + } + + /// Add a global value summary for a value of the given name. + void addGlobalValueSummary(StringRef ValueName, + std::unique_ptr<GlobalValueSummary> Summary) { + GlobalValueMap[GlobalValue::getGUID(ValueName)].push_back( + std::move(Summary)); + } + + /// Add a global value summary for a value of the given GUID. + void addGlobalValueSummary(GlobalValue::GUID ValueGUID, + std::unique_ptr<GlobalValueSummary> Summary) { + GlobalValueMap[ValueGUID].push_back(std::move(Summary)); + } + + /// Find the summary for global \p GUID in module \p ModuleId, or nullptr if + /// not found. + GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID, + StringRef ModuleId) const { + auto CalleeInfoList = findGlobalValueSummaryList(ValueGUID); + if (CalleeInfoList == end()) { + return nullptr; // This function does not have a summary + } + auto Summary = + llvm::find_if(CalleeInfoList->second, + [&](const std::unique_ptr<GlobalValueSummary> &Summary) { + return Summary->modulePath() == ModuleId; + }); + if (Summary == CalleeInfoList->second.end()) + return nullptr; + return Summary->get(); + } + + /// Returns the first GlobalValueSummary for \p GV, asserting that there + /// is only one if \p PerModuleIndex. + GlobalValueSummary *getGlobalValueSummary(const GlobalValue &GV, + bool PerModuleIndex = true) const { + assert(GV.hasName() && "Can't get GlobalValueSummary for GV with no name"); + return getGlobalValueSummary(GlobalValue::getGUID(GV.getName()), + PerModuleIndex); + } + + /// Returns the first GlobalValueSummary for \p ValueGUID, asserting that + /// there + /// is only one if \p PerModuleIndex. + GlobalValueSummary *getGlobalValueSummary(GlobalValue::GUID ValueGUID, + bool PerModuleIndex = true) const; + + /// Table of modules, containing module hash and id. + const StringMap<std::pair<uint64_t, ModuleHash>> &modulePaths() const { + return ModulePathStringTable; + } + + /// Table of modules, containing hash and id. + StringMap<std::pair<uint64_t, ModuleHash>> &modulePaths() { + return ModulePathStringTable; + } + + /// Get the module ID recorded for the given module path. + uint64_t getModuleId(const StringRef ModPath) const { + return ModulePathStringTable.lookup(ModPath).first; + } + + /// Get the module SHA1 hash recorded for the given module path. + const ModuleHash &getModuleHash(const StringRef ModPath) const { + auto It = ModulePathStringTable.find(ModPath); + assert(It != ModulePathStringTable.end() && "Module not registered"); + return It->second.second; + } + + /// Add the given per-module index into this module index/summary, + /// assigning it the given module ID. Each module merged in should have + /// a unique ID, necessary for consistent renaming of promoted + /// static (local) variables. + void mergeFrom(std::unique_ptr<ModuleSummaryIndex> Other, + uint64_t NextModuleId); + + /// Convenience method for creating a promoted global name + /// for the given value name of a local, and its original module's ID. + static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash) { + SmallString<256> NewName(Name); + NewName += ".llvm."; + NewName += utohexstr(ModHash[0]); // Take the first 32 bits + return NewName.str(); + } + + /// Helper to obtain the unpromoted name for a global value (or the original + /// name if not promoted). + static StringRef getOriginalNameBeforePromote(StringRef Name) { + std::pair<StringRef, StringRef> Pair = Name.split(".llvm."); + return Pair.first; + } + + /// Add a new module path with the given \p Hash, mapped to the given \p + /// ModID, and return an iterator to the entry in the index. + ModulePathStringTableTy::iterator + addModulePath(StringRef ModPath, uint64_t ModId, + ModuleHash Hash = ModuleHash{{0}}) { + return ModulePathStringTable.insert(std::make_pair( + ModPath, + std::make_pair(ModId, Hash))).first; + } + + /// Check if the given Module has any functions available for exporting + /// in the index. We consider any module present in the ModulePathStringTable + /// to have exported functions. + bool hasExportedFunctions(const Module &M) const { + return ModulePathStringTable.count(M.getModuleIdentifier()); + } + + /// Remove entries in the GlobalValueMap that have empty summaries due to the + /// eager nature of map entry creation during VST parsing. These would + /// also be suppressed during combined index generation in mergeFrom(), + /// but if there was only one module or this was the first module we might + /// not invoke mergeFrom. + void removeEmptySummaryEntries(); + + /// Collect for the given module the list of function it defines + /// (GUID -> Summary). + void collectDefinedFunctionsForModule(StringRef ModulePath, + GVSummaryMapTy &GVSummaryMap) const; + + /// Collect for each module the list of Summaries it defines (GUID -> + /// Summary). + void collectDefinedGVSummariesPerModule( + StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries) const; +}; + +} // End llvm namespace + +#endif diff --git a/include/llvm/IR/Operator.h b/include/llvm/IR/Operator.h index 372b254ab183..5880290f3d99 100644 --- a/include/llvm/IR/Operator.h +++ b/include/llvm/IR/Operator.h @@ -79,7 +79,7 @@ public: }; private: - friend class BinaryOperator; + friend class Instruction; friend class ConstantExpr; void setHasNoUnsignedWrap(bool B) { SubclassOptionalData = @@ -130,7 +130,7 @@ public: }; private: - friend class BinaryOperator; + friend class Instruction; friend class ConstantExpr; void setIsExact(bool B) { SubclassOptionalData = (SubclassOptionalData & ~IsExact) | (B * IsExact); @@ -401,6 +401,7 @@ public: } Type *getSourceElementType() const; + Type *getResultElementType() const; /// Method to return the address space of the pointer operand. unsigned getPointerAddressSpace() const { diff --git a/include/llvm/IR/OptBisect.h b/include/llvm/IR/OptBisect.h new file mode 100644 index 000000000000..9eee65e93e52 --- /dev/null +++ b/include/llvm/IR/OptBisect.h @@ -0,0 +1,81 @@ +//===----------- llvm/IR/OptBisect.h - LLVM Bisect support -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares the interface for bisecting optimizations. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_OPTBISECT_H +#define LLVM_IR_OPTBISECT_H + +namespace llvm { + +class Pass; +class StringRef; +class Twine; + +/// This class implements a mechanism to disable passes and individual +/// optimizations at compile time based on a command line option +/// (-opt-bisect-limit) in order to perform a bisecting search for +/// optimization-related problems. +class OptBisect { +public: + /// \brief Default constructor, initializes the OptBisect state based on the + /// -opt-bisect-limit command line argument. + /// + /// By default, bisection is disabled. + /// + /// Clients should not instantiate this class directly. All access should go + /// through LLVMContext. + OptBisect(); + + /// Checks the bisect limit to determine if the specified pass should run. + /// + /// This function will immediate return true if bisection is disabled. If the + /// bisect limit is set to -1, the function will print a message describing + /// the pass and the bisect number assigned to it and return true. Otherwise, + /// the function will print a message with the bisect number assigned to the + /// pass and indicating whether or not the pass will be run and return true if + /// the bisect limit has not yet been exceded or false if it has. + /// + /// Most passes should not call this routine directly. Instead, it is called + /// through a helper routine provided by the pass base class. For instance, + /// function passes should call FunctionPass::skipFunction(). + template <class UnitT> + bool shouldRunPass(const Pass *P, const UnitT &U); + + /// Checks the bisect limit to determine if the optimization described by the + /// /p Desc argument should run. + /// + /// This function will immediate return true if bisection is disabled. If the + /// bisect limit is set to -1, the function will print a message with the + /// bisect number assigned to the optimization along with the /p Desc + /// description and return true. Otherwise, the function will print a message + /// with the bisect number assigned to the optimization and indicating whether + /// or not the pass will be run and return true if the bisect limit has not + /// yet been exceded or false if it has. + /// + /// Passes may call this function to provide more fine grained control over + /// individual optimizations performed by the pass. Passes which cannot be + /// skipped entirely (such as non-optional code generation passes) may still + /// call this function to control whether or not individual optional + /// transformations are performed. + bool shouldRunCase(const Twine &Desc); + +private: + bool checkPass(const StringRef PassName, const StringRef TargetDesc); + + bool BisectEnabled = false; + unsigned LastBisectNum = 0; +}; + +} // end namespace llvm + +#endif // LLVM_IR_OPTBISECT_H diff --git a/include/llvm/IR/PassManager.h b/include/llvm/IR/PassManager.h index 2ceb53d21b7a..402d04a54a41 100644 --- a/include/llvm/IR/PassManager.h +++ b/include/llvm/IR/PassManager.h @@ -44,8 +44,8 @@ #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManagerInternal.h" -#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/TypeName.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/type_traits.h" #include <list> @@ -54,9 +54,6 @@ namespace llvm { -class Module; -class Function; - /// \brief An abstract set of preserved analyses following a transformation pass /// run. /// @@ -147,6 +144,16 @@ public: PreservedPassIDs.count(PassID); } + /// \brief Query whether all of the analyses in the set are preserved. + bool preserved(PreservedAnalyses Arg) { + if (Arg.areAllPreserved()) + return areAllPreserved(); + for (void *P : Arg.PreservedPassIDs) + if (!preserved(P)) + return false; + return true; + } + /// \brief Test whether all passes are preserved. /// /// This is used primarily to optimize for the case of no changes which will @@ -166,6 +173,44 @@ private: // Forward declare the analysis manager template. template <typename IRUnitT> class AnalysisManager; +/// A CRTP mix-in to automatically provide informational APIs needed for +/// passes. +/// +/// This provides some boiler plate for types that are passes. +template <typename DerivedT> struct PassInfoMixin { + /// Returns the name of the derived pass type. + static StringRef name() { + StringRef Name = getTypeName<DerivedT>(); + if (Name.startswith("llvm::")) + Name = Name.drop_front(strlen("llvm::")); + return Name; + } +}; + +/// A CRTP mix-in to automatically provide informational APIs needed for +/// analysis passes. +/// +/// This provides some boiler plate for types that are analysis passes. It +/// automatically mixes in \c PassInfoMixin and adds informational APIs +/// specifically used for analyses. +template <typename DerivedT> +struct AnalysisInfoMixin : PassInfoMixin<DerivedT> { + /// Returns an opaque, unique ID for this pass type. + /// + /// Note that this requires the derived type provide a static member whose + /// address can be converted to a void pointer. + /// + /// FIXME: The only reason the derived type needs to provide this rather than + /// this mixin providing it is due to broken implementations which cannot + /// correctly unique a templated static so that they have the same addresses + /// for each instantiation and are definitively emitted once for each + /// instantiation. The only currently known platform with this limitation are + /// Windows DLL builds, specifically building each part of LLVM as a DLL. If + /// we ever remove that build configuration, this mixin can provide the + /// static PassID as well. + static void *ID() { return (void *)&DerivedT::PassID; } +}; + /// \brief Manages a sequence of passes over units of IR. /// /// A pass manager contains a sequence of passes to run over units of IR. It is @@ -177,7 +222,8 @@ template <typename IRUnitT> class AnalysisManager; /// that analysis manager to each pass it runs, as well as calling the analysis /// manager's invalidation routine with the PreservedAnalyses of each pass it /// runs. -template <typename IRUnitT> class PassManager { +template <typename IRUnitT> +class PassManager : public PassInfoMixin<PassManager<IRUnitT>> { public: /// \brief Construct a pass manager. /// @@ -195,11 +241,11 @@ public: } /// \brief Run all of the passes in this manager over the IR. - PreservedAnalyses run(IRUnitT &IR, AnalysisManager<IRUnitT> *AM = nullptr) { + PreservedAnalyses run(IRUnitT &IR, AnalysisManager<IRUnitT> &AM) { PreservedAnalyses PA = PreservedAnalyses::all(); if (DebugLogging) - dbgs() << "Starting pass manager run.\n"; + dbgs() << "Starting " << getTypeName<IRUnitT>() << " pass manager run.\n"; for (unsigned Idx = 0, Size = Passes.size(); Idx != Size; ++Idx) { if (DebugLogging) @@ -208,13 +254,11 @@ public: PreservedAnalyses PassPA = Passes[Idx]->run(IR, AM); - // If we have an active analysis manager at this level we want to ensure - // we update it as each pass runs and potentially invalidates analyses. - // We also update the preserved set of analyses based on what analyses we - // have already handled the invalidation for here and don't need to - // invalidate when finished. - if (AM) - PassPA = AM->invalidate(IR, std::move(PassPA)); + // Update the analysis manager as each pass runs and potentially + // invalidates analyses. We also update the preserved set of analyses + // based on what analyses we have already handled the invalidation for + // here and don't need to invalidate when finished. + PassPA = AM.invalidate(IR, std::move(PassPA)); // Finally, we intersect the final preserved analyses to compute the // aggregate preserved set for this pass manager. @@ -228,7 +272,7 @@ public: } if (DebugLogging) - dbgs() << "Finished pass manager run.\n"; + dbgs() << "Finished " << getTypeName<IRUnitT>() << " pass manager run.\n"; return PA; } @@ -238,8 +282,6 @@ public: Passes.emplace_back(new PassModelT(std::move(Pass))); } - static StringRef name() { return "PassManager"; } - private: typedef detail::PassConcept<IRUnitT> PassConceptT; @@ -252,9 +294,11 @@ private: bool DebugLogging; }; +extern template class PassManager<Module>; /// \brief Convenience typedef for a pass manager over modules. typedef PassManager<Module> ModulePassManager; +extern template class PassManager<Function>; /// \brief Convenience typedef for a pass manager over functions. typedef PassManager<Function> FunctionPassManager; @@ -284,8 +328,7 @@ template <typename DerivedT, typename IRUnitT> class AnalysisManagerBase { } AnalysisManagerBase(const AnalysisManagerBase &) = delete; - AnalysisManagerBase & - operator=(const AnalysisManagerBase &) = delete; + AnalysisManagerBase &operator=(const AnalysisManagerBase &) = delete; protected: typedef detail::AnalysisResultConcept<IRUnitT> ResultConceptT; @@ -342,14 +385,34 @@ public: /// \brief Register an analysis pass with the manager. /// - /// This provides an initialized and set-up analysis pass to the analysis - /// manager. Whomever is setting up analysis passes must use this to populate - /// the manager with all of the analysis passes available. - template <typename PassT> void registerPass(PassT Pass) { - assert(!AnalysisPasses.count(PassT::ID()) && - "Registered the same analysis pass twice!"); + /// The argument is a callable whose result is a pass. This allows passing in + /// a lambda to construct the pass. + /// + /// The pass type registered is the result type of calling the argument. If + /// that pass has already been registered, then the argument will not be + /// called and this function will return false. Otherwise, the pass type + /// becomes registered, with the instance provided by calling the argument + /// once, and this function returns true. + /// + /// While this returns whether or not the pass type was already registered, + /// there in't an independent way to query that as that would be prone to + /// risky use when *querying* the analysis manager. Instead, the only + /// supported use case is avoiding duplicate registry of an analysis. This + /// interface also lends itself to minimizing the number of times we have to + /// do lookups for analyses or construct complex passes only to throw them + /// away. + template <typename PassBuilderT> bool registerPass(PassBuilderT PassBuilder) { + typedef decltype(PassBuilder()) PassT; typedef detail::AnalysisPassModel<IRUnitT, PassT> PassModelT; - AnalysisPasses[PassT::ID()].reset(new PassModelT(std::move(Pass))); + + auto &PassPtr = AnalysisPasses[PassT::ID()]; + if (PassPtr) + // Already registered this pass type! + return false; + + // Construct a new model around the instance returned by the builder. + PassPtr.reset(new PassModelT(PassBuilder())); + return true; } /// \brief Invalidate a specific analysis pass for an IR module. @@ -472,7 +535,7 @@ private: if (DebugLogging) dbgs() << "Running analysis: " << P.name() << "\n"; AnalysisResultListT &ResultList = AnalysisResultLists[&IR]; - ResultList.emplace_back(PassID, P.run(IR, this)); + ResultList.emplace_back(PassID, P.run(IR, *this)); // P.run may have inserted elements into AnalysisResults and invalidated // RI. @@ -513,8 +576,8 @@ private: return PA; if (DebugLogging) - dbgs() << "Invalidating all non-preserved analyses for: " - << IR.getName() << "\n"; + dbgs() << "Invalidating all non-preserved analyses for: " << IR.getName() + << "\n"; // Clear all the invalidated results associated specifically with this // function. @@ -574,7 +637,8 @@ private: /// \brief Map type from a pair of analysis ID and function pointer to an /// iterator into a particular result list. typedef DenseMap<std::pair<void *, IRUnitT *>, - typename AnalysisResultListT::iterator> AnalysisResultMapT; + typename AnalysisResultListT::iterator> + AnalysisResultMapT; /// \brief Map from an analysis ID and function to a particular cached /// analysis result. @@ -584,9 +648,11 @@ private: bool DebugLogging; }; +extern template class AnalysisManager<Module>; /// \brief Convenience typedef for the Module analysis manager. typedef AnalysisManager<Module> ModuleAnalysisManager; +extern template class AnalysisManager<Function>; /// \brief Convenience typedef for the Function analysis manager. typedef AnalysisManager<Function> FunctionAnalysisManager; @@ -598,26 +664,80 @@ typedef AnalysisManager<Function> FunctionAnalysisManager; /// never use a function analysis manager from within (transitively) a module /// pass manager unless your parent module pass has received a proxy result /// object for it. -class FunctionAnalysisManagerModuleProxy { +/// +/// Note that the proxy's result is a move-only object and represents ownership +/// of the validity of the analyses in the \c FunctionAnalysisManager it +/// provides. +template <typename AnalysisManagerT, typename IRUnitT> +class InnerAnalysisManagerProxy + : public AnalysisInfoMixin< + InnerAnalysisManagerProxy<AnalysisManagerT, IRUnitT>> { public: - class Result; + class Result { + public: + explicit Result(AnalysisManagerT &AM) : AM(&AM) {} + Result(Result &&Arg) : AM(std::move(Arg.AM)) { + // We have to null out the analysis manager in the moved-from state + // because we are taking ownership of the responsibilty to clear the + // analysis state. + Arg.AM = nullptr; + } + Result &operator=(Result &&RHS) { + AM = RHS.AM; + // We have to null out the analysis manager in the moved-from state + // because we are taking ownership of the responsibilty to clear the + // analysis state. + RHS.AM = nullptr; + return *this; + } + ~Result() { + // AM is cleared in a moved from state where there is nothing to do. + if (!AM) + return; + + // Clear out the analysis manager if we're being destroyed -- it means we + // didn't even see an invalidate call when we got invalidated. + AM->clear(); + } - static void *ID() { return (void *)&PassID; } + /// \brief Accessor for the analysis manager. + AnalysisManagerT &getManager() { return *AM; } + + /// \brief Handler for invalidation of the module. + /// + /// If this analysis itself is preserved, then we assume that the set of \c + /// Function objects in the \c Module hasn't changed and thus we don't need + /// to invalidate *all* cached data associated with a \c Function* in the \c + /// FunctionAnalysisManager. + /// + /// Regardless of whether this analysis is marked as preserved, all of the + /// analyses in the \c FunctionAnalysisManager are potentially invalidated + /// based on the set of preserved analyses. + bool invalidate(IRUnitT &IR, const PreservedAnalyses &PA) { + // If this proxy isn't marked as preserved, then we can't even invalidate + // individual function analyses, there may be an invalid set of Function + // objects in the cache making it impossible to incrementally preserve + // them. Just clear the entire manager. + if (!PA.preserved(InnerAnalysisManagerProxy::ID())) + AM->clear(); + + // Return false to indicate that this result is still a valid proxy. + return false; + } - static StringRef name() { return "FunctionAnalysisManagerModuleProxy"; } + private: + AnalysisManagerT *AM; + }; - explicit FunctionAnalysisManagerModuleProxy(FunctionAnalysisManager &FAM) - : FAM(&FAM) {} + explicit InnerAnalysisManagerProxy(AnalysisManagerT &AM) : AM(&AM) {} // We have to explicitly define all the special member functions because MSVC // refuses to generate them. - FunctionAnalysisManagerModuleProxy( - const FunctionAnalysisManagerModuleProxy &Arg) - : FAM(Arg.FAM) {} - FunctionAnalysisManagerModuleProxy(FunctionAnalysisManagerModuleProxy &&Arg) - : FAM(std::move(Arg.FAM)) {} - FunctionAnalysisManagerModuleProxy & - operator=(FunctionAnalysisManagerModuleProxy RHS) { - std::swap(FAM, RHS.FAM); + InnerAnalysisManagerProxy(const InnerAnalysisManagerProxy &Arg) + : AM(Arg.AM) {} + InnerAnalysisManagerProxy(InnerAnalysisManagerProxy &&Arg) + : AM(std::move(Arg.AM)) {} + InnerAnalysisManagerProxy &operator=(InnerAnalysisManagerProxy RHS) { + std::swap(AM, RHS.AM); return *this; } @@ -630,49 +750,24 @@ public: /// In debug builds, it will also assert that the analysis manager is empty /// as no queries should arrive at the function analysis manager prior to /// this analysis being requested. - Result run(Module &M); + Result run(IRUnitT &IR, AnalysisManager<IRUnitT> &) { return Result(*AM); } private: + friend AnalysisInfoMixin< + InnerAnalysisManagerProxy<AnalysisManagerT, IRUnitT>>; static char PassID; - FunctionAnalysisManager *FAM; + AnalysisManagerT *AM; }; -/// \brief The result proxy object for the -/// \c FunctionAnalysisManagerModuleProxy. -/// -/// See its documentation for more information. -class FunctionAnalysisManagerModuleProxy::Result { -public: - explicit Result(FunctionAnalysisManager &FAM) : FAM(&FAM) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - Result(const Result &Arg) : FAM(Arg.FAM) {} - Result(Result &&Arg) : FAM(std::move(Arg.FAM)) {} - Result &operator=(Result RHS) { - std::swap(FAM, RHS.FAM); - return *this; - } - ~Result(); - - /// \brief Accessor for the \c FunctionAnalysisManager. - FunctionAnalysisManager &getManager() { return *FAM; } - - /// \brief Handler for invalidation of the module. - /// - /// If this analysis itself is preserved, then we assume that the set of \c - /// Function objects in the \c Module hasn't changed and thus we don't need - /// to invalidate *all* cached data associated with a \c Function* in the \c - /// FunctionAnalysisManager. - /// - /// Regardless of whether this analysis is marked as preserved, all of the - /// analyses in the \c FunctionAnalysisManager are potentially invalidated - /// based on the set of preserved analyses. - bool invalidate(Module &M, const PreservedAnalyses &PA); +template <typename AnalysisManagerT, typename IRUnitT> +char InnerAnalysisManagerProxy<AnalysisManagerT, IRUnitT>::PassID; -private: - FunctionAnalysisManager *FAM; -}; +extern template class InnerAnalysisManagerProxy<FunctionAnalysisManager, + Module>; +/// Provide the \c FunctionAnalysisManager to \c Module proxy. +typedef InnerAnalysisManagerProxy<FunctionAnalysisManager, Module> + FunctionAnalysisManagerModuleProxy; /// \brief A function analysis which acts as a proxy for a module analysis /// manager. @@ -686,60 +781,67 @@ private: /// This proxy *doesn't* manage the invalidation in any way. That is handled by /// the recursive return path of each layer of the pass manager and the /// returned PreservedAnalysis set. -class ModuleAnalysisManagerFunctionProxy { +template <typename AnalysisManagerT, typename IRUnitT> +class OuterAnalysisManagerProxy + : public AnalysisInfoMixin< + OuterAnalysisManagerProxy<AnalysisManagerT, IRUnitT>> { public: - /// \brief Result proxy object for \c ModuleAnalysisManagerFunctionProxy. + /// \brief Result proxy object for \c OuterAnalysisManagerProxy. class Result { public: - explicit Result(const ModuleAnalysisManager &MAM) : MAM(&MAM) {} + explicit Result(const AnalysisManagerT &AM) : AM(&AM) {} // We have to explicitly define all the special member functions because // MSVC refuses to generate them. - Result(const Result &Arg) : MAM(Arg.MAM) {} - Result(Result &&Arg) : MAM(std::move(Arg.MAM)) {} + Result(const Result &Arg) : AM(Arg.AM) {} + Result(Result &&Arg) : AM(std::move(Arg.AM)) {} Result &operator=(Result RHS) { - std::swap(MAM, RHS.MAM); + std::swap(AM, RHS.AM); return *this; } - const ModuleAnalysisManager &getManager() const { return *MAM; } + const AnalysisManagerT &getManager() const { return *AM; } /// \brief Handle invalidation by ignoring it, this pass is immutable. - bool invalidate(Function &) { return false; } + bool invalidate(IRUnitT &) { return false; } private: - const ModuleAnalysisManager *MAM; + const AnalysisManagerT *AM; }; - static void *ID() { return (void *)&PassID; } - - static StringRef name() { return "ModuleAnalysisManagerFunctionProxy"; } - - ModuleAnalysisManagerFunctionProxy(const ModuleAnalysisManager &MAM) - : MAM(&MAM) {} + OuterAnalysisManagerProxy(const AnalysisManagerT &AM) : AM(&AM) {} // We have to explicitly define all the special member functions because MSVC // refuses to generate them. - ModuleAnalysisManagerFunctionProxy( - const ModuleAnalysisManagerFunctionProxy &Arg) - : MAM(Arg.MAM) {} - ModuleAnalysisManagerFunctionProxy(ModuleAnalysisManagerFunctionProxy &&Arg) - : MAM(std::move(Arg.MAM)) {} - ModuleAnalysisManagerFunctionProxy & - operator=(ModuleAnalysisManagerFunctionProxy RHS) { - std::swap(MAM, RHS.MAM); + OuterAnalysisManagerProxy(const OuterAnalysisManagerProxy &Arg) + : AM(Arg.AM) {} + OuterAnalysisManagerProxy(OuterAnalysisManagerProxy &&Arg) + : AM(std::move(Arg.AM)) {} + OuterAnalysisManagerProxy &operator=(OuterAnalysisManagerProxy RHS) { + std::swap(AM, RHS.AM); return *this; } /// \brief Run the analysis pass and create our proxy result object. - /// Nothing to see here, it just forwards the \c MAM reference into the + /// Nothing to see here, it just forwards the \c AM reference into the /// result. - Result run(Function &) { return Result(*MAM); } + Result run(IRUnitT &, AnalysisManager<IRUnitT> &) { return Result(*AM); } private: + friend AnalysisInfoMixin< + OuterAnalysisManagerProxy<AnalysisManagerT, IRUnitT>>; static char PassID; - const ModuleAnalysisManager *MAM; + const AnalysisManagerT *AM; }; +template <typename AnalysisManagerT, typename IRUnitT> +char OuterAnalysisManagerProxy<AnalysisManagerT, IRUnitT>::PassID; + +extern template class OuterAnalysisManagerProxy<ModuleAnalysisManager, + Function>; +/// Provide the \c ModuleAnalysisManager to \c Fucntion proxy. +typedef OuterAnalysisManagerProxy<ModuleAnalysisManager, Function> + ModuleAnalysisManagerFunctionProxy; + /// \brief Trivial adaptor that maps from a module to its functions. /// /// Designed to allow composition of a FunctionPass(Manager) and @@ -762,7 +864,9 @@ private: /// module. /// FIXME: Make the above true for all of LLVM's actual passes, some still /// violate this principle. -template <typename FunctionPassT> class ModuleToFunctionPassAdaptor { +template <typename FunctionPassT> +class ModuleToFunctionPassAdaptor + : public PassInfoMixin<ModuleToFunctionPassAdaptor<FunctionPassT>> { public: explicit ModuleToFunctionPassAdaptor(FunctionPassT Pass) : Pass(std::move(Pass)) {} @@ -783,11 +887,10 @@ public: } /// \brief Runs the function pass across every function in the module. - PreservedAnalyses run(Module &M, ModuleAnalysisManager *AM) { - FunctionAnalysisManager *FAM = nullptr; - if (AM) - // Setup the function analysis manager from its proxy. - FAM = &AM->getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) { + // Setup the function analysis manager from its proxy. + FunctionAnalysisManager &FAM = + AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); PreservedAnalyses PA = PreservedAnalyses::all(); for (Function &F : M) { @@ -801,8 +904,7 @@ public: // directly handle the function analysis manager's invalidation here and // update our preserved set to reflect that these have already been // handled. - if (FAM) - PassPA = FAM->invalidate(F, std::move(PassPA)); + PassPA = FAM.invalidate(F, std::move(PassPA)); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. @@ -817,8 +919,6 @@ public: return PA; } - static StringRef name() { return "ModuleToFunctionPassAdaptor"; } - private: FunctionPassT Pass; }; @@ -835,7 +935,8 @@ createModuleToFunctionPassAdaptor(FunctionPassT Pass) { /// /// This is a no-op pass which simply forces a specific analysis pass's result /// to be available when it is run. -template <typename AnalysisT> struct RequireAnalysisPass { +template <typename AnalysisT> +struct RequireAnalysisPass : PassInfoMixin<RequireAnalysisPass<AnalysisT>> { /// \brief Run this pass over some unit of IR. /// /// This pass can be run over any unit of IR and use any analysis manager @@ -843,14 +944,11 @@ template <typename AnalysisT> struct RequireAnalysisPass { /// created, these methods can be instantiated to satisfy whatever the /// context requires. template <typename IRUnitT> - PreservedAnalyses run(IRUnitT &Arg, AnalysisManager<IRUnitT> *AM) { - if (AM) - (void)AM->template getResult<AnalysisT>(Arg); + PreservedAnalyses run(IRUnitT &Arg, AnalysisManager<IRUnitT> &AM) { + (void)AM.template getResult<AnalysisT>(Arg); return PreservedAnalyses::all(); } - - static StringRef name() { return "RequireAnalysisPass"; } }; /// \brief A template utility pass to force an analysis result to be @@ -858,7 +956,9 @@ template <typename AnalysisT> struct RequireAnalysisPass { /// /// This is a no-op pass which simply forces a specific analysis result to be /// invalidated when it is run. -template <typename AnalysisT> struct InvalidateAnalysisPass { +template <typename AnalysisT> +struct InvalidateAnalysisPass + : PassInfoMixin<InvalidateAnalysisPass<AnalysisT>> { /// \brief Run this pass over some unit of IR. /// /// This pass can be run over any unit of IR and use any analysis manager @@ -866,29 +966,25 @@ template <typename AnalysisT> struct InvalidateAnalysisPass { /// created, these methods can be instantiated to satisfy whatever the /// context requires. template <typename IRUnitT> - PreservedAnalyses run(IRUnitT &Arg, AnalysisManager<IRUnitT> *AM) { - if (AM) - // We have to directly invalidate the analysis result as we can't - // enumerate all other analyses and use the preserved set to control it. - (void)AM->template invalidate<AnalysisT>(Arg); + PreservedAnalyses run(IRUnitT &Arg, AnalysisManager<IRUnitT> &AM) { + // We have to directly invalidate the analysis result as we can't + // enumerate all other analyses and use the preserved set to control it. + AM.template invalidate<AnalysisT>(Arg); return PreservedAnalyses::all(); } - - static StringRef name() { return "InvalidateAnalysisPass"; } }; /// \brief A utility pass that does nothing but preserves no analyses. /// /// As a consequence fo not preserving any analyses, this pass will force all /// analysis passes to be re-run to produce fresh results if any are needed. -struct InvalidateAllAnalysesPass { +struct InvalidateAllAnalysesPass : PassInfoMixin<InvalidateAllAnalysesPass> { /// \brief Run this pass over some unit of IR. - template <typename IRUnitT> PreservedAnalyses run(IRUnitT &Arg) { + template <typename IRUnitT> + PreservedAnalyses run(IRUnitT &, AnalysisManager<IRUnitT> &) { return PreservedAnalyses::none(); } - - static StringRef name() { return "InvalidateAllAnalysesPass"; } }; } diff --git a/include/llvm/IR/PassManagerInternal.h b/include/llvm/IR/PassManagerInternal.h index 92de10bcd75b..4351b5888283 100644 --- a/include/llvm/IR/PassManagerInternal.h +++ b/include/llvm/IR/PassManagerInternal.h @@ -18,8 +18,8 @@ #ifndef LLVM_IR_PASSMANAGERINTERNAL_H #define LLVM_IR_PASSMANAGERINTERNAL_H -#include "llvm/ADT/StringRef.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" namespace llvm { @@ -40,48 +40,20 @@ template <typename IRUnitT> struct PassConcept { /// Note that actual pass object can omit the analysis manager argument if /// desired. Also that the analysis manager may be null if there is no /// analysis manager in the pass pipeline. - virtual PreservedAnalyses run(IRUnitT &IR, AnalysisManager<IRUnitT> *AM) = 0; + virtual PreservedAnalyses run(IRUnitT &IR, AnalysisManager<IRUnitT> &AM) = 0; /// \brief Polymorphic method to access the name of a pass. virtual StringRef name() = 0; }; -/// \brief SFINAE metafunction for computing whether \c PassT has a run method -/// accepting an \c AnalysisManager<IRUnitT>. -template <typename IRUnitT, typename PassT, typename ResultT> -class PassRunAcceptsAnalysisManager { - typedef char SmallType; - struct BigType { - char a, b; - }; - - template <typename T, ResultT (T::*)(IRUnitT &, AnalysisManager<IRUnitT> *)> - struct Checker; - - template <typename T> static SmallType f(Checker<T, &T::run> *); - template <typename T> static BigType f(...); - -public: - enum { Value = sizeof(f<PassT>(nullptr)) == sizeof(SmallType) }; -}; - /// \brief A template wrapper used to implement the polymorphic API. /// /// Can be instantiated for any object which provides a \c run method accepting -/// an \c IRUnitT. It requires the pass to be a copyable object. When the -/// \c run method also accepts an \c AnalysisManager<IRUnitT>*, we pass it -/// along. +/// an \c IRUnitT& and an \c AnalysisManager<IRUnit>&. It requires the pass to +/// be a copyable object. When the template <typename IRUnitT, typename PassT, - typename PreservedAnalysesT = PreservedAnalyses, - bool AcceptsAnalysisManager = PassRunAcceptsAnalysisManager< - IRUnitT, PassT, PreservedAnalysesT>::Value> -struct PassModel; - -/// \brief Specialization of \c PassModel for passes that accept an analyis -/// manager. -template <typename IRUnitT, typename PassT, typename PreservedAnalysesT> -struct PassModel<IRUnitT, PassT, PreservedAnalysesT, true> - : PassConcept<IRUnitT> { + typename PreservedAnalysesT = PreservedAnalyses> +struct PassModel : PassConcept<IRUnitT> { explicit PassModel(PassT Pass) : Pass(std::move(Pass)) {} // We have to explicitly define all the special member functions because MSVC // refuses to generate them. @@ -96,39 +68,13 @@ struct PassModel<IRUnitT, PassT, PreservedAnalysesT, true> return *this; } - PreservedAnalysesT run(IRUnitT &IR, AnalysisManager<IRUnitT> *AM) override { + PreservedAnalysesT run(IRUnitT &IR, AnalysisManager<IRUnitT> &AM) override { return Pass.run(IR, AM); } StringRef name() override { return PassT::name(); } PassT Pass; }; -/// \brief Specialization of \c PassModel for passes that accept an analyis -/// manager. -template <typename IRUnitT, typename PassT, typename PreservedAnalysesT> -struct PassModel<IRUnitT, PassT, PreservedAnalysesT, false> - : PassConcept<IRUnitT> { - explicit PassModel(PassT Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - PassModel(const PassModel &Arg) : Pass(Arg.Pass) {} - PassModel(PassModel &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(PassModel &LHS, PassModel &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - } - PassModel &operator=(PassModel RHS) { - swap(*this, RHS); - return *this; - } - - PreservedAnalysesT run(IRUnitT &IR, AnalysisManager<IRUnitT> *AM) override { - return Pass.run(IR); - } - StringRef name() override { return PassT::name(); } - PassT Pass; -}; - /// \brief Abstract concept of an analysis result. /// /// This concept is parameterized over the IR unit that this result pertains @@ -252,7 +198,7 @@ template <typename IRUnitT> struct AnalysisPassConcept { /// \returns A unique_ptr to the analysis result object to be queried by /// users. virtual std::unique_ptr<AnalysisResultConcept<IRUnitT>> - run(IRUnitT &IR, AnalysisManager<IRUnitT> *AM) = 0; + run(IRUnitT &IR, AnalysisManager<IRUnitT> &AM) = 0; /// \brief Polymorphic method to access the name of a pass. virtual StringRef name() = 0; @@ -261,17 +207,10 @@ template <typename IRUnitT> struct AnalysisPassConcept { /// \brief Wrapper to model the analysis pass concept. /// /// Can wrap any type which implements a suitable \c run method. The method -/// must accept the IRUnitT as an argument and produce an object which can be -/// wrapped in a \c AnalysisResultModel. -template <typename IRUnitT, typename PassT, - bool AcceptsAnalysisManager = PassRunAcceptsAnalysisManager< - IRUnitT, PassT, typename PassT::Result>::Value> -struct AnalysisPassModel; - -/// \brief Specialization of \c AnalysisPassModel which passes an -/// \c AnalysisManager to PassT's run method. +/// must accept an \c IRUnitT& and an \c AnalysisManager<IRUnitT>& as arguments +/// and produce an object which can be wrapped in a \c AnalysisResultModel. template <typename IRUnitT, typename PassT> -struct AnalysisPassModel<IRUnitT, PassT, true> : AnalysisPassConcept<IRUnitT> { +struct AnalysisPassModel : AnalysisPassConcept<IRUnitT> { explicit AnalysisPassModel(PassT Pass) : Pass(std::move(Pass)) {} // We have to explicitly define all the special member functions because MSVC // refuses to generate them. @@ -294,7 +233,7 @@ struct AnalysisPassModel<IRUnitT, PassT, true> : AnalysisPassConcept<IRUnitT> { /// /// The return is wrapped in an \c AnalysisResultModel. std::unique_ptr<AnalysisResultConcept<IRUnitT>> - run(IRUnitT &IR, AnalysisManager<IRUnitT> *AM) override { + run(IRUnitT &IR, AnalysisManager<IRUnitT> &AM) override { return make_unique<ResultModelT>(Pass.run(IR, AM)); } @@ -306,44 +245,6 @@ struct AnalysisPassModel<IRUnitT, PassT, true> : AnalysisPassConcept<IRUnitT> { PassT Pass; }; -/// \brief Specialization of \c AnalysisPassModel which does not pass an -/// \c AnalysisManager to PassT's run method. -template <typename IRUnitT, typename PassT> -struct AnalysisPassModel<IRUnitT, PassT, false> : AnalysisPassConcept<IRUnitT> { - explicit AnalysisPassModel(PassT Pass) : Pass(std::move(Pass)) {} - // We have to explicitly define all the special member functions because MSVC - // refuses to generate them. - AnalysisPassModel(const AnalysisPassModel &Arg) : Pass(Arg.Pass) {} - AnalysisPassModel(AnalysisPassModel &&Arg) : Pass(std::move(Arg.Pass)) {} - friend void swap(AnalysisPassModel &LHS, AnalysisPassModel &RHS) { - using std::swap; - swap(LHS.Pass, RHS.Pass); - } - AnalysisPassModel &operator=(AnalysisPassModel RHS) { - swap(*this, RHS); - return *this; - } - - // FIXME: Replace PassT::Result with type traits when we use C++11. - typedef AnalysisResultModel<IRUnitT, PassT, typename PassT::Result> - ResultModelT; - - /// \brief The model delegates to the \c PassT::run method. - /// - /// The return is wrapped in an \c AnalysisResultModel. - std::unique_ptr<AnalysisResultConcept<IRUnitT>> - run(IRUnitT &IR, AnalysisManager<IRUnitT> *) override { - return make_unique<ResultModelT>(Pass.run(IR)); - } - - /// \brief The model delegates to a static \c PassT::name method. - /// - /// The returned string ref must point to constant immutable data! - StringRef name() override { return PassT::name(); } - - PassT Pass; -}; - } // End namespace detail } diff --git a/include/llvm/IR/PatternMatch.h b/include/llvm/IR/PatternMatch.h index f4d7d8c44416..7da9afcf9463 100644 --- a/include/llvm/IR/PatternMatch.h +++ b/include/llvm/IR/PatternMatch.h @@ -1312,6 +1312,43 @@ template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { return Signum_match<Val_t>(V); } +//===----------------------------------------------------------------------===// +// Matchers for two-operands operators with the operators in either order +// + +/// \brief Matches an ICmp with a predicate over LHS and RHS in either order. +/// Does not swap the predicate. +template<typename LHS, typename RHS> +inline match_combine_or<CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>, + CmpClass_match<RHS, LHS, ICmpInst, ICmpInst::Predicate>> +m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) { + return m_CombineOr(m_ICmp(Pred, L, R), m_ICmp(Pred, R, L)); +} + +/// \brief Matches an And with LHS and RHS in either order. +template<typename LHS, typename RHS> +inline match_combine_or<BinaryOp_match<LHS, RHS, Instruction::And>, + BinaryOp_match<RHS, LHS, Instruction::And>> +m_c_And(const LHS &L, const RHS &R) { + return m_CombineOr(m_And(L, R), m_And(R, L)); +} + +/// \brief Matches an Or with LHS and RHS in either order. +template<typename LHS, typename RHS> +inline match_combine_or<BinaryOp_match<LHS, RHS, Instruction::Or>, + BinaryOp_match<RHS, LHS, Instruction::Or>> +m_c_Or(const LHS &L, const RHS &R) { + return m_CombineOr(m_Or(L, R), m_Or(R, L)); +} + +/// \brief Matches an Xor with LHS and RHS in either order. +template<typename LHS, typename RHS> +inline match_combine_or<BinaryOp_match<LHS, RHS, Instruction::Xor>, + BinaryOp_match<RHS, LHS, Instruction::Xor>> +m_c_Xor(const LHS &L, const RHS &R) { + return m_CombineOr(m_Xor(L, R), m_Xor(R, L)); +} + } // end namespace PatternMatch } // end namespace llvm diff --git a/include/llvm/IR/ProfileSummary.h b/include/llvm/IR/ProfileSummary.h new file mode 100644 index 000000000000..f4248014c6e1 --- /dev/null +++ b/include/llvm/IR/ProfileSummary.h @@ -0,0 +1,85 @@ +//===-- ProfileSummary.h - Profile summary data structure. ------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the profile summary data structure. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_PROFILE_SUMMARY_H +#define LLVM_SUPPORT_PROFILE_SUMMARY_H + +#include <cstdint> +#include <utility> +#include <vector> + +#include "llvm/Support/Casting.h" + +namespace llvm { + +class LLVMContext; +class Metadata; +class MDTuple; +class MDNode; + +// The profile summary is one or more (Cutoff, MinCount, NumCounts) triplets. +// The semantics of counts depend on the type of profile. For instrumentation +// profile, counts are block counts and for sample profile, counts are +// per-line samples. Given a target counts percentile, we compute the minimum +// number of counts needed to reach this target and the minimum among these +// counts. +struct ProfileSummaryEntry { + uint32_t Cutoff; ///< The required percentile of counts. + uint64_t MinCount; ///< The minimum count for this percentile. + uint64_t NumCounts; ///< Number of counts >= the minimum count. + ProfileSummaryEntry(uint32_t TheCutoff, uint64_t TheMinCount, + uint64_t TheNumCounts) + : Cutoff(TheCutoff), MinCount(TheMinCount), NumCounts(TheNumCounts) {} +}; + +typedef std::vector<ProfileSummaryEntry> SummaryEntryVector; + +class ProfileSummary { +public: + enum Kind { PSK_Instr, PSK_Sample }; + +private: + const Kind PSK; + static const char *KindStr[2]; + SummaryEntryVector DetailedSummary; + uint64_t TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount; + uint32_t NumCounts, NumFunctions; + /// \brief Return detailed summary as metadata. + Metadata *getDetailedSummaryMD(LLVMContext &Context); + +public: + static const int Scale = 1000000; + ProfileSummary(Kind K, SummaryEntryVector DetailedSummary, + uint64_t TotalCount, uint64_t MaxCount, + uint64_t MaxInternalCount, uint64_t MaxFunctionCount, + uint32_t NumCounts, uint32_t NumFunctions) + : PSK(K), DetailedSummary(std::move(DetailedSummary)), + TotalCount(TotalCount), MaxCount(MaxCount), + MaxInternalCount(MaxInternalCount), MaxFunctionCount(MaxFunctionCount), + NumCounts(NumCounts), NumFunctions(NumFunctions) {} + Kind getKind() const { return PSK; } + /// \brief Return summary information as metadata. + Metadata *getMD(LLVMContext &Context); + /// \brief Construct profile summary from metdata. + static ProfileSummary *getFromMD(Metadata *MD); + SummaryEntryVector &getDetailedSummary() { return DetailedSummary; } + uint32_t getNumFunctions() { return NumFunctions; } + uint64_t getMaxFunctionCount() { return MaxFunctionCount; } + uint32_t getNumCounts() { return NumCounts; } + uint64_t getTotalCount() { return TotalCount; } + uint64_t getMaxCount() { return MaxCount; } + uint64_t getMaxInternalCount() { return MaxInternalCount; } +}; + +} // end namespace llvm +#endif diff --git a/include/llvm/IR/Statepoint.h b/include/llvm/IR/Statepoint.h index 51a0951a9798..5cd7fe1b576c 100644 --- a/include/llvm/IR/Statepoint.h +++ b/include/llvm/IR/Statepoint.h @@ -8,8 +8,9 @@ //===----------------------------------------------------------------------===// // // This file contains utility functions and a wrapper class analogous to -// CallSite for accessing the fields of gc.statepoint, gc.relocate, and -// gc.result intrinsics +// CallSite for accessing the fields of gc.statepoint, gc.relocate, +// gc.result intrinsics; and some general utilities helpful when dealing with +// gc.statepoint. // //===----------------------------------------------------------------------===// @@ -17,6 +18,7 @@ #define LLVM_IR_STATEPOINT_H #include "llvm/ADT/iterator_range.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" @@ -24,7 +26,6 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/Support/Compiler.h" namespace llvm { /// The statepoint intrinsic accepts a set of flags as its third argument. @@ -38,23 +39,22 @@ enum class StatepointFlags { }; class GCRelocateInst; +class GCResultInst; class ImmutableStatepoint; -bool isStatepoint(const ImmutableCallSite &CS); +bool isStatepoint(ImmutableCallSite CS); bool isStatepoint(const Value *V); bool isStatepoint(const Value &V); -bool isGCRelocate(const ImmutableCallSite &CS); - -bool isGCResult(const Value *V); -bool isGCResult(const ImmutableCallSite &CS); +bool isGCRelocate(ImmutableCallSite CS); +bool isGCResult(ImmutableCallSite CS); /// Analogous to CallSiteBase, this provides most of the actual /// functionality for Statepoint and ImmutableStatepoint. It is /// templatized to allow easily specializing of const and non-const /// concrete subtypes. This is structured analogous to CallSite -/// rather than the IntrinsicInst.h helpers since we want to support -/// invokable statepoints in the near future. +/// rather than the IntrinsicInst.h helpers since we need to support +/// invokable statepoints. template <typename FunTy, typename InstructionTy, typename ValueTy, typename CallSiteTy> class StatepointBase { @@ -252,11 +252,10 @@ public: /// Get the experimental_gc_result call tied to this statepoint. Can be /// nullptr if there isn't a gc_result tied to this statepoint. Guaranteed to /// be a CallInst if non-null. - InstructionTy *getGCResult() const { + const GCResultInst *getGCResult() const { for (auto *U : getInstruction()->users()) - if (isGCResult(U)) - return cast<CallInst>(U); - + if (auto *GRI = dyn_cast<GCResultInst>(U)) + return GRI; return nullptr; } @@ -305,11 +304,13 @@ public: explicit Statepoint(CallSite CS) : Base(CS) {} }; -/// This represents the gc.relocate intrinsic. -class GCRelocateInst : public IntrinsicInst { +/// Common base class for representing values projected from a statepoint. +/// Currently, the only projections available are gc.result and gc.relocate. +class GCProjectionInst : public IntrinsicInst { public: static inline bool classof(const IntrinsicInst *I) { - return I->getIntrinsicID() == Intrinsic::experimental_gc_relocate; + return I->getIntrinsicID() == Intrinsic::experimental_gc_relocate || + I->getIntrinsicID() == Intrinsic::experimental_gc_result; } static inline bool classof(const Value *V) { return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); @@ -330,6 +331,7 @@ public: // This takes care both of relocates for call statepoints and relocates // on normal path of invoke statepoint. if (!isa<LandingPadInst>(Token)) { + assert(isStatepoint(Token)); return cast<Instruction>(Token); } @@ -344,6 +346,17 @@ public: return InvokeBB->getTerminator(); } +}; + +/// Represents calls to the gc.relocate intrinsic. +class GCRelocateInst : public GCProjectionInst { +public: + static inline bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::experimental_gc_relocate; + } + static inline bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } /// The index into the associate statepoint's argument list /// which contains the base pointer of the pointer whose @@ -369,6 +382,17 @@ public: } }; +/// Represents calls to the gc.result intrinsic. +class GCResultInst : public GCProjectionInst { +public: + static inline bool classof(const IntrinsicInst *I) { + return I->getIntrinsicID() == Intrinsic::experimental_gc_result; + } + static inline bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } +}; + template <typename FunTy, typename InstructionTy, typename ValueTy, typename CallSiteTy> std::vector<const GCRelocateInst *> @@ -400,6 +424,26 @@ StatepointBase<FunTy, InstructionTy, ValueTy, CallSiteTy>::getRelocates() } return Result; } + +/// Call sites that get wrapped by a gc.statepoint (currently only in +/// RewriteStatepointsForGC and potentially in other passes in the future) can +/// have attributes that describe properties of gc.statepoint call they will be +/// eventually be wrapped in. This struct is used represent such directives. +struct StatepointDirectives { + Optional<uint32_t> NumPatchBytes; + Optional<uint64_t> StatepointID; + + static const uint64_t DefaultStatepointID = 0xABCDEF00; + static const uint64_t DeoptBundleStatepointID = 0xABCDEF0F; +}; + +/// Parse out statepoint directives from the function attributes present in \p +/// AS. +StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeSet AS); + +/// Return \c true if the the \p Attr is an attribute that is a statepoint +/// directive. +bool isStatepointDirectiveAttr(Attribute Attr); } #endif diff --git a/include/llvm/IR/SymbolTableListTraits.h b/include/llvm/IR/SymbolTableListTraits.h index 5fc48d10d63f..60e04e2f9eca 100644 --- a/include/llvm/IR/SymbolTableListTraits.h +++ b/include/llvm/IR/SymbolTableListTraits.h @@ -49,6 +49,7 @@ class Function; class Instruction; class GlobalVariable; class GlobalAlias; +class GlobalIFunc; class Module; #define DEFINE_SYMBOL_TABLE_PARENT_TYPE(NODE, PARENT) \ template <> struct SymbolTableListParentType<NODE> { typedef PARENT type; }; @@ -58,6 +59,7 @@ DEFINE_SYMBOL_TABLE_PARENT_TYPE(Argument, Function) DEFINE_SYMBOL_TABLE_PARENT_TYPE(Function, Module) DEFINE_SYMBOL_TABLE_PARENT_TYPE(GlobalVariable, Module) DEFINE_SYMBOL_TABLE_PARENT_TYPE(GlobalAlias, Module) +DEFINE_SYMBOL_TABLE_PARENT_TYPE(GlobalIFunc, Module) #undef DEFINE_SYMBOL_TABLE_PARENT_TYPE template <typename NodeTy> class SymbolTableList; diff --git a/include/llvm/IR/TrackingMDRef.h b/include/llvm/IR/TrackingMDRef.h index 97efaff7a377..fe513a8f9795 100644 --- a/include/llvm/IR/TrackingMDRef.h +++ b/include/llvm/IR/TrackingMDRef.h @@ -15,7 +15,6 @@ #define LLVM_IR_TRACKINGMDREF_H #include "llvm/IR/Metadata.h" -#include "llvm/Support/Casting.h" namespace llvm { diff --git a/include/llvm/IR/Type.h b/include/llvm/IR/Type.h index b2920dd3de63..ef7ad733f47a 100644 --- a/include/llvm/IR/Type.h +++ b/include/llvm/IR/Type.h @@ -16,6 +16,7 @@ #define LLVM_IR_TYPE_H #include "llvm/ADT/APFloat.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Casting.h" @@ -75,7 +76,7 @@ public: }; private: - /// Context - This refers to the LLVMContext in which this type was uniqued. + /// This refers to the LLVMContext in which this type was uniqued. LLVMContext &Context; TypeID ID : 8; // The current base type of this type. @@ -96,56 +97,64 @@ protected: assert(getSubclassData() == val && "Subclass data too large for field"); } - /// NumContainedTys - Keeps track of how many Type*'s there are in the - /// ContainedTys list. + /// Keeps track of how many Type*'s there are in the ContainedTys list. unsigned NumContainedTys; - /// ContainedTys - A pointer to the array of Types contained by this Type. - /// For example, this includes the arguments of a function type, the elements - /// of a structure, the pointee of a pointer, the element type of an array, - /// etc. This pointer may be 0 for types that don't contain other types - /// (Integer, Double, Float). + /// A pointer to the array of Types contained by this Type. For example, this + /// includes the arguments of a function type, the elements of a structure, + /// the pointee of a pointer, the element type of an array, etc. This pointer + /// may be 0 for types that don't contain other types (Integer, Double, + /// Float). Type * const *ContainedTys; + static bool isSequentialType(TypeID TyID) { + return TyID == ArrayTyID || TyID == PointerTyID || TyID == VectorTyID; + } + public: - void print(raw_ostream &O, bool IsForDebug = false) const; + /// Print the current type. + /// Omit the type details if \p NoDetails == true. + /// E.g., let %st = type { i32, i16 } + /// When \p NoDetails is true, we only print %st. + /// Put differently, \p NoDetails prints the type as if + /// inlined with the operands when printing an instruction. + void print(raw_ostream &O, bool IsForDebug = false, + bool NoDetails = false) const; void dump() const; - /// getContext - Return the LLVMContext in which this type was uniqued. + /// Return the LLVMContext in which this type was uniqued. LLVMContext &getContext() const { return Context; } //===--------------------------------------------------------------------===// // Accessors for working with types. // - /// getTypeID - Return the type id for the type. This will return one - /// of the TypeID enum elements defined above. - /// + /// Return the type id for the type. This will return one of the TypeID enum + /// elements defined above. TypeID getTypeID() const { return ID; } - /// isVoidTy - Return true if this is 'void'. + /// Return true if this is 'void'. bool isVoidTy() const { return getTypeID() == VoidTyID; } - /// isHalfTy - Return true if this is 'half', a 16-bit IEEE fp type. + /// Return true if this is 'half', a 16-bit IEEE fp type. bool isHalfTy() const { return getTypeID() == HalfTyID; } - /// isFloatTy - Return true if this is 'float', a 32-bit IEEE fp type. + /// Return true if this is 'float', a 32-bit IEEE fp type. bool isFloatTy() const { return getTypeID() == FloatTyID; } - /// isDoubleTy - Return true if this is 'double', a 64-bit IEEE fp type. + /// Return true if this is 'double', a 64-bit IEEE fp type. bool isDoubleTy() const { return getTypeID() == DoubleTyID; } - /// isX86_FP80Ty - Return true if this is x86 long double. + /// Return true if this is x86 long double. bool isX86_FP80Ty() const { return getTypeID() == X86_FP80TyID; } - /// isFP128Ty - Return true if this is 'fp128'. + /// Return true if this is 'fp128'. bool isFP128Ty() const { return getTypeID() == FP128TyID; } - /// isPPC_FP128Ty - Return true if this is powerpc long double. + /// Return true if this is powerpc long double. bool isPPC_FP128Ty() const { return getTypeID() == PPC_FP128TyID; } - /// isFloatingPointTy - Return true if this is one of the six floating point - /// types + /// Return true if this is one of the six floating-point types bool isFloatingPointTy() const { return getTypeID() == HalfTyID || getTypeID() == FloatTyID || getTypeID() == DoubleTyID || @@ -165,99 +174,81 @@ public: } } - /// isX86_MMXTy - Return true if this is X86 MMX. + /// Return true if this is X86 MMX. bool isX86_MMXTy() const { return getTypeID() == X86_MMXTyID; } - /// isFPOrFPVectorTy - Return true if this is a FP type or a vector of FP. - /// + /// Return true if this is a FP type or a vector of FP. bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); } - /// isLabelTy - Return true if this is 'label'. + /// Return true if this is 'label'. bool isLabelTy() const { return getTypeID() == LabelTyID; } - /// isMetadataTy - Return true if this is 'metadata'. + /// Return true if this is 'metadata'. bool isMetadataTy() const { return getTypeID() == MetadataTyID; } - /// isTokenTy - Return true if this is 'token'. + /// Return true if this is 'token'. bool isTokenTy() const { return getTypeID() == TokenTyID; } - /// isIntegerTy - True if this is an instance of IntegerType. - /// + /// True if this is an instance of IntegerType. bool isIntegerTy() const { return getTypeID() == IntegerTyID; } - /// isIntegerTy - Return true if this is an IntegerType of the given width. + /// Return true if this is an IntegerType of the given width. bool isIntegerTy(unsigned Bitwidth) const; - /// isIntOrIntVectorTy - Return true if this is an integer type or a vector of - /// integer types. - /// + /// Return true if this is an integer type or a vector of integer types. bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); } - /// isFunctionTy - True if this is an instance of FunctionType. - /// + /// True if this is an instance of FunctionType. bool isFunctionTy() const { return getTypeID() == FunctionTyID; } - /// isStructTy - True if this is an instance of StructType. - /// + /// True if this is an instance of StructType. bool isStructTy() const { return getTypeID() == StructTyID; } - /// isArrayTy - True if this is an instance of ArrayType. - /// + /// True if this is an instance of ArrayType. bool isArrayTy() const { return getTypeID() == ArrayTyID; } - /// isPointerTy - True if this is an instance of PointerType. - /// + /// True if this is an instance of PointerType. bool isPointerTy() const { return getTypeID() == PointerTyID; } - /// isPtrOrPtrVectorTy - Return true if this is a pointer type or a vector of - /// pointer types. - /// + /// Return true if this is a pointer type or a vector of pointer types. bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); } - /// isVectorTy - True if this is an instance of VectorType. - /// + /// True if this is an instance of VectorType. bool isVectorTy() const { return getTypeID() == VectorTyID; } - /// canLosslesslyBitCastTo - Return true if this type could be converted - /// with a lossless BitCast to type 'Ty'. For example, i8* to i32*. BitCasts - /// are valid for types of the same size only where no re-interpretation of - /// the bits is done. + /// Return true if this type could be converted with a lossless BitCast to + /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the + /// same size only where no re-interpretation of the bits is done. /// @brief Determine if this type could be losslessly bitcast to Ty bool canLosslesslyBitCastTo(Type *Ty) const; - /// isEmptyTy - Return true if this type is empty, that is, it has no - /// elements or all its elements are empty. + /// Return true if this type is empty, that is, it has no elements or all of + /// its elements are empty. bool isEmptyTy() const; - /// isFirstClassType - Return true if the type is "first class", meaning it - /// is a valid type for a Value. - /// + /// Return true if the type is "first class", meaning it is a valid type for a + /// Value. bool isFirstClassType() const { return getTypeID() != FunctionTyID && getTypeID() != VoidTyID; } - /// isSingleValueType - Return true if the type is a valid type for a - /// register in codegen. This includes all first-class types except struct - /// and array types. - /// + /// Return true if the type is a valid type for a register in codegen. This + /// includes all first-class types except struct and array types. bool isSingleValueType() const { return isFloatingPointTy() || isX86_MMXTy() || isIntegerTy() || isPointerTy() || isVectorTy(); } - /// isAggregateType - Return true if the type is an aggregate type. This - /// means it is valid as the first operand of an insertvalue or - /// extractvalue instruction. This includes struct and array types, but - /// does not include vector types. - /// + /// Return true if the type is an aggregate type. This means it is valid as + /// the first operand of an insertvalue or extractvalue instruction. This + /// includes struct and array types, but does not include vector types. bool isAggregateType() const { return getTypeID() == StructTyID || getTypeID() == ArrayTyID; } - /// isSized - Return true if it makes sense to take the size of this type. To - /// get the actual size for a particular target, it is reasonable to use the + /// Return true if it makes sense to take the size of this type. To get the + /// actual size for a particular target, it is reasonable to use the /// DataLayout subsystem to do this. - /// bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const { // If it's a primitive, it is always sized. if (getTypeID() == IntegerTyID || isFloatingPointTy() || @@ -273,8 +264,8 @@ public: return isSizedDerivedType(Visited); } - /// getPrimitiveSizeInBits - Return the basic size of this type if it is a - /// primitive type. These are fixed by LLVM and are not target dependent. + /// Return the basic size of this type if it is a primitive type. These are + /// fixed by LLVM and are not target-dependent. /// This will return zero if the type does not have a size or is not a /// primitive type. /// @@ -285,18 +276,18 @@ public: /// unsigned getPrimitiveSizeInBits() const LLVM_READONLY; - /// getScalarSizeInBits - If this is a vector type, return the - /// getPrimitiveSizeInBits value for the element type. Otherwise return the - /// getPrimitiveSizeInBits value for this type. + /// If this is a vector type, return the getPrimitiveSizeInBits value for the + /// element type. Otherwise return the getPrimitiveSizeInBits value for this + /// type. unsigned getScalarSizeInBits() const LLVM_READONLY; - /// getFPMantissaWidth - Return the width of the mantissa of this type. This - /// is only valid on floating point types. If the FP type does not - /// have a stable mantissa (e.g. ppc long double), this method returns -1. + /// Return the width of the mantissa of this type. This is only valid on + /// floating-point types. If the FP type does not have a stable mantissa (e.g. + /// ppc long double), this method returns -1. int getFPMantissaWidth() const; - /// getScalarType - If this is a vector type, return the element type, - /// otherwise return 'this'. + /// If this is a vector type, return the element type, otherwise return + /// 'this'. Type *getScalarType() const LLVM_READONLY; //===--------------------------------------------------------------------===// @@ -317,17 +308,15 @@ public: return subtype_reverse_iterator(subtype_begin()); } - /// getContainedType - This method is used to implement the type iterator - /// (defined at the end of the file). For derived types, this returns the - /// types 'contained' in the derived type. - /// + /// This method is used to implement the type iterator (defined at the end of + /// the file). For derived types, this returns the types 'contained' in the + /// derived type. Type *getContainedType(unsigned i) const { assert(i < NumContainedTys && "Index out of range!"); return ContainedTys[i]; } - /// getNumContainedTypes - Return the number of types in the derived type. - /// + /// Return the number of types in the derived type. unsigned getNumContainedTypes() const { return NumContainedTys; } //===--------------------------------------------------------------------===// @@ -347,7 +336,10 @@ public: inline unsigned getStructNumElements() const; inline Type *getStructElementType(unsigned N) const; - inline Type *getSequentialElementType() const; + inline Type *getSequentialElementType() const { + assert(isSequentialType(getTypeID()) && "Not a sequential type!"); + return ContainedTys[0]; + } inline uint64_t getArrayNumElements() const; Type *getArrayElementType() const { return getSequentialElementType(); } @@ -357,7 +349,7 @@ public: Type *getPointerElementType() const { return getSequentialElementType(); } - /// \brief Get the address space of this pointer or pointer vector type. + /// Get the address space of this pointer or pointer vector type. inline unsigned getPointerAddressSpace() const; //===--------------------------------------------------------------------===// @@ -365,7 +357,7 @@ public: // instances of Type. // - /// getPrimitiveType - Return a type based on an identifier. + /// Return a type based on an identifier. static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber); //===--------------------------------------------------------------------===// @@ -408,14 +400,14 @@ public: static PointerType *getInt32PtrTy(LLVMContext &C, unsigned AS = 0); static PointerType *getInt64PtrTy(LLVMContext &C, unsigned AS = 0); - /// getPointerTo - Return a pointer to the current type. This is equivalent - /// to PointerType::get(Foo, AddrSpace). + /// Return a pointer to the current type. This is equivalent to + /// PointerType::get(Foo, AddrSpace). PointerType *getPointerTo(unsigned AddrSpace = 0) const; private: - /// isSizedDerivedType - Derived types like structures and arrays are sized - /// iff all of the members of the type are sized as well. Since asking for - /// their size is relatively uncommon, move this operation out of line. + /// Derived types like structures and arrays are sized iff all of the members + /// of the type are sized as well. Since asking for their size is relatively + /// uncommon, move this operation out-of-line. bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const; }; diff --git a/include/llvm/IR/TypeFinder.h b/include/llvm/IR/TypeFinder.h index 5f3854377c16..d5baf7ab0b9e 100644 --- a/include/llvm/IR/TypeFinder.h +++ b/include/llvm/IR/TypeFinder.h @@ -15,6 +15,8 @@ #define LLVM_IR_TYPEFINDER_H #include "llvm/ADT/DenseSet.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Type.h" #include <vector> namespace llvm { @@ -22,7 +24,6 @@ namespace llvm { class MDNode; class Module; class StructType; -class Type; class Value; /// TypeFinder - Walk over a module, identifying all of the types that are diff --git a/include/llvm/IR/Use.h b/include/llvm/IR/Use.h index a738677f8e5b..e62eab56b1f1 100644 --- a/include/llvm/IR/Use.h +++ b/include/llvm/IR/Use.h @@ -27,9 +27,7 @@ #include "llvm/ADT/PointerIntPair.h" #include "llvm/Support/CBindingWrapping.h" -#include "llvm/Support/Compiler.h" #include <cstddef> -#include <iterator> namespace llvm { @@ -101,14 +99,8 @@ public: inline void set(Value *Val); - Value *operator=(Value *RHS) { - set(RHS); - return RHS; - } - const Use &operator=(const Use &RHS) { - set(RHS.Val); - return *this; - } + inline Value *operator=(Value *RHS); + inline const Use &operator=(const Use &RHS); Value *operator->() { return Val; } const Value *operator->() const { return Val; } diff --git a/include/llvm/IR/UseListOrder.h b/include/llvm/IR/UseListOrder.h index 1cabf03d1b00..b86425b6a697 100644 --- a/include/llvm/IR/UseListOrder.h +++ b/include/llvm/IR/UseListOrder.h @@ -15,8 +15,7 @@ #ifndef LLVM_IR_USELISTORDER_H #define LLVM_IR_USELISTORDER_H -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallVector.h" +#include <cstddef> #include <vector> namespace llvm { diff --git a/include/llvm/IR/User.h b/include/llvm/IR/User.h index 885ae197d228..4d6b30cd1124 100644 --- a/include/llvm/IR/User.h +++ b/include/llvm/IR/User.h @@ -19,7 +19,6 @@ #ifndef LLVM_IR_USER_H #define LLVM_IR_USER_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Value.h" @@ -28,6 +27,9 @@ namespace llvm { +template <typename T> class ArrayRef; +template <typename T> class MutableArrayRef; + /// \brief Compile-time customization of User operands. /// /// Customizes operand-related allocators and accessors. diff --git a/include/llvm/IR/Value.def b/include/llvm/IR/Value.def index 4c5d452fc3c3..48842d7f9cd5 100644 --- a/include/llvm/IR/Value.def +++ b/include/llvm/IR/Value.def @@ -54,21 +54,29 @@ HANDLE_VALUE(Argument) HANDLE_VALUE(BasicBlock) +HANDLE_VALUE(MemoryUse) +HANDLE_VALUE(MemoryDef) +HANDLE_VALUE(MemoryPhi) HANDLE_GLOBAL_VALUE(Function) HANDLE_GLOBAL_VALUE(GlobalAlias) +HANDLE_GLOBAL_VALUE(GlobalIFunc) HANDLE_GLOBAL_VALUE(GlobalVariable) -HANDLE_CONSTANT(UndefValue) HANDLE_CONSTANT(BlockAddress) HANDLE_CONSTANT(ConstantExpr) + +// ConstantAggregate. +HANDLE_CONSTANT(ConstantArray) +HANDLE_CONSTANT(ConstantStruct) +HANDLE_CONSTANT(ConstantVector) + +// ConstantData. +HANDLE_CONSTANT(UndefValue) HANDLE_CONSTANT(ConstantAggregateZero) HANDLE_CONSTANT(ConstantDataArray) HANDLE_CONSTANT(ConstantDataVector) HANDLE_CONSTANT(ConstantInt) HANDLE_CONSTANT(ConstantFP) -HANDLE_CONSTANT(ConstantArray) -HANDLE_CONSTANT(ConstantStruct) -HANDLE_CONSTANT(ConstantVector) HANDLE_CONSTANT(ConstantPointerNull) HANDLE_CONSTANT(ConstantTokenNone) @@ -81,6 +89,10 @@ HANDLE_INSTRUCTION(Instruction) HANDLE_CONSTANT_MARKER(ConstantFirstVal, Function) HANDLE_CONSTANT_MARKER(ConstantLastVal, ConstantTokenNone) +HANDLE_CONSTANT_MARKER(ConstantDataFirstVal, UndefValue) +HANDLE_CONSTANT_MARKER(ConstantDataLastVal, ConstantTokenNone) +HANDLE_CONSTANT_MARKER(ConstantAggregateFirstVal, ConstantArray) +HANDLE_CONSTANT_MARKER(ConstantAggregateLastVal, ConstantVector) #undef HANDLE_GLOBAL_VALUE #undef HANDLE_CONSTANT diff --git a/include/llvm/IR/Value.h b/include/llvm/IR/Value.h index 8918dcd38c93..f3a342dadf73 100644 --- a/include/llvm/IR/Value.h +++ b/include/llvm/IR/Value.h @@ -18,7 +18,6 @@ #include "llvm/IR/Use.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/Casting.h" -#include "llvm/Support/Compiler.h" namespace llvm { @@ -27,9 +26,13 @@ class Argument; class AssemblyAnnotationWriter; class BasicBlock; class Constant; +class ConstantData; +class ConstantAggregate; class DataLayout; class Function; class GlobalAlias; +class GlobalIFunc; +class GlobalIndirectSymbol; class GlobalObject; class GlobalValue; class GlobalVariable; @@ -106,10 +109,11 @@ protected: enum : unsigned { NumUserOperandsBits = 28 }; unsigned NumUserOperands : NumUserOperandsBits; - bool IsUsedByMD : 1; - bool HasName : 1; - bool HasHungOffUses : 1; - bool HasDescriptor : 1; + // Use the same type as the bitfield above so that MSVC will pack them. + unsigned IsUsedByMD : 1; + unsigned HasName : 1; + unsigned HasHungOffUses : 1; + unsigned HasDescriptor : 1; private: template <typename UseT> // UseT == 'Use' or 'const Use' @@ -347,13 +351,19 @@ public: assertModuleIsMaterialized(); return *materialized_user_begin(); } + iterator_range<user_iterator> materialized_users() { + return make_range(materialized_user_begin(), user_end()); + } + iterator_range<const_user_iterator> materialized_users() const { + return make_range(materialized_user_begin(), user_end()); + } iterator_range<user_iterator> users() { assertModuleIsMaterialized(); - return make_range(materialized_user_begin(), user_end()); + return materialized_users(); } iterator_range<const_user_iterator> users() const { assertModuleIsMaterialized(); - return make_range(materialized_user_begin(), user_end()); + return materialized_users(); } /// \brief Return true if there is exactly one user of this value. @@ -494,6 +504,20 @@ public: return const_cast<Value*>(this)->stripInBoundsOffsets(); } + /// \brief Returns the number of bytes known to be dereferenceable for the + /// pointer value. + /// + /// If CanBeNull is set by this function the pointer can either be null or be + /// dereferenceable up to the returned number of bytes. + unsigned getPointerDereferenceableBytes(const DataLayout &DL, + bool &CanBeNull) const; + + /// \brief Returns an alignment of the pointer value. + /// + /// Returns an alignment which is either specified explicitly, e.g. via + /// align attribute of a function argument, or guaranteed by DataLayout. + unsigned getPointerAlignment(const DataLayout &DL) const; + /// \brief Translate PHI node to its predecessor from the given basic block. /// /// If this value is a PHI node with CurBB as its parent, return the value in @@ -592,6 +616,16 @@ void Use::set(Value *V) { if (V) V->addUse(*this); } +Value *Use::operator=(Value *RHS) { + set(RHS); + return RHS; +} + +const Use &Use::operator=(const Use &RHS) { + set(RHS.Val); + return *this; +} + template <class Compare> void Value::sortUseList(Compare Cmp) { if (!UseList || !UseList->Next) // No need to sort 0 or 1 uses. @@ -669,6 +703,20 @@ template <> struct isa_impl<Constant, Value> { } }; +template <> struct isa_impl<ConstantData, Value> { + static inline bool doit(const Value &Val) { + return Val.getValueID() >= Value::ConstantDataFirstVal && + Val.getValueID() <= Value::ConstantDataLastVal; + } +}; + +template <> struct isa_impl<ConstantAggregate, Value> { + static inline bool doit(const Value &Val) { + return Val.getValueID() >= Value::ConstantAggregateFirstVal && + Val.getValueID() <= Value::ConstantAggregateLastVal; + } +}; + template <> struct isa_impl<Argument, Value> { static inline bool doit (const Value &Val) { return Val.getValueID() == Value::ArgumentVal; @@ -711,9 +759,21 @@ template <> struct isa_impl<GlobalAlias, Value> { } }; +template <> struct isa_impl<GlobalIFunc, Value> { + static inline bool doit(const Value &Val) { + return Val.getValueID() == Value::GlobalIFuncVal; + } +}; + +template <> struct isa_impl<GlobalIndirectSymbol, Value> { + static inline bool doit(const Value &Val) { + return isa<GlobalAlias>(Val) || isa<GlobalIFunc>(Val); + } +}; + template <> struct isa_impl<GlobalValue, Value> { static inline bool doit(const Value &Val) { - return isa<GlobalObject>(Val) || isa<GlobalAlias>(Val); + return isa<GlobalObject>(Val) || isa<GlobalIndirectSymbol>(Val); } }; @@ -738,8 +798,7 @@ public: // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_ISA_CONVERSION_FUNCTIONS(Value, LLVMValueRef) -/* Specialized opaque value conversions. - */ +// Specialized opaque value conversions. inline Value **unwrap(LLVMValueRef *Vals) { return reinterpret_cast<Value**>(Vals); } diff --git a/include/llvm/IR/ValueMap.h b/include/llvm/IR/ValueMap.h index ad518ac053b2..85379ad468c4 100644 --- a/include/llvm/IR/ValueMap.h +++ b/include/llvm/IR/ValueMap.h @@ -27,6 +27,7 @@ #define LLVM_IR_VALUEMAP_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/TrackingMDRef.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Mutex.h" @@ -84,8 +85,11 @@ class ValueMap { typedef DenseMap<const Metadata *, TrackingMDRef> MDMapT; typedef typename Config::ExtraData ExtraData; MapT Map; - std::unique_ptr<MDMapT> MDMap; + Optional<MDMapT> MDMap; ExtraData Data; + + bool MayMapMetadata = true; + ValueMap(const ValueMap&) = delete; ValueMap& operator=(const ValueMap&) = delete; public: @@ -99,12 +103,27 @@ public: explicit ValueMap(const ExtraData &Data, unsigned NumInitBuckets = 64) : Map(NumInitBuckets), Data(Data) {} - bool hasMD() const { return MDMap; } + bool hasMD() const { return bool(MDMap); } MDMapT &MD() { if (!MDMap) - MDMap.reset(new MDMapT); + MDMap.emplace(); return *MDMap; } + Optional<MDMapT> &getMDMap() { return MDMap; } + + bool mayMapMetadata() const { return MayMapMetadata; } + void enableMapMetadata() { MayMapMetadata = true; } + void disableMapMetadata() { MayMapMetadata = false; } + + /// Get the mapped metadata, if it's in the map. + Optional<Metadata *> getMappedMD(const Metadata *MD) const { + if (!MDMap) + return None; + auto Where = MDMap->find(MD); + if (Where == MDMap->end()) + return None; + return Where->second.get(); + } typedef ValueMapIterator<MapT, KeyT> iterator; typedef ValueMapConstIterator<MapT, KeyT> const_iterator; diff --git a/include/llvm/IR/ValueSymbolTable.h b/include/llvm/IR/ValueSymbolTable.h index 65bd7fc2fec1..61a12db403ea 100644 --- a/include/llvm/IR/ValueSymbolTable.h +++ b/include/llvm/IR/ValueSymbolTable.h @@ -14,13 +14,13 @@ #ifndef LLVM_IR_VALUESYMBOLTABLE_H #define LLVM_IR_VALUESYMBOLTABLE_H -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/Value.h" #include "llvm/Support/DataTypes.h" namespace llvm { template <typename ValueSubClass> class SymbolTableListTraits; + template <unsigned InternalLen> class SmallString; class BasicBlock; class Function; class NamedMDNode; @@ -39,6 +39,7 @@ class ValueSymbolTable { friend class SymbolTableListTraits<Function>; friend class SymbolTableListTraits<GlobalVariable>; friend class SymbolTableListTraits<GlobalAlias>; + friend class SymbolTableListTraits<GlobalIFunc>; /// @name Types /// @{ public: diff --git a/include/llvm/IR/Verifier.h b/include/llvm/IR/Verifier.h index 89039d24195e..fdb6ce400a8d 100644 --- a/include/llvm/IR/Verifier.h +++ b/include/llvm/IR/Verifier.h @@ -21,8 +21,7 @@ #ifndef LLVM_IR_VERIFIER_H #define LLVM_IR_VERIFIER_H -#include "llvm/ADT/StringRef.h" -#include <string> +#include "llvm/IR/PassManager.h" namespace llvm { @@ -30,7 +29,6 @@ class Function; class FunctionPass; class ModulePass; class Module; -class PreservedAnalyses; class raw_ostream; /// \brief Check a function for errors, useful for use when debugging a @@ -43,10 +41,38 @@ bool verifyFunction(const Function &F, raw_ostream *OS = nullptr); /// \brief Check a module for errors. /// -/// If there are no errors, the function returns false. If an error is found, -/// a message describing the error is written to OS (if non-null) and true is -/// returned. -bool verifyModule(const Module &M, raw_ostream *OS = nullptr); +/// If there are no errors, the function returns false. If an error is +/// found, a message describing the error is written to OS (if +/// non-null) and true is returned. +/// +/// \return true if the module is broken. If BrokenDebugInfo is +/// supplied, DebugInfo verification failures won't be considered as +/// error and instead *BrokenDebugInfo will be set to true. Debug +/// info errors can be "recovered" from by stripping the debug info. +bool verifyModule(const Module &M, raw_ostream *OS = nullptr, + bool *BrokenDebugInfo = nullptr); + +FunctionPass *createVerifierPass(bool FatalErrors = true); + +/// Check a module for errors, and report separate error states for IR +/// and debug info errors. +class VerifierAnalysis : public AnalysisInfoMixin<VerifierAnalysis> { + friend AnalysisInfoMixin<VerifierAnalysis>; + static char PassID; + +public: + struct Result { + bool IRBroken, DebugInfoBroken; + }; + static void *ID() { return (void *)&PassID; } + Result run(Module &M, ModuleAnalysisManager &); + Result run(Function &F, FunctionAnalysisManager &); +}; + +/// Check a module for errors, but report debug info errors separately. +/// Otherwise behaves as the normal verifyModule. Debug info errors can be +/// "recovered" from by stripping the debug info. +bool verifyModule(bool &BrokenDebugInfo, const Module &M, raw_ostream *OS); /// \brief Create a verifier pass. /// @@ -58,20 +84,17 @@ bool verifyModule(const Module &M, raw_ostream *OS = nullptr); /// /// Note that this creates a pass suitable for the legacy pass manager. It has /// nothing to do with \c VerifierPass. -FunctionPass *createVerifierPass(bool FatalErrors = true); - -class VerifierPass { +class VerifierPass : public PassInfoMixin<VerifierPass> { bool FatalErrors; public: explicit VerifierPass(bool FatalErrors = true) : FatalErrors(FatalErrors) {} - PreservedAnalyses run(Module &M); - PreservedAnalyses run(Function &F); - - static StringRef name() { return "VerifierPass"; } + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + } // End llvm namespace #endif |