diff options
Diffstat (limited to 'llvm/include/llvm')
382 files changed, 10561 insertions, 3243 deletions
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h index 1c4969733791..ed25b2cd89f1 100644 --- a/llvm/include/llvm/ADT/APFloat.h +++ b/llvm/include/llvm/ADT/APFloat.h @@ -38,6 +38,7 @@ class StringRef; class APFloat; class raw_ostream; +template <typename T> class Expected; template <typename T> class SmallVectorImpl; /// Enum that represents what fraction of the LSB truncated bits of an fp number @@ -143,7 +144,7 @@ struct APFloatBase { static const unsigned integerPartWidth = APInt::APINT_BITS_PER_WORD; /// A signed type to represent a floating point numbers unbiased exponent. - typedef signed short ExponentType; + typedef int32_t ExponentType; /// \name Floating Point Semantics. /// @{ @@ -299,7 +300,7 @@ public: bool, roundingMode); opStatus convertFromZeroExtendedInteger(const integerPart *, unsigned int, bool, roundingMode); - opStatus convertFromString(StringRef, roundingMode); + Expected<opStatus> convertFromString(StringRef, roundingMode); APInt bitcastToAPInt() const; double convertToDouble() const; float convertToFloat() const; @@ -486,7 +487,8 @@ private: integerPart addSignificand(const IEEEFloat &); integerPart subtractSignificand(const IEEEFloat &, integerPart); lostFraction addOrSubtractSignificand(const IEEEFloat &, bool subtract); - lostFraction multiplySignificand(const IEEEFloat &, const IEEEFloat *); + lostFraction multiplySignificand(const IEEEFloat &, IEEEFloat); + lostFraction multiplySignificand(const IEEEFloat&); lostFraction divideSignificand(const IEEEFloat &); void incrementSignificand(); void initialize(const fltSemantics *); @@ -525,8 +527,8 @@ private: bool *) const; opStatus convertFromUnsignedParts(const integerPart *, unsigned int, roundingMode); - opStatus convertFromHexadecimalString(StringRef, roundingMode); - opStatus convertFromDecimalString(StringRef, roundingMode); + Expected<opStatus> convertFromHexadecimalString(StringRef, roundingMode); + Expected<opStatus> convertFromDecimalString(StringRef, roundingMode); char *convertNormalToHexString(char *, unsigned int, bool, roundingMode) const; opStatus roundSignificandWithExponent(const integerPart *, unsigned int, int, @@ -648,7 +650,7 @@ public: cmpResult compare(const DoubleAPFloat &RHS) const; bool bitwiseIsEqual(const DoubleAPFloat &RHS) const; APInt bitcastToAPInt() const; - opStatus convertFromString(StringRef, roundingMode); + Expected<opStatus> convertFromString(StringRef, roundingMode); opStatus next(bool nextDown); opStatus convertToInteger(MutableArrayRef<integerPart> Input, @@ -851,6 +853,9 @@ public: APFloat(const fltSemantics &Semantics) : U(Semantics) {} APFloat(const fltSemantics &Semantics, StringRef S); APFloat(const fltSemantics &Semantics, integerPart I) : U(Semantics, I) {} + template <typename T, typename = typename std::enable_if< + std::is_floating_point<T>::value>::type> + APFloat(const fltSemantics &Semantics, T V) = delete; // TODO: Remove this constructor. This isn't faster than the first one. APFloat(const fltSemantics &Semantics, uninitializedTag) : U(Semantics, uninitialized) {} @@ -1105,7 +1110,7 @@ public: APFLOAT_DISPATCH_ON_SEMANTICS( convertFromZeroExtendedInteger(Input, InputSize, IsSigned, RM)); } - opStatus convertFromString(StringRef, roundingMode); + Expected<opStatus> convertFromString(StringRef, roundingMode); APInt bitcastToAPInt() const { APFLOAT_DISPATCH_ON_SEMANTICS(bitcastToAPInt()); } diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h index 8dce5a621bb3..0791a6d686a3 100644 --- a/llvm/include/llvm/ADT/APInt.h +++ b/llvm/include/llvm/ADT/APInt.h @@ -389,6 +389,11 @@ public: /// \returns true if this APInt is positive. bool isStrictlyPositive() const { return isNonNegative() && !isNullValue(); } + /// Determine if this APInt Value is non-positive (<= 0). + /// + /// \returns true if this APInt is non-positive. + bool isNonPositive() const { return !isStrictlyPositive(); } + /// Determine if all bits are set /// /// This checks to see if the value has all bits of the APInt are set or not. @@ -595,8 +600,8 @@ public: /// Constructs an APInt value that has a contiguous range of bits set. The /// bits from loBit (inclusive) to hiBit (exclusive) will be set. All other /// bits will be zero. For example, with parameters(32, 0, 16) you would get - /// 0x0000FFFF. If hiBit is less than loBit then the set bits "wrap". For - /// example, with parameters (32, 28, 4), you would get 0xF000000F. + /// 0x0000FFFF. Please call getBitsSetWithWrap if \p loBit may be greater than + /// \p hiBit. /// /// \param numBits the intended bit width of the result /// \param loBit the index of the lowest bit set. @@ -604,11 +609,23 @@ public: /// /// \returns An APInt value with the requested bits set. static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit) { + assert(loBit <= hiBit && "loBit greater than hiBit"); APInt Res(numBits, 0); Res.setBits(loBit, hiBit); return Res; } + /// Wrap version of getBitsSet. + /// If \p hiBit is no less than \p loBit, this is same with getBitsSet. + /// If \p hiBit is less than \p loBit, the set bits "wrap". For example, with + /// parameters (32, 28, 4), you would get 0xF000000F. + static APInt getBitsSetWithWrap(unsigned numBits, unsigned loBit, + unsigned hiBit) { + APInt Res(numBits, 0); + Res.setBitsWithWrap(loBit, hiBit); + return Res; + } + /// Get a value with upper bits starting at loBit set. /// /// Constructs an APInt value that has a contiguous range of bits set. The @@ -1109,6 +1126,10 @@ public: APInt uadd_sat(const APInt &RHS) const; APInt ssub_sat(const APInt &RHS) const; APInt usub_sat(const APInt &RHS) const; + APInt smul_sat(const APInt &RHS) const; + APInt umul_sat(const APInt &RHS) const; + APInt sshl_sat(const APInt &RHS) const; + APInt ushl_sat(const APInt &RHS) const; /// Array-indexing support. /// @@ -1245,7 +1266,7 @@ public: /// \returns true if *this <= RHS when considered signed. bool sle(uint64_t RHS) const { return !sgt(RHS); } - /// Unsigned greather than comparison + /// Unsigned greater than comparison /// /// Regards both *this and RHS as unsigned quantities and compares them for /// the validity of the greater-than relationship. @@ -1264,7 +1285,7 @@ public: return (!isSingleWord() && getActiveBits() > 64) || getZExtValue() > RHS; } - /// Signed greather than comparison + /// Signed greater than comparison /// /// Regards both *this and RHS as signed quantities and compares them for the /// validity of the greater-than relationship. @@ -1342,6 +1363,19 @@ public: /// that is greater than or equal to the current width. APInt trunc(unsigned width) const; + /// Truncate to new width with unsigned saturation. + /// + /// If the APInt, treated as unsigned integer, can be losslessly truncated to + /// the new bitwidth, then return truncated APInt. Else, return max value. + APInt truncUSat(unsigned width) const; + + /// Truncate to new width with signed saturation. + /// + /// If this APInt, treated as signed integer, can be losslessly truncated to + /// the new bitwidth, then return truncated APInt. Else, return either + /// signed min value if the APInt was negative, or signed max value. + APInt truncSSat(unsigned width) const; + /// Sign extend to a new width. /// /// This operation sign extends the APInt to a new width. If the high order @@ -1414,6 +1448,21 @@ public: } /// Set the bits from loBit (inclusive) to hiBit (exclusive) to 1. + /// This function handles "wrap" case when \p loBit > \p hiBit, and calls + /// setBits when \p loBit <= \p hiBit. + void setBitsWithWrap(unsigned loBit, unsigned hiBit) { + assert(hiBit <= BitWidth && "hiBit out of range"); + assert(loBit <= BitWidth && "loBit out of range"); + if (loBit <= hiBit) { + setBits(loBit, hiBit); + return; + } + setLowBits(hiBit); + setHighBits(BitWidth - loBit); + } + + /// Set the bits from loBit (inclusive) to hiBit (exclusive) to 1. + /// This function handles case when \p loBit <= \p hiBit. void setBits(unsigned loBit, unsigned hiBit) { assert(hiBit <= BitWidth && "hiBit out of range"); assert(loBit <= BitWidth && "loBit out of range"); @@ -1723,13 +1772,13 @@ public: return BitsToDouble(getWord(0)); } - /// Converts APInt bits to a double + /// Converts APInt bits to a float /// /// The conversion does not do a translation from integer to float, it just /// re-interprets the bits as a float. Note that it is valid to do this on /// any bit width. Exactly 32 bits will be translated. float bitsToFloat() const { - return BitsToFloat(getWord(0)); + return BitsToFloat(static_cast<uint32_t>(getWord(0))); } /// Converts a double to APInt bits. @@ -2158,7 +2207,7 @@ inline float RoundAPIntToFloat(const APInt &APIVal) { /// Converts the given APInt to a float value. /// -/// Treast the APInt as a signed value for conversion purposes. +/// Treats the APInt as a signed value for conversion purposes. inline float RoundSignedAPIntToFloat(const APInt &APIVal) { return float(APIVal.signedRoundToDouble()); } @@ -2194,7 +2243,7 @@ APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM); /// count as an overflow, but here we want to allow values to decrease /// and increase as long as they are within the same interval. /// Specifically, adding of two negative numbers should not cause an -/// overflow (as long as the magnitude does not exceed the bith width). +/// overflow (as long as the magnitude does not exceed the bit width). /// On the other hand, given a positive number, adding a negative /// number to it can give a negative result, which would cause the /// value to go from [-2^BW, 0) to [0, 2^BW). In that sense, zero is @@ -2216,6 +2265,12 @@ APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM); /// coefficients. Optional<APInt> SolveQuadraticEquationWrap(APInt A, APInt B, APInt C, unsigned RangeWidth); + +/// Compare two values, and if they are different, return the position of the +/// most significant bit that is different in the values. +Optional<unsigned> GetMostSignificantDifferentBit(const APInt &A, + const APInt &B); + } // End of APIntOps namespace // See friend declaration above. This additional declaration is required in diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h index f6455d3fa412..3d22442918cd 100644 --- a/llvm/include/llvm/ADT/ArrayRef.h +++ b/llvm/include/llvm/ADT/ArrayRef.h @@ -97,9 +97,19 @@ namespace llvm { /*implicit*/ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {} /// Construct an ArrayRef from a std::initializer_list. +#if LLVM_GNUC_PREREQ(9, 0, 0) +// Disable gcc's warning in this constructor as it generates an enormous amount +// of messages. Anyone using ArrayRef should already be aware of the fact that +// it does not do lifetime extension. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winit-list-lifetime" +#endif /*implicit*/ ArrayRef(const std::initializer_list<T> &Vec) : Data(Vec.begin() == Vec.end() ? (T*)nullptr : Vec.begin()), Length(Vec.size()) {} +#if LLVM_GNUC_PREREQ(9, 0, 0) +#pragma GCC diagnostic pop +#endif /// Construct an ArrayRef<const T*> from ArrayRef<T*>. This uses SFINAE to /// ensure that only ArrayRefs of pointers can be converted. diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h index fabf5d9cd348..5284be8c4a02 100644 --- a/llvm/include/llvm/ADT/BitVector.h +++ b/llvm/include/llvm/ADT/BitVector.h @@ -71,7 +71,7 @@ public: }; class BitVector { - typedef unsigned long BitWord; + typedef uintptr_t BitWord; enum { BITWORD_SIZE = (unsigned)sizeof(BitWord) * CHAR_BIT }; @@ -187,12 +187,12 @@ public: /// all - Returns true if all bits are set. bool all() const { for (unsigned i = 0; i < Size / BITWORD_SIZE; ++i) - if (Bits[i] != ~0UL) + if (Bits[i] != ~BitWord(0)) return false; // If bits remain check that they are ones. The unused bits are always zero. if (unsigned Remainder = Size % BITWORD_SIZE) - return Bits[Size / BITWORD_SIZE] == (1UL << Remainder) - 1; + return Bits[Size / BITWORD_SIZE] == (BitWord(1) << Remainder) - 1; return true; } @@ -285,7 +285,7 @@ public: unsigned LastBit = (End - 1) % BITWORD_SIZE; Copy |= maskTrailingZeros<BitWord>(LastBit + 1); } - if (Copy != ~0UL) { + if (Copy != ~BitWord(0)) { unsigned Result = i * BITWORD_SIZE + countTrailingOnes(Copy); return Result < size() ? Result : -1; } @@ -317,7 +317,7 @@ public: Copy |= maskTrailingOnes<BitWord>(FirstBit); } - if (Copy != ~0UL) { + if (Copy != ~BitWord(0)) { unsigned Result = (CurrentWord + 1) * BITWORD_SIZE - countLeadingOnes(Copy) - 1; return Result < Size ? Result : -1; @@ -414,21 +414,21 @@ public: if (I == E) return *this; if (I / BITWORD_SIZE == E / BITWORD_SIZE) { - BitWord EMask = 1UL << (E % BITWORD_SIZE); - BitWord IMask = 1UL << (I % BITWORD_SIZE); + BitWord EMask = BitWord(1) << (E % BITWORD_SIZE); + BitWord IMask = BitWord(1) << (I % BITWORD_SIZE); BitWord Mask = EMask - IMask; Bits[I / BITWORD_SIZE] |= Mask; return *this; } - BitWord PrefixMask = ~0UL << (I % BITWORD_SIZE); + BitWord PrefixMask = ~BitWord(0) << (I % BITWORD_SIZE); Bits[I / BITWORD_SIZE] |= PrefixMask; I = alignTo(I, BITWORD_SIZE); for (; I + BITWORD_SIZE <= E; I += BITWORD_SIZE) - Bits[I / BITWORD_SIZE] = ~0UL; + Bits[I / BITWORD_SIZE] = ~BitWord(0); - BitWord PostfixMask = (1UL << (E % BITWORD_SIZE)) - 1; + BitWord PostfixMask = (BitWord(1) << (E % BITWORD_SIZE)) - 1; if (I < E) Bits[I / BITWORD_SIZE] |= PostfixMask; @@ -453,21 +453,21 @@ public: if (I == E) return *this; if (I / BITWORD_SIZE == E / BITWORD_SIZE) { - BitWord EMask = 1UL << (E % BITWORD_SIZE); - BitWord IMask = 1UL << (I % BITWORD_SIZE); + BitWord EMask = BitWord(1) << (E % BITWORD_SIZE); + BitWord IMask = BitWord(1) << (I % BITWORD_SIZE); BitWord Mask = EMask - IMask; Bits[I / BITWORD_SIZE] &= ~Mask; return *this; } - BitWord PrefixMask = ~0UL << (I % BITWORD_SIZE); + BitWord PrefixMask = ~BitWord(0) << (I % BITWORD_SIZE); Bits[I / BITWORD_SIZE] &= ~PrefixMask; I = alignTo(I, BITWORD_SIZE); for (; I + BITWORD_SIZE <= E; I += BITWORD_SIZE) - Bits[I / BITWORD_SIZE] = 0UL; + Bits[I / BITWORD_SIZE] = BitWord(0); - BitWord PostfixMask = (1UL << (E % BITWORD_SIZE)) - 1; + BitWord PostfixMask = (BitWord(1) << (E % BITWORD_SIZE)) - 1; if (I < E) Bits[I / BITWORD_SIZE] &= ~PostfixMask; @@ -868,7 +868,7 @@ private: // Then set any stray high bits of the last used word. unsigned ExtraBits = Size % BITWORD_SIZE; if (ExtraBits) { - BitWord ExtraBitMask = ~0UL << ExtraBits; + BitWord ExtraBitMask = ~BitWord(0) << ExtraBits; if (t) Bits[UsedWords-1] |= ExtraBitMask; else diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h index 948a6e6bfb38..148d319c8603 100644 --- a/llvm/include/llvm/ADT/DenseMap.h +++ b/llvm/include/llvm/ADT/DenseMap.h @@ -1006,13 +1006,10 @@ public: } void grow(unsigned AtLeast) { - if (AtLeast >= InlineBuckets) + if (AtLeast > InlineBuckets) AtLeast = std::max<unsigned>(64, NextPowerOf2(AtLeast-1)); if (Small) { - if (AtLeast < InlineBuckets) - return; // Nothing to do. - // First move the inline buckets into a temporary storage. AlignedCharArrayUnion<BucketT[InlineBuckets]> TmpStorage; BucketT *TmpBegin = reinterpret_cast<BucketT *>(TmpStorage.buffer); @@ -1035,10 +1032,13 @@ public: P->getFirst().~KeyT(); } - // Now make this map use the large rep, and move all the entries back - // into it. - Small = false; - new (getLargeRep()) LargeRep(allocateBuckets(AtLeast)); + // AtLeast == InlineBuckets can happen if there are many tombstones, + // and grow() is used to remove them. Usually we always switch to the + // large rep here. + if (AtLeast > InlineBuckets) { + Small = false; + new (getLargeRep()) LargeRep(allocateBuckets(AtLeast)); + } this->moveFromOldBuckets(TmpBegin, TmpEnd); return; } diff --git a/llvm/include/llvm/ADT/DirectedGraph.h b/llvm/include/llvm/ADT/DirectedGraph.h index f6a358d99cd2..cfe98e178a91 100644 --- a/llvm/include/llvm/ADT/DirectedGraph.h +++ b/llvm/include/llvm/ADT/DirectedGraph.h @@ -48,6 +48,9 @@ public: static_cast<const DGEdge<NodeType, EdgeType> &>(*this).getTargetNode()); } + /// Set the target node this edge connects to. + void setTargetNode(const NodeType &N) { TargetNode = N; } + protected: // As the default implementation use address comparison for equality. bool isEqualTo(const EdgeType &E) const { return this == &E; } diff --git a/llvm/include/llvm/ADT/EnumeratedArray.h b/llvm/include/llvm/ADT/EnumeratedArray.h new file mode 100644 index 000000000000..a9528115618c --- /dev/null +++ b/llvm/include/llvm/ADT/EnumeratedArray.h @@ -0,0 +1,48 @@ +//===- llvm/ADT/EnumeratedArray.h - Enumerated Array-------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines an array type that can be indexed using scoped enum values. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_ENUMERATEDARRAY_H +#define LLVM_ADT_ENUMERATEDARRAY_H + +#include <cassert> + +namespace llvm { + +template <typename ValueType, typename Enumeration, + Enumeration LargestEnum = Enumeration::Last, typename IndexType = int, + IndexType Size = 1 + static_cast<IndexType>(LargestEnum)> +class EnumeratedArray { +public: + EnumeratedArray() = default; + EnumeratedArray(ValueType V) { + for (IndexType IX = 0; IX < Size; ++IX) { + Underlying[IX] = V; + } + } + inline const ValueType &operator[](const Enumeration Index) const { + auto IX = static_cast<const IndexType>(Index); + assert(IX >= 0 && IX < Size && "Index is out of bounds."); + return Underlying[IX]; + } + inline ValueType &operator[](const Enumeration Index) { + return const_cast<ValueType &>( + static_cast<const EnumeratedArray<ValueType, Enumeration, LargestEnum, + IndexType, Size> &>(*this)[Index]); + } + +private: + ValueType Underlying[Size]; +}; + +} // namespace llvm + +#endif // LLVM_ADT_ENUMERATEDARRAY_H diff --git a/llvm/include/llvm/ADT/FloatingPointMode.h b/llvm/include/llvm/ADT/FloatingPointMode.h new file mode 100644 index 000000000000..670b2368da9f --- /dev/null +++ b/llvm/include/llvm/ADT/FloatingPointMode.h @@ -0,0 +1,62 @@ +//===- llvm/Support/FloatingPointMode.h -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for dealing with flags related to floating point mode controls. +// +//===----------------------------------------------------------------------===/ + +#ifndef LLVM_FLOATINGPOINTMODE_H +#define LLVM_FLOATINGPOINTMODE_H + +#include "llvm/ADT/StringSwitch.h" + +namespace llvm { + +/// Represent handled modes for denormal (aka subnormal) modes in the floating +/// point environment. +enum class DenormalMode { + Invalid = -1, + + /// IEEE-754 denormal numbers preserved. + IEEE, + + /// The sign of a flushed-to-zero number is preserved in the sign of 0 + PreserveSign, + + /// Denormals are flushed to positive zero. + PositiveZero +}; + +/// Parse the expected names from the denormal-fp-math attribute. +inline DenormalMode parseDenormalFPAttribute(StringRef Str) { + // Assume ieee on unspecified attribute. + return StringSwitch<DenormalMode>(Str) + .Cases("", "ieee", DenormalMode::IEEE) + .Case("preserve-sign", DenormalMode::PreserveSign) + .Case("positive-zero", DenormalMode::PositiveZero) + .Default(DenormalMode::Invalid); +} + +/// Return the name used for the denormal handling mode used by the the +/// expected names from the denormal-fp-math attribute. +inline StringRef denormalModeName(DenormalMode Mode) { + switch (Mode) { + case DenormalMode::IEEE: + return "ieee"; + case DenormalMode::PreserveSign: + return "preserve-sign"; + case DenormalMode::PositiveZero: + return "positive-zero"; + default: + return ""; + } +} + +} + +#endif // LLVM_FLOATINGPOINTMODE_H diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h index d5837e51bcfc..4968b1ea7780 100644 --- a/llvm/include/llvm/ADT/FoldingSet.h +++ b/llvm/include/llvm/ADT/FoldingSet.h @@ -85,17 +85,17 @@ namespace llvm { /// /// MyNode *M = MyFoldingSet.FindNodeOrInsertPos(ID, InsertPoint); /// -/// If found then M with be non-NULL, else InsertPoint will point to where it +/// If found then M will be non-NULL, else InsertPoint will point to where it /// should be inserted using InsertNode. /// -/// 3) If you get a NULL result from FindNodeOrInsertPos then you can as a new -/// node with FindNodeOrInsertPos; +/// 3) If you get a NULL result from FindNodeOrInsertPos then you can insert a +/// new node with InsertNode; /// -/// InsertNode(N, InsertPoint); +/// MyFoldingSet.InsertNode(M, InsertPoint); /// /// 4) Finally, if you want to remove a node from the folding set call; /// -/// bool WasRemoved = RemoveNode(N); +/// bool WasRemoved = MyFoldingSet.RemoveNode(M); /// /// The result indicates whether the node existed in the folding set. diff --git a/llvm/include/llvm/ADT/Hashing.h b/llvm/include/llvm/ADT/Hashing.h index b22606bdb518..adcc5cf54da9 100644 --- a/llvm/include/llvm/ADT/Hashing.h +++ b/llvm/include/llvm/ADT/Hashing.h @@ -45,6 +45,7 @@ #define LLVM_ADT_HASHING_H #include "llvm/Support/DataTypes.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/SwapByteOrder.h" #include "llvm/Support/type_traits.h" #include <algorithm> @@ -256,7 +257,7 @@ inline uint64_t hash_short(const char *s, size_t length, uint64_t seed) { /// Currently, the algorithm for computing hash codes is based on CityHash and /// keeps 56 bytes of arbitrary state. struct hash_state { - uint64_t h0, h1, h2, h3, h4, h5, h6; + uint64_t h0 = 0, h1 = 0, h2 = 0, h3 = 0, h4 = 0, h5 = 0, h6 = 0; /// Create a new hash_state structure and initialize it based on the /// seed and the first 64-byte chunk. @@ -491,7 +492,7 @@ namespace detail { /// useful at minimizing the code in the recursive calls to ease the pain /// caused by a lack of variadic functions. struct hash_combine_recursive_helper { - char buffer[64]; + char buffer[64] = {}; hash_state state; const uint64_t seed; @@ -539,7 +540,7 @@ public: // store types smaller than the buffer. if (!store_and_advance(buffer_ptr, buffer_end, data, partial_store_size)) - abort(); + llvm_unreachable("buffer smaller than stored type"); } return buffer_ptr; } diff --git a/llvm/include/llvm/ADT/ImmutableSet.h b/llvm/include/llvm/ADT/ImmutableSet.h index 587105431533..a6a6abfd9600 100644 --- a/llvm/include/llvm/ADT/ImmutableSet.h +++ b/llvm/include/llvm/ADT/ImmutableSet.h @@ -205,8 +205,7 @@ public: ImutInfo::KeyOfValue(getValue()))) && "Value in left child is not less that current value"); - - assert(!(getRight() || + assert((!getRight() || ImutInfo::isLess(ImutInfo::KeyOfValue(getValue()), ImutInfo::KeyOfValue(getRight()->getValue()))) && "Current value is not less that value of right child"); diff --git a/llvm/include/llvm/ADT/Optional.h b/llvm/include/llvm/ADT/Optional.h index b45a74002e10..c84f9aa8b342 100644 --- a/llvm/include/llvm/ADT/Optional.h +++ b/llvm/include/llvm/ADT/Optional.h @@ -267,6 +267,14 @@ public: return hasValue() ? getValue() : std::forward<U>(value); } + /// Apply a function to the value if present; otherwise return None. + template <class Function> + auto map(const Function &F) const + -> Optional<decltype(F(getValue()))> { + if (*this) return F(getValue()); + return None; + } + #if LLVM_HAS_RVALUE_REFERENCE_THIS T &&getValue() && { return std::move(Storage.getValue()); } T &&operator*() && { return std::move(Storage.getValue()); } @@ -275,6 +283,14 @@ public: T getValueOr(U &&value) && { return hasValue() ? std::move(getValue()) : std::forward<U>(value); } + + /// Apply a function to the value if present; otherwise return None. + template <class Function> + auto map(const Function &F) && + -> Optional<decltype(F(std::move(*this).getValue()))> { + if (*this) return F(std::move(*this).getValue()); + return None; + } #endif }; diff --git a/llvm/include/llvm/ADT/PointerUnion.h b/llvm/include/llvm/ADT/PointerUnion.h index 98c905775a77..40b7b000da40 100644 --- a/llvm/include/llvm/ADT/PointerUnion.h +++ b/llvm/include/llvm/ADT/PointerUnion.h @@ -93,7 +93,7 @@ namespace pointer_union_detail { static constexpr int NumLowBitsAvailable = lowBitsAvailable<PTs...>(); }; - /// Implement assigment in terms of construction. + /// Implement assignment in terms of construction. template <typename Derived, typename T> struct AssignableFrom { Derived &operator=(T t) { return static_cast<Derived &>(*this) = Derived(t); @@ -272,16 +272,6 @@ struct PointerLikeTypeTraits<PointerUnion<PTs...>> { PointerUnion<PTs...>::Val)>::NumLowBitsAvailable; }; -/// A pointer union of three pointer types. See documentation for PointerUnion -/// for usage. -template <typename PT1, typename PT2, typename PT3> -using PointerUnion3 = PointerUnion<PT1, PT2, PT3>; - -/// A pointer union of four pointer types. See documentation for PointerUnion -/// for usage. -template <typename PT1, typename PT2, typename PT3, typename PT4> -using PointerUnion4 = PointerUnion<PT1, PT2, PT3, PT4>; - // Teach DenseMap how to use PointerUnions as keys. template <typename ...PTs> struct DenseMapInfo<PointerUnion<PTs...>> { using Union = PointerUnion<PTs...>; diff --git a/llvm/include/llvm/ADT/SCCIterator.h b/llvm/include/llvm/ADT/SCCIterator.h index eb1a5d0938cf..1e642b9f75d3 100644 --- a/llvm/include/llvm/ADT/SCCIterator.h +++ b/llvm/include/llvm/ADT/SCCIterator.h @@ -134,7 +134,10 @@ public: /// has been deleted, and \c New is to be used in its place. void ReplaceNode(NodeRef Old, NodeRef New) { assert(nodeVisitNumbers.count(Old) && "Old not in scc_iterator?"); - nodeVisitNumbers[New] = nodeVisitNumbers[Old]; + // Do the assignment in two steps, in case 'New' is not yet in the map, and + // inserting it causes the map to grow. + auto tempVal = nodeVisitNumbers[Old]; + nodeVisitNumbers[New] = tempVal; nodeVisitNumbers.erase(Old); } }; diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h index 274933bc5204..b61dab2459d1 100644 --- a/llvm/include/llvm/ADT/STLExtras.h +++ b/llvm/include/llvm/ADT/STLExtras.h @@ -17,7 +17,6 @@ #define LLVM_ADT_STLEXTRAS_H #include "llvm/ADT/Optional.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Config/abi-breaking.h" @@ -51,10 +50,6 @@ namespace detail { template <typename RangeT> using IterOfRange = decltype(std::begin(std::declval<RangeT &>())); -template <typename RangeT> -using ValueOfRange = typename std::remove_reference<decltype( - *std::begin(std::declval<RangeT &>()))>::type; - } // end namespace detail //===----------------------------------------------------------------------===// @@ -198,6 +193,15 @@ constexpr bool empty(const T &RangeOrContainer) { return adl_begin(RangeOrContainer) == adl_end(RangeOrContainer); } +/// Return a range covering \p RangeOrContainer with the first N elements +/// excluded. +template <typename T> +auto drop_begin(T &&RangeOrContainer, size_t N) -> + iterator_range<decltype(adl_begin(RangeOrContainer))> { + return make_range(std::next(adl_begin(RangeOrContainer), N), + adl_end(RangeOrContainer)); +} + // mapped_iterator - This is a simple iterator adapter that causes a function to // be applied whenever operator* is invoked on the iterator. @@ -1044,6 +1048,23 @@ inline int (*get_array_pod_sort_comparator(const T &)) return array_pod_sort_comparator<T>; } +#ifdef EXPENSIVE_CHECKS +namespace detail { + +inline unsigned presortShuffleEntropy() { + static unsigned Result(std::random_device{}()); + return Result; +} + +template <class IteratorTy> +inline void presortShuffle(IteratorTy Start, IteratorTy End) { + std::mt19937 Generator(presortShuffleEntropy()); + std::shuffle(Start, End, Generator); +} + +} // end namespace detail +#endif + /// array_pod_sort - This sorts an array with the specified start and end /// extent. This is just like std::sort, except that it calls qsort instead of /// using an inlined template. qsort is slightly slower than std::sort, but @@ -1065,8 +1086,7 @@ inline void array_pod_sort(IteratorTy Start, IteratorTy End) { auto NElts = End - Start; if (NElts <= 1) return; #ifdef EXPENSIVE_CHECKS - std::mt19937 Generator(std::random_device{}()); - std::shuffle(Start, End, Generator); + detail::presortShuffle<IteratorTy>(Start, End); #endif qsort(&*Start, NElts, sizeof(*Start), get_array_pod_sort_comparator(*Start)); } @@ -1082,8 +1102,7 @@ inline void array_pod_sort( auto NElts = End - Start; if (NElts <= 1) return; #ifdef EXPENSIVE_CHECKS - std::mt19937 Generator(std::random_device{}()); - std::shuffle(Start, End, Generator); + detail::presortShuffle<IteratorTy>(Start, End); #endif qsort(&*Start, NElts, sizeof(*Start), reinterpret_cast<int (*)(const void *, const void *)>(Compare)); @@ -1094,8 +1113,7 @@ inline void array_pod_sort( template <typename IteratorTy> inline void sort(IteratorTy Start, IteratorTy End) { #ifdef EXPENSIVE_CHECKS - std::mt19937 Generator(std::random_device{}()); - std::shuffle(Start, End, Generator); + detail::presortShuffle<IteratorTy>(Start, End); #endif std::sort(Start, End); } @@ -1107,8 +1125,7 @@ template <typename Container> inline void sort(Container &&C) { template <typename IteratorTy, typename Compare> inline void sort(IteratorTy Start, IteratorTy End, Compare Comp) { #ifdef EXPENSIVE_CHECKS - std::mt19937 Generator(std::random_device{}()); - std::shuffle(Start, End, Generator); + detail::presortShuffle<IteratorTy>(Start, End); #endif std::sort(Start, End, Comp); } @@ -1312,15 +1329,6 @@ bool is_splat(R &&Range) { std::equal(adl_begin(Range) + 1, adl_end(Range), adl_begin(Range))); } -/// Given a range of type R, iterate the entire range and return a -/// SmallVector with elements of the vector. This is useful, for example, -/// when you want to iterate a range and then sort the results. -template <unsigned Size, typename R> -SmallVector<typename std::remove_const<detail::ValueOfRange<R>>::type, Size> -to_vector(R &&Range) { - return {adl_begin(Range), adl_end(Range)}; -} - /// Provide a container algorithm similar to C++ Library Fundamentals v2's /// `erase_if` which is equivalent to: /// @@ -1407,6 +1415,8 @@ template <typename R> struct result_pair { result_pair(std::size_t Index, IterOfRange<R> Iter) : Index(Index), Iter(Iter) {} + result_pair<R>(const result_pair<R> &Other) + : Index(Other.Index), Iter(Other.Iter) {} result_pair<R> &operator=(const result_pair<R> &Other) { Index = Other.Index; Iter = Other.Iter; @@ -1455,6 +1465,7 @@ public: return Result.Iter == RHS.Result.Iter; } + enumerator_iter<R>(const enumerator_iter<R> &Other) : Result(Other.Result) {} enumerator_iter<R> &operator=(const enumerator_iter<R> &Other) { Result = Other.Result; return *this; diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h index 913518230d2d..1d8280063c80 100644 --- a/llvm/include/llvm/ADT/SmallPtrSet.h +++ b/llvm/include/llvm/ADT/SmallPtrSet.h @@ -409,6 +409,32 @@ private: } }; +/// Equality comparison for SmallPtrSet. +/// +/// Iterates over elements of LHS confirming that each value from LHS is also in +/// RHS, and that no additional values are in RHS. +template <typename PtrType> +bool operator==(const SmallPtrSetImpl<PtrType> &LHS, + const SmallPtrSetImpl<PtrType> &RHS) { + if (LHS.size() != RHS.size()) + return false; + + for (const auto *KV : LHS) + if (!RHS.count(KV)) + return false; + + return true; +} + +/// Inequality comparison for SmallPtrSet. +/// +/// Equivalent to !(LHS == RHS). +template <typename PtrType> +bool operator!=(const SmallPtrSetImpl<PtrType> &LHS, + const SmallPtrSetImpl<PtrType> &RHS) { + return !(LHS == RHS); +} + /// SmallPtrSet - This class implements a set which is optimized for holding /// SmallSize or less elements. This internally rounds up SmallSize to the next /// power of two if it is not already a power of two. See the comments above diff --git a/llvm/include/llvm/ADT/SmallSet.h b/llvm/include/llvm/ADT/SmallSet.h index 6b128c2e2992..a03fa7dd8423 100644 --- a/llvm/include/llvm/ADT/SmallSet.h +++ b/llvm/include/llvm/ADT/SmallSet.h @@ -248,6 +248,31 @@ private: template <typename PointeeType, unsigned N> class SmallSet<PointeeType*, N> : public SmallPtrSet<PointeeType*, N> {}; +/// Equality comparison for SmallSet. +/// +/// Iterates over elements of LHS confirming that each element is also a member +/// of RHS, and that RHS contains no additional values. +/// Equivalent to N calls to RHS.count. +/// For small-set mode amortized complexity is O(N^2) +/// For large-set mode amortized complexity is linear, worst case is O(N^2) (if +/// every hash collides). +template <typename T, unsigned LN, unsigned RN, typename C> +bool operator==(const SmallSet<T, LN, C> &LHS, const SmallSet<T, RN, C> &RHS) { + if (LHS.size() != RHS.size()) + return false; + + // All elements in LHS must also be in RHS + return all_of(LHS, [&RHS](const T &E) { return RHS.count(E); }); +} + +/// Inequality comparison for SmallSet. +/// +/// Equivalent to !(LHS == RHS). See operator== for performance notes. +template <typename T, unsigned LN, unsigned RN, typename C> +bool operator!=(const SmallSet<T, LN, C> &LHS, const SmallSet<T, RN, C> &RHS) { + return !(LHS == RHS); +} + } // end namespace llvm #endif // LLVM_ADT_SMALLSET_H diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h index 17586904d212..8c46aa906905 100644 --- a/llvm/include/llvm/ADT/SmallVector.h +++ b/llvm/include/llvm/ADT/SmallVector.h @@ -907,6 +907,17 @@ inline size_t capacity_in_bytes(const SmallVector<T, N> &X) { return X.capacity_in_bytes(); } +/// Given a range of type R, iterate the entire range and return a +/// SmallVector with elements of the vector. This is useful, for example, +/// when you want to iterate a range and then sort the results. +template <unsigned Size, typename R> +SmallVector<typename std::remove_const<typename std::remove_reference< + decltype(*std::begin(std::declval<R &>()))>::type>::type, + Size> +to_vector(R &&Range) { + return {std::begin(Range), std::end(Range)}; +} + } // end namespace llvm namespace std { diff --git a/llvm/include/llvm/ADT/Statistic.h b/llvm/include/llvm/ADT/Statistic.h index b7387ddcf1c7..d7aff6c5939a 100644 --- a/llvm/include/llvm/ADT/Statistic.h +++ b/llvm/include/llvm/ADT/Statistic.h @@ -173,7 +173,7 @@ using Statistic = NoopStatistic; static llvm::TrackingStatistic VARNAME = {DEBUG_TYPE, #VARNAME, DESC} /// Enable the collection and printing of statistics. -void EnableStatistics(bool PrintOnExit = true); +void EnableStatistics(bool DoPrintOnExit = true); /// Check if statistics are enabled. bool AreStatisticsEnabled(); diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 52baab17bede..9bfaaccd953e 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -21,6 +21,12 @@ #include <type_traits> #include <utility> +// Declare the __builtin_strlen intrinsic for MSVC so it can be used in +// constexpr context. +#if defined(_MSC_VER) +extern "C" size_t __builtin_strlen(const char *); +#endif + namespace llvm { class APInt; @@ -71,7 +77,7 @@ namespace llvm { static constexpr size_t strLen(const char *Str) { #if __cplusplus > 201402L return std::char_traits<char>::length(Str); -#elif __has_builtin(__builtin_strlen) || defined(__GNUC__) +#elif __has_builtin(__builtin_strlen) || defined(__GNUC__) || defined(_MSC_VER) return __builtin_strlen(Str); #else const char *Begin = Str; @@ -560,7 +566,8 @@ namespace llvm { /// /// If \p AllowInexact is false, the function will fail if the string /// cannot be represented exactly. Otherwise, the function only fails - /// in case of an overflow or underflow. + /// in case of an overflow or underflow, or an invalid floating point + /// representation. bool getAsDouble(double &Result, bool AllowInexact = true) const; /// @} diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h index edeb31efab80..76a754d671fb 100644 --- a/llvm/include/llvm/ADT/Triple.h +++ b/llvm/include/llvm/ADT/Triple.h @@ -95,7 +95,8 @@ public: wasm64, // WebAssembly with 64-bit pointers renderscript32, // 32-bit RenderScript renderscript64, // 64-bit RenderScript - LastArchType = renderscript64 + ve, // NEC SX-Aurora Vector Engine + LastArchType = ve }; enum SubArchType { NoSubArch, @@ -128,7 +129,9 @@ public: KalimbaSubArch_v4, KalimbaSubArch_v5, - MipsSubArch_r6 + MipsSubArch_r6, + + PPCSubArch_spe }; enum VendorType { UnknownVendor, @@ -203,8 +206,6 @@ public: CODE16, EABI, EABIHF, - ELFv1, - ELFv2, Android, Musl, MuslEABI, @@ -477,7 +478,7 @@ public: return getSubArch() == Triple::ARMSubArch_v7k; } - /// isOSDarwin - Is this a "Darwin" OS (OS X, iOS, or watchOS). + /// isOSDarwin - Is this a "Darwin" OS (macOS, iOS, tvOS or watchOS). bool isOSDarwin() const { return isMacOSX() || isiOS() || isWatchOS(); } @@ -730,6 +731,16 @@ public: return getArch() == Triple::riscv32 || getArch() == Triple::riscv64; } + /// Tests whether the target is x86 (32- or 64-bit). + bool isX86() const { + return getArch() == Triple::x86 || getArch() == Triple::x86_64; + } + + /// Tests whether the target is VE + bool isVE() const { + return getArch() == Triple::ve; + } + /// Tests whether the target supports comdat bool supportsCOMDAT() const { return !isOSBinFormatMachO(); diff --git a/llvm/include/llvm/ADT/Twine.h b/llvm/include/llvm/ADT/Twine.h index 4140c22aad3d..2dc7486c924f 100644 --- a/llvm/include/llvm/ADT/Twine.h +++ b/llvm/include/llvm/ADT/Twine.h @@ -153,11 +153,11 @@ namespace llvm { /// LHS - The prefix in the concatenation, which may be uninitialized for /// Null or Empty kinds. - Child LHS; + Child LHS = {0}; /// RHS - The suffix in the concatenation, which may be uninitialized for /// Null or Empty kinds. - Child RHS; + Child RHS = {0}; /// LHSKind - The NodeKind of the left hand side, \see getLHSKind(). NodeKind LHSKind = EmptyKind; diff --git a/llvm/include/llvm/ADT/iterator.h b/llvm/include/llvm/ADT/iterator.h index 467fd4c00ec5..8fd5c11a2dcb 100644 --- a/llvm/include/llvm/ADT/iterator.h +++ b/llvm/include/llvm/ADT/iterator.h @@ -333,6 +333,11 @@ make_pointer_range(RangeT &&Range) { PointerIteratorT(std::end(std::forward<RangeT>(Range)))); } +template <typename WrappedIteratorT, + typename T1 = typename std::remove_reference<decltype(**std::declval<WrappedIteratorT>())>::type, + typename T2 = typename std::add_pointer<T1>::type> +using raw_pointer_iterator = pointer_iterator<pointee_iterator<WrappedIteratorT, T1>, T2>; + // Wrapper iterator over iterator ItType, adding DataRef to the type of ItType, // to create NodeRef = std::pair<InnerTypeOfItType, DataRef>. template <typename ItType, typename NodeRef, typename DataRef> diff --git a/llvm/include/llvm/ADT/iterator_range.h b/llvm/include/llvm/ADT/iterator_range.h index aa8830943cab..f038f6bf2128 100644 --- a/llvm/include/llvm/ADT/iterator_range.h +++ b/llvm/include/llvm/ADT/iterator_range.h @@ -59,11 +59,6 @@ template <typename T> iterator_range<T> make_range(std::pair<T, T> p) { return iterator_range<T>(std::move(p.first), std::move(p.second)); } -template <typename T> -iterator_range<decltype(adl_begin(std::declval<T>()))> drop_begin(T &&t, - int n) { - return make_range(std::next(adl_begin(t), n), adl_end(t)); -} } #endif diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index 282142f51bb3..7c42a261ebb6 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -1179,14 +1179,9 @@ struct ExternalAAWrapperPass : ImmutablePass { static char ID; - ExternalAAWrapperPass() : ImmutablePass(ID) { - initializeExternalAAWrapperPassPass(*PassRegistry::getPassRegistry()); - } + ExternalAAWrapperPass(); - explicit ExternalAAWrapperPass(CallbackT CB) - : ImmutablePass(ID), CB(std::move(CB)) { - initializeExternalAAWrapperPassPass(*PassRegistry::getPassRegistry()); - } + explicit ExternalAAWrapperPass(CallbackT CB); void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h index 187317e3831b..e94a758b06ba 100644 --- a/llvm/include/llvm/Analysis/AliasSetTracker.h +++ b/llvm/include/llvm/Analysis/AliasSetTracker.h @@ -344,8 +344,8 @@ class AliasSetTracker { struct ASTCallbackVHDenseMapInfo : public DenseMapInfo<Value *> {}; AliasAnalysis &AA; - MemorySSA *MSSA; - Loop *L; + MemorySSA *MSSA = nullptr; + Loop *L = nullptr; ilist<AliasSet> AliasSets; using PointerMapType = DenseMap<ASTCallbackVH, AliasSet::PointerRec *, diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h index 97cb730d16c7..41d6c23b8d0d 100644 --- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h @@ -34,6 +34,7 @@ namespace llvm { class Function; class LoopInfo; class raw_ostream; +class PostDominatorTree; class TargetLibraryInfo; class Value; @@ -179,7 +180,7 @@ private: DenseMap<Edge, BranchProbability> Probs; /// Track the last function we run over for printing. - const Function *LastF; + const Function *LastF = nullptr; /// Track the set of blocks directly succeeded by a returning block. SmallPtrSet<const BasicBlock *, 16> PostDominatedByUnreachable; @@ -187,8 +188,10 @@ private: /// Track the set of blocks that always lead to a cold call. SmallPtrSet<const BasicBlock *, 16> PostDominatedByColdCall; - void updatePostDominatedByUnreachable(const BasicBlock *BB); - void updatePostDominatedByColdCall(const BasicBlock *BB); + void computePostDominatedByUnreachable(const Function &F, + PostDominatorTree *PDT); + void computePostDominatedByColdCall(const Function &F, + PostDominatorTree *PDT); bool calcUnreachableHeuristics(const BasicBlock *BB); bool calcMetadataWeights(const BasicBlock *BB); bool calcColdCallHeuristics(const BasicBlock *BB); @@ -233,10 +236,7 @@ class BranchProbabilityInfoWrapperPass : public FunctionPass { public: static char ID; - BranchProbabilityInfoWrapperPass() : FunctionPass(ID) { - initializeBranchProbabilityInfoWrapperPassPass( - *PassRegistry::getPassRegistry()); - } + BranchProbabilityInfoWrapperPass(); BranchProbabilityInfo &getBPI() { return BPI; } const BranchProbabilityInfo &getBPI() const { return BPI; } diff --git a/llvm/include/llvm/Analysis/DDG.h b/llvm/include/llvm/Analysis/DDG.h index 0e1eb9d2cda3..22df60efd84e 100644 --- a/llvm/include/llvm/Analysis/DDG.h +++ b/llvm/include/llvm/Analysis/DDG.h @@ -13,12 +13,12 @@ #ifndef LLVM_ANALYSIS_DDG_H #define LLVM_ANALYSIS_DDG_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DirectedGraph.h" #include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/DependenceGraphBuilder.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/Instructions.h" -#include <unordered_map> namespace llvm { class DDGNode; @@ -33,7 +33,11 @@ class LPMUpdater; /// 1. Single instruction node containing just one instruction. /// 2. Multiple instruction node where two or more instructions from /// the same basic block are merged into one node. -/// 3. Root node is a special node that connects to all components such that +/// 3. Pi-block node which is a group of other DDG nodes that are part of a +/// strongly-connected component of the graph. +/// A pi-block node contains more than one single or multiple instruction +/// nodes. The root node cannot be part of a pi-block. +/// 4. Root node is a special node that connects to all components such that /// there is always a path from it to any node in the graph. class DDGNode : public DDGNodeBase { public: @@ -43,6 +47,7 @@ public: Unknown, SingleInstruction, MultiInstruction, + PiBlock, Root, }; @@ -155,6 +160,55 @@ private: SmallVector<Instruction *, 2> InstList; }; +/// Subclass of DDGNode representing a pi-block. A pi-block represents a group +/// of DDG nodes that are part of a strongly-connected component of the graph. +/// Replacing all the SCCs with pi-blocks results in an acyclic representation +/// of the DDG. For example if we have: +/// {a -> b}, {b -> c, d}, {c -> a} +/// the cycle a -> b -> c -> a is abstracted into a pi-block "p" as follows: +/// {p -> d} with "p" containing: {a -> b}, {b -> c}, {c -> a} +class PiBlockDDGNode : public DDGNode { +public: + using PiNodeList = SmallVector<DDGNode *, 4>; + + PiBlockDDGNode() = delete; + PiBlockDDGNode(const PiNodeList &List); + PiBlockDDGNode(const PiBlockDDGNode &N); + PiBlockDDGNode(PiBlockDDGNode &&N); + ~PiBlockDDGNode(); + + PiBlockDDGNode &operator=(const PiBlockDDGNode &N) { + DDGNode::operator=(N); + NodeList = N.NodeList; + return *this; + } + + PiBlockDDGNode &operator=(PiBlockDDGNode &&N) { + DDGNode::operator=(std::move(N)); + NodeList = std::move(N.NodeList); + return *this; + } + + /// Get the list of nodes in this pi-block. + const PiNodeList &getNodes() const { + assert(!NodeList.empty() && "Node list is empty."); + return NodeList; + } + PiNodeList &getNodes() { + return const_cast<PiNodeList &>( + static_cast<const PiBlockDDGNode *>(this)->getNodes()); + } + + /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc. + static bool classof(const DDGNode *N) { + return N->getKind() == NodeKind::PiBlock; + } + +private: + /// List of nodes in this pi-block. + PiNodeList NodeList; +}; + /// Data Dependency Graph Edge. /// An edge in the DDG can represent a def-use relationship or /// a memory dependence based on the result of DependenceAnalysis. @@ -163,7 +217,13 @@ private: class DDGEdge : public DDGEdgeBase { public: /// The kind of edge in the DDG - enum class EdgeKind { Unknown, RegisterDefUse, MemoryDependence, Rooted }; + enum class EdgeKind { + Unknown, + RegisterDefUse, + MemoryDependence, + Rooted, + Last = Rooted // Must be equal to the largest enum value. + }; explicit DDGEdge(DDGNode &N) = delete; DDGEdge(DDGNode &N, EdgeKind K) : DDGEdgeBase(N), Kind(K) {} @@ -240,6 +300,7 @@ using DDGInfo = DependenceGraphInfo<DDGNode>; /// Data Dependency Graph class DataDependenceGraph : public DDGBase, public DDGInfo { + friend AbstractDependenceGraphBuilder<DataDependenceGraph>; friend class DDGBuilder; public: @@ -251,14 +312,25 @@ public: DataDependenceGraph(DataDependenceGraph &&G) : DDGBase(std::move(G)), DDGInfo(std::move(G)) {} DataDependenceGraph(Function &F, DependenceInfo &DI); - DataDependenceGraph(const Loop &L, DependenceInfo &DI); + DataDependenceGraph(Loop &L, LoopInfo &LI, DependenceInfo &DI); ~DataDependenceGraph(); + /// If node \p N belongs to a pi-block return a pointer to the pi-block, + /// otherwise return null. + const PiBlockDDGNode *getPiBlock(const NodeType &N) const; + protected: - /// Add node \p N to the graph, if it's not added yet, and keep track of - /// the root node. Return true if node is successfully added. + /// Add node \p N to the graph, if it's not added yet, and keep track of the + /// root node as well as pi-blocks and their members. Return true if node is + /// successfully added. bool addNode(NodeType &N); +private: + using PiBlockMapType = DenseMap<const NodeType *, const PiBlockDDGNode *>; + + /// Mapping from graph nodes to their containing pi-blocks. If a node is not + /// part of a pi-block, it will not appear in this map. + PiBlockMapType PiBlockMap; }; /// Concrete implementation of a pure data dependence graph builder. This class @@ -284,6 +356,12 @@ public: Graph.addNode(*SN); return *SN; } + DDGNode &createPiBlock(const NodeListType &L) final override { + auto *Pi = new PiBlockDDGNode(L); + assert(Pi && "Failed to allocate memory for pi-block node."); + Graph.addNode(*Pi); + return *Pi; + } DDGEdge &createDefUseEdge(DDGNode &Src, DDGNode &Tgt) final override { auto *E = new DDGEdge(Tgt, DDGEdge::EdgeKind::RegisterDefUse); assert(E && "Failed to allocate memory for edge"); @@ -304,6 +382,13 @@ public: return *E; } + const NodeListType &getNodesInPiBlock(const DDGNode &N) final override { + auto *PiNode = dyn_cast<const PiBlockDDGNode>(&N); + assert(PiNode && "Expected a pi-block node."); + return PiNode->getNodes(); + } + + bool shouldCreatePiBlocks() const final override; }; raw_ostream &operator<<(raw_ostream &OS, const DDGNode &N); diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h index 997013a5fc8e..0c4002c3c3ba 100644 --- a/llvm/include/llvm/Analysis/DependenceAnalysis.h +++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h @@ -926,6 +926,12 @@ template <typename T> class ArrayRef; bool tryDelinearize(Instruction *Src, Instruction *Dst, SmallVectorImpl<Subscript> &Pair); + + private: + /// checkSubscript - Helper function for checkSrcSubscript and + /// checkDstSubscript to avoid duplicate code + bool checkSubscript(const SCEV *Expr, const Loop *LoopNest, + SmallBitVector &Loops, bool IsSrc); }; // class DependenceInfo /// AnalysisPass to compute dependence information in a function @@ -954,10 +960,7 @@ template <typename T> class ArrayRef; class DependenceAnalysisWrapperPass : public FunctionPass { public: static char ID; // Class identification, replacement for typeinfo - DependenceAnalysisWrapperPass() : FunctionPass(ID) { - initializeDependenceAnalysisWrapperPassPass( - *PassRegistry::getPassRegistry()); - } + DependenceAnalysisWrapperPass(); bool runOnFunction(Function &F) override; void releaseMemory() override; diff --git a/llvm/include/llvm/Analysis/DependenceGraphBuilder.h b/llvm/include/llvm/Analysis/DependenceGraphBuilder.h index 5f4bdb47043b..08a13d967da2 100644 --- a/llvm/include/llvm/Analysis/DependenceGraphBuilder.h +++ b/llvm/include/llvm/Analysis/DependenceGraphBuilder.h @@ -44,7 +44,9 @@ public: /// The main entry to the graph construction algorithm. It starts by /// creating nodes in increasing order of granularity and then - /// adds def-use and memory edges. + /// adds def-use and memory edges. As one of the final stages, it + /// also creates pi-block nodes to facilitate codegen in transformations + /// that use dependence graphs. /// /// The algorithmic complexity of this implementation is O(V^2 * I^2), where V /// is the number of vertecies (nodes) and I is the number of instructions in @@ -52,12 +54,21 @@ public: /// therefore the worst-case time complexity is O(N^2). The average time /// complexity is O((N^2)/2). void populate() { + computeInstructionOrdinals(); createFineGrainedNodes(); createDefUseEdges(); createMemoryDependencyEdges(); createAndConnectRootNode(); + createPiBlocks(); + sortNodesTopologically(); } + /// Compute ordinal numbers for each instruction and store them in a map for + /// future look up. These ordinals are used to compute node ordinals which are + /// in turn used to order nodes that are part of a cycle. + /// Instruction ordinals are assigned based on lexical program order. + void computeInstructionOrdinals(); + /// Create fine grained nodes. These are typically atomic nodes that /// consist of a single instruction. void createFineGrainedNodes(); @@ -74,6 +85,16 @@ public: /// reachable from the root. void createAndConnectRootNode(); + /// Apply graph abstraction to groups of nodes that belong to a strongly + /// connected component of the graph to create larger compound nodes + /// called pi-blocks. The purpose of this abstraction is to isolate sets of + /// program elements that need to stay together during codegen and turn + /// the dependence graph into an acyclic graph. + void createPiBlocks(); + + /// Topologically sort the graph nodes. + void sortNodesTopologically(); + protected: /// Create the root node of the graph. virtual NodeType &createRootNode() = 0; @@ -81,6 +102,10 @@ protected: /// Create an atomic node in the graph given a single instruction. virtual NodeType &createFineGrainedNode(Instruction &I) = 0; + /// Create a pi-block node in the graph representing a group of nodes in an + /// SCC of the graph. + virtual NodeType &createPiBlock(const NodeListType &L) = 0; + /// Create a def-use edge going from \p Src to \p Tgt. virtual EdgeType &createDefUseEdge(NodeType &Src, NodeType &Tgt) = 0; @@ -90,15 +115,41 @@ protected: /// Create a rooted edge going from \p Src to \p Tgt . virtual EdgeType &createRootedEdge(NodeType &Src, NodeType &Tgt) = 0; + /// Given a pi-block node, return a vector of all the nodes contained within + /// it. + virtual const NodeListType &getNodesInPiBlock(const NodeType &N) = 0; + /// Deallocate memory of edge \p E. virtual void destroyEdge(EdgeType &E) { delete &E; } /// Deallocate memory of node \p N. virtual void destroyNode(NodeType &N) { delete &N; } + /// Return true if creation of pi-blocks are supported and desired, + /// and false otherwise. + virtual bool shouldCreatePiBlocks() const { return true; } + + /// Given an instruction \p I return its associated ordinal number. + size_t getOrdinal(Instruction &I) { + assert(InstOrdinalMap.find(&I) != InstOrdinalMap.end() && + "No ordinal computed for this instruction."); + return InstOrdinalMap[&I]; + } + + /// Given a node \p N return its associated ordinal number. + size_t getOrdinal(NodeType &N) { + assert(NodeOrdinalMap.find(&N) != NodeOrdinalMap.end() && + "No ordinal computed for this node."); + return NodeOrdinalMap[&N]; + } + /// Map types to map instructions to nodes used when populating the graph. using InstToNodeMap = DenseMap<Instruction *, NodeType *>; + /// Map Types to map instruction/nodes to an ordinal number. + using InstToOrdinalMap = DenseMap<Instruction *, size_t>; + using NodeToOrdinalMap = DenseMap<NodeType *, size_t>; + /// Reference to the graph that gets built by a concrete implementation of /// this builder. GraphType &Graph; @@ -112,6 +163,14 @@ protected: /// A mapping from instructions to the corresponding nodes in the graph. InstToNodeMap IMap; + + /// A mapping from each instruction to an ordinal number. This map is used to + /// populate the \p NodeOrdinalMap. + InstToOrdinalMap InstOrdinalMap; + + /// A mapping from nodes to an ordinal number. This map is used to sort nodes + /// in a pi-block based on program order. + NodeToOrdinalMap NodeOrdinalMap; }; } // namespace llvm diff --git a/llvm/include/llvm/Analysis/GlobalsModRef.h b/llvm/include/llvm/Analysis/GlobalsModRef.h index 5d1c5a05206a..fa5b16cf95eb 100644 --- a/llvm/include/llvm/Analysis/GlobalsModRef.h +++ b/llvm/include/llvm/Analysis/GlobalsModRef.h @@ -39,6 +39,9 @@ class GlobalsAAResult : public AAResultBase<GlobalsAAResult> { /// The globals that do not have their addresses taken. SmallPtrSet<const GlobalValue *, 8> NonAddressTakenGlobals; + /// Are there functions with local linkage that may modify globals. + bool UnknownFunctionsWithLocalLinkage = false; + /// IndirectGlobals - The memory pointed to by this global is known to be /// 'owned' by the global. SmallPtrSet<const GlobalValue *, 8> IndirectGlobals; diff --git a/llvm/include/llvm/Analysis/GuardUtils.h b/llvm/include/llvm/Analysis/GuardUtils.h index 41e7b7c06c75..b83211535ec2 100644 --- a/llvm/include/llvm/Analysis/GuardUtils.h +++ b/llvm/include/llvm/Analysis/GuardUtils.h @@ -15,6 +15,7 @@ namespace llvm { class BasicBlock; +class Use; class User; class Value; @@ -22,6 +23,10 @@ class Value; /// of llvm.experimental.guard intrinsic. bool isGuard(const User *U); +/// Returns true iff \p U is a widenable branch (that is, parseWidenableBranch +/// returns true). +bool isWidenableBranch(const User *U); + /// Returns true iff \p U has semantics of a guard expressed in a form of a /// widenable conditional branch to deopt block. bool isGuardAsWidenableBranch(const User *U); @@ -39,6 +44,11 @@ bool parseWidenableBranch(const User *U, Value *&Condition, Value *&WidenableCondition, BasicBlock *&IfTrueBB, BasicBlock *&IfFalseBB); +/// Analgous to the above, but return the Uses so that that they can be +/// modified. Unlike previous version, Condition is optional and may be null. +bool parseWidenableBranch(User *U, Use *&Cond, Use *&WC, BasicBlock *&IfTrueBB, + BasicBlock *&IfFalseBB); + } // llvm #endif // LLVM_ANALYSIS_GUARDUTILS_H diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h index a5ffca13046b..b661caee6848 100644 --- a/llvm/include/llvm/Analysis/InstructionSimplify.h +++ b/llvm/include/llvm/Analysis/InstructionSimplify.h @@ -259,6 +259,10 @@ Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, /// Given a callsite, fold the result or return null. Value *SimplifyCall(CallBase *Call, const SimplifyQuery &Q); +/// Given an operand for a Freeze, see if we can fold the result. +/// If not, this returns null. +Value *SimplifyFreezeInst(Value *Op, const SimplifyQuery &Q); + /// See if we can compute a simplified version of this instruction. If not, /// return null. Value *SimplifyInstruction(Instruction *I, const SimplifyQuery &Q, diff --git a/llvm/include/llvm/Analysis/IntervalPartition.h b/llvm/include/llvm/Analysis/IntervalPartition.h index 5b127c25a2b8..66a99fb15bfb 100644 --- a/llvm/include/llvm/Analysis/IntervalPartition.h +++ b/llvm/include/llvm/Analysis/IntervalPartition.h @@ -50,9 +50,7 @@ class IntervalPartition : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid - IntervalPartition() : FunctionPass(ID) { - initializeIntervalPartitionPass(*PassRegistry::getPassRegistry()); - } + IntervalPartition(); // run - Calculate the interval partition for this function bool runOnFunction(Function &F) override; diff --git a/llvm/include/llvm/Analysis/LazyValueInfo.h b/llvm/include/llvm/Analysis/LazyValueInfo.h index 570a5044f6f8..74e8f5072037 100644 --- a/llvm/include/llvm/Analysis/LazyValueInfo.h +++ b/llvm/include/llvm/Analysis/LazyValueInfo.h @@ -144,9 +144,7 @@ class LazyValueInfoWrapperPass : public FunctionPass { void operator=(const LazyValueInfoWrapperPass&) = delete; public: static char ID; - LazyValueInfoWrapperPass() : FunctionPass(ID) { - initializeLazyValueInfoWrapperPassPass(*PassRegistry::getPassRegistry()); - } + LazyValueInfoWrapperPass(); ~LazyValueInfoWrapperPass() override { assert(!Info.PImpl && "releaseMemory not called"); } diff --git a/llvm/include/llvm/Analysis/LegacyDivergenceAnalysis.h b/llvm/include/llvm/Analysis/LegacyDivergenceAnalysis.h index e33b8f4129f3..0770093bcd48 100644 --- a/llvm/include/llvm/Analysis/LegacyDivergenceAnalysis.h +++ b/llvm/include/llvm/Analysis/LegacyDivergenceAnalysis.h @@ -16,20 +16,18 @@ #define LLVM_ANALYSIS_LEGACY_DIVERGENCE_ANALYSIS_H #include "llvm/ADT/DenseSet.h" -#include "llvm/IR/Function.h" -#include "llvm/Pass.h" #include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/Pass.h" namespace llvm { class Value; +class Function; class GPUDivergenceAnalysis; class LegacyDivergenceAnalysis : public FunctionPass { public: static char ID; - LegacyDivergenceAnalysis() : FunctionPass(ID) { - initializeLegacyDivergenceAnalysisPass(*PassRegistry::getPassRegistry()); - } + LegacyDivergenceAnalysis(); void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 9e9aaa32c64f..7f8639ac90d1 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -175,8 +175,8 @@ public: }; MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L) - : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeRegisterWidth(-1U), - FoundNonConstantDistanceDependence(false), + : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeDepDistBytes(0), + MaxSafeRegisterWidth(-1U), FoundNonConstantDistanceDependence(false), Status(VectorizationSafetyStatus::Safe), RecordDependences(true) {} /// Register the location (instructions are given increasing numbers) @@ -724,9 +724,7 @@ class LoopAccessLegacyAnalysis : public FunctionPass { public: static char ID; - LoopAccessLegacyAnalysis() : FunctionPass(ID) { - initializeLoopAccessLegacyAnalysisPass(*PassRegistry::getPassRegistry()); - } + LoopAccessLegacyAnalysis(); bool runOnFunction(Function &F) override; @@ -750,11 +748,11 @@ private: DenseMap<Loop *, std::unique_ptr<LoopAccessInfo>> LoopAccessInfoMap; // The used analysis passes. - ScalarEvolution *SE; - const TargetLibraryInfo *TLI; - AliasAnalysis *AA; - DominatorTree *DT; - LoopInfo *LI; + ScalarEvolution *SE = nullptr; + const TargetLibraryInfo *TLI = nullptr; + AliasAnalysis *AA = nullptr; + DominatorTree *DT = nullptr; + LoopInfo *LI = nullptr; }; /// This analysis provides dependence information for the memory diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h index abf3863b0601..a01045124c7b 100644 --- a/llvm/include/llvm/Analysis/LoopInfo.h +++ b/llvm/include/llvm/Analysis/LoopInfo.h @@ -208,7 +208,7 @@ public: bool isLoopExiting(const BlockT *BB) const { assert(!isInvalid() && "Loop not in a valid state!"); assert(contains(BB) && "Exiting block must be part of the loop"); - for (const auto &Succ : children<const BlockT *>(BB)) { + for (const auto *Succ : children<const BlockT *>(BB)) { if (!contains(Succ)) return true; } @@ -756,6 +756,17 @@ public: /// - guarded by a loop guard branch. bool isGuarded() const { return (getLoopGuardBranch() != nullptr); } + /// Return true if the loop is in rotated form. + /// + /// This does not check if the loop was rotated by loop rotation, instead it + /// only checks if the loop is in rotated form (has a valid latch that exists + /// the loop). + bool isRotatedForm() const { + assert(!isInvalid() && "Loop not in a valid state!"); + BasicBlock *Latch = getLoopLatch(); + return Latch && isLoopExiting(Latch); + } + /// Return true if the loop induction variable starts at zero and increments /// by one each time through the loop. bool isCanonical(ScalarEvolution &SE) const; @@ -1211,9 +1222,7 @@ class LoopInfoWrapperPass : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid - LoopInfoWrapperPass() : FunctionPass(ID) { - initializeLoopInfoWrapperPassPass(*PassRegistry::getPassRegistry()); - } + LoopInfoWrapperPass(); LoopInfo &getLoopInfo() { return LI; } const LoopInfo &getLoopInfo() const { return LI; } diff --git a/llvm/include/llvm/Analysis/LoopInfoImpl.h b/llvm/include/llvm/Analysis/LoopInfoImpl.h index 8b11e848a195..99f192a59215 100644 --- a/llvm/include/llvm/Analysis/LoopInfoImpl.h +++ b/llvm/include/llvm/Analysis/LoopInfoImpl.h @@ -35,7 +35,7 @@ void LoopBase<BlockT, LoopT>::getExitingBlocks( SmallVectorImpl<BlockT *> &ExitingBlocks) const { assert(!isInvalid() && "Loop not in a valid state!"); for (const auto BB : blocks()) - for (const auto &Succ : children<BlockT *>(BB)) + for (auto *Succ : children<BlockT *>(BB)) if (!contains(Succ)) { // Not in current loop? It must be an exit block. ExitingBlocks.push_back(BB); @@ -63,7 +63,7 @@ void LoopBase<BlockT, LoopT>::getExitBlocks( SmallVectorImpl<BlockT *> &ExitBlocks) const { assert(!isInvalid() && "Loop not in a valid state!"); for (const auto BB : blocks()) - for (const auto &Succ : children<BlockT *>(BB)) + for (auto *Succ : children<BlockT *>(BB)) if (!contains(Succ)) // Not in current loop? It must be an exit block. ExitBlocks.push_back(Succ); @@ -142,7 +142,7 @@ void LoopBase<BlockT, LoopT>::getExitEdges( SmallVectorImpl<Edge> &ExitEdges) const { assert(!isInvalid() && "Loop not in a valid state!"); for (const auto BB : blocks()) - for (const auto &Succ : children<BlockT *>(BB)) + for (auto *Succ : children<BlockT *>(BB)) if (!contains(Succ)) // Not in current loop? It must be an exit block. ExitEdges.emplace_back(BB, Succ); diff --git a/llvm/include/llvm/Analysis/LoopPass.h b/llvm/include/llvm/Analysis/LoopPass.h index 9215ab34ec6d..04fed15f15f9 100644 --- a/llvm/include/llvm/Analysis/LoopPass.h +++ b/llvm/include/llvm/Analysis/LoopPass.h @@ -162,9 +162,7 @@ private: // pass manager calls lcssa verification for the current loop. struct LCSSAVerificationPass : public FunctionPass { static char ID; - LCSSAVerificationPass() : FunctionPass(ID) { - initializeLCSSAVerificationPassPass(*PassRegistry::getPassRegistry()); - } + LCSSAVerificationPass(); bool runOnFunction(Function &F) override { return false; } diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h index e89bf26a7234..9b393c9cdaa3 100644 --- a/llvm/include/llvm/Analysis/MemorySSA.h +++ b/llvm/include/llvm/Analysis/MemorySSA.h @@ -95,6 +95,7 @@ #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" #include <algorithm> #include <cassert> #include <cstddef> @@ -785,7 +786,7 @@ public: /// Used in various insertion functions to specify whether we are talking /// about the beginning or end of a block. - enum InsertionPlace { Beginning, End }; + enum InsertionPlace { Beginning, End, BeforeTerminator }; protected: // Used by Memory SSA annotater, dumpers, and wrapper pass @@ -793,11 +794,9 @@ protected: friend class MemorySSAPrinterLegacyPass; friend class MemorySSAUpdater; - void verifyPrevDefInPhis(Function &F) const; - void verifyDefUses(Function &F) const; - void verifyDomination(Function &F) const; - void verifyOrdering(Function &F) const; + void verifyOrderingDominationAndDefUses(Function &F) const; void verifyDominationNumbers(const Function &F) const; + void verifyPrevDefInPhis(Function &F) const; // This is used by the use optimizer and updater. AccessList *getWritableBlockAccesses(const BasicBlock *BB) const { diff --git a/llvm/include/llvm/Analysis/MustExecute.h b/llvm/include/llvm/Analysis/MustExecute.h index 87cf9f85c7f1..d88cdf40ed87 100644 --- a/llvm/include/llvm/Analysis/MustExecute.h +++ b/llvm/include/llvm/Analysis/MustExecute.h @@ -33,8 +33,13 @@ namespace llvm { +namespace { +template <typename T> using GetterTy = std::function<T *(const Function &F)>; +} + class Instruction; class DominatorTree; +class PostDominatorTree; class Loop; /// Captures loop safety information. @@ -374,8 +379,14 @@ struct MustBeExecutedContextExplorer { /// \param ExploreInterBlock Flag to indicate if instructions in blocks /// other than the parent of PP should be /// explored. - MustBeExecutedContextExplorer(bool ExploreInterBlock) - : ExploreInterBlock(ExploreInterBlock), EndIterator(*this, nullptr) {} + MustBeExecutedContextExplorer( + bool ExploreInterBlock, + GetterTy<const LoopInfo> LIGetter = + [](const Function &) { return nullptr; }, + GetterTy<const PostDominatorTree> PDTGetter = + [](const Function &) { return nullptr; }) + : ExploreInterBlock(ExploreInterBlock), LIGetter(LIGetter), + PDTGetter(PDTGetter), EndIterator(*this, nullptr) {} /// Clean up the dynamically allocated iterators. ~MustBeExecutedContextExplorer() { @@ -420,6 +431,30 @@ struct MustBeExecutedContextExplorer { } ///} + /// Helper to look for \p I in the context of \p PP. + /// + /// The context is expanded until \p I was found or no more expansion is + /// possible. + /// + /// \returns True, iff \p I was found. + bool findInContextOf(const Instruction *I, const Instruction *PP) { + auto EIt = begin(PP), EEnd = end(PP); + return findInContextOf(I, EIt, EEnd); + } + + /// Helper to look for \p I in the context defined by \p EIt and \p EEnd. + /// + /// The context is expanded until \p I was found or no more expansion is + /// possible. + /// + /// \returns True, iff \p I was found. + bool findInContextOf(const Instruction *I, iterator &EIt, iterator &EEnd) { + bool Found = EIt.count(I); + while (!Found && EIt != EEnd) + Found = (++EIt).getCurrentInst() == I; + return Found; + } + /// Return the next instruction that is guaranteed to be executed after \p PP. /// /// \param It The iterator that is used to traverse the must be @@ -430,6 +465,9 @@ struct MustBeExecutedContextExplorer { getMustBeExecutedNextInstruction(MustBeExecutedIterator &It, const Instruction *PP); + /// Find the next join point from \p InitBB in forward direction. + const BasicBlock *findForwardJoinPoint(const BasicBlock *InitBB); + /// Parameter that limit the performed exploration. See the constructor for /// their meaning. ///{ @@ -437,6 +475,19 @@ struct MustBeExecutedContextExplorer { ///} private: + /// Getters for common CFG analyses: LoopInfo, DominatorTree, and + /// PostDominatorTree. + ///{ + GetterTy<const LoopInfo> LIGetter; + GetterTy<const PostDominatorTree> PDTGetter; + ///} + + /// Map to cache isGuaranteedToTransferExecutionToSuccessor results. + DenseMap<const BasicBlock *, Optional<bool>> BlockTransferMap; + + /// Map to cache containsIrreducibleCFG results. + DenseMap<const Function*, Optional<bool>> IrreducibleControlMap; + /// Map from instructions to associated must be executed iterators. DenseMap<const Instruction *, MustBeExecutedIterator *> InstructionIteratorMap; diff --git a/llvm/include/llvm/Analysis/PhiValues.h b/llvm/include/llvm/Analysis/PhiValues.h index 124fa2191694..ee6eec85f198 100644 --- a/llvm/include/llvm/Analysis/PhiValues.h +++ b/llvm/include/llvm/Analysis/PhiValues.h @@ -108,7 +108,7 @@ private: /// Process a phi so that its entries in the depth and reachable maps are /// fully populated. - void processPhi(const PHINode *PN, SmallVector<const PHINode *, 8> &Stack); + void processPhi(const PHINode *PN, SmallVectorImpl<const PHINode *> &Stack); }; /// The analysis pass which yields a PhiValues diff --git a/llvm/include/llvm/Analysis/PostDominators.h b/llvm/include/llvm/Analysis/PostDominators.h index 87d2e0318d0a..801eb3d59673 100644 --- a/llvm/include/llvm/Analysis/PostDominators.h +++ b/llvm/include/llvm/Analysis/PostDominators.h @@ -34,6 +34,13 @@ public: /// Handle invalidation explicitly. bool invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &); + + // Ensure base-class overloads are visible. + using Base::dominates; + + /// Return true if \p I1 dominates \p I2. This checks if \p I2 comes before + /// \p I1 if they belongs to the same basic block. + bool dominates(const Instruction *I1, const Instruction *I2) const; }; /// Analysis pass which computes a \c PostDominatorTree. @@ -68,9 +75,7 @@ struct PostDominatorTreeWrapperPass : public FunctionPass { PostDominatorTree DT; - PostDominatorTreeWrapperPass() : FunctionPass(ID) { - initializePostDominatorTreeWrapperPassPass(*PassRegistry::getPassRegistry()); - } + PostDominatorTreeWrapperPass(); PostDominatorTree &getPostDomTree() { return DT; } const PostDominatorTree &getPostDomTree() const { return DT; } diff --git a/llvm/include/llvm/Analysis/PtrUseVisitor.h b/llvm/include/llvm/Analysis/PtrUseVisitor.h index fbf04c841d30..05bca2226742 100644 --- a/llvm/include/llvm/Analysis/PtrUseVisitor.h +++ b/llvm/include/llvm/Analysis/PtrUseVisitor.h @@ -222,9 +222,9 @@ public: // offsets on this pointer. // FIXME: Support a vector of pointers. assert(I.getType()->isPointerTy()); - IntegerType *IntPtrTy = cast<IntegerType>(DL.getIntPtrType(I.getType())); + IntegerType *IntIdxTy = cast<IntegerType>(DL.getIndexType(I.getType())); IsOffsetKnown = true; - Offset = APInt(IntPtrTy->getBitWidth(), 0); + Offset = APInt(IntIdxTy->getBitWidth(), 0); PI.reset(); // Enqueue the uses of this pointer. diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index 9c55f7a5090f..5286f6a220ec 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -435,35 +435,6 @@ public: } }; -struct ExitLimitQuery { - ExitLimitQuery(const Loop *L, BasicBlock *ExitingBlock, bool AllowPredicates) - : L(L), ExitingBlock(ExitingBlock), AllowPredicates(AllowPredicates) {} - - const Loop *L; - BasicBlock *ExitingBlock; - bool AllowPredicates; -}; - -template <> struct DenseMapInfo<ExitLimitQuery> { - static inline ExitLimitQuery getEmptyKey() { - return ExitLimitQuery(nullptr, nullptr, true); - } - - static inline ExitLimitQuery getTombstoneKey() { - return ExitLimitQuery(nullptr, nullptr, false); - } - - static unsigned getHashValue(ExitLimitQuery Val) { - return hash_combine(hash_combine(Val.L, Val.ExitingBlock), - Val.AllowPredicates); - } - - static bool isEqual(ExitLimitQuery LHS, ExitLimitQuery RHS) { - return LHS.L == RHS.L && LHS.ExitingBlock == RHS.ExitingBlock && - LHS.AllowPredicates == RHS.AllowPredicates; - } -}; - /// The main scalar evolution driver. Because client code (intentionally) /// can't do much with the SCEV objects directly, they must ask this class /// for services. @@ -748,13 +719,26 @@ public: unsigned getSmallConstantTripMultiple(const Loop *L, BasicBlock *ExitingBlock); + + /// The terms "backedge taken count" and "exit count" are used + /// interchangeably to refer to the number of times the backedge of a loop + /// has executed before the loop is exited. + enum ExitCountKind { + /// An expression exactly describing the number of times the backedge has + /// executed when a loop is exited. + Exact, + /// A constant which provides an upper bound on the exact trip count. + ConstantMaximum, + }; + /// Return the number of times the backedge executes before the given exit /// would be taken; if not exactly computable, return SCEVCouldNotCompute. /// For a single exit loop, this value is equivelent to the result of /// getBackedgeTakenCount. The loop is guaranteed to exit (via *some* exit) /// before the backedge is executed (ExitCount + 1) times. Note that there /// is no guarantee about *which* exit is taken on the exiting iteration. - const SCEV *getExitCount(const Loop *L, BasicBlock *ExitingBlock); + const SCEV *getExitCount(const Loop *L, BasicBlock *ExitingBlock, + ExitCountKind Kind = Exact); /// If the specified loop has a predictable backedge-taken count, return it, /// otherwise return a SCEVCouldNotCompute object. The backedge-taken count is @@ -766,7 +750,7 @@ public: /// Note that it is not valid to call this method on a loop without a /// loop-invariant backedge-taken count (see /// hasLoopInvariantBackedgeTakenCount). - const SCEV *getBackedgeTakenCount(const Loop *L); + const SCEV *getBackedgeTakenCount(const Loop *L, ExitCountKind Kind = Exact); /// Similar to getBackedgeTakenCount, except it will add a set of /// SCEV predicates to Predicates that are required to be true in order for @@ -779,7 +763,9 @@ public: /// to (i.e. a "conservative over-approximation") of the value returend by /// getBackedgeTakenCount. If such a value cannot be computed, it returns the /// SCEVCouldNotCompute object. - const SCEV *getConstantMaxBackedgeTakenCount(const Loop *L); + const SCEV *getConstantMaxBackedgeTakenCount(const Loop *L) { + return getBackedgeTakenCount(L, ConstantMaximum); + } /// Return true if the backedge taken count is either the value returned by /// getConstantMaxBackedgeTakenCount or zero. @@ -1227,6 +1213,9 @@ private: Predicates.insert(P); } + /// Construct either an exact exit limit from a constant, or an unknown + /// one from a SCEVCouldNotCompute. No other types of SCEVs are allowed + /// as arguments and asserts enforce that internally. /*implicit*/ ExitLimit(const SCEV *E); ExitLimit( @@ -1258,13 +1247,15 @@ private: struct ExitNotTakenInfo { PoisoningVH<BasicBlock> ExitingBlock; const SCEV *ExactNotTaken; + const SCEV *MaxNotTaken; std::unique_ptr<SCEVUnionPredicate> Predicate; explicit ExitNotTakenInfo(PoisoningVH<BasicBlock> ExitingBlock, const SCEV *ExactNotTaken, + const SCEV *MaxNotTaken, std::unique_ptr<SCEVUnionPredicate> Predicate) - : ExitingBlock(ExitingBlock), ExactNotTaken(ExactNotTaken), - Predicate(std::move(Predicate)) {} + : ExitingBlock(ExitingBlock), ExactNotTaken(ExactNotTaken), + MaxNotTaken(ExactNotTaken), Predicate(std::move(Predicate)) {} bool hasAlwaysTruePredicate() const { return !Predicate || Predicate->isAlwaysTrue(); @@ -1347,6 +1338,9 @@ private: /// Get the max backedge taken count for the loop. const SCEV *getMax(ScalarEvolution *SE) const; + /// Get the max backedge taken count for the particular loop exit. + const SCEV *getMax(BasicBlock *ExitingBlock, ScalarEvolution *SE) const; + /// Return true if the number of times this backedge is taken is either the /// value returned by getMax or zero. bool isMaxOrZero(ScalarEvolution *SE) const; @@ -1928,6 +1922,13 @@ public: ScalarEvolution run(Function &F, FunctionAnalysisManager &AM); }; +/// Verifier pass for the \c ScalarEvolutionAnalysis results. +class ScalarEvolutionVerifierPass + : public PassInfoMixin<ScalarEvolutionVerifierPass> { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + /// Printer pass for the \c ScalarEvolutionAnalysis results. class ScalarEvolutionPrinterPass : public PassInfoMixin<ScalarEvolutionPrinterPass> { diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h index d4b223863c54..1bd9db756bb3 100644 --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -9,6 +9,7 @@ #ifndef LLVM_ANALYSIS_TARGETLIBRARYINFO_H #define LLVM_ANALYSIS_TARGETLIBRARYINFO_H +#include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/Triple.h" @@ -197,6 +198,10 @@ public: /// Returns the size of the wchar_t type in bytes or 0 if the size is unknown. /// This queries the 'wchar_size' metadata. unsigned getWCharSize(const Module &M) const; + + /// Returns the largest vectorization factor used in the list of + /// vector functions. + unsigned getWidestVF(StringRef ScalarF) const; }; /// Provides information about what library functions are available for @@ -208,20 +213,50 @@ class TargetLibraryInfo { friend class TargetLibraryAnalysis; friend class TargetLibraryInfoWrapperPass; + /// The global (module level) TLI info. const TargetLibraryInfoImpl *Impl; + /// Support for -fno-builtin* options as function attributes, overrides + /// information in global TargetLibraryInfoImpl. + BitVector OverrideAsUnavailable; + public: - explicit TargetLibraryInfo(const TargetLibraryInfoImpl &Impl) : Impl(&Impl) {} + explicit TargetLibraryInfo(const TargetLibraryInfoImpl &Impl, + Optional<const Function *> F = None) + : Impl(&Impl), OverrideAsUnavailable(NumLibFuncs) { + if (!F) + return; + if ((*F)->hasFnAttribute("no-builtins")) + disableAllFunctions(); + else { + // Disable individual libc/libm calls in TargetLibraryInfo. + LibFunc LF; + AttributeSet FnAttrs = (*F)->getAttributes().getFnAttributes(); + for (const Attribute &Attr : FnAttrs) { + if (!Attr.isStringAttribute()) + continue; + auto AttrStr = Attr.getKindAsString(); + if (!AttrStr.consume_front("no-builtin-")) + continue; + if (getLibFunc(AttrStr, LF)) + setUnavailable(LF); + } + } + } // Provide value semantics. - TargetLibraryInfo(const TargetLibraryInfo &TLI) : Impl(TLI.Impl) {} - TargetLibraryInfo(TargetLibraryInfo &&TLI) : Impl(TLI.Impl) {} + TargetLibraryInfo(const TargetLibraryInfo &TLI) + : Impl(TLI.Impl), OverrideAsUnavailable(TLI.OverrideAsUnavailable) {} + TargetLibraryInfo(TargetLibraryInfo &&TLI) + : Impl(TLI.Impl), OverrideAsUnavailable(TLI.OverrideAsUnavailable) {} TargetLibraryInfo &operator=(const TargetLibraryInfo &TLI) { Impl = TLI.Impl; + OverrideAsUnavailable = TLI.OverrideAsUnavailable; return *this; } TargetLibraryInfo &operator=(TargetLibraryInfo &&TLI) { Impl = TLI.Impl; + OverrideAsUnavailable = TLI.OverrideAsUnavailable; return *this; } @@ -244,9 +279,27 @@ public: getLibFunc(*(CS.getCalledFunction()), F); } + /// Disables all builtins. + /// + /// This can be used for options like -fno-builtin. + void disableAllFunctions() LLVM_ATTRIBUTE_UNUSED { + OverrideAsUnavailable.set(); + } + + /// Forces a function to be marked as unavailable. + void setUnavailable(LibFunc F) LLVM_ATTRIBUTE_UNUSED { + OverrideAsUnavailable.set(F); + } + + TargetLibraryInfoImpl::AvailabilityState getState(LibFunc F) const { + if (OverrideAsUnavailable[F]) + return TargetLibraryInfoImpl::Unavailable; + return Impl->getState(F); + } + /// Tests whether a library function is available. bool has(LibFunc F) const { - return Impl->getState(F) != TargetLibraryInfoImpl::Unavailable; + return getState(F) != TargetLibraryInfoImpl::Unavailable; } bool isFunctionVectorizable(StringRef F, unsigned VF) const { return Impl->isFunctionVectorizable(F, VF); @@ -261,7 +314,7 @@ public: /// Tests if the function is both available and a candidate for optimized code /// generation. bool hasOptimizedCodeGen(LibFunc F) const { - if (Impl->getState(F) == TargetLibraryInfoImpl::Unavailable) + if (getState(F) == TargetLibraryInfoImpl::Unavailable) return false; switch (F) { default: break; @@ -291,7 +344,7 @@ public: } StringRef getName(LibFunc F) const { - auto State = Impl->getState(F); + auto State = getState(F); if (State == TargetLibraryInfoImpl::Unavailable) return StringRef(); if (State == TargetLibraryInfoImpl::StandardName) @@ -337,6 +390,12 @@ public: FunctionAnalysisManager::Invalidator &) { return false; } + + /// Returns the largest vectorization factor used in the list of + /// vector functions. + unsigned getWidestVF(StringRef ScalarF) const { + return Impl->getWidestVF(ScalarF); + } }; /// Analysis pass providing the \c TargetLibraryInfo. @@ -353,29 +412,24 @@ public: /// module. TargetLibraryAnalysis() {} - /// Construct a library analysis with preset info. + /// Construct a library analysis with baseline Module-level info. /// - /// This will directly copy the preset info into the result without - /// consulting the module's triple. - TargetLibraryAnalysis(TargetLibraryInfoImpl PresetInfoImpl) - : PresetInfoImpl(std::move(PresetInfoImpl)) {} + /// This will be supplemented with Function-specific info in the Result. + TargetLibraryAnalysis(TargetLibraryInfoImpl BaselineInfoImpl) + : BaselineInfoImpl(std::move(BaselineInfoImpl)) {} - TargetLibraryInfo run(Function &F, FunctionAnalysisManager &); + TargetLibraryInfo run(const Function &F, FunctionAnalysisManager &); private: friend AnalysisInfoMixin<TargetLibraryAnalysis>; static AnalysisKey Key; - Optional<TargetLibraryInfoImpl> PresetInfoImpl; - - StringMap<std::unique_ptr<TargetLibraryInfoImpl>> Impls; - - TargetLibraryInfoImpl &lookupInfoImpl(const Triple &T); + Optional<TargetLibraryInfoImpl> BaselineInfoImpl; }; class TargetLibraryInfoWrapperPass : public ImmutablePass { - TargetLibraryInfoImpl TLIImpl; - TargetLibraryInfo TLI; + TargetLibraryAnalysis TLA; + Optional<TargetLibraryInfo> TLI; virtual void anchor(); @@ -385,12 +439,10 @@ public: explicit TargetLibraryInfoWrapperPass(const Triple &T); explicit TargetLibraryInfoWrapperPass(const TargetLibraryInfoImpl &TLI); - TargetLibraryInfo &getTLI(const Function &F LLVM_ATTRIBUTE_UNUSED) { - return TLI; - } - const TargetLibraryInfo & - getTLI(const Function &F LLVM_ATTRIBUTE_UNUSED) const { - return TLI; + TargetLibraryInfo &getTLI(const Function &F) { + FunctionAnalysisManager DummyFAM; + TLI = TLA.run(F, DummyFAM); + return *TLI; } }; diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index d6fa88411654..5382d76813a7 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -36,16 +36,19 @@ namespace llvm { namespace Intrinsic { -enum ID : unsigned; +typedef unsigned ID; } class AssumptionCache; +class BlockFrequencyInfo; class BranchInst; class Function; class GlobalValue; class IntrinsicInst; class LoadInst; +class LoopAccessInfo; class Loop; +class ProfileSummaryInfo; class SCEV; class ScalarEvolution; class StoreInst; @@ -297,7 +300,9 @@ public: /// \p JTSize Set a jump table size only when \p SI is suitable for a jump /// table. unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, - unsigned &JTSize) const; + unsigned &JTSize, + ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI) const; /// Estimate the cost of a given IR user when lowered. /// @@ -514,6 +519,13 @@ public: TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const; + /// Query the target whether it would be prefered to create a predicated vector + /// loop, which can avoid the need to emit a scalar epilogue loop. + bool preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, ScalarEvolution &SE, + AssumptionCache &AC, TargetLibraryInfo *TLI, + DominatorTree *DT, + const LoopAccessInfo *LAI) const; + /// @} /// \name Scalar Target Information @@ -585,9 +597,9 @@ public: bool isLegalNTLoad(Type *DataType, Align Alignment) const; /// Return true if the target supports masked scatter. - bool isLegalMaskedScatter(Type *DataType) const; + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) const; /// Return true if the target supports masked gather. - bool isLegalMaskedGather(Type *DataType) const; + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) const; /// Return true if the target supports masked compress store. bool isLegalMaskedCompressStore(Type *DataType) const; @@ -742,10 +754,10 @@ public: /// Return the expected cost of materialization for the given integer /// immediate of the specified type for a given instruction. The cost can be /// zero if the immediate can be folded into the specified instruction. - int getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, - Type *Ty) const; - int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, - Type *Ty) const; + int getIntImmCostInst(unsigned Opc, unsigned Idx, const APInt &Imm, + Type *Ty) const; + int getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, + Type *Ty) const; /// Return the expected cost for the given integer when optimising /// for size. This is different than the other integer immediate cost @@ -889,12 +901,15 @@ public: /// \p Args is an optional argument which holds the instruction operands /// values so the TTI can analyze those values searching for special /// cases or optimizations based on those values. + /// \p CxtI is the optional original context instruction, if one exists, to + /// provide even more information. int getArithmeticInstrCost( unsigned Opcode, Type *Ty, OperandValueKind Opd1Info = OK_AnyValue, OperandValueKind Opd2Info = OK_AnyValue, OperandValueProperties Opd1PropInfo = OP_None, OperandValueProperties Opd2PropInfo = OP_None, - ArrayRef<const Value *> Args = ArrayRef<const Value *>()) const; + ArrayRef<const Value *> Args = ArrayRef<const Value *>(), + const Instruction *CxtI = nullptr) const; /// \return The cost of a shuffle instruction of kind Kind and of type Tp. /// The index and subtype parameters are used by the subvector insertion and @@ -930,8 +945,9 @@ public: int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index = -1) const; /// \return The cost of Load and Store instructions. - int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, - unsigned AddressSpace, const Instruction *I = nullptr) const; + int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, + unsigned AddressSpace, + const Instruction *I = nullptr) const; /// \return The cost of masked Load and Store instructions. int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, @@ -1176,7 +1192,9 @@ public: const User *U) = 0; virtual int getMemcpyCost(const Instruction *I) = 0; virtual unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, - unsigned &JTSize) = 0; + unsigned &JTSize, + ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI) = 0; virtual int getUserCost(const User *U, ArrayRef<const Value *> Operands) = 0; virtual bool hasBranchDivergence() = 0; @@ -1194,6 +1212,12 @@ public: AssumptionCache &AC, TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) = 0; + virtual bool preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, + ScalarEvolution &SE, + AssumptionCache &AC, + TargetLibraryInfo *TLI, + DominatorTree *DT, + const LoopAccessInfo *LAI) = 0; virtual bool isLegalAddImmediate(int64_t Imm) = 0; virtual bool isLegalICmpImmediate(int64_t Imm) = 0; virtual bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, @@ -1213,8 +1237,8 @@ public: virtual bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) = 0; virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0; virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0; - virtual bool isLegalMaskedScatter(Type *DataType) = 0; - virtual bool isLegalMaskedGather(Type *DataType) = 0; + virtual bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) = 0; + virtual bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) = 0; virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0; virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0; @@ -1254,10 +1278,10 @@ public: virtual int getIntImmCodeSizeCost(unsigned Opc, unsigned Idx, const APInt &Imm, Type *Ty) = 0; virtual int getIntImmCost(const APInt &Imm, Type *Ty) = 0; - virtual int getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, - Type *Ty) = 0; - virtual int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, - Type *Ty) = 0; + virtual int getIntImmCostInst(unsigned Opc, unsigned Idx, const APInt &Imm, + Type *Ty) = 0; + virtual int getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, + const APInt &Imm, Type *Ty) = 0; virtual unsigned getNumberOfRegisters(unsigned ClassID) const = 0; virtual unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const = 0; virtual const char* getRegisterClassName(unsigned ClassID) const = 0; @@ -1288,12 +1312,11 @@ public: virtual unsigned getMaxPrefetchIterationsAhead() const = 0; virtual unsigned getMaxInterleaveFactor(unsigned VF) = 0; - virtual unsigned - getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, - OperandValueKind Opd2Info, - OperandValueProperties Opd1PropInfo, - OperandValueProperties Opd2PropInfo, - ArrayRef<const Value *> Args) = 0; + virtual unsigned getArithmeticInstrCost( + unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, + OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo, + OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, + const Instruction *CxtI = nullptr) = 0; virtual int getShuffleCost(ShuffleKind Kind, Type *Tp, int Index, Type *SubTp) = 0; virtual int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, @@ -1305,7 +1328,7 @@ public: Type *CondTy, const Instruction *I) = 0; virtual int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) = 0; - virtual int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + virtual int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace, const Instruction *I) = 0; virtual int getMaskedMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, @@ -1464,6 +1487,12 @@ public: HardwareLoopInfo &HWLoopInfo) override { return Impl.isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); } + bool preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, ScalarEvolution &SE, + AssumptionCache &AC, TargetLibraryInfo *TLI, + DominatorTree *DT, + const LoopAccessInfo *LAI) override { + return Impl.preferPredicateOverEpilogue(L, LI, SE, AC, TLI, DT, LAI); + } bool isLegalAddImmediate(int64_t Imm) override { return Impl.isLegalAddImmediate(Imm); } @@ -1508,11 +1537,11 @@ public: bool isLegalNTLoad(Type *DataType, Align Alignment) override { return Impl.isLegalNTLoad(DataType, Alignment); } - bool isLegalMaskedScatter(Type *DataType) override { - return Impl.isLegalMaskedScatter(DataType); + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) override { + return Impl.isLegalMaskedScatter(DataType, Alignment); } - bool isLegalMaskedGather(Type *DataType) override { - return Impl.isLegalMaskedGather(DataType); + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) override { + return Impl.isLegalMaskedGather(DataType, Alignment); } bool isLegalMaskedCompressStore(Type *DataType) override { return Impl.isLegalMaskedCompressStore(DataType); @@ -1609,13 +1638,13 @@ public: int getIntImmCost(const APInt &Imm, Type *Ty) override { return Impl.getIntImmCost(Imm, Ty); } - int getIntImmCost(unsigned Opc, unsigned Idx, const APInt &Imm, - Type *Ty) override { - return Impl.getIntImmCost(Opc, Idx, Imm, Ty); + int getIntImmCostInst(unsigned Opc, unsigned Idx, const APInt &Imm, + Type *Ty) override { + return Impl.getIntImmCostInst(Opc, Idx, Imm, Ty); } - int getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, - Type *Ty) override { - return Impl.getIntImmCost(IID, Idx, Imm, Ty); + int getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, + Type *Ty) override { + return Impl.getIntImmCostIntrin(IID, Idx, Imm, Ty); } unsigned getNumberOfRegisters(unsigned ClassID) const override { return Impl.getNumberOfRegisters(ClassID); @@ -1677,17 +1706,20 @@ public: return Impl.getMaxInterleaveFactor(VF); } unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, - unsigned &JTSize) override { - return Impl.getEstimatedNumberOfCaseClusters(SI, JTSize); - } - unsigned - getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, - OperandValueKind Opd2Info, - OperandValueProperties Opd1PropInfo, - OperandValueProperties Opd2PropInfo, - ArrayRef<const Value *> Args) override { + unsigned &JTSize, + ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI) override { + return Impl.getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI); + } + unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, + OperandValueKind Opd1Info, + OperandValueKind Opd2Info, + OperandValueProperties Opd1PropInfo, + OperandValueProperties Opd2PropInfo, + ArrayRef<const Value *> Args, + const Instruction *CxtI = nullptr) override { return Impl.getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info, - Opd1PropInfo, Opd2PropInfo, Args); + Opd1PropInfo, Opd2PropInfo, Args, CxtI); } int getShuffleCost(ShuffleKind Kind, Type *Tp, int Index, Type *SubTp) override { @@ -1711,7 +1743,7 @@ public: int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) override { return Impl.getVectorInstrCost(Opcode, Val, Index); } - int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + int getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace, const Instruction *I) override { return Impl.getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, I); } diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index a431fa0d458b..ac0609e29270 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -114,7 +114,11 @@ public: } unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, - unsigned &JTSize) { + unsigned &JTSize, + ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI) { + (void)PSI; + (void)BFI; JTSize = 0; return SI.getNumCases(); } @@ -209,6 +213,13 @@ public: return false; } + bool preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, ScalarEvolution &SE, + AssumptionCache &AC, TargetLibraryInfo *TLI, + DominatorTree *DT, + const LoopAccessInfo *LAI) const { + return false; + } + void getUnrollingPreferences(Loop *, ScalarEvolution &, TTI::UnrollingPreferences &) {} @@ -261,9 +272,13 @@ public: return Alignment >= DataSize && isPowerOf2_32(DataSize); } - bool isLegalMaskedScatter(Type *DataType) { return false; } + bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) { + return false; + } - bool isLegalMaskedGather(Type *DataType) { return false; } + bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) { + return false; + } bool isLegalMaskedCompressStore(Type *DataType) { return false; } @@ -344,13 +359,13 @@ public: unsigned getIntImmCost(const APInt &Imm, Type *Ty) { return TTI::TCC_Basic; } - unsigned getIntImmCost(unsigned Opcode, unsigned Idx, const APInt &Imm, - Type *Ty) { + unsigned getIntImmCostInst(unsigned Opcode, unsigned Idx, const APInt &Imm, + Type *Ty) { return TTI::TCC_Free; } - unsigned getIntImmCost(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, - Type *Ty) { + unsigned getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, + const APInt &Imm, Type *Ty) { return TTI::TCC_Free; } @@ -419,7 +434,8 @@ public: TTI::OperandValueKind Opd2Info, TTI::OperandValueProperties Opd1PropInfo, TTI::OperandValueProperties Opd2PropInfo, - ArrayRef<const Value *> Args) { + ArrayRef<const Value *> Args, + const Instruction *CxtI = nullptr) { return 1; } @@ -447,7 +463,7 @@ public: return 1; } - unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, + unsigned getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace, const Instruction *I) { return 1; } diff --git a/llvm/include/llvm/Analysis/Utils/Local.h b/llvm/include/llvm/Analysis/Utils/Local.h index a63bcec9bc41..ca505960cbeb 100644 --- a/llvm/include/llvm/Analysis/Utils/Local.h +++ b/llvm/include/llvm/Analysis/Utils/Local.h @@ -14,6 +14,7 @@ #ifndef LLVM_ANALYSIS_UTILS_LOCAL_H #define LLVM_ANALYSIS_UTILS_LOCAL_H +#include "llvm/ADT/Twine.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/GetElementPtrTypeIterator.h" @@ -28,15 +29,15 @@ template <typename IRBuilderTy> Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, bool NoAssumptions = false) { GEPOperator *GEPOp = cast<GEPOperator>(GEP); - Type *IntPtrTy = DL.getIntPtrType(GEP->getType()); - Value *Result = Constant::getNullValue(IntPtrTy); + Type *IntIdxTy = DL.getIndexType(GEP->getType()); + Value *Result = Constant::getNullValue(IntIdxTy); // If the GEP is inbounds, we know that none of the addressing operations will // overflow in a signed sense. bool isInBounds = GEPOp->isInBounds() && !NoAssumptions; // Build a mask for high order bits. - unsigned IntPtrWidth = IntPtrTy->getScalarType()->getIntegerBitWidth(); + unsigned IntPtrWidth = IntIdxTy->getScalarType()->getIntegerBitWidth(); uint64_t PtrSizeMask = std::numeric_limits<uint64_t>::max() >> (64 - IntPtrWidth); @@ -55,17 +56,17 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, Size = DL.getStructLayout(STy)->getElementOffset(OpValue); if (Size) - Result = Builder->CreateAdd(Result, ConstantInt::get(IntPtrTy, Size), + Result = Builder->CreateAdd(Result, ConstantInt::get(IntIdxTy, Size), GEP->getName()+".offs"); continue; } // Splat the constant if needed. - if (IntPtrTy->isVectorTy() && !OpC->getType()->isVectorTy()) - OpC = ConstantVector::getSplat(IntPtrTy->getVectorNumElements(), OpC); + if (IntIdxTy->isVectorTy() && !OpC->getType()->isVectorTy()) + OpC = ConstantVector::getSplat(IntIdxTy->getVectorNumElements(), OpC); - Constant *Scale = ConstantInt::get(IntPtrTy, Size); - Constant *OC = ConstantExpr::getIntegerCast(OpC, IntPtrTy, true /*SExt*/); + Constant *Scale = ConstantInt::get(IntIdxTy, Size); + Constant *OC = ConstantExpr::getIntegerCast(OpC, IntIdxTy, true /*SExt*/); Scale = ConstantExpr::getMul(OC, Scale, false /*NUW*/, isInBounds /*NSW*/); // Emit an add instruction. @@ -74,15 +75,15 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP, } // Splat the index if needed. - if (IntPtrTy->isVectorTy() && !Op->getType()->isVectorTy()) - Op = Builder->CreateVectorSplat(IntPtrTy->getVectorNumElements(), Op); + if (IntIdxTy->isVectorTy() && !Op->getType()->isVectorTy()) + Op = Builder->CreateVectorSplat(IntIdxTy->getVectorNumElements(), Op); // Convert to correct type. - if (Op->getType() != IntPtrTy) - Op = Builder->CreateIntCast(Op, IntPtrTy, true, Op->getName()+".c"); + if (Op->getType() != IntIdxTy) + Op = Builder->CreateIntCast(Op, IntIdxTy, true, Op->getName()+".c"); if (Size != 1) { // We'll let instcombine(mul) convert this to a shl if possible. - Op = Builder->CreateMul(Op, ConstantInt::get(IntPtrTy, Size), + Op = Builder->CreateMul(Op, ConstantInt::get(IntIdxTy, Size), GEP->getName() + ".idx", false /*NUW*/, isInBounds /*NSW*/); } diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index 33b064fcf9d2..89cf9abdc8ba 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -203,6 +203,12 @@ class Value; /// x < -0 --> false bool CannotBeOrderedLessThanZero(const Value *V, const TargetLibraryInfo *TLI); + /// Return true if the floating-point scalar value is not an infinity or if + /// the floating-point vector value has no infinities. Return false if a value + /// could ever be infinity. + bool isKnownNeverInfinity(const Value *V, const TargetLibraryInfo *TLI, + unsigned Depth = 0); + /// Return true if the floating-point scalar value is not a NaN or if the /// floating-point vector value has no NaN elements. Return false if a value /// could ever be NaN. @@ -226,9 +232,9 @@ class Value; /// return undef. Value *isBytewiseValue(Value *V, const DataLayout &DL); - /// Given an aggregrate and an sequence of indices, see if the scalar value + /// Given an aggregate and an sequence of indices, see if the scalar value /// indexed is already around as a register, for example if it were inserted - /// directly into the aggregrate. + /// directly into the aggregate. /// /// If InsertBefore is not null, this function will duplicate (modified) /// insertvalues when a part of a nested struct is extracted. @@ -561,6 +567,10 @@ class Value; /// the parent of I. bool programUndefinedIfFullPoison(const Instruction *PoisonI); + /// Return true if this function can prove that V is never undef value + /// or poison value. + bool isGuaranteedNotToBeUndefOrPoison(const Value *V); + /// Specific patterns of select instructions we can match. enum SelectPatternFlavor { SPF_UNKNOWN = 0, diff --git a/llvm/include/llvm/Analysis/VecFuncs.def b/llvm/include/llvm/Analysis/VecFuncs.def index 4c9206266d9a..86bec0be7546 100644 --- a/llvm/include/llvm/Analysis/VecFuncs.def +++ b/llvm/include/llvm/Analysis/VecFuncs.def @@ -8,7 +8,14 @@ // This .def file will create mappings from scalar math functions to vector // functions along with their vectorization factor. The current support includes -// such mappings for Accelerate framework, MASS vector library, and SVML library. +// such mappings for Accelerate framework, MASS vector library, and SVML library. +// This .def file also allows creating an array of vector functions supported in +// the specified framework or library. + +#if defined(TLI_DEFINE_MASSV_VECFUNCS_NAMES) +#define TLI_DEFINE_MASSV_VECFUNCS +#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF) VEC, +#endif #if !(defined(TLI_DEFINE_VECFUNC)) #define TLI_DEFINE_VECFUNC(SCAL, VEC, VF) {SCAL, VEC, VF}, @@ -247,4 +254,4 @@ TLI_DEFINE_VECFUNC("llvm.log.f32", "__svml_logf16", 16) #undef TLI_DEFINE_ACCELERATE_VECFUNCS #undef TLI_DEFINE_MASSV_VECFUNCS #undef TLI_DEFINE_SVML_VECFUNCS - +#undef TLI_DEFINE_MASSV_VECFUNCS_NAMES diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h index 4a61c2bc35c7..0ca69bebc127 100644 --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -14,6 +14,7 @@ #define LLVM_ANALYSIS_VECTORUTILS_H #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/CheckedArithmetic.h" @@ -47,7 +48,9 @@ enum class VFISAKind { AVX, // x86 AVX AVX2, // x86 AVX2 AVX512, // x86 AVX512 - Unknown // Unknown ISA + LLVM, // LLVM internal ISA for functions that are not + // attached to an existing ABI via name mangling. + Unknown // Unknown ISA }; /// Encapsulates information needed to describe a parameter. @@ -79,13 +82,36 @@ struct VFParameter { struct VFShape { unsigned VF; // Vectorization factor. bool IsScalable; // True if the function is a scalable function. - VFISAKind ISA; // Instruction Set Architecture. SmallVector<VFParameter, 8> Parameters; // List of parameter informations. // Comparison operator. bool operator==(const VFShape &Other) const { - return std::tie(VF, IsScalable, ISA, Parameters) == - std::tie(Other.VF, Other.IsScalable, Other.ISA, Other.Parameters); + return std::tie(VF, IsScalable, Parameters) == + std::tie(Other.VF, Other.IsScalable, Other.Parameters); } + + /// Update the parameter in position P.ParamPos to P. + void updateParam(VFParameter P) { + assert(P.ParamPos < Parameters.size() && "Invalid parameter position."); + Parameters[P.ParamPos] = P; + assert(hasValidParameterList() && "Invalid parameter list"); + } + + // Retrieve the basic vectorization shape of the function, where all + // parameters are mapped to VFParamKind::Vector with \p EC + // lanes. Specifies whether the function has a Global Predicate + // argument via \p HasGlobalPred. + static VFShape get(const CallInst &CI, ElementCount EC, bool HasGlobalPred) { + SmallVector<VFParameter, 8> Parameters; + for (unsigned I = 0; I < CI.arg_size(); ++I) + Parameters.push_back(VFParameter({I, VFParamKind::Vector})); + if (HasGlobalPred) + Parameters.push_back( + VFParameter({CI.arg_size(), VFParamKind::GlobalPredicate})); + + return {EC.Min, EC.Scalable, Parameters}; + } + /// Sanity check on the Parameters in the VFShape. + bool hasValidParameterList() const; }; /// Holds the VFShape for a specific scalar to vector function mapping. @@ -93,15 +119,19 @@ struct VFInfo { VFShape Shape; // Classification of the vector function. StringRef ScalarName; // Scalar Function Name. StringRef VectorName; // Vector Function Name associated to this VFInfo. + VFISAKind ISA; // Instruction Set Architecture. // Comparison operator. bool operator==(const VFInfo &Other) const { - return std::tie(Shape, ScalarName, VectorName) == - std::tie(Shape, Other.ScalarName, Other.VectorName); + return std::tie(Shape, ScalarName, VectorName, ISA) == + std::tie(Shape, Other.ScalarName, Other.VectorName, Other.ISA); } }; namespace VFABI { +/// LLVM Internal VFABI ISA token for vector functions. +static constexpr char const *_LLVM_ = "_LLVM_"; + /// Function to contruct a VFInfo out of a mangled names in the /// following format: /// @@ -121,14 +151,20 @@ namespace VFABI { /// * x86 (libmvec): https://sourceware.org/glibc/wiki/libmvec and /// https://sourceware.org/glibc/wiki/libmvec?action=AttachFile&do=view&target=VectorABI.txt /// -/// -/// /// \param MangledName -> input string in the format /// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)]. Optional<VFInfo> tryDemangleForVFABI(StringRef MangledName); /// Retrieve the `VFParamKind` from a string token. VFParamKind getVFParamKindFromString(const StringRef Token); + +// Name of the attribute where the variant mappings are stored. +static constexpr char const *MappingsAttrName = "vector-function-abi-variant"; + +/// Populates a set of strings representing the Vector Function ABI variants +/// associated to the CallInst CI. +void getVectorVariantNames(const CallInst &CI, + SmallVectorImpl<std::string> &VariantMappings); } // end namespace VFABI template <typename T> class ArrayRef; @@ -137,13 +173,12 @@ class GetElementPtrInst; template <typename InstTy> class InterleaveGroup; class Loop; class ScalarEvolution; -class TargetLibraryInfo; class TargetTransformInfo; class Type; class Value; namespace Intrinsic { -enum ID : unsigned; +typedef unsigned ID; } /// Identify if the intrinsic is trivially vectorizable. @@ -542,13 +577,10 @@ public: /// formation for predicated accesses, we may be able to relax this limitation /// in the future once we handle more complicated blocks. void reset() { - SmallPtrSet<InterleaveGroup<Instruction> *, 4> DelSet; - // Avoid releasing a pointer twice. - for (auto &I : InterleaveGroupMap) - DelSet.insert(I.second); - for (auto *Ptr : DelSet) - delete Ptr; InterleaveGroupMap.clear(); + for (auto *Ptr : InterleaveGroups) + delete Ptr; + InterleaveGroups.clear(); RequiresScalarEpilogue = false; } diff --git a/llvm/include/llvm/BinaryFormat/COFF.h b/llvm/include/llvm/BinaryFormat/COFF.h index 0fe38a437725..626e0a431e93 100644 --- a/llvm/include/llvm/BinaryFormat/COFF.h +++ b/llvm/include/llvm/BinaryFormat/COFF.h @@ -547,7 +547,7 @@ struct PE32Header { uint32_t AddressOfEntryPoint; // RVA uint32_t BaseOfCode; // RVA uint32_t BaseOfData; // RVA - uint32_t ImageBase; + uint64_t ImageBase; uint32_t SectionAlignment; uint32_t FileAlignment; uint16_t MajorOperatingSystemVersion; @@ -563,10 +563,10 @@ struct PE32Header { uint16_t Subsystem; // FIXME: This should be DllCharacteristics to match the COFF spec. uint16_t DLLCharacteristics; - uint32_t SizeOfStackReserve; - uint32_t SizeOfStackCommit; - uint32_t SizeOfHeapReserve; - uint32_t SizeOfHeapCommit; + uint64_t SizeOfStackReserve; + uint64_t SizeOfStackCommit; + uint64_t SizeOfHeapReserve; + uint64_t SizeOfHeapCommit; uint32_t LoaderFlags; // FIXME: This should be NumberOfRvaAndSizes to match the COFF spec. uint32_t NumberOfRvaAndSize; diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def index 34a7410f7474..3faf3be65032 100644 --- a/llvm/include/llvm/BinaryFormat/Dwarf.def +++ b/llvm/include/llvm/BinaryFormat/Dwarf.def @@ -405,7 +405,7 @@ HANDLE_DW_AT(0x3b31, BORLAND_closure, 0, BORLAND) // LLVM project extensions. HANDLE_DW_AT(0x3e00, LLVM_include_path, 0, LLVM) HANDLE_DW_AT(0x3e01, LLVM_config_macros, 0, LLVM) -HANDLE_DW_AT(0x3e02, LLVM_isysroot, 0, LLVM) +HANDLE_DW_AT(0x3e02, LLVM_sysroot, 0, LLVM) HANDLE_DW_AT(0x3e03, LLVM_tag_offset, 0, LLVM) // Apple extensions. HANDLE_DW_AT(0x3fe1, APPLE_optimized, 0, APPLE) @@ -421,6 +421,7 @@ HANDLE_DW_AT(0x3fea, APPLE_property_setter, 0, APPLE) HANDLE_DW_AT(0x3feb, APPLE_property_attribute, 0, APPLE) HANDLE_DW_AT(0x3fec, APPLE_objc_complete_type, 0, APPLE) HANDLE_DW_AT(0x3fed, APPLE_property, 0, APPLE) +HANDLE_DW_AT(0x3fee, APPLE_objc_direct, 0, APPLE) // Attribute form encodings. HANDLE_DW_FORM(0x01, addr, 2, DWARF) @@ -647,6 +648,8 @@ HANDLE_DW_OP(0xa9, reinterpret, 5, DWARF) // Vendor extensions: // Extensions for GNU-style thread-local storage. HANDLE_DW_OP(0xe0, GNU_push_tls_address, 0, GNU) +// Extensions for WebAssembly. +HANDLE_DW_OP(0xed, WASM_location, 0, WASM) // The GNU entry value extension. HANDLE_DW_OP(0xf3, GNU_entry_value, 0, GNU) // Extensions for Fission proposal. diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h index 1c6aee48661c..2ad201831d2b 100644 --- a/llvm/include/llvm/BinaryFormat/Dwarf.h +++ b/llvm/include/llvm/BinaryFormat/Dwarf.h @@ -63,7 +63,8 @@ enum LLVMConstants : uint32_t { DWARF_VENDOR_GNU = 3, DWARF_VENDOR_GOOGLE = 4, DWARF_VENDOR_LLVM = 5, - DWARF_VENDOR_MIPS = 6 + DWARF_VENDOR_MIPS = 6, + DWARF_VENDOR_WASM = 7 }; /// Constants that define the DWARF format as 32 or 64 bit. @@ -458,6 +459,7 @@ StringRef AttributeEncodingString(unsigned Encoding); StringRef DecimalSignString(unsigned Sign); StringRef EndianityString(unsigned Endian); StringRef AccessibilityString(unsigned Access); +StringRef DefaultedMemberString(unsigned DefaultedEncodings); StringRef VisibilityString(unsigned Visibility); StringRef VirtualityString(unsigned Virtuality); StringRef LanguageString(unsigned Language); diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 46edfb6260be..caab91da9c83 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -1199,8 +1199,9 @@ enum { PT_SUNW_EH_FRAME = 0x6474e550, PT_SUNW_UNWIND = 0x6464e550, - PT_GNU_STACK = 0x6474e551, // Indicates stack executability. - PT_GNU_RELRO = 0x6474e552, // Read-only after relocation. + PT_GNU_STACK = 0x6474e551, // Indicates stack executability. + PT_GNU_RELRO = 0x6474e552, // Read-only after relocation. + PT_GNU_PROPERTY = 0x6474e553, // .note.gnu.property notes sections. PT_OPENBSD_RANDOMIZE = 0x65a3dbe6, // Fill with random data. PT_OPENBSD_WXNEEDED = 0x65a3dbe7, // Program does W^X violations. diff --git a/llvm/include/llvm/BinaryFormat/MinidumpConstants.def b/llvm/include/llvm/BinaryFormat/MinidumpConstants.def index aeef399af7a4..543305feea77 100644 --- a/llvm/include/llvm/BinaryFormat/MinidumpConstants.def +++ b/llvm/include/llvm/BinaryFormat/MinidumpConstants.def @@ -85,21 +85,22 @@ HANDLE_MDMP_STREAM_TYPE(0xFACECCCC, FacebookAppStateLog) HANDLE_MDMP_STREAM_TYPE(0xFACEDEAD, FacebookAbortReason) HANDLE_MDMP_STREAM_TYPE(0xFACEE000, FacebookThreadName) -HANDLE_MDMP_ARCH(0x0000, X86) // PROCESSOR_ARCHITECTURE_INTEL -HANDLE_MDMP_ARCH(0x0001, MIPS) // PROCESSOR_ARCHITECTURE_MIPS -HANDLE_MDMP_ARCH(0x0002, Alpha) // PROCESSOR_ARCHITECTURE_ALPHA -HANDLE_MDMP_ARCH(0x0003, PPC) // PROCESSOR_ARCHITECTURE_PPC -HANDLE_MDMP_ARCH(0x0004, SHX) // PROCESSOR_ARCHITECTURE_SHX (Super-H) -HANDLE_MDMP_ARCH(0x0005, ARM) // PROCESSOR_ARCHITECTURE_ARM -HANDLE_MDMP_ARCH(0x0006, IA64) // PROCESSOR_ARCHITECTURE_IA64 -HANDLE_MDMP_ARCH(0x0007, Alpha64) // PROCESSOR_ARCHITECTURE_ALPHA64 -HANDLE_MDMP_ARCH(0x0008, MSIL) // PROCESSOR_ARCHITECTURE_MSIL -HANDLE_MDMP_ARCH(0x0009, AMD64) // PROCESSOR_ARCHITECTURE_AMD64 -HANDLE_MDMP_ARCH(0x000a, X86Win64) // PROCESSOR_ARCHITECTURE_IA32_ON_WIN64 -HANDLE_MDMP_ARCH(0x8001, SPARC) // Breakpad-defined value for SPARC -HANDLE_MDMP_ARCH(0x8002, PPC64) // Breakpad-defined value for PPC64 -HANDLE_MDMP_ARCH(0x8003, ARM64) // Breakpad-defined value for ARM64 -HANDLE_MDMP_ARCH(0x8004, MIPS64) // Breakpad-defined value for MIPS64 +HANDLE_MDMP_ARCH(0x0000, X86) // PROCESSOR_ARCHITECTURE_INTEL +HANDLE_MDMP_ARCH(0x0001, MIPS) // PROCESSOR_ARCHITECTURE_MIPS +HANDLE_MDMP_ARCH(0x0002, Alpha) // PROCESSOR_ARCHITECTURE_ALPHA +HANDLE_MDMP_ARCH(0x0003, PPC) // PROCESSOR_ARCHITECTURE_PPC +HANDLE_MDMP_ARCH(0x0004, SHX) // PROCESSOR_ARCHITECTURE_SHX (Super-H) +HANDLE_MDMP_ARCH(0x0005, ARM) // PROCESSOR_ARCHITECTURE_ARM +HANDLE_MDMP_ARCH(0x0006, IA64) // PROCESSOR_ARCHITECTURE_IA64 +HANDLE_MDMP_ARCH(0x0007, Alpha64) // PROCESSOR_ARCHITECTURE_ALPHA64 +HANDLE_MDMP_ARCH(0x0008, MSIL) // PROCESSOR_ARCHITECTURE_MSIL +HANDLE_MDMP_ARCH(0x0009, AMD64) // PROCESSOR_ARCHITECTURE_AMD64 +HANDLE_MDMP_ARCH(0x000a, X86Win64) // PROCESSOR_ARCHITECTURE_IA32_ON_WIN64 +HANDLE_MDMP_ARCH(0x000c, ARM64) // PROCESSOR_ARCHITECTURE_ARM64 +HANDLE_MDMP_ARCH(0x8001, BP_SPARC) // Breakpad-defined value for SPARC +HANDLE_MDMP_ARCH(0x8002, BP_PPC64) // Breakpad-defined value for PPC64 +HANDLE_MDMP_ARCH(0x8003, BP_ARM64) // Breakpad-defined value for ARM64 +HANDLE_MDMP_ARCH(0x8004, BP_MIPS64) // Breakpad-defined value for MIPS64 HANDLE_MDMP_PLATFORM(0x0000, Win32S) // Win32 on Windows 3.1 HANDLE_MDMP_PLATFORM(0x0001, Win32Windows) // Windows 95-98-Me diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h index f550d880f68a..59f99cc8cd37 100644 --- a/llvm/include/llvm/BinaryFormat/Wasm.h +++ b/llvm/include/llvm/BinaryFormat/Wasm.h @@ -131,6 +131,7 @@ struct WasmFunction { uint32_t CodeSectionOffset; uint32_t Size; uint32_t CodeOffset; // start of Locals and Body + StringRef ExportName; // from the "export" section StringRef SymbolName; // from the "linking" section StringRef DebugName; // from the "name" section uint32_t Comdat; // from the "comdat info" section @@ -179,6 +180,7 @@ struct WasmSymbolInfo { uint32_t Flags; StringRef ImportModule; // For undefined symbols the module of the import StringRef ImportName; // For undefined symbols the name of the import + StringRef ExportName; // For symbols to be exported from the final module union { // For function or global symbols, the index in function or global index // space. diff --git a/llvm/include/llvm/BinaryFormat/XCOFF.h b/llvm/include/llvm/BinaryFormat/XCOFF.h index 20a0f446272f..b6c3aaa51fc4 100644 --- a/llvm/include/llvm/BinaryFormat/XCOFF.h +++ b/llvm/include/llvm/BinaryFormat/XCOFF.h @@ -13,6 +13,7 @@ #ifndef LLVM_BINARYFORMAT_XCOFF_H #define LLVM_BINARYFORMAT_XCOFF_H +#include "llvm/ADT/StringRef.h" #include <cstdint> namespace llvm { @@ -251,6 +252,8 @@ enum CFileCpuId : uint8_t { TCPU_970 = 19 ///< PPC970 - PowerPC 64-bit architecture. }; +StringRef getMappingClassString(XCOFF::StorageMappingClass SMC); + } // end namespace XCOFF } // end namespace llvm diff --git a/llvm/include/llvm/Bitcode/BitcodeWriter.h b/llvm/include/llvm/Bitcode/BitcodeWriter.h index 39061e09cda5..4beb89d30e00 100644 --- a/llvm/include/llvm/Bitcode/BitcodeWriter.h +++ b/llvm/include/llvm/Bitcode/BitcodeWriter.h @@ -17,6 +17,7 @@ #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/MC/StringTableBuilder.h" #include "llvm/Support/Allocator.h" +#include "llvm/Support/MemoryBuffer.h" #include <map> #include <memory> #include <string> @@ -151,6 +152,11 @@ class raw_ostream; const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex = nullptr); + /// Save a copy of the llvm IR as data in the __LLVM,__bitcode section. + void EmbedBitcodeInModule(Module &M, MemoryBufferRef Buf, bool EmbedBitcode, + bool EmbedMarker, + const std::vector<uint8_t> *CmdArgs); + } // end namespace llvm #endif // LLVM_BITCODE_BITCODEWRITER_H diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 1a397068caf0..2cfd66b96502 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -559,6 +559,7 @@ enum FunctionCodes { FUNC_CODE_INST_UNOP = 56, // UNOP: [opcode, ty, opval] FUNC_CODE_INST_CALLBR = 57, // CALLBR: [attr, cc, norm, transfs, // fnty, fnid, args...] + FUNC_CODE_INST_FREEZE = 58, // FREEZE: [opty, opval] }; enum UseListCodes { diff --git a/llvm/include/llvm/Bitstream/BitstreamReader.h b/llvm/include/llvm/Bitstream/BitstreamReader.h index b49a969a2d8b..c476f60420fa 100644 --- a/llvm/include/llvm/Bitstream/BitstreamReader.h +++ b/llvm/include/llvm/Bitstream/BitstreamReader.h @@ -39,7 +39,7 @@ public: /// This contains information emitted to BLOCKINFO_BLOCK blocks. These /// describe abbreviations that all blocks of the specified ID inherit. struct BlockInfo { - unsigned BlockID; + unsigned BlockID = 0; std::vector<std::shared_ptr<BitCodeAbbrev>> Abbrevs; std::string Name; std::vector<std::pair<unsigned, std::string>> RecordNames; diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index a4580da5aec9..b374fd3d80af 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -48,6 +48,7 @@ class GlobalObject; class GlobalValue; class GlobalVariable; class MachineBasicBlock; +class MachineBlockFrequencyInfo; class MachineConstantPoolValue; class MachineDominatorTree; class MachineFunction; @@ -58,7 +59,6 @@ class MachineModuleInfo; class MachineOptimizationRemarkEmitter; class MCAsmInfo; class MCCFIInstruction; -struct MCCodePaddingContext; class MCContext; class MCExpr; class MCInst; @@ -69,7 +69,9 @@ class MCSymbol; class MCTargetOptions; class MDNode; class Module; +class ProfileSummaryInfo; class raw_ostream; +class RemarkStreamer; class StackMaps; class TargetLoweringObjectFile; class TargetMachine; @@ -107,6 +109,10 @@ public: /// Optimization remark emitter. MachineOptimizationRemarkEmitter *ORE; + MachineBlockFrequencyInfo *MBFI; + + ProfileSummaryInfo *PSI; + /// The symbol for the current function. This is recalculated at the beginning /// of each call to runOnMachineFunction(). MCSymbol *CurrentFnSym = nullptr; @@ -280,6 +286,9 @@ public: /// Emit a table with all XRay instrumentation points. void emitXRayTable(); + DenseMap<const MCSection *, unsigned> PatchableFunctionEntryID; + void emitPatchableFunctionEntries(); + //===------------------------------------------------------------------===// // MachineFunctionPass Implementation. //===------------------------------------------------------------------===// @@ -319,7 +328,7 @@ public: void emitStackSizeSection(const MachineFunction &MF); - void emitRemarksSection(Module &M); + void emitRemarksSection(RemarkStreamer &RS); enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug }; CFIMoveType needsCFIMoves() const; @@ -689,8 +698,6 @@ private: GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy &S); /// Emit GlobalAlias or GlobalIFunc. void emitGlobalIndirectSymbol(Module &M, const GlobalIndirectSymbol &GIS); - void setupCodePaddingContext(const MachineBasicBlock &MBB, - MCCodePaddingContext &Context) const; }; } // end namespace llvm diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 2e57b4c9d332..30533d90bbf7 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -326,7 +326,9 @@ public: } unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, - unsigned &JumpTableSize) { + unsigned &JumpTableSize, + ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI) { /// Try to find the estimated number of clusters. Note that the number of /// clusters identified in this function could be different from the actual /// numbers found in lowering. This function ignore switches that are @@ -374,7 +376,7 @@ public: (MaxCaseVal - MinCaseVal) .getLimitedValue(std::numeric_limits<uint64_t>::max() - 1) + 1; // Check whether a range of clusters is dense enough for a jump table - if (TLI->isSuitableForJumpTable(&SI, N, Range)) { + if (TLI->isSuitableForJumpTable(&SI, N, Range, PSI, BFI)) { JumpTableSize = Range; return 1; } @@ -508,6 +510,13 @@ public: return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); } + bool preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, ScalarEvolution &SE, + AssumptionCache &AC, TargetLibraryInfo *TLI, + DominatorTree *DT, + const LoopAccessInfo *LAI) { + return BaseT::preferPredicateOverEpilogue(L, LI, SE, AC, TLI, DT, LAI); + } + int getInstructionLatency(const Instruction *I) { if (isa<LoadInst>(I)) return getST()->getSchedModel().DefaultLoadLatency; @@ -624,7 +633,8 @@ public: TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue, TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None, TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, - ArrayRef<const Value *> Args = ArrayRef<const Value *>()) { + ArrayRef<const Value *> Args = ArrayRef<const Value *>(), + const Instruction *CxtI = nullptr) { // Check if any of the operands are vector operands. const TargetLoweringBase *TLI = getTLI(); int ISD = TLI->InstructionOpcodeToISD(Opcode); @@ -772,9 +782,10 @@ public: // cost of the split itself. Count that as 1, to be consistent with // TLI->getTypeLegalizationCost(). if ((TLI->getTypeAction(Src->getContext(), TLI->getValueType(DL, Src)) == - TargetLowering::TypeSplitVector) || - (TLI->getTypeAction(Dst->getContext(), TLI->getValueType(DL, Dst)) == - TargetLowering::TypeSplitVector)) { + TargetLowering::TypeSplitVector || + TLI->getTypeAction(Dst->getContext(), TLI->getValueType(DL, Dst)) == + TargetLowering::TypeSplitVector) && + Src->getVectorNumElements() > 1 && Dst->getVectorNumElements() > 1) { Type *SplitDst = VectorType::get(Dst->getVectorElementType(), Dst->getVectorNumElements() / 2); Type *SplitSrc = VectorType::get(Src->getVectorElementType(), @@ -869,8 +880,9 @@ public: return LT.first; } - unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, - unsigned AddressSpace, const Instruction *I = nullptr) { + unsigned getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, + unsigned AddressSpace, + const Instruction *I = nullptr) { assert(!Src->isVoidTy() && "Invalid type"); std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(DL, Src); @@ -921,8 +933,8 @@ public: Cost = static_cast<T *>(this)->getMaskedMemoryOpCost( Opcode, VecTy, Alignment, AddressSpace); else - Cost = static_cast<T *>(this)->getMemoryOpCost(Opcode, VecTy, Alignment, - AddressSpace); + Cost = static_cast<T *>(this)->getMemoryOpCost( + Opcode, VecTy, MaybeAlign(Alignment), AddressSpace); // Legalize the vector type, and get the legalized and unlegalized type // sizes. diff --git a/llvm/include/llvm/CodeGen/CommandFlags.inc b/llvm/include/llvm/CodeGen/CommandFlags.inc index cb69e9f61405..8739b644873d 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.inc +++ b/llvm/include/llvm/CodeGen/CommandFlags.inc @@ -102,15 +102,15 @@ static cl::opt<llvm::ExceptionHandling> ExceptionModel( clEnumValN(ExceptionHandling::Wasm, "wasm", "WebAssembly exception handling"))); -static cl::opt<TargetMachine::CodeGenFileType> FileType( - "filetype", cl::init(TargetMachine::CGFT_AssemblyFile), +static cl::opt<CodeGenFileType> FileType( + "filetype", cl::init(CGFT_AssemblyFile), cl::desc( "Choose a file type (not all types are supported by all targets):"), - cl::values(clEnumValN(TargetMachine::CGFT_AssemblyFile, "asm", + cl::values(clEnumValN(CGFT_AssemblyFile, "asm", "Emit an assembly ('.s') file"), - clEnumValN(TargetMachine::CGFT_ObjectFile, "obj", + clEnumValN(CGFT_ObjectFile, "obj", "Emit a native object ('.o') file"), - clEnumValN(TargetMachine::CGFT_Null, "null", + clEnumValN(CGFT_Null, "null", "Emit nothing, for performance testing"))); static cl::opt<llvm::FramePointer::FP> FramePointerUsage( @@ -151,7 +151,7 @@ static cl::opt<bool> "attribute not to use exceptions"), cl::init(false)); -static cl::opt<llvm::FPDenormal::DenormalMode> DenormalMode( +static cl::opt<llvm::FPDenormal::DenormalMode> DenormalFPMath( "denormal-fp-math", cl::desc("Select which denormal numbers the code is permitted to require"), cl::init(FPDenormal::IEEE), @@ -238,6 +238,10 @@ static cl::opt<bool> cl::desc("Emit functions into separate sections"), cl::init(false)); +static cl::opt<unsigned> TLSSize("tls-size", + cl::desc("Bit size of immediate TLS offsets"), + cl::init(0)); + static cl::opt<bool> EmulatedTLS("emulated-tls", cl::desc("Use emulated TLS model"), cl::init(false)); @@ -276,6 +280,11 @@ static cl::opt<bool> cl::desc("Emit debug info about parameter's entry values"), cl::init(false)); +static cl::opt<bool> + ForceDwarfFrameSection("force-dwarf-frame-section", + cl::desc("Always emit a debug frame section."), + cl::init(false)); + // Common utility function tightly tied to the options listed here. Initializes // a TargetOptions object with CodeGen flags and returns it. static TargetOptions InitTargetOptionsFromCodeGenFlags() { @@ -286,7 +295,7 @@ static TargetOptions InitTargetOptionsFromCodeGenFlags() { Options.NoNaNsFPMath = EnableNoNaNsFPMath; Options.NoSignedZerosFPMath = EnableNoSignedZerosFPMath; Options.NoTrappingFPMath = EnableNoTrappingFPMath; - Options.FPDenormalMode = DenormalMode; + Options.FPDenormalMode = DenormalFPMath; Options.HonorSignDependentRoundingFPMathOption = EnableHonorSignDependentRoundingFPMath; if (FloatABIForCalls != FloatABI::Default) @@ -300,12 +309,14 @@ static TargetOptions InitTargetOptionsFromCodeGenFlags() { Options.DataSections = DataSections; Options.FunctionSections = FunctionSections; Options.UniqueSectionNames = UniqueSectionNames; + Options.TLSSize = TLSSize; Options.EmulatedTLS = EmulatedTLS; Options.ExplicitEmulatedTLS = EmulatedTLS.getNumOccurrences() > 0; Options.ExceptionModel = ExceptionModel; Options.EmitStackSizeSection = EnableStackSizeSection; Options.EmitAddrsig = EnableAddrsig; Options.EnableDebugEntryValues = EnableDebugEntryValues; + Options.ForceDwarfFrameSection = ForceDwarfFrameSection; Options.MCOptions = InitMCTargetOptionsFromFlags(); @@ -366,46 +377,52 @@ LLVM_ATTRIBUTE_UNUSED static std::vector<std::string> getFeatureList() { return Features.getFeatures(); } +/// Set function attributes of function \p F based on CPU, Features, and command +/// line flags. +LLVM_ATTRIBUTE_UNUSED static void +setFunctionAttributes(StringRef CPU, StringRef Features, Function &F) { + auto &Ctx = F.getContext(); + AttributeList Attrs = F.getAttributes(); + AttrBuilder NewAttrs; + + if (!CPU.empty() && !F.hasFnAttribute("target-cpu")) + NewAttrs.addAttribute("target-cpu", CPU); + if (!Features.empty()) + NewAttrs.addAttribute("target-features", Features); + if (FramePointerUsage.getNumOccurrences() > 0) { + if (FramePointerUsage == llvm::FramePointer::All) + NewAttrs.addAttribute("frame-pointer", "all"); + else if (FramePointerUsage == llvm::FramePointer::NonLeaf) + NewAttrs.addAttribute("frame-pointer", "non-leaf"); + else if (FramePointerUsage == llvm::FramePointer::None) + NewAttrs.addAttribute("frame-pointer", "none"); + } + if (DisableTailCalls.getNumOccurrences() > 0) + NewAttrs.addAttribute("disable-tail-calls", + toStringRef(DisableTailCalls)); + if (StackRealign) + NewAttrs.addAttribute("stackrealign"); + + if (TrapFuncName.getNumOccurrences() > 0) + for (auto &B : F) + for (auto &I : B) + if (auto *Call = dyn_cast<CallInst>(&I)) + if (const auto *F = Call->getCalledFunction()) + if (F->getIntrinsicID() == Intrinsic::debugtrap || + F->getIntrinsicID() == Intrinsic::trap) + Call->addAttribute( + llvm::AttributeList::FunctionIndex, + Attribute::get(Ctx, "trap-func-name", TrapFuncName)); + + // Let NewAttrs override Attrs. + F.setAttributes( + Attrs.addAttributes(Ctx, AttributeList::FunctionIndex, NewAttrs)); +} + /// Set function attributes of functions in Module M based on CPU, /// Features, and command line flags. LLVM_ATTRIBUTE_UNUSED static void setFunctionAttributes(StringRef CPU, StringRef Features, Module &M) { - for (auto &F : M) { - auto &Ctx = F.getContext(); - AttributeList Attrs = F.getAttributes(); - AttrBuilder NewAttrs; - - if (!CPU.empty()) - NewAttrs.addAttribute("target-cpu", CPU); - if (!Features.empty()) - NewAttrs.addAttribute("target-features", Features); - if (FramePointerUsage.getNumOccurrences() > 0) { - if (FramePointerUsage == llvm::FramePointer::All) - NewAttrs.addAttribute("frame-pointer", "all"); - else if (FramePointerUsage == llvm::FramePointer::NonLeaf) - NewAttrs.addAttribute("frame-pointer", "non-leaf"); - else if (FramePointerUsage == llvm::FramePointer::None) - NewAttrs.addAttribute("frame-pointer", "none"); - } - if (DisableTailCalls.getNumOccurrences() > 0) - NewAttrs.addAttribute("disable-tail-calls", - toStringRef(DisableTailCalls)); - if (StackRealign) - NewAttrs.addAttribute("stackrealign"); - - if (TrapFuncName.getNumOccurrences() > 0) - for (auto &B : F) - for (auto &I : B) - if (auto *Call = dyn_cast<CallInst>(&I)) - if (const auto *F = Call->getCalledFunction()) - if (F->getIntrinsicID() == Intrinsic::debugtrap || - F->getIntrinsicID() == Intrinsic::trap) - Call->addAttribute( - llvm::AttributeList::FunctionIndex, - Attribute::get(Ctx, "trap-func-name", TrapFuncName)); - - // Let NewAttrs override Attrs. - F.setAttributes( - Attrs.addAttributes(Ctx, AttributeList::FunctionIndex, NewAttrs)); - } + for (Function &F : M) + setFunctionAttributes(CPU, Features, F); } diff --git a/llvm/include/llvm/CodeGen/DFAPacketizer.h b/llvm/include/llvm/CodeGen/DFAPacketizer.h index 705465b15c4c..9cdaedc9e861 100644 --- a/llvm/include/llvm/CodeGen/DFAPacketizer.h +++ b/llvm/include/llvm/CodeGen/DFAPacketizer.h @@ -46,43 +46,18 @@ class MCInstrDesc; class SUnit; class TargetInstrInfo; -// -------------------------------------------------------------------- -// Definitions shared between DFAPacketizer.cpp and DFAPacketizerEmitter.cpp - -// DFA_MAX_RESTERMS * DFA_MAX_RESOURCES must fit within sizeof DFAInput. -// This is verified in DFAPacketizer.cpp:DFAPacketizer::DFAPacketizer. -// -// e.g. terms x resource bit combinations that fit in uint32_t: -// 4 terms x 8 bits = 32 bits -// 3 terms x 10 bits = 30 bits -// 2 terms x 16 bits = 32 bits -// -// e.g. terms x resource bit combinations that fit in uint64_t: -// 8 terms x 8 bits = 64 bits -// 7 terms x 9 bits = 63 bits -// 6 terms x 10 bits = 60 bits -// 5 terms x 12 bits = 60 bits -// 4 terms x 16 bits = 64 bits <--- current -// 3 terms x 21 bits = 63 bits -// 2 terms x 32 bits = 64 bits -// -#define DFA_MAX_RESTERMS 4 // The max # of AND'ed resource terms. -#define DFA_MAX_RESOURCES 16 // The max # of resource bits in one term. - -using DFAInput = uint64_t; -using DFAStateInput = int64_t; - -#define DFA_TBLTYPE "int64_t" // For generating DFAStateInputTable. -// -------------------------------------------------------------------- - class DFAPacketizer { private: const InstrItineraryData *InstrItins; - Automaton<DFAInput> A; + Automaton<uint64_t> A; + /// For every itinerary, an "action" to apply to the automaton. This removes + /// the redundancy in actions between itinerary classes. + ArrayRef<unsigned> ItinActions; public: - DFAPacketizer(const InstrItineraryData *InstrItins, Automaton<uint64_t> a) : - InstrItins(InstrItins), A(std::move(a)) { + DFAPacketizer(const InstrItineraryData *InstrItins, Automaton<uint64_t> a, + ArrayRef<unsigned> ItinActions) + : InstrItins(InstrItins), A(std::move(a)), ItinActions(ItinActions) { // Start off with resource tracking disabled. A.enableTranscription(false); } @@ -99,12 +74,6 @@ public: A.enableTranscription(Track); } - // Return the DFAInput for an instruction class. - DFAInput getInsnInput(unsigned InsnClass); - - // Return the DFAInput for an instruction class input vector. - static DFAInput getInsnInput(const std::vector<unsigned> &InsnClass); - // Check if the resources occupied by a MCInstrDesc are available in // the current state. bool canReserveResources(const MCInstrDesc *MID); diff --git a/llvm/include/llvm/CodeGen/DIE.h b/llvm/include/llvm/CodeGen/DIE.h index e8e7504a6cda..40f6b041e9b3 100644 --- a/llvm/include/llvm/CodeGen/DIE.h +++ b/llvm/include/llvm/CodeGen/DIE.h @@ -78,7 +78,7 @@ public: /// object. class DIEAbbrev : public FoldingSetNode { /// Unique number for node. - unsigned Number; + unsigned Number = 0; /// Dwarf tag code. dwarf::Tag Tag; diff --git a/llvm/include/llvm/CodeGen/FastISel.h b/llvm/include/llvm/CodeGen/FastISel.h index 03d681feb7aa..d9c680392e50 100644 --- a/llvm/include/llvm/CodeGen/FastISel.h +++ b/llvm/include/llvm/CodeGen/FastISel.h @@ -540,6 +540,11 @@ protected: bool selectXRayCustomEvent(const CallInst *II); bool selectXRayTypedEvent(const CallInst *II); + bool shouldOptForSize(const MachineFunction *MF) const { + // TODO: Implement PGSO. + return MF->getFunction().hasOptSize(); + } + private: /// Handle PHI nodes in successor blocks. /// diff --git a/llvm/include/llvm/CodeGen/FaultMaps.h b/llvm/include/llvm/CodeGen/FaultMaps.h index a1e2349c413e..da56c4dd13ab 100644 --- a/llvm/include/llvm/CodeGen/FaultMaps.h +++ b/llvm/include/llvm/CodeGen/FaultMaps.h @@ -36,7 +36,8 @@ public: static const char *faultTypeToString(FaultKind); - void recordFaultingOp(FaultKind FaultTy, const MCSymbol *HandlerLabel); + void recordFaultingOp(FaultKind FaultTy, const MCSymbol *FaultingLabel, + const MCSymbol *HandlerLabel); void serializeToFaultMapSection(); void reset() { FunctionInfos.clear(); diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h index f812a2f6c585..2d41f90fe053 100644 --- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -195,7 +195,7 @@ public: /// isExportedInst - Return true if the specified value is an instruction /// exported from its block. - bool isExportedInst(const Value *V) { + bool isExportedInst(const Value *V) const { return ValueMap.count(V); } diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h index 5a44e67992ad..e56177939f46 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -220,9 +220,7 @@ class GISelCSEAnalysisWrapperPass : public MachineFunctionPass { public: static char ID; - GISelCSEAnalysisWrapperPass() : MachineFunctionPass(ID) { - initializeGISelCSEAnalysisWrapperPassPass(*PassRegistry::getPassRegistry()); - } + GISelCSEAnalysisWrapperPass(); void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 4901a3748e4a..bc9774e09acf 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -107,7 +107,7 @@ public: /// make these decisions: function formal arguments, call /// instruction args, call instruction returns and function /// returns. However, once a decision has been made on where an - /// arugment should go, exactly what happens can vary slightly. This + /// argument should go, exactly what happens can vary slightly. This /// class abstracts the differences. struct ValueHandler { ValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 4c04dc52547d..e5ee21941e23 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -36,6 +36,18 @@ struct PreferredTuple { MachineInstr *MI; }; +struct IndexedLoadStoreMatchInfo { + Register Addr; + Register Base; + Register Offset; + bool IsPre; +}; + +struct PtrAddChain { + int64_t Imm; + Register Base; +}; + class CombinerHelper { protected: MachineIRBuilder &Builder; @@ -84,6 +96,8 @@ public: /// Combine \p MI into a pre-indexed or post-indexed load/store operation if /// legal and the surrounding code makes it useful. bool tryCombineIndexedLoadStore(MachineInstr &MI); + bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo); + void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo); bool matchElideBrByInvertingCond(MachineInstr &MI); void applyElideBrByInvertingCond(MachineInstr &MI); @@ -134,7 +148,7 @@ public: /// /// For example (pre-indexed): /// - /// $addr = G_GEP $base, $offset + /// $addr = G_PTR_ADD $base, $offset /// [...] /// $val = G_LOAD $addr /// [...] @@ -150,7 +164,7 @@ public: /// /// G_STORE $val, $base /// [...] - /// $addr = G_GEP $base, $offset + /// $addr = G_PTR_ADD $base, $offset /// [...] /// $whatever = COPY $addr /// @@ -161,6 +175,9 @@ public: /// $whatever = COPY $addr bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); + bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo); + bool applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo); + /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. bool tryCombine(MachineInstr &MI); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h index dfe5a7f3177d..d44612f54ae5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h @@ -40,6 +40,10 @@ public: const APInt &DemandedElts, unsigned Depth = 0); + unsigned computeNumSignBits(Register R, const APInt &DemandedElts, + unsigned Depth = 0); + unsigned computeNumSignBits(Register R, unsigned Depth = 0); + // KnownBitsAPI KnownBits getKnownBits(Register R); // Calls getKnownBits for first operand def of MI. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index bdb92aa4689d..6a2ea05f1b08 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -483,6 +483,9 @@ private: bool translateUserOp2(const User &U, MachineIRBuilder &MIRBuilder) { return false; } + bool translateFreeze(const User &U, MachineIRBuilder &MIRBuilder) { + return false; + } /// @} diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index fd3dc743000b..59d2540dd14e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -313,6 +313,14 @@ enum { /// - RendererFnID - Custom renderer function to call GIR_CustomRenderer, + /// Render operands to the specified instruction using a custom function, + /// reading from a specific operand. + /// - InsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to get the matched operand from + /// - OpIdx - Operand index in OldInsnID the render function should read from.. + /// - RendererFnID - Custom renderer function to call + GIR_CustomOperandRenderer, + /// Render a G_CONSTANT operator as a sign-extended immediate. /// - NewInsnID - Instruction ID to modify /// - OldInsnID - Instruction ID to copy from @@ -390,6 +398,10 @@ public: GISelKnownBits *KnownBits = nullptr; MachineFunction *MF = nullptr; + virtual void setupGeneratedPerFunctionState(MachineFunction &MF) { + llvm_unreachable("TableGen should have emitted implementation"); + } + /// Setup per-MF selector state. virtual void setupMF(MachineFunction &mf, GISelKnownBits &KB, @@ -397,6 +409,7 @@ public: CoverageInfo = &covinfo; KnownBits = &KB; MF = &mf; + setupGeneratedPerFunctionState(mf); } protected: @@ -487,7 +500,7 @@ protected: bool isOperandImmEqual(const MachineOperand &MO, int64_t Value, const MachineRegisterInfo &MRI) const; - /// Return true if the specified operand is a G_GEP with a G_CONSTANT on the + /// Return true if the specified operand is a G_PTR_ADD with a G_CONSTANT on the /// right-hand side. GlobalISel's separation of pointer and integer types /// means that we don't need to worry about G_OR with equivalent semantics. bool isBaseWithConstantOffset(const MachineOperand &Root, diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index 08f2f54bcf90..f866f42344f6 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -26,6 +26,7 @@ #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -573,7 +574,8 @@ bool InstructionSelector::executeMatchTable( assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); if (!MO.isReg() || - &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != + &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum), + MRI.getType(MO.getReg())) != RBI.getRegBank(MO.getReg(), MRI, TRI)) { if (handleReject() == RejectAndGiveUp) return false; @@ -640,10 +642,15 @@ bool InstructionSelector::executeMatchTable( << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); - if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { - if (handleReject() == RejectAndGiveUp) - return false; - } + if (MO.isImm() && MO.getImm() == Value) + break; + + if (MO.isCImm() && MO.getCImm()->equalsInt(Value)) + break; + + if (handleReject() == RejectAndGiveUp) + return false; + break; } @@ -945,8 +952,27 @@ bool InstructionSelector::executeMatchTable( dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" << InsnID << "], MIs[" << OldInsnID << "], " << RendererFnID << ")\n"); + (ISel.*ISelInfo.CustomRenderers[RendererFnID])( + OutMIs[InsnID], *State.MIs[OldInsnID], + -1); // Not a source operand of the old instruction. + break; + } + case GIR_CustomOperandRenderer: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RendererFnID = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + + DEBUG_WITH_TYPE( + TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs[" + << InsnID << "], MIs[" << OldInsnID << "]->getOperand(" + << OpIdx << "), " + << RendererFnID << ")\n"); (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], - *State.MIs[OldInsnID]); + *State.MIs[OldInsnID], + OpIdx); break; } case GIR_ConstrainOperandRC: { diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 7f960e727846..dd32a3b9e38e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -46,7 +46,8 @@ public: : Builder(B), MRI(MRI), LI(LI) {} bool tryCombineAnyExt(MachineInstr &MI, - SmallVectorImpl<MachineInstr *> &DeadInsts) { + SmallVectorImpl<MachineInstr *> &DeadInsts, + SmallVectorImpl<Register> &UpdatedDefs) { assert(MI.getOpcode() == TargetOpcode::G_ANYEXT); Builder.setInstr(MI); @@ -58,6 +59,7 @@ public: if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); Builder.buildAnyExtOrTrunc(DstReg, TruncSrc); + UpdatedDefs.push_back(DstReg); markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } @@ -70,6 +72,7 @@ public: m_GSExt(m_Reg(ExtSrc)), m_GZExt(m_Reg(ExtSrc)))))) { Builder.buildInstr(ExtMI->getOpcode(), {DstReg}, {ExtSrc}); + UpdatedDefs.push_back(DstReg); markInstAndDefDead(MI, *ExtMI, DeadInsts); return true; } @@ -83,15 +86,17 @@ public: auto &CstVal = SrcMI->getOperand(1); Builder.buildConstant( DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits())); + UpdatedDefs.push_back(DstReg); markInstAndDefDead(MI, *SrcMI, DeadInsts); return true; } } - return tryFoldImplicitDef(MI, DeadInsts); + return tryFoldImplicitDef(MI, DeadInsts, UpdatedDefs); } bool tryCombineZExt(MachineInstr &MI, - SmallVectorImpl<MachineInstr *> &DeadInsts) { + SmallVectorImpl<MachineInstr *> &DeadInsts, + SmallVectorImpl<Register> &UpdatedDefs) { assert(MI.getOpcode() == TargetOpcode::G_ZEXT); Builder.setInstr(MI); @@ -108,7 +113,8 @@ public: LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); LLT SrcTy = MRI.getType(SrcReg); APInt Mask = APInt::getAllOnesValue(SrcTy.getScalarSizeInBits()); - auto MIBMask = Builder.buildConstant(DstTy, Mask.getZExtValue()); + auto MIBMask = Builder.buildConstant( + DstTy, Mask.zext(DstTy.getScalarSizeInBits())); Builder.buildAnd(DstReg, Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), MIBMask); markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); @@ -124,15 +130,17 @@ public: auto &CstVal = SrcMI->getOperand(1); Builder.buildConstant( DstReg, CstVal.getCImm()->getValue().zext(DstTy.getSizeInBits())); + UpdatedDefs.push_back(DstReg); markInstAndDefDead(MI, *SrcMI, DeadInsts); return true; } } - return tryFoldImplicitDef(MI, DeadInsts); + return tryFoldImplicitDef(MI, DeadInsts, UpdatedDefs); } bool tryCombineSExt(MachineInstr &MI, - SmallVectorImpl<MachineInstr *> &DeadInsts) { + SmallVectorImpl<MachineInstr *> &DeadInsts, + SmallVectorImpl<Register> &UpdatedDefs) { assert(MI.getOpcode() == TargetOpcode::G_SEXT); Builder.setInstr(MI); @@ -154,12 +162,40 @@ public: markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } - return tryFoldImplicitDef(MI, DeadInsts); + return tryFoldImplicitDef(MI, DeadInsts, UpdatedDefs); + } + + bool tryCombineTrunc(MachineInstr &MI, + SmallVectorImpl<MachineInstr *> &DeadInsts, + SmallVectorImpl<Register> &UpdatedDefs) { + assert(MI.getOpcode() == TargetOpcode::G_TRUNC); + + Builder.setInstr(MI); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + + // Try to fold trunc(g_constant) when the smaller constant type is legal. + // Can't use MIPattern because we don't have a specific constant in mind. + auto *SrcMI = MRI.getVRegDef(SrcReg); + if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { + const LLT &DstTy = MRI.getType(DstReg); + if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) { + auto &CstVal = SrcMI->getOperand(1); + Builder.buildConstant( + DstReg, CstVal.getCImm()->getValue().trunc(DstTy.getSizeInBits())); + UpdatedDefs.push_back(DstReg); + markInstAndDefDead(MI, *SrcMI, DeadInsts); + return true; + } + } + + return false; } /// Try to fold G_[ASZ]EXT (G_IMPLICIT_DEF). bool tryFoldImplicitDef(MachineInstr &MI, - SmallVectorImpl<MachineInstr *> &DeadInsts) { + SmallVectorImpl<MachineInstr *> &DeadInsts, + SmallVectorImpl<Register> &UpdatedDefs) { unsigned Opcode = MI.getOpcode(); assert(Opcode == TargetOpcode::G_ANYEXT || Opcode == TargetOpcode::G_ZEXT || Opcode == TargetOpcode::G_SEXT); @@ -176,6 +212,7 @@ public: return false; LLVM_DEBUG(dbgs() << ".. Combine G_ANYEXT(G_IMPLICIT_DEF): " << MI;); Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, {DstReg}, {}); + UpdatedDefs.push_back(DstReg); } else { // G_[SZ]EXT (G_IMPLICIT_DEF) -> G_CONSTANT 0 because the top // bits will be 0 for G_ZEXT and 0/1 for the G_SEXT. @@ -183,6 +220,7 @@ public: return false; LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI;); Builder.buildConstant(DstReg, 0); + UpdatedDefs.push_back(DstReg); } markInstAndDefDead(MI, *DefMI, DeadInsts); @@ -191,37 +229,60 @@ public: return false; } - static unsigned canFoldMergeOpcode(unsigned MergeOp, unsigned ConvertOp, - LLT OpTy, LLT DestTy) { - if (OpTy.isVector() && DestTy.isVector()) - return MergeOp == TargetOpcode::G_CONCAT_VECTORS; - - if (OpTy.isVector() && !DestTy.isVector()) { - if (MergeOp == TargetOpcode::G_BUILD_VECTOR) + static bool canFoldMergeOpcode(unsigned MergeOp, unsigned ConvertOp, + LLT OpTy, LLT DestTy) { + // Check if we found a definition that is like G_MERGE_VALUES. + switch (MergeOp) { + default: + return false; + case TargetOpcode::G_BUILD_VECTOR: + case TargetOpcode::G_MERGE_VALUES: + // The convert operation that we will need to insert is + // going to convert the input of that type of instruction (scalar) + // to the destination type (DestTy). + // The conversion needs to stay in the same domain (scalar to scalar + // and vector to vector), so if we were to allow to fold the merge + // we would need to insert some bitcasts. + // E.g., + // <2 x s16> = build_vector s16, s16 + // <2 x s32> = zext <2 x s16> + // <2 x s16>, <2 x s16> = unmerge <2 x s32> + // + // As is the folding would produce: + // <2 x s16> = zext s16 <-- scalar to vector + // <2 x s16> = zext s16 <-- scalar to vector + // Which is invalid. + // Instead we would want to generate: + // s32 = zext s16 + // <2 x s16> = bitcast s32 + // s32 = zext s16 + // <2 x s16> = bitcast s32 + // + // That is not done yet. + if (ConvertOp == 0) return true; + return !DestTy.isVector(); + case TargetOpcode::G_CONCAT_VECTORS: { + if (ConvertOp == 0) + return true; + if (!DestTy.isVector()) + return false; - if (MergeOp == TargetOpcode::G_CONCAT_VECTORS) { - if (ConvertOp == 0) - return true; - - const unsigned OpEltSize = OpTy.getElementType().getSizeInBits(); - - // Don't handle scalarization with a cast that isn't in the same - // direction as the vector cast. This could be handled, but it would - // require more intermediate unmerges. - if (ConvertOp == TargetOpcode::G_TRUNC) - return DestTy.getSizeInBits() <= OpEltSize; - return DestTy.getSizeInBits() >= OpEltSize; - } + const unsigned OpEltSize = OpTy.getElementType().getSizeInBits(); - return false; + // Don't handle scalarization with a cast that isn't in the same + // direction as the vector cast. This could be handled, but it would + // require more intermediate unmerges. + if (ConvertOp == TargetOpcode::G_TRUNC) + return DestTy.getSizeInBits() <= OpEltSize; + return DestTy.getSizeInBits() >= OpEltSize; + } } - - return MergeOp == TargetOpcode::G_MERGE_VALUES; } bool tryCombineMerges(MachineInstr &MI, - SmallVectorImpl<MachineInstr *> &DeadInsts) { + SmallVectorImpl<MachineInstr *> &DeadInsts, + SmallVectorImpl<Register> &UpdatedDefs) { assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); unsigned NumDefs = MI.getNumOperands() - 1; @@ -271,8 +332,8 @@ public: SmallVector<Register, 2> TmpRegs; // This is a vector that is being scalarized and casted. Extract to // the element type, and do the conversion on the scalars. - LLT MergeEltTy - = MRI.getType(MergeI->getOperand(0).getReg()).getElementType(); + LLT MergeEltTy = + MRI.getType(MergeI->getOperand(0).getReg()).getElementType(); for (unsigned j = 0; j < NumMergeRegs; ++j) TmpRegs.push_back(MRI.createGenericVirtualRegister(MergeEltTy)); @@ -283,6 +344,7 @@ public: } else { Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg()); } + UpdatedDefs.append(DstRegs.begin(), DstRegs.end()); } } else if (NumMergeRegs > NumDefs) { @@ -304,31 +366,40 @@ public: ++j, ++Idx) Regs.push_back(MergeI->getOperand(Idx).getReg()); - Builder.buildMerge(MI.getOperand(DefIdx).getReg(), Regs); + Register DefReg = MI.getOperand(DefIdx).getReg(); + Builder.buildMerge(DefReg, Regs); + UpdatedDefs.push_back(DefReg); } } else { LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg()); + + if (!ConvertOp && DestTy != MergeSrcTy) + ConvertOp = TargetOpcode::G_BITCAST; + if (ConvertOp) { Builder.setInstr(MI); for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { Register MergeSrc = MergeI->getOperand(Idx + 1).getReg(); - Builder.buildInstr(ConvertOp, {MI.getOperand(Idx).getReg()}, - {MergeSrc}); + Register DefReg = MI.getOperand(Idx).getReg(); + Builder.buildInstr(ConvertOp, {DefReg}, {MergeSrc}); + UpdatedDefs.push_back(DefReg); } markInstAndDefDead(MI, *MergeI, DeadInsts); return true; } - // FIXME: is a COPY appropriate if the types mismatch? We know both - // registers are allocatable by now. - if (DestTy != MergeSrcTy) - return false; - for (unsigned Idx = 0; Idx < NumDefs; ++Idx) - MRI.replaceRegWith(MI.getOperand(Idx).getReg(), - MergeI->getOperand(Idx + 1).getReg()); + assert(DestTy == MergeSrcTy && + "Bitcast and the other kinds of conversions should " + "have happened earlier"); + + for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { + Register NewDef = MergeI->getOperand(Idx + 1).getReg(); + MRI.replaceRegWith(MI.getOperand(Idx).getReg(), NewDef); + UpdatedDefs.push_back(NewDef); + } } markInstAndDefDead(MI, *MergeI, DeadInsts); @@ -347,7 +418,8 @@ public: } bool tryCombineExtract(MachineInstr &MI, - SmallVectorImpl<MachineInstr *> &DeadInsts) { + SmallVectorImpl<MachineInstr *> &DeadInsts, + SmallVectorImpl<Register> &UpdatedDefs) { assert(MI.getOpcode() == TargetOpcode::G_EXTRACT); // Try to use the source registers from a G_MERGE_VALUES @@ -362,13 +434,14 @@ public: // for N >= %2.getSizeInBits() / 2 // %3 = G_EXTRACT %1, (N - %0.getSizeInBits() - unsigned Src = lookThroughCopyInstrs(MI.getOperand(1).getReg()); - MachineInstr *MergeI = MRI.getVRegDef(Src); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + MachineInstr *MergeI = MRI.getVRegDef(SrcReg); if (!MergeI || !isMergeLikeOpcode(MergeI->getOpcode())) return false; - LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); - LLT SrcTy = MRI.getType(Src); + Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + LLT SrcTy = MRI.getType(SrcReg); // TODO: Do we need to check if the resulting extract is supported? unsigned ExtractDstSize = DstTy.getSizeInBits(); @@ -386,10 +459,9 @@ public: // TODO: We could modify MI in place in most cases. Builder.setInstr(MI); - Builder.buildExtract( - MI.getOperand(0).getReg(), - MergeI->getOperand(MergeSrcIdx + 1).getReg(), - Offset - MergeSrcIdx * MergeSrcSize); + Builder.buildExtract(DstReg, MergeI->getOperand(MergeSrcIdx + 1).getReg(), + Offset - MergeSrcIdx * MergeSrcSize); + UpdatedDefs.push_back(DstReg); markInstAndDefDead(MI, *MergeI, DeadInsts); return true; } @@ -406,30 +478,79 @@ public: // etc, process the dead instructions now if any. if (!DeadInsts.empty()) deleteMarkedDeadInsts(DeadInsts, WrapperObserver); + + // Put here every vreg that was redefined in such a way that it's at least + // possible that one (or more) of its users (immediate or COPY-separated) + // could become artifact combinable with the new definition (or the + // instruction reachable from it through a chain of copies if any). + SmallVector<Register, 4> UpdatedDefs; + bool Changed = false; switch (MI.getOpcode()) { default: return false; case TargetOpcode::G_ANYEXT: - return tryCombineAnyExt(MI, DeadInsts); + Changed = tryCombineAnyExt(MI, DeadInsts, UpdatedDefs); + break; case TargetOpcode::G_ZEXT: - return tryCombineZExt(MI, DeadInsts); + Changed = tryCombineZExt(MI, DeadInsts, UpdatedDefs); + break; case TargetOpcode::G_SEXT: - return tryCombineSExt(MI, DeadInsts); + Changed = tryCombineSExt(MI, DeadInsts, UpdatedDefs); + break; case TargetOpcode::G_UNMERGE_VALUES: - return tryCombineMerges(MI, DeadInsts); + Changed = tryCombineMerges(MI, DeadInsts, UpdatedDefs); + break; case TargetOpcode::G_EXTRACT: - return tryCombineExtract(MI, DeadInsts); - case TargetOpcode::G_TRUNC: { - bool Changed = false; - for (auto &Use : MRI.use_instructions(MI.getOperand(0).getReg())) - Changed |= tryCombineInstruction(Use, DeadInsts, WrapperObserver); - return Changed; + Changed = tryCombineExtract(MI, DeadInsts, UpdatedDefs); + break; + case TargetOpcode::G_TRUNC: + Changed = tryCombineTrunc(MI, DeadInsts, UpdatedDefs); + if (!Changed) { + // Try to combine truncates away even if they are legal. As all artifact + // combines at the moment look only "up" the def-use chains, we achieve + // that by throwing truncates' users (with look through copies) into the + // ArtifactList again. + UpdatedDefs.push_back(MI.getOperand(0).getReg()); + } + break; } + // If the main loop through the ArtifactList found at least one combinable + // pair of artifacts, not only combine it away (as done above), but also + // follow the def-use chain from there to combine everything that can be + // combined within this def-use chain of artifacts. + while (!UpdatedDefs.empty()) { + Register NewDef = UpdatedDefs.pop_back_val(); + assert(NewDef.isVirtual() && "Unexpected redefinition of a physreg"); + for (MachineInstr &Use : MRI.use_instructions(NewDef)) { + switch (Use.getOpcode()) { + // Keep this list in sync with the list of all artifact combines. + case TargetOpcode::G_ANYEXT: + case TargetOpcode::G_ZEXT: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_UNMERGE_VALUES: + case TargetOpcode::G_EXTRACT: + case TargetOpcode::G_TRUNC: + // Adding Use to ArtifactList. + WrapperObserver.changedInstr(Use); + break; + case TargetOpcode::COPY: { + Register Copy = Use.getOperand(0).getReg(); + if (Copy.isVirtual()) + UpdatedDefs.push_back(Copy); + break; + } + default: + // If we do not have an artifact combine for the opcode, there is no + // point in adding it to the ArtifactList as nothing interesting will + // be done to it anyway. + break; + } + } } + return Changed; } private: - static unsigned getArtifactSrcReg(const MachineInstr &MI) { switch (MI.getOpcode()) { case TargetOpcode::COPY: diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h index 13cf3f7e694d..07173b9719bd 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h @@ -31,8 +31,12 @@ class Legalizer : public MachineFunctionPass { public: static char ID; -private: + struct MFResult { + bool Changed; + const MachineInstr *FailedOn; + }; +private: /// Initialize the field members using \p MF. void init(MachineFunction &MF); @@ -55,14 +59,19 @@ public: } MachineFunctionProperties getClearedProperties() const override { - return MachineFunctionProperties() - .set(MachineFunctionProperties::Property::NoPHIs); + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::NoPHIs); } bool combineExtracts(MachineInstr &MI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII); bool runOnMachineFunction(MachineFunction &MF) override; + + static MFResult + legalizeMachineFunction(MachineFunction &MF, const LegalizerInfo &LI, + ArrayRef<GISelChangeObserver *> AuxObservers, + MachineIRBuilder &MIRBuilder); }; } // End namespace llvm. diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index fbfe71255a38..ac7e5cbbdaa9 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -231,12 +231,16 @@ public: LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); LegalizeResult lowerFMad(MachineInstr &MI); + LegalizeResult lowerIntrinsicRound(MachineInstr &MI); LegalizeResult lowerUnmergeValues(MachineInstr &MI); LegalizeResult lowerShuffleVector(MachineInstr &MI); LegalizeResult lowerDynStackAlloc(MachineInstr &MI); LegalizeResult lowerExtract(MachineInstr &MI); LegalizeResult lowerInsert(MachineInstr &MI); LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); + LegalizeResult lowerBswap(MachineInstr &MI); + LegalizeResult lowerBitreverse(MachineInstr &MI); + LegalizeResult lowerReadRegister(MachineInstr &MI); private: MachineRegisterInfo &MRI; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 1cf62d1fde59..29f0d6575bac 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -22,8 +22,9 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/TargetOpcodes.h" -#include "llvm/Support/raw_ostream.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/LowLevelTypeImpl.h" +#include "llvm/Support/raw_ostream.h" #include <cassert> #include <cstdint> #include <tuple> @@ -660,6 +661,7 @@ public: /// The instruction is unsupported. LegalizeRuleSet &unsupported() { + markAllIdxsAsCovered(); return actionIf(LegalizeAction::Unsupported, always); } LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { @@ -1156,6 +1158,12 @@ public: virtual bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const; + /// Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while + /// widening a constant of type SmallTy which targets can override. + /// For eg, the DAG does (SmallTy.isByteSized() ? G_SEXT : G_ZEXT) which + /// will be the default. + virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const; + private: /// Determine what action should be taken to legalize the given generic /// instruction opcode, type-index and type. Requires computeTables to have @@ -1178,7 +1186,7 @@ private: /// {65, NarrowScalar} // bit sizes [65, +inf[ /// }); /// It may be that only 64-bit pointers are supported on your target: - /// setPointerAction(G_GEP, 0, LLT:pointer(1), + /// setPointerAction(G_PTR_ADD, 0, LLT:pointer(1), /// {{1, Unsupported}, // bit sizes [ 1, 63[ /// {64, Legal}, // bit sizes [64, 65[ /// {65, Unsupported}, // bit sizes [65, +inf[ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h index 06de5800b8b7..ad1904725dcd 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h @@ -42,6 +42,10 @@ public: static char ID; private: + /// An input function to decide if the pass should run or not + /// on the given MachineFunction. + std::function<bool(const MachineFunction &)> DoNotRunPass; + /// MRI contains all the register class/bank information that this /// pass uses and updates. MachineRegisterInfo *MRI; @@ -72,6 +76,7 @@ private: public: Localizer(); + Localizer(std::function<bool(const MachineFunction &)>); StringRef getPassName() const override { return "Localizer"; } diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 416f9c19f794..072a1411de8a 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -404,10 +404,11 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildGlobalValue(const DstOp &Res, const GlobalValue *GV); - /// Build and insert \p Res = G_GEP \p Op0, \p Op1 + /// Build and insert \p Res = G_PTR_ADD \p Op0, \p Op1 /// - /// G_GEP adds \p Op1 bytes to the pointer specified by \p Op0, - /// storing the resulting pointer in \p Res. + /// G_PTR_ADD adds \p Op1 addressible units to the pointer specified by \p Op0, + /// storing the resulting pointer in \p Res. Addressible units are typically + /// bytes but this can vary between targets. /// /// \pre setBasicBlock or setMI must have been called. /// \pre \p Res and \p Op0 must be generic virtual registers with pointer @@ -415,28 +416,28 @@ public: /// \pre \p Op1 must be a generic virtual register with scalar type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildGEP(const DstOp &Res, const SrcOp &Op0, - const SrcOp &Op1); + MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, + const SrcOp &Op1); - /// Materialize and insert \p Res = G_GEP \p Op0, (G_CONSTANT \p Value) + /// Materialize and insert \p Res = G_PTR_ADD \p Op0, (G_CONSTANT \p Value) /// - /// G_GEP adds \p Value bytes to the pointer specified by \p Op0, + /// G_PTR_ADD adds \p Value bytes to the pointer specified by \p Op0, /// storing the resulting pointer in \p Res. If \p Value is zero then no - /// G_GEP or G_CONSTANT will be created and \pre Op0 will be assigned to + /// G_PTR_ADD or G_CONSTANT will be created and \pre Op0 will be assigned to /// \p Res. /// /// \pre setBasicBlock or setMI must have been called. /// \pre \p Op0 must be a generic virtual register with pointer type. /// \pre \p ValueTy must be a scalar type. /// \pre \p Res must be 0. This is to detect confusion between - /// materializeGEP() and buildGEP(). + /// materializePtrAdd() and buildPtrAdd(). /// \post \p Res will either be a new generic virtual register of the same /// type as \p Op0 or \p Op0 itself. /// /// \return a MachineInstrBuilder for the newly created instruction. - Optional<MachineInstrBuilder> materializeGEP(Register &Res, Register Op0, - const LLT &ValueTy, - uint64_t Value); + Optional<MachineInstrBuilder> materializePtrAdd(Register &Res, Register Op0, + const LLT &ValueTy, + uint64_t Value); /// Build and insert \p Res = G_PTR_MASK \p Op0, \p NumBits /// @@ -517,6 +518,13 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op); + /// Build and insert \p Res = G_FPEXT \p Op + MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FPEXT, {Res}, {Op}, Flags); + } + + /// Build and insert a G_PTRTOINT instruction. MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src) { return buildInstr(TargetOpcode::G_PTRTOINT, {Dst}, {Src}); @@ -867,7 +875,8 @@ public: /// \pre \p Res must be smaller than \p Op /// /// \return The newly created instruction. - MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op); + MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, + Optional<unsigned> FLags = None); /// Build and insert \p Res = G_TRUNC \p Op /// @@ -1374,8 +1383,9 @@ public: /// Build and insert \p Res = G_FMA \p Op0, \p Op1, \p Op2 MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, - const SrcOp &Src1, const SrcOp &Src2) { - return buildInstr(TargetOpcode::G_FMA, {Dst}, {Src0, Src1, Src2}); + const SrcOp &Src1, const SrcOp &Src2, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_FMA, {Dst}, {Src0, Src1, Src2}, Flags); } /// Build and insert \p Res = G_FMAD \p Op0, \p Op1, \p Op2 @@ -1403,6 +1413,12 @@ public: return buildInstr(TargetOpcode::G_FCANONICALIZE, {Dst}, {Src0}, Flags); } + /// Build and insert \p Dst = G_INTRINSIC_TRUNC \p Src0 + MachineInstrBuilder buildIntrinsicTrunc(const DstOp &Dst, const SrcOp &Src0, + Optional<unsigned> Flags = None) { + return buildInstr(TargetOpcode::G_INTRINSIC_TRUNC, {Dst}, {Src0}, Flags); + } + /// Build and insert \p Res = G_FCOPYSIGN \p Op0, \p Op1 MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1) { diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h index e84b1c3ea8b1..8725d96efd82 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h @@ -20,6 +20,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/CodeGen/Register.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/LowLevelTypeImpl.h" #include <cassert> #include <initializer_list> #include <memory> @@ -543,7 +544,7 @@ public: const RegisterBank * getRegBankFromConstraints(const MachineInstr &MI, unsigned OpIdx, const TargetInstrInfo &TII, - const TargetRegisterInfo &TRI) const; + const MachineRegisterInfo &MRI) const; /// Helper method to apply something that is like the default mapping. /// Basically, that means that \p OpdMapper.getMI() is left untouched @@ -599,7 +600,7 @@ public: /// /// \todo This should be TableGen'ed. virtual const RegisterBank & - getRegBankFromRegClass(const TargetRegisterClass &RC) const { + getRegBankFromRegClass(const TargetRegisterClass &RC, LLT Ty) const { llvm_unreachable("The target must override this method"); } diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index 8af2853473c2..429d6db20e0c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -168,10 +168,5 @@ inline bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI) { return isKnownNeverNaN(Val, MRI, true); } -/// Get a rough equivalent of an MVT for a given LLT. -MVT getMVTForLLT(LLT Ty); -/// Get a rough equivalent of an LLT for a given MVT. -LLT getLLTForMVT(MVT Ty); - } // End namespace llvm. #endif diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 658ad31fa2a6..06140fae8790 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -13,6 +13,8 @@ #ifndef LLVM_CODEGEN_ISDOPCODES_H #define LLVM_CODEGEN_ISDOPCODES_H +#include "llvm/CodeGen/ValueTypes.h" + namespace llvm { /// ISD namespace - This namespace contains an enum which represents all of the @@ -283,6 +285,12 @@ namespace ISD { /// bits of the first 2 operands. SMULFIXSAT, UMULFIXSAT, + /// RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on + /// 2 integers with the same width and scale. SCALE represents the scale + /// of both operands as fixed point numbers. This SCALE parameter must be a + /// constant integer. + SDIVFIX, UDIVFIX, + /// Simple binary floating point operators. FADD, FSUB, FMUL, FDIV, FREM, @@ -302,6 +310,7 @@ namespace ISD { STRICT_FRINT, STRICT_FNEARBYINT, STRICT_FMAXNUM, STRICT_FMINNUM, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND, STRICT_FTRUNC, STRICT_LROUND, STRICT_LLROUND, STRICT_LRINT, STRICT_LLRINT, + STRICT_FMAXIMUM, STRICT_FMINIMUM, /// STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or /// unsigned integer. These have the same semantics as fptosi and fptoui @@ -310,6 +319,13 @@ namespace ISD { STRICT_FP_TO_SINT, STRICT_FP_TO_UINT, + /// STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to + /// a floating point value. These have the same semantics as sitofp and + /// uitofp in IR. + /// They are used to limit optimizations while the DAG is being optimized. + STRICT_SINT_TO_FP, + STRICT_UINT_TO_FP, + /// X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating /// point type down to the precision of the destination VT. TRUNC is a /// flag, which is always an integer that is zero or one. If TRUNC is 0, @@ -330,6 +346,12 @@ namespace ISD { /// It is used to limit optimizations while the DAG is being optimized. STRICT_FP_EXTEND, + /// STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used + /// for floating-point operands only. STRICT_FSETCC performs a quiet + /// comparison operation, while STRICT_FSETCCS performs a signaling + /// comparison operation. + STRICT_FSETCC, STRICT_FSETCCS, + /// FMA - Perform a * b + c with no intermediate rounding step. FMA, @@ -921,11 +943,16 @@ namespace ISD { BUILTIN_OP_END }; + /// FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations + /// which cannot raise FP exceptions should be less than this value. + /// Those that do must not be less than this value. + static const int FIRST_TARGET_STRICTFP_OPCODE = BUILTIN_OP_END+400; + /// FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations /// which do not reference a specific memory location should be less than /// this value. Those that do must not be less than this value, and can /// be used with SelectionDAG::getMemIntrinsicNode. - static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+400; + static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+500; //===--------------------------------------------------------------------===// /// MemIndexedMode enum - This enum defines the load / store indexed @@ -1076,7 +1103,17 @@ namespace ISD { /// Return the operation corresponding to !(X op Y), where 'op' is a valid /// SetCC operation. - CondCode getSetCCInverse(CondCode Operation, bool isInteger); + CondCode getSetCCInverse(CondCode Operation, EVT Type); + + namespace GlobalISel { + /// Return the operation corresponding to !(X op Y), where 'op' is a valid + /// SetCC operation. The U bit of the condition code has different meanings + /// between floating point and integer comparisons and LLT's don't provide + /// this distinction. As such we need to be told whether the comparison is + /// floating point or integer-like. Pointers should use integer-like + /// comparisons. + CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike); + } // end namespace GlobalISel /// Return the operation corresponding to (Y op X) when given the operation /// for (X op Y). @@ -1085,12 +1122,12 @@ namespace ISD { /// Return the result of a logical OR between different comparisons of /// identical values: ((X op1 Y) | (X op2 Y)). This function returns /// SETCC_INVALID if it is not possible to represent the resultant comparison. - CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, bool isInteger); + CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type); /// Return the result of a logical AND between different comparisons of /// identical values: ((X op1 Y) & (X op2 Y)). This function returns /// SETCC_INVALID if it is not possible to represent the resultant comparison. - CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, bool isInteger); + CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type); } // end llvm::ISD namespace diff --git a/llvm/include/llvm/CodeGen/LiveInterval.h b/llvm/include/llvm/CodeGen/LiveInterval.h index 290a2381d9c9..fe5adb59dac2 100644 --- a/llvm/include/llvm/CodeGen/LiveInterval.h +++ b/llvm/include/llvm/CodeGen/LiveInterval.h @@ -836,10 +836,35 @@ namespace llvm { /// don't defne the related lane masks after they get shrunk. E.g., /// when L000F gets split into L0007 and L0008 maybe only a subset /// of the VNIs that defined L000F defines L0007. + /// + /// The clean up of the VNIs need to look at the actual instructions + /// to decide what is or is not live at a definition point. If the + /// update of the subranges occurs while the IR does not reflect these + /// changes, \p ComposeSubRegIdx can be used to specify how the + /// definition are going to be rewritten. + /// E.g., let say we want to merge: + /// V1.sub1:<2 x s32> = COPY V2.sub3:<4 x s32> + /// We do that by choosing a class where sub1:<2 x s32> and sub3:<4 x s32> + /// overlap, i.e., by choosing a class where we can find "offset + 1 == 3". + /// Put differently we align V2's sub3 with V1's sub1: + /// V2: sub0 sub1 sub2 sub3 + /// V1: <offset> sub0 sub1 + /// + /// This offset will look like a composed subregidx in the the class: + /// V1.(composed sub2 with sub1):<4 x s32> = COPY V2.sub3:<4 x s32> + /// => V1.(composed sub2 with sub1):<4 x s32> = COPY V2.sub3:<4 x s32> + /// + /// Now if we didn't rewrite the uses and def of V1, all the checks for V1 + /// need to account for this offset. + /// This happens during coalescing where we update the live-ranges while + /// still having the old IR around because updating the IR on-the-fly + /// would actually clobber some information on how the live-ranges that + /// are being updated look like. void refineSubRanges(BumpPtrAllocator &Allocator, LaneBitmask LaneMask, std::function<void(LiveInterval::SubRange &)> Apply, const SlotIndexes &Indexes, - const TargetRegisterInfo &TRI); + const TargetRegisterInfo &TRI, + unsigned ComposeSubRegIdx = 0); bool operator<(const LiveInterval& other) const { const SlotIndex &thisIndex = beginIndex(); diff --git a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h index 05506d2c3bc6..c555763a4ec2 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h +++ b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h @@ -75,6 +75,7 @@ public: bool empty() const { return Segments.empty(); } SlotIndex startIndex() const { return Segments.start(); } + SlotIndex endIndex() const { return Segments.stop(); } // Provide public access to the underlying map to allow overlap iteration. using Map = LiveSegments; diff --git a/llvm/include/llvm/CodeGen/LiveIntervals.h b/llvm/include/llvm/CodeGen/LiveIntervals.h index 888d72b87bd1..2bfc99624937 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervals.h +++ b/llvm/include/llvm/CodeGen/LiveIntervals.h @@ -469,7 +469,7 @@ class VirtRegMap; void computeLiveInRegUnits(); void computeRegUnitRange(LiveRange&, unsigned Unit); - void computeVirtRegInterval(LiveInterval&); + bool computeVirtRegInterval(LiveInterval&); using ShrinkToUsesWorkList = SmallVector<std::pair<SlotIndex, VNInfo*>, 16>; void extendSegmentsToUses(LiveRange &Segments, diff --git a/llvm/include/llvm/CodeGen/LivePhysRegs.h b/llvm/include/llvm/CodeGen/LivePhysRegs.h index 50da0b3d5c48..085893462a08 100644 --- a/llvm/include/llvm/CodeGen/LivePhysRegs.h +++ b/llvm/include/llvm/CodeGen/LivePhysRegs.h @@ -137,6 +137,9 @@ public: /// Live out registers are the union of the live-in registers of the successor /// blocks and pristine registers. Live out registers of the end block are the /// callee saved registers. + /// If a register is not added by this method, it is guaranteed to not be + /// live out from MBB, although a sub-register may be. This is true + /// both before and after regalloc. void addLiveOuts(const MachineBasicBlock &MBB); /// Adds all live-out registers of basic block \p MBB but skips pristine diff --git a/llvm/include/llvm/CodeGen/LiveRegUnits.h b/llvm/include/llvm/CodeGen/LiveRegUnits.h index 314afad92970..1ed091e3bb5e 100644 --- a/llvm/include/llvm/CodeGen/LiveRegUnits.h +++ b/llvm/include/llvm/CodeGen/LiveRegUnits.h @@ -160,6 +160,19 @@ private: void addPristines(const MachineFunction &MF); }; +/// Returns an iterator range over all physical register and mask operands for +/// \p MI and bundled instructions. This also skips any debug operands. +inline iterator_range<filter_iterator< + ConstMIBundleOperands, std::function<bool(const MachineOperand &)>>> +phys_regs_and_masks(const MachineInstr &MI) { + std::function<bool(const MachineOperand &)> Pred = + [](const MachineOperand &MOP) { + return MOP.isRegMask() || (MOP.isReg() && !MOP.isDebug() && + Register::isPhysicalRegister(MOP.getReg())); + }; + return make_filter_range(const_mi_bundle_ops(MI), Pred); +} + } // end namespace llvm #endif // LLVM_CODEGEN_LIVEREGUNITS_H diff --git a/llvm/include/llvm/CodeGen/LiveStacks.h b/llvm/include/llvm/CodeGen/LiveStacks.h index 7c4c64d515df..1cbdb8bd86bd 100644 --- a/llvm/include/llvm/CodeGen/LiveStacks.h +++ b/llvm/include/llvm/CodeGen/LiveStacks.h @@ -17,6 +17,7 @@ #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include <cassert> #include <map> diff --git a/llvm/include/llvm/CodeGen/LiveVariables.h b/llvm/include/llvm/CodeGen/LiveVariables.h index 71de306e2942..7b45f7d76af5 100644 --- a/llvm/include/llvm/CodeGen/LiveVariables.h +++ b/llvm/include/llvm/CodeGen/LiveVariables.h @@ -36,6 +36,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/InitializePasses.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/LowLevelType.h b/llvm/include/llvm/CodeGen/LowLevelType.h index 687233e4e168..6295d86f749c 100644 --- a/llvm/include/llvm/CodeGen/LowLevelType.h +++ b/llvm/include/llvm/CodeGen/LowLevelType.h @@ -17,6 +17,7 @@ #define LLVM_CODEGEN_LOWLEVELTYPE_H #include "llvm/Support/LowLevelTypeImpl.h" +#include "llvm/Support/MachineValueType.h" namespace llvm { @@ -26,6 +27,14 @@ class Type; /// Construct a low-level type based on an LLVM type. LLT getLLTForType(Type &Ty, const DataLayout &DL); +/// Get a rough equivalent of an MVT for a given LLT. MVT can't distinguish +/// pointers, so these will convert to a plain integer. +MVT getMVTForLLT(LLT Ty); + +/// Get a rough equivalent of an LLT for a given MVT. LLT does not yet support +/// scalarable vector types, and will assert if used. +LLT getLLTForMVT(MVT Ty); + } #endif // LLVM_CODEGEN_LOWLEVELTYPE_H diff --git a/llvm/include/llvm/CodeGen/MIRFormatter.h b/llvm/include/llvm/CodeGen/MIRFormatter.h new file mode 100644 index 000000000000..e57c32c5ae61 --- /dev/null +++ b/llvm/include/llvm/CodeGen/MIRFormatter.h @@ -0,0 +1,83 @@ +//===-- llvm/CodeGen/MIRFormatter.h -----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains the declaration of the MIRFormatter class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MIRFORMATTER_H +#define LLVM_CODEGEN_MIRFORMATTER_H + +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/PseudoSourceValue.h" + +namespace llvm { + +struct PerFunctionMIParsingState; +struct SlotMapping; + +/// MIRFormater - Interface to format MIR operand based on target +class MIRFormatter { +public: + typedef function_ref<bool(StringRef::iterator Loc, const Twine &)> + ErrorCallbackType; + + MIRFormatter() {} + virtual ~MIRFormatter() = default; + + /// Implement target specific printing for machine operand immediate value, so + /// that we can have more meaningful mnemonic than a 64-bit integer. Passing + /// None to OpIdx means the index is unknown. + virtual void printImm(raw_ostream &OS, const MachineInstr &MI, + Optional<unsigned> OpIdx, int64_t Imm) const { + OS << Imm; + } + + /// Implement target specific parsing of immediate mnemonics. The mnemonic is + /// dot seperated strings. + virtual bool parseImmMnemonic(const unsigned OpCode, const unsigned OpIdx, + StringRef Src, int64_t &Imm, + ErrorCallbackType ErrorCallback) const { + llvm_unreachable("target did not implement parsing MIR immediate mnemonic"); + } + + /// Implement target specific printing of target custom pseudo source value. + /// Default implementation is not necessarily the correct MIR serialization + /// format. + virtual void + printCustomPseudoSourceValue(raw_ostream &OS, ModuleSlotTracker &MST, + const PseudoSourceValue &PSV) const { + PSV.printCustom(OS); + } + + /// Implement target specific parsing of target custom pseudo source value. + virtual bool parseCustomPseudoSourceValue( + StringRef Src, MachineFunction &MF, PerFunctionMIParsingState &PFS, + const PseudoSourceValue *&PSV, ErrorCallbackType ErrorCallback) const { + llvm_unreachable( + "target did not implement parsing MIR custom pseudo source value"); + } + + /// Helper functions to print IR value as MIR serialization format which will + /// be useful for target specific printer, e.g. for printing IR value in + /// custom pseudo source value. + static void printIRValue(raw_ostream &OS, const Value &V, + ModuleSlotTracker &MST); + + /// Helper functions to parse IR value from MIR serialization format which + /// will be useful for target specific parser, e.g. for parsing IR value for + /// custom pseudo source value. + static bool parseIRValue(StringRef Src, MachineFunction &MF, + PerFunctionMIParsingState &PFS, const Value *&V, + ErrorCallbackType ErrorCallback); +}; + +} // end namespace llvm + +#endif diff --git a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h index 4e32a04551c1..8ca665b23b28 100644 --- a/llvm/include/llvm/CodeGen/MIRParser/MIParser.h +++ b/llvm/include/llvm/CodeGen/MIRParser/MIParser.h @@ -171,12 +171,16 @@ struct PerFunctionMIParsingState { DenseMap<unsigned, unsigned> ConstantPoolSlots; DenseMap<unsigned, unsigned> JumpTableSlots; + /// Maps from slot numbers to function's unnamed values. + DenseMap<unsigned, const Value *> Slots2Values; + PerFunctionMIParsingState(MachineFunction &MF, SourceMgr &SM, const SlotMapping &IRSlots, PerTargetMIParsingState &Target); VRegInfo &getVRegInfo(unsigned Num); VRegInfo &getVRegInfoNamed(StringRef RegName); + const Value *getIRValue(unsigned Slot); }; /// Parse the machine basic block definitions, and skip the machine diff --git a/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h b/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h index 6a04e48e533c..385baea0446f 100644 --- a/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h +++ b/llvm/include/llvm/CodeGen/MIRParser/MIRParser.h @@ -23,10 +23,11 @@ namespace llvm { -class StringRef; +class Function; class MIRParserImpl; class MachineModuleInfo; class SMDiagnostic; +class StringRef; /// This class initializes machine functions by applying the state loaded from /// a MIR file. @@ -60,9 +61,11 @@ public: /// \param Filename - The name of the file to parse. /// \param Error - Error result info. /// \param Context - Context which will be used for the parsed LLVM IR module. -std::unique_ptr<MIRParser> createMIRParserFromFile(StringRef Filename, - SMDiagnostic &Error, - LLVMContext &Context); +/// \param ProcessIRFunction - function to run on every IR function or stub +/// loaded from the MIR file. +std::unique_ptr<MIRParser> createMIRParserFromFile( + StringRef Filename, SMDiagnostic &Error, LLVMContext &Context, + std::function<void(Function &)> ProcessIRFunction = nullptr); /// This function is another interface to the MIR serialization format parser. /// @@ -73,7 +76,8 @@ std::unique_ptr<MIRParser> createMIRParserFromFile(StringRef Filename, /// \param Contents - The MemoryBuffer containing the machine level IR. /// \param Context - Context which will be used for the parsed LLVM IR module. std::unique_ptr<MIRParser> -createMIRParser(std::unique_ptr<MemoryBuffer> Contents, LLVMContext &Context); +createMIRParser(std::unique_ptr<MemoryBuffer> Contents, LLVMContext &Context, + std::function<void(Function &)> ProcessIRFunction = nullptr); } // end namespace llvm diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h index a438ecfcc25e..2a826d0b64c0 100644 --- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h +++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h @@ -38,6 +38,9 @@ public: static char ID; MachineBlockFrequencyInfo(); + explicit MachineBlockFrequencyInfo(MachineFunction &F, + MachineBranchProbabilityInfo &MBPI, + MachineLoopInfo &MLI); ~MachineBlockFrequencyInfo() override; void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h index 2b9b2030eb97..cde3bc08692b 100644 --- a/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h +++ b/llvm/include/llvm/CodeGen/MachineBranchProbabilityInfo.h @@ -35,10 +35,7 @@ class MachineBranchProbabilityInfo : public ImmutablePass { public: static char ID; - MachineBranchProbabilityInfo() : ImmutablePass(ID) { - PassRegistry &Registry = *PassRegistry::getPassRegistry(); - initializeMachineBranchProbabilityInfoPass(Registry); - } + MachineBranchProbabilityInfo(); void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); diff --git a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h index 503227222207..149fe043d1f5 100644 --- a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h +++ b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h @@ -38,6 +38,51 @@ enum class MachineCombinerPattern { MULSUBX_OP2, MULADDXI_OP1, MULSUBXI_OP1, + // NEON integers vectors + MULADDv8i8_OP1, + MULADDv8i8_OP2, + MULADDv16i8_OP1, + MULADDv16i8_OP2, + MULADDv4i16_OP1, + MULADDv4i16_OP2, + MULADDv8i16_OP1, + MULADDv8i16_OP2, + MULADDv2i32_OP1, + MULADDv2i32_OP2, + MULADDv4i32_OP1, + MULADDv4i32_OP2, + + MULSUBv8i8_OP1, + MULSUBv8i8_OP2, + MULSUBv16i8_OP1, + MULSUBv16i8_OP2, + MULSUBv4i16_OP1, + MULSUBv4i16_OP2, + MULSUBv8i16_OP1, + MULSUBv8i16_OP2, + MULSUBv2i32_OP1, + MULSUBv2i32_OP2, + MULSUBv4i32_OP1, + MULSUBv4i32_OP2, + + MULADDv4i16_indexed_OP1, + MULADDv4i16_indexed_OP2, + MULADDv8i16_indexed_OP1, + MULADDv8i16_indexed_OP2, + MULADDv2i32_indexed_OP1, + MULADDv2i32_indexed_OP2, + MULADDv4i32_indexed_OP1, + MULADDv4i32_indexed_OP2, + + MULSUBv4i16_indexed_OP1, + MULSUBv4i16_indexed_OP2, + MULSUBv8i16_indexed_OP1, + MULSUBv8i16_indexed_OP2, + MULSUBv2i32_indexed_OP1, + MULSUBv2i32_indexed_OP2, + MULSUBv4i32_indexed_OP1, + MULSUBv4i32_indexed_OP2, + // Floating Point FMULADDH_OP1, FMULADDH_OP2, diff --git a/llvm/include/llvm/CodeGen/MachineDominators.h b/llvm/include/llvm/CodeGen/MachineDominators.h index e4d7a02f8c48..9d31232c9b95 100644 --- a/llvm/include/llvm/CodeGen/MachineDominators.h +++ b/llvm/include/llvm/CodeGen/MachineDominators.h @@ -81,6 +81,9 @@ public: static char ID; // Pass ID, replacement for typeid MachineDominatorTree(); + explicit MachineDominatorTree(MachineFunction &MF) : MachineFunctionPass(ID) { + calculate(MF); + } DomTreeT &getBase() { if (!DT) DT.reset(new DomTreeT()); @@ -111,6 +114,8 @@ public: bool runOnMachineFunction(MachineFunction &F) override; + void calculate(MachineFunction &F); + bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const { applySplitCriticalEdges(); diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h index 01fc50d14a7f..05b34d92651c 100644 --- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -553,7 +553,7 @@ public: void setStackSize(uint64_t Size) { StackSize = Size; } /// Estimate and return the size of the stack frame. - unsigned estimateStackSize(const MachineFunction &MF) const; + uint64_t estimateStackSize(const MachineFunction &MF) const; /// Return the correction for frame offsets. int getOffsetAdjustment() const { return OffsetAdjustment; } diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index 3a3176e51c51..7f4a3a8c2f97 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -20,6 +20,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/FloatingPointMode.h" #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" @@ -36,7 +37,6 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Recycler.h" -#include "llvm/Target/TargetMachine.h" #include <cassert> #include <cstdint> #include <memory> @@ -304,6 +304,10 @@ class MachineFunction { /// by debug and exception handling consumers. std::vector<MCCFIInstruction> FrameInstructions; + /// List of basic blocks immediately following calls to _setjmp. Used to + /// construct a table of valid longjmp targets for Windows Control Flow Guard. + std::vector<MCSymbol *> LongjmpTargets; + /// \name Exception Handling /// \{ @@ -322,10 +326,6 @@ class MachineFunction { /// CodeView label annotations. std::vector<std::pair<MCSymbol *, MDNode *>> CodeViewAnnotations; - /// CodeView heapallocsites. - std::vector<std::tuple<MCSymbol *, MCSymbol *, const DIType *>> - CodeViewHeapAllocSites; - bool CallsEHReturn = false; bool CallsUnwindInit = false; bool HasEHScopes = false; @@ -403,14 +403,7 @@ private: /// A helper function that returns call site info for a give call /// instruction if debug entry value support is enabled. - CallSiteInfoMap::iterator getCallSiteInfo(const MachineInstr *MI) { - assert(MI->isCall() && - "Call site info refers only to call instructions!"); - - if (!Target.Options.EnableDebugEntryValues) - return CallSitesInfo.end(); - return CallSitesInfo.find(MI); - } + CallSiteInfoMap::iterator getCallSiteInfo(const MachineInstr *MI); // Callbacks for insertion and removal. void handleInsertion(MachineInstr &MI); @@ -560,6 +553,9 @@ public: } void setHasWinCFI(bool v) { HasWinCFI = v; } + /// True if this function needs frame moves for debug or exceptions. + bool needsFrameMoves() const; + /// Get the function properties const MachineFunctionProperties &getProperties() const { return Properties; } MachineFunctionProperties &getProperties() { return Properties; } @@ -579,6 +575,10 @@ public: return const_cast<MachineFunction*>(this)->getInfo<Ty>(); } + /// Returns the denormal handling type for the default rounding mode of the + /// function. + DenormalMode getDenormalMode(const fltSemantics &FPType) const; + /// getBlockNumbered - MachineBasicBlocks are automatically numbered when they /// are inserted into the machine function. The block number for a machine /// basic block can be found by using the MBB::getNumber method, this method @@ -796,14 +796,15 @@ public: /// Allocate and initialize a register mask with @p NumRegister bits. uint32_t *allocateRegMask(); + ArrayRef<int> allocateShuffleMask(ArrayRef<int> Mask); + /// Allocate and construct an extra info structure for a `MachineInstr`. /// /// This is allocated on the function's allocator and so lives the life of /// the function. - MachineInstr::ExtraInfo * - createMIExtraInfo(ArrayRef<MachineMemOperand *> MMOs, - MCSymbol *PreInstrSymbol = nullptr, - MCSymbol *PostInstrSymbol = nullptr); + MachineInstr::ExtraInfo *createMIExtraInfo( + ArrayRef<MachineMemOperand *> MMOs, MCSymbol *PreInstrSymbol = nullptr, + MCSymbol *PostInstrSymbol = nullptr, MDNode *HeapAllocMarker = nullptr); /// Allocate a string and populate it with the given external symbol name. const char *createExternalSymbolName(StringRef Name); @@ -830,6 +831,17 @@ public: LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst); + /// Returns a reference to a list of symbols immediately following calls to + /// _setjmp in the function. Used to construct the longjmp target table used + /// by Windows Control Flow Guard. + const std::vector<MCSymbol *> &getLongjmpTargets() const { + return LongjmpTargets; + } + + /// Add the specified symbol to the list of valid longjmp targets for Windows + /// Control Flow Guard. + void addLongjmpTarget(MCSymbol *Target) { LongjmpTargets.push_back(Target); } + /// \name Exception Handling /// \{ @@ -947,14 +959,6 @@ public: return CodeViewAnnotations; } - /// Record heapallocsites - void addCodeViewHeapAllocSite(MachineInstr *I, const MDNode *MD); - - ArrayRef<std::tuple<MCSymbol *, MCSymbol *, const DIType *>> - getCodeViewHeapAllocSites() const { - return CodeViewHeapAllocSites; - } - /// Return a reference to the C++ typeinfo for the current function. const std::vector<const GlobalValue *> &getTypeInfos() const { return TypeInfos; diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index c94ad292ec96..6d4ab3b2a2a5 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -104,8 +104,8 @@ public: // no signed wrap. IsExact = 1 << 13, // Instruction supports division is // known to be exact. - FPExcept = 1 << 14, // Instruction may raise floating-point - // exceptions. + NoFPExcept = 1 << 14, // Instruction does not raise + // floatint-point exceptions. }; private: @@ -136,19 +136,23 @@ private: /// This has to be defined eagerly due to the implementation constraints of /// `PointerSumType` where it is used. class ExtraInfo final - : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *> { + : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *> { public: static ExtraInfo *create(BumpPtrAllocator &Allocator, ArrayRef<MachineMemOperand *> MMOs, MCSymbol *PreInstrSymbol = nullptr, - MCSymbol *PostInstrSymbol = nullptr) { + MCSymbol *PostInstrSymbol = nullptr, + MDNode *HeapAllocMarker = nullptr) { bool HasPreInstrSymbol = PreInstrSymbol != nullptr; bool HasPostInstrSymbol = PostInstrSymbol != nullptr; + bool HasHeapAllocMarker = HeapAllocMarker != nullptr; auto *Result = new (Allocator.Allocate( - totalSizeToAlloc<MachineMemOperand *, MCSymbol *>( - MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol), + totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *>( + MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, + HasHeapAllocMarker), alignof(ExtraInfo))) - ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol); + ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, + HasHeapAllocMarker); // Copy the actual data into the trailing objects. std::copy(MMOs.begin(), MMOs.end(), @@ -159,6 +163,8 @@ private: if (HasPostInstrSymbol) Result->getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] = PostInstrSymbol; + if (HasHeapAllocMarker) + Result->getTrailingObjects<MDNode *>()[0] = HeapAllocMarker; return Result; } @@ -177,6 +183,10 @@ private: : nullptr; } + MDNode *getHeapAllocMarker() const { + return HasHeapAllocMarker ? getTrailingObjects<MDNode *>()[0] : nullptr; + } + private: friend TrailingObjects; @@ -188,6 +198,7 @@ private: const int NumMMOs; const bool HasPreInstrSymbol; const bool HasPostInstrSymbol; + const bool HasHeapAllocMarker; // Implement the `TrailingObjects` internal API. size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const { @@ -196,12 +207,17 @@ private: size_t numTrailingObjects(OverloadToken<MCSymbol *>) const { return HasPreInstrSymbol + HasPostInstrSymbol; } + size_t numTrailingObjects(OverloadToken<MDNode *>) const { + return HasHeapAllocMarker; + } // Just a boring constructor to allow us to initialize the sizes. Always use // the `create` routine above. - ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol) + ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, + bool HasHeapAllocMarker) : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), - HasPostInstrSymbol(HasPostInstrSymbol) {} + HasPostInstrSymbol(HasPostInstrSymbol), + HasHeapAllocMarker(HasHeapAllocMarker) {} }; /// Enumeration of the kinds of inline extra info available. It is important @@ -592,6 +608,16 @@ public: return nullptr; } + /// Helper to extract a heap alloc marker if one has been added. + MDNode *getHeapAllocMarker() const { + if (!Info) + return nullptr; + if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) + return EI->getHeapAllocMarker(); + + return nullptr; + } + /// API for querying MachineInstr properties. They are the same as MCInstrDesc /// queries but they are bundle aware. @@ -692,7 +718,7 @@ public: /// block. The TargetInstrInfo::AnalyzeBranch method can be used to get more /// information about this branch. bool isConditionalBranch(QueryType Type = AnyInBundle) const { - return isBranch(Type) & !isBarrier(Type) & !isIndirectBranch(Type); + return isBranch(Type) && !isBarrier(Type) && !isIndirectBranch(Type); } /// Return true if this is a branch which always @@ -700,7 +726,7 @@ public: /// TargetInstrInfo::AnalyzeBranch method can be used to get more information /// about this branch. bool isUnconditionalBranch(QueryType Type = AnyInBundle) const { - return isBranch(Type) & isBarrier(Type) & !isIndirectBranch(Type); + return isBranch(Type) && isBarrier(Type) && !isIndirectBranch(Type); } /// Return true if this instruction has a predicate operand that @@ -859,10 +885,10 @@ public: /// instruction that can in principle raise an exception, as indicated /// by the MCID::MayRaiseFPException property, *and* at the same time, /// the instruction is used in a context where we expect floating-point - /// exceptions might be enabled, as indicated by the FPExcept MI flag. + /// exceptions are not disabled, as indicated by the NoFPExcept MI flag. bool mayRaiseFPException() const { return hasProperty(MCID::MayRaiseFPException) && - getFlag(MachineInstr::MIFlag::FPExcept); + !getFlag(MachineInstr::MIFlag::NoFPExcept); } //===--------------------------------------------------------------------===// @@ -1597,6 +1623,12 @@ public: /// replace ours with it. void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI); + /// Set a marker on instructions that denotes where we should create and emit + /// heap alloc site labels. This waits until after instruction selection and + /// optimizations to create the label, so it should still work if the + /// instruction is removed or duplicated. + void setHeapAllocMarker(MachineFunction &MF, MDNode *MD); + /// Return the MIFlags which represent both MachineInstrs. This /// should be used when merging two MachineInstrs into one. This routine does /// not modify the MIFlags of this MachineInstr. @@ -1619,7 +1651,8 @@ public: /// Add all implicit def and use operands to this instruction. void addImplicitDefUseOperands(MachineFunction &MF); - /// Scan instructions following MI and collect any matching DBG_VALUEs. + /// Scan instructions immediately following MI and collect any matching + /// DBG_VALUEs. void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues); /// Find all DBG_VALUEs that point to the register def in this instruction @@ -1657,6 +1690,12 @@ private: const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; + + /// Stores extra instruction information inline or allocates as ExtraInfo + /// based on the number of pointers. + void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs, + MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, + MDNode *HeapAllocMarker); }; /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index 880d4829ac7e..cabb9f1c97c9 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -250,7 +250,7 @@ public: return *this; } - const MachineInstrBuilder &addShuffleMask(const Constant *Val) const { + const MachineInstrBuilder &addShuffleMask(ArrayRef<int> Val) const { MI->addOperand(*MF, MachineOperand::CreateShuffleMask(Val)); return *this; } diff --git a/llvm/include/llvm/CodeGen/MachineInstrBundle.h b/llvm/include/llvm/CodeGen/MachineInstrBundle.h index 1810d23072d0..517f03e60933 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBundle.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBundle.h @@ -75,12 +75,12 @@ inline MachineBasicBlock::const_instr_iterator getBundleEnd( } //===----------------------------------------------------------------------===// -// MachineOperand iterator +// MachineBundleOperand iterator // -/// MachineOperandIteratorBase - Iterator that can visit all operands on a -/// MachineInstr, or all operands on a bundle of MachineInstrs. This class is -/// not intended to be used directly, use one of the sub-classes instead. +/// MIBundleOperandIteratorBase - Iterator that visits all operands in a bundle +/// of MachineInstrs. This class is not intended to be used directly, use one +/// of the sub-classes instead. /// /// Intended use: /// @@ -90,7 +90,10 @@ inline MachineBasicBlock::const_instr_iterator getBundleEnd( /// ... /// } /// -class MachineOperandIteratorBase { +template <typename ValueT> +class MIBundleOperandIteratorBase + : public iterator_facade_base<MIBundleOperandIteratorBase<ValueT>, + std::forward_iterator_tag, ValueT> { MachineBasicBlock::instr_iterator InstrI, InstrE; MachineInstr::mop_iterator OpI, OpE; @@ -99,35 +102,34 @@ class MachineOperandIteratorBase { void advance() { while (OpI == OpE) { // Don't advance off the basic block, or into a new bundle. - if (++InstrI == InstrE || !InstrI->isInsideBundle()) + if (++InstrI == InstrE || !InstrI->isInsideBundle()) { + InstrI = InstrE; break; + } OpI = InstrI->operands_begin(); OpE = InstrI->operands_end(); } } protected: - /// MachineOperandIteratorBase - Create an iterator that visits all operands + /// MIBundleOperandIteratorBase - Create an iterator that visits all operands /// on MI, or all operands on every instruction in the bundle containing MI. /// /// @param MI The instruction to examine. - /// @param WholeBundle When true, visit all operands on the entire bundle. /// - explicit MachineOperandIteratorBase(MachineInstr &MI, bool WholeBundle) { - if (WholeBundle) { - InstrI = getBundleStart(MI.getIterator()); - InstrE = MI.getParent()->instr_end(); - } else { - InstrI = InstrE = MI.getIterator(); - ++InstrE; - } + explicit MIBundleOperandIteratorBase(MachineInstr &MI) { + InstrI = getBundleStart(MI.getIterator()); + InstrE = MI.getParent()->instr_end(); OpI = InstrI->operands_begin(); OpE = InstrI->operands_end(); - if (WholeBundle) - advance(); + advance(); } - MachineOperand &deref() const { return *OpI; } + /// Constructor for an iterator past the last iteration: both instruction + /// iterators point to the end of the BB and OpI == OpE. + explicit MIBundleOperandIteratorBase(MachineBasicBlock::instr_iterator InstrE, + MachineInstr::mop_iterator OpE) + : InstrI(InstrE), InstrE(InstrE), OpI(OpE), OpE(OpE) {} public: /// isValid - Returns true until all the operands have been visited. @@ -140,123 +142,148 @@ public: advance(); } + ValueT &operator*() const { return *OpI; } + ValueT *operator->() const { return &*OpI; } + + bool operator==(const MIBundleOperandIteratorBase &Arg) const { + // Iterators are equal, if InstrI matches and either OpIs match or OpI == + // OpE match for both. The second condition allows us to construct an 'end' + // iterator, without finding the last instruction in a bundle up-front. + return InstrI == Arg.InstrI && + (OpI == Arg.OpI || (OpI == OpE && Arg.OpI == Arg.OpE)); + } /// getOperandNo - Returns the number of the current operand relative to its /// instruction. /// unsigned getOperandNo() const { return OpI - InstrI->operands_begin(); } - - /// VirtRegInfo - Information about a virtual register used by a set of operands. - /// - struct VirtRegInfo { - /// Reads - One of the operands read the virtual register. This does not - /// include undef or internal use operands, see MO::readsReg(). - bool Reads; - - /// Writes - One of the operands writes the virtual register. - bool Writes; - - /// Tied - Uses and defs must use the same register. This can be because of - /// a two-address constraint, or there may be a partial redefinition of a - /// sub-register. - bool Tied; - }; - - /// Information about how a physical register Reg is used by a set of - /// operands. - struct PhysRegInfo { - /// There is a regmask operand indicating Reg is clobbered. - /// \see MachineOperand::CreateRegMask(). - bool Clobbered; - - /// Reg or one of its aliases is defined. The definition may only cover - /// parts of the register. - bool Defined; - /// Reg or a super-register is defined. The definition covers the full - /// register. - bool FullyDefined; - - /// Reg or one of its aliases is read. The register may only be read - /// partially. - bool Read; - /// Reg or a super-register is read. The full register is read. - bool FullyRead; - - /// Either: - /// - Reg is FullyDefined and all defs of reg or an overlapping - /// register are dead, or - /// - Reg is completely dead because "defined" by a clobber. - bool DeadDef; - - /// Reg is Defined and all defs of reg or an overlapping register are - /// dead. - bool PartialDeadDef; - - /// There is a use operand of reg or a super-register with kill flag set. - bool Killed; - }; - - /// analyzeVirtReg - Analyze how the current instruction or bundle uses a - /// virtual register. This function should not be called after operator++(), - /// it expects a fresh iterator. - /// - /// @param Reg The virtual register to analyze. - /// @param Ops When set, this vector will receive an (MI, OpNum) entry for - /// each operand referring to Reg. - /// @returns A filled-in RegInfo struct. - VirtRegInfo analyzeVirtReg(unsigned Reg, - SmallVectorImpl<std::pair<MachineInstr*, unsigned> > *Ops = nullptr); - - /// analyzePhysReg - Analyze how the current instruction or bundle uses a - /// physical register. This function should not be called after operator++(), - /// it expects a fresh iterator. - /// - /// @param Reg The physical register to analyze. - /// @returns A filled-in PhysRegInfo struct. - PhysRegInfo analyzePhysReg(unsigned Reg, const TargetRegisterInfo *TRI); }; -/// MIOperands - Iterate over operands of a single instruction. +/// MIBundleOperands - Iterate over all operands in a bundle of machine +/// instructions. /// -class MIOperands : public MachineOperandIteratorBase { +class MIBundleOperands : public MIBundleOperandIteratorBase<MachineOperand> { + /// Constructor for an iterator past the last iteration. + MIBundleOperands(MachineBasicBlock::instr_iterator InstrE, + MachineInstr::mop_iterator OpE) + : MIBundleOperandIteratorBase(InstrE, OpE) {} + public: - MIOperands(MachineInstr &MI) : MachineOperandIteratorBase(MI, false) {} - MachineOperand &operator* () const { return deref(); } - MachineOperand *operator->() const { return &deref(); } + MIBundleOperands(MachineInstr &MI) : MIBundleOperandIteratorBase(MI) {} + + /// Returns an iterator past the last iteration. + static MIBundleOperands end(const MachineBasicBlock &MBB) { + return {const_cast<MachineBasicBlock &>(MBB).instr_end(), + const_cast<MachineBasicBlock &>(MBB).instr_begin()->operands_end()}; + } }; -/// ConstMIOperands - Iterate over operands of a single const instruction. +/// ConstMIBundleOperands - Iterate over all operands in a const bundle of +/// machine instructions. /// -class ConstMIOperands : public MachineOperandIteratorBase { +class ConstMIBundleOperands + : public MIBundleOperandIteratorBase<const MachineOperand> { + + /// Constructor for an iterator past the last iteration. + ConstMIBundleOperands(MachineBasicBlock::instr_iterator InstrE, + MachineInstr::mop_iterator OpE) + : MIBundleOperandIteratorBase(InstrE, OpE) {} + public: - ConstMIOperands(const MachineInstr &MI) - : MachineOperandIteratorBase(const_cast<MachineInstr &>(MI), false) {} - const MachineOperand &operator* () const { return deref(); } - const MachineOperand *operator->() const { return &deref(); } + ConstMIBundleOperands(const MachineInstr &MI) + : MIBundleOperandIteratorBase(const_cast<MachineInstr &>(MI)) {} + + /// Returns an iterator past the last iteration. + static ConstMIBundleOperands end(const MachineBasicBlock &MBB) { + return {const_cast<MachineBasicBlock &>(MBB).instr_end(), + const_cast<MachineBasicBlock &>(MBB).instr_begin()->operands_end()}; + } }; -/// MIBundleOperands - Iterate over all operands in a bundle of machine -/// instructions. +inline iterator_range<ConstMIBundleOperands> +const_mi_bundle_ops(const MachineInstr &MI) { + return make_range(ConstMIBundleOperands(MI), + ConstMIBundleOperands::end(*MI.getParent())); +} + +inline iterator_range<MIBundleOperands> mi_bundle_ops(MachineInstr &MI) { + return make_range(MIBundleOperands(MI), + MIBundleOperands::end(*MI.getParent())); +} + +/// VirtRegInfo - Information about a virtual register used by a set of +/// operands. /// -class MIBundleOperands : public MachineOperandIteratorBase { -public: - MIBundleOperands(MachineInstr &MI) : MachineOperandIteratorBase(MI, true) {} - MachineOperand &operator* () const { return deref(); } - MachineOperand *operator->() const { return &deref(); } +struct VirtRegInfo { + /// Reads - One of the operands read the virtual register. This does not + /// include undef or internal use operands, see MO::readsReg(). + bool Reads; + + /// Writes - One of the operands writes the virtual register. + bool Writes; + + /// Tied - Uses and defs must use the same register. This can be because of + /// a two-address constraint, or there may be a partial redefinition of a + /// sub-register. + bool Tied; }; -/// ConstMIBundleOperands - Iterate over all operands in a const bundle of -/// machine instructions. +/// AnalyzeVirtRegInBundle - Analyze how the current instruction or bundle uses +/// a virtual register. This function should not be called after operator++(), +/// it expects a fresh iterator. /// -class ConstMIBundleOperands : public MachineOperandIteratorBase { -public: - ConstMIBundleOperands(const MachineInstr &MI) - : MachineOperandIteratorBase(const_cast<MachineInstr &>(MI), true) {} - const MachineOperand &operator* () const { return deref(); } - const MachineOperand *operator->() const { return &deref(); } +/// @param Reg The virtual register to analyze. +/// @param Ops When set, this vector will receive an (MI, OpNum) entry for +/// each operand referring to Reg. +/// @returns A filled-in RegInfo struct. +VirtRegInfo AnalyzeVirtRegInBundle( + MachineInstr &MI, unsigned Reg, + SmallVectorImpl<std::pair<MachineInstr *, unsigned>> *Ops = nullptr); + +/// Information about how a physical register Reg is used by a set of +/// operands. +struct PhysRegInfo { + /// There is a regmask operand indicating Reg is clobbered. + /// \see MachineOperand::CreateRegMask(). + bool Clobbered; + + /// Reg or one of its aliases is defined. The definition may only cover + /// parts of the register. + bool Defined; + /// Reg or a super-register is defined. The definition covers the full + /// register. + bool FullyDefined; + + /// Reg or one of its aliases is read. The register may only be read + /// partially. + bool Read; + /// Reg or a super-register is read. The full register is read. + bool FullyRead; + + /// Either: + /// - Reg is FullyDefined and all defs of reg or an overlapping + /// register are dead, or + /// - Reg is completely dead because "defined" by a clobber. + bool DeadDef; + + /// Reg is Defined and all defs of reg or an overlapping register are + /// dead. + bool PartialDeadDef; + + /// There is a use operand of reg or a super-register with kill flag set. + bool Killed; }; +/// AnalyzePhysRegInBundle - Analyze how the current instruction or bundle uses +/// a physical register. This function should not be called after operator++(), +/// it expects a fresh iterator. +/// +/// @param Reg The physical register to analyze. +/// @returns A filled-in PhysRegInfo struct. +PhysRegInfo AnalyzePhysRegInBundle(const MachineInstr &MI, unsigned Reg, + const TargetRegisterInfo *TRI); + } // End llvm namespace #endif diff --git a/llvm/include/llvm/CodeGen/MachineLoopInfo.h b/llvm/include/llvm/CodeGen/MachineLoopInfo.h index da6df59c739c..8a93f91ae54d 100644 --- a/llvm/include/llvm/CodeGen/MachineLoopInfo.h +++ b/llvm/include/llvm/CodeGen/MachineLoopInfo.h @@ -37,6 +37,7 @@ namespace llvm { +class MachineDominatorTree; // Implementation in LoopInfoImpl.h class MachineLoop; extern template class LoopBase<MachineBasicBlock, MachineLoop>; @@ -88,8 +89,10 @@ class MachineLoopInfo : public MachineFunctionPass { public: static char ID; // Pass identification, replacement for typeid - MachineLoopInfo() : MachineFunctionPass(ID) { - initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry()); + MachineLoopInfo(); + explicit MachineLoopInfo(MachineDominatorTree &MDT) + : MachineFunctionPass(ID) { + calculate(MDT); } MachineLoopInfo(const MachineLoopInfo &) = delete; MachineLoopInfo &operator=(const MachineLoopInfo &) = delete; @@ -133,6 +136,7 @@ public: /// Calculate the natural loop information. bool runOnMachineFunction(MachineFunction &F) override; + void calculate(MachineDominatorTree &MDT); void releaseMemory() override { LI.releaseMemory(); } diff --git a/llvm/include/llvm/CodeGen/MachineLoopUtils.h b/llvm/include/llvm/CodeGen/MachineLoopUtils.h index 41379b75d00a..2cb0134ca848 100644 --- a/llvm/include/llvm/CodeGen/MachineLoopUtils.h +++ b/llvm/include/llvm/CodeGen/MachineLoopUtils.h @@ -10,6 +10,7 @@ #define LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H namespace llvm { +class MachineLoop; class MachineBasicBlock; class MachineRegisterInfo; class TargetInstrInfo; @@ -36,6 +37,10 @@ MachineBasicBlock *PeelSingleBlockLoop(LoopPeelDirection Direction, MachineRegisterInfo &MRI, const TargetInstrInfo *TII); +/// Return true if PhysReg is live outside the loop, i.e. determine if it +/// is live in the loop exit blocks, and false otherwise. +bool isRegLiveInExitBlocks(MachineLoop *Loop, int PhysReg); + } // namespace llvm #endif // LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h index 33a48a235e18..7ee700c62a25 100644 --- a/llvm/include/llvm/CodeGen/MachineMemOperand.h +++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h @@ -229,7 +229,7 @@ public: /// Return the minimum known alignment in bytes of the base address, without /// the offset. - uint64_t getBaseAlignment() const { return (1u << BaseAlignLog2) >> 1; } + uint64_t getBaseAlignment() const { return (1ull << BaseAlignLog2) >> 1; } /// Return the AA tags for the memory reference. AAMDNodes getAAInfo() const { return AAInfo; } diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h index df914dc2d85e..9ba2b01cb4bd 100644 --- a/llvm/include/llvm/CodeGen/MachineOperand.h +++ b/llvm/include/llvm/CodeGen/MachineOperand.h @@ -163,7 +163,8 @@ private: MachineInstr *ParentMI; /// Contents union - This contains the payload for the various operand types. - union { + union ContentsUnion { + ContentsUnion() {} MachineBasicBlock *MBB; // For MO_MachineBasicBlock. const ConstantFP *CFP; // For MO_FPImmediate. const ConstantInt *CI; // For MO_CImmediate. Integers > 64bit. @@ -174,7 +175,7 @@ private: unsigned CFIIndex; // For MO_CFI. Intrinsic::ID IntrinsicID; // For MO_IntrinsicID. unsigned Pred; // For MO_Predicate - const Constant *ShuffleMask; // For MO_ShuffleMask + ArrayRef<int> ShuffleMask; // For MO_ShuffleMask struct { // For MO_Register. // Register number is in SmallContents.RegNo. @@ -278,6 +279,9 @@ public: /// More complex way of printing a MachineOperand. /// \param TypeToPrint specifies the generic type to be printed on uses and /// defs. It can be determined using MachineInstr::getTypeToPrint. + /// \param OpIdx - specifies the index of the operand in machine instruction. + /// This will be used by target dependent MIR formatter. Could be None if the + /// index is unknown, e.g. called by dump(). /// \param PrintDef - whether we want to print `def` on an operand which /// isDef. Sometimes, if the operand is printed before '=', we don't print /// `def`. @@ -294,8 +298,9 @@ public: /// information from it's parent. /// \param IntrinsicInfo - same as \p TRI. void print(raw_ostream &os, ModuleSlotTracker &MST, LLT TypeToPrint, - bool PrintDef, bool IsStandalone, bool ShouldPrintRegisterTies, - unsigned TiedOperandIdx, const TargetRegisterInfo *TRI, + Optional<unsigned> OpIdx, bool PrintDef, bool IsStandalone, + bool ShouldPrintRegisterTies, unsigned TiedOperandIdx, + const TargetRegisterInfo *TRI, const TargetIntrinsicInfo *IntrinsicInfo) const; /// Same as print(os, TRI, IntrinsicInfo), but allows to specify the low-level @@ -583,7 +588,7 @@ public: return Contents.Pred; } - const Constant *getShuffleMask() const { + ArrayRef<int> getShuffleMask() const { assert(isShuffleMask() && "Wrong MachineOperand accessor"); return Contents.ShuffleMask; } @@ -911,9 +916,9 @@ public: return Op; } - static MachineOperand CreateShuffleMask(const Constant *C) { + static MachineOperand CreateShuffleMask(ArrayRef<int> Mask) { MachineOperand Op(MachineOperand::MO_ShuffleMask); - Op.Contents.ShuffleMask = C; + Op.Contents.ShuffleMask = Mask; return Op; } diff --git a/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h b/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h index a461a299917c..b2f8ad55fbd8 100644 --- a/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h +++ b/llvm/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h @@ -182,6 +182,10 @@ public: } } + MachineBlockFrequencyInfo *getBFI() { + return MBFI; + } + private: MachineFunction &MF; diff --git a/llvm/include/llvm/CodeGen/MachineOutliner.h b/llvm/include/llvm/CodeGen/MachineOutliner.h index 3868fa415579..4a1b04ab3e88 100644 --- a/llvm/include/llvm/CodeGen/MachineOutliner.h +++ b/llvm/include/llvm/CodeGen/MachineOutliner.h @@ -37,10 +37,10 @@ enum InstrType { Legal, LegalTerminator, Illegal, Invisible }; struct Candidate { private: /// The start index of this \p Candidate in the instruction list. - unsigned StartIdx; + unsigned StartIdx = 0; /// The number of instructions in this \p Candidate. - unsigned Len; + unsigned Len = 0; // The first instruction in this \p Candidate. MachineBasicBlock::iterator FirstInst; @@ -49,20 +49,20 @@ private: MachineBasicBlock::iterator LastInst; // The basic block that contains this Candidate. - MachineBasicBlock *MBB; + MachineBasicBlock *MBB = nullptr; /// Cost of calling an outlined function from this point as defined by the /// target. - unsigned CallOverhead; + unsigned CallOverhead = 0; public: /// The index of this \p Candidate's \p OutlinedFunction in the list of /// \p OutlinedFunctions. - unsigned FunctionIdx; + unsigned FunctionIdx = 0; /// Identifier denoting the instructions to emit to call an outlined function /// from this point. Defined by the target. - unsigned CallConstructionID; + unsigned CallConstructionID = 0; /// Contains physical register liveness information for the MBB containing /// this \p Candidate. diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h index e9cf7e115bff..24e85a953d47 100644 --- a/llvm/include/llvm/CodeGen/MachinePipeliner.h +++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h @@ -46,6 +46,7 @@ #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/ScheduleDAGInstrs.h" #include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/InitializePasses.h" namespace llvm { diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h index 333367943ac0..6cebaa47fe6a 100644 --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -757,7 +757,16 @@ public: unsigned getOtherResourceCount(unsigned &OtherCritIdx); - void releaseNode(SUnit *SU, unsigned ReadyCycle); + /// Release SU to make it ready. If it's not in hazard, remove it from + /// pending queue (if already in) and push into available queue. + /// Otherwise, push the SU into pending queue. + /// + /// @param SU The unit to be released. + /// @param ReadyCycle Until which cycle the unit is ready. + /// @param InPQueue Whether SU is already in pending queue. + /// @param Idx Position offset in pending queue (if in it). + void releaseNode(SUnit *SU, unsigned ReadyCycle, bool InPQueue, + unsigned Idx = 0); void bumpCycle(unsigned NextCycle); @@ -955,7 +964,7 @@ public: if (SU->isScheduled) return; - Top.releaseNode(SU, SU->TopReadyCycle); + Top.releaseNode(SU, SU->TopReadyCycle, false); TopCand.SU = nullptr; } @@ -963,7 +972,7 @@ public: if (SU->isScheduled) return; - Bot.releaseNode(SU, SU->BotReadyCycle); + Bot.releaseNode(SU, SU->BotReadyCycle, false); BotCand.SU = nullptr; } @@ -1009,7 +1018,7 @@ protected: /// initPolicy -> initialize(DAG) -> registerRoots -> pickNode ... class PostGenericScheduler : public GenericSchedulerBase { protected: - ScheduleDAGMI *DAG; + ScheduleDAGMI *DAG = nullptr; SchedBoundary Top; SmallVector<SUnit*, 8> BotRoots; @@ -1043,7 +1052,7 @@ public: void releaseTopNode(SUnit *SU) override { if (SU->isScheduled) return; - Top.releaseNode(SU, SU->TopReadyCycle); + Top.releaseNode(SU, SU->TopReadyCycle, false); } // Only called for roots. diff --git a/llvm/include/llvm/CodeGen/MachineSizeOpts.h b/llvm/include/llvm/CodeGen/MachineSizeOpts.h new file mode 100644 index 000000000000..3b02d0860ea1 --- /dev/null +++ b/llvm/include/llvm/CodeGen/MachineSizeOpts.h @@ -0,0 +1,39 @@ +//===- MachineSizeOpts.h - machine size optimization ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains some shared machine IR code size optimization related +// code. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_MACHINE_SIZEOPTS_H +#define LLVM_CODEGEN_MACHINE_SIZEOPTS_H + +#include "llvm/Transforms/Utils/SizeOpts.h" + +namespace llvm { + +class ProfileSummaryInfo; +class MachineBasicBlock; +class MachineBlockFrequencyInfo; +class MachineFunction; + +/// Returns true if machine function \p MF is suggested to be size-optimized +/// based on the profile. +bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, + const MachineBlockFrequencyInfo *BFI, + PGSOQueryType QueryType = PGSOQueryType::Other); +/// Returns true if machine basic block \p MBB is suggested to be size-optimized +/// based on the profile. +bool shouldOptimizeForSize(const MachineBasicBlock *MBB, + ProfileSummaryInfo *PSI, + const MachineBlockFrequencyInfo *MBFI, + PGSOQueryType QueryType = PGSOQueryType::Other); + +} // end namespace llvm + +#endif // LLVM_CODEGEN_MACHINE_SIZEOPTS_H diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h index 81a9b63b64ca..55c52f3447b0 100644 --- a/llvm/include/llvm/CodeGen/ModuloSchedule.h +++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h @@ -290,6 +290,9 @@ class PeelingModuloScheduleExpander { /// but not produced (in the epilog) or produced but not available (in the /// prolog). DenseMap<MachineBasicBlock *, BitVector> AvailableStages; + /// When peeling the epilogue keep track of the distance between the phi + /// nodes and the kernel. + DenseMap<MachineInstr *, unsigned> PhiNodeLoopIteration; /// CanonicalMIs and BlockMIs form a bidirectional map between any of the /// loop kernel clones. @@ -299,6 +302,8 @@ class PeelingModuloScheduleExpander { /// State passed from peelKernel to peelPrologAndEpilogs(). std::deque<MachineBasicBlock *> PeeledFront, PeeledBack; + /// Illegal phis that need to be deleted once we re-link stages. + SmallVector<MachineInstr *, 4> IllegalPhisToDelete; public: PeelingModuloScheduleExpander(MachineFunction &MF, ModuloSchedule &S, @@ -321,6 +326,13 @@ private: /// Peels one iteration of the rewritten kernel (BB) in the specified /// direction. MachineBasicBlock *peelKernel(LoopPeelDirection LPD); + // Delete instructions whose stage is less than MinStage in the given basic + // block. + void filterInstructions(MachineBasicBlock *MB, int MinStage); + // Move instructions of the given stage from sourceBB to DestBB. Remap the phi + // instructions to keep a valid IR. + void moveStageBetweenBlocks(MachineBasicBlock *DestBB, + MachineBasicBlock *SourceBB, unsigned Stage); /// Peel the kernel forwards and backwards to produce prologs and epilogs, /// and stitch them together. void peelPrologAndEpilogs(); @@ -342,6 +354,11 @@ private: MI = CanonicalMIs[MI]; return Schedule.getStage(MI); } + /// Helper function to find the right canonical register for a phi instruction + /// coming from a peeled out prologue. + Register getPhiCanonicalReg(MachineInstr* CanonicalPhi, MachineInstr* Phi); + /// Target loop info before kernel peeling. + std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> Info; }; /// Expander that simply annotates each scheduled instruction with a post-instr diff --git a/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h b/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h new file mode 100644 index 000000000000..56db30ff7d6d --- /dev/null +++ b/llvm/include/llvm/CodeGen/NonRelocatableStringpool.h @@ -0,0 +1,83 @@ +//===- NonRelocatableStringpool.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_NONRELOCATABLESTRINGPOOL_H +#define LLVM_CODEGEN_NONRELOCATABLESTRINGPOOL_H + +#include "llvm/CodeGen/DwarfStringPoolEntry.h" +#include "llvm/Support/Allocator.h" +#include <cstdint> +#include <vector> + +namespace llvm { + +/// A string table that doesn't need relocations. +/// +/// Use this class when a string table doesn't need relocations. +/// This class provides this ability by just associating offsets with strings. +class NonRelocatableStringpool { +public: + /// Entries are stored into the StringMap and simply linked together through + /// the second element of this pair in order to keep track of insertion + /// order. + using MapTy = StringMap<DwarfStringPoolEntry, BumpPtrAllocator>; + + NonRelocatableStringpool( + std::function<StringRef(StringRef Input)> Translator = nullptr, + bool PutEmptyString = false) + : Translator(Translator) { + if (PutEmptyString) + EmptyString = getEntry(""); + } + + DwarfStringPoolEntryRef getEntry(StringRef S); + + /// Get the offset of string \p S in the string table. This can insert a new + /// element or return the offset of a pre-existing one. + uint32_t getStringOffset(StringRef S) { return getEntry(S).getOffset(); } + + /// Get permanent storage for \p S (but do not necessarily emit \p S in the + /// output section). A latter call to getStringOffset() with the same string + /// will chain it though. + /// + /// \returns The StringRef that points to permanent storage to use + /// in place of \p S. + StringRef internString(StringRef S); + + uint64_t getSize() { return CurrentEndOffset; } + + /// Return the list of strings to be emitted. This does not contain the + /// strings which were added via internString only. + std::vector<DwarfStringPoolEntryRef> getEntriesForEmission() const; + +private: + MapTy Strings; + uint32_t CurrentEndOffset = 0; + unsigned NumEntries = 0; + DwarfStringPoolEntryRef EmptyString; + std::function<StringRef(StringRef Input)> Translator; +}; + +/// Helper for making strong types. +template <typename T, typename S> class StrongType : public T { +public: + template <typename... Args> + explicit StrongType(Args... A) : T(std::forward<Args>(A)...) {} +}; + +/// It's very easy to introduce bugs by passing the wrong string pool. +/// By using strong types the interface enforces that the right +/// kind of pool is used. +struct UniqueTag {}; +struct OffsetsTag {}; +using UniquingStringPool = StrongType<NonRelocatableStringpool, UniqueTag>; +using OffsetsStringPool = StrongType<NonRelocatableStringpool, OffsetsTag>; + +} // end namespace llvm + +#endif // LLVM_CODEGEN_NONRELOCATABLESTRINGPOOL_H diff --git a/llvm/include/llvm/CodeGen/ParallelCG.h b/llvm/include/llvm/CodeGen/ParallelCG.h index a44715d4fc4f..b4c761c2269e 100644 --- a/llvm/include/llvm/CodeGen/ParallelCG.h +++ b/llvm/include/llvm/CodeGen/ParallelCG.h @@ -39,7 +39,7 @@ std::unique_ptr<Module> splitCodeGen(std::unique_ptr<Module> M, ArrayRef<raw_pwrite_stream *> OSs, ArrayRef<llvm::raw_pwrite_stream *> BCOSs, const std::function<std::unique_ptr<TargetMachine>()> &TMFactory, - TargetMachine::CodeGenFileType FileType = TargetMachine::CGFT_ObjectFile, + CodeGenFileType FileType = CGFT_ObjectFile, bool PreserveLocals = false); } // namespace llvm diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index 1e765ce51e4a..4e3451d80572 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -275,6 +275,11 @@ namespace llvm { /// MachineCSE - This pass performs global CSE on machine instructions. extern char &MachineCSEID; + /// MIRCanonicalizer - This pass canonicalizes MIR by renaming vregs + /// according to the semantics of the instruction as well as hoists + /// code. + extern char &MIRCanonicalizerID; + /// ImplicitNullChecks - This pass folds null pointer checks into nearby /// memory operations. extern char &ImplicitNullChecksID; @@ -451,9 +456,16 @@ namespace llvm { /// Creates CFI Instruction Inserter pass. \see CFIInstrInserter.cpp FunctionPass *createCFIInstrInserter(); + /// Creates CFGuard longjmp target identification pass. + /// \see CFGuardLongjmp.cpp + FunctionPass *createCFGuardLongjmpPass(); + /// Create Hardware Loop pass. \see HardwareLoops.cpp FunctionPass *createHardwareLoopsPass(); + /// Create IR Type Promotion pass. \see TypePromotion.cpp + FunctionPass *createTypePromotionPass(); + } // End llvm namespace #endif diff --git a/llvm/include/llvm/CodeGen/PseudoSourceValue.h b/llvm/include/llvm/CodeGen/PseudoSourceValue.h index 4b3cc9145a13..593a865ea545 100644 --- a/llvm/include/llvm/CodeGen/PseudoSourceValue.h +++ b/llvm/include/llvm/CodeGen/PseudoSourceValue.h @@ -22,6 +22,7 @@ namespace llvm { class MachineFrameInfo; class MachineMemOperand; +class MIRFormatter; class raw_ostream; class TargetInstrInfo; @@ -52,6 +53,7 @@ private: const PseudoSourceValue* PSV); friend class MachineMemOperand; // For printCustom(). + friend class MIRFormatter; // For printCustom(). /// Implement printing for PseudoSourceValue. This is called from /// Value::print or Value's operator<<. diff --git a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h index a599fb62f5e2..5a747245a62e 100644 --- a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h +++ b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h @@ -25,6 +25,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/LoopTraversal.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/InitializePasses.h" namespace llvm { @@ -86,17 +87,58 @@ public: MachineFunctionProperties getRequiredProperties() const override { return MachineFunctionProperties().set( - MachineFunctionProperties::Property::NoVRegs); + MachineFunctionProperties::Property::NoVRegs).set( + MachineFunctionProperties::Property::TracksLiveness); } /// Provides the instruction id of the closest reaching def instruction of /// PhysReg that reaches MI, relative to the begining of MI's basic block. int getReachingDef(MachineInstr *MI, int PhysReg); + /// Provides the instruction of the closest reaching def instruction of + /// PhysReg that reaches MI, relative to the begining of MI's basic block. + MachineInstr *getReachingMIDef(MachineInstr *MI, int PhysReg); + + /// Provides the MI, from the given block, corresponding to the Id or a + /// nullptr if the id does not refer to the block. + MachineInstr *getInstFromId(MachineBasicBlock *MBB, int InstId); + + /// Return whether A and B use the same def of PhysReg. + bool hasSameReachingDef(MachineInstr *A, MachineInstr *B, int PhysReg); + + /// Return whether the reaching def for MI also is live out of its parent + /// block. + bool isReachingDefLiveOut(MachineInstr *MI, int PhysReg); + + /// Return the local MI that produces the live out value for PhysReg, or + /// nullptr for a non-live out or non-local def. + MachineInstr *getLocalLiveOutMIDef(MachineBasicBlock *MBB, + int PhysReg); + + /// Return whether the given register is used after MI, whether it's a local + /// use or a live out. + bool isRegUsedAfter(MachineInstr *MI, int PhysReg); + + /// Provides the first instruction before MI that uses PhysReg + MachineInstr *getInstWithUseBefore(MachineInstr *MI, int PhysReg); + + /// Provides all instructions before MI that uses PhysReg + void getAllInstWithUseBefore(MachineInstr *MI, int PhysReg, + SmallVectorImpl<MachineInstr*> &Uses); + /// Provides the clearance - the number of instructions since the closest /// reaching def instuction of PhysReg that reaches MI. int getClearance(MachineInstr *MI, MCPhysReg PhysReg); + /// Provides the uses, in the same block as MI, of register that MI defines. + /// This does not consider live-outs. + void getReachingLocalUses(MachineInstr *MI, int PhysReg, + SmallVectorImpl<MachineInstr*> &Uses); + + /// Provide the number of uses, in the same block as MI, of the register that + /// MI defines. + unsigned getNumUses(MachineInstr *MI, int PhysReg); + private: /// Set up LiveRegs by merging predecessor live-out values. void enterBasicBlock(const LoopTraversal::TraversedMBBInfo &TraversedMBB); diff --git a/llvm/include/llvm/CodeGen/RegisterUsageInfo.h b/llvm/include/llvm/CodeGen/RegisterUsageInfo.h index 33554550b9dc..53982ce5d4a2 100644 --- a/llvm/include/llvm/CodeGen/RegisterUsageInfo.h +++ b/llvm/include/llvm/CodeGen/RegisterUsageInfo.h @@ -21,6 +21,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/IR/Instructions.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include <cstdint> #include <vector> diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 6b8e2dd803ba..3bfde5b4ce1d 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -58,6 +58,7 @@ namespace llvm { class AAResults; class BlockAddress; +class BlockFrequencyInfo; class Constant; class ConstantFP; class ConstantInt; @@ -71,6 +72,7 @@ class MachineBasicBlock; class MachineConstantPoolValue; class MCSymbol; class OptimizationRemarkEmitter; +class ProfileSummaryInfo; class SDDbgValue; class SDDbgLabel; class SelectionDAG; @@ -235,6 +237,9 @@ class SelectionDAG { /// whenever manipulating the DAG. OptimizationRemarkEmitter *ORE; + ProfileSummaryInfo *PSI = nullptr; + BlockFrequencyInfo *BFI = nullptr; + /// The starting token. SDNode EntryNode; @@ -401,7 +406,8 @@ public: /// Prepare this SelectionDAG to process code in the given MachineFunction. void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, - LegacyDivergenceAnalysis * Divergence); + LegacyDivergenceAnalysis * Divergence, + ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin); void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) { FLI = FuncInfo; @@ -421,8 +427,10 @@ public: const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; } - LLVMContext *getContext() const {return Context; } + LLVMContext *getContext() const { return Context; } OptimizationRemarkEmitter &getORE() const { return *ORE; } + ProfileSummaryInfo *getPSI() const { return PSI; } + BlockFrequencyInfo *getBFI() const { return BFI; } /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'. void viewGraph(const std::string &Title); @@ -779,6 +787,20 @@ public: return getNode(ISD::BUILD_VECTOR, DL, VT, Ops); } + // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all + // elements. + SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) { + if (Op.getOpcode() == ISD::UNDEF) { + assert((VT.getVectorElementType() == Op.getValueType() || + (VT.isInteger() && + VT.getVectorElementType().bitsLE(Op.getValueType()))) && + "A splatted value must have a width equal or (for integers) " + "greater than the vector element type!"); + return getNode(ISD::UNDEF, SDLoc(), VT); + } + return getNode(ISD::SPLAT_VECTOR, DL, VT, Op); + } + /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to /// the shuffle node in input but with swapped operands. /// @@ -789,6 +811,11 @@ public: /// float type VT, by either extending or rounding (by truncation). SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT); + /// Convert Op, which must be a STRICT operation of float type, to the + /// float type VT, by either extending or rounding (by truncation). + std::pair<SDValue, SDValue> + getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT); + /// Convert Op, which must be of integer type, to the /// integer type VT, by either any-extending or truncating it. SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); @@ -826,22 +853,28 @@ public: /// Create a logical NOT operation as (XOR Val, BooleanOne). SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT); + /// Returns sum of the base pointer and offset. + /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default. + SDValue getMemBasePlusOffset(SDValue Base, int64_t Offset, const SDLoc &DL, + const SDNodeFlags Flags = SDNodeFlags()); + SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL, + const SDNodeFlags Flags = SDNodeFlags()); + /// Create an add instruction with appropriate flags when used for /// addressing some offset of an object. i.e. if a load is split into multiple /// components, create an add nuw from the base pointer to the offset. - SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset) { - EVT VT = Op.getValueType(); - return getObjectPtrOffset(SL, Op, getConstant(Offset, SL, VT)); + SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, int64_t Offset) { + SDNodeFlags Flags; + Flags.setNoUnsignedWrap(true); + return getMemBasePlusOffset(Ptr, Offset, SL, Flags); } - SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, SDValue Offset) { - EVT VT = Op.getValueType(); - + SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) { // The object itself can't wrap around the address space, so it shouldn't be // possible for the adds of the offsets to the split parts to overflow. SDNodeFlags Flags; Flags.setNoUnsignedWrap(true); - return getNode(ISD::ADD, SL, VT, Op, Offset, Flags); + return getMemBasePlusOffset(Ptr, Offset, SL, Flags); } /// Return a new CALLSEQ_START node, that starts new call frame, in which @@ -961,13 +994,17 @@ public: /// Helper function to make it easier to build SetCC's if you just have an /// ISD::CondCode instead of an SDValue. SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, - ISD::CondCode Cond) { + ISD::CondCode Cond, SDValue Chain = SDValue(), + bool IsSignaling = false) { assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() && "Cannot compare scalars to vectors"); assert(LHS.getValueType().isVector() == VT.isVector() && "Cannot compare scalars to vectors"); assert(Cond != ISD::SETCC_INVALID && "Cannot create a setCC of an invalid node."); + if (Chain) + return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL, + {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)}); return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond)); } @@ -1111,17 +1148,19 @@ public: SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM); - /// Returns sum of the base pointer and offset. - SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, const SDLoc &DL); - - SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, - SDValue Mask, SDValue Src0, EVT MemVT, - MachineMemOperand *MMO, ISD::LoadExtType, - bool IsExpanding = false); + SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, + SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, + MachineMemOperand *MMO, ISD::MemIndexedMode AM, + ISD::LoadExtType, bool IsExpanding = false); + SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, + SDValue Offset, ISD::MemIndexedMode AM); SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, - SDValue Ptr, SDValue Mask, EVT MemVT, - MachineMemOperand *MMO, bool IsTruncating = false, - bool IsCompressing = false); + SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, + MachineMemOperand *MMO, ISD::MemIndexedMode AM, + bool IsTruncating = false, bool IsCompressing = false); + SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, + SDValue Base, SDValue Offset, + ISD::MemIndexedMode AM); SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType); @@ -1697,6 +1736,14 @@ public: return It->second.HeapAllocSite; } + /// Return the current function's default denormal handling kind for the given + /// floating point type. + DenormalMode getDenormalMode(EVT VT) const { + return MF->getDenormalMode(EVTToAPFloatSemantics(VT)); + } + + bool shouldOptForSize() const; + private: void InsertNode(SDNode *N); bool RemoveNodeFromCSEMaps(SDNode *N); diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index de71a21d4671..9874d782c782 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -39,6 +39,8 @@ class SwiftErrorValueTracking; class GCFunctionInfo; class ScheduleDAGSDNodes; class LoadInst; +class ProfileSummaryInfo; +class BlockFrequencyInfo; /// SelectionDAGISel - This is the common base class used for SelectionDAG-based /// pattern-matching instruction selectors. @@ -46,12 +48,12 @@ class SelectionDAGISel : public MachineFunctionPass { public: TargetMachine &TM; const TargetLibraryInfo *LibInfo; - FunctionLoweringInfo *FuncInfo; + std::unique_ptr<FunctionLoweringInfo> FuncInfo; SwiftErrorValueTracking *SwiftError; MachineFunction *MF; MachineRegisterInfo *RegInfo; SelectionDAG *CurDAG; - SelectionDAGBuilder *SDB; + std::unique_ptr<SelectionDAGBuilder> SDB; AAResults *AA; GCFunctionInfo *GFI; CodeGenOpt::Level OptLevel; @@ -249,6 +251,11 @@ protected: virtual StringRef getIncludePathForIndex(unsigned index) { llvm_unreachable("Tblgen should generate the implementation of this!"); } + + bool shouldOptForSize(const MachineFunction *MF) const { + return CurDAG->shouldOptForSize(); + } + public: // Calls to these predicates are generated by tblgen. bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS, @@ -303,6 +310,9 @@ public: return false; } + /// Return whether the node may raise an FP exception. + bool mayRaiseFPException(SDNode *Node) const; + bool isOrEquivalentToAdd(const SDNode *N) const; private: diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index ceb8b72635a2..d81a4a8fd43f 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -42,6 +42,7 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MachineValueType.h" +#include "llvm/Support/TypeSize.h" #include <algorithm> #include <cassert> #include <climits> @@ -170,11 +171,15 @@ public: } /// Returns the size of the value in bits. - unsigned getValueSizeInBits() const { + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getValueSizeInBits() const { return getValueType().getSizeInBits(); } - unsigned getScalarValueSizeInBits() const { + TypeSize getScalarValueSizeInBits() const { return getValueType().getScalarType().getSizeInBits(); } @@ -382,7 +387,7 @@ public: Exact(false), NoNaNs(false), NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false), AllowContract(false), ApproximateFuncs(false), - AllowReassociation(false), NoFPExcept(true) {} + AllowReassociation(false), NoFPExcept(false) {} /// Propagate the fast-math-flags from an IR FPMathOperator. void copyFMF(const FPMathOperator &FPMO) { @@ -445,9 +450,9 @@ public: setDefined(); AllowReassociation = b; } - void setFPExcept(bool b) { + void setNoFPExcept(bool b) { setDefined(); - NoFPExcept = !b; + NoFPExcept = b; } // These are accessors for each flag. @@ -462,7 +467,7 @@ public: bool hasAllowContract() const { return AllowContract; } bool hasApproximateFuncs() const { return ApproximateFuncs; } bool hasAllowReassociation() const { return AllowReassociation; } - bool hasFPExcept() const { return !NoFPExcept; } + bool hasNoFPExcept() const { return NoFPExcept; } bool isFast() const { return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept && @@ -548,6 +553,7 @@ BEGIN_TWO_BYTE_PACK() class LSBaseSDNodeBitfields { friend class LSBaseSDNode; + friend class MaskedLoadStoreSDNode; friend class MaskedGatherScatterSDNode; uint16_t : NumMemSDNodeBits; @@ -555,6 +561,7 @@ BEGIN_TWO_BYTE_PACK() // This storage is shared between disparate class hierarchies to hold an // enumeration specific to the class hierarchy in use. // LSBaseSDNode => enum ISD::MemIndexedMode + // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode // MaskedGatherScatterSDNode => enum ISD::MemIndexType uint16_t AddressingMode : 3; }; @@ -659,6 +666,15 @@ public: /// \<target\>ISD namespace). bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } + /// Test if this node has a target-specific opcode that may raise + /// FP exceptions (in the \<target\>ISD namespace and greater than + /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory + /// opcode are currently automatically considered to possibly raise + /// FP exceptions as well. + bool isTargetStrictFPOpcode() const { + return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE; + } + /// Test if this node has a target-specific /// memory-referencing opcode (in the \<target\>ISD namespace and /// greater than FIRST_TARGET_MEMORY_OPCODE). @@ -685,38 +701,9 @@ public: switch (NodeType) { default: return false; - case ISD::STRICT_FADD: - case ISD::STRICT_FSUB: - case ISD::STRICT_FMUL: - case ISD::STRICT_FDIV: - case ISD::STRICT_FREM: - case ISD::STRICT_FMA: - case ISD::STRICT_FSQRT: - case ISD::STRICT_FPOW: - case ISD::STRICT_FPOWI: - case ISD::STRICT_FSIN: - case ISD::STRICT_FCOS: - case ISD::STRICT_FEXP: - case ISD::STRICT_FEXP2: - case ISD::STRICT_FLOG: - case ISD::STRICT_FLOG10: - case ISD::STRICT_FLOG2: - case ISD::STRICT_LRINT: - case ISD::STRICT_LLRINT: - case ISD::STRICT_FRINT: - case ISD::STRICT_FNEARBYINT: - case ISD::STRICT_FMAXNUM: - case ISD::STRICT_FMINNUM: - case ISD::STRICT_FCEIL: - case ISD::STRICT_FFLOOR: - case ISD::STRICT_LROUND: - case ISD::STRICT_LLROUND: - case ISD::STRICT_FROUND: - case ISD::STRICT_FTRUNC: - case ISD::STRICT_FP_TO_SINT: - case ISD::STRICT_FP_TO_UINT: - case ISD::STRICT_FP_ROUND: - case ISD::STRICT_FP_EXTEND: +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ + case ISD::STRICT_##DAGN: +#include "llvm/IR/ConstrainedOps.def" return true; } } @@ -1022,7 +1009,11 @@ public: } /// Returns MVT::getSizeInBits(getValueType(ResNo)). - unsigned getValueSizeInBits(unsigned ResNo) const { + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getValueSizeInBits(unsigned ResNo) const { return getValueType(ResNo).getSizeInBits(); } @@ -2293,19 +2284,38 @@ public: friend class SelectionDAG; MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, - const DebugLoc &dl, SDVTList VTs, EVT MemVT, + const DebugLoc &dl, SDVTList VTs, + ISD::MemIndexedMode AM, EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + LSBaseSDNodeBits.AddressingMode = AM; + assert(getAddressingMode() == AM && "Value truncated"); + } - // MaskedLoadSDNode (Chain, ptr, mask, passthru) - // MaskedStoreSDNode (Chain, data, ptr, mask) + // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru) + // MaskedStoreSDNode (Chain, data, ptr, offset, mask) // Mask is a vector of i1 elements const SDValue &getBasePtr() const { return getOperand(getOpcode() == ISD::MLOAD ? 1 : 2); } - const SDValue &getMask() const { + const SDValue &getOffset() const { return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3); } + const SDValue &getMask() const { + return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4); + } + + /// Return the addressing mode for this load or store: + /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. + ISD::MemIndexedMode getAddressingMode() const { + return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); + } + + /// Return true if this is a pre/post inc/dec load/store. + bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } + + /// Return true if this is NOT a pre/post inc/dec load/store. + bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD || @@ -2319,9 +2329,9 @@ public: friend class SelectionDAG; MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, - ISD::LoadExtType ETy, bool IsExpanding, EVT MemVT, - MachineMemOperand *MMO) - : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) { + ISD::MemIndexedMode AM, ISD::LoadExtType ETy, + bool IsExpanding, EVT MemVT, MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) { LoadSDNodeBits.ExtTy = ETy; LoadSDNodeBits.IsExpanding = IsExpanding; } @@ -2331,8 +2341,9 @@ public: } const SDValue &getBasePtr() const { return getOperand(1); } - const SDValue &getMask() const { return getOperand(2); } - const SDValue &getPassThru() const { return getOperand(3); } + const SDValue &getOffset() const { return getOperand(2); } + const SDValue &getMask() const { return getOperand(3); } + const SDValue &getPassThru() const { return getOperand(4); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MLOAD; @@ -2347,9 +2358,9 @@ public: friend class SelectionDAG; MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, - bool isTrunc, bool isCompressing, EVT MemVT, - MachineMemOperand *MMO) - : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { + ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing, + EVT MemVT, MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) { StoreSDNodeBits.IsTruncating = isTrunc; StoreSDNodeBits.IsCompressing = isCompressing; } @@ -2365,9 +2376,10 @@ public: /// memory at base_addr. bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } - const SDValue &getValue() const { return getOperand(1); } + const SDValue &getValue() const { return getOperand(1); } const SDValue &getBasePtr() const { return getOperand(2); } - const SDValue &getMask() const { return getOperand(3); } + const SDValue &getOffset() const { return getOperand(3); } + const SDValue &getMask() const { return getOperand(4); } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MSTORE; diff --git a/llvm/include/llvm/CodeGen/SlotIndexes.h b/llvm/include/llvm/CodeGen/SlotIndexes.h index 2b32a4d30dff..fb833806ca8e 100644 --- a/llvm/include/llvm/CodeGen/SlotIndexes.h +++ b/llvm/include/llvm/CodeGen/SlotIndexes.h @@ -347,14 +347,9 @@ class raw_ostream; public: static char ID; - SlotIndexes() : MachineFunctionPass(ID), mf(nullptr) { - initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); - } + SlotIndexes(); - ~SlotIndexes() override { - // The indexList's nodes are all allocated in the BumpPtrAllocator. - indexList.clearAndLeakNodesUnsafely(); - } + ~SlotIndexes() override; void getAnalysisUsage(AnalysisUsage &au) const override; void releaseMemory() override; diff --git a/llvm/include/llvm/CodeGen/StackMaps.h b/llvm/include/llvm/CodeGen/StackMaps.h index d7d88de6f682..63547e5b7c3e 100644 --- a/llvm/include/llvm/CodeGen/StackMaps.h +++ b/llvm/include/llvm/CodeGen/StackMaps.h @@ -266,13 +266,16 @@ public: /// Generate a stackmap record for a stackmap instruction. /// /// MI must be a raw STACKMAP, not a PATCHPOINT. - void recordStackMap(const MachineInstr &MI); + void recordStackMap(const MCSymbol &L, + const MachineInstr &MI); /// Generate a stackmap record for a patchpoint instruction. - void recordPatchPoint(const MachineInstr &MI); + void recordPatchPoint(const MCSymbol &L, + const MachineInstr &MI); /// Generate a stackmap record for a statepoint instruction. - void recordStatepoint(const MachineInstr &MI); + void recordStatepoint(const MCSymbol &L, + const MachineInstr &MI); /// If there is any stack map data, create a stack map section and serialize /// the map info into it. This clears the stack map data structures @@ -306,12 +309,15 @@ private: /// registers that need to be recorded in the stackmap. LiveOutVec parseRegisterLiveOutMask(const uint32_t *Mask) const; - /// This should be called by the MC lowering code _immediately_ before - /// lowering the MI to an MCInst. It records where the operands for the - /// instruction are stored, and outputs a label to record the offset of - /// the call from the start of the text section. In special cases (e.g. AnyReg - /// calling convention) the return register is also recorded if requested. - void recordStackMapOpers(const MachineInstr &MI, uint64_t ID, + /// Record the locations of the operands of the provided instruction in a + /// record keyed by the provided label. For instructions w/AnyReg calling + /// convention the return register is also recorded if requested. For + /// STACKMAP, and PATCHPOINT the label is expected to immediately *preceed* + /// lowering of the MI to MCInsts. For STATEPOINT, it expected to + /// immediately *follow*. It's not clear this difference was intentional, + /// but it exists today. + void recordStackMapOpers(const MCSymbol &L, + const MachineInstr &MI, uint64_t ID, MachineInstr::const_mop_iterator MOI, MachineInstr::const_mop_iterator MOE, bool recordResult = false); diff --git a/llvm/include/llvm/CodeGen/StackProtector.h b/llvm/include/llvm/CodeGen/StackProtector.h index ed52db3e6269..d2ab79cb235e 100644 --- a/llvm/include/llvm/CodeGen/StackProtector.h +++ b/llvm/include/llvm/CodeGen/StackProtector.h @@ -104,9 +104,7 @@ private: public: static char ID; // Pass identification, replacement for typeid. - StackProtector() : FunctionPass(ID), SSPBufferSize(8) { - initializeStackProtectorPass(*PassRegistry::getPassRegistry()); - } + StackProtector(); void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h index b8adcf759b19..4d6afa617d3a 100644 --- a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h +++ b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h @@ -19,6 +19,7 @@ namespace llvm { class FunctionLoweringInfo; class MachineBasicBlock; +class BlockFrequencyInfo; namespace SwitchCG { @@ -264,7 +265,8 @@ public: std::vector<BitTestBlock> BitTestCases; void findJumpTables(CaseClusterVector &Clusters, const SwitchInst *SI, - MachineBasicBlock *DefaultMBB); + MachineBasicBlock *DefaultMBB, + ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI); bool buildJumpTable(const CaseClusterVector &Clusters, unsigned First, unsigned Last, const SwitchInst *SI, @@ -295,4 +297,3 @@ private: } // namespace llvm #endif // LLVM_CODEGEN_SWITCHLOWERINGUTILS_H - diff --git a/llvm/include/llvm/CodeGen/TailDuplicator.h b/llvm/include/llvm/CodeGen/TailDuplicator.h index 358798d5ed60..e0623a3193e5 100644 --- a/llvm/include/llvm/CodeGen/TailDuplicator.h +++ b/llvm/include/llvm/CodeGen/TailDuplicator.h @@ -25,11 +25,13 @@ namespace llvm { class MachineBasicBlock; +class MachineBlockFrequencyInfo; class MachineBranchProbabilityInfo; class MachineFunction; class MachineInstr; class MachineModuleInfo; class MachineRegisterInfo; +class ProfileSummaryInfo; class TargetRegisterInfo; /// Utility class to perform tail duplication. @@ -40,6 +42,8 @@ class TailDuplicator { const MachineModuleInfo *MMI; MachineRegisterInfo *MRI; MachineFunction *MF; + const MachineBlockFrequencyInfo *MBFI; + ProfileSummaryInfo *PSI; bool PreRegAlloc; bool LayoutMode; unsigned TailDupSize; @@ -65,6 +69,8 @@ public: /// default implies using the command line value TailDupSize. void initMF(MachineFunction &MF, bool PreRegAlloc, const MachineBranchProbabilityInfo *MBPI, + const MachineBlockFrequencyInfo *MBFI, + ProfileSummaryInfo *PSI, bool LayoutMode, unsigned TailDupSize = 0); bool tailDuplicateBlocks(); diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h index db3d1175afee..f515050efadb 100644 --- a/llvm/include/llvm/CodeGen/TargetCallingConv.h +++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h @@ -38,6 +38,7 @@ namespace ISD { unsigned IsSplitEnd : 1; ///< Last part of a split unsigned IsSwiftSelf : 1; ///< Swift self parameter unsigned IsSwiftError : 1; ///< Swift error parameter + unsigned IsCFGuardTarget : 1; ///< Control Flow Guard target unsigned IsHva : 1; ///< HVA field for unsigned IsHvaStart : 1; ///< HVA structure start unsigned IsSecArgPass : 1; ///< Second argument @@ -56,8 +57,8 @@ namespace ISD { ArgFlagsTy() : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsNest(0), IsReturned(0), IsSplit(0), IsInAlloca(0), IsSplitEnd(0), - IsSwiftSelf(0), IsSwiftError(0), IsHva(0), IsHvaStart(0), - IsSecArgPass(0), ByValAlign(0), OrigAlign(0), + IsSwiftSelf(0), IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), + IsHvaStart(0), IsSecArgPass(0), ByValAlign(0), OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0), IsCopyElisionCandidate(0), IsPointer(0), ByValSize(0), PointerAddrSpace(0) { @@ -88,6 +89,9 @@ namespace ISD { bool isSwiftError() const { return IsSwiftError; } void setSwiftError() { IsSwiftError = 1; } + bool isCFGuardTarget() const { return IsCFGuardTarget; } + void setCFGuardTarget() { IsCFGuardTarget = 1; } + bool isHva() const { return IsHva; } void setHva() { IsHva = 1; } diff --git a/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/llvm/include/llvm/CodeGen/TargetFrameLowering.h index 72edb27964c4..c7d4c4d7e5d4 100644 --- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h +++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h @@ -282,6 +282,11 @@ public: return getFrameIndexReference(MF, FI, FrameReg); } + /// Returns the callee-saved registers as computed by determineCalleeSaves + /// in the BitVector \p SavedRegs. + virtual void getCalleeSaves(const MachineFunction &MF, + BitVector &SavedRegs) const; + /// This method determines which of the registers reported by /// TargetRegisterInfo::getCalleeSavedRegs() should actually get saved. /// The default implementation checks populates the \p SavedRegs bitset with @@ -289,6 +294,9 @@ public: /// this function to save additional registers. /// This method also sets up the register scavenger ensuring there is a free /// register or a frameindex available. + /// This method should not be called by any passes outside of PEI, because + /// it may change state passed in by \p MF and \p RS. The preferred + /// interface outside PEI is getCalleeSaves. virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS = nullptr) const; @@ -355,6 +363,11 @@ public: return true; } + /// Returns the StackID that scalable vectors should be associated with. + virtual TargetStackID::Value getStackIDForScalableVectors() const { + return TargetStackID::Default; + } + virtual bool isSupportedStackID(TargetStackID::Value ID) const { switch (ID) { default: diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index 5011cf34c0ee..ec3c0a0194f6 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -18,6 +18,7 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/None.h" #include "llvm/CodeGen/LiveRegUnits.h" +#include "llvm/CodeGen/MIRFormatter.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineCombinerPattern.h" #include "llvm/CodeGen/MachineFunction.h" @@ -51,6 +52,7 @@ class MCInst; struct MCSchedModel; class Module; class ScheduleDAG; +class ScheduleDAGMI; class ScheduleHazardRecognizer; class SDNode; class SelectionDAG; @@ -64,6 +66,22 @@ template <class T> class SmallVectorImpl; using ParamLoadedValue = std::pair<MachineOperand, DIExpression*>; +struct DestSourcePair { + const MachineOperand *Destination; + const MachineOperand *Source; + + DestSourcePair(const MachineOperand &Dest, const MachineOperand &Src) + : Destination(&Dest), Source(&Src) {} +}; + +/// Used to describe a register and immediate addition. +struct RegImmPair { + Register Reg; + int64_t Imm; + + RegImmPair(Register Reg, int64_t Imm) : Reg(Reg), Imm(Imm) {} +}; + //--------------------------------------------------------------------------- /// /// TargetInstrInfo - Interface to description of machine instruction set @@ -912,36 +930,42 @@ public: /// large registers. See for example the ARM target. virtual void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, - unsigned DestReg, unsigned SrcReg, + MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const { llvm_unreachable("Target didn't implement TargetInstrInfo::copyPhysReg!"); } protected: - /// Target-dependent implemenation for IsCopyInstr. + /// Target-dependent implementation for IsCopyInstr. /// If the specific machine instruction is a instruction that moves/copies - /// value from one register to another register return true along with - /// @Source machine operand and @Destination machine operand. - virtual bool isCopyInstrImpl(const MachineInstr &MI, - const MachineOperand *&Source, - const MachineOperand *&Destination) const { - return false; + /// value from one register to another register return destination and source + /// registers as machine operands. + virtual Optional<DestSourcePair> + isCopyInstrImpl(const MachineInstr &MI) const { + return None; } public: /// If the specific machine instruction is a instruction that moves/copies - /// value from one register to another register return true along with - /// @Source machine operand and @Destination machine operand. - /// For COPY-instruction the method naturally returns true, for all other - /// instructions the method calls target-dependent implementation. - bool isCopyInstr(const MachineInstr &MI, const MachineOperand *&Source, - const MachineOperand *&Destination) const { + /// value from one register to another register return destination and source + /// registers as machine operands. + /// For COPY-instruction the method naturally returns destination and source + /// registers as machine operands, for all other instructions the method calls + /// target-dependent implementation. + Optional<DestSourcePair> isCopyInstr(const MachineInstr &MI) const { if (MI.isCopy()) { - Destination = &MI.getOperand(0); - Source = &MI.getOperand(1); - return true; + return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; } - return isCopyInstrImpl(MI, Source, Destination); + return isCopyInstrImpl(MI); + } + + /// If the specific machine instruction is an instruction that adds an + /// immediate value and a physical register, and stores the result in + /// the given physical register \c Reg, return a pair of the source + /// register and the offset which has been added. + virtual Optional<RegImmPair> isAddImmediate(const MachineInstr &MI, + Register Reg) const { + return None; } /// Store the specified register of the given register class to the specified @@ -1213,6 +1237,10 @@ public: /// Get the base operand and byte offset of an instruction that reads/writes /// memory. + /// It returns false if MI does not read/write memory. + /// It returns false if no base operand and offset was found. + /// It is not guaranteed to always recognize base operand and offsets in all + /// cases. virtual bool getMemOperandWithOffset(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, @@ -1343,7 +1371,7 @@ public: /// scheduling the machine instructions before register allocation. virtual ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *, - const ScheduleDAG *DAG) const; + const ScheduleDAGMI *DAG) const; /// Allocate and return a hazard recognizer to use for this target when /// scheduling the machine instructions after register allocation. @@ -1621,9 +1649,9 @@ public: virtual bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const { - assert((MIa.mayLoad() || MIa.mayStore()) && + assert(MIa.mayLoadOrStore() && "MIa must load from or modify a memory location"); - assert((MIb.mayLoad() || MIb.mayStore()) && + assert(MIb.mayLoadOrStore() && "MIb must load from or modify a memory location"); return false; } @@ -1713,7 +1741,7 @@ public: virtual MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, - Register SrcSubReg, + unsigned SrcSubReg, Register Dst) const { return BuildMI(MBB, InsPt, DL, get(TargetOpcode::COPY), Dst) .addReg(Src, 0, SrcSubReg); @@ -1775,11 +1803,21 @@ public: } /// Produce the expression describing the \p MI loading a value into - /// the parameter's forwarding register. - virtual Optional<ParamLoadedValue> - describeLoadedValue(const MachineInstr &MI) const; + /// the physical register \p Reg. This hook should only be used with + /// \p MIs belonging to VReg-less functions. + virtual Optional<ParamLoadedValue> describeLoadedValue(const MachineInstr &MI, + Register Reg) const; + + /// Return MIR formatter to format/parse MIR operands. Target can override + /// this virtual function and return target specific MIR formatter. + virtual const MIRFormatter *getMIRFormatter() const { + if (!Formatter.get()) + Formatter = std::make_unique<MIRFormatter>(); + return Formatter.get(); + } private: + mutable std::unique_ptr<MIRFormatter> Formatter; unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode; unsigned CatchRetOpcode; unsigned ReturnOpcode; diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index a58fca7e73f5..24daf70dc008 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -28,6 +28,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/RuntimeLibcalls.h" @@ -53,6 +54,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/Utils/SizeOpts.h" #include <algorithm> #include <cassert> #include <climits> @@ -188,13 +190,14 @@ public: bool IsReturned : 1; bool IsSwiftSelf : 1; bool IsSwiftError : 1; + bool IsCFGuardTarget : 1; uint16_t Alignment = 0; Type *ByValType = nullptr; ArgListEntry() : IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false), IsNest(false), IsByVal(false), IsInAlloca(false), IsReturned(false), - IsSwiftSelf(false), IsSwiftError(false) {} + IsSwiftSelf(false), IsSwiftError(false), IsCFGuardTarget(false) {} void setAttributes(const CallBase *Call, unsigned ArgIdx); @@ -222,12 +225,16 @@ public: llvm_unreachable("Invalid content kind"); } - /// NOTE: The TargetMachine owns TLOF. explicit TargetLoweringBase(const TargetMachine &TM); TargetLoweringBase(const TargetLoweringBase &) = delete; TargetLoweringBase &operator=(const TargetLoweringBase &) = delete; virtual ~TargetLoweringBase() = default; + /// Return true if the target support strict float operation + bool isStrictFPEnabled() const { + return IsStrictFPEnabled; + } + protected: /// Initialize all of the actions to default values. void initActions(); @@ -469,6 +476,10 @@ public: return false; } + /// Return true if instruction generated for equality comparison is folded + /// with instruction generated for signed comparison. + virtual bool isEqualityCmpFoldedWithSignedCmp() const { return true; } + /// Return true if it is safe to transform an integer-domain bitwise operation /// into the equivalent floating-point operation. This should be set to true /// if the target has IEEE-754-compliant fabs/fneg operations for the input @@ -924,6 +935,8 @@ public: case ISD::SMULFIXSAT: case ISD::UMULFIX: case ISD::UMULFIXSAT: + case ISD::SDIVFIX: + case ISD::UDIVFIX: Supported = isSupportedFixedPointOperation(Op, VT, Scale); break; } @@ -937,38 +950,11 @@ public: unsigned EqOpc; switch (Op) { default: llvm_unreachable("Unexpected FP pseudo-opcode"); - case ISD::STRICT_FADD: EqOpc = ISD::FADD; break; - case ISD::STRICT_FSUB: EqOpc = ISD::FSUB; break; - case ISD::STRICT_FMUL: EqOpc = ISD::FMUL; break; - case ISD::STRICT_FDIV: EqOpc = ISD::FDIV; break; - case ISD::STRICT_FREM: EqOpc = ISD::FREM; break; - case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break; - case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break; - case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break; - case ISD::STRICT_FMA: EqOpc = ISD::FMA; break; - case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break; - case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break; - case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break; - case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break; - case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break; - case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break; - case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break; - case ISD::STRICT_LRINT: EqOpc = ISD::LRINT; break; - case ISD::STRICT_LLRINT: EqOpc = ISD::LLRINT; break; - case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break; - case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break; - case ISD::STRICT_FMAXNUM: EqOpc = ISD::FMAXNUM; break; - case ISD::STRICT_FMINNUM: EqOpc = ISD::FMINNUM; break; - case ISD::STRICT_FCEIL: EqOpc = ISD::FCEIL; break; - case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break; - case ISD::STRICT_LROUND: EqOpc = ISD::LROUND; break; - case ISD::STRICT_LLROUND: EqOpc = ISD::LLROUND; break; - case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break; - case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break; - case ISD::STRICT_FP_TO_SINT: EqOpc = ISD::FP_TO_SINT; break; - case ISD::STRICT_FP_TO_UINT: EqOpc = ISD::FP_TO_UINT; break; - case ISD::STRICT_FP_ROUND: EqOpc = ISD::FP_ROUND; break; - case ISD::STRICT_FP_EXTEND: EqOpc = ISD::FP_EXTEND; break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ + case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break; +#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ + case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break; +#include "llvm/IR/ConstrainedOps.def" } return getOperationAction(EqOpc, VT); @@ -1029,24 +1015,8 @@ public: /// Return true if lowering to a jump table is suitable for a set of case /// clusters which may contain \p NumCases cases, \p Range range of values. virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases, - uint64_t Range) const { - // FIXME: This function check the maximum table size and density, but the - // minimum size is not checked. It would be nice if the minimum size is - // also combined within this function. Currently, the minimum size check is - // performed in findJumpTable() in SelectionDAGBuiler and - // getEstimatedNumberOfCaseClusters() in BasicTTIImpl. - const bool OptForSize = SI->getParent()->getParent()->hasOptSize(); - const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize); - const unsigned MaxJumpTableSize = getMaximumJumpTableSize(); - - // Check whether the number of cases is small enough and - // the range is dense enough for a jump table. - if ((OptForSize || Range <= MaxJumpTableSize) && - (NumCases * 100 >= Range * MinDensity)) { - return true; - } - return false; - } + uint64_t Range, ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI) const; /// Return true if lowering to a bit test is suitable for a set of case /// clusters which contains \p NumDests unique destinations, \p Low and @@ -1143,12 +1113,8 @@ public: /// Return how the indexed load should be treated: either it is legal, needs /// to be promoted to a larger size, needs to be expanded to some other code /// sequence, or the target has a custom expander for it. - LegalizeAction - getIndexedLoadAction(unsigned IdxMode, MVT VT) const { - assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() && - "Table isn't big enough!"); - unsigned Ty = (unsigned)VT.SimpleTy; - return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] & 0xf0) >> 4); + LegalizeAction getIndexedLoadAction(unsigned IdxMode, MVT VT) const { + return getIndexedModeAction(IdxMode, VT, IMAB_Load); } /// Return true if the specified indexed load is legal on this target. @@ -1161,12 +1127,8 @@ public: /// Return how the indexed store should be treated: either it is legal, needs /// to be promoted to a larger size, needs to be expanded to some other code /// sequence, or the target has a custom expander for it. - LegalizeAction - getIndexedStoreAction(unsigned IdxMode, MVT VT) const { - assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() && - "Table isn't big enough!"); - unsigned Ty = (unsigned)VT.SimpleTy; - return (LegalizeAction)(IndexedModeActions[Ty][IdxMode] & 0x0f); + LegalizeAction getIndexedStoreAction(unsigned IdxMode, MVT VT) const { + return getIndexedModeAction(IdxMode, VT, IMAB_Store); } /// Return true if the specified indexed load is legal on this target. @@ -1176,6 +1138,34 @@ public: getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Custom); } + /// Return how the indexed load should be treated: either it is legal, needs + /// to be promoted to a larger size, needs to be expanded to some other code + /// sequence, or the target has a custom expander for it. + LegalizeAction getIndexedMaskedLoadAction(unsigned IdxMode, MVT VT) const { + return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad); + } + + /// Return true if the specified indexed load is legal on this target. + bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const { + return VT.isSimple() && + (getIndexedMaskedLoadAction(IdxMode, VT.getSimpleVT()) == Legal || + getIndexedMaskedLoadAction(IdxMode, VT.getSimpleVT()) == Custom); + } + + /// Return how the indexed store should be treated: either it is legal, needs + /// to be promoted to a larger size, needs to be expanded to some other code + /// sequence, or the target has a custom expander for it. + LegalizeAction getIndexedMaskedStoreAction(unsigned IdxMode, MVT VT) const { + return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore); + } + + /// Return true if the specified indexed load is legal on this target. + bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const { + return VT.isSimple() && + (getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Legal || + getIndexedMaskedStoreAction(IdxMode, VT.getSimpleVT()) == Custom); + } + /// Return how the condition code should be treated: either it is legal, needs /// to be expanded to some other code sequence, or the target has a custom /// expander for it. @@ -1265,7 +1255,7 @@ public: Elm = PointerTy.getTypeForEVT(Ty->getContext()); } return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(Elm, false), - VTy->getNumElements()); + VTy->getElementCount()); } return getValueType(DL, Ty, AllowUnknown); @@ -1550,16 +1540,6 @@ public: /// have to be legal as the hook is used before type legalization. virtual bool isSafeMemOpType(MVT /*VT*/) const { return true; } - /// Determine if we should use _setjmp or setjmp to implement llvm.setjmp. - bool usesUnderscoreSetJmp() const { - return UseUnderscoreSetJmp; - } - - /// Determine if we should use _longjmp or longjmp to implement llvm.longjmp. - bool usesUnderscoreLongJmp() const { - return UseUnderscoreLongJmp; - } - /// Return lower limit for number of blocks in a jump table. virtual unsigned getMinimumJumpTableEntries() const; @@ -1960,18 +1940,6 @@ protected: SchedPreferenceInfo = Pref; } - /// Indicate whether this target prefers to use _setjmp to implement - /// llvm.setjmp or the version without _. Defaults to false. - void setUseUnderscoreSetJmp(bool Val) { - UseUnderscoreSetJmp = Val; - } - - /// Indicate whether this target prefers to use _longjmp to implement - /// llvm.longjmp or the version without _. Defaults to false. - void setUseUnderscoreLongJmp(bool Val) { - UseUnderscoreLongJmp = Val; - } - /// Indicate the minimum number of blocks to generate jump tables. void setMinimumJumpTableEntries(unsigned Val); @@ -2063,13 +2031,8 @@ protected: /// /// NOTE: All indexed mode loads are initialized to Expand in /// TargetLowering.cpp - void setIndexedLoadAction(unsigned IdxMode, MVT VT, - LegalizeAction Action) { - assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE && - (unsigned)Action < 0xf && "Table isn't big enough!"); - // Load action are kept in the upper half. - IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] &= ~0xf0; - IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] |= ((uint8_t)Action) <<4; + void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action) { + setIndexedModeAction(IdxMode, VT, IMAB_Load, Action); } /// Indicate that the specified indexed store does or does not work with the @@ -2077,13 +2040,28 @@ protected: /// /// NOTE: All indexed mode stores are initialized to Expand in /// TargetLowering.cpp - void setIndexedStoreAction(unsigned IdxMode, MVT VT, - LegalizeAction Action) { - assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE && - (unsigned)Action < 0xf && "Table isn't big enough!"); - // Store action are kept in the lower half. - IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] &= ~0x0f; - IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] |= ((uint8_t)Action); + void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action) { + setIndexedModeAction(IdxMode, VT, IMAB_Store, Action); + } + + /// Indicate that the specified indexed masked load does or does not work with + /// the specified type and indicate what to do about it. + /// + /// NOTE: All indexed mode masked loads are initialized to Expand in + /// TargetLowering.cpp + void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, + LegalizeAction Action) { + setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action); + } + + /// Indicate that the specified indexed masked store does or does not work + /// with the specified type and indicate what to do about it. + /// + /// NOTE: All indexed mode masked stores are initialized to Expand in + /// TargetLowering.cpp + void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, + LegalizeAction Action) { + setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action); } /// Indicate that the specified condition code is or isn't supported on the @@ -2504,7 +2482,8 @@ public: /// Return true if an fpext operation input to an \p Opcode operation is free /// (for instance, because half-precision floating-point numbers are /// implicitly extended to float-precision) for an FMA instruction. - virtual bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const { + virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, + EVT DestVT, EVT SrcVT) const { assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && "invalid fpext types"); return isFPExtFree(DestVT, SrcVT); @@ -2536,10 +2515,24 @@ public: /// not legal, but should return true if those types will eventually legalize /// to types that support FMAs. After legalization, it will only be called on /// types that support FMAs (via Legal or Custom actions) - virtual bool isFMAFasterThanFMulAndFAdd(EVT) const { + virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, + EVT) const { return false; } + /// IR version + virtual bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *) const { + return false; + } + + /// Returns true if the FADD or FSUB node passed could legally be combined with + /// an fmul to form an ISD::FMAD. + virtual bool isFMADLegalForFAddFSub(const SelectionDAG &DAG, + const SDNode *N) const { + assert(N->getOpcode() == ISD::FADD || N->getOpcode() == ISD::FSUB); + return isOperationLegal(ISD::FMAD, N->getValueType(0)); + } + /// Return true if it's profitable to narrow operations of type VT1 to /// VT2. e.g. on x86, it's profitable to narrow from i32 to i8 but not from /// i32 to i16. @@ -2608,10 +2601,10 @@ public: // same blocks of its users. virtual bool shouldConsiderGEPOffsetSplit() const { return false; } - // Return the shift amount threshold for profitable transforms into shifts. - // Transforms creating shifts above the returned value will be avoided. - virtual unsigned getShiftAmountThreshold(EVT VT) const { - return VT.getScalarSizeInBits(); + /// Return true if creating a shift of the type by the given + /// amount is not profitable. + virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const { + return false; } //===--------------------------------------------------------------------===// @@ -2683,16 +2676,6 @@ private: /// predication. bool JumpIsExpensive; - /// This target prefers to use _setjmp to implement llvm.setjmp. - /// - /// Defaults to false. - bool UseUnderscoreSetJmp; - - /// This target prefers to use _longjmp to implement llvm.longjmp. - /// - /// Defaults to false. - bool UseUnderscoreLongJmp; - /// Information about the contents of the high-bits in boolean values held in /// a type wider than i1. See getBooleanContents. BooleanContent BooleanContents; @@ -2741,7 +2724,7 @@ private: /// This indicates the default register class to use for each ValueType the /// target supports natively. const TargetRegisterClass *RegClassForVT[MVT::LAST_VALUETYPE]; - unsigned char NumRegistersForVT[MVT::LAST_VALUETYPE]; + uint16_t NumRegistersForVT[MVT::LAST_VALUETYPE]; MVT RegisterTypeForVT[MVT::LAST_VALUETYPE]; /// This indicates the "representative" register class to use for each @@ -2781,13 +2764,13 @@ private: /// truncating store of a specific value type and truncating type is legal. LegalizeAction TruncStoreActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE]; - /// For each indexed mode and each value type, keep a pair of LegalizeAction + /// For each indexed mode and each value type, keep a quad of LegalizeAction /// that indicates how instruction selection should deal with the load / - /// store. + /// store / maskedload / maskedstore. /// /// The first dimension is the value_type for the reference. The second /// dimension represents the various modes for load store. - uint8_t IndexedModeActions[MVT::LAST_VALUETYPE][ISD::LAST_INDEXED_MODE]; + uint16_t IndexedModeActions[MVT::LAST_VALUETYPE][ISD::LAST_INDEXED_MODE]; /// For each condition code (ISD::CondCode) keep a LegalizeAction that /// indicates how instruction selection should deal with the condition code. @@ -2830,6 +2813,32 @@ private: /// Set default libcall names and calling conventions. void InitLibcalls(const Triple &TT); + /// The bits of IndexedModeActions used to store the legalisation actions + /// We store the data as | ML | MS | L | S | each taking 4 bits. + enum IndexedModeActionsBits { + IMAB_Store = 0, + IMAB_Load = 4, + IMAB_MaskedStore = 8, + IMAB_MaskedLoad = 12 + }; + + void setIndexedModeAction(unsigned IdxMode, MVT VT, unsigned Shift, + LegalizeAction Action) { + assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE && + (unsigned)Action < 0xf && "Table isn't big enough!"); + unsigned Ty = (unsigned)VT.SimpleTy; + IndexedModeActions[Ty][IdxMode] &= ~(0xf << Shift); + IndexedModeActions[Ty][IdxMode] |= ((uint16_t)Action) << Shift; + } + + LegalizeAction getIndexedModeAction(unsigned IdxMode, MVT VT, + unsigned Shift) const { + assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() && + "Table isn't big enough!"); + unsigned Ty = (unsigned)VT.SimpleTy; + return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf); + } + protected: /// Return true if the extension represented by \p I is free. /// \pre \p I is a sign, zero, or fp extension and @@ -2932,6 +2941,8 @@ protected: /// details. MachineBasicBlock *emitXRayTypedEvent(MachineInstr &MI, MachineBasicBlock *MBB) const; + + bool IsStrictFPEnabled; }; /// This class defines information used to lower LLVM code to legal SelectionDAG @@ -2947,7 +2958,6 @@ public: TargetLowering(const TargetLowering &) = delete; TargetLowering &operator=(const TargetLowering &) = delete; - /// NOTE: The TargetMachine owns TLOF. explicit TargetLowering(const TargetMachine &TM); bool isPositionIndependent() const; @@ -3024,12 +3034,19 @@ public: const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const; + void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, + SDValue &NewRHS, ISD::CondCode &CCCode, + const SDLoc &DL, const SDValue OldLHS, + const SDValue OldRHS, SDValue &Chain, + bool IsSignaling = false) const; + /// Returns a pair of (return value, chain). /// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC. std::pair<SDValue, SDValue> makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops, MakeLibCallOptions CallOptions, - const SDLoc &dl) const; + const SDLoc &dl, + SDValue Chain = SDValue()) const; /// Check whether parameters to a call that are passed in callee saved /// registers are the same as from the calling function. This needs to be @@ -3262,9 +3279,7 @@ public: bool isBeforeLegalize() const { return Level == BeforeLegalizeTypes; } bool isBeforeLegalizeOps() const { return Level < AfterLegalizeVectorOps; } - bool isAfterLegalizeDAG() const { - return Level == AfterLegalizeDAG; - } + bool isAfterLegalizeDAG() const { return Level >= AfterLegalizeDAG; } CombineLevel getDAGCombineLevel() { return Level; } bool isCalledByLegalizer() const { return CalledByLegalizer; } @@ -3695,7 +3710,7 @@ public: /// Return the register ID of the name passed in. Used by named register /// global variables extension. There is no target-independent behaviour /// so the default action is to bail. - virtual Register getRegisterByName(const char* RegName, EVT VT, + virtual Register getRegisterByName(const char* RegName, LLT Ty, const MachineFunction &MF) const { report_fatal_error("Named registers not implemented for this target"); } @@ -3758,7 +3773,7 @@ public: /// Should SelectionDAG lower an atomic store of the given kind as a normal /// StoreSDNode (as opposed to an AtomicSDNode)? NOTE: The intention is to /// eventually migrate all targets to the using StoreSDNodes, but porting is - /// being done target at a time. + /// being done target at a time. virtual bool lowerAtomicStoreAsStoreSDNode(const StoreInst &SI) const { assert(SI.isAtomic() && "violated precondition"); return false; @@ -3940,9 +3955,7 @@ public: StringRef Constraint, MVT VT) const; virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const { - if (ConstraintCode == "i") - return InlineAsm::Constraint_i; - else if (ConstraintCode == "m") + if (ConstraintCode == "m") return InlineAsm::Constraint_m; return InlineAsm::Constraint_Unknown; } @@ -4082,14 +4095,18 @@ public: /// Expand float to UINT conversion /// \param N Node to expand /// \param Result output after conversion + /// \param Chain output chain after conversion /// \returns True, if the expansion was successful, false otherwise - bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const; + bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, + SelectionDAG &DAG) const; /// Expand UINT(i64) to double(f64) conversion /// \param N Node to expand /// \param Result output after conversion + /// \param Chain output chain after conversion /// \returns True, if the expansion was successful, false otherwise - bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SelectionDAG &DAG) const; + bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain, + SelectionDAG &DAG) const; /// Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs. SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const; @@ -4125,12 +4142,13 @@ public: /// Turn load of vector type into a load of the individual elements. /// \param LD load to expand - /// \returns MERGE_VALUEs of the scalar loads with their chains. - SDValue scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const; + /// \returns BUILD_VECTOR and TokenFactor nodes. + std::pair<SDValue, SDValue> scalarizeVectorLoad(LoadSDNode *LD, + SelectionDAG &DAG) const; // Turn a store of a vector type into stores of the individual elements. /// \param ST Store with a vector value type - /// \returns MERGE_VALUs of the individual store chains. + /// \returns TokenFactor of the individual store chains. SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const; /// Expands an unaligned load to 2 half-size loads for an integer, and @@ -4168,6 +4186,14 @@ public: /// method accepts integers as its arguments. SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const; + /// Method for building the DAG expansion of ISD::[US]DIVFIX. This + /// method accepts integers as its arguments. + /// Note: This method may fail if the division could not be performed + /// within the type. Clients must retry with a wider type if this happens. + SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, + SDValue LHS, SDValue RHS, + unsigned Scale, SelectionDAG &DAG) const; + /// Method for building the DAG expansion of ISD::U(ADD|SUB)O. Expansion /// always suceeds and populates the Result and Overflow arguments. void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index 59f5ddbd9dac..4f58df93b93e 100644 --- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -233,6 +233,15 @@ public: MCSection *SelectSectionForGlobal(const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const override; + MCSection *getSectionForJumpTable(const Function &F, + const TargetMachine &TM) const override; + + /// Given a constant with the SectionKind, return a section that it should be + /// placed in. + MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind, + const Constant *C, + unsigned &Align) const override; + static XCOFF::StorageClass getStorageClassForGlobal(const GlobalObject *GO); }; diff --git a/llvm/include/llvm/CodeGen/TargetSchedule.h b/llvm/include/llvm/CodeGen/TargetSchedule.h index cce85c8d7b0d..aa6b82e14aa6 100644 --- a/llvm/include/llvm/CodeGen/TargetSchedule.h +++ b/llvm/include/llvm/CodeGen/TargetSchedule.h @@ -37,8 +37,12 @@ class TargetSchedModel { const TargetInstrInfo *TII = nullptr; SmallVector<unsigned, 16> ResourceFactors; - unsigned MicroOpFactor; // Multiply to normalize microops to resource units. - unsigned ResourceLCM; // Resource units per cycle. Latency normalization factor. + + // Multiply to normalize microops to resource units. + unsigned MicroOpFactor = 0; + + // Resource units per cycle. Latency normalization factor. + unsigned ResourceLCM = 0; unsigned computeInstrLatency(const MCSchedClassDesc &SCDesc) const; diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h index 56018eca8c27..6768cea89406 100644 --- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -206,6 +206,10 @@ public: /// which is the preferred way to influence this. virtual bool enablePostRAScheduler() const; + /// True if the subtarget should run a machine scheduler after register + /// allocation. + virtual bool enablePostRAMachineScheduler() const; + /// True if the subtarget should run the atomic expansion pass. virtual bool enableAtomicExpand() const; diff --git a/llvm/include/llvm/CodeGen/ValueTypes.h b/llvm/include/llvm/CodeGen/ValueTypes.h index cd4c4ca64081..bcf417762920 100644 --- a/llvm/include/llvm/CodeGen/ValueTypes.h +++ b/llvm/include/llvm/CodeGen/ValueTypes.h @@ -18,6 +18,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/TypeSize.h" #include <cassert> #include <cstdint> #include <string> @@ -209,11 +210,13 @@ namespace llvm { /// Return true if the bit size is a multiple of 8. bool isByteSized() const { - return (getSizeInBits() & 7) == 0; + return getSizeInBits().isByteSized(); } /// Return true if the size is a power-of-two number of bytes. bool isRound() const { + if (isScalableVector()) + return false; unsigned BitSize = getSizeInBits(); return BitSize >= 8 && !(BitSize & (BitSize - 1)); } @@ -288,25 +291,38 @@ namespace llvm { } /// Return the size of the specified value type in bits. - unsigned getSizeInBits() const { + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getSizeInBits() const { if (isSimple()) return V.getSizeInBits(); return getExtendedSizeInBits(); } - unsigned getScalarSizeInBits() const { + TypeSize getScalarSizeInBits() const { return getScalarType().getSizeInBits(); } /// Return the number of bytes overwritten by a store of the specified value /// type. - unsigned getStoreSize() const { - return (getSizeInBits() + 7) / 8; + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getStoreSize() const { + TypeSize BaseSize = getSizeInBits(); + return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()}; } /// Return the number of bits overwritten by a store of the specified value /// type. - unsigned getStoreSizeInBits() const { + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getStoreSizeInBits() const { return getStoreSize() * 8; } @@ -428,7 +444,7 @@ namespace llvm { bool isExtended2048BitVector() const LLVM_READONLY; EVT getExtendedVectorElementType() const; unsigned getExtendedVectorNumElements() const LLVM_READONLY; - unsigned getExtendedSizeInBits() const LLVM_READONLY; + TypeSize getExtendedSizeInBits() const LLVM_READONLY; }; } // end namespace llvm diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/DWARFLinker.h new file mode 100644 index 000000000000..80df01ca0539 --- /dev/null +++ b/llvm/include/llvm/DWARFLinker/DWARFLinker.h @@ -0,0 +1,198 @@ +//===- DWARFLinker.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DWARFLINKER_DWARFLINKER_H +#define LLVM_DWARFLINKER_DWARFLINKER_H + +#include "llvm/CodeGen/AccelTable.h" +#include "llvm/CodeGen/NonRelocatableStringpool.h" +#include "llvm/DWARFLinker/DWARFLinkerDeclContext.h" +#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/MC/MCDwarf.h" +#include <map> + +namespace llvm { + +enum class DwarfLinkerClient { Dsymutil, LLD, General }; + +/// Partial address range. Besides an offset, only the +/// HighPC is stored. The structure is stored in a map where the LowPC is the +/// key. +struct ObjFileAddressRange { + /// Function HighPC. + uint64_t HighPC; + /// Offset to apply to the linked address. + /// should be 0 for not-linked object file. + int64_t Offset; + + ObjFileAddressRange(uint64_t EndPC, int64_t Offset) + : HighPC(EndPC), Offset(Offset) {} + + ObjFileAddressRange() : HighPC(0), Offset(0) {} +}; + +/// Map LowPC to ObjFileAddressRange. +using RangesTy = std::map<uint64_t, ObjFileAddressRange>; + +/// AddressesMap represents information about valid addresses used +/// by debug information. Valid addresses are those which points to +/// live code sections. i.e. relocations for these addresses point +/// into sections which would be/are placed into resulting binary. +class AddressesMap { +public: + virtual ~AddressesMap(); + + /// Returns true if represented addresses are from linked file. + /// Returns false if represented addresses are from not-linked + /// object file. + virtual bool areRelocationsResolved() const = 0; + + /// Checks that there are valid relocations against a .debug_info + /// section. Reset current relocation pointer if neccessary. + virtual bool hasValidRelocs(bool ResetRelocsPtr = true) = 0; + + /// Checks that there is a relocation against .debug_info + /// table between \p StartOffset and \p NextOffset. + /// + /// This function must be called with offsets in strictly ascending + /// order because it never looks back at relocations it already 'went past'. + /// \returns true and sets Info.InDebugMap if it is the case. + virtual bool hasValidRelocationAt(uint64_t StartOffset, uint64_t EndOffset, + CompileUnit::DIEInfo &Info) = 0; + + /// Apply the valid relocations to the buffer \p Data, taking into + /// account that Data is at \p BaseOffset in the debug_info section. + /// + /// This function must be called with monotonic \p BaseOffset values. + /// + /// \returns true whether any reloc has been applied. + virtual bool applyValidRelocs(MutableArrayRef<char> Data, uint64_t BaseOffset, + bool IsLittleEndian) = 0; + + /// Returns all valid functions address ranges(i.e., those ranges + /// which points to sections with code). + virtual RangesTy &getValidAddressRanges() = 0; + + /// Erases all data. + virtual void clear() = 0; +}; + +/// DwarfEmitter presents interface to generate all debug info tables. +class DwarfEmitter { +public: + virtual ~DwarfEmitter(); + + /// Emit DIE containing warnings. + virtual void emitPaperTrailWarningsDie(const Triple &Triple, DIE &Die) = 0; + + /// Emit section named SecName with content equals to + /// corresponding section in Obj. + virtual void emitSectionContents(const object::ObjectFile &Obj, + StringRef SecName) = 0; + + /// Emit the abbreviation table \p Abbrevs to the debug_abbrev section. + virtual void + emitAbbrevs(const std::vector<std::unique_ptr<DIEAbbrev>> &Abbrevs, + unsigned DwarfVersion) = 0; + + /// Emit the string table described by \p Pool. + virtual void emitStrings(const NonRelocatableStringpool &Pool) = 0; + + /// Emit DWARF debug names. + virtual void + emitDebugNames(AccelTable<DWARF5AccelTableStaticData> &Table) = 0; + + /// Emit Apple namespaces accelerator table. + virtual void + emitAppleNamespaces(AccelTable<AppleAccelTableStaticOffsetData> &Table) = 0; + + /// Emit Apple names accelerator table. + virtual void + emitAppleNames(AccelTable<AppleAccelTableStaticOffsetData> &Table) = 0; + + /// Emit Apple Objective-C accelerator table. + virtual void + emitAppleObjc(AccelTable<AppleAccelTableStaticOffsetData> &Table) = 0; + + /// Emit Apple type accelerator table. + virtual void + emitAppleTypes(AccelTable<AppleAccelTableStaticTypeData> &Table) = 0; + + /// Emit debug_ranges for \p FuncRange by translating the + /// original \p Entries. + virtual void emitRangesEntries( + int64_t UnitPcOffset, uint64_t OrigLowPc, + const FunctionIntervals::const_iterator &FuncRange, + const std::vector<DWARFDebugRangeList::RangeListEntry> &Entries, + unsigned AddressSize) = 0; + + /// Emit debug_aranges entries for \p Unit and if \p DoRangesSection is true, + /// also emit the debug_ranges entries for the DW_TAG_compile_unit's + /// DW_AT_ranges attribute. + virtual void emitUnitRangesEntries(CompileUnit &Unit, + bool DoRangesSection) = 0; + + /// Copy the debug_line over to the updated binary while unobfuscating the + /// file names and directories. + virtual void translateLineTable(DataExtractor LineData, uint64_t Offset) = 0; + + /// Emit the line table described in \p Rows into the debug_line section. + virtual void emitLineTableForUnit(MCDwarfLineTableParams Params, + StringRef PrologueBytes, + unsigned MinInstLength, + std::vector<DWARFDebugLine::Row> &Rows, + unsigned AdddressSize) = 0; + + /// Emit the .debug_pubnames contribution for \p Unit. + virtual void emitPubNamesForUnit(const CompileUnit &Unit) = 0; + + /// Emit the .debug_pubtypes contribution for \p Unit. + virtual void emitPubTypesForUnit(const CompileUnit &Unit) = 0; + + /// Emit a CIE. + virtual void emitCIE(StringRef CIEBytes) = 0; + + /// Emit an FDE with data \p Bytes. + virtual void emitFDE(uint32_t CIEOffset, uint32_t AddreSize, uint32_t Address, + StringRef Bytes) = 0; + + /// Emit the debug_loc contribution for \p Unit by copying the entries from + /// \p Dwarf and offsetting them. Update the location attributes to point to + /// the new entries. + virtual void emitLocationsForUnit( + const CompileUnit &Unit, DWARFContext &Dwarf, + std::function<void(StringRef, SmallVectorImpl<uint8_t> &)> + ProcessExpr) = 0; + + /// Emit the compilation unit header for \p Unit in the + /// debug_info section. + /// + /// As a side effect, this also switches the current Dwarf version + /// of the MC layer to the one of U.getOrigUnit(). + virtual void emitCompileUnitHeader(CompileUnit &Unit) = 0; + + /// Recursively emit the DIE tree rooted at \p Die. + virtual void emitDIE(DIE &Die) = 0; + + /// Returns size of generated .debug_line section. + virtual uint64_t getLineSectionSize() const = 0; + + /// Returns size of generated .debug_frame section. + virtual uint64_t getFrameSectionSize() const = 0; + + /// Returns size of generated .debug_ranges section. + virtual uint64_t getRangesSectionSize() const = 0; + + /// Returns size of generated .debug_info section. + virtual uint64_t getDebugInfoSectionSize() const = 0; +}; + +} // end namespace llvm + +#endif // LLVM_DWARFLINKER_DWARFLINKER_H diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h new file mode 100644 index 000000000000..7873a16fea52 --- /dev/null +++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerCompileUnit.h @@ -0,0 +1,330 @@ +//===- DWARFLinkerCompileUnit.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H +#define LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H + +#include "llvm/ADT/IntervalMap.h" +#include "llvm/CodeGen/DIE.h" +#include "llvm/DebugInfo/DWARF/DWARFUnit.h" +#include "llvm/Support/DataExtractor.h" + +namespace llvm { + +class DeclContext; + +template <typename KeyT, typename ValT> +using HalfOpenIntervalMap = + IntervalMap<KeyT, ValT, IntervalMapImpl::NodeSizer<KeyT, ValT>::LeafSize, + IntervalMapHalfOpenInfo<KeyT>>; + +using FunctionIntervals = HalfOpenIntervalMap<uint64_t, int64_t>; + +// FIXME: Delete this structure. +struct PatchLocation { + DIE::value_iterator I; + + PatchLocation() = default; + PatchLocation(DIE::value_iterator I) : I(I) {} + + void set(uint64_t New) const { + assert(I); + const auto &Old = *I; + assert(Old.getType() == DIEValue::isInteger); + *I = DIEValue(Old.getAttribute(), Old.getForm(), DIEInteger(New)); + } + + uint64_t get() const { + assert(I); + return I->getDIEInteger().getValue(); + } +}; + +/// Stores all information relating to a compile unit, be it in its original +/// instance in the object file to its brand new cloned and generated DIE tree. +class CompileUnit { +public: + /// Information gathered about a DIE in the object file. + struct DIEInfo { + /// Address offset to apply to the described entity. + int64_t AddrAdjust; + + /// ODR Declaration context. + DeclContext *Ctxt; + + /// Cloned version of that DIE. + DIE *Clone; + + /// The index of this DIE's parent. + uint32_t ParentIdx; + + /// Is the DIE part of the linked output? + bool Keep : 1; + + /// Was this DIE's entity found in the map? + bool InDebugMap : 1; + + /// Is this a pure forward declaration we can strip? + bool Prune : 1; + + /// Does DIE transitively refer an incomplete decl? + bool Incomplete : 1; + }; + + CompileUnit(DWARFUnit &OrigUnit, unsigned ID, bool CanUseODR, + StringRef ClangModuleName) + : OrigUnit(OrigUnit), ID(ID), Ranges(RangeAlloc), + ClangModuleName(ClangModuleName) { + Info.resize(OrigUnit.getNumDIEs()); + + auto CUDie = OrigUnit.getUnitDIE(false); + if (!CUDie) { + HasODR = false; + return; + } + if (auto Lang = dwarf::toUnsigned(CUDie.find(dwarf::DW_AT_language))) + HasODR = CanUseODR && (*Lang == dwarf::DW_LANG_C_plus_plus || + *Lang == dwarf::DW_LANG_C_plus_plus_03 || + *Lang == dwarf::DW_LANG_C_plus_plus_11 || + *Lang == dwarf::DW_LANG_C_plus_plus_14 || + *Lang == dwarf::DW_LANG_ObjC_plus_plus); + else + HasODR = false; + } + + DWARFUnit &getOrigUnit() const { return OrigUnit; } + + unsigned getUniqueID() const { return ID; } + + void createOutputDIE() { + NewUnit.emplace(OrigUnit.getVersion(), OrigUnit.getAddressByteSize(), + OrigUnit.getUnitDIE().getTag()); + } + + DIE *getOutputUnitDIE() const { + if (NewUnit) + return &const_cast<BasicDIEUnit &>(*NewUnit).getUnitDie(); + return nullptr; + } + + bool hasODR() const { return HasODR; } + bool isClangModule() const { return !ClangModuleName.empty(); } + uint16_t getLanguage(); + + const std::string &getClangModuleName() const { return ClangModuleName; } + + DIEInfo &getInfo(unsigned Idx) { return Info[Idx]; } + const DIEInfo &getInfo(unsigned Idx) const { return Info[Idx]; } + + uint64_t getStartOffset() const { return StartOffset; } + uint64_t getNextUnitOffset() const { return NextUnitOffset; } + void setStartOffset(uint64_t DebugInfoSize) { StartOffset = DebugInfoSize; } + + uint64_t getLowPc() const { return LowPc; } + uint64_t getHighPc() const { return HighPc; } + bool hasLabelAt(uint64_t Addr) const { return Labels.count(Addr); } + + Optional<PatchLocation> getUnitRangesAttribute() const { + return UnitRangeAttribute; + } + + const FunctionIntervals &getFunctionRanges() const { return Ranges; } + + const std::vector<PatchLocation> &getRangesAttributes() const { + return RangeAttributes; + } + + const std::vector<std::pair<PatchLocation, int64_t>> & + getLocationAttributes() const { + return LocationAttributes; + } + + void setHasInterestingContent() { HasInterestingContent = true; } + bool hasInterestingContent() { return HasInterestingContent; } + + /// Mark every DIE in this unit as kept. This function also + /// marks variables as InDebugMap so that they appear in the + /// reconstructed accelerator tables. + void markEverythingAsKept(); + + /// Compute the end offset for this unit. Must be called after the CU's DIEs + /// have been cloned. \returns the next unit offset (which is also the + /// current debug_info section size). + uint64_t computeNextUnitOffset(); + + /// Keep track of a forward reference to DIE \p Die in \p RefUnit by \p + /// Attr. The attribute should be fixed up later to point to the absolute + /// offset of \p Die in the debug_info section or to the canonical offset of + /// \p Ctxt if it is non-null. + void noteForwardReference(DIE *Die, const CompileUnit *RefUnit, + DeclContext *Ctxt, PatchLocation Attr); + + /// Apply all fixups recorded by noteForwardReference(). + void fixupForwardReferences(); + + /// Add the low_pc of a label that is relocated by applying + /// offset \p PCOffset. + void addLabelLowPc(uint64_t LabelLowPc, int64_t PcOffset); + + /// Add a function range [\p LowPC, \p HighPC) that is relocated by applying + /// offset \p PCOffset. + void addFunctionRange(uint64_t LowPC, uint64_t HighPC, int64_t PCOffset); + + /// Keep track of a DW_AT_range attribute that we will need to patch up later. + void noteRangeAttribute(const DIE &Die, PatchLocation Attr); + + /// Keep track of a location attribute pointing to a location list in the + /// debug_loc section. + void noteLocationAttribute(PatchLocation Attr, int64_t PcOffset); + + /// Add a name accelerator entry for \a Die with \a Name. + void addNamespaceAccelerator(const DIE *Die, DwarfStringPoolEntryRef Name); + + /// Add a name accelerator entry for \a Die with \a Name. + void addNameAccelerator(const DIE *Die, DwarfStringPoolEntryRef Name, + bool SkipPubnamesSection = false); + + /// Add various accelerator entries for \p Die with \p Name which is stored + /// in the string table at \p Offset. \p Name must be an Objective-C + /// selector. + void addObjCAccelerator(const DIE *Die, DwarfStringPoolEntryRef Name, + bool SkipPubnamesSection = false); + + /// Add a type accelerator entry for \p Die with \p Name which is stored in + /// the string table at \p Offset. + void addTypeAccelerator(const DIE *Die, DwarfStringPoolEntryRef Name, + bool ObjcClassImplementation, + uint32_t QualifiedNameHash); + + struct AccelInfo { + /// Name of the entry. + DwarfStringPoolEntryRef Name; + + /// DIE this entry describes. + const DIE *Die; + + /// Hash of the fully qualified name. + uint32_t QualifiedNameHash; + + /// Emit this entry only in the apple_* sections. + bool SkipPubSection; + + /// Is this an ObjC class implementation? + bool ObjcClassImplementation; + + AccelInfo(DwarfStringPoolEntryRef Name, const DIE *Die, + bool SkipPubSection = false) + : Name(Name), Die(Die), SkipPubSection(SkipPubSection) {} + + AccelInfo(DwarfStringPoolEntryRef Name, const DIE *Die, + uint32_t QualifiedNameHash, bool ObjCClassIsImplementation) + : Name(Name), Die(Die), QualifiedNameHash(QualifiedNameHash), + SkipPubSection(false), + ObjcClassImplementation(ObjCClassIsImplementation) {} + }; + + const std::vector<AccelInfo> &getPubnames() const { return Pubnames; } + const std::vector<AccelInfo> &getPubtypes() const { return Pubtypes; } + const std::vector<AccelInfo> &getNamespaces() const { return Namespaces; } + const std::vector<AccelInfo> &getObjC() const { return ObjC; } + + /// Get the full path for file \a FileNum in the line table + StringRef getResolvedPath(unsigned FileNum) { + if (FileNum >= ResolvedPaths.size()) + return StringRef(); + return ResolvedPaths[FileNum]; + } + + /// Set the fully resolved path for the line-table's file \a FileNum + /// to \a Path. + void setResolvedPath(unsigned FileNum, StringRef Path) { + if (ResolvedPaths.size() <= FileNum) + ResolvedPaths.resize(FileNum + 1); + ResolvedPaths[FileNum] = Path; + } + + MCSymbol *getLabelBegin() { return LabelBegin; } + void setLabelBegin(MCSymbol *S) { LabelBegin = S; } + +private: + DWARFUnit &OrigUnit; + unsigned ID; + std::vector<DIEInfo> Info; ///< DIE info indexed by DIE index. + Optional<BasicDIEUnit> NewUnit; + MCSymbol *LabelBegin = nullptr; + + uint64_t StartOffset; + uint64_t NextUnitOffset; + + uint64_t LowPc = std::numeric_limits<uint64_t>::max(); + uint64_t HighPc = 0; + + /// A list of attributes to fixup with the absolute offset of + /// a DIE in the debug_info section. + /// + /// The offsets for the attributes in this array couldn't be set while + /// cloning because for cross-cu forward references the target DIE's offset + /// isn't known you emit the reference attribute. + std::vector< + std::tuple<DIE *, const CompileUnit *, DeclContext *, PatchLocation>> + ForwardDIEReferences; + + FunctionIntervals::Allocator RangeAlloc; + + /// The ranges in that interval map are the PC ranges for + /// functions in this unit, associated with the PC offset to apply + /// to the addresses to get the linked address. + FunctionIntervals Ranges; + + /// The DW_AT_low_pc of each DW_TAG_label. + SmallDenseMap<uint64_t, uint64_t, 1> Labels; + + /// DW_AT_ranges attributes to patch after we have gathered + /// all the unit's function addresses. + /// @{ + std::vector<PatchLocation> RangeAttributes; + Optional<PatchLocation> UnitRangeAttribute; + /// @} + + /// Location attributes that need to be transferred from the + /// original debug_loc section to the liked one. They are stored + /// along with the PC offset that is to be applied to their + /// function's address. + std::vector<std::pair<PatchLocation, int64_t>> LocationAttributes; + + /// Accelerator entries for the unit, both for the pub* + /// sections and the apple* ones. + /// @{ + std::vector<AccelInfo> Pubnames; + std::vector<AccelInfo> Pubtypes; + std::vector<AccelInfo> Namespaces; + std::vector<AccelInfo> ObjC; + /// @} + + /// Cached resolved paths from the line table. + /// Note, the StringRefs here point in to the intern (uniquing) string pool. + /// This means that a StringRef returned here doesn't need to then be uniqued + /// for the purposes of getting a unique address for each string. + std::vector<StringRef> ResolvedPaths; + + /// Is this unit subject to the ODR rule? + bool HasODR; + + /// Did a DIE actually contain a valid reloc? + bool HasInterestingContent; + + /// The DW_AT_language of this unit. + uint16_t Language = 0; + + /// If this is a Clang module, this holds the module's name. + std::string ClangModuleName; +}; + +} // end namespace llvm + +#endif // LLVM_DWARFLINKER_DWARFLINKERCOMPILEUNIT_H diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h b/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h new file mode 100644 index 000000000000..db40254bf600 --- /dev/null +++ b/llvm/include/llvm/DWARFLinker/DWARFLinkerDeclContext.h @@ -0,0 +1,169 @@ +//===- DWARFLinkerDeclContext.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DWARFLINKER_DWARFLINKERDECLCONTEXT_H +#define LLVM_DWARFLINKER_DWARFLINKERDECLCONTEXT_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/NonRelocatableStringpool.h" +#include "llvm/DWARFLinker/DWARFLinkerCompileUnit.h" +#include "llvm/DebugInfo/DWARF/DWARFDie.h" +#include "llvm/Support/Path.h" + +namespace llvm { + +struct DeclMapInfo; + +/// Small helper that resolves and caches file paths. This helps reduce the +/// number of calls to realpath which is expensive. We assume the input are +/// files, and cache the realpath of their parent. This way we can quickly +/// resolve different files under the same path. +class CachedPathResolver { +public: + /// Resolve a path by calling realpath and cache its result. The returned + /// StringRef is interned in the given \p StringPool. + StringRef resolve(std::string Path, NonRelocatableStringpool &StringPool) { + StringRef FileName = sys::path::filename(Path); + SmallString<256> ParentPath = sys::path::parent_path(Path); + + // If the ParentPath has not yet been resolved, resolve and cache it for + // future look-ups. + if (!ResolvedPaths.count(ParentPath)) { + SmallString<256> RealPath; + sys::fs::real_path(ParentPath, RealPath); + ResolvedPaths.insert({ParentPath, StringRef(RealPath).str()}); + } + + // Join the file name again with the resolved path. + SmallString<256> ResolvedPath(ResolvedPaths[ParentPath]); + sys::path::append(ResolvedPath, FileName); + return StringPool.internString(ResolvedPath); + } + +private: + StringMap<std::string> ResolvedPaths; +}; + +/// A DeclContext is a named program scope that is used for ODR uniquing of +/// types. +/// +/// The set of DeclContext for the ODR-subject parts of a Dwarf link is +/// expanded (and uniqued) with each new object file processed. We need to +/// determine the context of each DIE in an linked object file to see if the +/// corresponding type has already been emitted. +/// +/// The contexts are conceptually organized as a tree (eg. a function scope is +/// contained in a namespace scope that contains other scopes), but +/// storing/accessing them in an actual tree is too inefficient: we need to be +/// able to very quickly query a context for a given child context by name. +/// Storing a StringMap in each DeclContext would be too space inefficient. +/// +/// The solution here is to give each DeclContext a link to its parent (this +/// allows to walk up the tree), but to query the existence of a specific +/// DeclContext using a separate DenseMap keyed on the hash of the fully +/// qualified name of the context. +class DeclContext { +public: + using Map = DenseSet<DeclContext *, DeclMapInfo>; + + DeclContext() : DefinedInClangModule(0), Parent(*this) {} + + DeclContext(unsigned Hash, uint32_t Line, uint32_t ByteSize, uint16_t Tag, + StringRef Name, StringRef File, const DeclContext &Parent, + DWARFDie LastSeenDIE = DWARFDie(), unsigned CUId = 0) + : QualifiedNameHash(Hash), Line(Line), ByteSize(ByteSize), Tag(Tag), + DefinedInClangModule(0), Name(Name), File(File), Parent(Parent), + LastSeenDIE(LastSeenDIE), LastSeenCompileUnitID(CUId) {} + + uint32_t getQualifiedNameHash() const { return QualifiedNameHash; } + + bool setLastSeenDIE(CompileUnit &U, const DWARFDie &Die); + + uint32_t getCanonicalDIEOffset() const { return CanonicalDIEOffset; } + void setCanonicalDIEOffset(uint32_t Offset) { CanonicalDIEOffset = Offset; } + + bool isDefinedInClangModule() const { return DefinedInClangModule; } + void setDefinedInClangModule(bool Val) { DefinedInClangModule = Val; } + + uint16_t getTag() const { return Tag; } + StringRef getName() const { return Name; } + +private: + friend DeclMapInfo; + + unsigned QualifiedNameHash = 0; + uint32_t Line = 0; + uint32_t ByteSize = 0; + uint16_t Tag = dwarf::DW_TAG_compile_unit; + unsigned DefinedInClangModule : 1; + StringRef Name; + StringRef File; + const DeclContext &Parent; + DWARFDie LastSeenDIE; + uint32_t LastSeenCompileUnitID = 0; + uint32_t CanonicalDIEOffset = 0; +}; + +/// This class gives a tree-like API to the DenseMap that stores the +/// DeclContext objects. It holds the BumpPtrAllocator where these objects will +/// be allocated. +class DeclContextTree { +public: + /// Get the child of \a Context described by \a DIE in \a Unit. The + /// required strings will be interned in \a StringPool. + /// \returns The child DeclContext along with one bit that is set if + /// this context is invalid. + /// + /// An invalid context means it shouldn't be considered for uniquing, but its + /// not returning null, because some children of that context might be + /// uniquing candidates. + /// + /// FIXME: The invalid bit along the return value is to emulate some + /// dsymutil-classic functionality. + PointerIntPair<DeclContext *, 1> + getChildDeclContext(DeclContext &Context, const DWARFDie &DIE, + CompileUnit &Unit, UniquingStringPool &StringPool, + bool InClangModule); + + DeclContext &getRoot() { return Root; } + +private: + BumpPtrAllocator Allocator; + DeclContext Root; + DeclContext::Map Contexts; + + /// Cache resolved paths from the line table. + CachedPathResolver PathResolver; +}; + +/// Info type for the DenseMap storing the DeclContext pointers. +struct DeclMapInfo : private DenseMapInfo<DeclContext *> { + using DenseMapInfo<DeclContext *>::getEmptyKey; + using DenseMapInfo<DeclContext *>::getTombstoneKey; + + static unsigned getHashValue(const DeclContext *Ctxt) { + return Ctxt->QualifiedNameHash; + } + + static bool isEqual(const DeclContext *LHS, const DeclContext *RHS) { + if (RHS == getEmptyKey() || RHS == getTombstoneKey()) + return RHS == LHS; + return LHS->QualifiedNameHash == RHS->QualifiedNameHash && + LHS->Line == RHS->Line && LHS->ByteSize == RHS->ByteSize && + LHS->Name.data() == RHS->Name.data() && + LHS->File.data() == RHS->File.data() && + LHS->Parent.QualifiedNameHash == RHS->Parent.QualifiedNameHash; + } +}; + +} // end namespace llvm + +#endif // LLVM_DWARFLINKER_DWARFLINKERDECLCONTEXT_H diff --git a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h index a43ce20edde6..3b103c227708 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h +++ b/llvm/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h @@ -43,7 +43,7 @@ class GlobalTypeTableBuilder : public TypeCollection { /// Contains a list of all records indexed by TypeIndex.toArrayIndex(). SmallVector<ArrayRef<uint8_t>, 2> SeenRecords; - /// Contains a list of all hash values inexed by TypeIndex.toArrayIndex(). + /// Contains a list of all hash values indexed by TypeIndex.toArrayIndex(). SmallVector<GloballyHashedType, 2> SeenHashes; public: diff --git a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h index 1aafa3ca9f1d..1fcef9dd06c8 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h +++ b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h @@ -79,7 +79,7 @@ public: uint32_t Offset = 0; uint16_t Segment = 0; uint16_t Length = 0; - ThunkOrdinal Thunk; + ThunkOrdinal Thunk = ThunkOrdinal::Standard; StringRef Name; ArrayRef<uint8_t> VariantData; @@ -159,7 +159,7 @@ public: struct DecodedAnnotation { StringRef Name; ArrayRef<uint8_t> Bytes; - BinaryAnnotationsOpCode OpCode; + BinaryAnnotationsOpCode OpCode = BinaryAnnotationsOpCode::Invalid; uint32_t U1 = 0; uint32_t U2 = 0; int32_t S1 = 0; @@ -407,21 +407,21 @@ public: : SymbolRecord(SymbolRecordKind::LocalSym), RecordOffset(RecordOffset) {} TypeIndex Type; - LocalSymFlags Flags; + LocalSymFlags Flags = LocalSymFlags::None; StringRef Name; uint32_t RecordOffset = 0; }; struct LocalVariableAddrRange { - uint32_t OffsetStart; - uint16_t ISectStart; - uint16_t Range; + uint32_t OffsetStart = 0; + uint16_t ISectStart = 0; + uint16_t Range = 0; }; struct LocalVariableAddrGap { - uint16_t GapStartOffset; - uint16_t Range; + uint16_t GapStartOffset = 0; + uint16_t Range = 0; }; enum : uint16_t { MaxDefRange = 0xf000 }; @@ -628,7 +628,7 @@ public: uint32_t CodeOffset = 0; uint16_t Segment = 0; - ProcSymFlags Flags; + ProcSymFlags Flags = ProcSymFlags::None; StringRef Name; uint32_t RecordOffset = 0; @@ -670,7 +670,7 @@ public: : SymbolRecord(SymbolRecordKind::ExportSym), RecordOffset(RecordOffset) {} uint16_t Ordinal = 0; - ExportFlags Flags; + ExportFlags Flags = ExportFlags::None; StringRef Name; uint32_t RecordOffset = 0; @@ -686,7 +686,7 @@ public: TypeIndex Index; uint32_t ModFilenameOffset = 0; - LocalSymFlags Flags; + LocalSymFlags Flags = LocalSymFlags::None; StringRef Name; uint32_t RecordOffset = 0; @@ -700,7 +700,7 @@ public: : SymbolRecord(SymbolRecordKind::Compile2Sym), RecordOffset(RecordOffset) {} - CompileSym2Flags Flags; + CompileSym2Flags Flags = CompileSym2Flags::None; CPUType Machine; uint16_t VersionFrontendMajor = 0; uint16_t VersionFrontendMinor = 0; @@ -726,7 +726,7 @@ public: : SymbolRecord(SymbolRecordKind::Compile3Sym), RecordOffset(RecordOffset) {} - CompileSym3Flags Flags; + CompileSym3Flags Flags = CompileSym3Flags::None; CPUType Machine; uint16_t VersionFrontendMajor = 0; uint16_t VersionFrontendMinor = 0; @@ -771,7 +771,7 @@ public: uint32_t BytesOfCalleeSavedRegisters = 0; uint32_t OffsetOfExceptionHandler = 0; uint16_t SectionIdOfExceptionHandler = 0; - FrameProcedureOptions Flags; + FrameProcedureOptions Flags = FrameProcedureOptions::None; /// Extract the register this frame uses to refer to local variables. RegisterId getLocalFramePtrReg(CPUType CPU) const { diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h index b147dd6c3d05..35f5c0561138 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -144,7 +144,7 @@ public: ModifierOptions getModifiers() const { return Modifiers; } TypeIndex ModifiedType; - ModifierOptions Modifiers; + ModifierOptions Modifiers = ModifierOptions::None; }; // LF_PROCEDURE @@ -168,7 +168,7 @@ public: TypeIndex ReturnType; CallingConvention CallConv; FunctionOptions Options; - uint16_t ParameterCount; + uint16_t ParameterCount = 0; TypeIndex ArgumentList; }; @@ -202,9 +202,9 @@ public: TypeIndex ThisType; CallingConvention CallConv; FunctionOptions Options; - uint16_t ParameterCount; + uint16_t ParameterCount = 0; TypeIndex ArgumentList; - int32_t ThisPointerAdjustment; + int32_t ThisPointerAdjustment = 0; }; // LF_LABEL @@ -351,7 +351,7 @@ public: } TypeIndex ReferentType; - uint32_t Attrs; + uint32_t Attrs = 0; Optional<MemberPointerInfo> MemberInfo; void setAttrs(PointerKind PK, PointerMode PM, PointerOptions PO, @@ -414,7 +414,7 @@ public: TypeIndex ElementType; TypeIndex IndexType; - uint64_t Size; + uint64_t Size = 0; StringRef Name; }; @@ -459,7 +459,7 @@ public: StringRef getName() const { return Name; } StringRef getUniqueName() const { return UniqueName; } - uint16_t MemberCount; + uint16_t MemberCount = 0; ClassOptions Options; TypeIndex FieldList; StringRef Name; @@ -496,7 +496,7 @@ public: TypeIndex DerivationList; TypeIndex VTableShape; - uint64_t Size; + uint64_t Size = 0; }; // LF_UNION @@ -517,7 +517,7 @@ struct UnionRecord : public TagRecord { uint64_t getSize() const { return Size; } - uint64_t Size; + uint64_t Size = 0; }; // LF_ENUM @@ -550,8 +550,8 @@ public: uint8_t getBitSize() const { return BitSize; } TypeIndex Type; - uint8_t BitSize; - uint8_t BitOffset; + uint8_t BitSize = 0; + uint8_t BitOffset = 0; }; // LF_VTSHAPE @@ -592,7 +592,7 @@ public: StringRef getName() const { return Name; } GUID Guid; - uint32_t Age; + uint32_t Age = 0; StringRef Name; }; @@ -644,7 +644,7 @@ public: TypeIndex UDT; TypeIndex SourceFile; - uint32_t LineNumber; + uint32_t LineNumber = 0; }; // LF_UDT_MOD_SRC_LINE @@ -664,8 +664,8 @@ public: TypeIndex UDT; TypeIndex SourceFile; - uint32_t LineNumber; - uint16_t Module; + uint32_t LineNumber = 0; + uint16_t Module = 0; }; // LF_BUILDINFO @@ -717,7 +717,7 @@ public: TypeIndex CompleteClass; TypeIndex OverriddenVFTable; - uint32_t VFPtrOffset; + uint32_t VFPtrOffset = 0; std::vector<StringRef> MethodNames; }; @@ -749,7 +749,7 @@ public: TypeIndex Type; MemberAttributes Attrs; - int32_t VFTableOffset; + int32_t VFTableOffset = 0; StringRef Name; }; @@ -780,7 +780,7 @@ public: TypeIndex getMethodList() const { return MethodList; } StringRef getName() const { return Name; } - uint16_t NumOverloads; + uint16_t NumOverloads = 0; TypeIndex MethodList; StringRef Name; }; @@ -806,7 +806,7 @@ public: MemberAttributes Attrs; TypeIndex Type; - uint64_t FieldOffset; + uint64_t FieldOffset = 0; StringRef Name; }; @@ -883,7 +883,7 @@ public: MemberAttributes Attrs; TypeIndex Type; - uint64_t Offset; + uint64_t Offset = 0; }; // LF_VBCLASS, LF_IVBCLASS @@ -911,8 +911,8 @@ public: MemberAttributes Attrs; TypeIndex BaseType; TypeIndex VBPtrType; - uint64_t VBPtrOffset; - uint64_t VTableIndex; + uint64_t VBPtrOffset = 0; + uint64_t VTableIndex = 0; }; /// LF_INDEX - Used to chain two large LF_FIELDLIST or LF_METHODLIST records @@ -941,9 +941,9 @@ public: uint32_t getSignature() const { return Signature; } StringRef getPrecompFilePath() const { return PrecompFilePath; } - uint32_t StartTypeIndex; - uint32_t TypesCount; - uint32_t Signature; + uint32_t StartTypeIndex = 0; + uint32_t TypesCount = 0; + uint32_t Signature = 0; StringRef PrecompFilePath; }; @@ -955,7 +955,7 @@ public: uint32_t getSignature() const { return Signature; } - uint32_t Signature; + uint32_t Signature = 0; }; } // end namespace codeview diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h b/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h index e84704d99ddc..19492b93681c 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h +++ b/llvm/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h @@ -12,16 +12,35 @@ #include "llvm/DebugInfo/CodeView/TypeRecord.h" namespace llvm { - namespace codeview { - /// Given an arbitrary codeview type, determine if it is an LF_STRUCTURE, - /// LF_CLASS, LF_INTERFACE, LF_UNION, or LF_ENUM with the forward ref class - /// option. - bool isUdtForwardRef(CVType CVT); - - /// Given a CVType which is assumed to be an LF_MODIFIER, return the - /// TypeIndex of the type that the LF_MODIFIER modifies. - TypeIndex getModifiedType(const CVType &CVT); +namespace codeview { + +/// Given an arbitrary codeview type, determine if it is an LF_STRUCTURE, +/// LF_CLASS, LF_INTERFACE, LF_UNION, or LF_ENUM with the forward ref class +/// option. +bool isUdtForwardRef(CVType CVT); + +/// Given a CVType which is assumed to be an LF_MODIFIER, return the +/// TypeIndex of the type that the LF_MODIFIER modifies. +TypeIndex getModifiedType(const CVType &CVT); + +/// Return true if this record should be in the IPI stream of a PDB. In an +/// object file, these record kinds will appear mixed into the .debug$T section. +inline bool isIdRecord(TypeLeafKind K) { + switch (K) { + case TypeLeafKind::LF_FUNC_ID: + case TypeLeafKind::LF_MFUNC_ID: + case TypeLeafKind::LF_STRING_ID: + case TypeLeafKind::LF_SUBSTR_LIST: + case TypeLeafKind::LF_BUILDINFO: + case TypeLeafKind::LF_UDT_SRC_LINE: + case TypeLeafKind::LF_UDT_MOD_SRC_LINE: + return true; + default: + return false; } } +} // namespace codeview +} // namespace llvm + #endif diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h index 2d5f9f3c7658..7a728c2508cc 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h @@ -17,6 +17,7 @@ namespace llvm { class raw_ostream; +class DWARFObject; struct DWARFAddressRange { uint64_t LowPC; @@ -26,7 +27,9 @@ struct DWARFAddressRange { DWARFAddressRange() = default; /// Used for unit testing. - DWARFAddressRange(uint64_t LowPC, uint64_t HighPC, uint64_t SectionIndex = 0) + DWARFAddressRange( + uint64_t LowPC, uint64_t HighPC, + uint64_t SectionIndex = object::SectionedAddress::UndefSection) : LowPC(LowPC), HighPC(HighPC), SectionIndex(SectionIndex) {} /// Returns true if LowPC is smaller or equal to HighPC. This accounts for @@ -42,15 +45,20 @@ struct DWARFAddressRange { return LowPC < RHS.HighPC && RHS.LowPC < HighPC; } - void dump(raw_ostream &OS, uint32_t AddressSize, - DIDumpOptions DumpOpts = {}) const; + void dump(raw_ostream &OS, uint32_t AddressSize, DIDumpOptions DumpOpts = {}, + const DWARFObject *Obj = nullptr) const; }; -static inline bool operator<(const DWARFAddressRange &LHS, - const DWARFAddressRange &RHS) { +inline bool operator<(const DWARFAddressRange &LHS, + const DWARFAddressRange &RHS) { return std::tie(LHS.LowPC, LHS.HighPC) < std::tie(RHS.LowPC, RHS.HighPC); } +inline bool operator==(const DWARFAddressRange &LHS, + const DWARFAddressRange &RHS) { + return std::tie(LHS.LowPC, LHS.HighPC) == std::tie(RHS.LowPC, RHS.HighPC); +} + raw_ostream &operator<<(raw_ostream &OS, const DWARFAddressRange &R); /// DWARFAddressRangesVector - represents a set of absolute address ranges. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h index fae163622edb..a2a10d23433f 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -75,7 +75,7 @@ class DWARFContext : public DIContext { DWARFUnitVector DWOUnits; std::unique_ptr<DWARFDebugAbbrev> AbbrevDWO; - std::unique_ptr<DWARFDebugLoclists> LocDWO; + std::unique_ptr<DWARFDebugMacro> MacroDWO; /// The maximum DWARF version of all units. unsigned MaxVersion = 0; @@ -260,9 +260,6 @@ public: /// Get a pointer to the parsed dwo abbreviations object. const DWARFDebugAbbrev *getDebugAbbrevDWO(); - /// Get a pointer to the parsed DebugLoc object. - const DWARFDebugLoclists *getDebugLocDWO(); - /// Get a pointer to the parsed DebugAranges object. const DWARFDebugAranges *getDebugAranges(); @@ -275,6 +272,9 @@ public: /// Get a pointer to the parsed DebugMacro object. const DWARFDebugMacro *getDebugMacro(); + /// Get a pointer to the parsed DebugMacroDWO object. + const DWARFDebugMacro *getDebugMacroDWO(); + /// Get a reference to the parsed accelerator table object. const DWARFDebugNames &getDebugNames(); @@ -298,7 +298,7 @@ public: /// Report any recoverable parsing problems using the callback. Expected<const DWARFDebugLine::LineTable *> getLineTableForUnit(DWARFUnit *U, - std::function<void(Error)> RecoverableErrorCallback); + function_ref<void(Error)> RecoverableErrorCallback); DataExtractor getStringExtractor() const { return DataExtractor(DObj->getStrSection(), false, 0); diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h index 980724c525d2..6f7ddb2ef421 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h @@ -55,8 +55,6 @@ public: /// reflect the absolute address of this pointer. Optional<uint64_t> getEncodedPointer(uint64_t *Offset, uint8_t Encoding, uint64_t AbsPosOffset = 0) const; - - size_t size() const { return Section == nullptr ? 0 : Section->Data.size(); } }; } // end namespace llvm diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h index 172f1d2c9dbe..89b15d263580 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h @@ -21,7 +21,7 @@ class DWARFContext; class DWARFDebugAranges { public: void generate(DWARFContext *CTX); - uint32_t findAddress(uint64_t Address) const; + uint64_t findAddress(uint64_t Address) const; private: void clear(); @@ -33,7 +33,7 @@ private: struct Range { explicit Range(uint64_t LowPC = -1ULL, uint64_t HighPC = -1ULL, - uint32_t CUOffset = -1U) + uint64_t CUOffset = -1ULL) : LowPC(LowPC), Length(HighPC - LowPC), CUOffset(CUOffset) {} void setHighPC(uint64_t HighPC) { @@ -54,8 +54,8 @@ private: } uint64_t LowPC; /// Start of address range. - uint32_t Length; /// End of address range (not including this address). - uint32_t CUOffset; /// Offset of the compile unit or die. + uint64_t Length; /// End of address range (not including this address). + uint64_t CUOffset; /// Offset of the compile unit or die. }; struct RangeEndpoint { diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h index c2be8304ad84..d5b6c72c0461 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h @@ -280,11 +280,10 @@ public: void clear(); /// Parse prologue and all rows. - Error parse( - DWARFDataExtractor &DebugLineData, uint64_t *OffsetPtr, - const DWARFContext &Ctx, const DWARFUnit *U, - std::function<void(Error)> RecoverableErrorCallback, - raw_ostream *OS = nullptr); + Error parse(DWARFDataExtractor &DebugLineData, uint64_t *OffsetPtr, + const DWARFContext &Ctx, const DWARFUnit *U, + function_ref<void(Error)> RecoverableErrorCallback, + raw_ostream *OS = nullptr); using RowVector = std::vector<Row>; using RowIter = RowVector::const_iterator; @@ -309,10 +308,10 @@ public: }; const LineTable *getLineTable(uint64_t Offset) const; - Expected<const LineTable *> getOrParseLineTable( - DWARFDataExtractor &DebugLineData, uint64_t Offset, - const DWARFContext &Ctx, const DWARFUnit *U, - std::function<void(Error)> RecoverableErrorCallback); + Expected<const LineTable *> + getOrParseLineTable(DWARFDataExtractor &DebugLineData, uint64_t Offset, + const DWARFContext &Ctx, const DWARFUnit *U, + function_ref<void(Error)> RecoverableErrorCallback); /// Helper to allow for parsing of an entire .debug_line section in sequence. class SectionParser { diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h index c79d98e34f6e..3b141304f85f 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h @@ -13,6 +13,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/DebugInfo/DIContext.h" #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" +#include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h" #include "llvm/DebugInfo/DWARF/DWARFRelocMap.h" #include <cstdint> @@ -21,30 +22,73 @@ class DWARFUnit; class MCRegisterInfo; class raw_ostream; -class DWARFDebugLoc { +/// A single location within a location list. Entries are stored in the DWARF5 +/// form even if they originally come from a DWARF<=4 location list. +struct DWARFLocationEntry { + /// The entry kind (DW_LLE_***). + uint8_t Kind; + + /// The first value of the location entry (if applicable). + uint64_t Value0; + + /// The second value of the location entry (if applicable). + uint64_t Value1; + + /// The index of the section this entry is relative to (if applicable). + uint64_t SectionIndex; + + /// The location expression itself (if applicable). + SmallVector<uint8_t, 4> Loc; +}; + +/// An abstract base class for various kinds of location tables (.debug_loc, +/// .debug_loclists, and their dwo variants). +class DWARFLocationTable { public: - /// A single location within a location list. - struct Entry { - /// The beginning address of the instruction range. - uint64_t Begin; - /// The ending address of the instruction range. - uint64_t End; - /// The location of the variable within the specified range. - SmallVector<uint8_t, 4> Loc; - }; + DWARFLocationTable(DWARFDataExtractor Data) : Data(std::move(Data)) {} + virtual ~DWARFLocationTable() = default; + + /// Call the user-provided callback for each entry (including the end-of-list + /// entry) in the location list starting at \p Offset. The callback can return + /// false to terminate the iteration early. Returns an error if it was unable + /// to parse the entire location list correctly. Upon successful termination + /// \p Offset will be updated point past the end of the list. + virtual Error visitLocationList( + uint64_t *Offset, + function_ref<bool(const DWARFLocationEntry &)> Callback) const = 0; + + /// Dump the location list at the given \p Offset. The function returns true + /// iff it has successfully reched the end of the list. This means that one + /// can attempt to parse another list after the current one (\p Offset will be + /// updated to point past the end of the current list). + bool dumpLocationList(uint64_t *Offset, raw_ostream &OS, + Optional<object::SectionedAddress> BaseAddr, + const MCRegisterInfo *MRI, const DWARFObject &Obj, + DWARFUnit *U, DIDumpOptions DumpOpts, + unsigned Indent) const; + + Error visitAbsoluteLocationList( + uint64_t Offset, Optional<object::SectionedAddress> BaseAddr, + std::function<Optional<object::SectionedAddress>(uint32_t)> LookupAddr, + function_ref<bool(Expected<DWARFLocationExpression>)> Callback) const; + +protected: + DWARFDataExtractor Data; + + virtual void dumpRawEntry(const DWARFLocationEntry &Entry, raw_ostream &OS, + unsigned Indent, DIDumpOptions DumpOpts, + const DWARFObject &Obj) const = 0; +}; +class DWARFDebugLoc final : public DWARFLocationTable { +public: /// A list of locations that contain one variable. struct LocationList { /// The beginning offset where this location list is stored in the debug_loc /// section. uint64_t Offset; /// All the locations in which the variable is stored. - SmallVector<Entry, 2> Entries; - /// Dump this list on OS. - void dump(raw_ostream &OS, uint64_t BaseAddress, bool IsLittleEndian, - unsigned AddressSize, const MCRegisterInfo *MRI, DWARFUnit *U, - DIDumpOptions DumpOpts, - unsigned Indent) const; + SmallVector<DWARFLocationEntry, 2> Entries; }; private: @@ -54,67 +98,46 @@ private: /// the locations in which the variable is stored. LocationLists Locations; - unsigned AddressSize; - - bool IsLittleEndian; - public: + DWARFDebugLoc(DWARFDataExtractor Data) + : DWARFLocationTable(std::move(Data)) {} + /// Print the location lists found within the debug_loc section. - void dump(raw_ostream &OS, const MCRegisterInfo *RegInfo, DIDumpOptions DumpOpts, + void dump(raw_ostream &OS, const MCRegisterInfo *RegInfo, + const DWARFObject &Obj, DIDumpOptions DumpOpts, Optional<uint64_t> Offset) const; - /// Parse the debug_loc section accessible via the 'data' parameter using the - /// address size also given in 'data' to interpret the address ranges. - void parse(const DWARFDataExtractor &data); - - /// Return the location list at the given offset or nullptr. - LocationList const *getLocationListAtOffset(uint64_t Offset) const; + Error visitLocationList( + uint64_t *Offset, + function_ref<bool(const DWARFLocationEntry &)> Callback) const override; - Expected<LocationList> - parseOneLocationList(const DWARFDataExtractor &Data, uint64_t *Offset); +protected: + void dumpRawEntry(const DWARFLocationEntry &Entry, raw_ostream &OS, + unsigned Indent, DIDumpOptions DumpOpts, + const DWARFObject &Obj) const override; }; -class DWARFDebugLoclists { +class DWARFDebugLoclists final : public DWARFLocationTable { public: - struct Entry { - uint8_t Kind; - uint64_t Offset; - uint64_t Value0; - uint64_t Value1; - SmallVector<uint8_t, 4> Loc; - void dump(raw_ostream &OS, uint64_t &BaseAddr, bool IsLittleEndian, - unsigned AddressSize, const MCRegisterInfo *MRI, DWARFUnit *U, - DIDumpOptions DumpOpts, unsigned Indent, size_t MaxEncodingStringLength) const; - }; + DWARFDebugLoclists(DWARFDataExtractor Data, uint16_t Version) + : DWARFLocationTable(std::move(Data)), Version(Version) {} - struct LocationList { - uint64_t Offset; - SmallVector<Entry, 2> Entries; - void dump(raw_ostream &OS, uint64_t BaseAddr, bool IsLittleEndian, - unsigned AddressSize, const MCRegisterInfo *RegInfo, - DWARFUnit *U, DIDumpOptions DumpOpts, unsigned Indent) const; - }; + Error visitLocationList( + uint64_t *Offset, + function_ref<bool(const DWARFLocationEntry &)> Callback) const override; -private: - using LocationLists = SmallVector<LocationList, 4>; + /// Dump all location lists within the given range. + void dumpRange(uint64_t StartOffset, uint64_t Size, raw_ostream &OS, + const MCRegisterInfo *MRI, const DWARFObject &Obj, + DIDumpOptions DumpOpts); - LocationLists Locations; +protected: + void dumpRawEntry(const DWARFLocationEntry &Entry, raw_ostream &OS, + unsigned Indent, DIDumpOptions DumpOpts, + const DWARFObject &Obj) const override; - unsigned AddressSize; - - bool IsLittleEndian; - -public: - void parse(DataExtractor data, uint64_t Offset, uint64_t EndOffset, uint16_t Version); - void dump(raw_ostream &OS, uint64_t BaseAddr, const MCRegisterInfo *RegInfo, - DIDumpOptions DumpOpts, Optional<uint64_t> Offset) const; - - /// Return the location list at the given offset or nullptr. - LocationList const *getLocationListAtOffset(uint64_t Offset) const; - - static Expected<LocationList> parseOneLocationList(const DataExtractor &Data, - uint64_t *Offset, - unsigned Version); +private: + uint16_t Version; }; } // end namespace llvm diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h index a6c125990ca7..7880bcdf6881 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h @@ -42,7 +42,7 @@ class DWARFDebugMacro { using MacroList = SmallVector<Entry, 4>; /// A list of all the macro entries in the debug_macinfo section. - MacroList Macros; + std::vector<MacroList> MacroLists; public: DWARFDebugMacro() = default; @@ -54,7 +54,7 @@ public: void parse(DataExtractor data); /// Return whether the section has any entries. - bool empty() const { return Macros.empty(); } + bool empty() const { return MacroLists.empty(); } }; } // end namespace llvm diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h index 952c41e188c7..88e5432851d6 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h @@ -47,6 +47,12 @@ class DWARFDebugRnglist : public DWARFListType<RangeListEntry> { public: /// Build a DWARFAddressRangesVector from a rangelist. DWARFAddressRangesVector + getAbsoluteRanges(Optional<object::SectionedAddress> BaseAddr, + function_ref<Optional<object::SectionedAddress>(uint32_t)> + LookupPooledAddress) const; + + /// Build a DWARFAddressRangesVector from a rangelist. + DWARFAddressRangesVector getAbsoluteRanges(llvm::Optional<object::SectionedAddress> BaseAddr, DWARFUnit &U) const; }; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h index f7f08b4a499d..158bd82edee0 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDie.h @@ -18,6 +18,7 @@ #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" #include "llvm/DebugInfo/DWARF/DWARFAttribute.h" #include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" #include <cassert> #include <cstdint> #include <iterator> @@ -188,6 +189,7 @@ public: /// /// \returns anm optional absolute section offset value for the attribute. Optional<uint64_t> getRangesBaseAttribute() const; + Optional<uint64_t> getLocBaseAttribute() const; /// Get the DW_AT_high_pc attribute value as an address. /// @@ -230,6 +232,9 @@ public: bool addressRangeContainsAddress(const uint64_t Address) const; + Expected<DWARFLocationExpressionsVector> + getLocations(dwarf::Attribute Attr) const; + /// If a DIE represents a subprogram (or inlined subroutine), returns its /// mangled name (or short name, if mangled is missing). This name may be /// fetched from specification or abstract origin for this subprogram. diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h new file mode 100644 index 000000000000..35aa1a78e129 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFLocationExpression.h @@ -0,0 +1,49 @@ +//===- DWARFLocationExpression.h --------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_DWARF_DWARFLOCATIONEXPRESSION_H +#define LLVM_DEBUGINFO_DWARF_DWARFLOCATIONEXPRESSION_H + +#include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/DWARF/DWARFAddressRange.h" + +namespace llvm { + +class raw_ostream; + +/// Represents a single DWARF expression, whose value is location-dependent. +/// Typically used in DW_AT_location attributes to describe the location of +/// objects. +struct DWARFLocationExpression { + /// The address range in which this expression is valid. None denotes a + /// default entry which is valid in addresses not covered by other location + /// expressions, or everywhere if there are no other expressions. + Optional<DWARFAddressRange> Range; + + /// The expression itself. + SmallVector<uint8_t, 4> Expr; +}; + +inline bool operator==(const DWARFLocationExpression &L, + const DWARFLocationExpression &R) { + return L.Range == R.Range && L.Expr == R.Expr; +} + +inline bool operator!=(const DWARFLocationExpression &L, + const DWARFLocationExpression &R) { + return !(L == R); +} + +raw_ostream &operator<<(raw_ostream &OS, const DWARFLocationExpression &Loc); + +/// Represents a set of absolute location expressions. +using DWARFLocationExpressionsVector = std::vector<DWARFLocationExpression>; + +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_DWARF_DWARFLOCATIONEXPRESSION_H diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h index 88fe3f434edc..fbcde7d7cd78 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFObject.h @@ -48,6 +48,7 @@ public: virtual const DWARFSection &getRangesSection() const { return Dummy; } virtual const DWARFSection &getRnglistsSection() const { return Dummy; } virtual StringRef getMacinfoSection() const { return ""; } + virtual StringRef getMacinfoDWOSection() const { return ""; } virtual const DWARFSection &getPubnamesSection() const { return Dummy; } virtual const DWARFSection &getPubtypesSection() const { return Dummy; } virtual const DWARFSection &getGnuPubnamesSection() const { return Dummy; } @@ -60,6 +61,7 @@ public: virtual StringRef getAbbrevDWOSection() const { return ""; } virtual const DWARFSection &getLineDWOSection() const { return Dummy; } virtual const DWARFSection &getLocDWOSection() const { return Dummy; } + virtual const DWARFSection &getLoclistsDWOSection() const { return Dummy; } virtual StringRef getStrDWOSection() const { return ""; } virtual const DWARFSection &getStrOffsetsDWOSection() const { return Dummy; diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h index 51de114a3506..b2ddb7e36b0c 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h @@ -16,6 +16,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h" #include "llvm/DebugInfo/DWARF/DWARFDebugRnglists.h" #include "llvm/DebugInfo/DWARF/DWARFDie.h" @@ -199,17 +200,17 @@ class DWARFUnit { const DWARFDebugAbbrev *Abbrev; const DWARFSection *RangeSection; uint64_t RangeSectionBase; - /// We either keep track of the location list section or its data, depending - /// on whether we are handling a split DWARF section or not. - union { - const DWARFSection *LocSection; - StringRef LocSectionData; - }; + const DWARFSection *LocSection; + uint64_t LocSectionBase; + + /// Location table of this unit. + std::unique_ptr<DWARFLocationTable> LocTable; + const DWARFSection &LineSection; StringRef StringSection; const DWARFSection &StringOffsetSection; const DWARFSection *AddrOffsetSection; - uint32_t AddrOffsetSectionBase = 0; + Optional<uint64_t> AddrOffsetSectionBase; bool isLittleEndian; bool IsDWO; const DWARFUnitVector &UnitVector; @@ -220,6 +221,7 @@ class DWARFUnit { /// A table of range lists (DWARF v5 and later). Optional<DWARFDebugRnglistTable> RngListTable; + Optional<DWARFListTableHeader> LoclistTableHeader; mutable const DWARFAbbreviationDeclarationSet *Abbrevs; llvm::Optional<object::SectionedAddress> BaseAddr; @@ -274,8 +276,6 @@ public: bool isDWOUnit() const { return IsDWO; } DWARFContext& getContext() const { return Context; } const DWARFSection &getInfoSection() const { return InfoSection; } - const DWARFSection *getLocSection() const { return LocSection; } - StringRef getLocSectionData() const { return LocSectionData; } uint64_t getOffset() const { return Header.getOffset(); } const dwarf::FormParams &getFormParams() const { return Header.getFormParams(); @@ -308,6 +308,14 @@ public: RangeSection = RS; RangeSectionBase = Base; } + void setLocSection(const DWARFSection *LS, uint64_t Base) { + LocSection = LS; + LocSectionBase = Base; + } + + uint64_t getLocSectionBase() const { + return LocSectionBase; + } Optional<object::SectionedAddress> getAddrOffsetSectionItem(uint32_t Index) const; @@ -319,6 +327,8 @@ public: return DataExtractor(StringSection, false, 0); } + const DWARFLocationTable &getLocationTable() { return *LocTable; } + /// Extract the range list referenced by this compile unit from the /// .debug_ranges section. If the extraction is unsuccessful, an error /// is returned. Successful extraction requires that the compile unit @@ -417,13 +427,25 @@ public: /// an entry in the rangelist table's offset array and is supplied by /// DW_FORM_rnglistx. Optional<uint64_t> getRnglistOffset(uint32_t Index) { - if (RngListTable) - return RngListTable->getOffsetEntry(Index); + if (!RngListTable) + return None; + if (Optional<uint64_t> Off = RngListTable->getOffsetEntry(Index)) + return *Off + RangeSectionBase; return None; } + Optional<uint64_t> getLoclistOffset(uint32_t Index) { + if (!LoclistTableHeader) + return None; + if (Optional<uint64_t> Off = LoclistTableHeader->getOffsetEntry(Index)) + return *Off + getLocSectionBase(); + return None; + } Expected<DWARFAddressRangesVector> collectAddressRanges(); + Expected<DWARFLocationExpressionsVector> + findLoclistFromOffset(uint64_t Offset); + /// Returns subprogram DIE with address range encompassing the provided /// address. The pointer is alive as long as parsed compile unit DIEs are not /// cleared. diff --git a/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h b/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h index 63e18bb2ecd5..893cfc1eb07c 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h +++ b/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h @@ -12,6 +12,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/DebugInfo/GSYM/InlineInfo.h" #include "llvm/DebugInfo/GSYM/LineTable.h" +#include "llvm/DebugInfo/GSYM/LookupResult.h" #include "llvm/DebugInfo/GSYM/Range.h" #include "llvm/DebugInfo/GSYM/StringTable.h" #include <tuple> @@ -21,6 +22,7 @@ namespace llvm { class raw_ostream; namespace gsym { +class GsymReader; /// Function information in GSYM files encodes information for one contiguous /// address range. If a function has discontiguous address ranges, they will /// need to be encoded using multiple FunctionInfo objects. @@ -30,7 +32,7 @@ namespace gsym { /// The function information gets the function start address as an argument /// to the FunctionInfo::decode(...) function. This information is calculated /// from the GSYM header and an address offset from the GSYM address offsets -/// table. The encoded FunctionInfo information must be alinged to a 4 byte +/// table. The encoded FunctionInfo information must be aligned to a 4 byte /// boundary. /// /// The encoded data for a FunctionInfo starts with fixed data that all @@ -140,6 +142,33 @@ struct FunctionInfo { /// function info that was successfully written into the stream. llvm::Expected<uint64_t> encode(FileWriter &O) const; + + /// Lookup an address within a FunctionInfo object's data stream. + /// + /// Instead of decoding an entire FunctionInfo object when doing lookups, + /// we can decode only the information we need from the FunctionInfo's data + /// for the specific address. The lookup result information is returned as + /// a LookupResult. + /// + /// \param Data The binary stream to read the data from. This object must + /// have the data for the object starting at offset zero. The data + /// can contain more data than needed. + /// + /// \param GR The GSYM reader that contains the string and file table that + /// will be used to fill in information in the returned result. + /// + /// \param FuncAddr The function start address decoded from the GsymReader. + /// + /// \param Addr The address to lookup. + /// + /// \returns An LookupResult or an error describing the issue that was + /// encountered during decoding. An error should only be returned if the + /// address is not contained in the FunctionInfo or if the data is corrupted. + static llvm::Expected<LookupResult> lookup(DataExtractor &Data, + const GsymReader &GR, + uint64_t FuncAddr, + uint64_t Addr); + uint64_t startAddress() const { return Range.Start; } uint64_t endAddress() const { return Range.End; } uint64_t size() const { return Range.size(); } diff --git a/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h b/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h index 12c8187132ba..e61eb678c82e 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h +++ b/llvm/include/llvm/DebugInfo/GSYM/GsymCreator.h @@ -82,15 +82,15 @@ class FileWriter; /// The resulting GSYM size is smaller and causes fewer pages to be touched /// during address lookups when the address table is smaller. The size of the /// address offsets in the address table is specified in the header in -/// Header.AddrOffSize. The first offset in the address table is alinged to -/// Header.AddrOffSize alignement to ensure efficient access when loaded into +/// Header.AddrOffSize. The first offset in the address table is aligned to +/// Header.AddrOffSize alignment to ensure efficient access when loaded into /// memory. /// /// FUNCTION INFO OFFSETS TABLE /// /// The function info offsets table immediately follows the address table and /// consists of Header.NumAddresses 32 bit file offsets: one for each address -/// in the address table. This data is algined to a 4 byte boundary. The +/// in the address table. This data is aligned to a 4 byte boundary. The /// offsets in this table are the relative offsets from the start offset of the /// GSYM header and point to the function info data for each address in the /// address table. Keeping this data separate from the address table helps to diff --git a/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h b/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h index 113bcee9c9a3..5ba13f846798 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h +++ b/llvm/include/llvm/DebugInfo/GSYM/GsymReader.h @@ -1,9 +1,8 @@ //===- GsymReader.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -94,28 +93,45 @@ public: /// Get the full function info for an address. /// + /// This should be called when a client will store a copy of the complete + /// FunctionInfo for a given address. For one off lookups, use the lookup() + /// function below. + /// + /// Symbolication server processes might want to parse the entire function + /// info for a given address and cache it if the process stays around to + /// service many symbolication addresses, like for parsing profiling + /// information. + /// /// \param Addr A virtual address from the orignal object file to lookup. + /// /// \returns An expected FunctionInfo that contains the function info object /// or an error object that indicates reason for failing to lookup the - /// address, + /// address. llvm::Expected<FunctionInfo> getFunctionInfo(uint64_t Addr) const; + /// Lookup an address in the a GSYM. + /// + /// Lookup just the information needed for a specific address \a Addr. This + /// function is faster that calling getFunctionInfo() as it will only return + /// information that pertains to \a Addr and allows the parsing to skip any + /// extra information encoded for other addresses. For example the line table + /// parsing can stop when a matching LineEntry has been fouhnd, and the + /// InlineInfo can stop parsing early once a match has been found and also + /// skip information that doesn't match. This avoids memory allocations and + /// is much faster for lookups. + /// + /// \param Addr A virtual address from the orignal object file to lookup. + /// \returns An expected LookupResult that contains only the information + /// needed for the current address, or an error object that indicates reason + /// for failing to lookup the address. + llvm::Expected<LookupResult> lookup(uint64_t Addr) const; + /// Get a string from the string table. /// /// \param Offset The string table offset for the string to retrieve. /// \returns The string from the strin table. StringRef getString(uint32_t Offset) const { return StrTab[Offset]; } -protected: - /// Gets an address from the address table. - /// - /// Addresses are stored as offsets frrom the gsym::Header::BaseAddress. - /// - /// \param Index A index into the address table. - /// \returns A resolved virtual address for adddress in the address table - /// or llvm::None if Index is out of bounds. - Optional<uint64_t> getAddress(size_t Index) const; - /// Get the a file entry for the suppplied file index. /// /// Used to convert any file indexes in the FunctionInfo data back into @@ -131,6 +147,16 @@ protected: return llvm::None; } +protected: + /// Gets an address from the address table. + /// + /// Addresses are stored as offsets frrom the gsym::Header::BaseAddress. + /// + /// \param Index A index into the address table. + /// \returns A resolved virtual address for adddress in the address table + /// or llvm::None if Index is out of bounds. + Optional<uint64_t> getAddress(size_t Index) const; + /// Get an appropriate address info offsets array. /// /// The address table in the GSYM file is stored as array of 1, 2, 4 or 8 diff --git a/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h b/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h index 48fd9a7c1308..3b95e3e050bd 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h +++ b/llvm/include/llvm/DebugInfo/GSYM/InlineInfo.h @@ -10,6 +10,8 @@ #define LLVM_DEBUGINFO_GSYM_INLINEINFO_H #include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/GSYM/LineEntry.h" +#include "llvm/DebugInfo/GSYM/LookupResult.h" #include "llvm/DebugInfo/GSYM/Range.h" #include "llvm/Support/Error.h" #include <stdint.h> @@ -21,6 +23,7 @@ class raw_ostream; namespace gsym { +class GsymReader; /// Inline information stores the name of the inline function along with /// an array of address ranges. It also stores the call file and call line /// that called this inline function. This allows us to unwind inline call @@ -74,6 +77,52 @@ struct InlineInfo { using InlineArray = std::vector<const InlineInfo *>; + /// Lookup a single address within the inline info data. + /// + /// Clients have the option to decode an entire InlineInfo object (using + /// InlineInfo::decode() ) or just find the matching inline info using this + /// function. The benefit of using this function is that only the information + /// needed for the lookup will be extracted, other info can be skipped and + /// parsing can stop as soon as the deepest match is found. This allows + /// symbolication tools to be fast and efficient and avoid allocation costs + /// when doing lookups. + /// + /// This function will augment the SourceLocations array \a SrcLocs with any + /// inline information that pertains to \a Addr. If no inline information + /// exists for \a Addr, then \a SrcLocs will be left untouched. If there is + /// inline information for \a Addr, then \a SrcLocs will be modifiied to + /// contain the deepest most inline function's SourceLocation at index zero + /// in the array and proceed up the the concrete function source file and + /// line at the end of the array. + /// + /// \param GR The GSYM reader that contains the string and file table that + /// will be used to fill in the source locations. + /// + /// \param Data The binary stream to read the data from. This object must + /// have the data for the LineTable object starting at offset zero. The data + /// can contain more data than needed. + /// + /// \param BaseAddr The base address to use when decoding the line table. + /// This will be the FunctionInfo's start address and will be used to + /// decode the correct addresses for the inline information. + /// + /// \param Addr The address to lookup. + /// + /// \param SrcLocs The inline source locations that matches \a Addr. This + /// array must be initialized with the matching line entry + /// from the line table upon entry. The name of the concrete + /// function must be supplied since it will get pushed to + /// the last SourceLocation entry and the inline information + /// will fill in the source file and line from the inline + /// information. + /// + /// \returns An error if the inline information is corrupt, or + /// Error::success() for all other cases, even when no information + /// is added to \a SrcLocs. + static llvm::Error lookup(const GsymReader &GR, DataExtractor &Data, + uint64_t BaseAddr, uint64_t Addr, + SourceLocations &SrcLocs); + /// Lookup an address in the InlineInfo object /// /// This function is used to symbolicate an inline call stack and can diff --git a/llvm/include/llvm/DebugInfo/GSYM/LineTable.h b/llvm/include/llvm/DebugInfo/GSYM/LineTable.h index 3cdbccb08ced..22668e39d94c 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/LineTable.h +++ b/llvm/include/llvm/DebugInfo/GSYM/LineTable.h @@ -119,8 +119,25 @@ class LineTable { typedef std::vector<gsym::LineEntry> Collection; Collection Lines; ///< All line entries in the line table. public: - static LineEntry lookup(DataExtractor &Data, uint64_t BaseAddr, - uint64_t Addr); + /// Lookup a single address within a line table's data. + /// + /// Clients have the option to decode an entire line table using + /// LineTable::decode() or just find a single matching entry using this + /// function. The benefit of using this function is that parsed LineEntry + /// objects that do not match will not be stored in an array. This will avoid + /// memory allocation costs and parsing can stop once a match has been found. + /// + /// \param Data The binary stream to read the data from. This object must + /// have the data for the LineTable object starting at offset zero. The data + /// can contain more data than needed. + /// + /// \param BaseAddr The base address to use when decoding the line table. + /// This will be the FunctionInfo's start address and will be used to + /// initialize the line table row prior to parsing any opcodes. + /// + /// \returns An LineEntry object if a match is found, error otherwise. + static Expected<LineEntry> lookup(DataExtractor &Data, uint64_t BaseAddr, + uint64_t Addr); /// Decode an LineTable object from a binary data stream. /// diff --git a/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h b/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h new file mode 100644 index 000000000000..746fd36208e1 --- /dev/null +++ b/llvm/include/llvm/DebugInfo/GSYM/LookupResult.h @@ -0,0 +1,61 @@ +//===- LookupResult.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_LOOKUPRESULT_H +#define LLVM_DEBUGINFO_GSYM_LOOKUPRESULT_H + +#include "llvm/DebugInfo/GSYM/Range.h" +#include "llvm/ADT/StringRef.h" +#include <inttypes.h> +#include <vector> + +namespace llvm { +class raw_ostream; +namespace gsym { +struct FileEntry; + +struct SourceLocation { + StringRef Name; ///< Function or symbol name. + StringRef Dir; ///< Line entry source file directory path. + StringRef Base; ///< Line entry source file basename. + uint32_t Line = 0; ///< Source file line number. +}; + +inline bool operator==(const SourceLocation &LHS, const SourceLocation &RHS) { + return LHS.Name == RHS.Name && LHS.Dir == RHS.Dir && + LHS.Base == RHS.Base && LHS.Line == RHS.Line; +} + +raw_ostream &operator<<(raw_ostream &OS, const SourceLocation &R); + +using SourceLocations = std::vector<SourceLocation>; + + +struct LookupResult { + uint64_t LookupAddr = 0; ///< The address that this lookup pertains to. + AddressRange FuncRange; ///< The concrete function address range. + StringRef FuncName; ///< The concrete function name that contains LookupAddr. + /// The source locations that match this address. This information will only + /// be filled in if the FunctionInfo contains a line table. If an address is + /// for a concrete function with no inlined functions, this array will have + /// one entry. If an address points to an inline function, there will be one + /// SourceLocation for each inlined function with the last entry pointing to + /// the concrete function itself. This allows one address to generate + /// multiple locations and allows unwinding of inline call stacks. The + /// deepest inline function will appear at index zero in the source locations + /// array, and the concrete function will appear at the end of the array. + SourceLocations Locations; + std::string getSourceFile(uint32_t Index) const; +}; + +raw_ostream &operator<<(raw_ostream &OS, const LookupResult &R); + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_LOOKUPRESULT_H diff --git a/llvm/include/llvm/DebugInfo/GSYM/Range.h b/llvm/include/llvm/DebugInfo/GSYM/Range.h index 37cfec713f26..49e316eae3cf 100644 --- a/llvm/include/llvm/DebugInfo/GSYM/Range.h +++ b/llvm/include/llvm/DebugInfo/GSYM/Range.h @@ -61,6 +61,14 @@ struct AddressRange { void decode(DataExtractor &Data, uint64_t BaseAddr, uint64_t &Offset); void encode(FileWriter &O, uint64_t BaseAddr) const; /// @} + + /// Skip an address range object in the specified data a the specified + /// offset. + /// + /// \param Data The binary stream to read the data from. + /// + /// \param Offset The byte offset within \a Data. + static void skip(DataExtractor &Data, uint64_t &Offset); }; raw_ostream &operator<<(raw_ostream &OS, const AddressRange &R); @@ -100,6 +108,16 @@ public: void decode(DataExtractor &Data, uint64_t BaseAddr, uint64_t &Offset); void encode(FileWriter &O, uint64_t BaseAddr) const; /// @} + + /// Skip an address range object in the specified data a the specified + /// offset. + /// + /// \param Data The binary stream to read the data from. + /// + /// \param Offset The byte offset within \a Data. + /// + /// \returns The number of address ranges that were skipped. + static uint64_t skip(DataExtractor &Data, uint64_t &Offset); }; raw_ostream &operator<<(raw_ostream &OS, const AddressRanges &AR); diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h index 568f0c98c559..7b7337f532d8 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h @@ -25,9 +25,9 @@ class DbiModuleDescriptor { friend class DbiStreamBuilder; public: - DbiModuleDescriptor(); - DbiModuleDescriptor(const DbiModuleDescriptor &Info); - ~DbiModuleDescriptor(); + DbiModuleDescriptor() = default; + DbiModuleDescriptor(const DbiModuleDescriptor &Info) = default; + DbiModuleDescriptor &operator=(const DbiModuleDescriptor &Info) = default; static Error initialize(BinaryStreamRef Stream, DbiModuleDescriptor &Info); diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h index 14223273c898..5fb13ad30e91 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h @@ -39,6 +39,7 @@ public: DbiModuleSourceFilesIterator(const DbiModuleList &Modules, uint32_t Modi, uint16_t Filei); DbiModuleSourceFilesIterator() = default; + DbiModuleSourceFilesIterator(const DbiModuleSourceFilesIterator &R) = default; DbiModuleSourceFilesIterator & operator=(const DbiModuleSourceFilesIterator &R) = default; diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h b/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h index aa38417bcf4c..95c0a89551ed 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/HashTable.h @@ -56,6 +56,7 @@ public: } } + HashTableIterator(const HashTableIterator &R) = default; HashTableIterator &operator=(const HashTableIterator &R) { Map = R.Map; return *this; diff --git a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h index 11599fc1797d..8bfa5432b811 100644 --- a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h +++ b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h @@ -16,6 +16,7 @@ #include "llvm/DebugInfo/Symbolize/SymbolizableModule.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" +#include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Error.h" #include <algorithm> #include <cstdint> @@ -44,6 +45,7 @@ public: std::vector<std::string> DsymHints; std::string FallbackDebugPath; std::string DWPName; + std::vector<std::string> DebugFileDirectory; }; LLVMSymbolizer() = default; @@ -98,6 +100,9 @@ private: ObjectFile *lookUpDebuglinkObject(const std::string &Path, const ObjectFile *Obj, const std::string &ArchName); + ObjectFile *lookUpBuildIDObject(const std::string &Path, + const ELFObjectFileBase *Obj, + const std::string &ArchName); /// Returns pair of pointers to object and debug object. Expected<ObjectPair> getOrCreateObjectPair(const std::string &Path, diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h index 7784e842bfeb..376e0efea423 100644 --- a/llvm/include/llvm/Demangle/ItaniumDemangle.h +++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h @@ -607,48 +607,12 @@ public: } }; -class NodeOrString { - const void *First; - const void *Second; - -public: - /* implicit */ NodeOrString(StringView Str) { - const char *FirstChar = Str.begin(); - const char *SecondChar = Str.end(); - if (SecondChar == nullptr) { - assert(FirstChar == SecondChar); - ++FirstChar, ++SecondChar; - } - First = static_cast<const void *>(FirstChar); - Second = static_cast<const void *>(SecondChar); - } - - /* implicit */ NodeOrString(Node *N) - : First(static_cast<const void *>(N)), Second(nullptr) {} - NodeOrString() : First(nullptr), Second(nullptr) {} - - bool isString() const { return Second && First; } - bool isNode() const { return First && !Second; } - bool isEmpty() const { return !First && !Second; } - - StringView asString() const { - assert(isString()); - return StringView(static_cast<const char *>(First), - static_cast<const char *>(Second)); - } - - const Node *asNode() const { - assert(isNode()); - return static_cast<const Node *>(First); - } -}; - class ArrayType final : public Node { const Node *Base; - NodeOrString Dimension; + Node *Dimension; public: - ArrayType(const Node *Base_, NodeOrString Dimension_) + ArrayType(const Node *Base_, Node *Dimension_) : Node(KArrayType, /*RHSComponentCache=*/Cache::Yes, /*ArrayCache=*/Cache::Yes), @@ -665,10 +629,8 @@ public: if (S.back() != ']') S += " "; S += "["; - if (Dimension.isString()) - S += Dimension.asString(); - else if (Dimension.isNode()) - Dimension.asNode()->print(S); + if (Dimension) + Dimension->print(S); S += "]"; Base->printRight(S); } @@ -934,10 +896,10 @@ public: class VectorType final : public Node { const Node *BaseType; - const NodeOrString Dimension; + const Node *Dimension; public: - VectorType(const Node *BaseType_, NodeOrString Dimension_) + VectorType(const Node *BaseType_, Node *Dimension_) : Node(KVectorType), BaseType(BaseType_), Dimension(Dimension_) {} @@ -946,19 +908,17 @@ public: void printLeft(OutputStream &S) const override { BaseType->print(S); S += " vector["; - if (Dimension.isNode()) - Dimension.asNode()->print(S); - else if (Dimension.isString()) - S += Dimension.asString(); + if (Dimension) + Dimension->print(S); S += "]"; } }; class PixelVectorType final : public Node { - const NodeOrString Dimension; + const Node *Dimension; public: - PixelVectorType(NodeOrString Dimension_) + PixelVectorType(const Node *Dimension_) : Node(KPixelVectorType), Dimension(Dimension_) {} template<typename Fn> void match(Fn F) const { F(Dimension); } @@ -966,7 +926,7 @@ public: void printLeft(OutputStream &S) const override { // FIXME: This should demangle as "vector pixel". S += "pixel vector["; - S += Dimension.asString(); + Dimension->print(S); S += "]"; } }; @@ -2213,10 +2173,10 @@ class PODSmallVector { static_assert(std::is_pod<T>::value, "T is required to be a plain old data type"); - T* First; - T* Last; - T* Cap; - T Inline[N]; + T* First = nullptr; + T* Last = nullptr; + T* Cap = nullptr; + T Inline[N] = {0}; bool isInline() const { return First == Inline; } @@ -3548,7 +3508,9 @@ Node *AbstractManglingParser<Derived, Alloc>::parseVectorType() { if (!consumeIf("Dv")) return nullptr; if (look() >= '1' && look() <= '9') { - StringView DimensionNumber = parseNumber(); + Node *DimensionNumber = make<NameType>(parseNumber()); + if (!DimensionNumber) + return nullptr; if (!consumeIf('_')) return nullptr; if (consumeIf('p')) @@ -3573,7 +3535,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseVectorType() { Node *ElemType = getDerived().parseType(); if (!ElemType) return nullptr; - return make<VectorType>(ElemType, StringView()); + return make<VectorType>(ElemType, /*Dimension=*/nullptr); } // <decltype> ::= Dt <expression> E # decltype of an id-expression or class member access (C++0x) @@ -3599,10 +3561,12 @@ Node *AbstractManglingParser<Derived, Alloc>::parseArrayType() { if (!consumeIf('A')) return nullptr; - NodeOrString Dimension; + Node *Dimension = nullptr; if (std::isdigit(look())) { - Dimension = parseNumber(); + Dimension = make<NameType>(parseNumber()); + if (!Dimension) + return nullptr; if (!consumeIf('_')) return nullptr; } else if (!consumeIf('_')) { diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h index 81b279fe237d..ec40eec5a05e 100644 --- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h +++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h @@ -508,7 +508,7 @@ struct CustomTypeNode : public TypeNode { void outputPre(OutputStream &OS, OutputFlags Flags) const override; void outputPost(OutputStream &OS, OutputFlags Flags) const override; - IdentifierNode *Identifier; + IdentifierNode *Identifier = nullptr; }; struct NodeArrayNode : public Node { @@ -584,7 +584,7 @@ struct SpecialTableSymbolNode : public SymbolNode { void output(OutputStream &OS, OutputFlags Flags) const override; QualifiedNameNode *TargetName = nullptr; - Qualifiers Quals; + Qualifiers Quals = Qualifiers::Q_None; }; struct LocalStaticGuardVariableNode : public SymbolNode { diff --git a/llvm/include/llvm/Demangle/Utility.h b/llvm/include/llvm/Demangle/Utility.h index ec23859af46a..04e1936ebbe7 100644 --- a/llvm/include/llvm/Demangle/Utility.h +++ b/llvm/include/llvm/Demangle/Utility.h @@ -25,9 +25,9 @@ DEMANGLE_NAMESPACE_BEGIN // Stream that AST nodes write their string representation into after the AST // has been parsed. class OutputStream { - char *Buffer; - size_t CurrentPosition; - size_t BufferCapacity; + char *Buffer = nullptr; + size_t CurrentPosition = 0; + size_t BufferCapacity = 0; // Ensure there is at least n more positions in buffer. void grow(size_t N) { @@ -137,7 +137,7 @@ public: char *getBuffer() { return Buffer; } char *getBufferEnd() { return Buffer + CurrentPosition - 1; } - size_t getBufferCapacity() { return BufferCapacity; } + size_t getBufferCapacity() const { return BufferCapacity; } }; template <class T> class SwapAndRestore { diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h index b531127cf892..fa04653fa7bd 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -72,6 +72,7 @@ public: : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} OffsetT getOffset() const { return Offset; } + void setOffset(OffsetT Offset) { this->Offset = Offset; } Kind getKind() const { return K; } void setKind(Kind K) { this->K = K; } bool isRelocation() const { return K >= FirstRelocation; } @@ -130,7 +131,6 @@ private: uint64_t IsAbsolute : 1; }; -using BlockOrdinal = unsigned; using SectionOrdinal = unsigned; /// An Addressable with content and edges. @@ -139,10 +139,9 @@ class Block : public Addressable { private: /// Create a zero-fill defined addressable. - Block(Section &Parent, BlockOrdinal Ordinal, JITTargetAddress Size, - JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) - : Addressable(Address, true), Parent(Parent), Size(Size), - Ordinal(Ordinal) { + Block(Section &Parent, JITTargetAddress Size, JITTargetAddress Address, + uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(Parent), Size(Size) { assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); assert(AlignmentOffset < Alignment && "Alignment offset cannot exceed alignment"); @@ -153,10 +152,10 @@ private: } /// Create a defined addressable for the given content. - Block(Section &Parent, BlockOrdinal Ordinal, StringRef Content, - JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + Block(Section &Parent, StringRef Content, JITTargetAddress Address, + uint64_t Alignment, uint64_t AlignmentOffset) : Addressable(Address, true), Parent(Parent), Data(Content.data()), - Size(Content.size()), Ordinal(Ordinal) { + Size(Content.size()) { assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); assert(AlignmentOffset < Alignment && "Alignment offset cannot exceed alignment"); @@ -179,9 +178,6 @@ public: /// Return the parent section for this block. Section &getSection() const { return Parent; } - /// Return the ordinal for this block. - BlockOrdinal getOrdinal() const { return Ordinal; } - /// Returns true if this is a zero-fill block. /// /// If true, getSize is callable but getContent is not (the content is @@ -208,15 +204,32 @@ public: /// Get the alignment for this content. uint64_t getAlignment() const { return 1ull << P2Align; } + /// Set the alignment for this content. + void setAlignment(uint64_t Alignment) { + assert(isPowerOf2_64(Alignment) && "Alignment must be a power of two"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + } + /// Get the alignment offset for this content. uint64_t getAlignmentOffset() const { return AlignmentOffset; } + /// Set the alignment offset for this content. + void setAlignmentOffset(uint64_t AlignmentOffset) { + assert(AlignmentOffset < (1ull << P2Align) && + "Alignment offset can't exceed alignment"); + this->AlignmentOffset = AlignmentOffset; + } + /// Add an edge to this block. void addEdge(Edge::Kind K, Edge::OffsetT Offset, Symbol &Target, Edge::AddendT Addend) { Edges.push_back(Edge(K, Offset, Target, Addend)); } + /// Add an edge by copying an existing one. This is typically used when + /// moving edges between blocks. + void addEdge(const Edge &E) { Edges.push_back(E); } + /// Return the list of edges attached to this content. iterator_range<edge_iterator> edges() { return make_range(Edges.begin(), Edges.end()); @@ -233,6 +246,10 @@ public: /// Returns true if the list of edges is empty. bool edges_empty() const { return Edges.empty(); } + /// Remove the edge pointed to by the given iterator. + /// Invalidates all iterators that point to or past the given one. + void removeEdge(const_edge_iterator I) { Edges.erase(I); } + private: static constexpr uint64_t MaxAlignmentOffset = (1ULL << 57) - 1; @@ -241,7 +258,6 @@ private: Section &Parent; const char *Data = nullptr; size_t Size = 0; - BlockOrdinal Ordinal = 0; std::vector<Edge> Edges; }; @@ -287,6 +303,7 @@ private: JITTargetAddress Size, Linkage L, Scope S, bool IsLive, bool IsCallable) : Name(Name), Base(&Base), Offset(Offset), Size(Size) { + assert(Offset <= MaxOffset && "Offset out of range"); setLinkage(L); setScope(S); setLive(IsLive); @@ -307,14 +324,14 @@ private: } static Symbol &constructExternal(void *SymStorage, Addressable &Base, - StringRef Name, JITTargetAddress Size) { + StringRef Name, JITTargetAddress Size, + Linkage L) { assert(SymStorage && "Storage cannot be null"); assert(!Base.isDefined() && "Cannot create external symbol from defined block"); assert(!Name.empty() && "External symbol name cannot be empty"); auto *Sym = reinterpret_cast<Symbol *>(SymStorage); - new (Sym) Symbol(Base, 0, Name, Size, Linkage::Strong, Scope::Default, - false, false); + new (Sym) Symbol(Base, 0, Name, Size, L, Scope::Default, false, false); return *Sym; } @@ -334,6 +351,7 @@ private: JITTargetAddress Size, bool IsCallable, bool IsLive) { assert(SymStorage && "Storage cannot be null"); + assert(Offset < Base.getSize() && "Symbol offset is outside block"); auto *Sym = reinterpret_cast<Symbol *>(SymStorage); new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong, Scope::Local, IsLive, IsCallable); @@ -345,6 +363,7 @@ private: JITTargetAddress Size, Linkage L, Scope S, bool IsLive, bool IsCallable) { assert(SymStorage && "Storage cannot be null"); + assert(Offset < Base.getSize() && "Symbol offset is outside block"); assert(!Name.empty() && "Name cannot be empty"); auto *Sym = reinterpret_cast<Symbol *>(SymStorage); new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable); @@ -458,7 +477,7 @@ public: /// Set the linkage for this Symbol. void setLinkage(Linkage L) { - assert((L == Linkage::Strong || (Base->isDefined() && !Name.empty())) && + assert((L == Linkage::Strong || (!Base->isAbsolute() && !Name.empty())) && "Linkage can only be applied to defined named symbols"); this->L = static_cast<uint8_t>(L); } @@ -484,6 +503,13 @@ private: // note: Size and IsCallable fields left unchanged. } + void setBlock(Block &B) { Base = &B; } + + void setOffset(uint64_t NewOffset) { + assert(NewOffset <= MaxOffset && "Offset out of range"); + Offset = NewOffset; + } + static constexpr uint64_t MaxOffset = (1ULL << 59) - 1; // FIXME: A char* or SymbolStringPtr may pack better. @@ -532,6 +558,16 @@ public: /// Returns the ordinal for this section. SectionOrdinal getOrdinal() const { return SecOrdinal; } + /// Returns an iterator over the blocks defined in this section. + iterator_range<block_iterator> blocks() { + return make_range(Blocks.begin(), Blocks.end()); + } + + /// Returns an iterator over the blocks defined in this section. + iterator_range<const_block_iterator> blocks() const { + return make_range(Blocks.begin(), Blocks.end()); + } + /// Returns an iterator over the symbols defined in this section. iterator_range<symbol_iterator> symbols() { return make_range(Symbols.begin(), Symbols.end()); @@ -545,12 +581,6 @@ public: /// Return the number of symbols in this section. SymbolSet::size_type symbols_size() { return Symbols.size(); } - /// Return true if this section contains no symbols. - bool symbols_empty() const { return Symbols.empty(); } - - /// Returns the ordinal for the next block. - BlockOrdinal getNextBlockOrdinal() { return NextBlockOrdinal++; } - private: void addSymbol(Symbol &Sym) { assert(!Symbols.count(&Sym) && "Symbol is already in this section"); @@ -562,10 +592,20 @@ private: Symbols.erase(&Sym); } + void addBlock(Block &B) { + assert(!Blocks.count(&B) && "Block is already in this section"); + Blocks.insert(&B); + } + + void removeBlock(Block &B) { + assert(Blocks.count(&B) && "Block is not in this section"); + Blocks.erase(&B); + } + StringRef Name; sys::Memory::ProtectionFlags Prot; SectionOrdinal SecOrdinal = 0; - BlockOrdinal NextBlockOrdinal = 0; + BlockSet Blocks; SymbolSet Symbols; }; @@ -575,21 +615,21 @@ class SectionRange { public: SectionRange() = default; SectionRange(const Section &Sec) { - if (Sec.symbols_empty()) + if (llvm::empty(Sec.blocks())) return; - First = Last = *Sec.symbols().begin(); - for (auto *Sym : Sec.symbols()) { - if (Sym->getAddress() < First->getAddress()) - First = Sym; - if (Sym->getAddress() > Last->getAddress()) - Last = Sym; + First = Last = *Sec.blocks().begin(); + for (auto *B : Sec.blocks()) { + if (B->getAddress() < First->getAddress()) + First = B; + if (B->getAddress() > Last->getAddress()) + Last = B; } } - Symbol *getFirstSymbol() const { + Block *getFirstBlock() const { assert((!Last || First) && "First can not be null if end is non-null"); return First; } - Symbol *getLastSymbol() const { + Block *getLastBlock() const { assert((First || !Last) && "Last can not be null if start is non-null"); return Last; } @@ -598,17 +638,16 @@ public: return !First; } JITTargetAddress getStart() const { - return First ? First->getBlock().getAddress() : 0; + return First ? First->getAddress() : 0; } JITTargetAddress getEnd() const { - return Last ? Last->getBlock().getAddress() + Last->getBlock().getSize() - : 0; + return Last ? Last->getAddress() + Last->getSize() : 0; } uint64_t getSize() const { return getEnd() - getStart(); } private: - Symbol *First = nullptr; - Symbol *Last = nullptr; + Block *First = nullptr; + Block *Last = nullptr; }; class LinkGraph { @@ -633,12 +672,11 @@ private: template <typename... ArgTs> Block &createBlock(ArgTs &&... Args) { Block *B = reinterpret_cast<Block *>(Allocator.Allocate<Block>()); new (B) Block(std::forward<ArgTs>(Args)...); - Blocks.insert(B); + B->getSection().addBlock(*B); return *B; } void destroyBlock(Block &B) { - Blocks.erase(&B); B.~Block(); Allocator.Deallocate(&B); } @@ -648,69 +686,102 @@ private: Allocator.Deallocate(&S); } + static iterator_range<Section::block_iterator> getSectionBlocks(Section &S) { + return S.blocks(); + } + + static iterator_range<Section::const_block_iterator> + getSectionConstBlocks(Section &S) { + return S.blocks(); + } + + static iterator_range<Section::symbol_iterator> + getSectionSymbols(Section &S) { + return S.symbols(); + } + + static iterator_range<Section::const_symbol_iterator> + getSectionConstSymbols(Section &S) { + return S.symbols(); + } + public: using external_symbol_iterator = ExternalSymbolSet::iterator; - using block_iterator = BlockSet::iterator; - using section_iterator = pointee_iterator<SectionList::iterator>; using const_section_iterator = pointee_iterator<SectionList::const_iterator>; - template <typename SectionItrT, typename SymbolItrT, typename T> - class defined_symbol_iterator_impl + template <typename OuterItrT, typename InnerItrT, typename T, + iterator_range<InnerItrT> getInnerRange( + typename OuterItrT::reference)> + class nested_collection_iterator : public iterator_facade_base< - defined_symbol_iterator_impl<SectionItrT, SymbolItrT, T>, + nested_collection_iterator<OuterItrT, InnerItrT, T, getInnerRange>, std::forward_iterator_tag, T> { public: - defined_symbol_iterator_impl() = default; + nested_collection_iterator() = default; - defined_symbol_iterator_impl(SectionItrT SecI, SectionItrT SecE) - : SecI(SecI), SecE(SecE), - SymI(SecI != SecE ? SecI->symbols().begin() : SymbolItrT()) { - moveToNextSymbolOrEnd(); + nested_collection_iterator(OuterItrT OuterI, OuterItrT OuterE) + : OuterI(OuterI), OuterE(OuterE), + InnerI(getInnerBegin(OuterI, OuterE)) { + moveToNonEmptyInnerOrEnd(); } - bool operator==(const defined_symbol_iterator_impl &RHS) const { - return (SecI == RHS.SecI) && (SymI == RHS.SymI); + bool operator==(const nested_collection_iterator &RHS) const { + return (OuterI == RHS.OuterI) && (InnerI == RHS.InnerI); } T operator*() const { - assert(SymI != SecI->symbols().end() && "Dereferencing end?"); - return *SymI; + assert(InnerI != getInnerRange(*OuterI).end() && "Dereferencing end?"); + return *InnerI; } - defined_symbol_iterator_impl operator++() { - ++SymI; - moveToNextSymbolOrEnd(); + nested_collection_iterator operator++() { + ++InnerI; + moveToNonEmptyInnerOrEnd(); return *this; } private: - void moveToNextSymbolOrEnd() { - while (SecI != SecE && SymI == SecI->symbols().end()) { - ++SecI; - SymI = SecI == SecE ? SymbolItrT() : SecI->symbols().begin(); + static InnerItrT getInnerBegin(OuterItrT OuterI, OuterItrT OuterE) { + return OuterI != OuterE ? getInnerRange(*OuterI).begin() : InnerItrT(); + } + + void moveToNonEmptyInnerOrEnd() { + while (OuterI != OuterE && InnerI == getInnerRange(*OuterI).end()) { + ++OuterI; + InnerI = getInnerBegin(OuterI, OuterE); } } - SectionItrT SecI, SecE; - SymbolItrT SymI; + OuterItrT OuterI, OuterE; + InnerItrT InnerI; }; using defined_symbol_iterator = - defined_symbol_iterator_impl<const_section_iterator, - Section::symbol_iterator, Symbol *>; + nested_collection_iterator<const_section_iterator, + Section::symbol_iterator, Symbol *, + getSectionSymbols>; + + using const_defined_symbol_iterator = + nested_collection_iterator<const_section_iterator, + Section::const_symbol_iterator, const Symbol *, + getSectionConstSymbols>; + + using block_iterator = nested_collection_iterator<const_section_iterator, + Section::block_iterator, + Block *, getSectionBlocks>; - using const_defined_symbol_iterator = defined_symbol_iterator_impl< - const_section_iterator, Section::const_symbol_iterator, const Symbol *>; + using const_block_iterator = + nested_collection_iterator<const_section_iterator, + Section::const_block_iterator, const Block *, + getSectionConstBlocks>; LinkGraph(std::string Name, unsigned PointerSize, support::endianness Endianness) : Name(std::move(Name)), PointerSize(PointerSize), Endianness(Endianness) {} - ~LinkGraph(); - /// Returns the name of this graph (usually the name of the original /// underlying MemoryBuffer). const std::string &getName() { return Name; } @@ -732,23 +803,56 @@ public: Block &createContentBlock(Section &Parent, StringRef Content, uint64_t Address, uint64_t Alignment, uint64_t AlignmentOffset) { - return createBlock(Parent, Parent.getNextBlockOrdinal(), Content, Address, - Alignment, AlignmentOffset); + return createBlock(Parent, Content, Address, Alignment, AlignmentOffset); } /// Create a zero-fill block. Block &createZeroFillBlock(Section &Parent, uint64_t Size, uint64_t Address, uint64_t Alignment, uint64_t AlignmentOffset) { - return createBlock(Parent, Parent.getNextBlockOrdinal(), Size, Address, - Alignment, AlignmentOffset); + return createBlock(Parent, Size, Address, Alignment, AlignmentOffset); } + /// Cache type for the splitBlock function. + using SplitBlockCache = Optional<SmallVector<Symbol *, 8>>; + + /// Splits block B at the given index which must be greater than zero. + /// If SplitIndex == B.getSize() then this function is a no-op and returns B. + /// If SplitIndex < B.getSize() then this function returns a new block + /// covering the range [ 0, SplitIndex ), and B is modified to cover the range + /// [ SplitIndex, B.size() ). + /// + /// The optional Cache parameter can be used to speed up repeated calls to + /// splitBlock for a single block. If the value is None the cache will be + /// treated as uninitialized and splitBlock will populate it. Otherwise it + /// is assumed to contain the list of Symbols pointing at B, sorted in + /// descending order of offset. + /// + /// Notes: + /// + /// 1. The newly introduced block will have a new ordinal which will be + /// higher than any other ordinals in the section. Clients are responsible + /// for re-assigning block ordinals to restore a compatible order if + /// needed. + /// + /// 2. The cache is not automatically updated if new symbols are introduced + /// between calls to splitBlock. Any newly introduced symbols may be + /// added to the cache manually (descending offset order must be + /// preserved), or the cache can be set to None and rebuilt by + /// splitBlock on the next call. + Block &splitBlock(Block &B, size_t SplitIndex, + SplitBlockCache *Cache = nullptr); + /// Add an external symbol. /// Some formats (e.g. ELF) allow Symbols to have sizes. For Symbols whose /// size is not known, you should substitute '0'. - Symbol &addExternalSymbol(StringRef Name, uint64_t Size) { - auto &Sym = Symbol::constructExternal( - Allocator.Allocate<Symbol>(), createAddressable(0, false), Name, Size); + /// For external symbols Linkage determines whether the symbol must be + /// present during lookup: Externals with strong linkage must be found or + /// an error will be emitted. Externals with weak linkage are permitted to + /// be undefined, in which case they are assigned a value of 0. + Symbol &addExternalSymbol(StringRef Name, uint64_t Size, Linkage L) { + auto &Sym = + Symbol::constructExternal(Allocator.Allocate<Symbol>(), + createAddressable(0, false), Name, Size, L); ExternalSymbols.insert(&Sym); return Sym; } @@ -769,9 +873,8 @@ public: uint64_t Alignment, bool IsLive) { auto &Sym = Symbol::constructCommon( Allocator.Allocate<Symbol>(), - createBlock(Section, Section.getNextBlockOrdinal(), Address, Size, - Alignment, 0), - Name, Size, S, IsLive); + createBlock(Section, Size, Address, Alignment, 0), Name, Size, S, + IsLive); Section.addSymbol(Sym); return Sym; } @@ -811,6 +914,16 @@ public: return nullptr; } + iterator_range<block_iterator> blocks() { + return make_range(block_iterator(Sections.begin(), Sections.end()), + block_iterator(Sections.end(), Sections.end())); + } + + iterator_range<const_block_iterator> blocks() const { + return make_range(const_block_iterator(Sections.begin(), Sections.end()), + const_block_iterator(Sections.end(), Sections.end())); + } + iterator_range<external_symbol_iterator> external_symbols() { return make_range(ExternalSymbols.begin(), ExternalSymbols.end()); } @@ -830,10 +943,6 @@ public: const_defined_symbol_iterator(Sections.end(), Sections.end())); } - iterator_range<block_iterator> blocks() { - return make_range(Blocks.begin(), Blocks.end()); - } - /// Turn a defined symbol into an external one. void makeExternal(Symbol &Sym) { if (Sym.getAddressable().isAbsolute()) { @@ -881,7 +990,7 @@ public: /// Remove a block. void removeBlock(Block &B) { - Blocks.erase(&B); + B.getSection().removeBlock(B); destroyBlock(B); } @@ -902,12 +1011,150 @@ private: std::string Name; unsigned PointerSize; support::endianness Endianness; - BlockSet Blocks; SectionList Sections; ExternalSymbolSet ExternalSymbols; ExternalSymbolSet AbsoluteSymbols; }; +/// Enables easy lookup of blocks by addresses. +class BlockAddressMap { +public: + using AddrToBlockMap = std::map<JITTargetAddress, Block *>; + using const_iterator = AddrToBlockMap::const_iterator; + + /// A block predicate that always adds all blocks. + static bool includeAllBlocks(const Block &B) { return true; } + + /// A block predicate that always includes blocks with non-null addresses. + static bool includeNonNull(const Block &B) { return B.getAddress(); } + + BlockAddressMap() = default; + + /// Add a block to the map. Returns an error if the block overlaps with any + /// existing block. + template <typename PredFn = decltype(includeAllBlocks)> + Error addBlock(Block &B, PredFn Pred = includeAllBlocks) { + if (!Pred(B)) + return Error::success(); + + auto I = AddrToBlock.upper_bound(B.getAddress()); + + // If we're not at the end of the map, check for overlap with the next + // element. + if (I != AddrToBlock.end()) { + if (B.getAddress() + B.getSize() > I->second->getAddress()) + return overlapError(B, *I->second); + } + + // If we're not at the start of the map, check for overlap with the previous + // element. + if (I != AddrToBlock.begin()) { + auto &PrevBlock = *std::prev(I)->second; + if (PrevBlock.getAddress() + PrevBlock.getSize() > B.getAddress()) + return overlapError(B, PrevBlock); + } + + AddrToBlock.insert(I, std::make_pair(B.getAddress(), &B)); + return Error::success(); + } + + /// Add a block to the map without checking for overlap with existing blocks. + /// The client is responsible for ensuring that the block added does not + /// overlap with any existing block. + void addBlockWithoutChecking(Block &B) { AddrToBlock[B.getAddress()] = &B; } + + /// Add a range of blocks to the map. Returns an error if any block in the + /// range overlaps with any other block in the range, or with any existing + /// block in the map. + template <typename BlockPtrRange, + typename PredFn = decltype(includeAllBlocks)> + Error addBlocks(BlockPtrRange &&Blocks, PredFn Pred = includeAllBlocks) { + for (auto *B : Blocks) + if (auto Err = addBlock(*B, Pred)) + return Err; + return Error::success(); + } + + /// Add a range of blocks to the map without checking for overlap with + /// existing blocks. The client is responsible for ensuring that the block + /// added does not overlap with any existing block. + template <typename BlockPtrRange> + void addBlocksWithoutChecking(BlockPtrRange &&Blocks) { + for (auto *B : Blocks) + addBlockWithoutChecking(*B); + } + + /// Iterates over (Address, Block*) pairs in ascending order of address. + const_iterator begin() const { return AddrToBlock.begin(); } + const_iterator end() const { return AddrToBlock.end(); } + + /// Returns the block starting at the given address, or nullptr if no such + /// block exists. + Block *getBlockAt(JITTargetAddress Addr) const { + auto I = AddrToBlock.find(Addr); + if (I == AddrToBlock.end()) + return nullptr; + return I->second; + } + + /// Returns the block covering the given address, or nullptr if no such block + /// exists. + Block *getBlockCovering(JITTargetAddress Addr) const { + auto I = AddrToBlock.upper_bound(Addr); + if (I == AddrToBlock.begin()) + return nullptr; + auto *B = std::prev(I)->second; + if (Addr < B->getAddress() + B->getSize()) + return B; + return nullptr; + } + +private: + Error overlapError(Block &NewBlock, Block &ExistingBlock) { + auto NewBlockEnd = NewBlock.getAddress() + NewBlock.getSize(); + auto ExistingBlockEnd = + ExistingBlock.getAddress() + ExistingBlock.getSize(); + return make_error<JITLinkError>( + "Block at " + + formatv("{0:x16} -- {1:x16}", NewBlock.getAddress(), NewBlockEnd) + + " overlaps " + + formatv("{0:x16} -- {1:x16}", ExistingBlock.getAddress(), + ExistingBlockEnd)); + } + + AddrToBlockMap AddrToBlock; +}; + +/// A map of addresses to Symbols. +class SymbolAddressMap { +public: + using SymbolVector = SmallVector<Symbol *, 1>; + + /// Add a symbol to the SymbolAddressMap. + void addSymbol(Symbol &Sym) { + AddrToSymbols[Sym.getAddress()].push_back(&Sym); + } + + /// Add all symbols in a given range to the SymbolAddressMap. + template <typename SymbolPtrCollection> + void addSymbols(SymbolPtrCollection &&Symbols) { + for (auto *Sym : Symbols) + addSymbol(*Sym); + } + + /// Returns the list of symbols that start at the given address, or nullptr if + /// no such symbols exist. + const SymbolVector *getSymbolsAt(JITTargetAddress Addr) const { + auto I = AddrToSymbols.find(Addr); + if (I == AddrToSymbols.end()) + return nullptr; + return &I->second; + } + +private: + std::map<JITTargetAddress, SymbolVector> AddrToSymbols; +}; + /// A function for mutating LinkGraphs. using LinkGraphPassFunction = std::function<Error(LinkGraph &)>; @@ -943,6 +1190,14 @@ struct PassConfiguration { LinkGraphPassList PostFixupPasses; }; +/// Flags for symbol lookup. +/// +/// FIXME: These basically duplicate orc::SymbolLookupFlags -- We should merge +/// the two types once we have an OrcSupport library. +enum class SymbolLookupFlags { RequiredSymbol, WeaklyReferencedSymbol }; + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF); + /// A map of symbol names to resolved addresses. using AsyncLookupResult = DenseMap<StringRef, JITEvaluatedSymbol>; @@ -977,6 +1232,8 @@ createLookupContinuation(Continuation Cont) { /// Holds context for a single jitLink invocation. class JITLinkContext { public: + using LookupMap = DenseMap<StringRef, SymbolLookupFlags>; + /// Destroy a JITLinkContext. virtual ~JITLinkContext(); @@ -994,7 +1251,7 @@ public: /// Called by JITLink to resolve external symbols. This method is passed a /// lookup continutation which it must call with a result to continue the /// linking process. - virtual void lookup(const DenseSet<StringRef> &Symbols, + virtual void lookup(const LookupMap &Symbols, std::unique_ptr<JITLinkAsyncLookupContinuation> LC) = 0; /// Called by JITLink once all defined symbols in the graph have been assigned diff --git a/llvm/include/llvm/ExecutionEngine/JITSymbol.h b/llvm/include/llvm/ExecutionEngine/JITSymbol.h index c0f1ca4b9876..7a2a6cfa5203 100644 --- a/llvm/include/llvm/ExecutionEngine/JITSymbol.h +++ b/llvm/include/llvm/ExecutionEngine/JITSymbol.h @@ -41,6 +41,11 @@ class SymbolRef; using JITTargetAddress = uint64_t; /// Convert a JITTargetAddress to a pointer. +/// +/// Note: This is a raw cast of the address bit pattern to the given pointer +/// type. When casting to a function pointer in order to execute JIT'd code +/// jitTargetAddressToFunction should be preferred, as it will also perform +/// pointer signing on targets that require it. template <typename T> T jitTargetAddressToPointer(JITTargetAddress Addr) { static_assert(std::is_pointer<T>::value, "T must be a pointer type"); uintptr_t IntPtr = static_cast<uintptr_t>(Addr); @@ -48,6 +53,19 @@ template <typename T> T jitTargetAddressToPointer(JITTargetAddress Addr) { return reinterpret_cast<T>(IntPtr); } +/// Convert a JITTargetAddress to a callable function pointer. +/// +/// Casts the given address to a callable function pointer. This operation +/// will perform pointer signing for platforms that require it (e.g. arm64e). +template <typename T> T jitTargetAddressToFunction(JITTargetAddress Addr) { + static_assert( + std::is_pointer<T>::value && + std::is_function<typename std::remove_pointer<T>::type>::value, + "T must be a function pointer type"); + return jitTargetAddressToPointer<T>(Addr); +} + +/// Convert a pointer to a JITTargetAddress. template <typename T> JITTargetAddress pointerToJITTargetAddress(T *Ptr) { return static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Ptr)); } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index 4f22a4c38796..d0a9ca5c0580 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -45,8 +45,11 @@ using VModuleKey = uint64_t; // efficiency). using SymbolNameSet = DenseSet<SymbolStringPtr>; +/// A vector of symbol names. +using SymbolNameVector = std::vector<SymbolStringPtr>; + /// A map from symbol names (as SymbolStringPtrs) to JITSymbols -/// (address/flags pairs). +/// (address/flags pairs). using SymbolMap = DenseMap<SymbolStringPtr, JITEvaluatedSymbol>; /// A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags. @@ -55,8 +58,244 @@ using SymbolFlagsMap = DenseMap<SymbolStringPtr, JITSymbolFlags>; /// A map from JITDylibs to sets of symbols. using SymbolDependenceMap = DenseMap<JITDylib *, SymbolNameSet>; -/// A list of (JITDylib*, bool) pairs. -using JITDylibSearchList = std::vector<std::pair<JITDylib *, bool>>; +/// Lookup flags that apply to each dylib in the search order for a lookup. +/// +/// If MatchHiddenSymbolsOnly is used (the default) for a given dylib, then +/// only symbols in that Dylib's interface will be searched. If +/// MatchHiddenSymbols is used then symbols with hidden visibility will match +/// as well. +enum class JITDylibLookupFlags { MatchExportedSymbolsOnly, MatchAllSymbols }; + +/// Lookup flags that apply to each symbol in a lookup. +/// +/// If RequiredSymbol is used (the default) for a given symbol then that symbol +/// must be found during the lookup or the lookup will fail returning a +/// SymbolNotFound error. If WeaklyReferencedSymbol is used and the given +/// symbol is not found then the query will continue, and no result for the +/// missing symbol will be present in the result (assuming the rest of the +/// lookup succeeds). +enum class SymbolLookupFlags { RequiredSymbol, WeaklyReferencedSymbol }; + +/// Describes the kind of lookup being performed. The lookup kind is passed to +/// symbol generators (if they're invoked) to help them determine what +/// definitions to generate. +/// +/// Static -- Lookup is being performed as-if at static link time (e.g. +/// generators representing static archives should pull in new +/// definitions). +/// +/// DLSym -- Lookup is being performed as-if at runtime (e.g. generators +/// representing static archives should not pull in new definitions). +enum class LookupKind { Static, DLSym }; + +/// A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search +/// order during symbol lookup. +using JITDylibSearchOrder = + std::vector<std::pair<JITDylib *, JITDylibLookupFlags>>; + +/// Convenience function for creating a search order from an ArrayRef of +/// JITDylib*, all with the same flags. +inline JITDylibSearchOrder makeJITDylibSearchOrder( + ArrayRef<JITDylib *> JDs, + JITDylibLookupFlags Flags = JITDylibLookupFlags::MatchExportedSymbolsOnly) { + JITDylibSearchOrder O; + O.reserve(JDs.size()); + for (auto *JD : JDs) + O.push_back(std::make_pair(JD, Flags)); + return O; +} + +/// A set of symbols to look up, each associated with a SymbolLookupFlags +/// value. +/// +/// This class is backed by a vector and optimized for fast insertion, +/// deletion and iteration. It does not guarantee a stable order between +/// operations, and will not automatically detect duplicate elements (they +/// can be manually checked by calling the validate method). +class SymbolLookupSet { +public: + using value_type = std::pair<SymbolStringPtr, SymbolLookupFlags>; + using UnderlyingVector = std::vector<value_type>; + using iterator = UnderlyingVector::iterator; + using const_iterator = UnderlyingVector::const_iterator; + + SymbolLookupSet() = default; + + explicit SymbolLookupSet( + SymbolStringPtr Name, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + add(std::move(Name), Flags); + } + + /// Construct a SymbolLookupSet from an initializer list of SymbolStringPtrs. + explicit SymbolLookupSet( + std::initializer_list<SymbolStringPtr> Names, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.reserve(Names.size()); + for (auto &Name : Names) + add(std::move(Name), Flags); + } + + /// Construct a SymbolLookupSet from a SymbolNameSet with the given + /// Flags used for each value. + explicit SymbolLookupSet( + const SymbolNameSet &Names, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.reserve(Names.size()); + for (const auto &Name : Names) + add(Name, Flags); + } + + /// Construct a SymbolLookupSet from a vector of symbols with the given Flags + /// used for each value. + /// If the ArrayRef contains duplicates it is up to the client to remove these + /// before using this instance for lookup. + explicit SymbolLookupSet( + ArrayRef<SymbolStringPtr> Names, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.reserve(Names.size()); + for (const auto &Name : Names) + add(Name, Flags); + } + + /// Add an element to the set. The client is responsible for checking that + /// duplicates are not added. + void add(SymbolStringPtr Name, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.push_back(std::make_pair(std::move(Name), Flags)); + } + + bool empty() const { return Symbols.empty(); } + UnderlyingVector::size_type size() const { return Symbols.size(); } + iterator begin() { return Symbols.begin(); } + iterator end() { return Symbols.end(); } + const_iterator begin() const { return Symbols.begin(); } + const_iterator end() const { return Symbols.end(); } + + /// Removes the Ith element of the vector, replacing it with the last element. + void remove(UnderlyingVector::size_type I) { + std::swap(Symbols[I], Symbols.back()); + Symbols.pop_back(); + } + + /// Removes the element pointed to by the given iterator. This iterator and + /// all subsequent ones (including end()) are invalidated. + void remove(iterator I) { remove(I - begin()); } + + /// Removes all elements matching the given predicate, which must be callable + /// as bool(const SymbolStringPtr &, SymbolLookupFlags Flags). + template <typename PredFn> void remove_if(PredFn &&Pred) { + UnderlyingVector::size_type I = 0; + while (I != Symbols.size()) { + const auto &Name = Symbols[I].first; + auto Flags = Symbols[I].second; + if (Pred(Name, Flags)) + remove(I); + else + ++I; + } + } + + /// Loop over the elements of this SymbolLookupSet, applying the Body function + /// to each one. Body must be callable as + /// bool(const SymbolStringPtr &, SymbolLookupFlags). + /// If Body returns true then the element just passed in is removed from the + /// set. If Body returns false then the element is retained. + template <typename BodyFn> + auto forEachWithRemoval(BodyFn &&Body) -> typename std::enable_if< + std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(), + std::declval<SymbolLookupFlags>())), + bool>::value>::type { + UnderlyingVector::size_type I = 0; + while (I != Symbols.size()) { + const auto &Name = Symbols[I].first; + auto Flags = Symbols[I].second; + if (Body(Name, Flags)) + remove(I); + else + ++I; + } + } + + /// Loop over the elements of this SymbolLookupSet, applying the Body function + /// to each one. Body must be callable as + /// Expected<bool>(const SymbolStringPtr &, SymbolLookupFlags). + /// If Body returns a failure value, the loop exits immediately. If Body + /// returns true then the element just passed in is removed from the set. If + /// Body returns false then the element is retained. + template <typename BodyFn> + auto forEachWithRemoval(BodyFn &&Body) -> typename std::enable_if< + std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(), + std::declval<SymbolLookupFlags>())), + Expected<bool>>::value, + Error>::type { + UnderlyingVector::size_type I = 0; + while (I != Symbols.size()) { + const auto &Name = Symbols[I].first; + auto Flags = Symbols[I].second; + auto Remove = Body(Name, Flags); + if (!Remove) + return Remove.takeError(); + if (*Remove) + remove(I); + else + ++I; + } + return Error::success(); + } + + /// Construct a SymbolNameVector from this instance by dropping the Flags + /// values. + SymbolNameVector getSymbolNames() const { + SymbolNameVector Names; + Names.reserve(Symbols.size()); + for (auto &KV : Symbols) + Names.push_back(KV.first); + return Names; + } + + /// Sort the lookup set by pointer value. This sort is fast but sensitive to + /// allocation order and so should not be used where a consistent order is + /// required. + void sortByAddress() { + llvm::sort(Symbols, [](const value_type &LHS, const value_type &RHS) { + return LHS.first < RHS.first; + }); + } + + /// Sort the lookup set lexicographically. This sort is slow but the order + /// is unaffected by allocation order. + void sortByName() { + llvm::sort(Symbols, [](const value_type &LHS, const value_type &RHS) { + return *LHS.first < *RHS.first; + }); + } + + /// Remove any duplicate elements. If a SymbolLookupSet is not duplicate-free + /// by construction, this method can be used to turn it into a proper set. + void removeDuplicates() { + sortByAddress(); + auto LastI = std::unique(Symbols.begin(), Symbols.end()); + Symbols.erase(LastI, Symbols.end()); + } + +#ifndef NDEBUG + /// Returns true if this set contains any duplicates. This should only be used + /// in assertions. + bool containsDuplicates() { + if (Symbols.size() < 2) + return false; + sortByAddress(); + for (UnderlyingVector::size_type I = 1; I != Symbols.size(); ++I) + if (Symbols[I].first == Symbols[I - 1].first) + return true; + return true; + } +#endif + +private: + UnderlyingVector Symbols; +}; struct SymbolAliasMapEntry { SymbolAliasMapEntry() = default; @@ -76,6 +315,9 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym); /// Render a SymbolNameSet. raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols); +/// Render a SymbolNameVector. +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols); + /// Render a SymbolFlagsMap entry. raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV); @@ -98,8 +340,25 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps); /// Render a MaterializationUnit. raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU); -/// Render a JITDylibSearchList. -raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs); +//// Render a JITDylibLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibLookupFlags &JDLookupFlags); + +/// Rendar a SymbolLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags); + +/// Render a JITDylibLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); + +/// Render a SymbolLookupSet entry. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet::value_type &KV); + +/// Render a SymbolLookupSet. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet); + +/// Render a JITDylibSearchOrder. +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibSearchOrder &SearchOrder); /// Render a SymbolAliasMap. raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); @@ -107,6 +366,9 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); /// Render a SymbolState. raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); +/// Render a LookupKind. +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); + /// Callback to notify client that symbols have been resolved. using SymbolsResolvedCallback = unique_function<void(Expected<SymbolMap>)>; @@ -139,12 +401,13 @@ public: static char ID; SymbolsNotFound(SymbolNameSet Symbols); + SymbolsNotFound(SymbolNameVector Symbols); std::error_code convertToErrorCode() const override; void log(raw_ostream &OS) const override; - const SymbolNameSet &getSymbols() const { return Symbols; } + const SymbolNameVector &getSymbols() const { return Symbols; } private: - SymbolNameSet Symbols; + SymbolNameVector Symbols; }; /// Used to notify clients that a set of symbols could not be removed. @@ -376,7 +639,8 @@ public: /// Note: Care must be taken that no sets of aliases form a cycle, as such /// a cycle will result in a deadlock when any symbol in the cycle is /// resolved. - ReExportsMaterializationUnit(JITDylib *SourceJD, bool MatchNonExported, + ReExportsMaterializationUnit(JITDylib *SourceJD, + JITDylibLookupFlags SourceJDLookupFlags, SymbolAliasMap Aliases, VModuleKey K); StringRef getName() const override; @@ -387,7 +651,7 @@ private: static SymbolFlagsMap extractFlags(const SymbolAliasMap &Aliases); JITDylib *SourceJD = nullptr; - bool MatchNonExported = false; + JITDylibLookupFlags SourceJDLookupFlags; SymbolAliasMap Aliases; }; @@ -405,25 +669,26 @@ private: inline std::unique_ptr<ReExportsMaterializationUnit> symbolAliases(SymbolAliasMap Aliases, VModuleKey K = VModuleKey()) { return std::make_unique<ReExportsMaterializationUnit>( - nullptr, true, std::move(Aliases), std::move(K)); + nullptr, JITDylibLookupFlags::MatchAllSymbols, std::move(Aliases), + std::move(K)); } /// Create a materialization unit for re-exporting symbols from another JITDylib /// with alternative names/flags. -/// If MatchNonExported is true then non-exported symbols from SourceJD can be -/// re-exported. If it is false, attempts to re-export a non-exported symbol -/// will result in a "symbol not found" error. +/// SourceJD will be searched using the given JITDylibLookupFlags. inline std::unique_ptr<ReExportsMaterializationUnit> reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, - bool MatchNonExported = false, VModuleKey K = VModuleKey()) { + JITDylibLookupFlags SourceJDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly, + VModuleKey K = VModuleKey()) { return std::make_unique<ReExportsMaterializationUnit>( - &SourceJD, MatchNonExported, std::move(Aliases), std::move(K)); + &SourceJD, SourceJDLookupFlags, std::move(Aliases), std::move(K)); } /// Build a SymbolAliasMap for the common case where you want to re-export /// symbols from another JITDylib with the same linkage/flags. Expected<SymbolAliasMap> -buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols); +buildSimpleReexportsAAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols); /// Represents the state that a symbol has reached during materialization. enum class SymbolState : uint8_t { @@ -448,7 +713,7 @@ public: /// Create a query for the given symbols. The NotifyComplete /// callback will be called once all queried symbols reach the given /// minimum state. - AsynchronousSymbolQuery(const SymbolNameSet &Symbols, + AsynchronousSymbolQuery(const SymbolLookupSet &Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete); @@ -456,6 +721,15 @@ public: void notifySymbolMetRequiredState(const SymbolStringPtr &Name, JITEvaluatedSymbol Sym); + /// Remove a symbol from the query. This is used to drop weakly referenced + /// symbols that are not found. + void dropSymbol(const SymbolStringPtr &Name) { + assert(ResolvedSymbols.count(Name) && + "Redundant removal of weakly-referenced symbol"); + ResolvedSymbols.erase(Name); + --OutstandingSymbolsCount; + } + /// Returns true if all symbols covered by this query have been /// resolved. bool isComplete() const { return OutstandingSymbolsCount == 0; } @@ -497,11 +771,21 @@ class JITDylib { friend class ExecutionSession; friend class MaterializationResponsibility; public: + /// Definition generators can be attached to JITDylibs to generate new + /// definitions for otherwise unresolved symbols during lookup. class DefinitionGenerator { public: virtual ~DefinitionGenerator(); - virtual Expected<SymbolNameSet> - tryToGenerate(JITDylib &Parent, const SymbolNameSet &Names) = 0; + + /// DefinitionGenerators should override this method to insert new + /// definitions into the parent JITDylib. K specifies the kind of this + /// lookup. JD specifies the target JITDylib being searched, and + /// JDLookupFlags specifies whether the search should match against + /// hidden symbols. Finally, Symbols describes the set of unresolved + /// symbols and their associated lookup flags. + virtual Error tryToGenerate(LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) = 0; }; using AsynchronousSymbolQuerySet = @@ -552,18 +836,20 @@ public: /// as the first in the search order (instead of this dylib) ensures that /// definitions within this dylib resolve to the lazy-compiling stubs, /// rather than immediately materializing the definitions in this dylib. - void setSearchOrder(JITDylibSearchList NewSearchOrder, - bool SearchThisJITDylibFirst = true, - bool MatchNonExportedInThisDylib = true); + void setSearchOrder(JITDylibSearchOrder NewSearchOrder, + bool SearchThisJITDylibFirst = true); /// Add the given JITDylib to the search order for definitions in this /// JITDylib. - void addToSearchOrder(JITDylib &JD, bool MatcNonExported = false); + void addToSearchOrder(JITDylib &JD, + JITDylibLookupFlags JDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly); /// Replace OldJD with NewJD in the search order if OldJD is present. /// Otherwise this operation is a no-op. void replaceInSearchOrder(JITDylib &OldJD, JITDylib &NewJD, - bool MatchNonExported = false); + JITDylibLookupFlags JDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly); /// Remove the given JITDylib from the search order for this JITDylib if it is /// present. Otherwise this operation is a no-op. @@ -572,7 +858,7 @@ public: /// Do something with the search order (run under the session lock). template <typename Func> auto withSearchOrderDo(Func &&F) - -> decltype(F(std::declval<const JITDylibSearchList &>())); + -> decltype(F(std::declval<const JITDylibSearchOrder &>())); /// Define all symbols provided by the materialization unit to be part of this /// JITDylib. @@ -605,8 +891,11 @@ public: Error remove(const SymbolNameSet &Names); /// Search the given JITDylib for the symbols in Symbols. If found, store - /// the flags for each symbol in Flags. Returns any unresolved symbols. - Expected<SymbolFlagsMap> lookupFlags(const SymbolNameSet &Names); + /// the flags for each symbol in Flags. If any required symbols are not found + /// then an error will be returned. + Expected<SymbolFlagsMap> lookupFlags(LookupKind K, + JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet LookupSet); /// Dump current JITDylib state to OS. void dump(raw_ostream &OS); @@ -709,20 +998,23 @@ private: Error defineImpl(MaterializationUnit &MU); - Expected<SymbolNameSet> lookupFlagsImpl(SymbolFlagsMap &Flags, - const SymbolNameSet &Names); + void lookupFlagsImpl(SymbolFlagsMap &Result, LookupKind K, + JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Unresolved); - Error lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs); + Error lodgeQuery(MaterializationUnitList &MUs, + std::shared_ptr<AsynchronousSymbolQuery> &Q, LookupKind K, + JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Unresolved); - Error lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs); + Error lodgeQueryImpl(MaterializationUnitList &MUs, + std::shared_ptr<AsynchronousSymbolQuery> &Q, + LookupKind K, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Unresolved); bool lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, std::vector<std::unique_ptr<MaterializationUnit>> &MUs, - SymbolNameSet &Unresolved); + SymbolLookupSet &Unresolved); void detachQueryHelper(AsynchronousSymbolQuery &Q, const SymbolNameSet &QuerySymbols); @@ -754,7 +1046,7 @@ private: UnmaterializedInfosMap UnmaterializedInfos; MaterializingInfosMap MaterializingInfos; std::vector<std::unique_ptr<DefinitionGenerator>> DefGenerators; - JITDylibSearchList SearchOrder; + JITDylibSearchOrder SearchOrder; }; /// An ExecutionSession represents a running JIT program. @@ -787,10 +1079,6 @@ public: return F(); } - /// Get the "main" JITDylib, which is created automatically on construction of - /// the ExecutionSession. - JITDylib &getMainJITDylib(); - /// Return a pointer to the "name" JITDylib. /// Ownership of JITDylib remains within Execution Session JITDylib *getJITDylibByName(StringRef Name); @@ -800,8 +1088,7 @@ public: /// The JITDylib Name is required to be unique. Clients should verify that /// names are not being re-used (e.g. by calling getJITDylibByName) if names /// are based on user input. - JITDylib &createJITDylib(std::string Name, - bool AddToMainDylibSearchOrder = true); + JITDylib &createJITDylib(std::string Name); /// Allocate a module key for a new module to add to the JIT. VModuleKey allocateVModule() { @@ -863,8 +1150,9 @@ public: /// dependenant symbols for this query (e.g. it is being made by a top level /// client to get an address to call) then the value NoDependenciesToRegister /// can be used. - void lookup(const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, - SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, + void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder, + SymbolLookupSet Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies); /// Blocking version of lookup above. Returns the resolved symbol map. @@ -874,8 +1162,9 @@ public: /// or an error occurs. If WaitUntilReady is false and an error occurs /// after resolution, the function will return a success value, but the /// error will be reported via reportErrors. - Expected<SymbolMap> lookup(const JITDylibSearchList &SearchOrder, - const SymbolNameSet &Symbols, + Expected<SymbolMap> lookup(const JITDylibSearchOrder &SearchOrder, + const SymbolLookupSet &Symbols, + LookupKind K = LookupKind::Static, SymbolState RequiredState = SymbolState::Ready, RegisterDependenciesFunction RegisterDependencies = NoDependenciesToRegister); @@ -883,7 +1172,7 @@ public: /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. - Expected<JITEvaluatedSymbol> lookup(const JITDylibSearchList &SearchOrder, + Expected<JITEvaluatedSymbol> lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Symbol); /// Convenience version of blocking lookup. @@ -951,7 +1240,7 @@ GeneratorT &JITDylib::addGenerator(std::unique_ptr<GeneratorT> DefGenerator) { template <typename Func> auto JITDylib::withSearchOrderDo(Func &&F) - -> decltype(F(std::declval<const JITDylibSearchList &>())) { + -> decltype(F(std::declval<const JITDylibSearchOrder &>())) { return ES.runSessionLocked([&]() { return F(SearchOrder); }); } @@ -988,7 +1277,7 @@ Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU) { }); } -/// ReexportsGenerator can be used with JITDylib::setGenerator to automatically +/// ReexportsGenerator can be used with JITDylib::addGenerator to automatically /// re-export a subset of the source JITDylib's symbols in the target. class ReexportsGenerator : public JITDylib::DefinitionGenerator { public: @@ -997,15 +1286,17 @@ public: /// Create a reexports generator. If an Allow predicate is passed, only /// symbols for which the predicate returns true will be reexported. If no /// Allow predicate is passed, all symbols will be exported. - ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, + ReexportsGenerator(JITDylib &SourceJD, + JITDylibLookupFlags SourceJDLookupFlags, SymbolPredicate Allow = SymbolPredicate()); - Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, - const SymbolNameSet &Names) override; + Error tryToGenerate(LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) override; private: JITDylib &SourceJD; - bool MatchNonExported = false; + JITDylibLookupFlags SourceJDLookupFlags; SymbolPredicate Allow; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h new file mode 100644 index 000000000000..b2ef29d65ffe --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/DebugUtils.h @@ -0,0 +1,58 @@ +//===----- DebugUtils.h - Utilities for debugging ORC JITs ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for debugging ORC-based JITs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H + +#include "llvm/Support/Error.h" +#include <memory> +#include <string> + +namespace llvm { + +class MemoryBuffer; + +namespace orc { + +/// A function object that can be used as an ObjectTransformLayer transform +/// to dump object files to disk at a specified path. +class DumpObjects { +public: + /// Construct a DumpObjects transform that will dump objects to disk. + /// + /// @param DumpDir specifies the path to write dumped objects to. DumpDir may + /// be empty, in which case files will be dumped to the working directory. If + /// DumpDir is non-empty then any trailing separators will be discarded. + /// + /// @param IdentifierOverride specifies a file name stem to use when dumping + /// objects. If empty, each MemoryBuffer's identifier will be used (with a .o + /// suffix added if not already present). If an identifier override is + /// supplied it will be used instead (since all buffers will use the same + /// identifier, the resulting files will be named <ident>.o, <ident>.2.o, + /// <ident>.3.o, and so on). IdentifierOverride should not contain an + /// extension, as a .o suffix will be added by DumpObjects. + DumpObjects(std::string DumpDir = "", std::string IdentifierOverride = ""); + + /// Dumps the given buffer to disk. + Expected<std::unique_ptr<MemoryBuffer>> + operator()(std::unique_ptr<MemoryBuffer> Obj); + +private: + StringRef getBufferIdentifier(MemoryBuffer &B); + std::string DumpDir; + std::string IdentifierOverride; +}; + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index cf0a428662ef..f7255c5af845 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -40,6 +40,17 @@ namespace orc { class ObjectLayer; +/// Run a main function, returning the result. +/// +/// If the optional ProgramName argument is given then it will be inserted +/// before the strings in Args as the first argument to the called function. +/// +/// It is legal to have an empty argument list and no program name, however +/// many main functions will expect a name argument at least, and will fail +/// if none is provided. +int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, + Optional<StringRef> ProgramName = None); + /// This iterator provides a convenient way to iterate over the elements /// of an llvm.global_ctors/llvm.global_dtors instance. /// @@ -242,7 +253,7 @@ public: /// passes the 'Allow' predicate will be added to the JITDylib. class DynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator { public: - using SymbolPredicate = std::function<bool(SymbolStringPtr)>; + using SymbolPredicate = std::function<bool(const SymbolStringPtr &)>; /// Create a DynamicLibrarySearchGenerator that searches for symbols in the /// given sys::DynamicLibrary. @@ -268,8 +279,9 @@ public: return Load(nullptr, GlobalPrefix, std::move(Allow)); } - Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, - const SymbolNameSet &Names) override; + Error tryToGenerate(LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &Symbols) override; private: sys::DynamicLibrary Dylib; @@ -292,13 +304,14 @@ public: Load(ObjectLayer &L, const char *FileName); /// Try to create a StaticLibrarySearchGenerator from the given memory buffer. - /// Thhis call will succeed if the buffer contains a valid archive, otherwise + /// This call will succeed if the buffer contains a valid archive, otherwise /// it will return an error. static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer); - Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, - const SymbolNameSet &Names) override; + Error tryToGenerate(LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &Symbols) override; private: StaticLibraryDefinitionGenerator(ObjectLayer &L, @@ -307,8 +320,7 @@ private: ObjectLayer &L; std::unique_ptr<MemoryBuffer> ArchiveBuffer; - object::Archive Archive; - size_t UnrealizedObjects = 0; + std::unique_ptr<object::Archive> Archive; }; } // end namespace orc diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index a7ed5372d1e4..a9ab3a630a64 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -84,7 +84,7 @@ public: return std::move(LTP); } - /// Get a free trampoline. Returns an error if one can not be provide (e.g. + /// Get a free trampoline. Returns an error if one can not be provided (e.g. /// because the pool is empty and can not be grown). Expected<JITTargetAddress> getTrampoline() override { std::lock_guard<std::mutex> Lock(LTPMutex); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h b/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h index bcbd72e68f15..4f2f55770996 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h @@ -79,18 +79,30 @@ public: return *this; } + /// Get the relocation model. + const Optional<Reloc::Model> &getRelocationModel() const { return RM; } + /// Set the code model. JITTargetMachineBuilder &setCodeModel(Optional<CodeModel::Model> CM) { this->CM = std::move(CM); return *this; } + /// Get the code model. + const Optional<CodeModel::Model> &getCodeModel() const { return CM; } + /// Set the LLVM CodeGen optimization level. JITTargetMachineBuilder &setCodeGenOptLevel(CodeGenOpt::Level OptLevel) { this->OptLevel = OptLevel; return *this; } + /// Set subtarget features. + JITTargetMachineBuilder &setFeatures(StringRef FeatureString) { + Features = SubtargetFeatures(FeatureString); + return *this; + } + /// Add subtarget features. JITTargetMachineBuilder & addFeatures(const std::vector<std::string> &FeatureVec); @@ -101,6 +113,17 @@ public: /// Access subtarget features. const SubtargetFeatures &getFeatures() const { return Features; } + /// Set TargetOptions. + /// + /// Note: This operation will overwrite any previously configured options, + /// including EmulatedTLS and ExplicitEmulatedTLS which + /// the JITTargetMachineBuilder sets by default. Clients are responsible + /// for re-enabling these overwritten options. + JITTargetMachineBuilder &setOptions(TargetOptions Options) { + this->Options = std::move(Options); + return *this; + } + /// Access TargetOptions. TargetOptions &getOptions() { return Options; } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h index b1e47d77557c..c048ff3d5522 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -117,6 +117,9 @@ public: /// Returns a reference to the ObjLinkingLayer ObjectLayer &getObjLinkingLayer() { return *ObjLinkingLayer; } + /// Returns a reference to the object transform layer. + ObjectTransformLayer &getObjTransformLayer() { return ObjTransformLayer; } + protected: static std::unique_ptr<ObjectLayer> createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES); @@ -140,6 +143,7 @@ protected: std::unique_ptr<ThreadPool> CompileThreads; std::unique_ptr<ObjectLayer> ObjLinkingLayer; + ObjectTransformLayer ObjTransformLayer; std::unique_ptr<IRCompileLayer> CompileLayer; CtorDtorRunner CtorRunner, DtorRunner; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index caf8e707516d..50d25f18891e 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -79,7 +79,7 @@ public: /// Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyEmitted functors. ObjectLinkingLayer(ExecutionSession &ES, - jitlink::JITLinkMemoryManager &MemMgr); + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr); /// Destruct an ObjectLinkingLayer. ~ObjectLinkingLayer(); @@ -145,7 +145,7 @@ private: Error removeAllModules(); mutable std::mutex LayerMutex; - jitlink::JITLinkMemoryManager &MemMgr; + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr; bool OverrideObjectFlags = false; bool AutoClaimObjectSymbols = false; ReturnObjectBufferFunction ReturnObjectBuffer; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h index eac1cc3e097a..bf989cc8677c 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h @@ -29,11 +29,15 @@ public: std::unique_ptr<MemoryBuffer>)>; ObjectTransformLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, - TransformFunction Transform); + TransformFunction Transform = TransformFunction()); void emit(MaterializationResponsibility R, std::unique_ptr<MemoryBuffer> O) override; + void setTransform(TransformFunction Transform) { + this->Transform = std::move(Transform); + } + private: ObjectLayer &BaseLayer; TransformFunction Transform; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h index 38246bc480b6..2e58ddd75d31 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcABISupport.h @@ -29,7 +29,7 @@ namespace orc { /// Generic ORC ABI support. /// -/// This class can be substituted as the target architecure support class for +/// This class can be substituted as the target architecture support class for /// ORC templates that require one (e.g. IndirectStubsManagers). It does not /// support lazy JITing however, and any attempt to use that functionality /// will result in execution of an llvm_unreachable. @@ -71,7 +71,7 @@ public: }; /// Provide information about stub blocks generated by the -/// makeIndirectStubsBlock function. +/// makeIndirectStubsBlock function. template <unsigned StubSizeVal> class GenericIndirectStubsInfo { public: const static unsigned StubSize = StubSizeVal; @@ -95,13 +95,13 @@ public: unsigned getNumStubs() const { return NumStubs; } /// Get a pointer to the stub at the given index, which must be in - /// the range 0 .. getNumStubs() - 1. + /// the range 0 .. getNumStubs() - 1. void *getStub(unsigned Idx) const { return static_cast<char *>(StubsMem.base()) + Idx * StubSize; } /// Get a pointer to the implementation-pointer at the given index, - /// which must be in the range 0 .. getNumStubs() - 1. + /// which must be in the range 0 .. getNumStubs() - 1. void **getPtr(unsigned Idx) const { char *PtrsBase = static_cast<char *>(StubsMem.base()) + NumStubs * StubSize; return reinterpret_cast<void **>(PtrsBase) + Idx; @@ -123,19 +123,19 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// Write the resolver code into the given memory. The user is be - /// responsible for allocating the memory and setting permissions. + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); - /// Write the requsted number of trampolines into the given memory, - /// which must be big enough to hold 1 pointer, plus NumTrampolines - /// trampolines. + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, unsigned NumTrampolines); /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. + /// the nearest page size. /// /// E.g. Asking for 4 stubs on x86-64, where stubs are 8-bytes, with 4k /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 @@ -154,14 +154,14 @@ public: using IndirectStubsInfo = GenericIndirectStubsInfo<8>; - /// Write the requsted number of trampolines into the given memory, - /// which must be big enough to hold 1 pointer, plus NumTrampolines - /// trampolines. + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, unsigned NumTrampolines); /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. + /// the nearest page size. /// /// E.g. Asking for 4 stubs on x86-64, where stubs are 8-bytes, with 4k /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 @@ -180,8 +180,8 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// Write the resolver code into the given memory. The user is be - /// responsible for allocating the memory and setting permissions. + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); }; @@ -196,8 +196,8 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// Write the resolver code into the given memory. The user is be - /// responsible for allocating the memory and setting permissions. + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); }; @@ -216,19 +216,19 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// Write the resolver code into the given memory. The user is be - /// responsible for allocating the memory and setting permissions. + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry, void *CallbackMgr); - /// Write the requsted number of trampolines into the given memory, - /// which must be big enough to hold 1 pointer, plus NumTrampolines - /// trampolines. + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, unsigned NumTrampolines); /// Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. + /// the nearest page size. /// /// E.g. Asking for 4 stubs on i386, where stubs are 8-bytes, with 4k /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 @@ -249,16 +249,16 @@ public: using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// @brief Write the requsted number of trampolines into the given memory, - /// which must be big enough to hold 1 pointer, plus NumTrampolines - /// trampolines. + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr,unsigned NumTrampolines); - /// @brief Write the resolver code into the given memory. The user is be - /// responsible for allocating the memory and setting permissions. + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry,void *CallbackMgr, bool isBigEndian); - /// @brief Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. + /// Emit at least MinStubs worth of indirect call stubs, rounded out to + /// the nearest page size. /// /// E.g. Asking for 4 stubs on Mips32, where stubs are 8-bytes, with 4k /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 @@ -291,17 +291,17 @@ public: using IndirectStubsInfo = GenericIndirectStubsInfo<32>; using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr, void *TrampolineId); - /// @brief Write the resolver code into the given memory. The user is be - /// responsible for allocating the memory and setting permissions. + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. static void writeResolverCode(uint8_t *ResolveMem, JITReentryFn Reentry,void *CallbackMgr); - /// @brief Write the requsted number of trampolines into the given memory, - /// which must be big enough to hold 1 pointer, plus NumTrampolines - /// trampolines. + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. static void writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr,unsigned NumTrampolines); - /// @brief Emit at least MinStubs worth of indirect call stubs, rounded out to - /// the nearest page size. + /// Emit at least MinStubs worth of indirect call stubs, rounded out to + /// the nearest page size. /// /// E.g. Asking for 4 stubs on Mips64, where stubs are 8-bytes, with 4k /// pages will return a block of 512 stubs (4096 / 8 = 512). Asking for 513 diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h index e5d6a3eca85f..61e2e49a872a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcError.h @@ -14,6 +14,8 @@ #define LLVM_EXECUTIONENGINE_ORC_ORCERROR_H #include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include <string> #include <system_error> namespace llvm { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h index e7b598d8f812..3ff5a5f6e90e 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h @@ -16,8 +16,8 @@ #define LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETRPCAPI_H #include "llvm/ExecutionEngine/JITSymbol.h" -#include "llvm/ExecutionEngine/Orc/RPCUtils.h" -#include "llvm/ExecutionEngine/Orc/RawByteChannel.h" +#include "llvm/ExecutionEngine/Orc/RPC/RPCUtils.h" +#include "llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h" namespace llvm { namespace orc { diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h index 752a0a34e0a1..9c69a84f4c67 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h @@ -1,4 +1,4 @@ -//===- llvm/ExecutionEngine/Orc/RPCSerialization.h --------------*- C++ -*-===// +//===- llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h --------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -9,7 +9,7 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_RPCSERIALIZATION_H #define LLVM_EXECUTIONENGINE_ORC_RPCSERIALIZATION_H -#include "OrcError.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/Support/thread.h" #include <map> #include <mutex> diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h index ee9c2cc69c30..ed09363dcecc 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h @@ -23,7 +23,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" -#include "llvm/ExecutionEngine/Orc/RPCSerialization.h" +#include "llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h" #include "llvm/Support/MSVCErrorWorkarounds.h" #include <future> diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RawByteChannel.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h index 46b7c59450e6..50e26f8449df 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RawByteChannel.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h @@ -1,4 +1,4 @@ -//===- llvm/ExecutionEngine/Orc/RawByteChannel.h ----------------*- C++ -*-===// +//===- llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h ----------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -10,7 +10,7 @@ #define LLVM_EXECUTIONENGINE_ORC_RAWBYTECHANNEL_H #include "llvm/ADT/StringRef.h" -#include "llvm/ExecutionEngine/Orc/RPCSerialization.h" +#include "llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include <cstdint> diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index c5106cf09ecc..091394795c0b 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -54,6 +54,8 @@ public: RTDyldObjectLinkingLayer(ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager); + ~RTDyldObjectLinkingLayer(); + /// Emit the object. void emit(MaterializationResponsibility R, std::unique_ptr<MemoryBuffer> O) override; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h index 766a6b070f12..f6b86bb23167 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Speculation.h @@ -100,23 +100,27 @@ private: SymbolsInJD.insert(ImplSymbolName); } - DEBUG_WITH_TYPE("orc", for (auto &I - : SpeculativeLookUpImpls) { - llvm::dbgs() << "\n In " << I.first->getName() << " JITDylib "; - for (auto &N : I.second) - llvm::dbgs() << "\n Likely Symbol : " << N; + DEBUG_WITH_TYPE("orc", { + for (auto &I : SpeculativeLookUpImpls) { + llvm::dbgs() << "\n In " << I.first->getName() << " JITDylib "; + for (auto &N : I.second) + llvm::dbgs() << "\n Likely Symbol : " << N; + } }); // for a given symbol, there may be no symbol qualified for speculatively // compile try to fix this before jumping to this code if possible. for (auto &LookupPair : SpeculativeLookUpImpls) - ES.lookup(JITDylibSearchList({{LookupPair.first, true}}), - LookupPair.second, SymbolState::Ready, - [this](Expected<SymbolMap> Result) { - if (auto Err = Result.takeError()) - ES.reportError(std::move(Err)); - }, - NoDependenciesToRegister); + ES.lookup( + LookupKind::Static, + makeJITDylibSearchOrder(LookupPair.first, + JITDylibLookupFlags::MatchAllSymbols), + SymbolLookupSet(LookupPair.second), SymbolState::Ready, + [this](Expected<SymbolMap> Result) { + if (auto Err = Result.takeError()) + ES.reportError(std::move(Err)); + }, + NoDependenciesToRegister); } public: @@ -151,8 +155,11 @@ public: this->getES().reportError(ReadySymbol.takeError()); }; // Include non-exported symbols also. - ES.lookup(JITDylibSearchList({{JD, true}}), SymbolNameSet({Target}), - SymbolState::Ready, OnReadyFixUp, NoDependenciesToRegister); + ES.lookup( + LookupKind::Static, + makeJITDylibSearchOrder(JD, JITDylibLookupFlags::MatchAllSymbols), + SymbolLookupSet(Target, SymbolLookupFlags::WeaklyReferencedSymbol), + SymbolState::Ready, OnReadyFixUp, NoDependenciesToRegister); } } diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h new file mode 100644 index 000000000000..2f9a5ee71e67 --- /dev/null +++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h @@ -0,0 +1,109 @@ +//===- OMPConstants.h - OpenMP related constants and helpers ------ C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file defines constans and helpers used when dealing with OpenMP. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OPENMP_CONSTANTS_H +#define LLVM_OPENMP_CONSTANTS_H + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/StringRef.h" + +namespace llvm { +class Type; +class Module; +class StructType; +class PointerType; +class FunctionType; + +namespace omp { +LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); + +/// IDs for all OpenMP directives. +enum class Directive { +#define OMP_DIRECTIVE(Enum, ...) Enum, +#include "llvm/Frontend/OpenMP/OMPKinds.def" +}; + +/// Make the enum values available in the llvm::omp namespace. This allows us to +/// write something like OMPD_parallel if we have a `using namespace omp`. At +/// the same time we do not loose the strong type guarantees of the enum class, +/// that is we cannot pass an unsigned as Directive without an explicit cast. +#define OMP_DIRECTIVE(Enum, ...) constexpr auto Enum = omp::Directive::Enum; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + +/// IDs for all omp runtime library (RTL) functions. +enum class RuntimeFunction { +#define OMP_RTL(Enum, ...) Enum, +#include "llvm/Frontend/OpenMP/OMPKinds.def" +}; + +#define OMP_RTL(Enum, ...) constexpr auto Enum = omp::RuntimeFunction::Enum; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + +/// IDs for the different proc bind kinds. +enum class ProcBindKind { +#define OMP_PROC_BIND_KIND(Enum, Str, Value) Enum = Value, +#include "llvm/Frontend/OpenMP/OMPKinds.def" +}; + +#define OMP_PROC_BIND_KIND(Enum, ...) \ + constexpr auto Enum = omp::ProcBindKind::Enum; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + +/// IDs for all omp runtime library ident_t flag encodings (see +/// their defintion in openmp/runtime/src/kmp.h). +enum class IdentFlag { +#define OMP_IDENT_FLAG(Enum, Str, Value) Enum = Value, +#include "llvm/Frontend/OpenMP/OMPKinds.def" + LLVM_MARK_AS_BITMASK_ENUM(0x7FFFFFFF) +}; + +#define OMP_IDENT_FLAG(Enum, ...) constexpr auto Enum = omp::IdentFlag::Enum; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + +/// Parse \p Str and return the directive it matches or OMPD_unknown if none. +Directive getOpenMPDirectiveKind(StringRef Str); + +/// Return a textual representation of the directive \p D. +StringRef getOpenMPDirectiveName(Directive D); + +/// Forward declarations for LLVM-IR types (simple, function and structure) are +/// generated below. Their names are defined and used in OpenMP/OMPKinds.def. +/// Here we provide the forward declarations, the initializeTypes function will +/// provide the values. +/// +///{ +namespace types { + +#define OMP_TYPE(VarName, InitValue) extern Type *VarName; +#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \ + extern FunctionType *VarName; \ + extern PointerType *VarName##Ptr; +#define OMP_STRUCT_TYPE(VarName, StrName, ...) \ + extern StructType *VarName; \ + extern PointerType *VarName##Ptr; +#include "llvm/Frontend/OpenMP/OMPKinds.def" + +/// Helper to initialize all types defined in OpenMP/OMPKinds.def. +void initializeTypes(Module &M); + +/// Helper to uninitialize all types defined in OpenMP/OMPKinds.def. +void uninitializeTypes(); + +} // namespace types +///} + +} // end namespace omp + +} // end namespace llvm + +#endif // LLVM_OPENMP_CONSTANTS_H diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h new file mode 100644 index 000000000000..e1e1d5a30f3c --- /dev/null +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -0,0 +1,250 @@ +//===- IR/OpenMPIRBuilder.h - OpenMP encoding builder for LLVM IR - C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the OpenMPIRBuilder class and helpers used as a convenient +// way to create LLVM instructions for OpenMP directives. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OPENMP_IR_IRBUILDER_H +#define LLVM_OPENMP_IR_IRBUILDER_H + +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/Frontend/OpenMP/OMPConstants.h" + +namespace llvm { + +/// An interface to create LLVM-IR for OpenMP directives. +/// +/// Each OpenMP directive has a corresponding public generator method. +class OpenMPIRBuilder { +public: + /// Create a new OpenMPIRBuilder operating on the given module \p M. This will + /// not have an effect on \p M (see initialize). + OpenMPIRBuilder(Module &M) : M(M), Builder(M.getContext()) {} + + /// Initialize the internal state, this will put structures types and + /// potentially other helpers into the underlying module. Must be called + /// before any other method and only once! + void initialize(); + + /// Add attributes known for \p FnID to \p Fn. + void addAttributes(omp::RuntimeFunction FnID, Function &Fn); + + /// Type used throughout for insertion points. + using InsertPointTy = IRBuilder<>::InsertPoint; + + /// Callback type for variable finalization (think destructors). + /// + /// \param CodeGenIP is the insertion point at which the finalization code + /// should be placed. + /// + /// A finalize callback knows about all objects that need finalization, e.g. + /// destruction, when the scope of the currently generated construct is left + /// at the time, and location, the callback is invoked. + using FinalizeCallbackTy = std::function<void(InsertPointTy CodeGenIP)>; + + struct FinalizationInfo { + /// The finalization callback provided by the last in-flight invocation of + /// CreateXXXX for the directive of kind DK. + FinalizeCallbackTy FiniCB; + + /// The directive kind of the innermost directive that has an associated + /// region which might require finalization when it is left. + omp::Directive DK; + + /// Flag to indicate if the directive is cancellable. + bool IsCancellable; + }; + + /// Push a finalization callback on the finalization stack. + /// + /// NOTE: Temporary solution until Clang CG is gone. + void pushFinalizationCB(const FinalizationInfo &FI) { + FinalizationStack.push_back(FI); + } + + /// Pop the last finalization callback from the finalization stack. + /// + /// NOTE: Temporary solution until Clang CG is gone. + void popFinalizationCB() { FinalizationStack.pop_back(); } + + /// Callback type for body (=inner region) code generation + /// + /// The callback takes code locations as arguments, each describing a + /// location at which code might need to be generated or a location that is + /// the target of control transfer. + /// + /// \param AllocaIP is the insertion point at which new alloca instructions + /// should be placed. + /// \param CodeGenIP is the insertion point at which the body code should be + /// placed. + /// \param ContinuationBB is the basic block target to leave the body. + /// + /// Note that all blocks pointed to by the arguments have terminators. + using BodyGenCallbackTy = + function_ref<void(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, + BasicBlock &ContinuationBB)>; + + /// Callback type for variable privatization (think copy & default + /// constructor). + /// + /// \param AllocaIP is the insertion point at which new alloca instructions + /// should be placed. + /// \param CodeGenIP is the insertion point at which the privatization code + /// should be placed. + /// \param Val The value beeing copied/created. + /// \param ReplVal The replacement value, thus a copy or new created version + /// of \p Val. + /// + /// \returns The new insertion point where code generation continues and + /// \p ReplVal the replacement of \p Val. + using PrivatizeCallbackTy = function_ref<InsertPointTy( + InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &Val, + Value *&ReplVal)>; + + /// Description of a LLVM-IR insertion point (IP) and a debug/source location + /// (filename, line, column, ...). + struct LocationDescription { + template <typename T, typename U> + LocationDescription(const IRBuilder<T, U> &IRB) + : IP(IRB.saveIP()), DL(IRB.getCurrentDebugLocation()) {} + LocationDescription(const InsertPointTy &IP) : IP(IP) {} + LocationDescription(const InsertPointTy &IP, const DebugLoc &DL) + : IP(IP), DL(DL) {} + InsertPointTy IP; + DebugLoc DL; + }; + + /// Emitter methods for OpenMP directives. + /// + ///{ + + /// Generator for '#omp barrier' + /// + /// \param Loc The location where the barrier directive was encountered. + /// \param DK The kind of directive that caused the barrier. + /// \param ForceSimpleCall Flag to force a simple (=non-cancellation) barrier. + /// \param CheckCancelFlag Flag to indicate a cancel barrier return value + /// should be checked and acted upon. + /// + /// \returns The insertion point after the barrier. + InsertPointTy CreateBarrier(const LocationDescription &Loc, omp::Directive DK, + bool ForceSimpleCall = false, + bool CheckCancelFlag = true); + + /// Generator for '#omp cancel' + /// + /// \param Loc The location where the directive was encountered. + /// \param IfCondition The evaluated 'if' clause expression, if any. + /// \param CanceledDirective The kind of directive that is cancled. + /// + /// \returns The insertion point after the barrier. + InsertPointTy CreateCancel(const LocationDescription &Loc, + Value *IfCondition, + omp::Directive CanceledDirective); + + /// Generator for '#omp parallel' + /// + /// \param Loc The insert and source location description. + /// \param BodyGenCB Callback that will generate the region code. + /// \param PrivCB Callback to copy a given variable (think copy constructor). + /// \param FiniCB Callback to finalize variable copies. + /// \param IfCondition The evaluated 'if' clause expression, if any. + /// \param NumThreads The evaluated 'num_threads' clause expression, if any. + /// \param ProcBind The value of the 'proc_bind' clause (see ProcBindKind). + /// \param IsCancellable Flag to indicate a cancellable parallel region. + /// + /// \returns The insertion position *after* the parallel. + IRBuilder<>::InsertPoint + CreateParallel(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, + PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, + Value *IfCondition, Value *NumThreads, + omp::ProcBindKind ProcBind, bool IsCancellable); + + ///} + +private: + /// Update the internal location to \p Loc. + bool updateToLocation(const LocationDescription &Loc) { + Builder.restoreIP(Loc.IP); + Builder.SetCurrentDebugLocation(Loc.DL); + return Loc.IP.getBlock() != nullptr; + } + + /// Return the function declaration for the runtime function with \p FnID. + Function *getOrCreateRuntimeFunction(omp::RuntimeFunction FnID); + + /// Return the (LLVM-IR) string describing the source location \p LocStr. + Constant *getOrCreateSrcLocStr(StringRef LocStr); + + /// Return the (LLVM-IR) string describing the default source location. + Constant *getOrCreateDefaultSrcLocStr(); + + /// Return the (LLVM-IR) string describing the source location \p Loc. + Constant *getOrCreateSrcLocStr(const LocationDescription &Loc); + + /// Return an ident_t* encoding the source location \p SrcLocStr and \p Flags. + Value *getOrCreateIdent(Constant *SrcLocStr, + omp::IdentFlag Flags = omp::IdentFlag(0)); + + /// Generate control flow and cleanup for cancellation. + /// + /// \param CancelFlag Flag indicating if the cancellation is performed. + /// \param CanceledDirective The kind of directive that is cancled. + void emitCancelationCheckImpl(Value *CancelFlag, + omp::Directive CanceledDirective); + + /// Generate a barrier runtime call. + /// + /// \param Loc The location at which the request originated and is fulfilled. + /// \param DK The directive which caused the barrier + /// \param ForceSimpleCall Flag to force a simple (=non-cancellation) barrier. + /// \param CheckCancelFlag Flag to indicate a cancel barrier return value + /// should be checked and acted upon. + /// + /// \returns The insertion point after the barrier. + InsertPointTy emitBarrierImpl(const LocationDescription &Loc, + omp::Directive DK, bool ForceSimpleCall, + bool CheckCancelFlag); + + /// The finalization stack made up of finalize callbacks currently in-flight, + /// wrapped into FinalizationInfo objects that reference also the finalization + /// target block and the kind of cancellable directive. + SmallVector<FinalizationInfo, 8> FinalizationStack; + + /// Return true if the last entry in the finalization stack is of kind \p DK + /// and cancellable. + bool isLastFinalizationInfoCancellable(omp::Directive DK) { + return !FinalizationStack.empty() && + FinalizationStack.back().IsCancellable && + FinalizationStack.back().DK == DK; + } + + /// Return the current thread ID. + /// + /// \param Ident The ident (ident_t*) describing the query origin. + Value *getOrCreateThreadID(Value *Ident); + + /// The underlying LLVM-IR module + Module &M; + + /// The LLVM-IR Builder used to create IR. + IRBuilder<> Builder; + + /// Map to remember source location strings + StringMap<Constant *> SrcLocStrMap; + + /// Map to remember existing ident_t*. + DenseMap<std::pair<Constant *, uint64_t>, GlobalVariable *> IdentMap; +}; + +} // end namespace llvm + +#endif // LLVM_IR_IRBUILDER_H diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def new file mode 100644 index 000000000000..3ec27e5c08a8 --- /dev/null +++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def @@ -0,0 +1,289 @@ +//===--- OMPKinds.def - OpenMP directives, clauses, rt-calls -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file defines the list of supported OpenMP directives, clauses, runtime +/// calls, and other things that need to be listed in enums. +/// +//===----------------------------------------------------------------------===// + +/// OpenMP Directives and combined directives +/// +///{ + +#ifndef OMP_DIRECTIVE +#define OMP_DIRECTIVE(Enum, Str) +#endif + +#define __OMP_DIRECTIVE_EXT(Name, Str) OMP_DIRECTIVE(OMPD_##Name, Str) +#define __OMP_DIRECTIVE(Name) __OMP_DIRECTIVE_EXT(Name, #Name) + +__OMP_DIRECTIVE(threadprivate) +__OMP_DIRECTIVE(parallel) +__OMP_DIRECTIVE(task) +__OMP_DIRECTIVE(simd) +__OMP_DIRECTIVE(for) +__OMP_DIRECTIVE(sections) +__OMP_DIRECTIVE(section) +__OMP_DIRECTIVE(single) +__OMP_DIRECTIVE(master) +__OMP_DIRECTIVE(critical) +__OMP_DIRECTIVE(taskyield) +__OMP_DIRECTIVE(barrier) +__OMP_DIRECTIVE(taskwait) +__OMP_DIRECTIVE(taskgroup) +__OMP_DIRECTIVE(flush) +__OMP_DIRECTIVE(ordered) +__OMP_DIRECTIVE(atomic) +__OMP_DIRECTIVE(target) +__OMP_DIRECTIVE(teams) +__OMP_DIRECTIVE(cancel) +__OMP_DIRECTIVE(requires) +__OMP_DIRECTIVE_EXT(target_data, "target data") +__OMP_DIRECTIVE_EXT(target_enter_data, "target enter data") +__OMP_DIRECTIVE_EXT(target_exit_data, "target exit data") +__OMP_DIRECTIVE_EXT(target_parallel, "target parallel") +__OMP_DIRECTIVE_EXT(target_parallel_for, "target parallel for") +__OMP_DIRECTIVE_EXT(target_update, "target update") +__OMP_DIRECTIVE_EXT(parallel_for, "parallel for") +__OMP_DIRECTIVE_EXT(parallel_for_simd, "parallel for simd") +__OMP_DIRECTIVE_EXT(parallel_master, "parallel master") +__OMP_DIRECTIVE_EXT(parallel_sections, "parallel sections") +__OMP_DIRECTIVE_EXT(for_simd, "for simd") +__OMP_DIRECTIVE_EXT(cancellation_point, "cancellation point") +__OMP_DIRECTIVE_EXT(declare_reduction, "declare reduction") +__OMP_DIRECTIVE_EXT(declare_mapper, "declare mapper") +__OMP_DIRECTIVE_EXT(declare_simd, "declare simd") +__OMP_DIRECTIVE(taskloop) +__OMP_DIRECTIVE_EXT(taskloop_simd, "taskloop simd") +__OMP_DIRECTIVE(distribute) +__OMP_DIRECTIVE_EXT(declare_target, "declare target") +__OMP_DIRECTIVE_EXT(end_declare_target, "end declare target") +__OMP_DIRECTIVE_EXT(distribute_parallel_for, "distribute parallel for") +__OMP_DIRECTIVE_EXT(distribute_parallel_for_simd, + "distribute parallel for simd") +__OMP_DIRECTIVE_EXT(distribute_simd, "distribute simd") +__OMP_DIRECTIVE_EXT(target_parallel_for_simd, "target parallel for simd") +__OMP_DIRECTIVE_EXT(target_simd, "target simd") +__OMP_DIRECTIVE_EXT(teams_distribute, "teams distribute") +__OMP_DIRECTIVE_EXT(teams_distribute_simd, "teams distribute simd") +__OMP_DIRECTIVE_EXT(teams_distribute_parallel_for_simd, + "teams distribute parallel for simd") +__OMP_DIRECTIVE_EXT(teams_distribute_parallel_for, + "teams distribute parallel for") +__OMP_DIRECTIVE_EXT(target_teams, "target teams") +__OMP_DIRECTIVE_EXT(target_teams_distribute, "target teams distribute") +__OMP_DIRECTIVE_EXT(target_teams_distribute_parallel_for, + "target teams distribute parallel for") +__OMP_DIRECTIVE_EXT(target_teams_distribute_parallel_for_simd, + "target teams distribute parallel for simd") +__OMP_DIRECTIVE_EXT(target_teams_distribute_simd, + "target teams distribute simd") +__OMP_DIRECTIVE(allocate) +__OMP_DIRECTIVE_EXT(declare_variant, "declare variant") +__OMP_DIRECTIVE_EXT(master_taskloop, "master taskloop") +__OMP_DIRECTIVE_EXT(parallel_master_taskloop, "parallel master taskloop") +__OMP_DIRECTIVE_EXT(master_taskloop_simd, "master taskloop simd") +__OMP_DIRECTIVE_EXT(parallel_master_taskloop_simd, + "parallel master taskloop simd") + +// Has to be the last because Clang implicitly expects it to be. +__OMP_DIRECTIVE(unknown) + +#undef __OMP_DIRECTIVE_EXT +#undef __OMP_DIRECTIVE +#undef OMP_DIRECTIVE + +///} + +/// Types used in runtime structs or runtime functions +/// +///{ + +#ifndef OMP_TYPE +#define OMP_TYPE(VarName, InitValue) +#endif + +#define __OMP_TYPE(VarName) OMP_TYPE(VarName, Type::get##VarName##Ty(Ctx)) + +__OMP_TYPE(Void) +__OMP_TYPE(Int8) +__OMP_TYPE(Int32) +__OMP_TYPE(Int8Ptr) +__OMP_TYPE(Int32Ptr) + +#undef __OMP_TYPE +#undef OMP_TYPE + +///} + +/// Struct and function types +/// +///{ + +#ifndef OMP_STRUCT_TYPE +#define OMP_STRUCT_TYPE(VarName, StructName, ...) +#endif + +#define __OMP_STRUCT_TYPE(VarName, Name, ...) \ + OMP_STRUCT_TYPE(VarName, "struct." #Name, __VA_ARGS__) + +__OMP_STRUCT_TYPE(Ident, ident_t, Int32, Int32, Int32, Int32, Int8Ptr) + +#undef __OMP_STRUCT_TYPE +#undef OMP_STRUCT_TYPE + +#ifndef OMP_FUNCTION_TYPE +#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) +#endif + +#define __OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \ + OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, __VA_ARGS__) + +__OMP_FUNCTION_TYPE(ParallelTask, true, Void, Int32Ptr, Int32Ptr) + +#undef __OMP_FUNCTION_TYPE +#undef OMP_FUNCTION_TYPE + +///} + +/// Runtime library function (and their attributes) +/// +///{ + +#ifndef OMP_RTL +#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) +#endif + +#define __OMP_RTL(Name, IsVarArg, ReturnType, ...) \ + OMP_RTL(OMPRTL_##Name, #Name, IsVarArg, ReturnType, __VA_ARGS__) + +__OMP_RTL(__kmpc_barrier, false, Void, IdentPtr, Int32) +__OMP_RTL(__kmpc_cancel, false, Int32, IdentPtr, Int32, Int32) +__OMP_RTL(__kmpc_cancel_barrier, false, Int32, IdentPtr, Int32) +__OMP_RTL(__kmpc_global_thread_num, false, Int32, IdentPtr) +__OMP_RTL(__kmpc_fork_call, true, Void, IdentPtr, Int32, ParallelTaskPtr) +__OMP_RTL(__kmpc_push_num_threads, false, Void, IdentPtr, Int32, + /* Int */ Int32) +__OMP_RTL(__kmpc_push_proc_bind, false, Void, IdentPtr, Int32, /* Int */ Int32) +__OMP_RTL(__kmpc_serialized_parallel, false, Void, IdentPtr, Int32) +__OMP_RTL(__kmpc_end_serialized_parallel, false, Void, IdentPtr, Int32) + +__OMP_RTL(omp_get_thread_num, false, Int32, ) + +#undef __OMP_RTL +#undef OMP_RTL + +#define EnumAttr(Kind) Attribute::get(Ctx, Attribute::AttrKind::Kind) +#define AttributeSet(...) \ + AttributeSet::get(Ctx, ArrayRef<Attribute>({__VA_ARGS__})) + +#ifndef OMP_ATTRS_SET +#define OMP_ATTRS_SET(VarName, AttrSet) +#endif + +#define __OMP_ATTRS_SET(VarName, AttrSet) OMP_ATTRS_SET(VarName, AttrSet) + +__OMP_ATTRS_SET(GetterAttrs, + OptimisticAttributes + ? AttributeSet(EnumAttr(NoUnwind), EnumAttr(ReadOnly), + EnumAttr(NoSync), EnumAttr(NoFree)) + : AttributeSet(EnumAttr(NoUnwind))) + +#undef __OMP_ATTRS_SET +#undef OMP_ATTRS_SET + +#ifndef OMP_RTL_ATTRS +#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) +#endif + +#define __OMP_RTL_ATTRS(Name, FnAttrSet, RetAttrSet, ArgAttrSets) \ + OMP_RTL_ATTRS(OMPRTL_##Name, FnAttrSet, RetAttrSet, ArgAttrSets) + +__OMP_RTL_ATTRS(__kmpc_fork_call, AttributeSet(EnumAttr(NoUnwind)), + AttributeSet(), {}) + +__OMP_RTL_ATTRS(__kmpc_global_thread_num, GetterAttrs, AttributeSet(), {}) +__OMP_RTL_ATTRS(omp_get_thread_num, GetterAttrs, AttributeSet(), {}) + +#undef __OMP_RTL_ATTRS +#undef OMP_RTL_ATTRS +#undef AttributeSet +#undef EnumAttr + +///} + +/// KMP ident_t bit flags +/// +/// In accordance with the values in `openmp/runtime/src/kmp.h`. +/// +///{ + +#ifndef OMP_IDENT_FLAG +#define OMP_IDENT_FLAG(Enum, Str, Value) +#endif + +#define __OMP_IDENT_FLAG(Name, Value) \ + OMP_IDENT_FLAG(OMP_IDENT_FLAG_##Name, #Name, Value) + +__OMP_IDENT_FLAG(KMPC, 0x02) +__OMP_IDENT_FLAG(BARRIER_EXPL, 0x20) +__OMP_IDENT_FLAG(BARRIER_IMPL, 0x0040) +__OMP_IDENT_FLAG(BARRIER_IMPL_MASK, 0x01C0) +__OMP_IDENT_FLAG(BARRIER_IMPL_FOR, 0x0040) +__OMP_IDENT_FLAG(BARRIER_IMPL_SECTIONS, 0x00C0) +__OMP_IDENT_FLAG(BARRIER_IMPL_SINGLE, 0x0140) +__OMP_IDENT_FLAG(BARRIER_IMPL_WORKSHARE, 0x01C0) + +#undef __OMP_IDENT_FLAG +#undef OMP_IDENT_FLAG + +///} + +/// KMP cancel kind +/// +///{ + +#ifndef OMP_CANCEL_KIND +#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) +#endif + +#define __OMP_CANCEL_KIND(Name, Value) \ + OMP_CANCEL_KIND(OMP_CANCEL_KIND_##Name, #Name, OMPD_##Name, Value) + +__OMP_CANCEL_KIND(parallel, 1) +__OMP_CANCEL_KIND(for, 2) +__OMP_CANCEL_KIND(sections, 3) +__OMP_CANCEL_KIND(taskgroup, 4) + +#undef __OMP_CANCEL_KIND +#undef OMP_CANCEL_KIND + +///} + +/// Proc bind kinds +/// +///{ + +#ifndef OMP_PROC_BIND_KIND +#define OMP_PROC_BIND_KIND(Enum, Str, Value) +#endif + +#define __OMP_PROC_BIND_KIND(Name, Value) \ + OMP_PROC_BIND_KIND(OMP_PROC_BIND_##Name, #Name, Value) + +__OMP_PROC_BIND_KIND(master, 2) +__OMP_PROC_BIND_KIND(close, 3) +__OMP_PROC_BIND_KIND(spread, 4) +__OMP_PROC_BIND_KIND(default, 6) +__OMP_PROC_BIND_KIND(unknown, 7) + +#undef __OMP_PROC_BIND_KIND +#undef OMP_PROC_BIND_KIND + +///} diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h index 5f514b9c47d2..244878bd3155 100644 --- a/llvm/include/llvm/IR/Argument.h +++ b/llvm/include/llvm/IR/Argument.h @@ -76,8 +76,13 @@ public: bool hasByValOrInAllocaAttr() const; /// If this is a byval or inalloca argument, return its alignment. + /// FIXME: Remove this function once transition to Align is over. + /// Use getParamAlign() instead. unsigned getParamAlignment() const; + /// If this is a byval or inalloca argument, return its alignment. + MaybeAlign getParamAlign() const; + /// If this is a byval argument, return its type. Type *getParamByValType() const; diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index 153046d2311c..5d4a5f6743b7 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -220,6 +220,7 @@ def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">; def NoNansFPMath : StrBoolAttr<"no-nans-fp-math">; def UnsafeFPMath : StrBoolAttr<"unsafe-fp-math">; def NoJumpTables : StrBoolAttr<"no-jump-tables">; +def NoInlineLineTables : StrBoolAttr<"no-inline-line-tables">; def ProfileSampleAccurate : StrBoolAttr<"profile-sample-accurate">; class CompatRule<string F> { diff --git a/llvm/include/llvm/IR/AutoUpgrade.h b/llvm/include/llvm/IR/AutoUpgrade.h index 66f38e5b55d1..42f50cc991de 100644 --- a/llvm/include/llvm/IR/AutoUpgrade.h +++ b/llvm/include/llvm/IR/AutoUpgrade.h @@ -16,6 +16,7 @@ #include "llvm/ADT/StringRef.h" namespace llvm { + class AttrBuilder; class CallInst; class Constant; class Function; @@ -91,6 +92,10 @@ namespace llvm { /// pointers. std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple); + /// Upgrade function attributes "no-frame-pointer-elim" and + /// "no-frame-pointer-elim-non-leaf" to "frame-pointer". + void UpgradeFramePointerAttributes(AttrBuilder &B); + } // End llvm namespace #endif diff --git a/llvm/include/llvm/IR/CallSite.h b/llvm/include/llvm/IR/CallSite.h index 13b1ae8d0e32..0e957c4797e8 100644 --- a/llvm/include/llvm/IR/CallSite.h +++ b/llvm/include/llvm/IR/CallSite.h @@ -45,7 +45,7 @@ namespace llvm { namespace Intrinsic { -enum ID : unsigned; +typedef unsigned ID; } template <typename FunTy = const Function, typename BBTy = const BasicBlock, @@ -693,6 +693,18 @@ private: User::op_iterator getCallee() const; }; +/// Establish a view to a call site for examination. +class ImmutableCallSite : public CallSiteBase<> { +public: + ImmutableCallSite() = default; + ImmutableCallSite(const CallInst *CI) : CallSiteBase(CI) {} + ImmutableCallSite(const InvokeInst *II) : CallSiteBase(II) {} + ImmutableCallSite(const CallBrInst *CBI) : CallSiteBase(CBI) {} + explicit ImmutableCallSite(const Instruction *II) : CallSiteBase(II) {} + explicit ImmutableCallSite(const Value *V) : CallSiteBase(V) {} + ImmutableCallSite(CallSite CS) : CallSiteBase(CS.getInstruction()) {} +}; + /// AbstractCallSite /// /// An abstract call site is a wrapper that allows to treat direct, @@ -765,6 +777,13 @@ public: /// as well as the callee of the abstract call site. AbstractCallSite(const Use *U); + /// Add operand uses of \p ICS that represent callback uses into \p CBUses. + /// + /// All uses added to \p CBUses can be used to create abstract call sites for + /// which AbstractCallSite::isCallbackCall() will return true. + static void getCallbackUses(ImmutableCallSite ICS, + SmallVectorImpl<const Use *> &CBUses); + /// Conversion operator to conveniently check for a valid/initialized ACS. explicit operator bool() const { return (bool)CS; } @@ -850,7 +869,7 @@ public: /// callee of this ACS. Only valid for callback calls! int getCallArgOperandNoForCallee() const { assert(isCallbackCall()); - assert(CI.ParameterEncoding.size() && CI.ParameterEncoding[0] > 0); + assert(CI.ParameterEncoding.size() && CI.ParameterEncoding[0] >= 0); return CI.ParameterEncoding[0]; } @@ -902,18 +921,6 @@ template <> struct DenseMapInfo<CallSite> { } }; -/// Establish a view to a call site for examination. -class ImmutableCallSite : public CallSiteBase<> { -public: - ImmutableCallSite() = default; - ImmutableCallSite(const CallInst *CI) : CallSiteBase(CI) {} - ImmutableCallSite(const InvokeInst *II) : CallSiteBase(II) {} - ImmutableCallSite(const CallBrInst *CBI) : CallSiteBase(CBI) {} - explicit ImmutableCallSite(const Instruction *II) : CallSiteBase(II) {} - explicit ImmutableCallSite(const Value *V) : CallSiteBase(V) {} - ImmutableCallSite(CallSite CS) : CallSiteBase(CS.getInstruction()) {} -}; - } // end namespace llvm #endif // LLVM_IR_CALLSITE_H diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h index c1c979c2e2ab..d0906de3ea4e 100644 --- a/llvm/include/llvm/IR/CallingConv.h +++ b/llvm/include/llvm/IR/CallingConv.h @@ -80,6 +80,12 @@ namespace CallingConv { /// be performed. Tail = 18, + /// Special calling convention on Windows for calling the Control + /// Guard Check ICall funtion. The function takes exactly one argument + /// (address of the target function) passed in the first argument register, + /// and has no return value. All register values are preserved. + CFGuard_Check = 19, + // Target - This is the start of the target-specific calling conventions, // e.g. fastcall and thiscall on X86. FirstTargetCC = 64, diff --git a/llvm/include/llvm/IR/Constant.h b/llvm/include/llvm/IR/Constant.h index 2b6a6e4141b9..174e7364c524 100644 --- a/llvm/include/llvm/IR/Constant.h +++ b/llvm/include/llvm/IR/Constant.h @@ -53,6 +53,10 @@ public: /// Returns true if the value is one. bool isOneValue() const; + /// Return true if the value is not the one value, or, + /// for vectors, does not contain one value elements. + bool isNotOneValue() const; + /// Return true if this is the value that would be returned by /// getAllOnesValue. bool isAllOnesValue() const; @@ -64,7 +68,8 @@ public: /// Return true if the value is negative zero or null value. bool isZeroValue() const; - /// Return true if the value is not the smallest signed value. + /// Return true if the value is not the smallest signed value, or, + /// for vectors, does not contain smallest signed value elements. bool isNotMinSignedValue() const; /// Return true if the value is the smallest signed value. @@ -128,9 +133,10 @@ public: Constant *getAggregateElement(unsigned Elt) const; Constant *getAggregateElement(Constant *Elt) const; - /// If this is a splat vector constant, meaning that all of the elements have - /// the same value, return that value. Otherwise return 0. - Constant *getSplatValue() const; + /// If all elements of the vector constant have the same value, return that + /// value. Otherwise, return nullptr. Ignore undefined elements by setting + /// AllowUndefs to true. + Constant *getSplatValue(bool AllowUndefs = false) const; /// If C is a constant integer then return its value, otherwise C must be a /// vector of constant integers, all equal, and the common value is returned. @@ -188,6 +194,10 @@ public: return const_cast<Constant*>( static_cast<const Constant *>(this)->stripPointerCasts()); } + + /// Try to replace undefined constant C or undefined elements in C with + /// Replacement. If no changes are made, the constant C is returned. + static Constant *replaceUndefsWith(Constant *C, Constant *Replacement); }; } // end namespace llvm diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h index 964f9e8e9bc9..e6bac8a5f933 100644 --- a/llvm/include/llvm/IR/ConstantRange.h +++ b/llvm/include/llvm/IR/ConstantRange.h @@ -327,6 +327,14 @@ public: const ConstantRange &Other) const; /// Return a new range representing the possible values resulting + /// from an application of the specified overflowing binary operator to a + /// left hand side of this range and a right hand side of \p Other given + /// the provided knowledge about lack of wrapping \p NoWrapKind. + ConstantRange overflowingBinaryOp(Instruction::BinaryOps BinOp, + const ConstantRange &Other, + unsigned NoWrapKind) const; + + /// Return a new range representing the possible values resulting /// from an addition of a value in this range and a value in \p Other. ConstantRange add(const ConstantRange &Other) const; @@ -343,6 +351,14 @@ public: ConstantRange sub(const ConstantRange &Other) const; /// Return a new range representing the possible values resulting + /// from an subtraction with wrap type \p NoWrapKind of a value in this + /// range and a value in \p Other. + /// If the result range is disjoint, the preferred range is determined by the + /// \p PreferredRangeType. + ConstantRange subWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind, + PreferredRangeType RangeType = Smallest) const; + + /// Return a new range representing the possible values resulting /// from a multiplication of a value in this range and a value in \p Other, /// treating both this and \p Other as unsigned ranges. ConstantRange multiply(const ConstantRange &Other) const; @@ -418,6 +434,20 @@ public: /// Perform a signed saturating subtraction of two constant ranges. ConstantRange ssub_sat(const ConstantRange &Other) const; + /// Perform an unsigned saturating multiplication of two constant ranges. + ConstantRange umul_sat(const ConstantRange &Other) const; + + /// Perform a signed saturating multiplication of two constant ranges. + ConstantRange smul_sat(const ConstantRange &Other) const; + + /// Perform an unsigned saturating left shift of this constant range by a + /// value in \p Other. + ConstantRange ushl_sat(const ConstantRange &Other) const; + + /// Perform a signed saturating left shift of this constant range by a + /// value in \p Other. + ConstantRange sshl_sat(const ConstantRange &Other) const; + /// Return a new range that is the logical not of the current set. ConstantRange inverse() const; diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index ca56e8b9328c..262ab439df65 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -522,9 +522,10 @@ public: return cast<VectorType>(Value::getType()); } - /// If this is a splat constant, meaning that all of the elements have the - /// same value, return that value. Otherwise return NULL. - Constant *getSplatValue() const; + /// If all elements of the vector constant have the same value, return that + /// value. Otherwise, return nullptr. Ignore undefined elements by setting + /// AllowUndefs to true. + Constant *getSplatValue(bool AllowUndefs = false) const; /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Value *V) { @@ -1250,7 +1251,7 @@ public: /// which would take a ConstantExpr parameter, but that would have spread /// implementation details of ConstantExpr outside of Constants.cpp, which /// would make it harder to remove ConstantExprs altogether. - Instruction *getAsInstruction(); + Instruction *getAsInstruction() const; /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Value *V) { diff --git a/llvm/include/llvm/IR/ConstrainedOps.def b/llvm/include/llvm/IR/ConstrainedOps.def new file mode 100644 index 000000000000..7e24684ca654 --- /dev/null +++ b/llvm/include/llvm/IR/ConstrainedOps.def @@ -0,0 +1,86 @@ +//===- llvm/IR/ConstrainedOps.def - Constrained intrinsics ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines properties of constrained intrinsics, in particular corresponding +// floating point operations and DAG nodes. +// +//===----------------------------------------------------------------------===// + +#ifndef INSTRUCTION +#define INSTRUCTION(N,A,R,I,D) +#endif + +// In most cases intrinsic function is handled similar to instruction. +#ifndef FUNCTION +#define FUNCTION INSTRUCTION +#endif + +// Likewise for compare instructions. +#ifndef CMP_INSTRUCTION +#define CMP_INSTRUCTION INSTRUCTION +#endif + +// Arguments of the entries are: +// - instruction or intrinsic function name. +// - Number of original instruction/intrinsic arguments. +// - 1 if the corresponding constrained intrinsic has rounding mode argument. +// - name of the constrained intrinsic to represent this instruction/function. +// - DAG node corresponding to the constrained intrinsic without prefix STRICT_. + +// These are definitions for instructions, that are converted into constrained +// intrinsics. +// +INSTRUCTION(FAdd, 2, 1, experimental_constrained_fadd, FADD) +INSTRUCTION(FSub, 2, 1, experimental_constrained_fsub, FSUB) +INSTRUCTION(FMul, 2, 1, experimental_constrained_fmul, FMUL) +INSTRUCTION(FDiv, 2, 1, experimental_constrained_fdiv, FDIV) +INSTRUCTION(FRem, 2, 1, experimental_constrained_frem, FREM) +INSTRUCTION(FPExt, 1, 0, experimental_constrained_fpext, FP_EXTEND) +INSTRUCTION(SIToFP, 1, 1, experimental_constrained_sitofp, SINT_TO_FP) +INSTRUCTION(UIToFP, 1, 1, experimental_constrained_uitofp, UINT_TO_FP) +INSTRUCTION(FPToSI, 1, 0, experimental_constrained_fptosi, FP_TO_SINT) +INSTRUCTION(FPToUI, 1, 0, experimental_constrained_fptoui, FP_TO_UINT) +INSTRUCTION(FPTrunc, 1, 1, experimental_constrained_fptrunc, FP_ROUND) + +// These are definitions for compare instructions (signaling and quiet version). +// Both of these match to FCmp / SETCC. +CMP_INSTRUCTION(FCmp, 2, 0, experimental_constrained_fcmp, FSETCC) +CMP_INSTRUCTION(FCmp, 2, 0, experimental_constrained_fcmps, FSETCCS) + +// Theses are definitions for intrinsic functions, that are converted into +// constrained intrinsics. +// +FUNCTION(ceil, 1, 0, experimental_constrained_ceil, FCEIL) +FUNCTION(cos, 1, 1, experimental_constrained_cos, FCOS) +FUNCTION(exp, 1, 1, experimental_constrained_exp, FEXP) +FUNCTION(exp2, 1, 1, experimental_constrained_exp2, FEXP2) +FUNCTION(floor, 1, 0, experimental_constrained_floor, FFLOOR) +FUNCTION(fma, 3, 1, experimental_constrained_fma, FMA) +FUNCTION(log, 1, 1, experimental_constrained_log, FLOG) +FUNCTION(log10, 1, 1, experimental_constrained_log10, FLOG10) +FUNCTION(log2, 1, 1, experimental_constrained_log2, FLOG2) +FUNCTION(lrint, 1, 1, experimental_constrained_lrint, LRINT) +FUNCTION(llrint, 1, 1, experimental_constrained_llrint, LLRINT) +FUNCTION(lround, 1, 0, experimental_constrained_lround, LROUND) +FUNCTION(llround, 1, 0, experimental_constrained_llround, LLROUND) +FUNCTION(maxnum, 2, 0, experimental_constrained_maxnum, FMAXNUM) +FUNCTION(minnum, 2, 0, experimental_constrained_minnum, FMINNUM) +FUNCTION(maximum, 2, 0, experimental_constrained_maximum, FMAXIMUM) +FUNCTION(minimum, 2, 0, experimental_constrained_minimum, FMINIMUM) +FUNCTION(nearbyint, 1, 1, experimental_constrained_nearbyint, FNEARBYINT) +FUNCTION(pow, 2, 1, experimental_constrained_pow, FPOW) +FUNCTION(powi, 2, 1, experimental_constrained_powi, FPOWI) +FUNCTION(rint, 1, 1, experimental_constrained_rint, FRINT) +FUNCTION(round, 1, 0, experimental_constrained_round, FROUND) +FUNCTION(sin, 1, 1, experimental_constrained_sin, FSIN) +FUNCTION(sqrt, 1, 1, experimental_constrained_sqrt, FSQRT) +FUNCTION(trunc, 1, 0, experimental_constrained_trunc, FTRUNC) + +#undef INSTRUCTION +#undef FUNCTION +#undef CMP_INSTRUCTION diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h index ad9a35b55414..f7c242554f6a 100644 --- a/llvm/include/llvm/IR/DIBuilder.h +++ b/llvm/include/llvm/IR/DIBuilder.h @@ -237,8 +237,10 @@ namespace llvm { /// \param File File where this type is defined. /// \param LineNo Line number. /// \param Context The surrounding context for the typedef. + /// \param AlignInBits Alignment. (optional) DIDerivedType *createTypedef(DIType *Ty, StringRef Name, DIFile *File, - unsigned LineNo, DIScope *Context); + unsigned LineNo, DIScope *Context, + uint32_t AlignInBits = 0); /// Create debugging information entry for a 'friend'. DIDerivedType *createFriend(DIType *Ty, DIType *FriendTy); @@ -572,7 +574,7 @@ namespace llvm { /// \param File File where this variable is defined. /// \param LineNo Line number. /// \param Ty Variable Type. - /// \param isLocalToUnit Boolean flag indicate whether this variable is + /// \param IsLocalToUnit Boolean flag indicate whether this variable is /// externally visible or not. /// \param Expr The location of the global relative to the attached /// GlobalVariable. @@ -581,16 +583,16 @@ namespace llvm { /// specified) DIGlobalVariableExpression *createGlobalVariableExpression( DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File, - unsigned LineNo, DIType *Ty, bool isLocalToUnit, + unsigned LineNo, DIType *Ty, bool IsLocalToUnit, bool isDefined = true, DIExpression *Expr = nullptr, MDNode *Decl = nullptr, - MDTuple *templateParams = nullptr, uint32_t AlignInBits = 0); + MDTuple *TemplateParams = nullptr, uint32_t AlignInBits = 0); /// Identical to createGlobalVariable /// except that the resulting DbgNode is temporary and meant to be RAUWed. DIGlobalVariable *createTempGlobalVariableFwdDecl( DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File, - unsigned LineNo, DIType *Ty, bool isLocalToUnit, MDNode *Decl = nullptr, - MDTuple *templateParams = nullptr, uint32_t AlignInBits = 0); + unsigned LineNo, DIType *Ty, bool IsLocalToUnit, MDNode *Decl = nullptr, + MDTuple *TemplateParams= nullptr, uint32_t AlignInBits = 0); /// Create a new descriptor for an auto variable. This is a local variable /// that is not a subprogram parameter. @@ -732,11 +734,11 @@ namespace llvm { /// A space-separated shell-quoted list of -D macro /// definitions as they would appear on a command line. /// \param IncludePath The path to the module map file. - /// \param ISysRoot The clang system root (value of -isysroot). + /// \param SysRoot The clang system root (value of -isysroot). DIModule *createModule(DIScope *Scope, StringRef Name, StringRef ConfigurationMacros, StringRef IncludePath, - StringRef ISysRoot); + StringRef SysRoot); /// This creates a descriptor for a lexical block with a new file /// attached. This merely extends the existing diff --git a/llvm/include/llvm/IR/DebugInfoFlags.def b/llvm/include/llvm/IR/DebugInfoFlags.def index f90c580f10ef..df375b6c68e8 100644 --- a/llvm/include/llvm/IR/DebugInfoFlags.def +++ b/llvm/include/llvm/IR/DebugInfoFlags.def @@ -50,7 +50,6 @@ HANDLE_DI_FLAG((3 << 16), VirtualInheritance) HANDLE_DI_FLAG((1 << 18), IntroducedVirtual) HANDLE_DI_FLAG((1 << 19), BitField) HANDLE_DI_FLAG((1 << 20), NoReturn) -HANDLE_DI_FLAG((1 << 21), ArgumentNotModified) HANDLE_DI_FLAG((1 << 22), TypePassByValue) HANDLE_DI_FLAG((1 << 23), TypePassByReference) HANDLE_DI_FLAG((1 << 24), EnumClass) @@ -88,11 +87,15 @@ HANDLE_DISP_FLAG((1u << 5), Pure) HANDLE_DISP_FLAG((1u << 6), Elemental) HANDLE_DISP_FLAG((1u << 7), Recursive) HANDLE_DISP_FLAG((1u << 8), MainSubprogram) +// May also utilize this Flag in future, when adding support +// for defaulted functions +HANDLE_DISP_FLAG((1u << 9), Deleted) +HANDLE_DISP_FLAG((1u << 11), ObjCDirect) #ifdef DISP_FLAG_LARGEST_NEEDED // Intended to be used with ADT/BitmaskEnum.h. // NOTE: Always must be equal to largest flag, check this when adding new flags. -HANDLE_DISP_FLAG((1 << 8), Largest) +HANDLE_DISP_FLAG((1 << 11), Largest) #undef DISP_FLAG_LARGEST_NEEDED #endif diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h index 28a59576b7c6..d6bfe504dd94 100644 --- a/llvm/include/llvm/IR/DebugInfoMetadata.h +++ b/llvm/include/llvm/IR/DebugInfoMetadata.h @@ -1758,6 +1758,13 @@ public: bool isPure() const { return getSPFlags() & SPFlagPure; } bool isElemental() const { return getSPFlags() & SPFlagElemental; } bool isRecursive() const { return getSPFlags() & SPFlagRecursive; } + bool isObjCDirect() const { return getSPFlags() & SPFlagObjCDirect; } + + /// Check if this is deleted member function. + /// + /// Return true if this subprogram is a C++11 special + /// member function declared deleted. + bool isDeleted() const { return getSPFlags() & SPFlagDeleted; } /// Check if this is reference-qualified. /// @@ -2077,34 +2084,34 @@ class DIModule : public DIScope { static DIModule *getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name, StringRef ConfigurationMacros, - StringRef IncludePath, StringRef ISysRoot, + StringRef IncludePath, StringRef SysRoot, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, Scope, getCanonicalMDString(Context, Name), getCanonicalMDString(Context, ConfigurationMacros), getCanonicalMDString(Context, IncludePath), - getCanonicalMDString(Context, ISysRoot), + getCanonicalMDString(Context, SysRoot), Storage, ShouldCreate); } static DIModule *getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, MDString *ConfigurationMacros, - MDString *IncludePath, MDString *ISysRoot, + MDString *IncludePath, MDString *SysRoot, StorageType Storage, bool ShouldCreate = true); TempDIModule cloneImpl() const { return getTemporary(getContext(), getScope(), getName(), getConfigurationMacros(), getIncludePath(), - getISysRoot()); + getSysRoot()); } public: DEFINE_MDNODE_GET(DIModule, (DIScope *Scope, StringRef Name, StringRef ConfigurationMacros, StringRef IncludePath, - StringRef ISysRoot), - (Scope, Name, ConfigurationMacros, IncludePath, ISysRoot)) + StringRef SysRoot), + (Scope, Name, ConfigurationMacros, IncludePath, SysRoot)) DEFINE_MDNODE_GET(DIModule, (Metadata *Scope, MDString *Name, MDString *ConfigurationMacros, - MDString *IncludePath, MDString *ISysRoot), - (Scope, Name, ConfigurationMacros, IncludePath, ISysRoot)) + MDString *IncludePath, MDString *SysRoot), + (Scope, Name, ConfigurationMacros, IncludePath, SysRoot)) TempDIModule clone() const { return cloneImpl(); } @@ -2112,13 +2119,13 @@ public: StringRef getName() const { return getStringOperand(1); } StringRef getConfigurationMacros() const { return getStringOperand(2); } StringRef getIncludePath() const { return getStringOperand(3); } - StringRef getISysRoot() const { return getStringOperand(4); } + StringRef getSysRoot() const { return getStringOperand(4); } Metadata *getRawScope() const { return getOperand(0); } MDString *getRawName() const { return getOperandAs<MDString>(1); } MDString *getRawConfigurationMacros() const { return getOperandAs<MDString>(2); } MDString *getRawIncludePath() const { return getOperandAs<MDString>(3); } - MDString *getRawISysRoot() const { return getOperandAs<MDString>(4); } + MDString *getRawSysRoot() const { return getOperandAs<MDString>(4); } static bool classof(const Metadata *MD) { return MD->getMetadataID() == DIModuleKind; @@ -2545,6 +2552,16 @@ public: return 0; } + using ExtOps = std::array<uint64_t, 6>; + + /// Returns the ops for a zero- or sign-extension in a DIExpression. + static ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed); + + /// Append a zero- or sign-extension to \p Expr. Converts the expression to a + /// stack value if it isn't one already. + static DIExpression *appendExt(const DIExpression *Expr, unsigned FromSize, + unsigned ToSize, bool Signed); + /// Check if fragments overlap between a pair of FragmentInfos. static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B) { return fragmentCmp(A, B) == 0; @@ -2809,11 +2826,6 @@ public: bool isArtificial() const { return getFlags() & FlagArtificial; } bool isObjectPointer() const { return getFlags() & FlagObjectPointer; } - /// Check that an argument is unmodified. - bool isNotModified() const { return getFlags() & FlagArgumentNotModified; } - /// Set the flag if an argument is unmodified. - void setIsNotModified() { Flags |= FlagArgumentNotModified; } - /// Check that a location is valid for this variable. /// /// Check that \c DL exists, is in the same subprogram, and has the same @@ -3247,6 +3259,89 @@ public: } }; +/// Identifies a unique instance of a variable. +/// +/// Storage for identifying a potentially inlined instance of a variable, +/// or a fragment thereof. This guarantees that exactly one variable instance +/// may be identified by this class, even when that variable is a fragment of +/// an aggregate variable and/or there is another inlined instance of the same +/// source code variable nearby. +/// This class does not necessarily uniquely identify that variable: it is +/// possible that a DebugVariable with different parameters may point to the +/// same variable instance, but not that one DebugVariable points to multiple +/// variable instances. +class DebugVariable { + using FragmentInfo = DIExpression::FragmentInfo; + + const DILocalVariable *Variable; + Optional<FragmentInfo> Fragment; + const DILocation *InlinedAt; + + /// Fragment that will overlap all other fragments. Used as default when + /// caller demands a fragment. + static const FragmentInfo DefaultFragment; + +public: + DebugVariable(const DILocalVariable *Var, Optional<FragmentInfo> FragmentInfo, + const DILocation *InlinedAt) + : Variable(Var), Fragment(FragmentInfo), InlinedAt(InlinedAt) {} + + DebugVariable(const DILocalVariable *Var, const DIExpression *DIExpr, + const DILocation *InlinedAt) + : Variable(Var), + Fragment(DIExpr ? DIExpr->getFragmentInfo() : NoneType()), + InlinedAt(InlinedAt) {} + + const DILocalVariable *getVariable() const { return Variable; } + const Optional<FragmentInfo> getFragment() const { return Fragment; } + const DILocation *getInlinedAt() const { return InlinedAt; } + + const FragmentInfo getFragmentOrDefault() const { + return Fragment.getValueOr(DefaultFragment); + } + + static bool isDefaultFragment(const FragmentInfo F) { + return F == DefaultFragment; + } + + bool operator==(const DebugVariable &Other) const { + return std::tie(Variable, Fragment, InlinedAt) == + std::tie(Other.Variable, Other.Fragment, Other.InlinedAt); + } + + bool operator<(const DebugVariable &Other) const { + return std::tie(Variable, Fragment, InlinedAt) < + std::tie(Other.Variable, Other.Fragment, Other.InlinedAt); + } +}; + +template <> struct DenseMapInfo<DebugVariable> { + using FragmentInfo = DIExpression::FragmentInfo; + + /// Empty key: no key should be generated that has no DILocalVariable. + static inline DebugVariable getEmptyKey() { + return DebugVariable(nullptr, NoneType(), nullptr); + } + + /// Difference in tombstone is that the Optional is meaningful. + static inline DebugVariable getTombstoneKey() { + return DebugVariable(nullptr, {{0, 0}}, nullptr); + } + + static unsigned getHashValue(const DebugVariable &D) { + unsigned HV = 0; + const Optional<FragmentInfo> Fragment = D.getFragment(); + if (Fragment) + HV = DenseMapInfo<FragmentInfo>::getHashValue(*Fragment); + + return hash_combine(D.getVariable(), HV, D.getInlinedAt()); + } + + static bool isEqual(const DebugVariable &A, const DebugVariable &B) { + return A == B; + } +}; + } // end namespace llvm #undef DEFINE_MDNODE_GET_UNPACK_IMPL diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h index fef1c6abf8c2..6a14785a6cc3 100644 --- a/llvm/include/llvm/IR/Dominators.h +++ b/llvm/include/llvm/IR/Dominators.h @@ -262,9 +262,7 @@ class DominatorTreeWrapperPass : public FunctionPass { public: static char ID; - DominatorTreeWrapperPass() : FunctionPass(ID) { - initializeDominatorTreeWrapperPassPass(*PassRegistry::getPassRegistry()); - } + DominatorTreeWrapperPass(); DominatorTree &getDomTree() { return DT; } const DominatorTree &getDomTree() const { return DT; } diff --git a/llvm/include/llvm/IR/FPEnv.h b/llvm/include/llvm/IR/FPEnv.h new file mode 100644 index 000000000000..a1e0665d4112 --- /dev/null +++ b/llvm/include/llvm/IR/FPEnv.h @@ -0,0 +1,70 @@ +//===- FPEnv.h ---- FP Environment ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// @file +/// This file contains the declarations of entities that describe floating +/// point environment and related functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_FLOATINGPOINT_H +#define LLVM_IR_FLOATINGPOINT_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include <stdint.h> + +namespace llvm { + +namespace fp { + +/// Rounding mode used for floating point operations. +/// +/// Each of these values correspond to some metadata argument value of a +/// constrained floating point intrinsic. See the LLVM Language Reference Manual +/// for details. +enum RoundingMode : uint8_t { + rmDynamic, ///< This corresponds to "fpround.dynamic". + rmToNearest, ///< This corresponds to "fpround.tonearest". + rmDownward, ///< This corresponds to "fpround.downward". + rmUpward, ///< This corresponds to "fpround.upward". + rmTowardZero ///< This corresponds to "fpround.tozero". +}; + +/// Exception behavior used for floating point operations. +/// +/// Each of these values correspond to some metadata argument value of a +/// constrained floating point intrinsic. See the LLVM Language Reference Manual +/// for details. +enum ExceptionBehavior : uint8_t { + ebIgnore, ///< This corresponds to "fpexcept.ignore". + ebMayTrap, ///< This corresponds to "fpexcept.maytrap". + ebStrict ///< This corresponds to "fpexcept.strict". +}; + +} + +/// Returns a valid RoundingMode enumerator when given a string +/// that is valid as input in constrained intrinsic rounding mode +/// metadata. +Optional<fp::RoundingMode> StrToRoundingMode(StringRef); + +/// For any RoundingMode enumerator, returns a string valid as input in +/// constrained intrinsic rounding mode metadata. +Optional<StringRef> RoundingModeToStr(fp::RoundingMode); + +/// Returns a valid ExceptionBehavior enumerator when given a string +/// valid as input in constrained intrinsic exception behavior metadata. +Optional<fp::ExceptionBehavior> StrToExceptionBehavior(StringRef); + +/// For any ExceptionBehavior enumerator, returns a string valid as +/// input in constrained intrinsic exception behavior metadata. +Optional<StringRef> ExceptionBehaviorToStr(fp::ExceptionBehavior); + +} +#endif diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h index d586a9460d2b..d9cbcc63fa62 100644 --- a/llvm/include/llvm/IR/Function.h +++ b/llvm/include/llvm/IR/Function.h @@ -43,7 +43,7 @@ namespace llvm { namespace Intrinsic { -enum ID : unsigned; +typedef unsigned ID; } class AssemblyAnnotationWriter; @@ -435,12 +435,18 @@ public: void addDereferenceableOrNullParamAttr(unsigned ArgNo, uint64_t Bytes); /// Extract the alignment for a call or parameter (0=unknown). + /// FIXME: Remove this function once transition to Align is over. + /// Use getParamAlign() instead. unsigned getParamAlignment(unsigned ArgNo) const { - if (const auto MA = AttributeSets.getParamAlignment(ArgNo)) + if (const auto MA = getParamAlign(ArgNo)) return MA->value(); return 0; } + MaybeAlign getParamAlign(unsigned ArgNo) const { + return AttributeSets.getParamAlignment(ArgNo); + } + /// Extract the byval type for a parameter. Type *getParamByValType(unsigned ArgNo) const { Type *Ty = AttributeSets.getParamByValType(ArgNo); diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h index 2209881dbda6..0171356914d6 100644 --- a/llvm/include/llvm/IR/GlobalValue.h +++ b/llvm/include/llvm/IR/GlobalValue.h @@ -38,7 +38,7 @@ class GlobalObject; class Module; namespace Intrinsic { - enum ID : unsigned; +typedef unsigned ID; } // end namespace Intrinsic class GlobalValue : public Constant { diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index d1ddb75cde9b..a6252b298001 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -97,8 +97,8 @@ protected: FastMathFlags FMF; bool IsFPConstrained; - ConstrainedFPIntrinsic::ExceptionBehavior DefaultConstrainedExcept; - ConstrainedFPIntrinsic::RoundingMode DefaultConstrainedRounding; + fp::ExceptionBehavior DefaultConstrainedExcept; + fp::RoundingMode DefaultConstrainedRounding; ArrayRef<OperandBundleDef> DefaultOperandBundles; @@ -106,8 +106,8 @@ public: IRBuilderBase(LLVMContext &context, MDNode *FPMathTag = nullptr, ArrayRef<OperandBundleDef> OpBundles = None) : Context(context), DefaultFPMathTag(FPMathTag), IsFPConstrained(false), - DefaultConstrainedExcept(ConstrainedFPIntrinsic::ebStrict), - DefaultConstrainedRounding(ConstrainedFPIntrinsic::rmDynamic), + DefaultConstrainedExcept(fp::ebStrict), + DefaultConstrainedRounding(fp::rmDynamic), DefaultOperandBundles(OpBundles) { ClearInsertionPoint(); } @@ -234,27 +234,39 @@ public: bool getIsFPConstrained() { return IsFPConstrained; } /// Set the exception handling to be used with constrained floating point - void setDefaultConstrainedExcept( - ConstrainedFPIntrinsic::ExceptionBehavior NewExcept) { + void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) { DefaultConstrainedExcept = NewExcept; } /// Set the rounding mode handling to be used with constrained floating point - void setDefaultConstrainedRounding( - ConstrainedFPIntrinsic::RoundingMode NewRounding) { + void setDefaultConstrainedRounding(fp::RoundingMode NewRounding) { DefaultConstrainedRounding = NewRounding; } /// Get the exception handling used with constrained floating point - ConstrainedFPIntrinsic::ExceptionBehavior getDefaultConstrainedExcept() { + fp::ExceptionBehavior getDefaultConstrainedExcept() { return DefaultConstrainedExcept; } /// Get the rounding mode handling used with constrained floating point - ConstrainedFPIntrinsic::RoundingMode getDefaultConstrainedRounding() { + fp::RoundingMode getDefaultConstrainedRounding() { return DefaultConstrainedRounding; } + void setConstrainedFPFunctionAttr() { + assert(BB && "Must have a basic block to set any function attributes!"); + + Function *F = BB->getParent(); + if (!F->hasFnAttribute(Attribute::StrictFP)) { + F->addFnAttr(Attribute::StrictFP); + } + } + + void setConstrainedFPCallAttr(CallInst *I) { + if (!I->hasFnAttr(Attribute::StrictFP)) + I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP); + } + //===--------------------------------------------------------------------===// // RAII helpers. //===--------------------------------------------------------------------===// @@ -437,15 +449,15 @@ public: /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is /// specified, it will be added to the instruction. Likewise with alias.scope /// and noalias tags. - CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, unsigned Align, - bool isVolatile = false, MDNode *TBAATag = nullptr, - MDNode *ScopeTag = nullptr, + CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, + MaybeAlign Align, bool isVolatile = false, + MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) { return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile, TBAATag, ScopeTag, NoAliasTag); } - CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align, + CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr); @@ -456,19 +468,45 @@ public: /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is /// specified, it will be added to the instruction. Likewise with alias.scope /// and noalias tags. + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes Align instead of this one. + LLVM_ATTRIBUTE_DEPRECATED( + CallInst *CreateElementUnorderedAtomicMemSet( + Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment, + uint32_t ElementSize, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr), + "Use the version that takes Align instead of this one") { + return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size), + Align(Alignment), ElementSize, + TBAATag, ScopeTag, NoAliasTag); + } + CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, - uint64_t Size, unsigned Align, + uint64_t Size, Align Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) { - return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size), Align, + return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size), + Align(Alignment), ElementSize, + TBAATag, ScopeTag, NoAliasTag); + } + + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes Align instead of this one. + LLVM_ATTRIBUTE_DEPRECATED( + CallInst *CreateElementUnorderedAtomicMemSet( + Value *Ptr, Value *Val, Value *Size, unsigned Alignment, + uint32_t ElementSize, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr), + "Use the version that takes Align instead of this one") { + return CreateElementUnorderedAtomicMemSet(Ptr, Val, Size, Align(Alignment), ElementSize, TBAATag, ScopeTag, NoAliasTag); } CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, - Value *Size, unsigned Align, + Value *Size, Align Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, @@ -479,8 +517,23 @@ public: /// If the pointers aren't i8*, they will be converted. If a TBAA tag is /// specified, it will be added to the instruction. Likewise with alias.scope /// and noalias tags. - CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, - unsigned SrcAlign, uint64_t Size, + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. + LLVM_ATTRIBUTE_DEPRECATED( + CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, + unsigned SrcAlign, uint64_t Size, + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *TBAAStructTag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr), + "Use the version that takes MaybeAlign instead") { + return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign), + getInt64(Size), isVolatile, TBAATag, TBAAStructTag, + ScopeTag, NoAliasTag); + } + + CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, + MaybeAlign SrcAlign, uint64_t Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, @@ -490,8 +543,18 @@ public: NoAliasTag); } - CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, - unsigned SrcAlign, Value *Size, + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. + LLVM_ATTRIBUTE_DEPRECATED( + CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, + unsigned SrcAlign, Value *Size, + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *TBAAStructTag = nullptr, + MDNode *ScopeTag = nullptr, + MDNode *NoAliasTag = nullptr), + "Use the version that takes MaybeAlign instead"); + CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, + MaybeAlign SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, @@ -527,16 +590,40 @@ public: /// If the pointers aren't i8*, they will be converted. If a TBAA tag is /// specified, it will be added to the instruction. Likewise with alias.scope /// and noalias tags. - CallInst *CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, - uint64_t Size, bool isVolatile = false, - MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. + LLVM_ATTRIBUTE_DEPRECATED( + CallInst *CreateMemMove( + Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, + uint64_t Size, bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr), + "Use the version that takes MaybeAlign") { + return CreateMemMove(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign), + getInt64(Size), isVolatile, TBAATag, ScopeTag, + NoAliasTag); + } + CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, + MaybeAlign SrcAlign, uint64_t Size, + bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) { - return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size), isVolatile, - TBAATag, ScopeTag, NoAliasTag); - } - - CallInst *CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, - Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, + return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size), + isVolatile, TBAATag, ScopeTag, NoAliasTag); + } + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. + LLVM_ATTRIBUTE_DEPRECATED( + CallInst *CreateMemMove( + Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, + Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, + MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr), + "Use the version that takes MaybeAlign") { + return CreateMemMove(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign), + Size, isVolatile, TBAATag, ScopeTag, NoAliasTag); + } + CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, + MaybeAlign SrcAlign, Value *Size, + bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr); @@ -1082,38 +1169,44 @@ private: return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr; } - Value *getConstrainedFPRounding( - Optional<ConstrainedFPIntrinsic::RoundingMode> Rounding) { - ConstrainedFPIntrinsic::RoundingMode UseRounding = - DefaultConstrainedRounding; + Value *getConstrainedFPRounding(Optional<fp::RoundingMode> Rounding) { + fp::RoundingMode UseRounding = DefaultConstrainedRounding; if (Rounding.hasValue()) UseRounding = Rounding.getValue(); - Optional<StringRef> RoundingStr = - ConstrainedFPIntrinsic::RoundingModeToStr(UseRounding); + Optional<StringRef> RoundingStr = RoundingModeToStr(UseRounding); assert(RoundingStr.hasValue() && "Garbage strict rounding mode!"); auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue()); return MetadataAsValue::get(Context, RoundingMDS); } - Value *getConstrainedFPExcept( - Optional<ConstrainedFPIntrinsic::ExceptionBehavior> Except) { - ConstrainedFPIntrinsic::ExceptionBehavior UseExcept = - DefaultConstrainedExcept; + Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) { + fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept; if (Except.hasValue()) UseExcept = Except.getValue(); - Optional<StringRef> ExceptStr = - ConstrainedFPIntrinsic::ExceptionBehaviorToStr(UseExcept); + Optional<StringRef> ExceptStr = ExceptionBehaviorToStr(UseExcept); assert(ExceptStr.hasValue() && "Garbage strict exception behavior!"); auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue()); return MetadataAsValue::get(Context, ExceptMDS); } + Value *getConstrainedFPPredicate(CmpInst::Predicate Predicate) { + assert(CmpInst::isFPPredicate(Predicate) && + Predicate != CmpInst::FCMP_FALSE && + Predicate != CmpInst::FCMP_TRUE && + "Invalid constrained FP comparison predicate!"); + + StringRef PredicateStr = CmpInst::getPredicateName(Predicate); + auto *PredicateMDS = MDString::get(Context, PredicateStr); + + return MetadataAsValue::get(Context, PredicateMDS); + } + public: Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { @@ -1468,8 +1561,8 @@ public: CallInst *CreateConstrainedFPBinOp( Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr, const Twine &Name = "", MDNode *FPMathTag = nullptr, - Optional<ConstrainedFPIntrinsic::RoundingMode> Rounding = None, - Optional<ConstrainedFPIntrinsic::ExceptionBehavior> Except = None) { + Optional<fp::RoundingMode> Rounding = None, + Optional<fp::ExceptionBehavior> Except = None) { Value *RoundingV = getConstrainedFPRounding(Rounding); Value *ExceptV = getConstrainedFPExcept(Except); @@ -1479,6 +1572,7 @@ public: CallInst *C = CreateIntrinsic(ID, {L->getType()}, {L, R, RoundingV, ExceptV}, nullptr, Name); + setConstrainedFPCallAttr(C); setFPAttrs(C, FPMathTag, UseFMF); return C; } @@ -1608,22 +1702,40 @@ public: /// Provided to resolve 'CreateAlignedLoad(Ptr, Align, "...")' /// correctly, instead of converting the string to 'bool' for the isVolatile /// parameter. + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, const char *Name) { + return CreateAlignedLoad(Ty, Ptr, MaybeAlign(Align), Name); + } + LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, + const char *Name) { LoadInst *LI = CreateLoad(Ty, Ptr, Name); - LI->setAlignment(MaybeAlign(Align)); + LI->setAlignment(Align); return LI; } + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, const Twine &Name = "") { + return CreateAlignedLoad(Ty, Ptr, MaybeAlign(Align), Name); + } + LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, + const Twine &Name = "") { LoadInst *LI = CreateLoad(Ty, Ptr, Name); - LI->setAlignment(MaybeAlign(Align)); + LI->setAlignment(Align); return LI; } + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, bool isVolatile, const Twine &Name = "") { + return CreateAlignedLoad(Ty, Ptr, MaybeAlign(Align), isVolatile, Name); + } + LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, + bool isVolatile, const Twine &Name = "") { LoadInst *LI = CreateLoad(Ty, Ptr, isVolatile, Name); - LI->setAlignment(MaybeAlign(Align)); + LI->setAlignment(Align); return LI; } @@ -1644,6 +1756,23 @@ public: return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, Align, isVolatile, Name); } + // Deprecated [opaque pointer types] + LoadInst *CreateAlignedLoad(Value *Ptr, MaybeAlign Align, const char *Name) { + return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, + Align, Name); + } + // Deprecated [opaque pointer types] + LoadInst *CreateAlignedLoad(Value *Ptr, MaybeAlign Align, + const Twine &Name = "") { + return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, + Align, Name); + } + // Deprecated [opaque pointer types] + LoadInst *CreateAlignedLoad(Value *Ptr, MaybeAlign Align, bool isVolatile, + const Twine &Name = "") { + return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, + Align, isVolatile, Name); + } StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, unsigned Align, bool isVolatile = false) { @@ -1651,7 +1780,10 @@ public: SI->setAlignment(MaybeAlign(Align)); return SI; } - + StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, + bool isVolatile = false) { + return CreateAlignedStore(Val, Ptr, Align ? Align->value() : 0, isVolatile); + } FenceInst *CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID = SyncScope::System, const Twine &Name = "") { @@ -1927,10 +2059,16 @@ public: } Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){ + if (IsFPConstrained) + return CreateConstrainedFPCast(Intrinsic::experimental_constrained_uitofp, + V, DestTy, nullptr, Name); return CreateCast(Instruction::UIToFP, V, DestTy, Name); } Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){ + if (IsFPConstrained) + return CreateConstrainedFPCast(Intrinsic::experimental_constrained_sitofp, + V, DestTy, nullptr, Name); return CreateCast(Instruction::SIToFP, V, DestTy, Name); } @@ -2062,8 +2200,8 @@ public: Intrinsic::ID ID, Value *V, Type *DestTy, Instruction *FMFSource = nullptr, const Twine &Name = "", MDNode *FPMathTag = nullptr, - Optional<ConstrainedFPIntrinsic::RoundingMode> Rounding = None, - Optional<ConstrainedFPIntrinsic::ExceptionBehavior> Except = None) { + Optional<fp::RoundingMode> Rounding = None, + Optional<fp::ExceptionBehavior> Except = None) { Value *ExceptV = getConstrainedFPExcept(Except); FastMathFlags UseFMF = FMF; @@ -2071,19 +2209,26 @@ public: UseFMF = FMFSource->getFastMathFlags(); CallInst *C; + bool HasRoundingMD = false; switch (ID) { - default: { + default: + break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ + case Intrinsic::INTRINSIC: \ + HasRoundingMD = ROUND_MODE; \ + break; +#include "llvm/IR/ConstrainedOps.def" + } + if (HasRoundingMD) { Value *RoundingV = getConstrainedFPRounding(Rounding); C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, nullptr, Name); - } break; - case Intrinsic::experimental_constrained_fpext: - case Intrinsic::experimental_constrained_fptoui: - case Intrinsic::experimental_constrained_fptosi: + } else C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, Name); - break; - } + + setConstrainedFPCallAttr(C); + if (isa<FPMathOperator>(C)) setFPAttrs(C, FPMathTag, UseFMF); return C; @@ -2216,14 +2361,49 @@ public: return Insert(new ICmpInst(P, LHS, RHS), Name); } + // Create a quiet floating-point comparison (i.e. one that raises an FP + // exception only in the case where an input is a signaling NaN). + // Note that this differs from CreateFCmpS only if IsFPConstrained is true. Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name = "", MDNode *FPMathTag = nullptr) { + if (IsFPConstrained) + return CreateConstrainedFPCmp(Intrinsic::experimental_constrained_fcmp, + P, LHS, RHS, Name); + + if (auto *LC = dyn_cast<Constant>(LHS)) + if (auto *RC = dyn_cast<Constant>(RHS)) + return Insert(Folder.CreateFCmp(P, LC, RC), Name); + return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); + } + + // Create a signaling floating-point comparison (i.e. one that raises an FP + // exception whenever an input is any NaN, signaling or quiet). + // Note that this differs from CreateFCmp only if IsFPConstrained is true. + Value *CreateFCmpS(CmpInst::Predicate P, Value *LHS, Value *RHS, + const Twine &Name = "", MDNode *FPMathTag = nullptr) { + if (IsFPConstrained) + return CreateConstrainedFPCmp(Intrinsic::experimental_constrained_fcmps, + P, LHS, RHS, Name); + if (auto *LC = dyn_cast<Constant>(LHS)) if (auto *RC = dyn_cast<Constant>(RHS)) return Insert(Folder.CreateFCmp(P, LC, RC), Name); return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); } + CallInst *CreateConstrainedFPCmp( + Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, + const Twine &Name = "", + Optional<fp::ExceptionBehavior> Except = None) { + Value *PredicateV = getConstrainedFPPredicate(P); + Value *ExceptV = getConstrainedFPExcept(Except); + + CallInst *C = CreateIntrinsic(ID, {L->getType()}, + {L, R, PredicateV, ExceptV}, nullptr, Name); + setConstrainedFPCallAttr(C); + return C; + } + //===--------------------------------------------------------------------===// // Instruction creation methods: Other Instructions //===--------------------------------------------------------------------===// @@ -2240,6 +2420,8 @@ public: ArrayRef<Value *> Args = None, const Twine &Name = "", MDNode *FPMathTag = nullptr) { CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles); + if (IsFPConstrained) + setConstrainedFPCallAttr(CI); if (isa<FPMathOperator>(CI)) setFPAttrs(CI, FPMathTag, FMF); return Insert(CI, Name); @@ -2249,6 +2431,8 @@ public: ArrayRef<OperandBundleDef> OpBundles, const Twine &Name = "", MDNode *FPMathTag = nullptr) { CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles); + if (IsFPConstrained) + setConstrainedFPCallAttr(CI); if (isa<FPMathOperator>(CI)) setFPAttrs(CI, FPMathTag, FMF); return Insert(CI, Name); @@ -2284,6 +2468,33 @@ public: Args, OpBundles, Name, FPMathTag); } + CallInst *CreateConstrainedFPCall( + Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "", + Optional<fp::RoundingMode> Rounding = None, + Optional<fp::ExceptionBehavior> Except = None) { + llvm::SmallVector<Value *, 6> UseArgs; + + for (auto *OneArg : Args) + UseArgs.push_back(OneArg); + bool HasRoundingMD = false; + switch (Callee->getIntrinsicID()) { + default: + break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ + case Intrinsic::INTRINSIC: \ + HasRoundingMD = ROUND_MODE; \ + break; +#include "llvm/IR/ConstrainedOps.def" + } + if (HasRoundingMD) + UseArgs.push_back(getConstrainedFPRounding(Rounding)); + UseArgs.push_back(getConstrainedFPExcept(Except)); + + CallInst *C = CreateCall(Callee, UseArgs, Name); + setConstrainedFPCallAttr(C); + return C; + } + Value *CreateSelect(Value *C, Value *True, Value *False, const Twine &Name = "", Instruction *MDFrom = nullptr) { if (auto *CC = dyn_cast<Constant>(C)) @@ -2370,6 +2581,10 @@ public: return Insert(LandingPadInst::Create(Ty, NumClauses), Name); } + Value *CreateFreeze(Value *V, const Twine &Name = "") { + return Insert(new FreezeInst(V), Name); + } + //===--------------------------------------------------------------------===// // Utility creation methods //===--------------------------------------------------------------------===// @@ -2499,8 +2714,9 @@ public: return V; } - Value *CreatePreserveArrayAccessIndex(Value *Base, unsigned Dimension, - unsigned LastIndex, MDNode *DbgInfo) { + Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base, + unsigned Dimension, unsigned LastIndex, + MDNode *DbgInfo) { assert(isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.array.access.index."); auto *BaseType = Base->getType(); @@ -2513,7 +2729,7 @@ public: IdxList.push_back(LastIndexV); Type *ResultType = - GetElementPtrInst::getGEPReturnType(Base, IdxList); + GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList); Module *M = BB->getParent()->getParent(); Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( @@ -2547,8 +2763,9 @@ public: return Fn; } - Value *CreatePreserveStructAccessIndex(Value *Base, unsigned Index, - unsigned FieldIndex, MDNode *DbgInfo) { + Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base, + unsigned Index, unsigned FieldIndex, + MDNode *DbgInfo) { assert(isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.struct.access.index."); auto *BaseType = Base->getType(); @@ -2556,7 +2773,7 @@ public: Value *GEPIndex = getInt32(Index); Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); Type *ResultType = - GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex}); + GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex}); Module *M = BB->getParent()->getParent(); Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( diff --git a/llvm/include/llvm/IR/IRPrintingPasses.h b/llvm/include/llvm/IR/IRPrintingPasses.h index 3be9449c1a93..230db988f737 100644 --- a/llvm/include/llvm/IR/IRPrintingPasses.h +++ b/llvm/include/llvm/IR/IRPrintingPasses.h @@ -23,7 +23,6 @@ namespace llvm { class Pass; -class BasicBlockPass; class Function; class FunctionPass; class Module; @@ -43,11 +42,6 @@ ModulePass *createPrintModulePass(raw_ostream &OS, FunctionPass *createPrintFunctionPass(raw_ostream &OS, const std::string &Banner = ""); -/// Create and return a pass that writes the BB to the specified -/// \c raw_ostream. -BasicBlockPass *createPrintBasicBlockPass(raw_ostream &OS, - const std::string &Banner = ""); - /// Print out a name of an LLVM value without any prefixes. /// /// The name is surrounded with ""'s and escaped if it has any special or diff --git a/llvm/include/llvm/IR/InstVisitor.h b/llvm/include/llvm/IR/InstVisitor.h index fbeb2caf14e6..6168c877a2be 100644 --- a/llvm/include/llvm/IR/InstVisitor.h +++ b/llvm/include/llvm/IR/InstVisitor.h @@ -199,6 +199,7 @@ public: RetTy visitFuncletPadInst(FuncletPadInst &I) { DELEGATE(Instruction); } RetTy visitCleanupPadInst(CleanupPadInst &I) { DELEGATE(FuncletPadInst); } RetTy visitCatchPadInst(CatchPadInst &I) { DELEGATE(FuncletPadInst); } + RetTy visitFreezeInst(FreezeInst &I) { DELEGATE(Instruction); } // Handle the special instrinsic instruction classes. RetTy visitDbgDeclareInst(DbgDeclareInst &I) { DELEGATE(DbgVariableIntrinsic);} diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h index 7fb94e9d8c22..b2cdd58a5046 100644 --- a/llvm/include/llvm/IR/InstrTypes.h +++ b/llvm/include/llvm/IR/InstrTypes.h @@ -47,7 +47,7 @@ namespace llvm { namespace Intrinsic { -enum ID : unsigned; +typedef unsigned ID; } //===----------------------------------------------------------------------===// @@ -1039,6 +1039,11 @@ struct OperandBundleUse { return getTagID() == LLVMContext::OB_funclet; } + /// Return true if this is a "cfguardtarget" operand bundle. + bool isCFGuardTargetOperandBundle() const { + return getTagID() == LLVMContext::OB_cfguardtarget; + } + private: /// Pointer to an entry in LLVMContextImpl::getOrInsertBundleTag. StringMapEntry<uint32_t> *Tag; @@ -1267,6 +1272,19 @@ public: return isArgOperand(&UI.getUse()); } + /// Given a use for a arg operand, get the arg operand number that + /// corresponds to it. + unsigned getArgOperandNo(const Use *U) const { + assert(isArgOperand(U) && "Arg operand # out of range!"); + return U - arg_begin(); + } + + /// Given a value use iterator, return the arg operand number corresponding to + /// it. Iterator must actually correspond to a data operand. + unsigned getArgOperandNo(Value::const_user_iterator UI) const { + return getArgOperandNo(&UI.getUse()); + } + /// Returns true if this CallSite passes the given Value* as an argument to /// the called function. bool hasArgument(const Value *V) const { @@ -1567,19 +1585,31 @@ public: } /// Extract the alignment of the return value. + /// FIXME: Remove this function once transition to Align is over. + /// Use getRetAlign() instead. unsigned getRetAlignment() const { if (const auto MA = Attrs.getRetAlignment()) return MA->value(); return 0; } + /// Extract the alignment of the return value. + MaybeAlign getRetAlign() const { return Attrs.getRetAlignment(); } + /// Extract the alignment for a call or parameter (0=unknown). + /// FIXME: Remove this function once transition to Align is over. + /// Use getParamAlign() instead. unsigned getParamAlignment(unsigned ArgNo) const { if (const auto MA = Attrs.getParamAlignment(ArgNo)) return MA->value(); return 0; } + /// Extract the alignment for a call or parameter (0=unknown). + MaybeAlign getParamAlign(unsigned ArgNo) const { + return Attrs.getParamAlignment(ArgNo); + } + /// Extract the byval type for a call or parameter. Type *getParamByValType(unsigned ArgNo) const { Type *Ty = Attrs.getParamByValType(ArgNo); @@ -1917,7 +1947,7 @@ public: /// Is the function attribute S disallowed by some operand bundle on /// this operand bundle user? bool isFnAttrDisallowedByOpBundle(StringRef S) const { - // Operand bundles only possibly disallow readnone, readonly and argmenonly + // Operand bundles only possibly disallow readnone, readonly and argmemonly // attributes. All String attributes are fine. return false; } diff --git a/llvm/include/llvm/IR/Instruction.def b/llvm/include/llvm/IR/Instruction.def index 41cdf613ad64..a5ad92f58f94 100644 --- a/llvm/include/llvm/IR/Instruction.def +++ b/llvm/include/llvm/IR/Instruction.def @@ -217,7 +217,8 @@ HANDLE_OTHER_INST(63, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. HANDLE_OTHER_INST(64, ExtractValue, ExtractValueInst)// extract from aggregate HANDLE_OTHER_INST(65, InsertValue, InsertValueInst) // insert into aggregate HANDLE_OTHER_INST(66, LandingPad, LandingPadInst) // Landing pad instruction. - LAST_OTHER_INST(66) +HANDLE_OTHER_INST(67, Freeze, FreezeInst) // Freeze instruction. + LAST_OTHER_INST(67) #undef FIRST_TERM_INST #undef HANDLE_TERM_INST diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h index 803f6977b32c..3bfa0e4afc39 100644 --- a/llvm/include/llvm/IR/Instruction.h +++ b/llvm/include/llvm/IR/Instruction.h @@ -129,7 +129,7 @@ public: bool isUnaryOp() const { return isUnaryOp(getOpcode()); } bool isBinaryOp() const { return isBinaryOp(getOpcode()); } bool isIntDivRem() const { return isIntDivRem(getOpcode()); } - bool isShift() { return isShift(getOpcode()); } + bool isShift() const { return isShift(getOpcode()); } bool isCast() const { return isCast(getOpcode()); } bool isFuncletPad() const { return isFuncletPad(getOpcode()); } bool isExceptionalTerminator() const { diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index fa980df03ef0..b73d5274238c 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -78,9 +78,9 @@ public: AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name, BasicBlock *InsertAtEnd); - AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, unsigned Align, + AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, MaybeAlign Align, const Twine &Name = "", Instruction *InsertBefore = nullptr); - AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, unsigned Align, + AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, MaybeAlign Align, const Twine &Name, BasicBlock *InsertAtEnd); /// Return true if there is an allocation size parameter to the allocation @@ -239,14 +239,20 @@ public: } /// Return the alignment of the access that is being performed. + /// FIXME: Remove this function once transition to Align is over. + /// Use getAlign() instead. unsigned getAlignment() const { - if (const auto MA = - decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31)) + if (const auto MA = getAlign()) return MA->value(); return 0; } - void setAlignment(MaybeAlign Align); + /// Return the alignment of the access that is being performed. + MaybeAlign getAlign() const { + return decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31); + } + + void setAlignment(MaybeAlign Alignment); /// Returns the ordering constraint of this load instruction. AtomicOrdering getOrdering() const { @@ -365,14 +371,19 @@ public: DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); /// Return the alignment of the access that is being performed + /// FIXME: Remove this function once transition to Align is over. + /// Use getAlign() instead. unsigned getAlignment() const { - if (const auto MA = - decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31)) + if (const auto MA = getAlign()) return MA->value(); return 0; } - void setAlignment(MaybeAlign Align); + MaybeAlign getAlign() const { + return decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31); + } + + void setAlignment(MaybeAlign Alignment); /// Returns the ordering constraint of this store instruction. AtomicOrdering getOrdering() const { @@ -1039,11 +1050,6 @@ public: /// Returns the pointer type returned by the GEP /// instruction, which may be a vector of pointers. - static Type *getGEPReturnType(Value *Ptr, ArrayRef<Value *> IdxList) { - return getGEPReturnType( - cast<PointerType>(Ptr->getType()->getScalarType())->getElementType(), - Ptr, IdxList); - } static Type *getGEPReturnType(Type *ElTy, Value *Ptr, ArrayRef<Value *> IdxList) { Type *PtrTy = PointerType::get(checkGEPType(getIndexedType(ElTy, IdxList)), @@ -4142,13 +4148,9 @@ CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles, int NumOperands, const Twine &NameStr, BasicBlock *InsertAtEnd) - : CallBase( - cast<FunctionType>( - cast<PointerType>(Func->getType())->getElementType()) - ->getReturnType(), - Instruction::CallBr, - OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands, - InsertAtEnd) { + : CallBase(Ty->getReturnType(), Instruction::CallBr, + OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands, + InsertAtEnd) { init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr); } @@ -5299,6 +5301,35 @@ inline unsigned getLoadStoreAddressSpace(Value *I) { return cast<StoreInst>(I)->getPointerAddressSpace(); } +//===----------------------------------------------------------------------===// +// FreezeInst Class +//===----------------------------------------------------------------------===// + +/// This class represents a freeze function that returns random concrete +/// value if an operand is either a poison value or an undef value +class FreezeInst : public UnaryInstruction { +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + + /// Clone an identical FreezeInst + FreezeInst *cloneImpl() const; + +public: + explicit FreezeInst(Value *S, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr); + FreezeInst(Value *S, const Twine &NameStr, BasicBlock *InsertAtEnd); + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static inline bool classof(const Instruction *I) { + return I->getOpcode() == Freeze; + } + static inline bool classof(const Value *V) { + return isa<Instruction>(V) && classof(cast<Instruction>(V)); + } +}; + } // end namespace llvm #endif // LLVM_IR_INSTRUCTIONS_H diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index c989b4a2e72a..42a5564a4488 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -25,6 +25,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/FPEnv.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instructions.h" @@ -208,83 +209,28 @@ namespace llvm { /// This is the common base class for constrained floating point intrinsics. class ConstrainedFPIntrinsic : public IntrinsicInst { public: - /// Specifies the rounding mode to be assumed. This is only used when - /// when constrained floating point is enabled. See the LLVM Language - /// Reference Manual for details. - enum RoundingMode : uint8_t { - rmDynamic, ///< This corresponds to "fpround.dynamic". - rmToNearest, ///< This corresponds to "fpround.tonearest". - rmDownward, ///< This corresponds to "fpround.downward". - rmUpward, ///< This corresponds to "fpround.upward". - rmTowardZero ///< This corresponds to "fpround.tozero". - }; - - /// Specifies the required exception behavior. This is only used when - /// when constrained floating point is used. See the LLVM Language - /// Reference Manual for details. - enum ExceptionBehavior : uint8_t { - ebIgnore, ///< This corresponds to "fpexcept.ignore". - ebMayTrap, ///< This corresponds to "fpexcept.maytrap". - ebStrict ///< This corresponds to "fpexcept.strict". - }; - bool isUnaryOp() const; bool isTernaryOp() const; - Optional<RoundingMode> getRoundingMode() const; - Optional<ExceptionBehavior> getExceptionBehavior() const; - - /// Returns a valid RoundingMode enumerator when given a string - /// that is valid as input in constrained intrinsic rounding mode - /// metadata. - static Optional<RoundingMode> StrToRoundingMode(StringRef); - - /// For any RoundingMode enumerator, returns a string valid as input in - /// constrained intrinsic rounding mode metadata. - static Optional<StringRef> RoundingModeToStr(RoundingMode); + Optional<fp::RoundingMode> getRoundingMode() const; + Optional<fp::ExceptionBehavior> getExceptionBehavior() const; - /// Returns a valid ExceptionBehavior enumerator when given a string - /// valid as input in constrained intrinsic exception behavior metadata. - static Optional<ExceptionBehavior> StrToExceptionBehavior(StringRef); + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const IntrinsicInst *I); + static bool classof(const Value *V) { + return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V)); + } + }; - /// For any ExceptionBehavior enumerator, returns a string valid as - /// input in constrained intrinsic exception behavior metadata. - static Optional<StringRef> ExceptionBehaviorToStr(ExceptionBehavior); + /// Constrained floating point compare intrinsics. + class ConstrainedFPCmpIntrinsic : public ConstrainedFPIntrinsic { + public: + FCmpInst::Predicate getPredicate() const; // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { switch (I->getIntrinsicID()) { - case Intrinsic::experimental_constrained_fadd: - case Intrinsic::experimental_constrained_fsub: - case Intrinsic::experimental_constrained_fmul: - case Intrinsic::experimental_constrained_fdiv: - case Intrinsic::experimental_constrained_frem: - case Intrinsic::experimental_constrained_fma: - case Intrinsic::experimental_constrained_fptosi: - case Intrinsic::experimental_constrained_fptoui: - case Intrinsic::experimental_constrained_fptrunc: - case Intrinsic::experimental_constrained_fpext: - case Intrinsic::experimental_constrained_sqrt: - case Intrinsic::experimental_constrained_pow: - case Intrinsic::experimental_constrained_powi: - case Intrinsic::experimental_constrained_sin: - case Intrinsic::experimental_constrained_cos: - case Intrinsic::experimental_constrained_exp: - case Intrinsic::experimental_constrained_exp2: - case Intrinsic::experimental_constrained_log: - case Intrinsic::experimental_constrained_log10: - case Intrinsic::experimental_constrained_log2: - case Intrinsic::experimental_constrained_lrint: - case Intrinsic::experimental_constrained_llrint: - case Intrinsic::experimental_constrained_rint: - case Intrinsic::experimental_constrained_nearbyint: - case Intrinsic::experimental_constrained_maxnum: - case Intrinsic::experimental_constrained_minnum: - case Intrinsic::experimental_constrained_ceil: - case Intrinsic::experimental_constrained_floor: - case Intrinsic::experimental_constrained_lround: - case Intrinsic::experimental_constrained_llround: - case Intrinsic::experimental_constrained_round: - case Intrinsic::experimental_constrained_trunc: + case Intrinsic::experimental_constrained_fcmp: + case Intrinsic::experimental_constrained_fcmps: return true; default: return false; } @@ -402,7 +348,10 @@ namespace llvm { return cast<PointerType>(getRawDest()->getType())->getAddressSpace(); } + /// FIXME: Remove this function once transition to Align is over. + /// Use getDestAlign() instead. unsigned getDestAlignment() const { return getParamAlignment(ARG_DEST); } + MaybeAlign getDestAlign() const { return getParamAlign(ARG_DEST); } /// Set the specified arguments of the instruction. void setDest(Value *Ptr) { @@ -411,11 +360,21 @@ namespace llvm { setArgOperand(ARG_DEST, Ptr); } + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. void setDestAlignment(unsigned Alignment) { + setDestAlignment(MaybeAlign(Alignment)); + } + void setDestAlignment(MaybeAlign Alignment) { + removeParamAttr(ARG_DEST, Attribute::Alignment); + if (Alignment) + addParamAttr(ARG_DEST, + Attribute::getWithAlignment(getContext(), *Alignment)); + } + void setDestAlignment(Align Alignment) { removeParamAttr(ARG_DEST, Attribute::Alignment); - if (Alignment > 0) - addParamAttr(ARG_DEST, Attribute::getWithAlignment(getContext(), - Align(Alignment))); + addParamAttr(ARG_DEST, + Attribute::getWithAlignment(getContext(), Alignment)); } void setLength(Value *L) { @@ -450,22 +409,37 @@ namespace llvm { return cast<PointerType>(getRawSource()->getType())->getAddressSpace(); } + /// FIXME: Remove this function once transition to Align is over. + /// Use getSourceAlign() instead. unsigned getSourceAlignment() const { return BaseCL::getParamAlignment(ARG_SOURCE); } + MaybeAlign getSourceAlign() const { + return BaseCL::getParamAlign(ARG_SOURCE); + } + void setSource(Value *Ptr) { assert(getRawSource()->getType() == Ptr->getType() && "setSource called with pointer of wrong type!"); BaseCL::setArgOperand(ARG_SOURCE, Ptr); } + /// FIXME: Remove this function once transition to Align is over. + /// Use the version that takes MaybeAlign instead of this one. void setSourceAlignment(unsigned Alignment) { + setSourceAlignment(MaybeAlign(Alignment)); + } + void setSourceAlignment(MaybeAlign Alignment) { + BaseCL::removeParamAttr(ARG_SOURCE, Attribute::Alignment); + if (Alignment) + BaseCL::addParamAttr(ARG_SOURCE, Attribute::getWithAlignment( + BaseCL::getContext(), *Alignment)); + } + void setSourceAlignment(Align Alignment) { BaseCL::removeParamAttr(ARG_SOURCE, Attribute::Alignment); - if (Alignment > 0) - BaseCL::addParamAttr(ARG_SOURCE, - Attribute::getWithAlignment(BaseCL::getContext(), - Align(Alignment))); + BaseCL::addParamAttr(ARG_SOURCE, Attribute::getWithAlignment( + BaseCL::getContext(), Alignment)); } }; diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index 9e4ebd915afc..58e7725fc0df 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -33,14 +33,17 @@ class AttributeList; /// function known by LLVM. The enum values are returned by /// Function::getIntrinsicID(). namespace Intrinsic { - enum ID : unsigned { - not_intrinsic = 0, // Must be zero + // Intrinsic ID type. This is an opaque typedef to facilitate splitting up + // the enum into target-specific enums. + typedef unsigned ID; - // Get the intrinsic enums generated from Intrinsics.td + enum IndependentIntrinsics : unsigned { + not_intrinsic = 0, // Must be zero + + // Get the intrinsic enums generated from Intrinsics.td #define GET_INTRINSIC_ENUM_VALUES #include "llvm/IR/IntrinsicEnums.inc" #undef GET_INTRINSIC_ENUM_VALUES - , num_intrinsics }; /// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx". diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 7a0263f88c2a..865e4ccc9bc4 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -582,12 +582,6 @@ def int_maximum : Intrinsic<[llvm_anyfloat_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative] >; -// NOTE: these are internal interfaces. -def int_setjmp : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty]>; -def int_longjmp : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [IntrNoReturn]>; -def int_sigsetjmp : Intrinsic<[llvm_i32_ty] , [llvm_ptr_ty, llvm_i32_ty]>; -def int_siglongjmp : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [IntrNoReturn]>; - // Internal interface for object size checking def int_objectsize : Intrinsic<[llvm_anyint_ty], [llvm_anyptr_ty, llvm_i1_ty, @@ -640,6 +634,16 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn] in { [ llvm_anyfloat_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_sitofp : Intrinsic<[ llvm_anyfloat_ty ], + [ llvm_anyint_ty, + llvm_metadata_ty, + llvm_metadata_ty ]>; + + def int_experimental_constrained_uitofp : Intrinsic<[ llvm_anyfloat_ty ], + [ llvm_anyint_ty, + llvm_metadata_ty, + llvm_metadata_ty ]>; + def int_experimental_constrained_fptrunc : Intrinsic<[ llvm_anyfloat_ty ], [ llvm_anyfloat_ty, llvm_metadata_ty, @@ -714,20 +718,24 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn] in { def int_experimental_constrained_maxnum : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, - llvm_metadata_ty, llvm_metadata_ty ]>; def int_experimental_constrained_minnum : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, LLVMMatchType<0>, - llvm_metadata_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_maximum : Intrinsic<[ llvm_anyfloat_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_metadata_ty ]>; + def int_experimental_constrained_minimum : Intrinsic<[ llvm_anyfloat_ty ], + [ LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_metadata_ty ]>; def int_experimental_constrained_ceil : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, - llvm_metadata_ty, llvm_metadata_ty ]>; def int_experimental_constrained_floor : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, - llvm_metadata_ty, llvm_metadata_ty ]>; def int_experimental_constrained_lround : Intrinsic<[ llvm_anyint_ty ], [ llvm_anyfloat_ty, @@ -737,14 +745,22 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn] in { llvm_metadata_ty ]>; def int_experimental_constrained_round : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, - llvm_metadata_ty, llvm_metadata_ty ]>; def int_experimental_constrained_trunc : Intrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0>, - llvm_metadata_ty, llvm_metadata_ty ]>; + + // Constrained floating-point comparison (quiet and signaling variants). + // Third operand is the predicate represented as a metadata string. + def int_experimental_constrained_fcmp + : Intrinsic<[ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty> ], + [ llvm_anyfloat_ty, LLVMMatchType<0>, + llvm_metadata_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_fcmps + : Intrinsic<[ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty> ], + [ llvm_anyfloat_ty, LLVMMatchType<0>, + llvm_metadata_ty, llvm_metadata_ty ]>; } -// FIXME: Add intrinsic for fcmp. // FIXME: Consider maybe adding intrinsics for sitofp, uitofp. //===------------------------- Expect Intrinsics --------------------------===// @@ -914,6 +930,14 @@ def int_umul_fix : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative, ImmArg<2>]>; +def int_sdiv_fix : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; + +def int_udiv_fix : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; + //===------------------- Fixed Point Saturation Arithmetic Intrinsics ----------------===// // def int_smul_fix_sat : Intrinsic<[llvm_anyint_ty], @@ -1070,7 +1094,7 @@ def int_experimental_guard : Intrinsic<[], [llvm_i1_ty, llvm_vararg_ty], // Supports widenable conditions for guards represented as explicit branches. def int_experimental_widenable_condition : Intrinsic<[llvm_i1_ty], [], - [IntrInaccessibleMemOnly, IntrWillReturn]>; + [IntrInaccessibleMemOnly, IntrWillReturn, IntrSpeculatable]>; // NOP: calls/invokes to this intrinsic are removed by codegen def int_donothing : Intrinsic<[], [], [IntrNoMem, IntrWillReturn]>; @@ -1232,6 +1256,42 @@ let IntrProperties = [IntrNoMem, IntrWillReturn] in { [llvm_anyvector_ty]>; } +//===----- Matrix intrinsics ---------------------------------------------===// + +def int_matrix_transpose : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + llvm_i32_ty, + llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, + IntrWillReturn, ImmArg<1>, ImmArg<2>]>; + +def int_matrix_multiply : Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty, + llvm_anyvector_ty, + llvm_i32_ty, + llvm_i32_ty, + llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, + IntrWillReturn, ImmArg<2>, ImmArg<3>, + ImmArg<4>]>; + +def int_matrix_columnwise_load : Intrinsic<[llvm_anyvector_ty], + [LLVMAnyPointerType<LLVMMatchType<0>>, + llvm_i32_ty, + llvm_i32_ty, + llvm_i32_ty], + [IntrReadMem, IntrWillReturn, + ImmArg<2>, ImmArg<3>]>; + +def int_matrix_columnwise_store : Intrinsic<[], + [llvm_anyvector_ty, + LLVMAnyPointerType<LLVMMatchType<0>>, + llvm_i32_ty, + llvm_i32_ty, + llvm_i32_ty], + [WriteOnly<1>, IntrWillReturn, + ImmArg<3>, ImmArg<4>]>; + //===---------- Intrinsics to control hardware supported loops ----------===// // Specify that the value given is the number of iterations that the next loop @@ -1254,7 +1314,9 @@ def int_loop_decrement : // maximum number of elements processed in an iteration). Return the remaining // number of iterations still to be executed. This is effectively a sub which // can be used with a phi, icmp and br to control the number of iterations -// executed, as usual. +// executed, as usual. Any optimisations are allowed to treat it is a sub, and +// it's scevable, so it's the backends responsibility to handle cases where it +// may be optimised. def int_loop_decrement_reg : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty, llvm_anyint_ty], [IntrNoDuplicate]>; diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index db01700f409f..27a2550d1857 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -33,6 +33,9 @@ def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; +def int_aarch64_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_aarch64_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>; + //===----------------------------------------------------------------------===// // HINT @@ -443,6 +446,10 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic; def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic; def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic; + + // v8.3-A Floating-point complex add + def int_aarch64_neon_vcadd_rot90 : AdvSIMD_2VectorArg_Intrinsic; + def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic; } let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". @@ -761,6 +768,20 @@ def llvm_nxv4f32_ty : LLVMType<nxv4f32>; def llvm_nxv2f64_ty : LLVMType<nxv2f64>; let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". + + class AdvSIMD_1Vec_PredLoad_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerTo<0>], + [IntrReadMem, IntrArgMemOnly]>; + + class AdvSIMD_1Vec_PredStore_Intrinsic + : Intrinsic<[], + [llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerTo<0>], + [IntrArgMemOnly, NoCapture<2>]>; + class AdvSIMD_Merged1VectorArg_Intrinsic : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, @@ -768,6 +789,79 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". LLVMMatchType<0>], [IntrNoMem]>; + class AdvSIMD_2VectorArgIndexed_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty], + [IntrNoMem]>; + + class AdvSIMD_3VectorArgIndexed_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty], + [IntrNoMem]>; + + class AdvSIMD_Pred1VectorArg_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>], + [IntrNoMem]>; + + class AdvSIMD_Pred2VectorArg_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMMatchType<0>], + [IntrNoMem]>; + + class AdvSIMD_Pred3VectorArg_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMMatchType<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_Compare_Intrinsic + : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty, + LLVMMatchType<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_CompareWide_Intrinsic + : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty, + llvm_nxv2i64_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_Saturating_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [IntrNoMem]>; + + class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + llvm_i32_ty, + llvm_i32_ty], + [IntrNoMem, ImmArg<1>, ImmArg<2>]>; + + class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T> + : Intrinsic<[T], + [T, llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T> + : Intrinsic<[T], + [T, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>, ImmArg<2>]>; + class AdvSIMD_SVE_CNT_Intrinsic : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], [LLVMVectorOfBitcastsToInt<0>, @@ -775,16 +869,136 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". llvm_anyvector_ty], [IntrNoMem]>; + class AdvSIMD_SVE_FP_Reduce_Intrinsic + : Intrinsic<[llvm_anyfloat_ty], + [LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, + llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_ReduceWithInit_Intrinsic + : Intrinsic<[LLVMVectorElementType<0>], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMVectorElementType<0>, + llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_FP_ReduceWithInit_Intrinsic + : Intrinsic<[llvm_anyfloat_ty], + [LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, + LLVMMatchType<0>, + llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_ShiftByImm_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + llvm_i32_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_ShiftWide_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + llvm_nxv2i64_ty], + [IntrNoMem]>; + class AdvSIMD_SVE_Unpack_Intrinsic - : Intrinsic<[llvm_anyvector_ty], + : Intrinsic<[llvm_anyvector_ty], [LLVMSubdivide2VectorType<0>], [IntrNoMem]>; + class AdvSIMD_SVE_CADD_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_CMLA_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_CMLA_LANE_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMMatchType<0>, + LLVMMatchType<0>, + llvm_i32_ty, + llvm_i32_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_EXPA_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMVectorOfBitcastsToInt<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_FCVT_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_FCVTZS_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMVectorOfBitcastsToInt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_INSR_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMVectorElementType<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_PTRUE_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [llvm_i32_ty], + [IntrNoMem, ImmArg<0>]>; + class AdvSIMD_SVE_PUNPKHI_Intrinsic : Intrinsic<[LLVMHalfElementsVectorType<0>], [llvm_anyvector_ty], [IntrNoMem]>; + class AdvSIMD_SVE_SCALE_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMMatchType<0>, + LLVMVectorOfBitcastsToInt<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_SCVTF_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty], + [IntrNoMem]>; + + class AdvSIMD_SVE_TSMUL_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMVectorOfBitcastsToInt<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_CNTB_Intrinsic + : Intrinsic<[llvm_i64_ty], + [llvm_i32_ty], + [IntrNoMem, ImmArg<0>]>; + + class AdvSIMD_SVE_CNTP_Intrinsic + : Intrinsic<[llvm_i64_ty], + [llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; + class AdvSIMD_SVE_DOT_Intrinsic : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, @@ -800,6 +1014,70 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". llvm_i32_ty], [IntrNoMem]>; + class AdvSIMD_SVE_PTEST_Intrinsic + : Intrinsic<[llvm_i1_ty], + [llvm_anyvector_ty, + LLVMMatchType<0>], + [IntrNoMem]>; + + class AdvSIMD_SVE_TBL_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMVectorOfBitcastsToInt<0>], + [IntrNoMem]>; + + class SVE2_3VectorArg_Long_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMSubdivide2VectorType<0>, + LLVMSubdivide2VectorType<0>], + [IntrNoMem]>; + + class SVE2_3VectorArgIndexed_Long_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, + LLVMSubdivide2VectorType<0>, + LLVMSubdivide2VectorType<0>, + llvm_i32_ty], + [IntrNoMem]>; + + class SVE2_1VectorArg_Narrowing_Intrinsic + : Intrinsic<[LLVMSubdivide2VectorType<0>], + [llvm_anyvector_ty], + [IntrNoMem]>; + + class SVE2_Merged1VectorArg_Narrowing_Intrinsic + : Intrinsic<[LLVMSubdivide2VectorType<0>], + [LLVMSubdivide2VectorType<0>, + llvm_anyvector_ty], + [IntrNoMem]>; + class SVE2_2VectorArg_Narrowing_Intrinsic + : Intrinsic< + [LLVMSubdivide2VectorType<0>], + [llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; + + class SVE2_Merged2VectorArg_Narrowing_Intrinsic + : Intrinsic< + [LLVMSubdivide2VectorType<0>], + [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; + + class SVE2_1VectorArg_Imm_Narrowing_Intrinsic + : Intrinsic<[LLVMSubdivide2VectorType<0>], + [llvm_anyvector_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; + + class SVE2_2VectorArg_Imm_Narrowing_Intrinsic + : Intrinsic<[LLVMSubdivide2VectorType<0>], + [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, + llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; + + // NOTE: There is no relationship between these intrinsics beyond an attempt + // to reuse currently identical class definitions. + class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic; + // This class of intrinsics are not intended to be useful within LLVM IR but // are instead here to support some of the more regid parts of the ACLE. class Builtin_SVCVT<string name, LLVMType OUT, LLVMType IN> @@ -812,10 +1090,132 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". +class AdvSIMD_SVE_Reduce_Intrinsic + : Intrinsic<[LLVMVectorElementType<0>], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty], + [IntrNoMem]>; + +class AdvSIMD_SVE_SADDV_Reduce_Intrinsic + : Intrinsic<[llvm_i64_ty], + [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty], + [IntrNoMem]>; + +class AdvSIMD_SVE_WHILE_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [llvm_anyint_ty, LLVMMatchType<1>], + [IntrNoMem]>; + +class AdvSIMD_GatherLoad_64bitOffset_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [ + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> + ], + [IntrReadMem, IntrArgMemOnly]>; + +class AdvSIMD_GatherLoad_32bitOffset_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [ + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> + ], + [IntrReadMem, IntrArgMemOnly]>; + +class AdvSIMD_GatherLoad_VecTorBase_Intrinsic + : Intrinsic<[llvm_anyvector_ty], + [ + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty, + llvm_i64_ty + ], + [IntrReadMem, IntrArgMemOnly]>; + +class AdvSIMD_ScatterStore_64bitOffset_Intrinsic + : Intrinsic<[], + [ + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> + ], + [IntrWriteMem, IntrArgMemOnly]>; + +class AdvSIMD_ScatterStore_32bitOffset_Intrinsic + : Intrinsic<[], + [ + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMPointerToElt<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> + ], + [IntrWriteMem, IntrArgMemOnly]>; + +class AdvSIMD_ScatterStore_VectorBase_Intrinsic + : Intrinsic<[], + [ + llvm_anyvector_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + llvm_anyvector_ty, llvm_i64_ty + ], + [IntrWriteMem, IntrArgMemOnly, ImmArg<3>]>; + +// +// Loads +// + +def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic; + +// +// Stores +// + +def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic; + // // Integer arithmetic // +def int_aarch64_sve_add : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_subr : AdvSIMD_Pred2VectorArg_Intrinsic; + +def int_aarch64_sve_mul : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_smulh : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_umulh : AdvSIMD_Pred2VectorArg_Intrinsic; + +def int_aarch64_sve_sdiv : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_udiv : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_sdivr : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_udivr : AdvSIMD_Pred2VectorArg_Intrinsic; + +def int_aarch64_sve_smax : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_umax : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_smin : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_umin : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic; + +def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic; + +def int_aarch64_sve_saddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; +def int_aarch64_sve_uaddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; + +def int_aarch64_sve_smaxv : AdvSIMD_SVE_Reduce_Intrinsic; +def int_aarch64_sve_umaxv : AdvSIMD_SVE_Reduce_Intrinsic; +def int_aarch64_sve_sminv : AdvSIMD_SVE_Reduce_Intrinsic; +def int_aarch64_sve_uminv : AdvSIMD_SVE_Reduce_Intrinsic; + +def int_aarch64_sve_orv : AdvSIMD_SVE_Reduce_Intrinsic; +def int_aarch64_sve_eorv : AdvSIMD_SVE_Reduce_Intrinsic; +def int_aarch64_sve_andv : AdvSIMD_SVE_Reduce_Intrinsic; + def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; @@ -825,32 +1225,480 @@ def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic; def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; +// Shifts + +def int_aarch64_sve_asr : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; +def int_aarch64_sve_asrd : AdvSIMD_SVE_ShiftByImm_Intrinsic; +def int_aarch64_sve_insr : AdvSIMD_SVE_INSR_Intrinsic; +def int_aarch64_sve_lsl : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; +def int_aarch64_sve_lsr : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; + +// +// Integer comparisons +// + +def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic; + +def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; +def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic; + // // Counting bits // +def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic; def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic; // +// Counting elements +// + +def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic; +def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic; +def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic; +def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic; + +def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; + +// +// Saturating scalar arithmetic +// + +def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic; + +def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; + +def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic; + +def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; + +def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic; + +def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; + +def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; +def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic; + +def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; +def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; +def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; + +// +// Reversal +// + +def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic; + +// // Permutations and selection // +def int_aarch64_sve_clasta : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_clasta_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; +def int_aarch64_sve_clastb : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_clastb_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; +def int_aarch64_sve_compact : AdvSIMD_Pred1VectorArg_Intrinsic; +def int_aarch64_sve_ext : AdvSIMD_2VectorArgIndexed_Intrinsic; +def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic; +def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic; +def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic; +def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic; def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; - +def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic; +def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic; def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; +def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic; + +// +// Logical operations +// + +def int_aarch64_sve_and : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_bic : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_eor : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_not : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_orr : AdvSIMD_Pred2VectorArg_Intrinsic; + +// +// Conversion +// + +def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic; + +// +// While comparisons +// + +def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic; +def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic; +def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic; +def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic; +def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic; +def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic; +def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic; +def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic; + +// +// Floating-point arithmetic +// + +def int_aarch64_sve_fabd : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fabs : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_fadd : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fcadd : AdvSIMD_SVE_CADD_Intrinsic; +def int_aarch64_sve_fcmla : AdvSIMD_SVE_CMLA_Intrinsic; +def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic; +def int_aarch64_sve_fdiv : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fdivr : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fexpa_x : AdvSIMD_SVE_EXPA_Intrinsic; +def int_aarch64_sve_fmad : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_fmax : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fmaxnm : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fmin : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fminnm : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fmla : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_fmla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; +def int_aarch64_sve_fmls : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_fmls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; +def int_aarch64_sve_fmsb : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_fmul : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fmulx : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fneg : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_fmul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; +def int_aarch64_sve_fnmad : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_fnmla : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_fnmls : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_fnmsb : AdvSIMD_Pred3VectorArg_Intrinsic; +def int_aarch64_sve_frecpe_x : AdvSIMD_1VectorArg_Intrinsic; +def int_aarch64_sve_frecps_x : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_frecpx : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frinta : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frinti : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frintm : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frintn : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frintp : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frintx : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frintz : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_frsqrte_x : AdvSIMD_1VectorArg_Intrinsic; +def int_aarch64_sve_frsqrts_x : AdvSIMD_2VectorArg_Intrinsic; +def int_aarch64_sve_fscale : AdvSIMD_SVE_SCALE_Intrinsic; +def int_aarch64_sve_fsqrt : AdvSIMD_Merged1VectorArg_Intrinsic; +def int_aarch64_sve_fsub : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fsubr : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_ftmad_x : AdvSIMD_2VectorArgIndexed_Intrinsic; +def int_aarch64_sve_ftsmul_x : AdvSIMD_SVE_TSMUL_Intrinsic; +def int_aarch64_sve_ftssel_x : AdvSIMD_SVE_TSMUL_Intrinsic; + +// +// Floating-point reductions +// + +def int_aarch64_sve_fadda : AdvSIMD_SVE_FP_ReduceWithInit_Intrinsic; +def int_aarch64_sve_faddv : AdvSIMD_SVE_FP_Reduce_Intrinsic; +def int_aarch64_sve_fmaxv : AdvSIMD_SVE_FP_Reduce_Intrinsic; +def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_FP_Reduce_Intrinsic; +def int_aarch64_sve_fminv : AdvSIMD_SVE_FP_Reduce_Intrinsic; +def int_aarch64_sve_fminnmv : AdvSIMD_SVE_FP_Reduce_Intrinsic; + +// +// Floating-point conversions +// + +def int_aarch64_sve_fcvt : AdvSIMD_SVE_FCVT_Intrinsic; +def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic; +def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic; +def int_aarch64_sve_scvtf : AdvSIMD_SVE_SCVTF_Intrinsic; +def int_aarch64_sve_ucvtf : AdvSIMD_SVE_SCVTF_Intrinsic; // // Floating-point comparisons // -def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic; + +def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic; +def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic; + +def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_fcvtzs_i32f64 : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2f64_ty>; +def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv4f32_ty>; + +def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2f64_ty>; +def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_fcvtzu_i64f32 : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv4f32_ty>; + +def int_aarch64_sve_fcvt_f16f32 : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4f32_ty>; +def int_aarch64_sve_fcvt_f16f64 : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2f64_ty>; +def int_aarch64_sve_fcvt_f32f64 : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2f64_ty>; + +def int_aarch64_sve_fcvt_f32f16 : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_fcvt_f64f16 : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_fcvt_f64f32 : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv4f32_ty>; + +def int_aarch64_sve_fcvtlt_f32f16 : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv8f16_ty>; +def int_aarch64_sve_fcvtlt_f64f32 : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv4f32_ty>; +def int_aarch64_sve_fcvtnt_f16f32 : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4f32_ty>; +def int_aarch64_sve_fcvtnt_f32f64 : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2f64_ty>; + +def int_aarch64_sve_fcvtx_f32f64 : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2f64_ty>; +def int_aarch64_sve_fcvtxnt_f32f64 : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2f64_ty>; + +def int_aarch64_sve_scvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i32_ty>; +def int_aarch64_sve_scvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i64_ty>; +def int_aarch64_sve_scvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i64_ty>; +def int_aarch64_sve_scvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv4i32_ty>; + +def int_aarch64_sve_ucvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i32_ty>; +def int_aarch64_sve_ucvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i64_ty>; +def int_aarch64_sve_ucvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i64_ty>; +def int_aarch64_sve_ucvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv4i32_ty>; + +// +// Predicate creation +// + +def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic; // // Predicate operations // +def int_aarch64_sve_and_z : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_bic_z : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_eor_z : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_nand_z : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_nor_z : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_orn_z : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_orr_z : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_pfirst : AdvSIMD_Pred1VectorArg_Intrinsic; +def int_aarch64_sve_pnext : AdvSIMD_Pred1VectorArg_Intrinsic; def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic; def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic; + +// +// Testing predicates +// + +def int_aarch64_sve_ptest_any : AdvSIMD_SVE_PTEST_Intrinsic; +def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic; +def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic; + +// +// Gather loads: +// + +// scalar + vector, 64 bit unscaled offsets +def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_64bitOffset_Intrinsic; + +// scalar + vector, 64 bit scaled offsets +def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_64bitOffset_Intrinsic; + +// scalar + vector, 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) +// extended to 64 bits +def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_32bitOffset_Intrinsic; +def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_32bitOffset_Intrinsic; + +// scalar + vector, 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended +// to 64 bits +def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_32bitOffset_Intrinsic; +def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_32bitOffset_Intrinsic; + +// vector base + immediate index +def int_aarch64_sve_ld1_gather_imm : AdvSIMD_GatherLoad_VecTorBase_Intrinsic; + +// +// Scatter stores: +// + +// scalar + vector, 64 bit unscaled offsets +def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_64bitOffset_Intrinsic; + +// scalar + vector, 64 bit scaled offsets +def int_aarch64_sve_st1_scatter_index + : AdvSIMD_ScatterStore_64bitOffset_Intrinsic; + +// scalar + vector, 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) +// extended to 64 bits +def int_aarch64_sve_st1_scatter_sxtw + : AdvSIMD_ScatterStore_32bitOffset_Intrinsic; + +def int_aarch64_sve_st1_scatter_uxtw + : AdvSIMD_ScatterStore_32bitOffset_Intrinsic; + +// scalar + vector, 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended +// to 64 bits +def int_aarch64_sve_st1_scatter_sxtw_index + : AdvSIMD_ScatterStore_32bitOffset_Intrinsic; + +def int_aarch64_sve_st1_scatter_uxtw_index + : AdvSIMD_ScatterStore_32bitOffset_Intrinsic; + +// vector base + immediate index +def int_aarch64_sve_st1_scatter_imm : AdvSIMD_ScatterStore_VectorBase_Intrinsic; + +// +// SVE2 - Non-widening pairwise arithmetic +// + +def int_aarch64_sve_faddp : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fmaxp : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fminp : AdvSIMD_Pred2VectorArg_Intrinsic; +def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic; + +// +// SVE2 - Floating-point widening multiply-accumulate +// + +def int_aarch64_sve_fmlalb : SVE2_3VectorArg_Long_Intrinsic; +def int_aarch64_sve_fmlalb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; +def int_aarch64_sve_fmlalt : SVE2_3VectorArg_Long_Intrinsic; +def int_aarch64_sve_fmlalt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; +def int_aarch64_sve_fmlslb : SVE2_3VectorArg_Long_Intrinsic; +def int_aarch64_sve_fmlslb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; +def int_aarch64_sve_fmlslt : SVE2_3VectorArg_Long_Intrinsic; +def int_aarch64_sve_fmlslt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; + +// +// SVE2 - Floating-point integer binary logarithm +// + +def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic; + +// +// SVE2 - Unary narrowing operations +// + +def int_aarch64_sve_sqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_sqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_uqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_uqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; + +// +// SVE2 - Binary narrowing DSP operations +// +def int_aarch64_sve_addhnb : SVE2_2VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_addhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; + +def int_aarch64_sve_raddhnb : SVE2_2VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_raddhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; + +def int_aarch64_sve_subhnb : SVE2_2VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_subhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; + +def int_aarch64_sve_rsubhnb : SVE2_2VectorArg_Narrowing_Intrinsic; +def int_aarch64_sve_rsubhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; + +// Narrowing shift right +def int_aarch64_sve_shrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_shrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +def int_aarch64_sve_rshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_rshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +// Saturating shift right - signed input/output +def int_aarch64_sve_sqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_sqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +def int_aarch64_sve_sqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_sqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +// Saturating shift right - unsigned input/output +def int_aarch64_sve_uqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_uqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +def int_aarch64_sve_uqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_uqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +// Saturating shift right - signed input, unsigned output +def int_aarch64_sve_sqshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_sqshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; + +def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; +def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; } diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index ab6ee7f92dd1..07ca3a9229d6 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -218,7 +218,7 @@ def int_amdgcn_s_waitcnt : GCCBuiltin<"__builtin_amdgcn_s_waitcnt">, def int_amdgcn_div_scale : Intrinsic< // 1st parameter: Numerator // 2nd parameter: Denominator - // 3rd parameter: Constant to select select between first and + // 3rd parameter: Constant to select between first and // second. (0 = first, 1 = second). [llvm_anyfloat_ty, llvm_i1_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty], diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td index e13da6157e04..518ad7079225 100644 --- a/llvm/include/llvm/IR/IntrinsicsARM.td +++ b/llvm/include/llvm/IR/IntrinsicsARM.td @@ -426,8 +426,6 @@ let IntrProperties = [IntrNoMem, Commutative] in { def int_arm_neon_vhaddu : Neon_2Arg_Intrinsic; def int_arm_neon_vrhadds : Neon_2Arg_Intrinsic; def int_arm_neon_vrhaddu : Neon_2Arg_Intrinsic; - def int_arm_neon_vqadds : Neon_2Arg_Intrinsic; - def int_arm_neon_vqaddu : Neon_2Arg_Intrinsic; def int_arm_neon_vraddhn : Neon_2Arg_Narrow_Intrinsic; // Vector Multiply. @@ -459,8 +457,6 @@ let IntrProperties = [IntrNoMem, Commutative] in { // Vector Subtract. def int_arm_neon_vhsubs : Neon_2Arg_Intrinsic; def int_arm_neon_vhsubu : Neon_2Arg_Intrinsic; -def int_arm_neon_vqsubs : Neon_2Arg_Intrinsic; -def int_arm_neon_vqsubu : Neon_2Arg_Intrinsic; def int_arm_neon_vrsubhn : Neon_2Arg_Narrow_Intrinsic; // Vector Absolute Compare. @@ -777,14 +773,352 @@ class Neon_Dot_Intrinsic def int_arm_neon_udot : Neon_Dot_Intrinsic; def int_arm_neon_sdot : Neon_Dot_Intrinsic; -def int_arm_vctp8 : Intrinsic<[llvm_v16i1_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_arm_vctp16 : Intrinsic<[llvm_v8i1_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_arm_vctp32 : Intrinsic<[llvm_v4i1_ty], [llvm_i32_ty], [IntrNoMem]>; -def int_arm_vctp64 : Intrinsic<[llvm_v2i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>; + +def int_arm_mve_vctp8 : Intrinsic<[llvm_v16i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_mve_vctp16 : Intrinsic<[llvm_v8i1_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_mve_vctp32 : Intrinsic<[llvm_v4i1_ty], [llvm_i32_ty], [IntrNoMem]>; +// vctp64 takes v4i1, to work around v2i1 not being a legal MVE type +def int_arm_mve_vctp64 : Intrinsic<[llvm_v4i1_ty], [llvm_i32_ty], [IntrNoMem]>; + +// v8.3-A Floating-point complex add +def int_arm_neon_vcadd_rot90 : Neon_2Arg_Intrinsic; +def int_arm_neon_vcadd_rot270 : Neon_2Arg_Intrinsic; // GNU eabi mcount def int_arm_gnu_eabi_mcount : Intrinsic<[], [], [IntrReadMem, IntrWriteMem]>; +def int_arm_mve_pred_i2v : Intrinsic< + [llvm_anyvector_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_arm_mve_pred_v2i : Intrinsic< + [llvm_i32_ty], [llvm_anyvector_ty], [IntrNoMem]>; + +multiclass IntrinsicSignSuffix<list<LLVMType> rets, list<LLVMType> params = [], + list<IntrinsicProperty> props = [], + string name = "", + list<SDNodeProperty> sdprops = []> { + def _s: Intrinsic<rets, params, props, name, sdprops>; + def _u: Intrinsic<rets, params, props, name, sdprops>; +} + +def int_arm_mve_min_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_max_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_abd_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], [IntrNoMem]>; +def int_arm_mve_add_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_and_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_bic_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_eor_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_orn_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_orr_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_sub_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_mul_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_mulh_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_qdmulh_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_rmulh_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_qrdmulh_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_mull_int_predicated: Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty /* unsigned */, + llvm_i32_ty /* top */, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_mull_poly_predicated: Intrinsic<[llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty, llvm_anyvector_ty, + LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_qadd_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], [IntrNoMem]>; +def int_arm_mve_hadd_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], [IntrNoMem]>; +def int_arm_mve_rhadd_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], [IntrNoMem]>; +def int_arm_mve_qsub_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], [IntrNoMem]>; +def int_arm_mve_hsub_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */, + llvm_anyvector_ty, LLVMMatchType<0>], [IntrNoMem]>; + +defm int_arm_mve_minv: IntrinsicSignSuffix<[llvm_i32_ty], + [llvm_i32_ty, llvm_anyvector_ty], [IntrNoMem]>; +defm int_arm_mve_maxv: IntrinsicSignSuffix<[llvm_i32_ty], + [llvm_i32_ty, llvm_anyvector_ty], [IntrNoMem]>; + +multiclass MVEPredicated<list<LLVMType> rets, list<LLVMType> params, + LLVMType pred = llvm_anyvector_ty, + list<IntrinsicProperty> props = [IntrNoMem]> { + def "": Intrinsic<rets, params, props>; + def _predicated: Intrinsic<rets, params # [pred], props>; +} +multiclass MVEPredicatedM<list<LLVMType> rets, list<LLVMType> params, + LLVMType pred = llvm_anyvector_ty, + list<IntrinsicProperty> props = [IntrNoMem]> { + def "": Intrinsic<rets, params, props>; + def _predicated: Intrinsic<rets, params # [pred, + !if(!eq(!cast<string>(rets[0]), "llvm_anyvector_ty"), + LLVMMatchType<0>, rets[0])], props>; +} + +defm int_arm_mve_vcvt_narrow: MVEPredicated<[llvm_v8f16_ty], + [llvm_v8f16_ty, llvm_v4f32_ty, llvm_i32_ty], llvm_v4i1_ty>; + +defm int_arm_mve_vldr_gather_base: MVEPredicated< + [llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty], + llvm_anyvector_ty, [IntrReadMem]>; +defm int_arm_mve_vldr_gather_base_wb: MVEPredicated< + [llvm_anyvector_ty, llvm_anyvector_ty], + [LLVMMatchType<1>, llvm_i32_ty], llvm_anyvector_ty, [IntrReadMem]>; +defm int_arm_mve_vstr_scatter_base: MVEPredicated< + [], [llvm_anyvector_ty, llvm_i32_ty, llvm_anyvector_ty], + llvm_anyvector_ty, [IntrWriteMem]>; +defm int_arm_mve_vstr_scatter_base_wb: MVEPredicated< + [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty], + llvm_anyvector_ty, [IntrWriteMem]>; + +// gather_offset takes three i32 parameters. The first is the size of +// memory element loaded, in bits. The second is a left bit shift to +// apply to each offset in the vector parameter (must be either 0, or +// correspond to the element size of the destination vector type). The +// last is 1 to indicate zero extension (if the load is widening), or +// 0 for sign extension. +// +// scatter_offset has the first two of those parameters, but since it +// narrows rather than widening, it doesn't have the last one. +defm int_arm_mve_vldr_gather_offset: MVEPredicated< + [llvm_anyvector_ty], [llvm_anyptr_ty, llvm_anyvector_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], llvm_anyvector_ty, [IntrReadMem]>; +defm int_arm_mve_vstr_scatter_offset: MVEPredicated< + [], [llvm_anyptr_ty, llvm_anyvector_ty, llvm_anyvector_ty, + llvm_i32_ty, llvm_i32_ty], llvm_anyvector_ty, [IntrWriteMem]>; + +def int_arm_mve_shl_imm_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; +def int_arm_mve_shr_imm_predicated: Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, // extra i32 is unsigned flag + llvm_anyvector_ty, LLVMMatchType<0>], + [IntrNoMem]>; + +defm int_arm_mve_vqshl_imm: MVEPredicatedM<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_i32_ty /*shiftcount*/, llvm_i32_ty /*unsigned*/]>; +defm int_arm_mve_vrshr_imm: MVEPredicatedM<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_i32_ty /*shiftcount*/, llvm_i32_ty /*unsigned*/]>; +defm int_arm_mve_vqshlu_imm: MVEPredicatedM<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_i32_ty /*shiftcount*/]>; +defm int_arm_mve_vshll_imm: MVEPredicatedM<[llvm_anyvector_ty], + [llvm_anyvector_ty, llvm_i32_ty /*shiftcount*/, llvm_i32_ty /*unsigned*/, + llvm_i32_ty /*top-half*/]>; + +defm int_arm_mve_vsli: MVEPredicated< + [llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty]>; +defm int_arm_mve_vsri: MVEPredicated< + [llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty]>; + +defm int_arm_mve_vshrn: MVEPredicated< + [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty, + llvm_i32_ty /*shiftcount*/, llvm_i32_ty /*saturate*/, llvm_i32_ty /*round*/, + llvm_i32_ty /*unsigned-out*/, llvm_i32_ty /*unsigned-in*/, + llvm_i32_ty /*top-half*/]>; + +defm int_arm_mve_vshl_scalar: MVEPredicated< + [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_i32_ty /*shiftcount*/, + llvm_i32_ty /*saturate*/, llvm_i32_ty /*round*/, llvm_i32_ty /*unsigned*/]>; +defm int_arm_mve_vshl_vector: MVEPredicatedM< + [llvm_anyvector_ty], [LLVMMatchType<0>, llvm_anyvector_ty /*shiftcounts*/, + llvm_i32_ty /*saturate*/, llvm_i32_ty /*round*/, llvm_i32_ty /*unsigned*/]>; + +// MVE scalar shifts. +class ARM_MVE_qrshift_single<list<LLVMType> value, + list<LLVMType> saturate = []> : + Intrinsic<value, value # [llvm_i32_ty] # saturate, [IntrNoMem]>; +multiclass ARM_MVE_qrshift<list<LLVMType> saturate = []> { + // Most of these shifts come in 32- and 64-bit versions. But only + // the 64-bit ones have the extra saturation argument (if any). + def "": ARM_MVE_qrshift_single<[llvm_i32_ty]>; + def l: ARM_MVE_qrshift_single<[llvm_i32_ty, llvm_i32_ty], saturate>; +} +defm int_arm_mve_urshr: ARM_MVE_qrshift; +defm int_arm_mve_uqshl: ARM_MVE_qrshift; +defm int_arm_mve_srshr: ARM_MVE_qrshift; +defm int_arm_mve_sqshl: ARM_MVE_qrshift; +defm int_arm_mve_uqrshl: ARM_MVE_qrshift<[llvm_i32_ty]>; +defm int_arm_mve_sqrshr: ARM_MVE_qrshift<[llvm_i32_ty]>; +// LSLL and ASRL only have 64-bit versions, not 32. +def int_arm_mve_lsll: ARM_MVE_qrshift_single<[llvm_i32_ty, llvm_i32_ty]>; +def int_arm_mve_asrl: ARM_MVE_qrshift_single<[llvm_i32_ty, llvm_i32_ty]>; + +def int_arm_mve_vabd: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */], + [IntrNoMem]>; +def int_arm_mve_vadc: Intrinsic< + [llvm_anyvector_ty, llvm_i32_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], [IntrNoMem]>; +def int_arm_mve_vadc_predicated: Intrinsic< + [llvm_anyvector_ty, llvm_i32_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, + llvm_i32_ty, llvm_anyvector_ty], [IntrNoMem]>; +def int_arm_mve_vmulh: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */], + [IntrNoMem]>; +def int_arm_mve_vqdmulh: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; +def int_arm_mve_vhadd: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */], + [IntrNoMem]>; +def int_arm_mve_vrhadd: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */], + [IntrNoMem]>; +def int_arm_mve_vhsub: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */], + [IntrNoMem]>; +def int_arm_mve_vrmulh: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty /* unsigned */], + [IntrNoMem]>; +def int_arm_mve_vqrdmulh: Intrinsic< + [llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; +def int_arm_mve_vmull: Intrinsic< + [llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty /* unsigned */, + llvm_i32_ty /* top */], [IntrNoMem]>; +def int_arm_mve_vmull_poly: Intrinsic< + [llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty], [IntrNoMem]>; + +// Intrinsic with a predicated and a non-predicated case. The predicated case +// has two additional parameters: inactive (the value for inactive lanes, can +// be undef) and predicate. +multiclass MVEMXPredicated<list<LLVMType> rets, list<LLVMType> flags, + list<LLVMType> params, LLVMType inactive, + LLVMType predicate, + list<IntrinsicProperty> props = [IntrNoMem]> { + def "": Intrinsic<rets, flags # params, props>; + def _predicated: Intrinsic<rets, flags # [inactive] # params # [predicate], + props>; +} + +// The first two parameters are compile-time constants: +// * Halving: 0 means halving (vhcaddq), 1 means non-halving (vcaddq) +// instruction. Note: the flag is inverted to match the corresonding +// bit in the instruction encoding +// * Rotation angle: 0 mean 90 deg, 1 means 180 deg +defm int_arm_mve_vcaddq : MVEMXPredicated< + [llvm_anyvector_ty], + [llvm_i32_ty, llvm_i32_ty], [LLVMMatchType<0>, LLVMMatchType<0>], + LLVMMatchType<0>, llvm_anyvector_ty>; + +// The first operand of the following two intrinsics is the rotation angle +// (must be a compile-time constant): +// 0 - 0 deg +// 1 - 90 deg +// 2 - 180 deg +// 3 - 270 deg +defm int_arm_mve_vcmulq : MVEMXPredicated< + [llvm_anyvector_ty], + [llvm_i32_ty], [LLVMMatchType<0>, LLVMMatchType<0>], + LLVMMatchType<0>, llvm_anyvector_ty>; + +defm int_arm_mve_vcmlaq : MVEPredicated< + [llvm_anyvector_ty], + [llvm_i32_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + llvm_anyvector_ty>; + +def int_arm_mve_vld2q: Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_anyptr_ty], [IntrReadMem]>; +def int_arm_mve_vld4q: Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [llvm_anyptr_ty], [IntrReadMem]>; + +def int_arm_mve_vst2q: Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty], [IntrWriteMem]>; +def int_arm_mve_vst4q: Intrinsic<[], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, LLVMMatchType<1>, llvm_i32_ty], [IntrWriteMem] +>; + +// MVE vector absolute difference and accumulate across vector +// The first operand is an 'unsigned' flag. The remaining operands are: +// * accumulator +// * first vector operand +// * second vector operand +// * mask (only in predicated versions) +defm int_arm_mve_vabav: MVEPredicated< + [llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_anyvector_ty, LLVMMatchType<0>], llvm_anyvector_ty>; + +// The following 3 instrinsics are MVE vector reductions with two vector +// operands. +// The first 3 operands are boolean flags (must be compile-time constants): +// * unsigned - the instruction operates on vectors of unsigned values and +// unsigned scalars +// * subtract - the instruction performs subtraction after multiplication of +// lane pairs (e.g., vmlsdav vs vmladav) +// * exchange - the instruction exchanges successive even and odd lanes of +// the first operands before multiplication of lane pairs +// (e.g., vmladavx vs vmladav) +// The remaining operands are: +// * accumulator +// * first vector operand +// * second vector operand +// * mask (only in predicated versions) + +// Version with 32-bit result, vml{a,s}dav[a][x] +defm int_arm_mve_vmldava: MVEPredicated< + [llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_anyvector_ty, LLVMMatchType<0>], + llvm_anyvector_ty>; + +// Version with 64-bit result, vml{a,s}ldav[a][x] +defm int_arm_mve_vmlldava: MVEPredicated< + [llvm_i32_ty, llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_anyvector_ty, LLVMMatchType<0>], + llvm_anyvector_ty>; + +// Version with 72-bit rounded result, vrml{a,s}ldavh[a][x] +defm int_arm_mve_vrmlldavha: MVEPredicated< + [llvm_i32_ty, llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_anyvector_ty, LLVMMatchType<0>], + llvm_anyvector_ty>; } // end TargetPrefix diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td index 0483d965ba64..ec328d69a8dd 100644 --- a/llvm/include/llvm/IR/IntrinsicsNVVM.td +++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td @@ -53,6 +53,10 @@ class WMMA_REGS<string Geom, string Frag, string PtxEltType> { string gft = Geom#":"#Frag#":"#ptx_elt_type; string ft = frag#":"#ptx_elt_type; list<LLVMType> regs = !cond( + // mma.sync.m8n8k4 uses smaller a/b fragments than wmma fp ops + !eq(gft,"m8n8k4:a:f16") : RepLLVMType<2, llvm_v2f16_ty>.ret, + !eq(gft,"m8n8k4:b:f16") : RepLLVMType<2, llvm_v2f16_ty>.ret, + // fp16 -> fp16/fp32 @ m16n16k16/m8n32k16/m32n8k16 // All currently supported geometries use the same fragment format, // so we only need to consider {fragment, type}. @@ -137,13 +141,19 @@ class MMA_SIGNATURE<WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { class WMMA_NAME_MMA<string ALayout, string BLayout, int Satfinite, WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> { string signature = MMA_SIGNATURE<A, B, C, D>.ret; - string llvm = "llvm.nvvm.wmma." - # A.geom - # ".mma" - # "." # ALayout - # "." # BLayout - # signature - # !if(Satfinite, ".satfinite", ""); + string llvm = !if( + !eq(A.geom, "m8n8k4"), + "llvm.nvvm.mma.m8n8k4" + # "." # ALayout + # "." # BLayout + # signature, + "llvm.nvvm.wmma." + # A.geom + # ".mma" + # "." # ALayout + # "." # BLayout + # signature + # !if(Satfinite, ".satfinite", "")); string record = !subst(".", "_", !subst("llvm.", "int_", llvm)); @@ -160,7 +170,7 @@ class MMA_OPS<list<string> Geom, list<string> TypeA, list<string> TypeB, !foldl([]<list<WMMA_REGS>>, TypeA, t2, type_a, !listconcat(t2, !foldl([]<list<WMMA_REGS>>, !if(!size(TypeB), TypeB, [type_a]), t3, type_b, !listconcat(t3, !foldl([]<list<WMMA_REGS>>, TypeC, t4, type_c, !listconcat(t4, - !foldl([]<list<WMMA_REGS>>, !if(!size(TypeC), TypeC, [type_c]), t5, type_d, !listconcat(t5, + !foldl([]<list<WMMA_REGS>>, !if(!size(TypeD), TypeD, [type_c]), t5, type_d, !listconcat(t5, [[WMMA_REGS<geom, "a", type_a>, WMMA_REGS<geom, "b", type_b>, WMMA_REGS<geom, "c", type_c>, @@ -185,19 +195,23 @@ class MMA_LDST_OPS<list<string> Geom, list<string> Frags, list<string> Types> { // drives generation of corresponding intrinsics and instructions. class NVVM_MMA_OPS<int _ = 0> { list<list<WMMA_REGS>> fp_mma_ops = MMA_OPS< + ["m8n8k4"], + ["f16"], [], ["f16", "f32"], ["f16", "f32"]>.ret; + list<list<WMMA_REGS>> fp_wmma_ops = MMA_OPS< ["m16n16k16", "m32n8k16", "m8n32k16"], ["f16"], [], ["f16", "f32"], ["f16", "f32"]>.ret; - list<list<WMMA_REGS>> int_mma_ops = MMA_OPS< + list<list<WMMA_REGS>> int_wmma_ops = MMA_OPS< ["m16n16k16", "m32n8k16", "m8n32k16"], ["s8", "u8"], [], ["s32"], []>.ret; - list<list<WMMA_REGS>> subint_mma_ops = MMA_OPS< + list<list<WMMA_REGS>> subint_wmma_ops = MMA_OPS< ["m8n8k32"], ["s4", "u4"], [], ["s32"], []>.ret; - list<list<WMMA_REGS>> bit_mma_ops = MMA_OPS< + list<list<WMMA_REGS>> bit_wmma_ops = MMA_OPS< ["m8n8k128"], ["b1"], [], ["s32"], []>.ret; - list<list<WMMA_REGS>> all_mma_ops = !listconcat(fp_mma_ops, int_mma_ops, - subint_mma_ops, bit_mma_ops); + list<list<WMMA_REGS>> all_mma_ops = !listconcat( + fp_mma_ops, fp_wmma_ops, int_wmma_ops, + subint_wmma_ops, bit_wmma_ops); list<WMMA_REGS> ldst_ab_ops = MMA_LDST_OPS< ["m16n16k16", "m32n8k16", "m8n32k16"], @@ -245,10 +259,25 @@ class NVVM_MMA_SUPPORTED<list<WMMA_REGS> frags, string layout_a, string layout_b # ":" # frags[0].frag ; string t = frags[0].ptx_elt_type; + + // gcd is a shortcut used to identify instructions that depend on + // geom+frag_c+frag_d. Not all instances of this class have all fragments + // specified. If there are not enough fragments, the tail evaluates to '?'. + string gcd = frags[0].geom + # ":" + # !if(!eq(!size(frags), 4), + frags[2].ptx_elt_type # frags[3].ptx_elt_type, + "?"); list<int> ret = !cond( // Sub-int MMA only supports fixed A/B layout. // b1 does not support .satf. !eq(mma#":"#satf, "b1:row:col:0") : [1], + // mma.m8n8k4 has no .satf modifier. + !and(!eq(frags[0].geom, "m8n8k4"), + !ne(satf, 0)): [], + + // mma.m8n8k4 has no C=f32 D=f16 variant. + !eq(gcd, "m8n8k4:f32f16"): [], !eq(mma, "s4:row:col") : [1], !eq(mma, "u4:row:col") : [1], !eq(mma, "s4:row:col") : [1], @@ -4094,7 +4123,7 @@ class NVVM_WMMA_ST<WMMA_REGS Frag, string Layout, int WithStride> [IntrWriteMem, IntrArgMemOnly, WriteOnly<0>, NoCapture<0>], WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.intr>; -// Create all load/store variants +// Create all load/store variants foreach layout = ["row", "col"] in { foreach stride = [0, 1] in { foreach frag = NVVM_MMA_OPS.all_ld_ops in diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 60393189b830..2039ad1a26b8 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -10,59 +10,59 @@ // //===----------------------------------------------------------------------===// -let TargetPrefix = "riscv" in { - //===----------------------------------------------------------------------===// // Atomics -class MaskedAtomicRMW32Intrinsic - : Intrinsic<[llvm_i32_ty], - [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrArgMemOnly, NoCapture<0>, ImmArg<3>]>; - -class MaskedAtomicRMW32WithSextIntrinsic - : Intrinsic<[llvm_i32_ty], - [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty], - [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; +// Atomic Intrinsics have multiple versions for different access widths, which +// all follow one of the following signatures (depending on how many arguments +// they require). We carefully instantiate only specific versions of these for +// specific integer widths, rather than using `llvm_anyint_ty`. +// +// In fact, as these intrinsics take `llvm_anyptr_ty`, the given names are the +// canonical names, and the intrinsics used in the code will have a name +// suffixed with the pointer type they are specialised for (denoted `<p>` in the +// names below), in order to avoid type conflicts. -def int_riscv_masked_atomicrmw_xchg_i32 : MaskedAtomicRMW32Intrinsic; -def int_riscv_masked_atomicrmw_add_i32 : MaskedAtomicRMW32Intrinsic; -def int_riscv_masked_atomicrmw_sub_i32 : MaskedAtomicRMW32Intrinsic; -def int_riscv_masked_atomicrmw_nand_i32 : MaskedAtomicRMW32Intrinsic; -def int_riscv_masked_atomicrmw_max_i32 : MaskedAtomicRMW32WithSextIntrinsic; -def int_riscv_masked_atomicrmw_min_i32 : MaskedAtomicRMW32WithSextIntrinsic; -def int_riscv_masked_atomicrmw_umax_i32 : MaskedAtomicRMW32Intrinsic; -def int_riscv_masked_atomicrmw_umin_i32 : MaskedAtomicRMW32Intrinsic; +let TargetPrefix = "riscv" in { -def int_riscv_masked_cmpxchg_i32 - : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], - [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; + // T @llvm.<name>.T.<p>(any*, T, T, T imm); + class MaskedAtomicRMWFourArg<LLVMType itype> + : Intrinsic<[itype], [llvm_anyptr_ty, itype, itype, itype], + [IntrArgMemOnly, NoCapture<0>, ImmArg<3>]>; + // T @llvm.<name>.T.<p>(any*, T, T, T, T imm); + class MaskedAtomicRMWFiveArg<LLVMType itype> + : Intrinsic<[itype], [llvm_anyptr_ty, itype, itype, itype, itype], + [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; -class MaskedAtomicRMW64Intrinsic - : Intrinsic<[llvm_i64_ty], - [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], - [IntrArgMemOnly, NoCapture<0>, ImmArg<3>]>; + // We define 32-bit and 64-bit variants of the above, where T stands for i32 + // or i64 respectively: + multiclass MaskedAtomicRMWFourArgIntrinsics { + // i32 @llvm.<name>.i32.<p>(any*, i32, i32, i32 imm); + def _i32 : MaskedAtomicRMWFourArg<llvm_i32_ty>; + // i64 @llvm.<name>.i32.<p>(any*, i64, i64, i64 imm); + def _i64 : MaskedAtomicRMWFourArg<llvm_i64_ty>; + } -class MaskedAtomicRMW64WithSextIntrinsic - : Intrinsic<[llvm_i64_ty], - [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty, - llvm_i64_ty], - [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; + multiclass MaskedAtomicRMWFiveArgIntrinsics { + // i32 @llvm.<name>.i32.<p>(any*, i32, i32, i32, i32 imm); + def _i32 : MaskedAtomicRMWFiveArg<llvm_i32_ty>; + // i64 @llvm.<name>.i64.<p>(any*, i64, i64, i64, i64 imm); + def _i64 : MaskedAtomicRMWFiveArg<llvm_i64_ty>; + } -def int_riscv_masked_atomicrmw_xchg_i64 : MaskedAtomicRMW64Intrinsic; -def int_riscv_masked_atomicrmw_add_i64 : MaskedAtomicRMW64Intrinsic; -def int_riscv_masked_atomicrmw_sub_i64 : MaskedAtomicRMW64Intrinsic; -def int_riscv_masked_atomicrmw_nand_i64 : MaskedAtomicRMW64Intrinsic; -def int_riscv_masked_atomicrmw_max_i64 : MaskedAtomicRMW64WithSextIntrinsic; -def int_riscv_masked_atomicrmw_min_i64 : MaskedAtomicRMW64WithSextIntrinsic; -def int_riscv_masked_atomicrmw_umax_i64 : MaskedAtomicRMW64Intrinsic; -def int_riscv_masked_atomicrmw_umin_i64 : MaskedAtomicRMW64Intrinsic; + // @llvm.riscv.masked.atomicrmw.*.{i32,i64}.<p>(...) + defm int_riscv_masked_atomicrmw_xchg : MaskedAtomicRMWFourArgIntrinsics; + defm int_riscv_masked_atomicrmw_add : MaskedAtomicRMWFourArgIntrinsics; + defm int_riscv_masked_atomicrmw_sub : MaskedAtomicRMWFourArgIntrinsics; + defm int_riscv_masked_atomicrmw_nand : MaskedAtomicRMWFourArgIntrinsics; + // Signed min and max need an extra operand to do sign extension with. + defm int_riscv_masked_atomicrmw_max : MaskedAtomicRMWFiveArgIntrinsics; + defm int_riscv_masked_atomicrmw_min : MaskedAtomicRMWFiveArgIntrinsics; + // Unsigned min and max don't need the extra operand. + defm int_riscv_masked_atomicrmw_umax : MaskedAtomicRMWFourArgIntrinsics; + defm int_riscv_masked_atomicrmw_umin : MaskedAtomicRMWFourArgIntrinsics; -def int_riscv_masked_cmpxchg_i64 - : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, - llvm_i64_ty, llvm_i64_ty], - [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; + // @llvm.riscv.masked.cmpxchg.{i32,i64}.<p>(...) + defm int_riscv_masked_cmpxchg : MaskedAtomicRMWFiveArgIntrinsics; } // TargetPrefix = "riscv" diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td index 810979b99934..e97700ad724a 100644 --- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td @@ -112,6 +112,11 @@ def int_wasm_sub_saturate_unsigned : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; +def int_wasm_avgr_unsigned : + Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>], + [IntrNoMem, IntrSpeculatable]>; + def int_wasm_bitselect : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], @@ -132,6 +137,10 @@ def int_wasm_qfms : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; +def int_wasm_dot : + Intrinsic<[llvm_v4i32_ty], + [llvm_v8i16_ty, llvm_v8i16_ty], + [IntrNoMem, IntrSpeculatable]>; def int_wasm_narrow_signed : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>], diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h index 91bd57dc5ac0..39d19b7cffd9 100644 --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -17,7 +17,6 @@ #include "llvm-c/Types.h" #include "llvm/IR/DiagnosticHandler.h" #include "llvm/Support/CBindingWrapping.h" -#include "llvm/Support/Options.h" #include <cstdint> #include <memory> #include <string> @@ -85,6 +84,7 @@ public: OB_deopt = 0, // "deopt" OB_funclet = 1, // "funclet" OB_gc_transition = 2, // "gc-transition" + OB_cfguardtarget = 3, // "cfguardtarget" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. @@ -287,14 +287,6 @@ public: void emitError(const Instruction *I, const Twine &ErrorStr); void emitError(const Twine &ErrorStr); - /// Query for a debug option's value. - /// - /// This function returns typed data populated from command line parsing. - template <typename ValT, typename Base, ValT(Base::*Mem)> - ValT getOption() const { - return OptionRegistry::instance().template get<ValT, Base, Mem>(); - } - /// Access the object which can disable optional passes and individual /// optimizations at compile time. OptPassGate &getOptPassGate() const; diff --git a/llvm/include/llvm/IR/LegacyPassManager.h b/llvm/include/llvm/IR/LegacyPassManager.h index d6bb79ab6019..2b87143276b9 100644 --- a/llvm/include/llvm/IR/LegacyPassManager.h +++ b/llvm/include/llvm/IR/LegacyPassManager.h @@ -63,7 +63,7 @@ private: PassManagerImpl *PM; }; -/// FunctionPassManager manages FunctionPasses and BasicBlockPassManagers. +/// FunctionPassManager manages FunctionPasses. class FunctionPassManager : public PassManagerBase { public: /// FunctionPassManager ctor - This initializes the pass manager. It needs, diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h index 72bc80fb5381..5044c1f6ed31 100644 --- a/llvm/include/llvm/IR/LegacyPassManagers.h +++ b/llvm/include/llvm/IR/LegacyPassManagers.h @@ -53,10 +53,6 @@ // a place to implement common pass manager APIs. All pass managers derive from // PMDataManager. // -// [o] class BBPassManager : public FunctionPass, public PMDataManager; -// -// BBPassManager manages BasicBlockPasses. -// // [o] class FunctionPassManager; // // This is a external interface used to manage FunctionPasses. This @@ -103,7 +99,6 @@ enum PassDebuggingString { EXECUTION_MSG, // "Executing Pass '" + PassName MODIFICATION_MSG, // "Made Modification '" + PassName FREEING_MSG, // " Freeing Pass '" + PassName - ON_BASICBLOCK_MSG, // "' on BasicBlock '" + InstructionName + "'...\n" ON_FUNCTION_MSG, // "' on Function '" + FunctionName + "'...\n" ON_MODULE_MSG, // "' on Module '" + ModuleName + "'...\n" ON_REGION_MSG, // "' on Region '" + Msg + "'...\n'" diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h index f62b1e246cca..dda939b97575 100644 --- a/llvm/include/llvm/IR/Metadata.h +++ b/llvm/include/llvm/IR/Metadata.h @@ -22,6 +22,7 @@ #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/ilist_node.h" #include "llvm/ADT/iterator_range.h" @@ -641,26 +642,32 @@ public: /// A collection of metadata nodes that might be associated with a /// memory access used by the alias-analysis infrastructure. struct AAMDNodes { - explicit AAMDNodes(MDNode *T = nullptr, MDNode *S = nullptr, - MDNode *N = nullptr) - : TBAA(T), Scope(S), NoAlias(N) {} + explicit AAMDNodes() = default; + explicit AAMDNodes(MDNode *T, MDNode *TS, MDNode *S, MDNode *N) + : TBAA(T), TBAAStruct(TS), Scope(S), NoAlias(N) {} bool operator==(const AAMDNodes &A) const { - return TBAA == A.TBAA && Scope == A.Scope && NoAlias == A.NoAlias; + return TBAA == A.TBAA && TBAAStruct == A.TBAAStruct && Scope == A.Scope && + NoAlias == A.NoAlias; } bool operator!=(const AAMDNodes &A) const { return !(*this == A); } - explicit operator bool() const { return TBAA || Scope || NoAlias; } + explicit operator bool() const { + return TBAA || TBAAStruct || Scope || NoAlias; + } /// The tag for type-based alias analysis. - MDNode *TBAA; + MDNode *TBAA = nullptr; + + /// The tag for type-based alias analysis (tbaa struct). + MDNode *TBAAStruct = nullptr; /// The tag for alias scope specification (used with noalias). - MDNode *Scope; + MDNode *Scope = nullptr; /// The tag specifying the noalias scope. - MDNode *NoAlias; + MDNode *NoAlias = nullptr; /// Given two sets of AAMDNodes that apply to the same pointer, /// give the best AAMDNodes that are compatible with both (i.e. a set of @@ -670,6 +677,7 @@ struct AAMDNodes { AAMDNodes intersect(const AAMDNodes &Other) { AAMDNodes Result; Result.TBAA = Other.TBAA == TBAA ? TBAA : nullptr; + Result.TBAAStruct = Other.TBAAStruct == TBAAStruct ? TBAAStruct : nullptr; Result.Scope = Other.Scope == Scope ? Scope : nullptr; Result.NoAlias = Other.NoAlias == NoAlias ? NoAlias : nullptr; return Result; @@ -681,16 +689,17 @@ template<> struct DenseMapInfo<AAMDNodes> { static inline AAMDNodes getEmptyKey() { return AAMDNodes(DenseMapInfo<MDNode *>::getEmptyKey(), - nullptr, nullptr); + nullptr, nullptr, nullptr); } static inline AAMDNodes getTombstoneKey() { return AAMDNodes(DenseMapInfo<MDNode *>::getTombstoneKey(), - nullptr, nullptr); + nullptr, nullptr, nullptr); } static unsigned getHashValue(const AAMDNodes &Val) { return DenseMapInfo<MDNode *>::getHashValue(Val.TBAA) ^ + DenseMapInfo<MDNode *>::getHashValue(Val.TBAAStruct) ^ DenseMapInfo<MDNode *>::getHashValue(Val.Scope) ^ DenseMapInfo<MDNode *>::getHashValue(Val.NoAlias); } diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h index 59331142766a..68cd583c136c 100644 --- a/llvm/include/llvm/IR/Module.h +++ b/llvm/include/llvm/IR/Module.h @@ -660,24 +660,8 @@ public: concat_iterator<const GlobalObject, const_iterator, const_global_iterator>; - iterator_range<global_object_iterator> global_objects() { - return concat<GlobalObject>(functions(), globals()); - } - iterator_range<const_global_object_iterator> global_objects() const { - return concat<const GlobalObject>(functions(), globals()); - } - - global_object_iterator global_object_begin() { - return global_objects().begin(); - } - global_object_iterator global_object_end() { return global_objects().end(); } - - const_global_object_iterator global_object_begin() const { - return global_objects().begin(); - } - const_global_object_iterator global_object_end() const { - return global_objects().end(); - } + iterator_range<global_object_iterator> global_objects(); + iterator_range<const_global_object_iterator> global_objects() const; using global_value_iterator = concat_iterator<GlobalValue, iterator, global_iterator, alias_iterator, @@ -686,23 +670,8 @@ public: concat_iterator<const GlobalValue, const_iterator, const_global_iterator, const_alias_iterator, const_ifunc_iterator>; - iterator_range<global_value_iterator> global_values() { - return concat<GlobalValue>(functions(), globals(), aliases(), ifuncs()); - } - iterator_range<const_global_value_iterator> global_values() const { - return concat<const GlobalValue>(functions(), globals(), aliases(), - ifuncs()); - } - - global_value_iterator global_value_begin() { return global_values().begin(); } - global_value_iterator global_value_end() { return global_values().end(); } - - const_global_value_iterator global_value_begin() const { - return global_values().begin(); - } - const_global_value_iterator global_value_end() const { - return global_values().end(); - } + iterator_range<global_value_iterator> global_values(); + iterator_range<const_global_value_iterator> global_values() const; /// @} /// @name Named Metadata Iteration diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index be60447abd87..aa4054c8409e 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -29,6 +29,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/ScaledNumber.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/raw_ostream.h" #include <algorithm> #include <array> #include <cassert> @@ -172,7 +173,7 @@ struct ValueInfo { RefAndFlags.setInt(HaveGVs); } - operator bool() const { return getRef(); } + explicit operator bool() const { return getRef(); } GlobalValue::GUID getGUID() const { return getRef()->first; } const GlobalValue *getValue() const { @@ -547,6 +548,8 @@ public: // Indicate if the global value cannot be inlined. unsigned NoInline : 1; + // Indicate if function should be always inlined. + unsigned AlwaysInline : 1; }; /// Create an empty FunctionSummary (with specified call edges). @@ -941,6 +944,11 @@ private: /// considered live. bool WithGlobalValueDeadStripping = false; + /// Indicates that summary-based attribute propagation has run and + /// GVarFlags::MaybeReadonly / GVarFlags::MaybeWriteonly are really + /// read/write only. + bool WithAttributePropagation = false; + /// Indicates that summary-based synthetic entry count propagation has run bool HasSyntheticEntryCounts = false; @@ -987,6 +995,13 @@ public: : HaveGVs(HaveGVs), EnableSplitLTOUnit(EnableSplitLTOUnit), Saver(Alloc) { } + // Current version for the module summary in bitcode files. + // The BitcodeSummaryVersion should be bumped whenever we introduce changes + // in the way some record are interpreted, like flags for instance. + // Note that incrementing this may require changes in both BitcodeReader.cpp + // and BitcodeWriter.cpp. + static constexpr uint64_t BitcodeSummaryVersion = 8; + bool haveGVs() const { return HaveGVs; } gvsummary_iterator begin() { return GlobalValueMap.begin(); } @@ -1065,6 +1080,18 @@ public: WithGlobalValueDeadStripping = true; } + bool withAttributePropagation() const { return WithAttributePropagation; } + void setWithAttributePropagation() { + WithAttributePropagation = true; + } + + bool isReadOnly(const GlobalVarSummary *GVS) const { + return WithAttributePropagation && GVS->maybeReadOnly(); + } + bool isWriteOnly(const GlobalVarSummary *GVS) const { + return WithAttributePropagation && GVS->maybeWriteOnly(); + } + bool hasSyntheticEntryCounts() const { return HasSyntheticEntryCounts; } void setHasSyntheticEntryCounts() { HasSyntheticEntryCounts = true; } @@ -1241,9 +1268,11 @@ public: } /// Helper to obtain the unpromoted name for a global value (or the original - /// name if not promoted). + /// name if not promoted). Split off the rightmost ".llvm.${hash}" suffix, + /// because it is possible in certain clients (not clang at the moment) for + /// two rounds of ThinLTO optimization and therefore promotion to occur. static StringRef getOriginalNameBeforePromote(StringRef Name) { - std::pair<StringRef, StringRef> Pair = Name.split(".llvm."); + std::pair<StringRef, StringRef> Pair = Name.rsplit(".llvm."); return Pair.first; } @@ -1349,13 +1378,18 @@ public: void dump() const; /// Export summary to dot file for GraphViz. - void exportToDot(raw_ostream& OS) const; + void + exportToDot(raw_ostream &OS, + const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) const; /// Print out strongly connected components for debugging. void dumpSCCs(raw_ostream &OS); /// Analyze index and detect unmodified globals void propagateAttributes(const DenseSet<GlobalValue::GUID> &PreservedSymbols); + + /// Checks if we can import global variable from another module. + bool canImportGlobalVar(GlobalValueSummary *S, bool AnalyzeRefs) const; }; /// GraphTraits definition to build SCC for the index @@ -1427,15 +1461,6 @@ struct GraphTraits<ModuleSummaryIndex *> : public GraphTraits<ValueInfo> { return ValueInfo(I->haveGVs(), &P); } }; - -static inline bool canImportGlobalVar(GlobalValueSummary *S) { - assert(isa<GlobalVarSummary>(S->getBaseObject())); - - // We don't import GV with references, because it can result - // in promotion of local variables in the source module. - return !GlobalValue::isInterposableLinkage(S->linkage()) && - !S->notEligibleToImport() && S->refs().empty(); -} } // end namespace llvm #endif // LLVM_IR_MODULESUMMARYINDEX_H diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h index 0e3c19f4947f..835236b1eac0 100644 --- a/llvm/include/llvm/IR/NoFolder.h +++ b/llvm/include/llvm/IR/NoFolder.h @@ -196,7 +196,7 @@ public: } Instruction *CreateFNeg(Constant *C) const { - return BinaryOperator::CreateFNeg(C); + return UnaryOperator::CreateFNeg(C); } Instruction *CreateNot(Constant *C) const { diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h index 037f5aed03ee..35e08d9215e2 100644 --- a/llvm/include/llvm/IR/Operator.h +++ b/llvm/include/llvm/IR/Operator.h @@ -66,6 +66,7 @@ public: class OverflowingBinaryOperator : public Operator { public: enum { + AnyWrap = 0, NoUnsignedWrap = (1 << 0), NoSignedWrap = (1 << 1) }; @@ -394,8 +395,12 @@ public: return true; case Instruction::PHI: case Instruction::Select: - case Instruction::Call: - return V->getType()->isFPOrFPVectorTy(); + case Instruction::Call: { + Type *Ty = V->getType(); + while (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) + Ty = ArrTy->getElementType(); + return Ty->isFPOrFPVectorTy(); + } default: return false; } diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h index 1e1f4a92f844..58591ab380cc 100644 --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -1165,9 +1165,9 @@ public: /// Result proxy object for \c OuterAnalysisManagerProxy. class Result { public: - explicit Result(const AnalysisManagerT &AM) : AM(&AM) {} + explicit Result(const AnalysisManagerT &OuterAM) : OuterAM(&OuterAM) {} - const AnalysisManagerT &getManager() const { return *AM; } + const AnalysisManagerT &getManager() const { return *OuterAM; } /// When invalidation occurs, remove any registered invalidation events. bool invalidate( @@ -1219,7 +1219,7 @@ public: } private: - const AnalysisManagerT *AM; + const AnalysisManagerT *OuterAM; /// A map from an outer analysis ID to the set of this IR-unit's analyses /// which need to be invalidated. @@ -1227,14 +1227,15 @@ public: OuterAnalysisInvalidationMap; }; - OuterAnalysisManagerProxy(const AnalysisManagerT &AM) : AM(&AM) {} + OuterAnalysisManagerProxy(const AnalysisManagerT &OuterAM) + : OuterAM(&OuterAM) {} /// Run the analysis pass and create our proxy result object. - /// Nothing to see here, it just forwards the \c AM reference into the + /// Nothing to see here, it just forwards the \c OuterAM reference into the /// result. Result run(IRUnitT &, AnalysisManager<IRUnitT, ExtraArgTs...> &, ExtraArgTs...) { - return Result(*AM); + return Result(*OuterAM); } private: @@ -1243,7 +1244,7 @@ private: static AnalysisKey Key; - const AnalysisManagerT *AM; + const AnalysisManagerT *OuterAM; }; template <typename AnalysisManagerT, typename IRUnitT, typename... ExtraArgTs> diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 2851b24c05ae..6621fc9f819c 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -35,6 +35,7 @@ #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Operator.h" #include "llvm/IR/Value.h" @@ -365,7 +366,7 @@ inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { struct is_nonnegative { bool isValue(const APInt &C) { return C.isNonNegative(); } }; -/// Match an integer or vector of nonnegative values. +/// Match an integer or vector of non-negative values. /// For vectors, this includes constants with undefined elements. inline cst_pred_ty<is_nonnegative> m_NonNegative() { return cst_pred_ty<is_nonnegative>(); @@ -374,6 +375,28 @@ inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { return V; } +struct is_strictlypositive { + bool isValue(const APInt &C) { return C.isStrictlyPositive(); } +}; +/// Match an integer or vector of strictly positive values. +/// For vectors, this includes constants with undefined elements. +inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() { + return cst_pred_ty<is_strictlypositive>(); +} +inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) { + return V; +} + +struct is_nonpositive { + bool isValue(const APInt &C) { return C.isNonPositive(); } +}; +/// Match an integer or vector of non-positive values. +/// For vectors, this includes constants with undefined elements. +inline cst_pred_ty<is_nonpositive> m_NonPositive() { + return cst_pred_ty<is_nonpositive>(); +} +inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; } + struct is_one { bool isValue(const APInt &C) { return C.isOneValue(); } }; @@ -558,6 +581,8 @@ inline bind_ty<const Value> m_Value(const Value *&V) { return V; } inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; } /// Match a binary operator, capturing it if we match. inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; } +/// Match a with overflow intrinsic, capturing it if we match. +inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) { return I; } /// Match a ConstantInt, capturing the value if we match. inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; } @@ -1230,6 +1255,12 @@ m_SelectCst(const Cond &C) { return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>()); } +/// Matches FreezeInst. +template <typename OpTy> +inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) { + return OneOps_match<OpTy, Instruction::Freeze>(Op); +} + /// Matches InsertElementInst. template <typename Val_t, typename Elt_t, typename Idx_t> inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement> @@ -1727,6 +1758,12 @@ struct m_Intrinsic_Ty<T0, T1, T2, T3> { Argument_match<T3>>; }; +template <typename T0, typename T1, typename T2, typename T3, typename T4> +struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> { + using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty, + Argument_match<T4>>; +}; + /// Match intrinsic calls like this: /// m_Intrinsic<Intrinsic::fabs>(m_Value(X)) template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { @@ -1757,6 +1794,15 @@ m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); } +template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, + typename T3, typename T4> +inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty +m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3, + const T4 &Op4) { + return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3), + m_Argument<4>(Op4)); +} + // Helper intrinsic matching specializations. template <typename Opnd0> inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) { @@ -1937,6 +1983,25 @@ template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) { return Signum_match<Val_t>(V); } +template <int Ind, typename Opnd_t> struct ExtractValue_match { + Opnd_t Val; + ExtractValue_match(const Opnd_t &V) : Val(V) {} + + template <typename OpTy> bool match(OpTy *V) { + if (auto *I = dyn_cast<ExtractValueInst>(V)) + return I->getNumIndices() == 1 && I->getIndices()[0] == Ind && + Val.match(I->getAggregateOperand()); + return false; + } +}; + +/// Match a single index ExtractValue instruction. +/// For example m_ExtractValue<1>(...) +template <int Ind, typename Val_t> +inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) { + return ExtractValue_match<Ind, Val_t>(V); +} + } // end namespace PatternMatch } // end namespace llvm diff --git a/llvm/include/llvm/IR/RemarkStreamer.h b/llvm/include/llvm/IR/RemarkStreamer.h index 2abf6f99cb08..9ea12e8389f0 100644 --- a/llvm/include/llvm/IR/RemarkStreamer.h +++ b/llvm/include/llvm/IR/RemarkStreamer.h @@ -53,6 +53,8 @@ public: Error setFilter(StringRef Filter); /// Emit a diagnostic through the streamer. void emit(const DiagnosticInfoOptimizationBase &Diag); + /// Check if the remarks also need to have associated metadata in a section. + bool needsSection() const; }; template <typename ThisError> diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def index f6c74d497b18..fe2c32e3c975 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.def +++ b/llvm/include/llvm/IR/RuntimeLibcalls.def @@ -23,7 +23,7 @@ // Declare the enumerator for each libcall, along with its default name. Some // libcalls have different names on particular OSes or architectures. These -// are set in InitLibcallNames() in TargetLoweringBase.cpp and/or by targets +// are set in InitLibcalls() in TargetLoweringBase.cpp and/or by targets // using TargetLoweringBase::setLibcallName() #ifndef HANDLE_LIBCALL #error "HANDLE_LIBCALL must be defined" @@ -386,10 +386,6 @@ HANDLE_LIBCALL(UO_F32, "__unordsf2") HANDLE_LIBCALL(UO_F64, "__unorddf2") HANDLE_LIBCALL(UO_F128, "__unordtf2") HANDLE_LIBCALL(UO_PPCF128, "__gcc_qunord") -HANDLE_LIBCALL(O_F32, "__unordsf2") -HANDLE_LIBCALL(O_F64, "__unorddf2") -HANDLE_LIBCALL(O_F128, "__unordtf2") -HANDLE_LIBCALL(O_PPCF128, "__gcc_qunord") // Memory HANDLE_LIBCALL(MEMCPY, "memcpy") diff --git a/llvm/include/llvm/IR/ValueHandle.h b/llvm/include/llvm/IR/ValueHandle.h index 1135d796f7ed..50b7701f6716 100644 --- a/llvm/include/llvm/IR/ValueHandle.h +++ b/llvm/include/llvm/IR/ValueHandle.h @@ -171,6 +171,25 @@ template <> struct simplify_type<const WeakVH> { static SimpleType getSimplifiedValue(const WeakVH &WVH) { return WVH; } }; +// Specialize DenseMapInfo to allow WeakVH to participate in DenseMap. +template <> struct DenseMapInfo<WeakVH> { + static inline WeakVH getEmptyKey() { + return WeakVH(DenseMapInfo<Value *>::getEmptyKey()); + } + + static inline WeakVH getTombstoneKey() { + return WeakVH(DenseMapInfo<Value *>::getTombstoneKey()); + } + + static unsigned getHashValue(const WeakVH &Val) { + return DenseMapInfo<Value *>::getHashValue(Val); + } + + static bool isEqual(const WeakVH &LHS, const WeakVH &RHS) { + return DenseMapInfo<Value *>::isEqual(LHS, RHS); + } +}; + /// Value handle that is nullable, but tries to track the Value. /// /// This is a value handle that tries hard to point to a Value, even across @@ -264,6 +283,7 @@ public: #else AssertingVH() : ThePtr(nullptr) {} AssertingVH(ValueTy *P) : ThePtr(GetAsValue(P)) {} + AssertingVH(const AssertingVH<ValueTy> &) = default; #endif operator ValueTy*() const { diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 49f69340c828..a5e1310e28b9 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -91,6 +91,8 @@ void initializeCFGOnlyPrinterLegacyPassPass(PassRegistry&); void initializeCFGOnlyViewerLegacyPassPass(PassRegistry&); void initializeCFGPrinterLegacyPassPass(PassRegistry&); void initializeCFGSimplifyPassPass(PassRegistry&); +void initializeCFGuardPass(PassRegistry&); +void initializeCFGuardLongjmpPass(PassRegistry&); void initializeCFGViewerLegacyPassPass(PassRegistry&); void initializeCFIInstrInserterPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); @@ -178,6 +180,7 @@ void initializeIndVarSimplifyLegacyPassPass(PassRegistry&); void initializeIndirectBrExpandPassPass(PassRegistry&); void initializeInferAddressSpacesPass(PassRegistry&); void initializeInferFunctionAttrsLegacyPassPass(PassRegistry&); +void initializeInjectTLIMappingsLegacyPass(PassRegistry &); void initializeInlineCostAnalysisPass(PassRegistry&); void initializeInstCountPass(PassRegistry&); void initializeInstNamerPass(PassRegistry&); @@ -252,6 +255,7 @@ void initializeLowerIntrinsicsPass(PassRegistry&); void initializeLowerInvokeLegacyPassPass(PassRegistry&); void initializeLowerSwitchPass(PassRegistry&); void initializeLowerTypeTestsPass(PassRegistry&); +void initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry &); void initializeMIRCanonicalizerPass(PassRegistry &); void initializeMIRNamerPass(PassRegistry &); void initializeMIRPrintingPassPass(PassRegistry&); @@ -284,7 +288,7 @@ void initializeMemoryDependenceWrapperPassPass(PassRegistry&); void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&); void initializeMemorySSAWrapperPassPass(PassRegistry&); void initializeMemorySanitizerLegacyPassPass(PassRegistry&); -void initializeMergeFunctionsPass(PassRegistry&); +void initializeMergeFunctionsLegacyPassPass(PassRegistry&); void initializeMergeICmpsLegacyPassPass(PassRegistry &); void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&); void initializeMetaRenamerPass(PassRegistry&); @@ -332,7 +336,6 @@ void initializePostRAMachineSinkingPass(PassRegistry&); void initializePostRASchedulerPass(PassRegistry&); void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry&); void initializePredicateInfoPrinterLegacyPassPass(PassRegistry&); -void initializePrintBasicBlockPassPass(PassRegistry&); void initializePrintFunctionPassWrapperPass(PassRegistry&); void initializePrintModulePassWrapperPass(PassRegistry&); void initializeProcessImplicitDefsPass(PassRegistry&); @@ -343,6 +346,7 @@ void initializeRABasicPass(PassRegistry&); void initializeRAGreedyPass(PassRegistry&); void initializeReachingDefAnalysisPass(PassRegistry&); void initializeReassociateLegacyPassPass(PassRegistry&); +void initializeRedundantDbgInstEliminationPass(PassRegistry&); void initializeRegAllocFastPass(PassRegistry&); void initializeRegBankSelectPass(PassRegistry&); void initializeRegToMemPass(PassRegistry&); @@ -406,6 +410,7 @@ void initializeTargetTransformInfoWrapperPassPass(PassRegistry&); void initializeThreadSanitizerLegacyPassPass(PassRegistry&); void initializeTwoAddressInstructionPassPass(PassRegistry&); void initializeTypeBasedAAWrapperPassPass(PassRegistry&); +void initializeTypePromotionPass(PassRegistry&); void initializeUnifyFunctionExitNodesPass(PassRegistry&); void initializeUnpackMachineBundlesPass(PassRegistry&); void initializeUnreachableBlockElimLegacyPassPass(PassRegistry&); diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h index daa6585b1113..50147300f7f7 100644 --- a/llvm/include/llvm/LTO/Config.h +++ b/llvm/include/llvm/LTO/Config.h @@ -14,9 +14,12 @@ #ifndef LLVM_LTO_CONFIG_H #define LLVM_LTO_CONFIG_H +#include "llvm/ADT/DenseSet.h" #include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Passes/PassBuilder.h" #include "llvm/Support/CodeGen.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include <functional> @@ -41,7 +44,7 @@ struct Config { Optional<Reloc::Model> RelocModel = Reloc::PIC_; Optional<CodeModel::Model> CodeModel = None; CodeGenOpt::Level CGOptLevel = CodeGenOpt::Default; - TargetMachine::CodeGenFileType CGFileType = TargetMachine::CGFT_ObjectFile; + CodeGenFileType CGFileType = CGFT_ObjectFile; unsigned OptLevel = 2; bool DisableVerify = false; @@ -126,6 +129,9 @@ struct Config { /// with llvm-lto2. std::unique_ptr<raw_ostream> ResolutionFile; + /// Tunable parameters for passes in the default pipelines. + PipelineTuningOptions PTO; + /// The following callbacks deal with tasks, which normally represent the /// entire optimization and code generation pipeline for what will become a /// single native object file. Each task has a unique identifier between 0 and @@ -183,8 +189,9 @@ struct Config { /// /// It is called regardless of whether the backend is in-process, although it /// is not called from individual backend processes. - using CombinedIndexHookFn = - std::function<bool(const ModuleSummaryIndex &Index)>; + using CombinedIndexHookFn = std::function<bool( + const ModuleSummaryIndex &Index, + const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)>; CombinedIndexHookFn CombinedIndexHook; /// This is a convenience function that configures this Config object to write diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 0a1e3e1d0e42..aa21f963d3a8 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -59,7 +59,7 @@ void thinLTOResolvePrevailingInIndex( /// must apply the changes to the Module via thinLTOInternalizeModule. void thinLTOInternalizeAndPromoteInIndex( ModuleSummaryIndex &Index, - function_ref<bool(StringRef, GlobalValue::GUID)> isExported, + function_ref<bool(StringRef, ValueInfo)> isExported, function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing); @@ -222,7 +222,7 @@ using NativeObjectCache = /// The details of this type definition aren't important; clients can only /// create a ThinBackend using one of the create*ThinBackend() functions below. using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>( - Config &C, ModuleSummaryIndex &CombinedIndex, + const Config &C, ModuleSummaryIndex &CombinedIndex, StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, AddStreamFn AddStream, NativeObjectCache Cache)>; @@ -306,7 +306,8 @@ private: Config Conf; struct RegularLTOState { - RegularLTOState(unsigned ParallelCodeGenParallelismLevel, Config &Conf); + RegularLTOState(unsigned ParallelCodeGenParallelismLevel, + const Config &Conf); struct CommonResolution { uint64_t Size = 0; MaybeAlign Align; diff --git a/llvm/include/llvm/LTO/LTOBackend.h b/llvm/include/llvm/LTO/LTOBackend.h index 4ff8a1993d49..de4fa308fde7 100644 --- a/llvm/include/llvm/LTO/LTOBackend.h +++ b/llvm/include/llvm/LTO/LTOBackend.h @@ -35,13 +35,13 @@ namespace lto { /// Runs a regular LTO backend. The regular LTO backend can also act as the /// regular LTO phase of ThinLTO, which may need to access the combined index. -Error backend(Config &C, AddStreamFn AddStream, +Error backend(const Config &C, AddStreamFn AddStream, unsigned ParallelCodeGenParallelismLevel, std::unique_ptr<Module> M, ModuleSummaryIndex &CombinedIndex); /// Runs a ThinLTO backend. -Error thinBackend(Config &C, unsigned Task, AddStreamFn AddStream, Module &M, - const ModuleSummaryIndex &CombinedIndex, +Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, + Module &M, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, MapVector<StringRef, BitcodeModule> &ModuleMap); diff --git a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h index 8718df4b88e6..114ba85947a5 100644 --- a/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h +++ b/llvm/include/llvm/LTO/legacy/LTOCodeGenerator.h @@ -35,11 +35,13 @@ #define LLVM_LTO_LTOCODEGENERATOR_H #include "llvm-c/lto.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Module.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Target/TargetMachine.h" @@ -87,8 +89,8 @@ struct LTOCodeGenerator { void setCodePICModel(Optional<Reloc::Model> Model) { RelocModel = Model; } /// Set the file type to be emitted (assembly or object code). - /// The default is TargetMachine::CGFT_ObjectFile. - void setFileType(TargetMachine::CodeGenFileType FT) { FileType = FT; } + /// The default is CGFT_ObjectFile. + void setFileType(CodeGenFileType FT) { FileType = FT; } void setCpu(StringRef MCpu) { this->MCpu = MCpu; } void setAttr(StringRef MAttr) { this->MAttr = MAttr; } @@ -121,7 +123,7 @@ struct LTOCodeGenerator { /// name is misleading). This function should be called before /// LTOCodeGenerator::compilexxx(), and /// LTOCodeGenerator::writeMergedModules(). - void setCodeGenDebugOptions(StringRef Opts); + void setCodeGenDebugOptions(ArrayRef<const char *> Opts); /// Parse the options set in setCodeGenDebugOptions. /// @@ -238,7 +240,7 @@ private: bool ShouldInternalize = EnableLTOInternalization; bool ShouldEmbedUselists = false; bool ShouldRestoreGlobalsLinkage = false; - TargetMachine::CodeGenFileType FileType = TargetMachine::CGFT_ObjectFile; + CodeGenFileType FileType = CGFT_ObjectFile; std::unique_ptr<ToolOutputFile> DiagnosticOutputFile; bool Freestanding = false; std::unique_ptr<ToolOutputFile> StatsFile = nullptr; diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index ac88165845d3..aa64296f9428 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -159,6 +159,7 @@ namespace { (void) llvm::createPostDomOnlyViewerPass(); (void) llvm::createPostDomViewerPass(); (void) llvm::createReassociatePass(); + (void) llvm::createRedundantDbgInstEliminationPass(); (void) llvm::createRegionInfoPass(); (void) llvm::createRegionOnlyPrinterPass(); (void) llvm::createRegionOnlyViewerPass(); @@ -200,7 +201,6 @@ namespace { llvm::raw_string_ostream os(buf); (void) llvm::createPrintModulePass(os); (void) llvm::createPrintFunctionPass(os); - (void) llvm::createPrintBasicBlockPass(os); (void) llvm::createModuleDebugInfoPrinterPass(); (void) llvm::createPartialInliningPass(); (void) llvm::createLintPass(); @@ -226,6 +226,7 @@ namespace { (void) llvm::createScalarizeMaskedMemIntrinPass(); (void) llvm::createWarnMissedTransformationsPass(); (void) llvm::createHardwareLoopsPass(); + (void)llvm::createInjectTLIMappingsLegacyPass(); (void)new llvm::IntervalPartition(); (void)new llvm::ScalarEvolutionWrapperPass(); diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h index 1f3ad6c1e547..ed7d5c7f01f4 100644 --- a/llvm/include/llvm/MC/MCAsmBackend.h +++ b/llvm/include/llvm/MC/MCAsmBackend.h @@ -17,21 +17,18 @@ #include "llvm/MC/MCFragment.h" #include "llvm/Support/Endian.h" #include <cstdint> -#include <memory> namespace llvm { class MCAsmLayout; class MCAssembler; class MCCFIInstruction; -class MCCodePadder; struct MCFixupKindInfo; class MCFragment; class MCInst; class MCObjectStreamer; class MCObjectTargetWriter; class MCObjectWriter; -struct MCCodePaddingContext; class MCRelaxableFragment; class MCSubtargetInfo; class MCValue; @@ -39,8 +36,6 @@ class raw_pwrite_stream; /// Generic interface to target specific assembler backends. class MCAsmBackend { - std::unique_ptr<MCCodePadder> CodePadder; - protected: // Can only create subclasses. MCAsmBackend(support::endianness Endian); @@ -51,6 +46,16 @@ public: const support::endianness Endian; + /// Return true if this target might automatically pad instructions and thus + /// need to emit padding enable/disable directives around sensative code. + virtual bool allowAutoPadding() const { return false; } + + /// Give the target a chance to manipulate state related to instruction + /// alignment (e.g. padding for optimization) before and after actually + /// emitting the instruction. + virtual void alignBranchesBegin(MCObjectStreamer &OS, const MCInst &Inst) {} + virtual void alignBranchesEnd(MCObjectStreamer &OS, const MCInst &Inst) {} + /// lifetime management virtual void reset() {} @@ -184,40 +189,6 @@ public: virtual bool isMicroMips(const MCSymbol *Sym) const { return false; } - - /// Handles all target related code padding when starting to write a new - /// basic block to an object file. - /// - /// \param OS The streamer used for writing the padding data and function. - /// \param Context the context of the padding, Embeds the basic block's - /// parameters. - void handleCodePaddingBasicBlockStart(MCObjectStreamer *OS, - const MCCodePaddingContext &Context); - /// Handles all target related code padding after writing a block to an object - /// file. - /// - /// \param Context the context of the padding, Embeds the basic block's - /// parameters. - void handleCodePaddingBasicBlockEnd(const MCCodePaddingContext &Context); - /// Handles all target related code padding before writing a new instruction - /// to an object file. - /// - /// \param Inst the instruction. - void handleCodePaddingInstructionBegin(const MCInst &Inst); - /// Handles all target related code padding after writing an instruction to an - /// object file. - /// - /// \param Inst the instruction. - void handleCodePaddingInstructionEnd(const MCInst &Inst); - - /// Relaxes a fragment (changes the size of the padding) according to target - /// requirements. The new size computation is done w.r.t a layout. - /// - /// \param PF The fragment to relax. - /// \param Layout Code layout information. - /// - /// \returns true iff any relaxation occurred. - bool relaxFragment(MCPaddingFragment *PF, MCAsmLayout &Layout); }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h index 3261c483e0d8..5a6dff64caef 100644 --- a/llvm/include/llvm/MC/MCAsmInfo.h +++ b/llvm/include/llvm/MC/MCAsmInfo.h @@ -333,6 +333,10 @@ protected: /// protected visibility. Defaults to MCSA_Protected MCSymbolAttr ProtectedVisibilityAttr = MCSA_Protected; + // This attribute is used to indicate symbols such as commons on AIX may have + // a storage mapping class embedded in the name. + bool SymbolsHaveSMC = false; + //===--- Dwarf Emission Directives -----------------------------------===// /// True if target supports emission of debugging information. Defaults to @@ -454,6 +458,9 @@ public: unsigned Encoding, MCStreamer &Streamer) const; + /// Return true if C is an acceptable character inside a symbol name. + virtual bool isAcceptableChar(char C) const; + /// Return true if the identifier \p Name does not need quotes to be /// syntactically correct. virtual bool isValidUnquotedName(StringRef Name) const; @@ -584,6 +591,8 @@ public: return ProtectedVisibilityAttr; } + bool getSymbolsHaveSMC() const { return SymbolsHaveSMC; } + bool doesSupportDebugInformation() const { return SupportsDebugInformation; } bool doesSupportExceptionHandling() const { diff --git a/llvm/include/llvm/MC/MCAsmInfoELF.h b/llvm/include/llvm/MC/MCAsmInfoELF.h index aa2e5873e2c6..408d4df76412 100644 --- a/llvm/include/llvm/MC/MCAsmInfoELF.h +++ b/llvm/include/llvm/MC/MCAsmInfoELF.h @@ -18,10 +18,6 @@ class MCAsmInfoELF : public MCAsmInfo { MCSection *getNonexecutableStackSection(MCContext &Ctx) const final; protected: - /// Targets which have non-executable stacks by default can set this to false - /// to disable the special section which requests a non-executable stack. - bool UsesNonexecutableStackSection = true; - MCAsmInfoELF(); }; diff --git a/llvm/include/llvm/MC/MCAsmInfoXCOFF.h b/llvm/include/llvm/MC/MCAsmInfoXCOFF.h index 4a3bacc954e0..5483899d5875 100644 --- a/llvm/include/llvm/MC/MCAsmInfoXCOFF.h +++ b/llvm/include/llvm/MC/MCAsmInfoXCOFF.h @@ -20,9 +20,9 @@ protected: MCAsmInfoXCOFF(); public: - // Return true only when the identifier Name does not need quotes to be - // syntactically correct for XCOFF. - bool isValidUnquotedName(StringRef Name) const override; + // Return true only when C is an acceptable character inside a + // MCSymbolXCOFF. + bool isAcceptableChar(char C) const override; }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h index 4543018901a4..8c76f30222e5 100644 --- a/llvm/include/llvm/MC/MCAssembler.h +++ b/llvm/include/llvm/MC/MCAssembler.h @@ -191,11 +191,8 @@ private: bool layoutSectionOnce(MCAsmLayout &Layout, MCSection &Sec); bool relaxInstruction(MCAsmLayout &Layout, MCRelaxableFragment &IF); - - bool relaxPaddingFragment(MCAsmLayout &Layout, MCPaddingFragment &PF); - bool relaxLEB(MCAsmLayout &Layout, MCLEBFragment &IF); - + bool relaxBoundaryAlign(MCAsmLayout &Layout, MCBoundaryAlignFragment &BF); bool relaxDwarfLineAddr(MCAsmLayout &Layout, MCDwarfLineAddrFragment &DF); bool relaxDwarfCallFrameFragment(MCAsmLayout &Layout, MCDwarfCallFrameFragment &DF); diff --git a/llvm/include/llvm/MC/MCCodeEmitter.h b/llvm/include/llvm/MC/MCCodeEmitter.h index 04b4367ada7b..2794acc0753f 100644 --- a/llvm/include/llvm/MC/MCCodeEmitter.h +++ b/llvm/include/llvm/MC/MCCodeEmitter.h @@ -30,6 +30,12 @@ public: /// Lifetime management virtual void reset() {} + /// Emit the prefixes of given instruction on the output stream. + /// + /// \param Inst a single low-level machine instruction. + /// \param OS output stream. + virtual void emitPrefix(const MCInst &Inst, raw_ostream &OS, + const MCSubtargetInfo &STI) const {} /// EncodeInstruction - Encode the given \p Inst to bytes on the output /// stream \p OS. virtual void encodeInstruction(const MCInst &Inst, raw_ostream &OS, diff --git a/llvm/include/llvm/MC/MCCodePadder.h b/llvm/include/llvm/MC/MCCodePadder.h deleted file mode 100644 index f7b1a2113a9a..000000000000 --- a/llvm/include/llvm/MC/MCCodePadder.h +++ /dev/null @@ -1,241 +0,0 @@ -//===- llvm/MC/MCCodePadder.h - MC Code Padder ------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_MC_MCCODEPADDER_H -#define LLVM_MC_MCCODEPADDER_H - -#include "MCFragment.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" - -namespace llvm { - -class MCAsmLayout; -class MCCodePaddingPolicy; -class MCFragment; -class MCInst; -class MCObjectStreamer; -class MCSection; - -typedef SmallVector<const MCPaddingFragment *, 8> MCPFRange; - -struct MCCodePaddingContext { - bool IsPaddingActive; - bool IsBasicBlockReachableViaFallthrough; - bool IsBasicBlockReachableViaBranch; -}; - -/// Target-independent base class incharge of all code padding decisions for a -/// target. During encoding it determines if and where MCPaddingFragments will -/// be located, as later on, when layout information is available, it determines -/// their sizes. -class MCCodePadder { - MCCodePadder(const MCCodePadder &) = delete; - void operator=(const MCCodePadder &) = delete; - - /// Determines if the MCCodePaddingPolicies are active. - bool ArePoliciesActive; - - /// All the supported MCCodePaddingPolicies. - SmallPtrSet<MCCodePaddingPolicy *, 4> CodePaddingPolicies; - - /// A pointer to the fragment of the instruction whose padding is currently - /// done for. - MCPaddingFragment *CurrHandledInstFragment; - - /// A map holding the jurisdiction for each padding fragment. Key: padding - /// fragment. Value: The fragment's jurisdiction. A jurisdiction is a vector - /// of padding fragments whose conditions are being controlled by another - /// fragment, the key fragment. - DenseMap<MCPaddingFragment *, MCPFRange> FragmentToJurisdiction; - MCPFRange &getJurisdiction(MCPaddingFragment *Fragment, MCAsmLayout &Layout); - - /// A map holding the maximal instruction window size relevant for a padding - /// fragment. - DenseMap<MCPaddingFragment *, uint64_t> FragmentToMaxWindowSize; - uint64_t getMaxWindowSize(MCPaddingFragment *Fragment, MCAsmLayout &Layout); - -protected: - /// The current streamer, used to stream code padding. - MCObjectStreamer *OS; - - bool addPolicy(MCCodePaddingPolicy *Policy); - - virtual bool - basicBlockRequiresInsertionPoint(const MCCodePaddingContext &Context) { - return false; - } - - virtual bool instructionRequiresInsertionPoint(const MCInst &Inst) { - return false; - } - - virtual bool usePoliciesForBasicBlock(const MCCodePaddingContext &Context) { - return Context.IsPaddingActive; - } - -public: - MCCodePadder() - : ArePoliciesActive(false), CurrHandledInstFragment(nullptr), - OS(nullptr) {} - virtual ~MCCodePadder(); - - /// Handles all target related code padding when starting to write a new - /// basic block to an object file. - /// - /// \param OS The streamer used for writing the padding data and function. - /// \param Context the context of the padding, Embeds the basic block's - /// parameters. - void handleBasicBlockStart(MCObjectStreamer *OS, - const MCCodePaddingContext &Context); - /// Handles all target related code padding when done writing a block to an - /// object file. - /// - /// \param Context the context of the padding, Embeds the basic block's - /// parameters. - void handleBasicBlockEnd(const MCCodePaddingContext &Context); - /// Handles all target related code padding before writing a new instruction - /// to an object file. - /// - /// \param Inst the instruction. - void handleInstructionBegin(const MCInst &Inst); - /// Handles all target related code padding after writing an instruction to an - /// object file. - /// - /// \param Inst the instruction. - void handleInstructionEnd(const MCInst &Inst); - - /// Relaxes a fragment (changes the size of the padding) according to target - /// requirements. The new size computation is done w.r.t a layout. - /// - /// \param Fragment The fragment to relax. - /// \param Layout Code layout information. - /// - /// \returns true iff any relaxation occurred. - bool relaxFragment(MCPaddingFragment *Fragment, MCAsmLayout &Layout); -}; - -/// The base class for all padding policies, i.e. a rule or set of rules to pad -/// the generated code. -class MCCodePaddingPolicy { - MCCodePaddingPolicy() = delete; - MCCodePaddingPolicy(const MCCodePaddingPolicy &) = delete; - void operator=(const MCCodePaddingPolicy &) = delete; - -protected: - /// A mask holding the kind of this policy, i.e. only the i'th bit will be set - /// where i is the kind number. - const uint64_t KindMask; - /// Instruction window size relevant to this policy. - const uint64_t WindowSize; - /// A boolean indicating which byte of the instruction determies its - /// instruction window. If true - the last byte of the instructions, o.w. - - /// the first byte of the instruction. - const bool InstByteIsLastByte; - - MCCodePaddingPolicy(uint64_t Kind, uint64_t WindowSize, - bool InstByteIsLastByte) - : KindMask(UINT64_C(1) << Kind), WindowSize(WindowSize), - InstByteIsLastByte(InstByteIsLastByte) {} - - /// Computes and returns the offset of the consecutive fragment of a given - /// fragment. - /// - /// \param Fragment The fragment whose consecutive offset will be computed. - /// \param Layout Code layout information. - /// - /// \returns the offset of the consecutive fragment of \p Fragment. - static uint64_t getNextFragmentOffset(const MCFragment *Fragment, - const MCAsmLayout &Layout); - /// Returns the instruction byte of an instruction pointed by a given - /// MCPaddingFragment. An instruction byte is the address of the byte of an - /// instruction which determines its instruction window. - /// - /// \param Fragment The fragment pointing to the instruction. - /// \param Layout Code layout information. - /// - /// \returns the instruction byte of an instruction pointed by \p Fragment. - uint64_t getFragmentInstByte(const MCPaddingFragment *Fragment, - MCAsmLayout &Layout) const; - uint64_t computeWindowEndAddress(const MCPaddingFragment *Fragment, - uint64_t Offset, MCAsmLayout &Layout) const; - - /// Computes and returns the penalty weight of a first instruction window in a - /// range. This requires a special function since the first window does not - /// contain all the padding fragments in that window. It only contains all the - /// padding fragments starting from the relevant insertion point. - /// - /// \param Window The first window. - /// \param Offset The offset of the parent section relative to the beginning - /// of the file, mod the window size. - /// \param Layout Code layout information. - /// - /// \returns the penalty weight of a first instruction window in a range, \p - /// Window. - double computeFirstWindowPenaltyWeight(const MCPFRange &Window, - uint64_t Offset, - MCAsmLayout &Layout) const; - /// Computes and returns the penalty caused by an instruction window. - /// - /// \param Window The instruction window. - /// \param Offset The offset of the parent section relative to the beginning - /// of the file, mod the window size. - /// \param Layout Code layout information. - /// - /// \returns the penalty caused by \p Window. - virtual double computeWindowPenaltyWeight(const MCPFRange &Window, - uint64_t Offset, - MCAsmLayout &Layout) const = 0; - -public: - virtual ~MCCodePaddingPolicy() {} - - /// Returns the kind mask of this policy - A mask holding the kind of this - /// policy, i.e. only the i'th bit will be set where i is the kind number. - uint64_t getKindMask() const { return KindMask; } - /// Returns the instruction window size relevant to this policy. - uint64_t getWindowSize() const { return WindowSize; } - /// Returns true if the last byte of an instruction determines its instruction - /// window, or false if the first of an instruction determines it. - bool isInstByteLastByte() const { return InstByteIsLastByte; } - - /// Returns true iff this policy needs padding for a given basic block. - /// - /// \param Context the context of the padding, Embeds the basic block's - /// parameters. - /// - /// \returns true iff this policy needs padding for the basic block. - virtual bool - basicBlockRequiresPaddingFragment(const MCCodePaddingContext &Context) const { - return false; - } - /// Returns true iff this policy needs padding for a given instruction. - /// - /// \param Inst The given instruction. - /// - /// \returns true iff this policy needs padding for \p Inst. - virtual bool instructionRequiresPaddingFragment(const MCInst &Inst) const { - return false; - } - /// Computes and returns the penalty caused by a range of instruction windows. - /// The weight is computed for each window separelty and then accumulated. - /// - /// \param Range The range. - /// \param Offset The offset of the parent section relative to the beginning - /// of the file, mod the window size. - /// \param Layout Code layout information. - /// - /// \returns the penalty caused by \p Range. - double computeRangePenaltyWeight(const MCPFRange &Range, uint64_t Offset, - MCAsmLayout &Layout) const; -}; - -} // namespace llvm - -#endif // LLVM_MC_MCCODEPADDER_H diff --git a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h index 268f3ccad889..76c5215264bc 100644 --- a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h +++ b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h @@ -69,7 +69,6 @@ public: /// \param Address - The address, in the memory space of region, of the first /// byte of the instruction. /// \param Bytes - A reference to the actual bytes of the instruction. - /// \param VStream - The stream to print warnings and diagnostic messages on. /// \param CStream - The stream to print comments and annotations on. /// \return - MCDisassembler::Success if the instruction is valid, /// MCDisassembler::SoftFail if the instruction was @@ -77,7 +76,6 @@ public: /// MCDisassembler::Fail if the instruction was invalid. virtual DecodeStatus getInstruction(MCInst &Instr, uint64_t &Size, ArrayRef<uint8_t> Bytes, uint64_t Address, - raw_ostream &VStream, raw_ostream &CStream) const = 0; /// May parse any prelude that precedes instructions after the start of a @@ -88,13 +86,11 @@ public: /// \param Address - The address, in the memory space of region, of the first /// byte of the symbol. /// \param Bytes - A reference to the actual bytes at the symbol location. - /// \param VStream - The stream to print warnings and diagnostic messages on. /// \param CStream - The stream to print comments and annotations on. /// \return - MCDisassembler::Success if the bytes are valid, /// MCDisassembler::Fail if the bytes were invalid. virtual DecodeStatus onSymbolStart(StringRef Name, uint64_t &Size, ArrayRef<uint8_t> Bytes, uint64_t Address, - raw_ostream &VStream, raw_ostream &CStream) const; private: diff --git a/llvm/include/llvm/MC/MCDwarf.h b/llvm/include/llvm/MC/MCDwarf.h index a33b4b31bb06..b8ee585bb017 100644 --- a/llvm/include/llvm/MC/MCDwarf.h +++ b/llvm/include/llvm/MC/MCDwarf.h @@ -54,7 +54,7 @@ struct MCDwarfFile { std::string Name; // The index into the list of directory names for this file name. - unsigned DirIndex; + unsigned DirIndex = 0; /// The MD5 checksum, if there is one. Non-owning pointer to data allocated /// in MCContext. diff --git a/llvm/include/llvm/MC/MCELFStreamer.h b/llvm/include/llvm/MC/MCELFStreamer.h index 8838d53d75b5..85534ed6c085 100644 --- a/llvm/include/llvm/MC/MCELFStreamer.h +++ b/llvm/include/llvm/MC/MCELFStreamer.h @@ -41,7 +41,8 @@ public: void InitSections(bool NoExecStack) override; void ChangeSection(MCSection *Section, const MCExpr *Subsection) override; void EmitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override; - void EmitLabel(MCSymbol *Symbol, SMLoc Loc, MCFragment *F) override; + void EmitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, MCFragment *F, + uint64_t Offset) override; void EmitAssemblerFlag(MCAssemblerFlag Flag) override; void EmitThumbFunc(MCSymbol *Func) override; void EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override; @@ -101,7 +102,7 @@ MCELFStreamer *createARMELFStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB, std::unique_ptr<MCObjectWriter> OW, std::unique_ptr<MCCodeEmitter> Emitter, - bool RelaxAll, bool IsThumb); + bool RelaxAll, bool IsThumb, bool IsAndroid); } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h index b0def566c46a..ed237eb1e884 100644 --- a/llvm/include/llvm/MC/MCFragment.h +++ b/llvm/include/llvm/MC/MCFragment.h @@ -16,6 +16,7 @@ #include "llvm/ADT/ilist_node.h" #include "llvm/MC/MCFixup.h" #include "llvm/MC/MCInst.h" +#include "llvm/Support/Alignment.h" #include "llvm/Support/Casting.h" #include "llvm/Support/SMLoc.h" #include <cstdint> @@ -41,7 +42,7 @@ public: FT_Dwarf, FT_DwarfFrame, FT_LEB, - FT_Padding, + FT_BoundaryAlign, FT_SymbolId, FT_CVInlineLines, FT_CVDefRange, @@ -49,39 +50,27 @@ public: }; private: - FragmentType Kind; - -protected: - bool HasInstructions; - -private: - /// LayoutOrder - The layout order of this fragment. - unsigned LayoutOrder; - /// The data for the section this fragment is in. MCSection *Parent; - /// Atom - The atom this fragment is in, as represented by its defining - /// symbol. + /// The atom this fragment is in, as represented by its defining symbol. const MCSymbol *Atom; - /// \name Assembler Backend Data - /// @{ - // - // FIXME: This could all be kept private to the assembler implementation. - - /// Offset - The offset of this fragment in its section. This is ~0 until + /// The offset of this fragment in its section. This is ~0 until /// initialized. uint64_t Offset; - /// @} + /// The layout order of this fragment. + unsigned LayoutOrder; + + FragmentType Kind; protected: + bool HasInstructions; + MCFragment(FragmentType Kind, bool HasInstructions, MCSection *Parent = nullptr); - ~MCFragment(); - public: MCFragment() = delete; MCFragment(const MCFragment &) = delete; @@ -108,9 +97,6 @@ public: /// this is false, but specific fragment types may set it to true. bool hasInstructions() const { return HasInstructions; } - /// Return true if given frgment has FT_Dummy type. - bool isDummy() const { return Kind == FT_Dummy; } - void dump() const; }; @@ -135,8 +121,8 @@ protected: MCSection *Sec) : MCFragment(FType, HasInstructions, Sec) {} - /// STI - The MCSubtargetInfo in effect when the instruction was encoded. - /// must be non-null for instructions. + /// The MCSubtargetInfo in effect when the instruction was encoded. + /// It must be non-null for instructions. const MCSubtargetInfo *STI = nullptr; public: @@ -206,7 +192,7 @@ template<unsigned ContentsSize, unsigned FixupsSize> class MCEncodedFragmentWithFixups : public MCEncodedFragmentWithContents<ContentsSize> { - /// Fixups - The list of fixups in this fragment. + /// The list of fixups in this fragment. SmallVector<MCFixup, FixupsSize> Fixups; protected: @@ -271,7 +257,7 @@ public: /// class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> { - /// Inst - The instruction this is a fragment for. + /// The instruction this is a fragment for. MCInst Inst; public: @@ -289,21 +275,21 @@ public: }; class MCAlignFragment : public MCFragment { - /// Alignment - The alignment to ensure, in bytes. + /// The alignment to ensure, in bytes. unsigned Alignment; - /// EmitNops - Flag to indicate that (optimal) NOPs should be emitted instead + /// Flag to indicate that (optimal) NOPs should be emitted instead /// of using the provided value. The exact interpretation of this flag is /// target dependent. bool EmitNops : 1; - /// Value - Value to use for filling padding bytes. + /// Value to use for filling padding bytes. int64_t Value; - /// ValueSize - The size of the integer (in bytes) of \p Value. + /// The size of the integer (in bytes) of \p Value. unsigned ValueSize; - /// MaxBytesToEmit - The maximum number of bytes to emit; if the alignment + /// The maximum number of bytes to emit; if the alignment /// cannot be satisfied in this width then this fragment is ignored. unsigned MaxBytesToEmit; @@ -313,9 +299,6 @@ public: : MCFragment(FT_Align, false, Sec), Alignment(Alignment), EmitNops(false), Value(Value), ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {} - /// \name Accessors - /// @{ - unsigned getAlignment() const { return Alignment; } int64_t getValue() const { return Value; } @@ -327,109 +310,15 @@ public: bool hasEmitNops() const { return EmitNops; } void setEmitNops(bool Value) { EmitNops = Value; } - /// @} - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_Align; } }; -/// Fragment for adding required padding. -/// This fragment is always inserted before an instruction, and holds that -/// instruction as context information (as well as a mask of kinds) for -/// determining the padding size. -/// -class MCPaddingFragment : public MCFragment { - /// A mask containing all the kinds relevant to this fragment. i.e. the i'th - /// bit will be set iff kind i is relevant to this fragment. - uint64_t PaddingPoliciesMask; - /// A boolean indicating if this fragment will actually hold padding. If its - /// value is false, then this fragment serves only as a placeholder, - /// containing data to assist other insertion point in their decision making. - bool IsInsertionPoint; - - uint64_t Size; - - struct MCInstInfo { - bool IsInitialized; - MCInst Inst; - /// A boolean indicating whether the instruction pointed by this fragment is - /// a fixed size instruction or a relaxable instruction held by a - /// MCRelaxableFragment. - bool IsImmutableSizedInst; - union { - /// If the instruction is a fixed size instruction, hold its size. - size_t InstSize; - /// Otherwise, hold a pointer to the MCRelaxableFragment holding it. - MCRelaxableFragment *InstFragment; - }; - }; - MCInstInfo InstInfo; - -public: - static const uint64_t PFK_None = UINT64_C(0); - - enum MCPaddingFragmentKind { - // values 0-7 are reserved for future target independet values. - - FirstTargetPerfNopFragmentKind = 8, - - /// Limit range of target MCPerfNopFragment kinds to fit in uint64_t - MaxTargetPerfNopFragmentKind = 63 - }; - - MCPaddingFragment(MCSection *Sec = nullptr) - : MCFragment(FT_Padding, false, Sec), PaddingPoliciesMask(PFK_None), - IsInsertionPoint(false), Size(UINT64_C(0)), - InstInfo({false, MCInst(), false, {0}}) {} - - bool isInsertionPoint() const { return IsInsertionPoint; } - void setAsInsertionPoint() { IsInsertionPoint = true; } - uint64_t getPaddingPoliciesMask() const { return PaddingPoliciesMask; } - void setPaddingPoliciesMask(uint64_t Value) { PaddingPoliciesMask = Value; } - bool hasPaddingPolicy(uint64_t PolicyMask) const { - assert(isPowerOf2_64(PolicyMask) && - "Policy mask must contain exactly one policy"); - return (getPaddingPoliciesMask() & PolicyMask) != PFK_None; - } - const MCInst &getInst() const { - assert(isInstructionInitialized() && "Fragment has no instruction!"); - return InstInfo.Inst; - } - size_t getInstSize() const { - assert(isInstructionInitialized() && "Fragment has no instruction!"); - if (InstInfo.IsImmutableSizedInst) - return InstInfo.InstSize; - assert(InstInfo.InstFragment != nullptr && - "Must have a valid InstFragment to retrieve InstSize from"); - return InstInfo.InstFragment->getContents().size(); - } - void setInstAndInstSize(const MCInst &Inst, size_t InstSize) { - InstInfo.IsInitialized = true; - InstInfo.IsImmutableSizedInst = true; - InstInfo.Inst = Inst; - InstInfo.InstSize = InstSize; - } - void setInstAndInstFragment(const MCInst &Inst, - MCRelaxableFragment *InstFragment) { - InstInfo.IsInitialized = true; - InstInfo.IsImmutableSizedInst = false; - InstInfo.Inst = Inst; - InstInfo.InstFragment = InstFragment; - } - uint64_t getSize() const { return Size; } - void setSize(uint64_t Value) { Size = Value; } - bool isInstructionInitialized() const { return InstInfo.IsInitialized; } - - static bool classof(const MCFragment *F) { - return F->getKind() == MCFragment::FT_Padding; - } -}; - class MCFillFragment : public MCFragment { + uint8_t ValueSize; /// Value to use for filling bytes. uint64_t Value; - uint8_t ValueSize; /// The number of bytes to insert. const MCExpr &NumValues; @@ -439,7 +328,7 @@ class MCFillFragment : public MCFragment { public: MCFillFragment(uint64_t Value, uint8_t VSize, const MCExpr &NumValues, SMLoc Loc, MCSection *Sec = nullptr) - : MCFragment(FT_Fill, false, Sec), Value(Value), ValueSize(VSize), + : MCFragment(FT_Fill, false, Sec), ValueSize(VSize), Value(Value), NumValues(NumValues), Loc(Loc) {} uint64_t getValue() const { return Value; } @@ -454,22 +343,20 @@ public: }; class MCOrgFragment : public MCFragment { - /// The offset this fragment should start at. - const MCExpr *Offset; - /// Value to use for filling bytes. int8_t Value; + /// The offset this fragment should start at. + const MCExpr *Offset; + /// Source location of the directive that this fragment was created for. SMLoc Loc; public: MCOrgFragment(const MCExpr &Offset, int8_t Value, SMLoc Loc, MCSection *Sec = nullptr) - : MCFragment(FT_Org, false, Sec), Offset(&Offset), Value(Value), Loc(Loc) {} - - /// \name Accessors - /// @{ + : MCFragment(FT_Org, false, Sec), Value(Value), Offset(&Offset), + Loc(Loc) {} const MCExpr &getOffset() const { return *Offset; } @@ -477,31 +364,26 @@ public: SMLoc getLoc() const { return Loc; } - /// @} - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_Org; } }; class MCLEBFragment : public MCFragment { - /// Value - The value this fragment should contain. - const MCExpr *Value; - - /// IsSigned - True if this is a sleb128, false if uleb128. + /// True if this is a sleb128, false if uleb128. bool IsSigned; + /// The value this fragment should contain. + const MCExpr *Value; + SmallString<8> Contents; public: MCLEBFragment(const MCExpr &Value_, bool IsSigned_, MCSection *Sec = nullptr) - : MCFragment(FT_LEB, false, Sec), Value(&Value_), IsSigned(IsSigned_) { + : MCFragment(FT_LEB, false, Sec), IsSigned(IsSigned_), Value(&Value_) { Contents.push_back(0); } - /// \name Accessors - /// @{ - const MCExpr &getValue() const { return *Value; } bool isSigned() const { return IsSigned; } @@ -517,11 +399,11 @@ public: }; class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> { - /// LineDelta - the value of the difference between the two line numbers + /// The value of the difference between the two line numbers /// between two .loc dwarf directives. int64_t LineDelta; - /// AddrDelta - The expression for the difference of the two symbols that + /// The expression for the difference of the two symbols that /// make up the address delta between two .loc dwarf directives. const MCExpr *AddrDelta; @@ -531,22 +413,17 @@ public: : MCEncodedFragmentWithFixups<8, 1>(FT_Dwarf, false, Sec), LineDelta(LineDelta), AddrDelta(&AddrDelta) {} - /// \name Accessors - /// @{ - int64_t getLineDelta() const { return LineDelta; } const MCExpr &getAddrDelta() const { return *AddrDelta; } - /// @} - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_Dwarf; } }; class MCDwarfCallFrameFragment : public MCEncodedFragmentWithFixups<8, 1> { - /// AddrDelta - The expression for the difference of the two symbols that + /// The expression for the difference of the two symbols that /// make up the address delta between two .cfi_* dwarf directives. const MCExpr *AddrDelta; @@ -555,13 +432,8 @@ public: : MCEncodedFragmentWithFixups<8, 1>(FT_DwarfFrame, false, Sec), AddrDelta(&AddrDelta) {} - /// \name Accessors - /// @{ - const MCExpr &getAddrDelta() const { return *AddrDelta; } - /// @} - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_DwarfFrame; } @@ -575,14 +447,9 @@ public: MCSymbolIdFragment(const MCSymbol *Sym, MCSection *Sec = nullptr) : MCFragment(FT_SymbolId, false, Sec), Sym(Sym) {} - /// \name Accessors - /// @{ - const MCSymbol *getSymbol() { return Sym; } const MCSymbol *getSymbol() const { return Sym; } - /// @} - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_SymbolId; } @@ -611,17 +478,12 @@ public: StartFileId(StartFileId), StartLineNum(StartLineNum), FnStartSym(FnStartSym), FnEndSym(FnEndSym) {} - /// \name Accessors - /// @{ - const MCSymbol *getFnStartSym() const { return FnStartSym; } const MCSymbol *getFnEndSym() const { return FnEndSym; } SmallString<8> &getContents() { return Contents; } const SmallString<8> &getContents() const { return Contents; } - /// @} - static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_CVInlineLines; } @@ -644,20 +506,53 @@ public: Ranges(Ranges.begin(), Ranges.end()), FixedSizePortion(FixedSizePortion) {} - /// \name Accessors - /// @{ ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> getRanges() const { return Ranges; } StringRef getFixedSizePortion() const { return FixedSizePortion; } - /// @} static bool classof(const MCFragment *F) { return F->getKind() == MCFragment::FT_CVDefRange; } }; +/// Represents required padding such that a particular other set of fragments +/// does not cross a particular power-of-two boundary. The other fragments must +/// follow this one within the same section. +class MCBoundaryAlignFragment : public MCFragment { + /// The alignment requirement of the branch to be aligned. + Align AlignBoundary; + /// Flag to indicate whether the branch is fused. Use in determining the + /// region of fragments being aligned. + bool Fused : 1; + /// Flag to indicate whether NOPs should be emitted. + bool EmitNops : 1; + /// The size of the fragment. The size is lazily set during relaxation, and + /// is not meaningful before that. + uint64_t Size = 0; + +public: + MCBoundaryAlignFragment(Align AlignBoundary, bool Fused = false, + bool EmitNops = false, MCSection *Sec = nullptr) + : MCFragment(FT_BoundaryAlign, false, Sec), AlignBoundary(AlignBoundary), + Fused(Fused), EmitNops(EmitNops) {} + + uint64_t getSize() const { return Size; } + void setSize(uint64_t Value) { Size = Value; } + + Align getAlignment() const { return AlignBoundary; } + + bool isFused() const { return Fused; } + void setFused(bool Value) { Fused = Value; } + + bool canEmitNops() const { return EmitNops; } + void setEmitNops(bool Value) { EmitNops = Value; } + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_BoundaryAlign; + } +}; } // end namespace llvm #endif // LLVM_MC_MCFRAGMENT_H diff --git a/llvm/include/llvm/MC/MCInst.h b/llvm/include/llvm/MC/MCInst.h index 8df8096bba94..360dbda58fcb 100644 --- a/llvm/include/llvm/MC/MCInst.h +++ b/llvm/include/llvm/MC/MCInst.h @@ -157,13 +157,14 @@ public: /// instruction. class MCInst { unsigned Opcode = 0; - SMLoc Loc; - SmallVector<MCOperand, 8> Operands; // These flags could be used to pass some info from one target subcomponent // to another, for example, from disassembler to asm printer. The values of // the flags have any sense on target level only (e.g. prefixes on x86). unsigned Flags = 0; + SMLoc Loc; + SmallVector<MCOperand, 8> Operands; + public: MCInst() = default; diff --git a/llvm/include/llvm/MC/MCInstPrinter.h b/llvm/include/llvm/MC/MCInstPrinter.h index 4501ce3084c8..97290e73c28f 100644 --- a/llvm/include/llvm/MC/MCInstPrinter.h +++ b/llvm/include/llvm/MC/MCInstPrinter.h @@ -16,6 +16,7 @@ namespace llvm { class MCAsmInfo; class MCInst; +class MCOperand; class MCInstrInfo; class MCRegisterInfo; class MCSubtargetInfo; @@ -34,6 +35,8 @@ enum Style { } // end namespace HexStyle +struct AliasMatchingData; + /// This is an instance of a target assembly language printer that /// converts an MCInst to valid target assembly syntax. class MCInstPrinter { @@ -58,6 +61,10 @@ protected: /// Utility function for printing annotations. void printAnnotation(raw_ostream &OS, StringRef Annot); + /// Helper for matching MCInsts to alias patterns when printing instructions. + const char *matchAliasPatterns(const MCInst *MI, const MCSubtargetInfo *STI, + const AliasMatchingData &M); + public: MCInstPrinter(const MCAsmInfo &mai, const MCInstrInfo &mii, const MCRegisterInfo &mri) : MAI(mai), MII(mii), MRI(mri) {} @@ -72,8 +79,8 @@ public: void setCommentStream(raw_ostream &OS) { CommentStream = &OS; } /// Print the specified MCInst to the specified raw_ostream. - virtual void printInst(const MCInst *MI, raw_ostream &OS, StringRef Annot, - const MCSubtargetInfo &STI) = 0; + virtual void printInst(const MCInst *MI, uint64_t Address, StringRef Annot, + const MCSubtargetInfo &STI, raw_ostream &OS) = 0; /// Return the name of the specified opcode enum (e.g. "MOV32ri") or /// empty if we can't resolve it. @@ -104,6 +111,48 @@ public: format_object<uint64_t> formatHex(uint64_t Value) const; }; +/// Map from opcode to pattern list by binary search. +struct PatternsForOpcode { + uint32_t Opcode; + uint16_t PatternStart; + uint16_t NumPatterns; +}; + +/// Data for each alias pattern. Includes feature bits, string, number of +/// operands, and a variadic list of conditions to check. +struct AliasPattern { + uint32_t AsmStrOffset; + uint32_t AliasCondStart; + uint8_t NumOperands; + uint8_t NumConds; +}; + +struct AliasPatternCond { + enum CondKind : uint8_t { + K_Feature, // Match only if a feature is enabled. + K_NegFeature, // Match only if a feature is disabled. + K_Ignore, // Match any operand. + K_Reg, // Match a specific register. + K_TiedReg, // Match another already matched register. + K_Imm, // Match a specific immediate. + K_RegClass, // Match registers in a class. + K_Custom, // Call custom matcher by index. + }; + + CondKind Kind; + uint32_t Value; +}; + +/// Tablegenerated data structures needed to match alias patterns. +struct AliasMatchingData { + ArrayRef<PatternsForOpcode> OpToPatterns; + ArrayRef<AliasPattern> Patterns; + ArrayRef<AliasPatternCond> PatternConds; + StringRef AsmStrings; + bool (*ValidateMCOperand)(const MCOperand &MCOp, const MCSubtargetInfo &STI, + unsigned PredicateIndex); +}; + } // end namespace llvm #endif // LLVM_MC_MCINSTPRINTER_H diff --git a/llvm/include/llvm/MC/MCInstrDesc.h b/llvm/include/llvm/MC/MCInstrDesc.h index e75a27614a22..506f2c09304c 100644 --- a/llvm/include/llvm/MC/MCInstrDesc.h +++ b/llvm/include/llvm/MC/MCInstrDesc.h @@ -37,7 +37,12 @@ enum OperandConstraint { /// These are flags set on operands, but should be considered /// private, all access should go through the MCOperandInfo accessors. /// See the accessors for a description of what these are. -enum OperandFlags { LookupPtrRegClass = 0, Predicate, OptionalDef }; +enum OperandFlags { + LookupPtrRegClass = 0, + Predicate, + OptionalDef, + BranchTarget +}; /// Operands are tagged with one of the values of this enum. enum OperandType { @@ -98,6 +103,9 @@ public: /// Set if this operand is a optional def. bool isOptionalDef() const { return Flags & (1 << MCOI::OptionalDef); } + /// Set if this operand is a branch target. + bool isBranchTarget() const { return Flags & (1 << MCOI::BranchTarget); } + bool isGenericType() const { return OperandType >= MCOI::OPERAND_FIRST_GENERIC && OperandType <= MCOI::OPERAND_LAST_GENERIC; @@ -168,6 +176,7 @@ enum Flag { Add, Trap, VariadicOpsAreDefs, + Authenticated, }; } @@ -304,7 +313,7 @@ public: /// block. The TargetInstrInfo::AnalyzeBranch method can be used to get more /// information about this branch. bool isConditionalBranch() const { - return isBranch() & !isBarrier() & !isIndirectBranch(); + return isBranch() && !isBarrier() && !isIndirectBranch(); } /// Return true if this is a branch which always @@ -312,7 +321,7 @@ public: /// TargetInstrInfo::AnalyzeBranch method can be used to get more information /// about this branch. bool isUnconditionalBranch() const { - return isBranch() & isBarrier() & !isIndirectBranch(); + return isBranch() && isBarrier() && !isIndirectBranch(); } /// Return true if this is a branch or an instruction which directly @@ -408,6 +417,15 @@ public: return Flags & (1ULL << MCID::VariadicOpsAreDefs); } + /// Return true if this instruction authenticates a pointer (e.g. LDRAx/BRAx + /// from ARMv8.3, which perform loads/branches with authentication). + /// + /// An authenticated instruction may fail in an ABI-defined manner when + /// operating on an invalid signed pointer. + bool isAuthenticated() const { + return Flags & (1ULL << MCID::Authenticated); + } + //===--------------------------------------------------------------------===// // Side Effect Analysis //===--------------------------------------------------------------------===// diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h index 278aebee99ac..853e5066f039 100644 --- a/llvm/include/llvm/MC/MCMachObjectWriter.h +++ b/llvm/include/llvm/MC/MCMachObjectWriter.h @@ -28,7 +28,9 @@ class MachObjectWriter; class MCMachObjectTargetWriter : public MCObjectTargetWriter { const unsigned Is64Bit : 1; const uint32_t CPUType; - const uint32_t CPUSubtype; +protected: + uint32_t CPUSubtype; +public: unsigned LocalDifference_RIT; protected: diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h index abc87bf27748..2f7f5d64b466 100644 --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -27,20 +27,20 @@ class MCObjectFileInfo { protected: /// True if .comm supports alignment. This is a hack for as long as we /// support 10.4 Tiger, whose assembler doesn't support alignment on comm. - bool CommDirectiveSupportsAlignment; + bool CommDirectiveSupportsAlignment = false; /// True if target object file supports a weak_definition of constant 0 for an /// omitted EH frame. - bool SupportsWeakOmittedEHFrame; + bool SupportsWeakOmittedEHFrame = false; /// True if the target object file supports emitting a compact unwind section /// without an associated EH frame section. - bool SupportsCompactUnwindWithoutEHFrame; + bool SupportsCompactUnwindWithoutEHFrame = false; /// OmitDwarfIfHaveCompactUnwind - True if the target object file /// supports having some functions with compact unwind and other with /// dwarf unwind. - bool OmitDwarfIfHaveCompactUnwind; + bool OmitDwarfIfHaveCompactUnwind = false; /// FDE CFI encoding. Controls the encoding of the begin label in the /// .eh_frame section. Unlike the LSDA encoding, personality encoding, and @@ -49,134 +49,136 @@ protected: unsigned FDECFIEncoding = 0; /// Compact unwind encoding indicating that we should emit only an EH frame. - unsigned CompactUnwindDwarfEHFrameOnly; + unsigned CompactUnwindDwarfEHFrameOnly = 0; /// Section directive for standard text. - MCSection *TextSection; + MCSection *TextSection = nullptr; /// Section directive for standard data. - MCSection *DataSection; + MCSection *DataSection = nullptr; /// Section that is default initialized to zero. - MCSection *BSSSection; + MCSection *BSSSection = nullptr; /// Section that is readonly and can contain arbitrary initialized data. /// Targets are not required to have a readonly section. If they don't, /// various bits of code will fall back to using the data section for /// constants. - MCSection *ReadOnlySection; + MCSection *ReadOnlySection = nullptr; /// If exception handling is supported by the target, this is the section the /// Language Specific Data Area information is emitted to. - MCSection *LSDASection; + MCSection *LSDASection = nullptr; /// If exception handling is supported by the target and the target can /// support a compact representation of the CIE and FDE, this is the section /// to emit them into. - MCSection *CompactUnwindSection; + MCSection *CompactUnwindSection = nullptr; // Dwarf sections for debug info. If a target supports debug info, these must // be set. - MCSection *DwarfAbbrevSection; - MCSection *DwarfInfoSection; - MCSection *DwarfLineSection; - MCSection *DwarfLineStrSection; - MCSection *DwarfFrameSection; - MCSection *DwarfPubTypesSection; - const MCSection *DwarfDebugInlineSection; - MCSection *DwarfStrSection; - MCSection *DwarfLocSection; - MCSection *DwarfARangesSection; - MCSection *DwarfRangesSection; - MCSection *DwarfMacinfoSection; + MCSection *DwarfAbbrevSection = nullptr; + MCSection *DwarfInfoSection = nullptr; + MCSection *DwarfLineSection = nullptr; + MCSection *DwarfLineStrSection = nullptr; + MCSection *DwarfFrameSection = nullptr; + MCSection *DwarfPubTypesSection = nullptr; + const MCSection *DwarfDebugInlineSection = nullptr; + MCSection *DwarfStrSection = nullptr; + MCSection *DwarfLocSection = nullptr; + MCSection *DwarfARangesSection = nullptr; + MCSection *DwarfRangesSection = nullptr; + MCSection *DwarfMacinfoSection = nullptr; // The pubnames section is no longer generated by default. The generation // can be enabled by a compiler flag. - MCSection *DwarfPubNamesSection; + MCSection *DwarfPubNamesSection = nullptr; /// Accelerator table sections. DwarfDebugNamesSection is the DWARF v5 /// accelerator table, while DwarfAccelNamesSection, DwarfAccelObjCSection, /// DwarfAccelNamespaceSection, DwarfAccelTypesSection are pre-DWARF v5 /// extensions. - MCSection *DwarfDebugNamesSection; - MCSection *DwarfAccelNamesSection; - MCSection *DwarfAccelObjCSection; - MCSection *DwarfAccelNamespaceSection; - MCSection *DwarfAccelTypesSection; + MCSection *DwarfDebugNamesSection = nullptr; + MCSection *DwarfAccelNamesSection = nullptr; + MCSection *DwarfAccelObjCSection = nullptr; + MCSection *DwarfAccelNamespaceSection = nullptr; + MCSection *DwarfAccelTypesSection = nullptr; // These are used for the Fission separate debug information files. - MCSection *DwarfInfoDWOSection; - MCSection *DwarfTypesDWOSection; - MCSection *DwarfAbbrevDWOSection; - MCSection *DwarfStrDWOSection; - MCSection *DwarfLineDWOSection; - MCSection *DwarfLocDWOSection; - MCSection *DwarfStrOffDWOSection; + MCSection *DwarfInfoDWOSection = nullptr; + MCSection *DwarfTypesDWOSection = nullptr; + MCSection *DwarfAbbrevDWOSection = nullptr; + MCSection *DwarfStrDWOSection = nullptr; + MCSection *DwarfLineDWOSection = nullptr; + MCSection *DwarfLocDWOSection = nullptr; + MCSection *DwarfStrOffDWOSection = nullptr; + MCSection *DwarfMacinfoDWOSection = nullptr; /// The DWARF v5 string offset and address table sections. - MCSection *DwarfStrOffSection; - MCSection *DwarfAddrSection; + MCSection *DwarfStrOffSection = nullptr; + MCSection *DwarfAddrSection = nullptr; /// The DWARF v5 range list section. - MCSection *DwarfRnglistsSection; + MCSection *DwarfRnglistsSection = nullptr; /// The DWARF v5 locations list section. - MCSection *DwarfLoclistsSection; + MCSection *DwarfLoclistsSection = nullptr; - /// The DWARF v5 range list section for fission. - MCSection *DwarfRnglistsDWOSection; + /// The DWARF v5 range and location list sections for fission. + MCSection *DwarfRnglistsDWOSection = nullptr; + MCSection *DwarfLoclistsDWOSection = nullptr; // These are for Fission DWP files. - MCSection *DwarfCUIndexSection; - MCSection *DwarfTUIndexSection; + MCSection *DwarfCUIndexSection = nullptr; + MCSection *DwarfTUIndexSection = nullptr; /// Section for newer gnu pubnames. - MCSection *DwarfGnuPubNamesSection; + MCSection *DwarfGnuPubNamesSection = nullptr; /// Section for newer gnu pubtypes. - MCSection *DwarfGnuPubTypesSection; + MCSection *DwarfGnuPubTypesSection = nullptr; // Section for Swift AST - MCSection *DwarfSwiftASTSection; + MCSection *DwarfSwiftASTSection = nullptr; - MCSection *COFFDebugSymbolsSection; - MCSection *COFFDebugTypesSection; - MCSection *COFFGlobalTypeHashesSection; + MCSection *COFFDebugSymbolsSection = nullptr; + MCSection *COFFDebugTypesSection = nullptr; + MCSection *COFFGlobalTypeHashesSection = nullptr; /// Extra TLS Variable Data section. /// /// If the target needs to put additional information for a TLS variable, /// it'll go here. - MCSection *TLSExtraDataSection; + MCSection *TLSExtraDataSection = nullptr; /// Section directive for Thread Local data. ELF, MachO, COFF, and Wasm. - MCSection *TLSDataSection; // Defaults to ".tdata". + MCSection *TLSDataSection = nullptr; // Defaults to ".tdata". /// Section directive for Thread Local uninitialized data. /// /// Null if this target doesn't support a BSS section. ELF and MachO only. - MCSection *TLSBSSSection; // Defaults to ".tbss". + MCSection *TLSBSSSection = nullptr; // Defaults to ".tbss". /// StackMap section. - MCSection *StackMapSection; + MCSection *StackMapSection = nullptr; /// FaultMap section. - MCSection *FaultMapSection; + MCSection *FaultMapSection = nullptr; /// Remarks section. - MCSection *RemarksSection; + MCSection *RemarksSection = nullptr; /// EH frame section. /// /// It is initialized on demand so it can be overwritten (with uniquing). - MCSection *EHFrameSection; + MCSection *EHFrameSection = nullptr; /// Section containing metadata on function stack sizes. - MCSection *StackSizesSection; + MCSection *StackSizesSection = nullptr; mutable DenseMap<const MCSymbol *, unsigned> StackSizesUniquing; // ELF specific sections. - MCSection *DataRelROSection; - MCSection *MergeableConst4Section; - MCSection *MergeableConst8Section; - MCSection *MergeableConst16Section; - MCSection *MergeableConst32Section; + MCSection *DataRelROSection = nullptr; + MCSection *MergeableConst4Section = nullptr; + MCSection *MergeableConst8Section = nullptr; + MCSection *MergeableConst16Section = nullptr; + MCSection *MergeableConst32Section = nullptr; // MachO specific sections. @@ -184,33 +186,35 @@ protected: /// /// Contains the source code name of the variable, visibility and a pointer to /// the initial value (.tdata or .tbss). - MCSection *TLSTLVSection; // Defaults to ".tlv". + MCSection *TLSTLVSection = nullptr; // Defaults to ".tlv". /// Section for thread local data initialization functions. - const MCSection *TLSThreadInitSection; // Defaults to ".thread_init_func". - - MCSection *CStringSection; - MCSection *UStringSection; - MCSection *TextCoalSection; - MCSection *ConstTextCoalSection; - MCSection *ConstDataSection; - MCSection *DataCoalSection; - MCSection *ConstDataCoalSection; - MCSection *DataCommonSection; - MCSection *DataBSSSection; - MCSection *FourByteConstantSection; - MCSection *EightByteConstantSection; - MCSection *SixteenByteConstantSection; - MCSection *LazySymbolPointerSection; - MCSection *NonLazySymbolPointerSection; - MCSection *ThreadLocalPointerSection; + // Defaults to ".thread_init_func". + const MCSection *TLSThreadInitSection = nullptr; + + MCSection *CStringSection = nullptr; + MCSection *UStringSection = nullptr; + MCSection *TextCoalSection = nullptr; + MCSection *ConstTextCoalSection = nullptr; + MCSection *ConstDataSection = nullptr; + MCSection *DataCoalSection = nullptr; + MCSection *ConstDataCoalSection = nullptr; + MCSection *DataCommonSection = nullptr; + MCSection *DataBSSSection = nullptr; + MCSection *FourByteConstantSection = nullptr; + MCSection *EightByteConstantSection = nullptr; + MCSection *SixteenByteConstantSection = nullptr; + MCSection *LazySymbolPointerSection = nullptr; + MCSection *NonLazySymbolPointerSection = nullptr; + MCSection *ThreadLocalPointerSection = nullptr; /// COFF specific sections. - MCSection *DrectveSection; - MCSection *PDataSection; - MCSection *XDataSection; - MCSection *SXDataSection; - MCSection *GFIDsSection; + MCSection *DrectveSection = nullptr; + MCSection *PDataSection = nullptr; + MCSection *XDataSection = nullptr; + MCSection *SXDataSection = nullptr; + MCSection *GFIDsSection = nullptr; + MCSection *GLJMPSection = nullptr; public: void InitMCObjectFileInfo(const Triple &TT, bool PIC, MCContext &ctx, @@ -297,6 +301,12 @@ public: MCSection *getDwarfRnglistsDWOSection() const { return DwarfRnglistsDWOSection; } + MCSection *getDwarfLoclistsDWOSection() const { + return DwarfLoclistsDWOSection; + } + MCSection *getDwarfMacinfoDWOSection() const { + return DwarfMacinfoDWOSection; + } MCSection *getDwarfCUIndexSection() const { return DwarfCUIndexSection; } MCSection *getDwarfTUIndexSection() const { return DwarfTUIndexSection; } MCSection *getDwarfSwiftASTSection() const { return DwarfSwiftASTSection; } @@ -379,6 +389,7 @@ public: MCSection *getXDataSection() const { return XDataSection; } MCSection *getSXDataSection() const { return SXDataSection; } MCSection *getGFIDsSection() const { return GFIDsSection; } + MCSection *getGLJMPSection() const { return GLJMPSection; } MCSection *getEHFrameSection() { return EHFrameSection; @@ -391,8 +402,8 @@ public: private: Environment Env; - bool PositionIndependent; - MCContext *Ctx; + bool PositionIndependent = false; + MCContext *Ctx = nullptr; Triple TT; VersionTuple SDKVersion; diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index 8affca49490f..9e3f87565e26 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -38,6 +38,8 @@ class MCObjectStreamer : public MCStreamer { bool EmitEHFrame; bool EmitDebugFrame; SmallVector<MCSymbol *, 2> PendingLabels; + SmallVector<MCSection*, 2> PendingLabelSections; + unsigned CurSubsectionIdx; struct PendingMCFixup { const MCSymbol *Sym; MCFixup Fixup; @@ -84,22 +86,27 @@ public: /// Optionally a \p STI can be passed in so that a new fragment is created /// if the Subtarget differs from the current fragment. MCDataFragment *getOrCreateDataFragment(const MCSubtargetInfo* STI = nullptr); - MCPaddingFragment *getOrCreatePaddingFragment(); protected: bool changeSectionImpl(MCSection *Section, const MCExpr *Subsection); - /// If any labels have been emitted but not assigned fragments, ensure that - /// they get assigned, either to F if possible or to a new data fragment. - /// Optionally, it is also possible to provide an offset \p FOffset, which - /// will be used as a symbol offset within the fragment. + /// Assign a label to the current Section and Subsection even though a + /// fragment is not yet present. Use flushPendingLabels(F) to associate + /// a fragment with this label. + void addPendingLabel(MCSymbol* label); + + /// If any labels have been emitted but not assigned fragments in the current + /// Section and Subsection, ensure that they get assigned, either to fragment + /// F if possible or to a new data fragment. Optionally, one can provide an + /// offset \p FOffset as a symbol offset within the fragment. void flushPendingLabels(MCFragment *F, uint64_t FOffset = 0); public: void visitUsedSymbol(const MCSymbol &Sym) override; - /// Create a dummy fragment to assign any pending labels. - void flushPendingLabels() { flushPendingLabels(nullptr); } + /// Create a data fragment for any pending labels across all Sections + /// and Subsections. + void flushPendingLabels(); MCAssembler &getAssembler() { return *Assembler; } MCAssembler *getAssemblerPtr() override; @@ -107,7 +114,8 @@ public: /// @{ void EmitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override; - virtual void EmitLabel(MCSymbol *Symbol, SMLoc Loc, MCFragment *F); + virtual void EmitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, MCFragment *F, + uint64_t Offset); void EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) override; void EmitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc = SMLoc()) override; @@ -132,10 +140,6 @@ public: unsigned MaxBytesToEmit = 0) override; void emitValueToOffset(const MCExpr *Offset, unsigned char Value, SMLoc Loc) override; - void - EmitCodePaddingBasicBlockStart(const MCCodePaddingContext &Context) override; - void - EmitCodePaddingBasicBlockEnd(const MCCodePaddingContext &Context) override; void EmitDwarfLocDirective(unsigned FileNo, unsigned Line, unsigned Column, unsigned Flags, unsigned Isa, unsigned Discriminator, diff --git a/llvm/include/llvm/MC/MCParser/AsmCond.h b/llvm/include/llvm/MC/MCParser/AsmCond.h index ea2155010081..44edd2b758ff 100644 --- a/llvm/include/llvm/MC/MCParser/AsmCond.h +++ b/llvm/include/llvm/MC/MCParser/AsmCond.h @@ -30,8 +30,6 @@ public: ConditionalAssemblyType TheCond = NoCond; bool CondMet = false; bool Ignore = false; - - AsmCond() = default; }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h index 2b6e2aa48b8f..abb95628c2a9 100644 --- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -71,10 +71,10 @@ public: /// variable/label? Only valid when parsing MS-style inline assembly. virtual bool needAddressOf() const { return false; } - /// isOffsetOf - Do we need to emit code to get the offset of the variable, - /// rather then the value of the variable? Only valid when parsing MS-style - /// inline assembly. - virtual bool isOffsetOf() const { return false; } + /// isOffsetOfLocal - Do we need to emit code to get the offset of the local + /// variable, rather than its value? Only valid when parsing MS-style inline + /// assembly. + virtual bool isOffsetOfLocal() const { return false; } /// getOffsetOfLoc - Get the location of the offset operator. virtual SMLoc getOffsetOfLoc() const { return SMLoc(); } diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h index 849dbd57f1aa..6e4821cbc7b9 100644 --- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -35,6 +35,7 @@ enum AsmRewriteKind { AOK_Align, // Rewrite align as .align. AOK_EVEN, // Rewrite even as .even. AOK_Emit, // Rewrite _emit as .byte. + AOK_CallInput, // Rewrite in terms of ${N:P}. AOK_Input, // Rewrite in terms of $N. AOK_Output, // Rewrite in terms of $N. AOK_SizeDirective, // Add a sizing directive (e.g., dword ptr). @@ -49,6 +50,7 @@ const char AsmRewritePrecedence [] = { 2, // AOK_EVEN 2, // AOK_Emit 3, // AOK_Input + 3, // AOK_CallInput 3, // AOK_Output 5, // AOK_SizeDirective 1, // AOK_Label @@ -64,39 +66,27 @@ struct IntelExpr { int64_t Imm; StringRef BaseReg; StringRef IndexReg; + StringRef OffsetName; unsigned Scale; - IntelExpr(bool needBracs = false) : NeedBracs(needBracs), Imm(0), - BaseReg(StringRef()), IndexReg(StringRef()), - Scale(1) {} - // Compund immediate expression - IntelExpr(int64_t imm, bool needBracs) : IntelExpr(needBracs) { - Imm = imm; - } - // [Reg + ImmediateExpression] - // We don't bother to emit an immediate expression evaluated to zero - IntelExpr(StringRef reg, int64_t imm = 0, unsigned scale = 0, - bool needBracs = true) : - IntelExpr(imm, needBracs) { - IndexReg = reg; + IntelExpr() + : NeedBracs(false), Imm(0), BaseReg(StringRef()), IndexReg(StringRef()), + OffsetName(StringRef()), Scale(1) {} + // [BaseReg + IndexReg * ScaleExpression + OFFSET name + ImmediateExpression] + IntelExpr(StringRef baseReg, StringRef indexReg, unsigned scale, + StringRef offsetName, int64_t imm, bool needBracs) + : NeedBracs(needBracs), Imm(imm), BaseReg(baseReg), IndexReg(indexReg), + OffsetName(offsetName), Scale(1) { if (scale) Scale = scale; } - // [BaseReg + IndexReg * ScaleExpression + ImmediateExpression] - IntelExpr(StringRef baseReg, StringRef indexReg, unsigned scale = 0, - int64_t imm = 0, bool needBracs = true) : - IntelExpr(indexReg, imm, scale, needBracs) { - BaseReg = baseReg; - } - bool hasBaseReg() const { - return BaseReg.size(); - } - bool hasIndexReg() const { - return IndexReg.size(); - } - bool hasRegs() const { - return hasBaseReg() || hasIndexReg(); - } + bool hasBaseReg() const { return !BaseReg.empty(); } + bool hasIndexReg() const { return !IndexReg.empty(); } + bool hasRegs() const { return hasBaseReg() || hasIndexReg(); } + bool hasOffset() const { return !OffsetName.empty(); } + // Normally we won't emit immediates unconditionally, + // unless we've got no other components + bool emitImm() const { return !(hasRegs() || hasOffset()); } bool isValid() const { return (Scale == 1) || (hasIndexReg() && (Scale == 2 || Scale == 4 || Scale == 8)); @@ -107,13 +97,14 @@ struct AsmRewrite { AsmRewriteKind Kind; SMLoc Loc; unsigned Len; + bool Done; int64_t Val; StringRef Label; IntelExpr IntelExp; public: AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len = 0, int64_t val = 0) - : Kind(kind), Loc(loc), Len(len), Val(val) {} + : Kind(kind), Loc(loc), Len(len), Done(false), Val(val) {} AsmRewrite(AsmRewriteKind kind, SMLoc loc, unsigned len, StringRef label) : AsmRewrite(kind, loc, len) { Label = label; } AsmRewrite(SMLoc loc, unsigned len, IntelExpr exp) @@ -174,6 +165,7 @@ struct DiagnosticPredicate { : DiagnosticPredicateTy::NearMatch) {} DiagnosticPredicate(DiagnosticPredicateTy T) : Type(T) {} DiagnosticPredicate(const DiagnosticPredicate &) = default; + DiagnosticPredicate& operator=(const DiagnosticPredicate &) = default; operator bool() const { return Type == DiagnosticPredicateTy::Match; } bool isMatch() const { return Type == DiagnosticPredicateTy::Match; } diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h index c7dc56ea588e..9864d95d19e0 100644 --- a/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/llvm/include/llvm/MC/MCRegisterInfo.h @@ -16,11 +16,13 @@ #define LLVM_MC_MCREGISTERINFO_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/MC/LaneBitmask.h" #include "llvm/MC/MCRegister.h" #include <cassert> #include <cstdint> +#include <iterator> #include <utility> namespace llvm { @@ -177,6 +179,9 @@ private: DenseMap<MCRegister, int> L2CVRegs; // LLVM to CV regs mapping public: + // Forward declaration to become a friend class of DiffListIterator. + template <class SubT> class mc_difflist_iterator; + /// DiffListIterator - Base iterator class that can traverse the /// differentially encoded register and regunit lists in DiffLists. /// Don't use this class directly, use one of the specialized sub-classes @@ -220,8 +225,113 @@ public: if (!advance()) List = nullptr; } + + template <class SubT> friend class MCRegisterInfo::mc_difflist_iterator; + }; + + /// Forward iterator using DiffListIterator. + template <class SubT> + class mc_difflist_iterator + : public iterator_facade_base<mc_difflist_iterator<SubT>, + std::forward_iterator_tag, MCPhysReg> { + MCRegisterInfo::DiffListIterator Iter; + /// Current value as MCPhysReg, so we can return a reference to it. + MCPhysReg Val; + + protected: + mc_difflist_iterator(MCRegisterInfo::DiffListIterator Iter) : Iter(Iter) {} + + // Allow conversion between instantiations where valid. + mc_difflist_iterator(MCRegister Reg, const MCPhysReg *DiffList) { + Iter.init(Reg, DiffList); + Val = *Iter; + } + + public: + // Allow default construction to build variables, but this doesn't build + // a useful iterator. + mc_difflist_iterator() = default; + + /// Return an iterator past the last element. + static SubT end() { + SubT End; + End.Iter.List = nullptr; + return End; + } + + bool operator==(const mc_difflist_iterator &Arg) const { + return Iter.List == Arg.Iter.List; + } + + const MCPhysReg &operator*() const { return Val; } + + using mc_difflist_iterator::iterator_facade_base::operator++; + void operator++() { + assert(Iter.List && "Cannot increment the end iterator!"); + ++Iter; + Val = *Iter; + } }; + /// Forward iterator over all sub-registers. + /// TODO: Replace remaining uses of MCSubRegIterator. + class mc_subreg_iterator : public mc_difflist_iterator<mc_subreg_iterator> { + public: + mc_subreg_iterator(MCRegisterInfo::DiffListIterator Iter) + : mc_difflist_iterator(Iter) {} + mc_subreg_iterator() = default; + mc_subreg_iterator(MCRegister Reg, const MCRegisterInfo *MCRI) + : mc_difflist_iterator(Reg, MCRI->DiffLists + MCRI->get(Reg).SubRegs) {} + }; + + /// Forward iterator over all super-registers. + /// TODO: Replace remaining uses of MCSuperRegIterator. + class mc_superreg_iterator + : public mc_difflist_iterator<mc_superreg_iterator> { + public: + mc_superreg_iterator(MCRegisterInfo::DiffListIterator Iter) + : mc_difflist_iterator(Iter) {} + mc_superreg_iterator() = default; + mc_superreg_iterator(MCRegister Reg, const MCRegisterInfo *MCRI) + : mc_difflist_iterator(Reg, + MCRI->DiffLists + MCRI->get(Reg).SuperRegs) {} + }; + + /// Return an iterator range over all sub-registers of \p Reg, excluding \p + /// Reg. + iterator_range<mc_subreg_iterator> subregs(MCRegister Reg) const { + return make_range(std::next(mc_subreg_iterator(Reg, this)), + mc_subreg_iterator::end()); + } + + /// Return an iterator range over all sub-registers of \p Reg, including \p + /// Reg. + iterator_range<mc_subreg_iterator> subregs_inclusive(MCRegister Reg) const { + return make_range({Reg, this}, mc_subreg_iterator::end()); + } + + /// Return an iterator range over all super-registers of \p Reg, excluding \p + /// Reg. + iterator_range<mc_superreg_iterator> superregs(MCRegister Reg) const { + return make_range(std::next(mc_superreg_iterator(Reg, this)), + mc_superreg_iterator::end()); + } + + /// Return an iterator range over all super-registers of \p Reg, including \p + /// Reg. + iterator_range<mc_superreg_iterator> + superregs_inclusive(MCRegister Reg) const { + return make_range({Reg, this}, mc_superreg_iterator::end()); + } + + /// Return an iterator range over all sub- and super-registers of \p Reg, + /// including \p Reg. + detail::concat_range<const MCPhysReg, iterator_range<mc_subreg_iterator>, + iterator_range<mc_superreg_iterator>> + sub_and_superregs_inclusive(MCRegister Reg) const { + return concat<const MCPhysReg>(subregs_inclusive(Reg), superregs(Reg)); + } + // These iterators are allowed to sub-class DiffListIterator and access // internal list pointers. friend class MCSubRegIterator; diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index d057feda87d8..d80cc5b086b3 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -78,10 +78,6 @@ private: /// Whether this section has had instructions emitted into it. bool HasInstructions : 1; - /// Whether this section has had data emitted into it. - /// Right now this is only used by the ARM backend. - bool HasData : 1; - bool IsRegistered : 1; MCDummyFragment DummyFragment; @@ -92,6 +88,15 @@ private: /// below that number. SmallVector<std::pair<unsigned, MCFragment *>, 1> SubsectionFragmentMap; + /// State for tracking labels that don't yet have Fragments + struct PendingLabel { + MCSymbol* Sym; + unsigned Subsection; + PendingLabel(MCSymbol* Sym, unsigned Subsection = 0) + : Sym(Sym), Subsection(Subsection) {} + }; + SmallVector<PendingLabel, 2> PendingLabels; + protected: SectionVariant Variant; SectionKind Kind; @@ -141,9 +146,6 @@ public: bool hasInstructions() const { return HasInstructions; } void setHasInstructions(bool Value) { HasInstructions = Value; } - bool hasData() const { return HasData; } - void setHasData(bool Value) { HasData = Value; } - bool isRegistered() const { return IsRegistered; } void setIsRegistered(bool Value) { IsRegistered = Value; } @@ -166,12 +168,6 @@ public: iterator end() { return Fragments.end(); } const_iterator end() const { return Fragments.end(); } - reverse_iterator rbegin() { return Fragments.rbegin(); } - const_reverse_iterator rbegin() const { return Fragments.rbegin(); } - - reverse_iterator rend() { return Fragments.rend(); } - const_reverse_iterator rend() const { return Fragments.rend(); } - MCSection::iterator getSubsectionInsertionPoint(unsigned Subsection); void dump() const; @@ -187,6 +183,18 @@ public: /// Check whether this section is "virtual", that is has no actual object /// file contents. virtual bool isVirtualSection() const = 0; + + /// Add a pending label for the requested subsection. This label will be + /// associated with a fragment in flushPendingLabels() + void addPendingLabel(MCSymbol* label, unsigned Subsection = 0); + + /// Associate all pending labels in a subsection with a fragment. + void flushPendingLabels(MCFragment *F, uint64_t FOffset = 0, + unsigned Subsection = 0); + + /// Associate all pending labels with empty data fragments. One fragment + /// will be created for each subsection as necessary. + void flushPendingLabels(); }; } // end namespace llvm diff --git a/llvm/include/llvm/MC/MCSectionXCOFF.h b/llvm/include/llvm/MC/MCSectionXCOFF.h index ee302ed5ecec..611eb69c1493 100644 --- a/llvm/include/llvm/MC/MCSectionXCOFF.h +++ b/llvm/include/llvm/MC/MCSectionXCOFF.h @@ -16,11 +16,10 @@ #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/XCOFF.h" #include "llvm/MC/MCSection.h" +#include "llvm/MC/MCSymbolXCOFF.h" namespace llvm { -class MCSymbol; - // This class represents an XCOFF `Control Section`, more commonly referred to // as a csect. A csect represents the smallest possible unit of data/code which // will be relocated as a single block. A csect can either be: @@ -38,14 +37,18 @@ class MCSectionXCOFF final : public MCSection { XCOFF::StorageMappingClass MappingClass; XCOFF::SymbolType Type; XCOFF::StorageClass StorageClass; + MCSymbolXCOFF *const QualName; MCSectionXCOFF(StringRef Section, XCOFF::StorageMappingClass SMC, XCOFF::SymbolType ST, XCOFF::StorageClass SC, SectionKind K, - MCSymbol *Begin) + MCSymbolXCOFF *QualName, MCSymbol *Begin) : MCSection(SV_XCOFF, K, Begin), Name(Section), MappingClass(SMC), - Type(ST), StorageClass(SC) { - assert((ST == XCOFF::XTY_SD || ST == XCOFF::XTY_CM) && + Type(ST), StorageClass(SC), QualName(QualName) { + assert((ST == XCOFF::XTY_SD || ST == XCOFF::XTY_CM || ST == XCOFF::XTY_ER) && "Invalid or unhandled type for csect."); + assert(QualName != nullptr && "QualName is needed."); + QualName->setStorageClass(SC); + QualName->setContainingCsect(this); } public: @@ -59,6 +62,7 @@ public: XCOFF::StorageMappingClass getMappingClass() const { return MappingClass; } XCOFF::StorageClass getStorageClass() const { return StorageClass; } XCOFF::SymbolType getCSectType() const { return Type; } + MCSymbolXCOFF *getQualNameSymbol() const { return QualName; } void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, raw_ostream &OS, diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 6b48580ae57c..ba1649d33d12 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -103,8 +103,9 @@ public: // Allow a target to add behavior to the emitAssignment of MCStreamer. virtual void emitAssignment(MCSymbol *Symbol, const MCExpr *Value); - virtual void prettyPrintAsm(MCInstPrinter &InstPrinter, raw_ostream &OS, - const MCInst &Inst, const MCSubtargetInfo &STI); + virtual void prettyPrintAsm(MCInstPrinter &InstPrinter, uint64_t Address, + const MCInst &Inst, const MCSubtargetInfo &STI, + raw_ostream &OS); virtual void emitDwarfFileDirective(StringRef Directive); @@ -222,6 +223,13 @@ class MCStreamer { bool UseAssemblerInfoForParsing; + /// Is the assembler allowed to insert padding automatically? For + /// correctness reasons, we sometimes need to ensure instructions aren't + /// seperated in unexpected ways. At the moment, this feature is only + /// useable from an integrated assembler, but assembly syntax is under + /// discussion for future inclusion. + bool AllowAutoPadding = false; + protected: MCStreamer(MCContext &Ctx); @@ -266,6 +274,9 @@ public: return TargetStreamer.get(); } + void setAllowAutoPadding(bool v) { AllowAutoPadding = v; } + bool getAllowAutoPadding() const { return AllowAutoPadding; } + /// When emitting an object file, create and emit a real label. When emitting /// textual assembly, this should do nothing to avoid polluting our output. virtual MCSymbol *EmitCFILabel(); @@ -546,11 +557,13 @@ public: /// Emits an lcomm directive with XCOFF csect information. /// - /// \param Symbol - The symbol we are emiting. + /// \param LabelSym - Label on the block of storage. /// \param Size - The size of the block of storage. - /// \param ByteAlignment - The alignment of the symbol in bytes. Must be a power - /// of 2. - virtual void EmitXCOFFLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, + /// \param CsectSym - Csect name for the block of storage. + /// \param ByteAlignment - The alignment of the symbol in bytes. Must be a + /// power of 2. + virtual void EmitXCOFFLocalCommonSymbol(MCSymbol *LabelSym, uint64_t Size, + MCSymbol *CsectSym, unsigned ByteAlignment); /// Emit an ELF .size directive. diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h index 189484deac7e..84263bf94035 100644 --- a/llvm/include/llvm/MC/MCSymbol.h +++ b/llvm/include/llvm/MC/MCSymbol.h @@ -178,14 +178,6 @@ private: llvm_unreachable("Constructor throws?"); } - MCSection *getSectionPtr() const { - if (MCFragment *F = getFragment()) { - assert(F != AbsolutePseudoFragment); - return F->getParent(); - } - return nullptr; - } - /// Get a reference to the name field. Requires that we have a name const StringMapEntry<bool> *&getNameEntryPtr() { assert(FragmentAndHasName.getInt() && "Name is required"); @@ -267,7 +259,7 @@ public: /// Get the section associated with a defined, non-absolute symbol. MCSection &getSection() const { assert(isInSection() && "Invalid accessor!"); - return *getSectionPtr(); + return *getFragment()->getParent(); } /// Mark the symbol as defined in the fragment \p F. diff --git a/llvm/include/llvm/MC/MCSymbolWasm.h b/llvm/include/llvm/MC/MCSymbolWasm.h index 95beebe3f75a..ba2068a46071 100644 --- a/llvm/include/llvm/MC/MCSymbolWasm.h +++ b/llvm/include/llvm/MC/MCSymbolWasm.h @@ -21,6 +21,7 @@ class MCSymbolWasm : public MCSymbol { mutable bool IsUsedInGOT = false; Optional<std::string> ImportModule; Optional<std::string> ImportName; + Optional<std::string> ExportName; wasm::WasmSignature *Signature = nullptr; Optional<wasm::WasmGlobalType> GlobalType; Optional<wasm::WasmEventType> EventType; @@ -78,6 +79,7 @@ public: } void setImportModule(StringRef Name) { ImportModule = Name; } + bool hasImportName() const { return ImportName.hasValue(); } const StringRef getImportName() const { if (ImportName.hasValue()) { return ImportName.getValue(); @@ -86,6 +88,10 @@ public: } void setImportName(StringRef Name) { ImportName = Name; } + bool hasExportName() const { return ExportName.hasValue(); } + const StringRef getExportName() const { return ExportName.getValue(); } + void setExportName(StringRef Name) { ExportName = Name; } + void setUsedInGOT() const { IsUsedInGOT = true; } bool isUsedInGOT() const { return IsUsedInGOT; } diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h index 98ecd2466926..07dfb5d29977 100644 --- a/llvm/include/llvm/MC/MCSymbolXCOFF.h +++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h @@ -9,6 +9,7 @@ #define LLVM_MC_MCSYMBOLXCOFF_H #include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/XCOFF.h" #include "llvm/MC/MCSymbol.h" @@ -48,6 +49,19 @@ public: return ContainingCsect; } + bool hasContainingCsect() const { return ContainingCsect != nullptr; } + + StringRef getUnqualifiedName() const { + const StringRef name = getName(); + if (name.back() == ']') { + StringRef lhs, rhs; + std::tie(lhs, rhs) = name.rsplit('['); + assert(!rhs.empty() && "Invalid SMC format in XCOFF symbol."); + return lhs; + } + return name; + } + private: Optional<XCOFF::StorageClass> StorageClass; MCSectionXCOFF *ContainingCsect = nullptr; diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h index f184620ff047..51a5fc9aa26a 100644 --- a/llvm/include/llvm/MC/MCTargetOptions.h +++ b/llvm/include/llvm/MC/MCTargetOptions.h @@ -46,7 +46,6 @@ public: bool MCSaveTempLabels : 1; bool MCUseDwarfDirectory : 1; bool MCIncrementalLinkerCompatible : 1; - bool MCPIECopyRelocations : 1; bool ShowMCEncoding : 1; bool ShowMCInst : 1; bool AsmVerbose : 1; diff --git a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.inc b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.inc index 9f1177f470b9..93e21b626eac 100644 --- a/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.inc +++ b/llvm/include/llvm/MC/MCTargetOptionsCommandFlags.inc @@ -28,8 +28,6 @@ static cl::opt<bool> IncrementalLinkerCompatible( "When used with filetype=obj, " "emit an object file which can be used with an incremental linker")); -static cl::opt<bool> PIECopyRelocations("pie-copy-relocations", cl::desc("PIE Copy Relocations")); - static cl::opt<int> DwarfVersion("dwarf-version", cl::desc("Dwarf version"), cl::init(0)); @@ -55,7 +53,6 @@ static MCTargetOptions InitMCTargetOptionsFromFlags() { MCTargetOptions Options; Options.MCRelaxAll = RelaxAll; Options.MCIncrementalLinkerCompatible = IncrementalLinkerCompatible; - Options.MCPIECopyRelocations = PIECopyRelocations; Options.DwarfVersion = DwarfVersion; Options.ShowMCInst = ShowMCInst; Options.ABIName = ABIName; diff --git a/llvm/include/llvm/MC/MCXCOFFStreamer.h b/llvm/include/llvm/MC/MCXCOFFStreamer.h index b13b0031d18e..f6f8e56977d3 100644 --- a/llvm/include/llvm/MC/MCXCOFFStreamer.h +++ b/llvm/include/llvm/MC/MCXCOFFStreamer.h @@ -26,7 +26,8 @@ public: uint64_t Size = 0, unsigned ByteAlignment = 0, SMLoc Loc = SMLoc()) override; void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override; - void EmitXCOFFLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size, + void EmitXCOFFLocalCommonSymbol(MCSymbol *LabelSym, uint64_t Size, + MCSymbol *CsectSym, unsigned ByteAlign) override; }; diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h index 28b00c8413de..42c5b67ac3fa 100644 --- a/llvm/include/llvm/Object/ELF.h +++ b/llvm/include/llvm/Object/ELF.h @@ -402,12 +402,17 @@ ELFFile<ELFT>::getSectionContentsAsArray(const Elf_Shdr *Sec) const { " has an invalid sh_size (" + Twine(Size) + ") which is not a multiple of its sh_entsize (" + Twine(Sec->sh_entsize) + ")"); - if ((std::numeric_limits<uintX_t>::max() - Offset < Size) || - Offset + Size > Buf.size()) + if (std::numeric_limits<uintX_t>::max() - Offset < Size) return createError("section " + getSecIndexForError(this, Sec) + " has a sh_offset (0x" + Twine::utohexstr(Offset) + - ") + sh_size (0x" + Twine(Size) + + ") + sh_size (0x" + Twine::utohexstr(Size) + ") that cannot be represented"); + if (Offset + Size > Buf.size()) + return createError("section " + getSecIndexForError(this, Sec) + + " has a sh_offset (0x" + Twine::utohexstr(Offset) + + ") + sh_size (0x" + Twine::utohexstr(Size) + + ") that is greater than the file size (0x" + + Twine::utohexstr(Buf.size()) + ")"); if (Offset % alignof(T)) // TODO: this error is untested. @@ -641,11 +646,12 @@ ELFFile<ELFT>::getSHNDXTable(const Elf_Shdr &Section, SymTable.sh_type) + " section (expected SHT_SYMTAB/SHT_DYNSYM)"); - if (V.size() != (SymTable.sh_size / sizeof(Elf_Sym))) - return createError("SHT_SYMTAB_SHNDX section has sh_size (" + - Twine(SymTable.sh_size) + - ") which is not equal to the number of symbols (" + - Twine(V.size()) + ")"); + uint64_t Syms = SymTable.sh_size / sizeof(Elf_Sym); + if (V.size() != Syms) + return createError("SHT_SYMTAB_SHNDX has " + Twine(V.size()) + + " entries, but the symbol table associated has " + + Twine(Syms)); + return V; } diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h index 424289a9ccaa..8a68e49477fd 100644 --- a/llvm/include/llvm/Object/ELFObjectFile.h +++ b/llvm/include/llvm/Object/ELFObjectFile.h @@ -723,6 +723,8 @@ template <class ELFT> Expected<ArrayRef<uint8_t>> ELFObjectFile<ELFT>::getSectionContents(DataRefImpl Sec) const { const Elf_Shdr *EShdr = getSection(Sec); + if (EShdr->sh_type == ELF::SHT_NOBITS) + return makeArrayRef((const uint8_t *)base(), 0); if (std::error_code EC = checkOffset(getMemoryBufferRef(), (uintptr_t)base() + EShdr->sh_offset, EShdr->sh_size)) diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h index 76be8049a7d4..c3ecdd93563f 100644 --- a/llvm/include/llvm/Object/MachO.h +++ b/llvm/include/llvm/Object/MachO.h @@ -567,7 +567,7 @@ public: static StringRef guessLibraryShortName(StringRef Name, bool &isFramework, StringRef &Suffix); - static Triple::ArchType getArch(uint32_t CPUType); + static Triple::ArchType getArch(uint32_t CPUType, uint32_t CPUSubType); static Triple getArchTriple(uint32_t CPUType, uint32_t CPUSubType, const char **McpuDefault = nullptr, const char **ArchFlag = nullptr); diff --git a/llvm/include/llvm/Object/ObjectFile.h b/llvm/include/llvm/Object/ObjectFile.h index adc9dbc189af..2f1493457605 100644 --- a/llvm/include/llvm/Object/ObjectFile.h +++ b/llvm/include/llvm/Object/ObjectFile.h @@ -155,6 +155,8 @@ inline bool operator==(const SectionedAddress &LHS, std::tie(RHS.SectionIndex, RHS.Address); } +raw_ostream &operator<<(raw_ostream &OS, const SectionedAddress &Addr); + /// This is a value type class that represents a single symbol in the list of /// symbols in the object file. class SymbolRef : public BasicSymbolRef { diff --git a/llvm/include/llvm/Object/Wasm.h b/llvm/include/llvm/Object/Wasm.h index e130ea32ed21..8af94c4963b6 100644 --- a/llvm/include/llvm/Object/Wasm.h +++ b/llvm/include/llvm/Object/Wasm.h @@ -280,6 +280,7 @@ private: uint32_t StartFunction = -1; bool HasLinkingSection = false; bool HasDylinkSection = false; + bool SeenCodeSection = false; wasm::WasmLinkingData LinkingData; uint32_t NumImportedGlobals = 0; uint32_t NumImportedFunctions = 0; diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h index 84073ce5f6cf..fcdbf7a8095c 100644 --- a/llvm/include/llvm/Object/XCOFFObjectFile.h +++ b/llvm/include/llvm/Object/XCOFFObjectFile.h @@ -47,7 +47,26 @@ struct XCOFFFileHeader64 { support::ubig32_t NumberOfSymTableEntries; }; -struct XCOFFSectionHeader32 { +template <typename T> struct XCOFFSectionHeader { + // Least significant 3 bits are reserved. + static constexpr unsigned SectionFlagsReservedMask = 0x7; + + // The low order 16 bits of section flags denotes the section type. + static constexpr unsigned SectionFlagsTypeMask = 0xffffu; + +public: + StringRef getName() const; + uint16_t getSectionType() const; + bool isReservedSectionType() const; +}; + +// Explicit extern template declarations. +struct XCOFFSectionHeader32; +struct XCOFFSectionHeader64; +extern template struct XCOFFSectionHeader<XCOFFSectionHeader32>; +extern template struct XCOFFSectionHeader<XCOFFSectionHeader64>; + +struct XCOFFSectionHeader32 : XCOFFSectionHeader<XCOFFSectionHeader32> { char Name[XCOFF::NameSize]; support::ubig32_t PhysicalAddress; support::ubig32_t VirtualAddress; @@ -58,11 +77,9 @@ struct XCOFFSectionHeader32 { support::ubig16_t NumberOfRelocations; support::ubig16_t NumberOfLineNumbers; support::big32_t Flags; - - StringRef getName() const; }; -struct XCOFFSectionHeader64 { +struct XCOFFSectionHeader64 : XCOFFSectionHeader<XCOFFSectionHeader64> { char Name[XCOFF::NameSize]; support::ubig64_t PhysicalAddress; support::ubig64_t VirtualAddress; @@ -74,8 +91,6 @@ struct XCOFFSectionHeader64 { support::ubig32_t NumberOfLineNumbers; support::big32_t Flags; char Padding[4]; - - StringRef getName() const; }; struct XCOFFSymbolEntry { diff --git a/llvm/include/llvm/ObjectYAML/DWARFYAML.h b/llvm/include/llvm/ObjectYAML/DWARFYAML.h index 525fd9a89242..26dabfcf27fe 100644 --- a/llvm/include/llvm/ObjectYAML/DWARFYAML.h +++ b/llvm/include/llvm/ObjectYAML/DWARFYAML.h @@ -161,8 +161,6 @@ struct Data { } // end namespace DWARFYAML } // end namespace llvm -LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::Hex64) -LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::Hex8) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::AttributeAbbrev) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::Abbrev) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::DWARFYAML::ARangeDescriptor) diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h index 0898a0e7d532..f87135e6a1b5 100644 --- a/llvm/include/llvm/ObjectYAML/ELFYAML.h +++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h @@ -64,6 +64,8 @@ LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_ASE) LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_AFL_FLAGS1) LLVM_YAML_STRONG_TYPEDEF(uint32_t, MIPS_ISA) +LLVM_YAML_STRONG_TYPEDEF(StringRef, YAMLFlowString) + // For now, hardcode 64 bits everywhere that 32 or 64 would be needed // since 64-bit can hold 32-bit values too. struct FileHeader { @@ -124,24 +126,43 @@ struct StackSizeEntry { llvm::yaml::Hex64 Size; }; -struct Section { - enum class SectionKind { +struct NoteEntry { + StringRef Name; + yaml::BinaryRef Desc; + llvm::yaml::Hex32 Type; +}; + +struct Chunk { + enum class ChunkKind { Dynamic, Group, RawContent, Relocation, + Relr, NoBits, + Note, Hash, + GnuHash, Verdef, Verneed, StackSizes, SymtabShndxSection, Symver, MipsABIFlags, - Addrsig + Addrsig, + Fill, + LinkerOptions, + DependentLibraries, }; - SectionKind Kind; + + ChunkKind Kind; StringRef Name; + + Chunk(ChunkKind K) : Kind(K) {} + virtual ~Chunk(); +}; + +struct Section : public Chunk { ELF_SHT Type; Optional<ELF_SHF> Flags; llvm::yaml::Hex64 Address; @@ -153,9 +174,10 @@ struct Section { // When they are, this flag is used to signal about that. bool IsImplicit; - Section(SectionKind Kind, bool IsImplicit = false) - : Kind(Kind), IsImplicit(IsImplicit) {} - virtual ~Section(); + Section(ChunkKind Kind, bool IsImplicit = false) + : Chunk(Kind), IsImplicit(IsImplicit) {} + + static bool classof(const Chunk *S) { return S->Kind != ChunkKind::Fill; } // The following members are used to override section fields which is // useful for creating invalid objects. @@ -171,6 +193,26 @@ struct Section { // This can be used to override the sh_size field. It does not affect the // content written. Optional<llvm::yaml::Hex64> ShSize; + + // This can be used to override the sh_flags field. + Optional<llvm::yaml::Hex64> ShFlags; +}; + +// Fill is a block of data which is placed outside of sections. It is +// not present in the sections header table, but it might affect the output file +// size and program headers produced. +struct Fill : Chunk { + Optional<yaml::BinaryRef> Pattern; + llvm::yaml::Hex64 Size; + + // We have to remember the offset of the fill, because it does not have + // a corresponding section header, unlike a section. We might need this + // information when writing the output. + uint64_t ShOffset; + + Fill() : Chunk(ChunkKind::Fill) {} + + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Fill; } }; struct StackSizesSection : Section { @@ -178,10 +220,10 @@ struct StackSizesSection : Section { Optional<llvm::yaml::Hex64> Size; Optional<std::vector<StackSizeEntry>> Entries; - StackSizesSection() : Section(SectionKind::StackSizes) {} + StackSizesSection() : Section(ChunkKind::StackSizes) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::StackSizes; + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::StackSizes; } static bool nameMatches(StringRef Name) { @@ -193,11 +235,9 @@ struct DynamicSection : Section { std::vector<DynamicEntry> Entries; Optional<yaml::BinaryRef> Content; - DynamicSection() : Section(SectionKind::Dynamic) {} + DynamicSection() : Section(ChunkKind::Dynamic) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::Dynamic; - } + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Dynamic; } }; struct RawContentSection : Section { @@ -205,21 +245,29 @@ struct RawContentSection : Section { Optional<llvm::yaml::Hex64> Size; Optional<llvm::yaml::Hex64> Info; - RawContentSection() : Section(SectionKind::RawContent) {} + RawContentSection() : Section(ChunkKind::RawContent) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::RawContent; + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::RawContent; } }; struct NoBitsSection : Section { llvm::yaml::Hex64 Size; - NoBitsSection() : Section(SectionKind::NoBits) {} + NoBitsSection() : Section(ChunkKind::NoBits) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::NoBits; - } + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::NoBits; } +}; + +struct NoteSection : Section { + Optional<yaml::BinaryRef> Content; + Optional<llvm::yaml::Hex64> Size; + Optional<std::vector<ELFYAML::NoteEntry>> Notes; + + NoteSection() : Section(ChunkKind::Note) {} + + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Note; } }; struct HashSection : Section { @@ -228,9 +276,42 @@ struct HashSection : Section { Optional<std::vector<uint32_t>> Bucket; Optional<std::vector<uint32_t>> Chain; - HashSection() : Section(SectionKind::Hash) {} + HashSection() : Section(ChunkKind::Hash) {} + + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Hash; } +}; + +struct GnuHashHeader { + // The number of hash buckets. + // Not used when dumping the object, but can be used to override + // the real number of buckets when emiting an object from a YAML document. + Optional<llvm::yaml::Hex32> NBuckets; - static bool classof(const Section *S) { return S->Kind == SectionKind::Hash; } + // Index of the first symbol in the dynamic symbol table + // included in the hash table. + llvm::yaml::Hex32 SymNdx; + + // The number of words in the Bloom filter. + // Not used when dumping the object, but can be used to override the real + // number of words in the Bloom filter when emiting an object from a YAML + // document. + Optional<llvm::yaml::Hex32> MaskWords; + + // A shift constant used by the Bloom filter. + llvm::yaml::Hex32 Shift2; +}; + +struct GnuHashSection : Section { + Optional<yaml::BinaryRef> Content; + + Optional<GnuHashHeader> Header; + Optional<std::vector<llvm::yaml::Hex64>> BloomFilter; + Optional<std::vector<llvm::yaml::Hex32>> HashBuckets; + Optional<std::vector<llvm::yaml::Hex32>> HashValues; + + GnuHashSection() : Section(ChunkKind::GnuHash) {} + + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::GnuHash; } }; struct VernauxEntry { @@ -247,13 +328,14 @@ struct VerneedEntry { }; struct VerneedSection : Section { - std::vector<VerneedEntry> VerneedV; + Optional<yaml::BinaryRef> Content; + Optional<std::vector<VerneedEntry>> VerneedV; llvm::yaml::Hex64 Info; - VerneedSection() : Section(SectionKind::Verneed) {} + VerneedSection() : Section(ChunkKind::Verneed) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::Verneed; + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::Verneed; } }; @@ -271,20 +353,44 @@ struct AddrsigSection : Section { Optional<llvm::yaml::Hex64> Size; Optional<std::vector<AddrsigSymbol>> Symbols; - AddrsigSection() : Section(SectionKind::Addrsig) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::Addrsig; + AddrsigSection() : Section(ChunkKind::Addrsig) {} + + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Addrsig; } +}; + +struct LinkerOption { + StringRef Key; + StringRef Value; +}; + +struct LinkerOptionsSection : Section { + Optional<std::vector<LinkerOption>> Options; + Optional<yaml::BinaryRef> Content; + + LinkerOptionsSection() : Section(ChunkKind::LinkerOptions) {} + + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::LinkerOptions; + } +}; + +struct DependentLibrariesSection : Section { + Optional<std::vector<YAMLFlowString>> Libs; + Optional<yaml::BinaryRef> Content; + + DependentLibrariesSection() : Section(ChunkKind::DependentLibraries) {} + + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::DependentLibraries; } }; struct SymverSection : Section { std::vector<uint16_t> Entries; - SymverSection() : Section(SectionKind::Symver) {} + SymverSection() : Section(ChunkKind::Symver) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::Symver; - } + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Symver; } }; struct VerdefEntry { @@ -296,27 +402,25 @@ struct VerdefEntry { }; struct VerdefSection : Section { - std::vector<VerdefEntry> Entries; + Optional<std::vector<VerdefEntry>> Entries; + Optional<yaml::BinaryRef> Content; + llvm::yaml::Hex64 Info; - VerdefSection() : Section(SectionKind::Verdef) {} + VerdefSection() : Section(ChunkKind::Verdef) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::Verdef; - } + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Verdef; } }; struct Group : Section { // Members of a group contain a flag and a list of section indices // that are part of the group. std::vector<SectionOrType> Members; - StringRef Signature; /* Info */ + Optional<StringRef> Signature; /* Info */ - Group() : Section(SectionKind::Group) {} + Group() : Section(ChunkKind::Group) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::Group; - } + static bool classof(const Chunk *S) { return S->Kind == ChunkKind::Group; } }; struct Relocation { @@ -330,20 +434,31 @@ struct RelocationSection : Section { std::vector<Relocation> Relocations; StringRef RelocatableSec; /* Info */ - RelocationSection() : Section(SectionKind::Relocation) {} + RelocationSection() : Section(ChunkKind::Relocation) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::Relocation; + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::Relocation; + } +}; + +struct RelrSection : Section { + Optional<std::vector<llvm::yaml::Hex64>> Entries; + Optional<yaml::BinaryRef> Content; + + RelrSection() : Section(ChunkKind::Relr) {} + + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::Relr; } }; struct SymtabShndxSection : Section { std::vector<uint32_t> Entries; - SymtabShndxSection() : Section(SectionKind::SymtabShndxSection) {} + SymtabShndxSection() : Section(ChunkKind::SymtabShndxSection) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::SymtabShndxSection; + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::SymtabShndxSection; } }; @@ -361,23 +476,35 @@ struct MipsABIFlags : Section { MIPS_AFL_FLAGS1 Flags1; llvm::yaml::Hex32 Flags2; - MipsABIFlags() : Section(SectionKind::MipsABIFlags) {} + MipsABIFlags() : Section(ChunkKind::MipsABIFlags) {} - static bool classof(const Section *S) { - return S->Kind == SectionKind::MipsABIFlags; + static bool classof(const Chunk *S) { + return S->Kind == ChunkKind::MipsABIFlags; } }; struct Object { FileHeader Header; std::vector<ProgramHeader> ProgramHeaders; - std::vector<std::unique_ptr<Section>> Sections; + + // An object might contain output section descriptions as well as + // custom data that does not belong to any section. + std::vector<std::unique_ptr<Chunk>> Chunks; + // Although in reality the symbols reside in a section, it is a lot // cleaner and nicer if we read them from the YAML as a separate // top-level key, which automatically ensures that invariants like there // being a single SHT_SYMTAB section are upheld. Optional<std::vector<Symbol>> Symbols; - std::vector<Symbol> DynamicSymbols; + Optional<std::vector<Symbol>> DynamicSymbols; + + std::vector<Section *> getSections() { + std::vector<Section *> Ret; + for (const std::unique_ptr<Chunk> &Sec : Chunks) + if (auto S = dyn_cast<ELFYAML::Section>(Sec.get())) + Ret.push_back(S); + return Ret; + } }; } // end namespace ELFYAML @@ -386,8 +513,10 @@ struct Object { LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::AddrsigSymbol) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::StackSizeEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::DynamicEntry) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::LinkerOption) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::NoteEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::ProgramHeader) -LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::ELFYAML::Section>) +LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::ELFYAML::Chunk>) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::Symbol) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::VerdefEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::VernauxEntry) @@ -524,10 +653,18 @@ template <> struct MappingTraits<ELFYAML::StackSizeEntry> { static void mapping(IO &IO, ELFYAML::StackSizeEntry &Rel); }; +template <> struct MappingTraits<ELFYAML::GnuHashHeader> { + static void mapping(IO &IO, ELFYAML::GnuHashHeader &Rel); +}; + template <> struct MappingTraits<ELFYAML::DynamicEntry> { static void mapping(IO &IO, ELFYAML::DynamicEntry &Rel); }; +template <> struct MappingTraits<ELFYAML::NoteEntry> { + static void mapping(IO &IO, ELFYAML::NoteEntry &N); +}; + template <> struct MappingTraits<ELFYAML::VerdefEntry> { static void mapping(IO &IO, ELFYAML::VerdefEntry &E); }; @@ -544,14 +681,17 @@ template <> struct MappingTraits<ELFYAML::AddrsigSymbol> { static void mapping(IO &IO, ELFYAML::AddrsigSymbol &Sym); }; +template <> struct MappingTraits<ELFYAML::LinkerOption> { + static void mapping(IO &IO, ELFYAML::LinkerOption &Sym); +}; + template <> struct MappingTraits<ELFYAML::Relocation> { static void mapping(IO &IO, ELFYAML::Relocation &Rel); }; -template <> -struct MappingTraits<std::unique_ptr<ELFYAML::Section>> { - static void mapping(IO &IO, std::unique_ptr<ELFYAML::Section> &Section); - static StringRef validate(IO &io, std::unique_ptr<ELFYAML::Section> &Section); +template <> struct MappingTraits<std::unique_ptr<ELFYAML::Chunk>> { + static void mapping(IO &IO, std::unique_ptr<ELFYAML::Chunk> &C); + static StringRef validate(IO &io, std::unique_ptr<ELFYAML::Chunk> &C); }; template <> diff --git a/llvm/include/llvm/ObjectYAML/YAML.h b/llvm/include/llvm/ObjectYAML/YAML.h index 37014109a615..3bf6527a7e2d 100644 --- a/llvm/include/llvm/ObjectYAML/YAML.h +++ b/llvm/include/llvm/ObjectYAML/YAML.h @@ -85,7 +85,8 @@ public: /// Write the contents (regardless of whether it is binary or a /// hex string) as binary to the given raw_ostream. - void writeAsBinary(raw_ostream &OS) const; + /// N can be used to specify the maximum number of bytes. + void writeAsBinary(raw_ostream &OS, uint64_t N = UINT64_MAX) const; /// Write the contents (regardless of whether it is binary or a /// hex string) as hex to the given raw_ostream. diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h index 1d53ae32cf37..49419844e7ad 100644 --- a/llvm/include/llvm/Pass.h +++ b/llvm/include/llvm/Pass.h @@ -57,13 +57,11 @@ enum PassManagerType { PMT_FunctionPassManager, ///< FPPassManager PMT_LoopPassManager, ///< LPPassManager PMT_RegionPassManager, ///< RGPassManager - PMT_BasicBlockPassManager, ///< BBPassManager PMT_Last }; // Different types of passes. enum PassKind { - PT_BasicBlock, PT_Region, PT_Loop, PT_Function, @@ -305,56 +303,6 @@ protected: bool skipFunction(const Function &F) const; }; -//===----------------------------------------------------------------------===// -/// Deprecated - do not create new passes as BasicBlockPasses. Use FunctionPass -/// with a loop over the BasicBlocks instead. -// -/// BasicBlockPass class - This class is used to implement most local -/// optimizations. Optimizations should subclass this class if they -/// meet the following constraints: -/// 1. Optimizations are local, operating on either a basic block or -/// instruction at a time. -/// 2. Optimizations do not modify the CFG of the contained function, or any -/// other basic block in the function. -/// 3. Optimizations conform to all of the constraints of FunctionPasses. -/// -class BasicBlockPass : public Pass { -public: - explicit BasicBlockPass(char &pid) : Pass(PT_BasicBlock, pid) {} - - /// createPrinterPass - Get a basic block printer pass. - Pass *createPrinterPass(raw_ostream &OS, - const std::string &Banner) const override; - - using llvm::Pass::doInitialization; - using llvm::Pass::doFinalization; - - /// doInitialization - Virtual method overridden by BasicBlockPass subclasses - /// to do any necessary per-function initialization. - virtual bool doInitialization(Function &); - - /// runOnBasicBlock - Virtual method overriden by subclasses to do the - /// per-basicblock processing of the pass. - virtual bool runOnBasicBlock(BasicBlock &BB) = 0; - - /// doFinalization - Virtual method overriden by BasicBlockPass subclasses to - /// do any post processing needed after all passes have run. - virtual bool doFinalization(Function &); - - void preparePassManager(PMStack &PMS) override; - - void assignPassManager(PMStack &PMS, PassManagerType T) override; - - /// Return what kind of Pass Manager can manage this pass. - PassManagerType getPotentialPassManagerType() const override; - -protected: - /// Optional passes call this function to check whether the pass should be - /// skipped. This is the case when Attribute::OptimizeNone is set or when - /// optimization bisect is over the limit. - bool skipBasicBlock(const BasicBlock &BB) const; -}; - /// If the user specifies the -time-passes argument on an LLVM tool command line /// then the value of this boolean will be true, otherwise false. /// This is the storage for the -time-passes option. @@ -364,7 +312,6 @@ extern bool TimePassesIsEnabled; // Include support files that contain important APIs commonly used by Passes, // but that we want to separate out to make it easier to read the header files. -#include "llvm/InitializePasses.h" #include "llvm/PassAnalysisSupport.h" #include "llvm/PassSupport.h" diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index f73e4b42dd4b..e7db8fd421fe 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -151,10 +151,6 @@ public: /// Optimize quickly without destroying debuggability. /// - /// FIXME: The current and historical behavior of this level does *not* - /// agree with this goal, but we would like to move toward this goal in the - /// future. - /// /// This level is tuned to produce a result from the optimizer as quickly /// as possible and to avoid destroying debuggability. This tends to result /// in a very good development mode where the compiled code will be @@ -164,9 +160,9 @@ public: /// debugging of the resulting binary. /// /// As an example, complex loop transformations such as versioning, - /// vectorization, or fusion might not make sense here due to the degree to - /// which the executed code would differ from the source code, and the - /// potential compile time cost. + /// vectorization, or fusion don't make sense here due to the degree to + /// which the executed code differs from the source code, and the compile time + /// cost. O1, /// Optimize for fast execution as much as possible without triggering @@ -635,6 +631,13 @@ public: std::string ProfileFile, std::string ProfileRemappingFile); + + /// Returns PIC. External libraries can use this to register pass + /// instrumentation callbacks. + PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const { + return PIC; + } + private: static Optional<std::vector<PipelineElement>> parsePipelineText(StringRef Text); diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h index 0dd0c7ec8065..f272e8c03903 100644 --- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -311,7 +311,7 @@ struct FunctionRecord { /// Regions in the function along with their counts. std::vector<CountedRegion> CountedRegions; /// The number of times this function was executed. - uint64_t ExecutionCount; + uint64_t ExecutionCount = 0; FunctionRecord(StringRef Name, ArrayRef<StringRef> Filenames) : Name(Name), Filenames(Filenames.begin(), Filenames.end()) {} diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index c26f76949992..1f8872947c64 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -885,7 +885,7 @@ uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, std::unique_ptr<InstrProfValueData[]> InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC) const { - uint64_t Dummy; + uint64_t Dummy = 0; uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC); uint32_t N = getNumValueDataForSite(ValueKind, Site); if (N == 0) { diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 749781b9ac2d..99f41d8fef07 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -130,7 +130,9 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) @@ -628,7 +630,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 4 +#define INSTR_PROF_RAW_VERSION 5 /* Indexed profile format version (start from 1). */ #define INSTR_PROF_INDEX_VERSION 5 /* Coverage mapping format vresion (start from 0). */ @@ -742,7 +744,7 @@ typedef struct InstrProfValueData { #endif /* INSTR_PROF_DATA_INC */ #ifndef INSTR_ORDER_FILE_INC -// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). +/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ #define INSTR_ORDER_FILE_BUFFER_SIZE 131072 #define INSTR_ORDER_FILE_BUFFER_BITS 17 #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index 55418d9d0f9c..f8be89c569b7 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -387,7 +387,10 @@ public: if (FS != iter->second.end()) return &FS->second; // If we cannot find exact match of the callee name, return the FS with - // the max total count. + // the max total count. Only do this when CalleeName is not provided, + // i.e., only for indirect calls. + if (!CalleeName.empty()) + return nullptr; uint64_t MaxTotalSamples = 0; const FunctionSamples *R = nullptr; for (const auto &NameFS : iter->second) diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h index 5a5d4cfde224..72b178edc260 100644 --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -358,6 +358,15 @@ public: return getSamplesFor(CanonName); } + /// Return the samples collected for function \p F, create empty + /// FunctionSamples if it doesn't exist. + FunctionSamples *getOrCreateSamplesFor(const Function &F) { + std::string FGUID; + StringRef CanonName = FunctionSamples::getCanonicalFnName(F); + CanonName = getRepInFormat(CanonName, getFormat(), FGUID); + return &Profiles[CanonName]; + } + /// Return the samples collected for function \p F. virtual FunctionSamples *getSamplesFor(StringRef Fname) { if (Remapper) { diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h index cc951594c9e2..5814f69fdcab 100644 --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -80,7 +80,7 @@ protected: void computeSummary(const StringMap<FunctionSamples> &ProfileMap); /// Profile format. - SampleProfileFormat Format; + SampleProfileFormat Format = SPF_None; }; /// Sample-based profile writer (text format). @@ -227,7 +227,7 @@ private: // Save the start of SecLBRProfile so we can compute the offset to the // start of SecLBRProfile for each Function's Profile and will keep it // in FuncOffsetTable. - uint64_t SecLBRProfileStart; + uint64_t SecLBRProfileStart = 0; // FuncOffsetTable maps function name to its profile offset in SecLBRProfile // section. It is used to load function profile on demand. MapVector<StringRef, uint64_t> FuncOffsetTable; diff --git a/llvm/include/llvm/Remarks/Remark.h b/llvm/include/llvm/Remarks/Remark.h index 1243311fb8c5..6211db4a8e96 100644 --- a/llvm/include/llvm/Remarks/Remark.h +++ b/llvm/include/llvm/Remarks/Remark.h @@ -30,8 +30,8 @@ constexpr uint64_t CurrentRemarkVersion = 0; struct RemarkLocation { /// Absolute path of the source file corresponding to this remark. StringRef SourceFilePath; - unsigned SourceLine; - unsigned SourceColumn; + unsigned SourceLine = 0; + unsigned SourceColumn = 0; }; // Create wrappers for C Binding types (see CBindingWrapping.h). @@ -110,6 +110,21 @@ private: DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Remark, LLVMRemarkEntryRef) /// Comparison operators for Remark objects and dependent objects. + +template <typename T> +bool operator<(const Optional<T> &LHS, const Optional<T> &RHS) { + // Sorting based on optionals should result in all `None` entries to appear + // before the valid entries. For example, remarks with no debug location will + // appear first. + if (!LHS && !RHS) + return false; + if (!LHS && RHS) + return true; + if (LHS && !RHS) + return false; + return *LHS < *RHS; +} + inline bool operator==(const RemarkLocation &LHS, const RemarkLocation &RHS) { return LHS.SourceFilePath == RHS.SourceFilePath && LHS.SourceLine == RHS.SourceLine && @@ -120,6 +135,11 @@ inline bool operator!=(const RemarkLocation &LHS, const RemarkLocation &RHS) { return !(LHS == RHS); } +inline bool operator<(const RemarkLocation &LHS, const RemarkLocation &RHS) { + return std::make_tuple(LHS.SourceFilePath, LHS.SourceLine, LHS.SourceColumn) < + std::make_tuple(RHS.SourceFilePath, RHS.SourceLine, RHS.SourceColumn); +} + inline bool operator==(const Argument &LHS, const Argument &RHS) { return LHS.Key == RHS.Key && LHS.Val == RHS.Val && LHS.Loc == RHS.Loc; } @@ -128,6 +148,11 @@ inline bool operator!=(const Argument &LHS, const Argument &RHS) { return !(LHS == RHS); } +inline bool operator<(const Argument &LHS, const Argument &RHS) { + return std::make_tuple(LHS.Key, LHS.Val, LHS.Loc) < + std::make_tuple(RHS.Key, RHS.Val, RHS.Loc); +} + inline bool operator==(const Remark &LHS, const Remark &RHS) { return LHS.RemarkType == RHS.RemarkType && LHS.PassName == RHS.PassName && LHS.RemarkName == RHS.RemarkName && @@ -139,6 +164,13 @@ inline bool operator!=(const Remark &LHS, const Remark &RHS) { return !(LHS == RHS); } +inline bool operator<(const Remark &LHS, const Remark &RHS) { + return std::make_tuple(LHS.RemarkType, LHS.PassName, LHS.RemarkName, + LHS.FunctionName, LHS.Loc, LHS.Hotness, LHS.Args) < + std::make_tuple(RHS.RemarkType, RHS.PassName, RHS.RemarkName, + RHS.FunctionName, RHS.Loc, RHS.Hotness, RHS.Args); +} + } // end namespace remarks } // end namespace llvm diff --git a/llvm/include/llvm/Remarks/RemarkFormat.h b/llvm/include/llvm/Remarks/RemarkFormat.h index 6dd32b226099..a432c5adf59e 100644 --- a/llvm/include/llvm/Remarks/RemarkFormat.h +++ b/llvm/include/llvm/Remarks/RemarkFormat.h @@ -27,6 +27,9 @@ enum class Format { Unknown, YAML, YAMLStrTab, Bitstream }; /// Parse and validate a string for the remark format. Expected<Format> parseFormat(StringRef FormatStr); +/// Parse and validate a magic number to a remark format. +Expected<Format> magicToFormat(StringRef Magic); + } // end namespace remarks } // end namespace llvm diff --git a/llvm/include/llvm/Remarks/RemarkLinker.h b/llvm/include/llvm/Remarks/RemarkLinker.h new file mode 100644 index 000000000000..c82c73d8c94f --- /dev/null +++ b/llvm/include/llvm/Remarks/RemarkLinker.h @@ -0,0 +1,100 @@ +//===-- llvm/Remarks/RemarkLinker.h -----------------------------*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an interface to link together multiple remark files. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_REMARK_LINKER_H +#define LLVM_REMARKS_REMARK_LINKER_H + +#include "llvm/Object/ObjectFile.h" +#include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkFormat.h" +#include "llvm/Remarks/RemarkStringTable.h" +#include "llvm/Support/Error.h" +#include <memory> +#include <set> + +namespace llvm { +namespace remarks { + +struct RemarkLinker { +private: + /// Compare through the pointers. + struct RemarkPtrCompare { + bool operator()(const std::unique_ptr<Remark> &LHS, + const std::unique_ptr<Remark> &RHS) const { + assert(LHS && RHS && "Invalid pointers to compare."); + return *LHS < *RHS; + }; + }; + + /// The main string table for the remarks. + /// Note: all remarks should use the strings from this string table to avoid + /// dangling references. + StringTable StrTab; + + /// A set holding unique remarks. + /// FIXME: std::set is probably not the most appropriate data structure here. + /// Due to the limitation of having a move-only key, there isn't another + /// obvious choice for now. + std::set<std::unique_ptr<Remark>, RemarkPtrCompare> Remarks; + + /// A path to append before the external file path found in remark metadata. + Optional<std::string> PrependPath; + + /// Keep this remark. If it's already in the set, discard it. + Remark &keep(std::unique_ptr<Remark> Remark); + +public: + /// Set a path to prepend to the external file path. + void setExternalFilePrependPath(StringRef PrependPath); + + /// Link the remarks found in \p Buffer. + /// If \p RemarkFormat is not provided, try to deduce it from the metadata in + /// \p Buffer. + /// \p Buffer can be either a standalone remark container or just + /// metadata. This takes care of uniquing and merging the remarks. + Error link(StringRef Buffer, Optional<Format> RemarkFormat = None); + + /// Link the remarks found in \p Obj by looking for the right section and + /// calling the method above. + Error link(const object::ObjectFile &Obj, + Optional<Format> RemarkFormat = None); + + /// Serialize the linked remarks to the stream \p OS, using the format \p + /// RemarkFormat. + /// This clears internal state such as the string table. + /// Note: this implies that the serialization mode is standalone. + Error serialize(raw_ostream &OS, Format RemarksFormat) const; + + /// Check whether there are any remarks linked. + bool empty() const { return Remarks.empty(); } + + /// Return a collection of the linked unique remarks to iterate on. + /// Ex: + /// for (const Remark &R : RL.remarks() { [...] } + using iterator = + pointee_iterator<std::set<std::unique_ptr<Remark>>::iterator>; + + iterator_range<iterator> remarks() const { + return {Remarks.begin(), Remarks.end()}; + } +}; + +/// Returns a buffer with the contents of the remarks section depending on the +/// format of the file. If the section doesn't exist, this returns an empty +/// optional. +Expected<Optional<StringRef>> +getRemarksSectionContents(const object::ObjectFile &Obj); + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_LINKER_H */ diff --git a/llvm/include/llvm/Support/AArch64TargetParser.def b/llvm/include/llvm/Support/AArch64TargetParser.def index 15737265dfc3..6b25ef2ca435 100644 --- a/llvm/include/llvm/Support/AArch64TargetParser.def +++ b/llvm/include/llvm/Support/AArch64TargetParser.def @@ -120,10 +120,24 @@ AARCH64_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, AArch64::AEK_SSBS)) AARCH64_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_NONE)) -AARCH64_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) -AARCH64_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) +AARCH64_CPU_NAME("apple-a7", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_NONE)) +AARCH64_CPU_NAME("apple-a8", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_NONE)) +AARCH64_CPU_NAME("apple-a9", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_NONE)) +AARCH64_CPU_NAME("apple-a10", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_CRC | AArch64::AEK_RDM)) +AARCH64_CPU_NAME("apple-a11", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_NONE)) +AARCH64_CPU_NAME("apple-a12", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16)) +AARCH64_CPU_NAME("apple-a13", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16 | AArch64::AEK_FP16FML)) +AARCH64_CPU_NAME("apple-s4", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16)) +AARCH64_CPU_NAME("apple-s5", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16)) AARCH64_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_CRC)) AARCH64_CPU_NAME("exynos-m4", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, diff --git a/llvm/include/llvm/Support/AArch64TargetParser.h b/llvm/include/llvm/Support/AArch64TargetParser.h index 94f341c83260..fbe08945a038 100644 --- a/llvm/include/llvm/Support/AArch64TargetParser.h +++ b/llvm/include/llvm/Support/AArch64TargetParser.h @@ -123,6 +123,15 @@ void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values); bool isX18ReservedByDefault(const Triple &TT); +struct ParsedBranchProtection { + StringRef Scope; + StringRef Key; + bool BranchTargetEnforcement; +}; + +bool parseBranchProtection(StringRef Spec, ParsedBranchProtection &PBP, + StringRef &Err); + } // namespace AArch64 } // namespace llvm diff --git a/llvm/include/llvm/Support/AMDGPUMetadata.h b/llvm/include/llvm/Support/AMDGPUMetadata.h index f7f1ec40dde9..eeef4e699c3e 100644 --- a/llvm/include/llvm/Support/AMDGPUMetadata.h +++ b/llvm/include/llvm/Support/AMDGPUMetadata.h @@ -75,6 +75,7 @@ enum class ValueKind : uint8_t { HiddenDefaultQueue = 12, HiddenCompletionAction = 13, HiddenMultiGridSyncArg = 14, + HiddenHostcallBuffer = 15, Unknown = 0xff }; diff --git a/llvm/include/llvm/Support/ARMTargetParser.def b/llvm/include/llvm/Support/ARMTargetParser.def index 3e77e20762c1..7f03d9a1320a 100644 --- a/llvm/include/llvm/Support/ARMTargetParser.def +++ b/llvm/include/llvm/Support/ARMTargetParser.def @@ -277,8 +277,6 @@ ARM_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) ARM_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) -ARM_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m4", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h index 106b90c35bf5..670335ffecbc 100644 --- a/llvm/include/llvm/Support/Allocator.h +++ b/llvm/include/llvm/Support/Allocator.h @@ -269,7 +269,7 @@ public: inline LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * Allocate(size_t Size, size_t Alignment) { - assert(Alignment > 0 && "0-byte alignnment is not allowed. Use 1 instead."); + assert(Alignment > 0 && "0-byte alignment is not allowed. Use 1 instead."); return Allocate(Size, Align(Alignment)); } diff --git a/llvm/include/llvm/Support/Automaton.h b/llvm/include/llvm/Support/Automaton.h index 7c13a698e492..c2b921311a8c 100644 --- a/llvm/include/llvm/Support/Automaton.h +++ b/llvm/include/llvm/Support/Automaton.h @@ -117,6 +117,10 @@ public: reset(); } + ArrayRef<NfaStatePair> getTransitionInfo() const { + return TransitionInfo; + } + void reset() { Paths.clear(); Heads.clear(); @@ -198,7 +202,13 @@ public: M->emplace(std::make_pair(I.FromDfaState, I.Action), std::make_pair(I.ToDfaState, I.InfoIdx)); } - Automaton(const Automaton &) = default; + Automaton(const Automaton &Other) + : M(Other.M), State(Other.State), Transcribe(Other.Transcribe) { + // Transcriber is not thread-safe, so create a new instance on copy. + if (Other.Transcriber) + Transcriber = std::make_shared<internal::NfaTranscriber>( + Other.Transcriber->getTransitionInfo()); + } /// Reset the automaton to its initial state. void reset() { diff --git a/llvm/include/llvm/Support/BinaryStreamArray.h b/llvm/include/llvm/Support/BinaryStreamArray.h index 67ba2e4189be..1634983d26ce 100644 --- a/llvm/include/llvm/Support/BinaryStreamArray.h +++ b/llvm/include/llvm/Support/BinaryStreamArray.h @@ -133,9 +133,9 @@ public: Extractor &getExtractor() { return E; } BinaryStreamRef getUnderlyingStream() const { return Stream; } - void setUnderlyingStream(BinaryStreamRef S, uint32_t Skew = 0) { - Stream = S; - this->Skew = Skew; + void setUnderlyingStream(BinaryStreamRef NewStream, uint32_t NewSkew = 0) { + Stream = NewStream; + Skew = NewSkew; } void drop_front() { Skew += begin()->length(); } @@ -143,7 +143,7 @@ public: private: BinaryStreamRef Stream; Extractor E; - uint32_t Skew; + uint32_t Skew = 0; }; template <typename ValueType, typename Extractor> @@ -274,6 +274,7 @@ public: return !(*this == Other); } + FixedStreamArray(const FixedStreamArray &) = default; FixedStreamArray &operator=(const FixedStreamArray &) = default; const T &operator[](uint32_t Index) const { @@ -323,6 +324,8 @@ public: FixedStreamArrayIterator(const FixedStreamArray<T> &Array, uint32_t Index) : Array(Array), Index(Index) {} + FixedStreamArrayIterator<T>(const FixedStreamArrayIterator<T> &Other) + : Array(Other.Array), Index(Other.Index) {} FixedStreamArrayIterator<T> & operator=(const FixedStreamArrayIterator<T> &Other) { Array = Other.Array; diff --git a/llvm/include/llvm/Support/BinaryStreamReader.h b/llvm/include/llvm/Support/BinaryStreamReader.h index 9e16ce227ff8..b7d61c02667b 100644 --- a/llvm/include/llvm/Support/BinaryStreamReader.h +++ b/llvm/include/llvm/Support/BinaryStreamReader.h @@ -148,14 +148,14 @@ public: /// returns an appropriate error code. Error readStreamRef(BinaryStreamRef &Ref, uint32_t Length); - /// Read \p Length bytes from the underlying stream into \p Stream. This is + /// Read \p Length bytes from the underlying stream into \p Ref. This is /// equivalent to calling getUnderlyingStream().slice(Offset, Length). /// Updates the stream's offset to point after the newly read object. Never /// causes a copy. /// /// \returns a success error code if the data was successfully read, otherwise /// returns an appropriate error code. - Error readSubstream(BinarySubstreamRef &Stream, uint32_t Size); + Error readSubstream(BinarySubstreamRef &Ref, uint32_t Length); /// Get a pointer to an object of type T from the underlying stream, as if by /// memcpy, and store the result into \p Dest. It is up to the caller to diff --git a/llvm/include/llvm/Support/BinaryStreamRef.h b/llvm/include/llvm/Support/BinaryStreamRef.h index 7427b8da5b43..5375d6a3a761 100644 --- a/llvm/include/llvm/Support/BinaryStreamRef.h +++ b/llvm/include/llvm/Support/BinaryStreamRef.h @@ -198,7 +198,7 @@ public: }; struct BinarySubstreamRef { - uint32_t Offset; // Offset in the parent stream + uint32_t Offset = 0; // Offset in the parent stream BinaryStreamRef StreamData; // Stream Data BinarySubstreamRef slice(uint32_t Off, uint32_t Size) const { @@ -211,8 +211,8 @@ struct BinarySubstreamRef { BinarySubstreamRef keep_front(uint32_t N) const { return slice(0, N); } std::pair<BinarySubstreamRef, BinarySubstreamRef> - split(uint32_t Offset) const { - return std::make_pair(keep_front(Offset), drop_front(Offset)); + split(uint32_t Off) const { + return std::make_pair(keep_front(Off), drop_front(Off)); } uint32_t size() const { return StreamData.getLength(); } diff --git a/llvm/include/llvm/Support/CodeGen.h b/llvm/include/llvm/Support/CodeGen.h index a3f423e558cf..e2aa2b68b9d2 100644 --- a/llvm/include/llvm/Support/CodeGen.h +++ b/llvm/include/llvm/Support/CodeGen.h @@ -57,6 +57,15 @@ namespace llvm { }; } + /// These enums are meant to be passed into addPassesToEmitFile to indicate + /// what type of file to emit, and returned by it to indicate what type of + /// file could actually be made. + enum CodeGenFileType { + CGFT_AssemblyFile, + CGFT_ObjectFile, + CGFT_Null // Do not emit any output. + }; + // Specify effect of frame pointer elimination optimization. namespace FramePointer { enum FP {All, NonLeaf, None}; diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h index 63784463e171..05374e34aa7d 100644 --- a/llvm/include/llvm/Support/CommandLine.h +++ b/llvm/include/llvm/Support/CommandLine.h @@ -20,6 +20,8 @@ #define LLVM_SUPPORT_COMMANDLINE_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" @@ -29,6 +31,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" #include <cassert> #include <climits> @@ -468,6 +471,43 @@ struct sub { template <class Opt> void apply(Opt &O) const { O.addSubCommand(Sub); } }; +// Specify a callback function to be called when an option is seen. +// Can be used to set other options automatically. +template <typename R, typename Ty> struct cb { + std::function<R(Ty)> CB; + + cb(std::function<R(Ty)> CB) : CB(CB) {} + + template <typename Opt> void apply(Opt &O) const { O.setCallback(CB); } +}; + +namespace detail { +template <typename F> +struct callback_traits : public callback_traits<decltype(&F::operator())> {}; + +template <typename R, typename C, typename... Args> +struct callback_traits<R (C::*)(Args...) const> { + using result_type = R; + using arg_type = typename std::tuple_element<0, std::tuple<Args...>>::type; + static_assert(sizeof...(Args) == 1, "callback function must have one and only one parameter"); + static_assert(std::is_same<result_type, void>::value, + "callback return type must be void"); + static_assert( + std::is_lvalue_reference<arg_type>::value && + std::is_const<typename std::remove_reference<arg_type>::type>::value, + "callback arg_type must be a const lvalue reference"); +}; +} // namespace detail + +template <typename F> +cb<typename detail::callback_traits<F>::result_type, + typename detail::callback_traits<F>::arg_type> +callback(F CB) { + using result_type = typename detail::callback_traits<F>::result_type; + using arg_type = typename detail::callback_traits<F>::arg_type; + return cb<result_type, arg_type>(CB); +} + //===----------------------------------------------------------------------===// // OptionValue class @@ -952,6 +992,50 @@ public: extern template class basic_parser<int>; //-------------------------------------------------- +// parser<long> +// +template <> class parser<long> final : public basic_parser<long> { +public: + parser(Option &O) : basic_parser(O) {} + + // parse - Return true on error. + bool parse(Option &O, StringRef ArgName, StringRef Arg, long &Val); + + // getValueName - Overload in subclass to provide a better default value. + StringRef getValueName() const override { return "long"; } + + void printOptionDiff(const Option &O, long V, OptVal Default, + size_t GlobalWidth) const; + + // An out-of-line virtual method to provide a 'home' for this class. + void anchor() override; +}; + +extern template class basic_parser<long>; + +//-------------------------------------------------- +// parser<long long> +// +template <> class parser<long long> : public basic_parser<long long> { +public: + parser(Option &O) : basic_parser(O) {} + + // parse - Return true on error. + bool parse(Option &O, StringRef ArgName, StringRef Arg, long long &Val); + + // getValueName - Overload in subclass to provide a better default value. + StringRef getValueName() const override { return "long"; } + + void printOptionDiff(const Option &O, long long V, OptVal Default, + size_t GlobalWidth) const; + + // An out-of-line virtual method to provide a 'home' for this class. + void anchor() override; +}; + +extern template class basic_parser<long long>; + +//-------------------------------------------------- // parser<unsigned> // template <> class parser<unsigned> : public basic_parser<unsigned> { @@ -1341,6 +1425,7 @@ class opt : public Option, return true; // Parse error! this->setValue(Val); this->setPosition(pos); + Callback(Val); return false; } @@ -1399,6 +1484,7 @@ public: template <class T> DataType &operator=(const T &Val) { this->setValue(Val); + Callback(Val); return this->getValue(); } @@ -1408,6 +1494,14 @@ public: apply(this, Ms...); done(); } + + void setCallback( + std::function<void(const typename ParserClass::parser_data_type &)> CB) { + Callback = CB; + } + + std::function<void(const typename ParserClass::parser_data_type &)> Callback = + [](const typename ParserClass::parser_data_type &) {}; }; extern template class opt<unsigned>; @@ -1547,6 +1641,7 @@ class list : public Option, public list_storage<DataType, StorageClass> { list_storage<DataType, StorageClass>::addValue(Val); setPosition(pos); Positions.push_back(pos); + Callback(Val); return false; } @@ -1593,6 +1688,14 @@ public: apply(this, Ms...); done(); } + + void setCallback( + std::function<void(const typename ParserClass::parser_data_type &)> CB) { + Callback = CB; + } + + std::function<void(const typename ParserClass::parser_data_type &)> Callback = + [](const typename ParserClass::parser_data_type &) {}; }; // multi_val - Modifier to set the number of additional values. @@ -1693,6 +1796,7 @@ class bits : public Option, public bits_storage<DataType, Storage> { this->addValue(Val); setPosition(pos); Positions.push_back(pos); + Callback(Val); return false; } @@ -1734,6 +1838,14 @@ public: apply(this, Ms...); done(); } + + void setCallback( + std::function<void(const typename ParserClass::parser_data_type &)> CB) { + Callback = CB; + } + + std::function<void(const typename ParserClass::parser_data_type &)> Callback = + [](const typename ParserClass::parser_data_type &) {}; }; //===----------------------------------------------------------------------===// @@ -1831,7 +1943,7 @@ void PrintHelpMessage(bool Hidden = false, bool Categorized = false); // /// Use this to get a StringMap to all registered named options -/// (e.g. -help). Note \p Map Should be an empty StringMap. +/// (e.g. -help). /// /// \return A reference to the StringMap used by the cl APIs to parse options. /// @@ -1964,10 +2076,16 @@ bool readConfigFile(StringRef CfgFileName, StringSaver &Saver, /// with nullptrs in the Argv vector. /// \param [in] RelativeNames true if names of nested response files must be /// resolved relative to including file. +/// \param [in] FS File system used for all file access when running the tool. +/// \param [in] CurrentDir Path used to resolve relative rsp files. If set to +/// None, process' cwd is used instead. /// \return true if all @files were expanded successfully or there were none. -bool ExpandResponseFiles(StringSaver &Saver, TokenizerCallback Tokenizer, - SmallVectorImpl<const char *> &Argv, - bool MarkEOLs = false, bool RelativeNames = false); +bool ExpandResponseFiles( + StringSaver &Saver, TokenizerCallback Tokenizer, + SmallVectorImpl<const char *> &Argv, bool MarkEOLs = false, + bool RelativeNames = false, + llvm::vfs::FileSystem &FS = *llvm::vfs::getRealFileSystem(), + llvm::Optional<llvm::StringRef> CurrentDir = llvm::None); /// Mark all options not part of this category as cl::ReallyHidden. /// diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h index cb7e57d4cd21..6f6f65cad6f5 100644 --- a/llvm/include/llvm/Support/Compiler.h +++ b/llvm/include/llvm/Support/Compiler.h @@ -117,11 +117,17 @@ /// not accessible from outside it. Can also be used to mark variables and /// functions, making them private to any shared library they are linked into. /// On PE/COFF targets, library visibility is the default, so this isn't needed. +/// +/// LLVM_EXTERNAL_VISIBILITY - classes, functions, and variables marked with +/// this attribute will be made public and visible outside of any shared library +/// they are linked in to. #if (__has_attribute(visibility) || LLVM_GNUC_PREREQ(4, 0, 0)) && \ !defined(__MINGW32__) && !defined(__CYGWIN__) && !defined(_WIN32) #define LLVM_LIBRARY_VISIBILITY __attribute__ ((visibility("hidden"))) +#define LLVM_EXTERNAL_VISIBILITY __attribute__ ((visibility("default"))) #else #define LLVM_LIBRARY_VISIBILITY +#define LLVM_EXTERNAL_VISIBILITY #endif #if defined(__GNUC__) @@ -406,10 +412,12 @@ #if __has_feature(memory_sanitizer) # define LLVM_MEMORY_SANITIZER_BUILD 1 # include <sanitizer/msan_interface.h> +# define LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE __attribute__((no_sanitize_memory)) #else # define LLVM_MEMORY_SANITIZER_BUILD 0 # define __msan_allocated_memory(p, size) # define __msan_unpoison(p, size) +# define LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE #endif /// \macro LLVM_ADDRESS_SANITIZER_BUILD @@ -504,19 +512,15 @@ void AnnotateIgnoreWritesEnd(const char *file, int line); /// extern globals, and static globals. /// /// This is essentially an extremely restricted analog to C++11's thread_local -/// support, and uses that when available. However, it falls back on -/// platform-specific or vendor-provided extensions when necessary. These -/// extensions don't support many of the C++11 thread_local's features. You -/// should only use this for PODs that you can statically initialize to -/// some constant value. In almost all circumstances this is most appropriate -/// for use with a pointer, integer, or small aggregation of pointers and -/// integers. +/// support. It uses thread_local if available, falling back on gcc __thread +/// if not. __thread doesn't support many of the C++11 thread_local's +/// features. You should only use this for PODs that you can statically +/// initialize to some constant value. In almost all circumstances this is most +/// appropriate for use with a pointer, integer, or small aggregation of +/// pointers and integers. #if LLVM_ENABLE_THREADS -#if __has_feature(cxx_thread_local) +#if __has_feature(cxx_thread_local) || defined(_MSC_VER) #define LLVM_THREAD_LOCAL thread_local -#elif defined(_MSC_VER) -// MSVC supports this with a __declspec. -#define LLVM_THREAD_LOCAL __declspec(thread) #else // Clang, GCC, and other compatible compilers used __thread prior to C++11 and // we only need the restricted functionality that provides. diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h index feb449e2899c..9522c4742244 100644 --- a/llvm/include/llvm/Support/CrashRecoveryContext.h +++ b/llvm/include/llvm/Support/CrashRecoveryContext.h @@ -100,6 +100,14 @@ public: /// Explicitly trigger a crash recovery in the current process, and /// return failure from RunSafely(). This function does not return. void HandleCrash(); + + /// In case of a crash, this is the crash identifier. + int RetCode = 0; + + /// Selects whether handling of failures should be done in the same way as + /// for regular crashes. When this is active, a crash would print the + /// callstack, clean-up any temporary files and create a coredump/minidump. + bool DumpStackAndCleanupOnFailure = false; }; /// Abstract base class of cleanup handlers. @@ -111,12 +119,12 @@ public: /// a crash recovery context. class CrashRecoveryContextCleanup { protected: - CrashRecoveryContext *context; + CrashRecoveryContext *context = nullptr; CrashRecoveryContextCleanup(CrashRecoveryContext *context) - : context(context), cleanupFired(false) {} + : context(context) {} public: - bool cleanupFired; + bool cleanupFired = false; virtual ~CrashRecoveryContextCleanup(); virtual void recoverResources() = 0; @@ -127,7 +135,7 @@ public: private: friend class CrashRecoveryContext; - CrashRecoveryContextCleanup *prev, *next; + CrashRecoveryContextCleanup *prev = nullptr, *next = nullptr; }; /// Base class of cleanup handler that controls recovery of resources of the diff --git a/llvm/include/llvm/Support/DataExtractor.h b/llvm/include/llvm/Support/DataExtractor.h index f590a1e104fb..0be478811b22 100644 --- a/llvm/include/llvm/Support/DataExtractor.h +++ b/llvm/include/llvm/Support/DataExtractor.h @@ -534,14 +534,14 @@ public: /// error state of the cursor. The only way both eof and error states can be /// true is if one attempts a read while the cursor is at the very end of the /// data buffer. - bool eof(const Cursor &C) const { return Data.size() == C.Offset; } + bool eof(const Cursor &C) const { return size() == C.Offset; } /// Test the validity of \a offset. /// /// @return /// \b true if \a offset is a valid offset into the data in this /// object, \b false otherwise. - bool isValidOffset(uint64_t offset) const { return Data.size() > offset; } + bool isValidOffset(uint64_t offset) const { return size() > offset; } /// Test the availability of \a length bytes of data from \a offset. /// @@ -563,6 +563,9 @@ public: return isValidOffsetForDataOfSize(offset, AddressSize); } + /// Return the number of bytes in the underlying buffer. + size_t size() const { return Data.size(); } + protected: // Make it possible for subclasses to access these fields without making them // public. diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h index 350877a219bf..44676338808b 100644 --- a/llvm/include/llvm/Support/Error.h +++ b/llvm/include/llvm/Support/Error.h @@ -155,10 +155,10 @@ private: /// they're moved-assigned or constructed from Success values that have already /// been checked. This enforces checking through all levels of the call stack. class LLVM_NODISCARD Error { - // Both ErrorList and FileError need to be able to yank ErrorInfoBase - // pointers out of this class to add to the error list. + // ErrorList needs to be able to yank ErrorInfoBase pointers out of Errors + // to add to the error list. It can't rely on handleErrors for this, since + // handleErrors does not support ErrorList handlers. friend class ErrorList; - friend class FileError; // handleErrors needs to be able to set the Checked flag. template <typename... HandlerTs> @@ -1232,6 +1232,8 @@ public: Err->log(OS); } + StringRef getFileName() { return FileName; } + Error takeError() { return Error(std::move(Err)); } std::error_code convertToErrorCode() const override; @@ -1251,8 +1253,14 @@ private: } static Error build(const Twine &F, Optional<size_t> Line, Error E) { + std::unique_ptr<ErrorInfoBase> Payload; + handleAllErrors(std::move(E), + [&](std::unique_ptr<ErrorInfoBase> EIB) -> Error { + Payload = std::move(EIB); + return Error::success(); + }); return Error( - std::unique_ptr<FileError>(new FileError(F, Line, E.takePayload()))); + std::unique_ptr<FileError>(new FileError(F, Line, std::move(Payload)))); } std::string FileName; diff --git a/llvm/include/llvm/Support/FileCheck.h b/llvm/include/llvm/Support/FileCheck.h index 2547449246a8..429e36cfcbb5 100644 --- a/llvm/include/llvm/Support/FileCheck.h +++ b/llvm/include/llvm/Support/FileCheck.h @@ -71,6 +71,7 @@ class FileCheckType { public: FileCheckType(FileCheckKind Kind = CheckNone) : Kind(Kind), Count(1) {} FileCheckType(const FileCheckType &) = default; + FileCheckType &operator=(const FileCheckType &) = default; operator FileCheckKind() const { return Kind; } diff --git a/llvm/include/llvm/Support/FileCollector.h b/llvm/include/llvm/Support/FileCollector.h index 19429bd3e9b4..079fe3efab9d 100644 --- a/llvm/include/llvm/Support/FileCollector.h +++ b/llvm/include/llvm/Support/FileCollector.h @@ -46,7 +46,11 @@ public: private: void addFileImpl(StringRef SrcPath); - bool markAsSeen(StringRef Path) { return Seen.insert(Path).second; } + bool markAsSeen(StringRef Path) { + if (Path.empty()) + return false; + return Seen.insert(Path).second; + } bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result); diff --git a/llvm/include/llvm/Support/FileOutputBuffer.h b/llvm/include/llvm/Support/FileOutputBuffer.h index 999f551ebf2d..bdc1425d4361 100644 --- a/llvm/include/llvm/Support/FileOutputBuffer.h +++ b/llvm/include/llvm/Support/FileOutputBuffer.h @@ -32,6 +32,10 @@ public: enum { /// set the 'x' bit on the resulting file F_executable = 1, + + /// Don't use mmap and instead write an in-memory buffer to a file when this + /// buffer is closed. + F_no_mmap = 2, }; /// Factory method to create an OutputBuffer object which manages a read/write diff --git a/llvm/include/llvm/Support/FormatVariadic.h b/llvm/include/llvm/Support/FormatVariadic.h index 5bbda9dd626e..86a9d30cc138 100644 --- a/llvm/include/llvm/Support/FormatVariadic.h +++ b/llvm/include/llvm/Support/FormatVariadic.h @@ -57,7 +57,7 @@ struct ReplacementItem { size_t Index = 0; size_t Align = 0; AlignStyle Where = AlignStyle::Right; - char Pad; + char Pad = 0; StringRef Options; }; diff --git a/llvm/include/llvm/Support/GenericDomTree.h b/llvm/include/llvm/Support/GenericDomTree.h index 9169379f746d..2545a075062a 100644 --- a/llvm/include/llvm/Support/GenericDomTree.h +++ b/llvm/include/llvm/Support/GenericDomTree.h @@ -778,13 +778,13 @@ protected: NodeRef NewBBSucc = *GraphT::child_begin(NewBB); std::vector<NodeRef> PredBlocks; - for (const auto &Pred : children<Inverse<N>>(NewBB)) + for (auto Pred : children<Inverse<N>>(NewBB)) PredBlocks.push_back(Pred); assert(!PredBlocks.empty() && "No predblocks?"); bool NewBBDominatesNewBBSucc = true; - for (const auto &Pred : children<Inverse<N>>(NewBBSucc)) { + for (auto Pred : children<Inverse<N>>(NewBBSucc)) { if (Pred != NewBB && !dominates(NewBBSucc, Pred) && isReachableFromEntry(Pred)) { NewBBDominatesNewBBSucc = false; diff --git a/llvm/include/llvm/Support/InitLLVM.h b/llvm/include/llvm/Support/InitLLVM.h index 8069859a3e0b..3be8d6b6d2e0 100644 --- a/llvm/include/llvm/Support/InitLLVM.h +++ b/llvm/include/llvm/Support/InitLLVM.h @@ -17,7 +17,8 @@ // the following one-time initializations: // // 1. Setting up a signal handler so that pretty stack trace is printed out -// if a process crashes. +// if a process crashes. A signal handler that exits when a failed write to +// a pipe occurs may optionally be installed: this is on-by-default. // // 2. Set up the global new-handler which is called when a memory allocation // attempt fails. @@ -32,9 +33,11 @@ namespace llvm { class InitLLVM { public: - InitLLVM(int &Argc, const char **&Argv); - InitLLVM(int &Argc, char **&Argv) - : InitLLVM(Argc, const_cast<const char **&>(Argv)) {} + InitLLVM(int &Argc, const char **&Argv, + bool InstallPipeSignalExitHandler = true); + InitLLVM(int &Argc, char **&Argv, bool InstallPipeSignalExitHandler = true) + : InitLLVM(Argc, const_cast<const char **&>(Argv), + InstallPipeSignalExitHandler) {} ~InitLLVM(); diff --git a/llvm/include/llvm/Support/JSON.h b/llvm/include/llvm/Support/JSON.h index 0ca41097dddd..2c63468c401a 100644 --- a/llvm/include/llvm/Support/JSON.h +++ b/llvm/include/llvm/Support/JSON.h @@ -122,6 +122,8 @@ public: std::pair<iterator, bool> try_emplace(ObjectKey &&K, Ts &&... Args) { return M.try_emplace(std::move(K), std::forward<Ts>(Args)...); } + bool erase(StringRef K); + void erase(iterator I) { M.erase(I); } iterator find(StringRef K) { return M.find_as(K); } const_iterator find(StringRef K) const { return M.find_as(K); } @@ -555,6 +557,9 @@ inline Object::Object(std::initializer_list<KV> Properties) { inline std::pair<Object::iterator, bool> Object::insert(KV E) { return try_emplace(std::move(E.K), std::move(E.V)); } +inline bool Object::erase(StringRef K) { + return M.erase(ObjectKey(K)); +} // Standard deserializers are provided for primitive types. // See comments on Value. @@ -709,7 +714,7 @@ public: /// J.attribute("timestamp", int64_t(E.Time)); /// J.attributeArray("participants", [&] { /// for (const Participant &P : E.Participants) -/// J.string(P.toString()); +/// J.value(P.toString()); /// }); /// }); /// }); diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h index 07fd94e29a1f..ff25b6fc572c 100644 --- a/llvm/include/llvm/Support/KnownBits.h +++ b/llvm/include/llvm/Support/KnownBits.h @@ -97,6 +97,9 @@ public: /// Returns true if this value is known to be non-negative. bool isNonNegative() const { return Zero.isSignBitSet(); } + /// Returns true if this value is known to be positive. + bool isStrictlyPositive() const { return Zero.isSignBitSet() && !One.isNullValue(); } + /// Make this value negative. void makeNegative() { One.setSignBit(); @@ -107,6 +110,18 @@ public: Zero.setSignBit(); } + /// Return the minimal value possible given these KnownBits. + APInt getMinValue() const { + // Assume that all bits that aren't known-ones are zeros. + return One; + } + + /// Return the maximal value possible given these KnownBits. + APInt getMaxValue() const { + // Assume that all bits that aren't known-zeros are ones. + return ~Zero; + } + /// Truncate the underlying known Zero and One bits. This is equivalent /// to truncating the value we're tracking. KnownBits trunc(unsigned BitWidth) const { diff --git a/llvm/include/llvm/Support/LineIterator.h b/llvm/include/llvm/Support/LineIterator.h index c9f10ca975ae..2a1e47bfe5b7 100644 --- a/llvm/include/llvm/Support/LineIterator.h +++ b/llvm/include/llvm/Support/LineIterator.h @@ -30,16 +30,16 @@ class MemoryBuffer; /// Note that this iterator requires the buffer to be nul terminated. class line_iterator : public std::iterator<std::forward_iterator_tag, StringRef> { - const MemoryBuffer *Buffer; - char CommentMarker; - bool SkipBlanks; + const MemoryBuffer *Buffer = nullptr; + char CommentMarker = '\0'; + bool SkipBlanks = true; - unsigned LineNumber; + unsigned LineNumber = 1; StringRef CurrentLine; public: /// Default construct an "end" iterator. - line_iterator() : Buffer(nullptr) {} + line_iterator() = default; /// Construct a new iterator around some memory buffer. explicit line_iterator(const MemoryBuffer &Buffer, bool SkipBlanks = true, diff --git a/llvm/include/llvm/Support/LockFileManager.h b/llvm/include/llvm/Support/LockFileManager.h index 57e4fbd84cd9..2efeca3b6200 100644 --- a/llvm/include/llvm/Support/LockFileManager.h +++ b/llvm/include/llvm/Support/LockFileManager.h @@ -77,7 +77,9 @@ public: operator LockFileState() const { return getState(); } /// For a shared lock, wait until the owner releases the lock. - WaitForUnlockResult waitForUnlock(); + /// Total timeout for the file to appear is ~1.5 minutes. + /// \param MaxSeconds the maximum wait time per iteration in seconds. + WaitForUnlockResult waitForUnlock(const unsigned MaxSeconds = 40); /// Remove the lock file. This may delete a different lock file than /// the one previously read if there is a race. diff --git a/llvm/include/llvm/Support/LowLevelTypeImpl.h b/llvm/include/llvm/Support/LowLevelTypeImpl.h index 0e02b6e7d750..6ef7c298bc28 100644 --- a/llvm/include/llvm/Support/LowLevelTypeImpl.h +++ b/llvm/include/llvm/Support/LowLevelTypeImpl.h @@ -137,6 +137,8 @@ public: : LLT::scalar(NewEltSize); } + bool isByteSized() const { return (getSizeInBits() & 7) == 0; } + unsigned getScalarSizeInBits() const { assert(RawData != 0 && "Invalid Type"); if (!IsVector) { diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h index 7f9f0b85c55e..26b45a602763 100644 --- a/llvm/include/llvm/Support/MachineValueType.h +++ b/llvm/include/llvm/Support/MachineValueType.h @@ -671,7 +671,12 @@ namespace llvm { return { getVectorNumElements(), isScalableVector() }; } - unsigned getSizeInBits() const { + /// Returns the size of the specified MVT in bits. + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getSizeInBits() const { switch (SimpleTy) { default: llvm_unreachable("getSizeInBits called on extended MVT."); @@ -691,25 +696,25 @@ namespace llvm { case Metadata: llvm_unreachable("Value type is metadata."); case i1: - case v1i1: - case nxv1i1: return 1; - case v2i1: - case nxv2i1: return 2; - case v4i1: - case nxv4i1: return 4; + case v1i1: return TypeSize::Fixed(1); + case nxv1i1: return TypeSize::Scalable(1); + case v2i1: return TypeSize::Fixed(2); + case nxv2i1: return TypeSize::Scalable(2); + case v4i1: return TypeSize::Fixed(4); + case nxv4i1: return TypeSize::Scalable(4); case i8 : case v1i8: - case v8i1: + case v8i1: return TypeSize::Fixed(8); case nxv1i8: - case nxv8i1: return 8; + case nxv8i1: return TypeSize::Scalable(8); case i16 : case f16: case v16i1: case v2i8: - case v1i16: + case v1i16: return TypeSize::Fixed(16); case nxv16i1: case nxv2i8: - case nxv1i16: return 16; + case nxv1i16: return TypeSize::Scalable(16); case f32 : case i32 : case v32i1: @@ -717,15 +722,15 @@ namespace llvm { case v2i16: case v2f16: case v1f32: - case v1i32: + case v1i32: return TypeSize::Fixed(32); case nxv32i1: case nxv4i8: case nxv2i16: case nxv1i32: case nxv2f16: - case nxv1f32: return 32; + case nxv1f32: return TypeSize::Scalable(32); case v3i16: - case v3f16: return 48; + case v3f16: return TypeSize::Fixed(48); case x86mmx: case f64 : case i64 : @@ -736,17 +741,17 @@ namespace llvm { case v1i64: case v4f16: case v2f32: - case v1f64: + case v1f64: return TypeSize::Fixed(64); case nxv8i8: case nxv4i16: case nxv2i32: case nxv1i64: case nxv4f16: case nxv2f32: - case nxv1f64: return 64; - case f80 : return 80; + case nxv1f64: return TypeSize::Scalable(64); + case f80 : return TypeSize::Fixed(80); case v3i32: - case v3f32: return 96; + case v3f32: return TypeSize::Fixed(96); case f128: case ppcf128: case i128: @@ -758,16 +763,16 @@ namespace llvm { case v1i128: case v8f16: case v4f32: - case v2f64: + case v2f64: return TypeSize::Fixed(128); case nxv16i8: case nxv8i16: case nxv4i32: case nxv2i64: case nxv8f16: case nxv4f32: - case nxv2f64: return 128; + case nxv2f64: return TypeSize::Scalable(128); case v5i32: - case v5f32: return 160; + case v5f32: return TypeSize::Fixed(160); case v256i1: case v32i8: case v16i16: @@ -775,13 +780,13 @@ namespace llvm { case v4i64: case v16f16: case v8f32: - case v4f64: + case v4f64: return TypeSize::Fixed(256); case nxv32i8: case nxv16i16: case nxv8i32: case nxv4i64: case nxv8f32: - case nxv4f64: return 256; + case nxv4f64: return TypeSize::Scalable(256); case v512i1: case v64i8: case v32i16: @@ -789,56 +794,71 @@ namespace llvm { case v8i64: case v32f16: case v16f32: - case v8f64: + case v8f64: return TypeSize::Fixed(512); case nxv32i16: case nxv16i32: case nxv8i64: case nxv16f32: - case nxv8f64: return 512; + case nxv8f64: return TypeSize::Scalable(512); case v1024i1: case v128i8: case v64i16: case v32i32: case v16i64: - case v32f32: + case v32f32: return TypeSize::Fixed(1024); case nxv32i32: - case nxv16i64: return 1024; + case nxv16i64: return TypeSize::Scalable(1024); case v256i8: case v128i16: case v64i32: case v32i64: - case v64f32: - case nxv32i64: return 2048; + case v64f32: return TypeSize::Fixed(2048); + case nxv32i64: return TypeSize::Scalable(2048); case v128i32: - case v128f32: return 4096; + case v128f32: return TypeSize::Fixed(4096); case v256i32: - case v256f32: return 8192; + case v256f32: return TypeSize::Fixed(8192); case v512i32: - case v512f32: return 16384; + case v512f32: return TypeSize::Fixed(16384); case v1024i32: - case v1024f32: return 32768; + case v1024f32: return TypeSize::Fixed(32768); case v2048i32: - case v2048f32: return 65536; - case exnref: return 0; // opaque type + case v2048f32: return TypeSize::Fixed(65536); + case exnref: return TypeSize::Fixed(0); // opaque type } } - unsigned getScalarSizeInBits() const { + TypeSize getScalarSizeInBits() const { return getScalarType().getSizeInBits(); } /// Return the number of bytes overwritten by a store of the specified value /// type. - unsigned getStoreSize() const { - return (getSizeInBits() + 7) / 8; + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getStoreSize() const { + TypeSize BaseSize = getSizeInBits(); + return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()}; } /// Return the number of bits overwritten by a store of the specified value /// type. - unsigned getStoreSizeInBits() const { + /// + /// If the value type is a scalable vector type, the scalable property will + /// be set and the runtime size will be a positive integer multiple of the + /// base size. + TypeSize getStoreSizeInBits() const { return getStoreSize() * 8; } + /// Returns true if the number of bits for the type is a multiple of an + /// 8-bit byte. + bool isByteSized() const { + return getSizeInBits().isByteSized(); + } + /// Return true if this has more bits than VT. bool bitsGT(MVT VT) const { return getSizeInBits() > VT.getSizeInBits(); diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h index 004a6f5f6eb8..37b9669cbeed 100644 --- a/llvm/include/llvm/Support/MathExtras.h +++ b/llvm/include/llvm/Support/MathExtras.h @@ -732,6 +732,11 @@ inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) { return alignTo(Numerator, Denominator) / Denominator; } +/// Returns the integer nearest(Numerator / Denominator). +inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) { + return (Numerator + (Denominator / 2)) / Denominator; +} + /// Returns the largest uint64_t less than or equal to \p Value and is /// \p Skew mod \p Align. \p Align must be non-zero inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { diff --git a/llvm/include/llvm/Support/Memory.h b/llvm/include/llvm/Support/Memory.h index 6f22dd7080cd..c0454223b2fd 100644 --- a/llvm/include/llvm/Support/Memory.h +++ b/llvm/include/llvm/Support/Memory.h @@ -57,6 +57,17 @@ namespace sys { MF_WRITE = 0x2000000, MF_EXEC = 0x4000000, MF_RWE_MASK = 0x7000000, + + /// The \p MF_HUGE_HINT flag is used to indicate that the request for + /// a memory block should be satisfied with large pages if possible. + /// This is only a hint and small pages will be used as fallback. + /// + /// The presence or absence of this flag in the returned memory block + /// is (at least currently) *not* a reliable indicator that the memory + /// block will use or will not use large pages. On some systems a request + /// without this flag can be backed by large pages without this flag being + /// set, and on some other systems a request with this flag can fallback + /// to small pages without this flag being cleared. MF_HUGE_HINT = 0x0000001 }; diff --git a/llvm/include/llvm/Support/Options.h b/llvm/include/llvm/Support/Options.h deleted file mode 100644 index d02ef85a75bf..000000000000 --- a/llvm/include/llvm/Support/Options.h +++ /dev/null @@ -1,119 +0,0 @@ -//===- llvm/Support/Options.h - Debug options support -----------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// \file -/// This file declares helper objects for defining debug options that can be -/// configured via the command line. The new API currently builds on the cl::opt -/// API, but does not require the use of static globals. -/// -/// With this API options are registered during initialization. For passes, this -/// happens during pass initialization. Passes with options will call a static -/// registerOptions method during initialization that registers options with the -/// OptionRegistry. An example implementation of registerOptions is: -/// -/// static void registerOptions() { -/// OptionRegistry::registerOption<bool, Scalarizer, -/// &Scalarizer::ScalarizeLoadStore>( -/// "scalarize-load-store", -/// "Allow the scalarizer pass to scalarize loads and store", false); -/// } -/// -/// When reading data for options the interface is via the LLVMContext. Option -/// data for passes should be read from the context during doInitialization. An -/// example of reading the above option would be: -/// -/// ScalarizeLoadStore = -/// M.getContext().getOption<bool, -/// Scalarizer, -/// &Scalarizer::ScalarizeLoadStore>(); -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_OPTIONS_H -#define LLVM_SUPPORT_OPTIONS_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/Support/CommandLine.h" - -namespace llvm { - -namespace detail { - -// Options are keyed of the unique address of a static character synthesized -// based on template arguments. -template <typename ValT, typename Base, ValT(Base::*Mem)> class OptionKey { -public: - static char ID; -}; - -template <typename ValT, typename Base, ValT(Base::*Mem)> -char OptionKey<ValT, Base, Mem>::ID = 0; - -} // namespace detail - -/// Singleton class used to register debug options. -/// -/// The OptionRegistry is responsible for managing lifetimes of the options and -/// provides interfaces for option registration and reading values from options. -/// This object is a singleton, only one instance should ever exist so that all -/// options are registered in the same place. -class OptionRegistry { -private: - DenseMap<void *, cl::Option *> Options; - - /// Adds a cl::Option to the registry. - /// - /// \param Key unique key for option - /// \param O option to map to \p Key - /// - /// Allocated cl::Options are owned by the OptionRegistry and are deallocated - /// on destruction or removal - void addOption(void *Key, cl::Option *O); - -public: - ~OptionRegistry(); - OptionRegistry() {} - - /// Returns a reference to the singleton instance. - static OptionRegistry &instance(); - - /// Registers an option with the OptionRegistry singleton. - /// - /// \tparam ValT type of the option's data - /// \tparam Base class used to key the option - /// \tparam Mem member of \p Base used for keying the option - /// - /// Options are keyed off the template parameters to generate unique static - /// characters. The template parameters are (1) the type of the data the - /// option stores (\p ValT), the class that will read the option (\p Base), - /// and the member that the class will store the data into (\p Mem). - template <typename ValT, typename Base, ValT(Base::*Mem)> - static void registerOption(StringRef ArgStr, StringRef Desc, - const ValT &InitValue) { - cl::opt<ValT> *Option = new cl::opt<ValT>(ArgStr, cl::desc(Desc), - cl::Hidden, cl::init(InitValue)); - instance().addOption(&detail::OptionKey<ValT, Base, Mem>::ID, Option); - } - - /// Returns the value of the option. - /// - /// \tparam ValT type of the option's data - /// \tparam Base class used to key the option - /// \tparam Mem member of \p Base used for keying the option - /// - /// Reads option values based on the key generated by the template parameters. - /// Keying for get() is the same as keying for registerOption. - template <typename ValT, typename Base, ValT(Base::*Mem)> ValT get() const { - auto It = Options.find(&detail::OptionKey<ValT, Base, Mem>::ID); - assert(It != Options.end() && "Option not in OptionRegistry"); - return *(cl::opt<ValT> *)It->second; - } -}; - -} // namespace llvm - -#endif diff --git a/llvm/include/llvm/Support/Path.h b/llvm/include/llvm/Support/Path.h index 5c0bee58f188..97955f882d51 100644 --- a/llvm/include/llvm/Support/Path.h +++ b/llvm/include/llvm/Support/Path.h @@ -52,10 +52,10 @@ enum class Style { windows, posix, native }; class const_iterator : public iterator_facade_base<const_iterator, std::input_iterator_tag, const StringRef> { - StringRef Path; ///< The entire path. - StringRef Component; ///< The current component. Not necessarily in Path. - size_t Position; ///< The iterators current position within Path. - Style S; ///< The path style to use. + StringRef Path; ///< The entire path. + StringRef Component; ///< The current component. Not necessarily in Path. + size_t Position = 0; ///< The iterators current position within Path. + Style S = Style::native; ///< The path style to use. // An end iterator has Position = Path.size() + 1. friend const_iterator begin(StringRef path, Style style); @@ -78,10 +78,10 @@ public: class reverse_iterator : public iterator_facade_base<reverse_iterator, std::input_iterator_tag, const StringRef> { - StringRef Path; ///< The entire path. - StringRef Component; ///< The current component. Not necessarily in Path. - size_t Position; ///< The iterators current position within Path. - Style S; ///< The path style to use. + StringRef Path; ///< The entire path. + StringRef Component; ///< The current component. Not necessarily in Path. + size_t Position = 0; ///< The iterators current position within Path. + Style S = Style::native; ///< The path style to use. friend reverse_iterator rbegin(StringRef path, Style style); friend reverse_iterator rend(StringRef path); @@ -121,6 +121,8 @@ reverse_iterator rend(StringRef path); /// Remove the last component from \a path unless it is the root dir. /// +/// Similar to the POSIX "dirname" utility. +/// /// @code /// directory/filename.cpp => directory/ /// directory/ => directory @@ -150,18 +152,33 @@ void replace_extension(SmallVectorImpl<char> &path, const Twine &extension, /// /// @code /// /foo, /old, /new => /foo +/// /old, /old, /new => /new +/// /old, /old/, /new, false => /old +/// /old, /old/, /new, true => /new /// /old/foo, /old, /new => /new/foo +/// /old/foo, /old/, /new => /new/foo +/// /old/foo, /old/, /new/ => /new/foo +/// /oldfoo, /old, /new => /oldfoo /// /foo, <empty>, /new => /new/foo -/// /old/foo, /old, <empty> => /foo +/// /foo, <empty>, new => new/foo +/// /old/foo, /old, <empty>, false => /foo +/// /old/foo, /old, <empty>, true => foo /// @endcode /// /// @param Path If \a Path starts with \a OldPrefix modify to instead /// start with \a NewPrefix. -/// @param OldPrefix The path prefix to strip from \a Path. +/// @param OldPrefix The path prefix to strip from \a Path. Any trailing +/// path separator is ignored if strict is true. /// @param NewPrefix The path prefix to replace \a NewPrefix with. -void replace_path_prefix(SmallVectorImpl<char> &Path, +/// @param style The path separator style +/// @param strict If strict is true, a directory separator following +/// \a OldPrefix will also be stripped. Otherwise, directory +/// separators will only be matched and stripped when present +/// in \a OldPrefix. +/// @result true if \a Path begins with OldPrefix +bool replace_path_prefix(SmallVectorImpl<char> &Path, const StringRef &OldPrefix, const StringRef &NewPrefix, - Style style = Style::native); + Style style = Style::native, bool strict = false); /// Append to path. /// @@ -295,7 +312,7 @@ StringRef parent_path(StringRef path, Style style = Style::native); /// /// @param path Input path. /// @result The filename part of \a path. This is defined as the last component -/// of \a path. +/// of \a path. Similar to the POSIX "basename" utility. StringRef filename(StringRef path, Style style = Style::native); /// Get stem. diff --git a/llvm/include/llvm/Support/Signals.h b/llvm/include/llvm/Support/Signals.h index a4f1fad22dd5..e0a18e72f2a7 100644 --- a/llvm/include/llvm/Support/Signals.h +++ b/llvm/include/llvm/Support/Signals.h @@ -85,16 +85,36 @@ namespace sys { /// thread on some platforms. void SetInfoSignalFunction(void (*Handler)()); - /// Registers a function to be called when a "pipe" signal is delivered to - /// the process. + /// Registers a function to be called in a "one-shot" manner when a pipe + /// signal is delivered to the process (i.e., on a failed write to a pipe). + /// After the pipe signal is handled once, the handler is unregistered. /// - /// The "pipe" signal typically indicates a failed write to a pipe (SIGPIPE). - /// The default installed handler calls `exit(EX_IOERR)`, causing the process - /// to immediately exit with an IO error exit code. + /// The LLVM signal handling code will not install any handler for the pipe + /// signal unless one is provided with this API (see \ref + /// DefaultOneShotPipeSignalHandler). This handler must be provided before + /// any other LLVM signal handlers are installed: the \ref InitLLVM + /// constructor has a flag that can simplify this setup. /// - /// This function is only applicable on POSIX systems. - void SetPipeSignalFunction(void (*Handler)()); - + /// Note that the handler is not allowed to call any non-reentrant + /// functions. A null handler pointer disables the current installed + /// function. Note also that the handler may be executed on a + /// different thread on some platforms. + /// + /// This is a no-op on Windows. + void SetOneShotPipeSignalFunction(void (*Handler)()); + + /// On Unix systems, this function exits with an "IO error" exit code. + /// This is a no-op on Windows. + void DefaultOneShotPipeSignalHandler(); + + /// This function does the following: + /// - clean up any temporary files registered with RemoveFileOnSignal() + /// - dump the callstack from the exception context + /// - call any relevant interrupt/signal handlers + /// - create a core/mini dump of the exception context whenever possible + /// Context is a system-specific failure context: it is the signal type on + /// Unix; the ExceptionContext on Windows. + void CleanupOnSignal(uintptr_t Context); } // End sys namespace } // End llvm namespace diff --git a/llvm/include/llvm/Support/SourceMgr.h b/llvm/include/llvm/Support/SourceMgr.h index aa6026c23d07..1b005519e5d4 100644 --- a/llvm/include/llvm/Support/SourceMgr.h +++ b/llvm/include/llvm/Support/SourceMgr.h @@ -61,10 +61,10 @@ private: /// into relatively small files (often smaller than 2^8 or 2^16 bytes), /// we select the offset vector element type dynamically based on the /// size of Buffer. - using VariableSizeOffsets = PointerUnion4<std::vector<uint8_t> *, - std::vector<uint16_t> *, - std::vector<uint32_t> *, - std::vector<uint64_t> *>; + using VariableSizeOffsets = PointerUnion<std::vector<uint8_t> *, + std::vector<uint16_t> *, + std::vector<uint32_t> *, + std::vector<uint64_t> *>; /// Vector of offsets into Buffer at which there are line-endings /// (lazily populated). Once populated, the '\n' that marks the end of diff --git a/llvm/include/llvm/Support/SpecialCaseList.h b/llvm/include/llvm/Support/SpecialCaseList.h index b7400266f4df..5b5b7f6124d6 100644 --- a/llvm/include/llvm/Support/SpecialCaseList.h +++ b/llvm/include/llvm/Support/SpecialCaseList.h @@ -55,6 +55,7 @@ #include "llvm/ADT/StringSet.h" #include "llvm/Support/Regex.h" #include "llvm/Support/TrigramIndex.h" +#include "llvm/Support/VirtualFileSystem.h" #include <string> #include <vector> @@ -68,7 +69,8 @@ public: /// Parses the special case list entries from files. On failure, returns /// 0 and writes an error message to string. static std::unique_ptr<SpecialCaseList> - create(const std::vector<std::string> &Paths, std::string &Error); + create(const std::vector<std::string> &Paths, llvm::vfs::FileSystem &FS, + std::string &Error); /// Parses the special case list from a memory buffer. On failure, returns /// 0 and writes an error message to string. static std::unique_ptr<SpecialCaseList> create(const MemoryBuffer *MB, @@ -76,7 +78,7 @@ public: /// Parses the special case list entries from files. On failure, reports a /// fatal error. static std::unique_ptr<SpecialCaseList> - createOrDie(const std::vector<std::string> &Paths); + createOrDie(const std::vector<std::string> &Paths, llvm::vfs::FileSystem &FS); ~SpecialCaseList(); @@ -103,7 +105,7 @@ protected: // Implementations of the create*() functions that can also be used by derived // classes. bool createInternal(const std::vector<std::string> &Paths, - std::string &Error); + vfs::FileSystem &VFS, std::string &Error); bool createInternal(const MemoryBuffer *MB, std::string &Error); SpecialCaseList() = default; diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 11731ac35415..e004550059d4 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -285,6 +285,9 @@ HANDLE_TARGET_OPCODE(G_INTRINSIC_TRUNC) /// INTRINSIC round intrinsic. HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUND) +/// INTRINSIC readcyclecounter +HANDLE_TARGET_OPCODE(G_READCYCLECOUNTER) + /// Generic load (including anyext load) HANDLE_TARGET_OPCODE(G_LOAD) @@ -524,7 +527,7 @@ HANDLE_TARGET_OPCODE(G_FMINIMUM) HANDLE_TARGET_OPCODE(G_FMAXIMUM) /// Generic pointer offset -HANDLE_TARGET_OPCODE(G_GEP) +HANDLE_TARGET_OPCODE(G_PTR_ADD) /// Clear the specified number of low bits in a pointer. This rounds the value /// *down* to the given alignment. @@ -611,12 +614,16 @@ HANDLE_TARGET_OPCODE(G_JUMP_TABLE) /// Generic dynamic stack allocation. HANDLE_TARGET_OPCODE(G_DYN_STACKALLOC) -// TODO: Add more generic opcodes as we move along. +/// read_register intrinsic +HANDLE_TARGET_OPCODE(G_READ_REGISTER) + +/// write_register intrinsic +HANDLE_TARGET_OPCODE(G_WRITE_REGISTER) /// Marker for the end of the generic opcode. /// This is used to check if an opcode is in the range of the /// generic opcodes. -HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_DYN_STACKALLOC) +HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_WRITE_REGISTER) /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific post-isel opcode values start here. diff --git a/llvm/include/llvm/Support/TargetRegistry.h b/llvm/include/llvm/Support/TargetRegistry.h index f4bc26b858c8..d91eabae8235 100644 --- a/llvm/include/llvm/Support/TargetRegistry.h +++ b/llvm/include/llvm/Support/TargetRegistry.h @@ -128,7 +128,8 @@ public: using ArchMatchFnTy = bool (*)(Triple::ArchType Arch); using MCAsmInfoCtorFnTy = MCAsmInfo *(*)(const MCRegisterInfo &MRI, - const Triple &TT); + const Triple &TT, + const MCTargetOptions &Options); using MCInstrInfoCtorFnTy = MCInstrInfo *(*)(); using MCInstrAnalysisCtorFnTy = MCInstrAnalysis *(*)(const MCInstrInfo *Info); using MCRegInfoCtorFnTy = MCRegisterInfo *(*)(const Triple &TT); @@ -335,11 +336,11 @@ public: /// feature set; it should always be provided. Generally this should be /// either the target triple from the module, or the target triple of the /// host if that does not exist. - MCAsmInfo *createMCAsmInfo(const MCRegisterInfo &MRI, - StringRef TheTriple) const { + MCAsmInfo *createMCAsmInfo(const MCRegisterInfo &MRI, StringRef TheTriple, + const MCTargetOptions &Options) const { if (!MCAsmInfoCtorFn) return nullptr; - return MCAsmInfoCtorFn(MRI, Triple(TheTriple)); + return MCAsmInfoCtorFn(MRI, Triple(TheTriple), Options); } /// createMCInstrInfo - Create a MCInstrInfo implementation. @@ -473,7 +474,7 @@ public: const MCSubtargetInfo &STI, bool RelaxAll, bool IncrementalLinkerCompatible, bool DWARFMustBeAtTheEnd) const { - MCStreamer *S; + MCStreamer *S = nullptr; switch (T.getObjectFormat()) { case Triple::UnknownObjectFormat: llvm_unreachable("Unknown object format"); @@ -948,9 +949,9 @@ template <class MCAsmInfoImpl> struct RegisterMCAsmInfo { } private: - static MCAsmInfo *Allocator(const MCRegisterInfo & /*MRI*/, - const Triple &TT) { - return new MCAsmInfoImpl(TT); + static MCAsmInfo *Allocator(const MCRegisterInfo & /*MRI*/, const Triple &TT, + const MCTargetOptions &Options) { + return new MCAsmInfoImpl(TT, Options); } }; diff --git a/llvm/include/llvm/Support/Threading.h b/llvm/include/llvm/Support/Threading.h index 46d413dc487b..bacab8fa23b6 100644 --- a/llvm/include/llvm/Support/Threading.h +++ b/llvm/include/llvm/Support/Threading.h @@ -14,6 +14,7 @@ #ifndef LLVM_SUPPORT_THREADING_H #define LLVM_SUPPORT_THREADING_H +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX #include "llvm/Support/Compiler.h" @@ -52,9 +53,8 @@ class Twine; /// false otherwise. bool llvm_is_multithreaded(); -/// llvm_execute_on_thread - Execute the given \p UserFn on a separate -/// thread, passing it the provided \p UserData and waits for thread -/// completion. +/// Execute the given \p UserFn on a separate thread, passing it the provided \p +/// UserData and waits for thread completion. /// /// This function does not guarantee that the code will actually be executed /// on a separate thread or honoring the requested stack size, but tries to do @@ -62,10 +62,26 @@ bool llvm_is_multithreaded(); /// /// \param UserFn - The callback to execute. /// \param UserData - An argument to pass to the callback function. -/// \param RequestedStackSize - If non-zero, a requested size (in bytes) for -/// the thread stack. -void llvm_execute_on_thread(void (*UserFn)(void *), void *UserData, - unsigned RequestedStackSize = 0); +/// \param StackSizeInBytes - A requested size (in bytes) for the thread stack +/// (or None for default) +void llvm_execute_on_thread( + void (*UserFn)(void *), void *UserData, + llvm::Optional<unsigned> StackSizeInBytes = llvm::None); + +/// Schedule the given \p Func for execution on a separate thread, then return +/// to the caller immediately. Roughly equivalent to +/// `std::thread(Func).detach()`, except it allows requesting a specific stack +/// size, if supported for the platform. +/// +/// This function would report a fatal error if it can't execute the code +/// on a separate thread. +/// +/// \param Func - The callback to execute. +/// \param StackSizeInBytes - A requested size (in bytes) for the thread stack +/// (or None for default) +void llvm_execute_on_thread_async( + llvm::unique_function<void()> Func, + llvm::Optional<unsigned> StackSizeInBytes = llvm::None); #if LLVM_THREADING_USE_STD_CALL_ONCE diff --git a/llvm/include/llvm/Support/TimeProfiler.h b/llvm/include/llvm/Support/TimeProfiler.h index 8cc430d0bc72..678f8c136811 100644 --- a/llvm/include/llvm/Support/TimeProfiler.h +++ b/llvm/include/llvm/Support/TimeProfiler.h @@ -19,7 +19,8 @@ extern TimeTraceProfiler *TimeTraceProfilerInstance; /// Initialize the time trace profiler. /// This sets up the global \p TimeTraceProfilerInstance /// variable to be the profiler instance. -void timeTraceProfilerInitialize(unsigned TimeTraceGranularity); +void timeTraceProfilerInitialize(unsigned TimeTraceGranularity, + StringRef ProcName); /// Cleanup the time trace profiler, if it was initialized. void timeTraceProfilerCleanup(); @@ -57,6 +58,10 @@ struct TimeTraceScope { TimeTraceScope(TimeTraceScope &&) = delete; TimeTraceScope &operator=(TimeTraceScope &&) = delete; + TimeTraceScope(StringRef Name) { + if (TimeTraceProfilerInstance != nullptr) + timeTraceProfilerBegin(Name, StringRef("")); + } TimeTraceScope(StringRef Name, StringRef Detail) { if (TimeTraceProfilerInstance != nullptr) timeTraceProfilerBegin(Name, Detail); diff --git a/llvm/include/llvm/Support/Timer.h b/llvm/include/llvm/Support/Timer.h index 76c9bc7b6863..a298ecd90404 100644 --- a/llvm/include/llvm/Support/Timer.h +++ b/llvm/include/llvm/Support/Timer.h @@ -78,18 +78,18 @@ class Timer { TimeRecord StartTime; ///< The time startTimer() was last called. std::string Name; ///< The name of this time variable. std::string Description; ///< Description of this time variable. - bool Running; ///< Is the timer currently running? - bool Triggered; ///< Has the timer ever been triggered? + bool Running = false; ///< Is the timer currently running? + bool Triggered = false; ///< Has the timer ever been triggered? TimerGroup *TG = nullptr; ///< The TimerGroup this Timer is in. - Timer **Prev; ///< Pointer to \p Next of previous timer in group. - Timer *Next; ///< Next timer in the group. + Timer **Prev = nullptr; ///< Pointer to \p Next of previous timer in group. + Timer *Next = nullptr; ///< Next timer in the group. public: - explicit Timer(StringRef Name, StringRef Description) { - init(Name, Description); + explicit Timer(StringRef TimerName, StringRef TimerDescription) { + init(TimerName, TimerDescription); } - Timer(StringRef Name, StringRef Description, TimerGroup &tg) { - init(Name, Description, tg); + Timer(StringRef TimerName, StringRef TimerDescription, TimerGroup &tg) { + init(TimerName, TimerDescription, tg); } Timer(const Timer &RHS) { assert(!RHS.TG && "Can only copy uninitialized timers"); @@ -102,8 +102,8 @@ public: /// Create an uninitialized timer, client must use 'init'. explicit Timer() {} - void init(StringRef Name, StringRef Description); - void init(StringRef Name, StringRef Description, TimerGroup &tg); + void init(StringRef TimerName, StringRef TimerDescription); + void init(StringRef TimerName, StringRef TimerDescription, TimerGroup &tg); const std::string &getName() const { return Name; } const std::string &getDescription() const { return Description; } @@ -174,6 +174,7 @@ class TimerGroup { std::string Description; PrintRecord(const PrintRecord &Other) = default; + PrintRecord &operator=(const PrintRecord &Other) = default; PrintRecord(const TimeRecord &Time, const std::string &Name, const std::string &Description) : Time(Time), Name(Name), Description(Description) {} diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h index 711679cdcacb..7ea651f0f22c 100644 --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -138,6 +138,11 @@ public: return IsScalable; } + // Returns true if the number of bits is a multiple of an 8-bit byte. + bool isByteSized() const { + return (MinSize & 7) == 0; + } + // Casts to a uint64_t if this is a fixed-width size. // // NOTE: This interface is obsolete and will be removed in a future version diff --git a/llvm/include/llvm/Support/VersionTuple.h b/llvm/include/llvm/Support/VersionTuple.h index 14736d6b28f0..f3eeea2f7b44 100644 --- a/llvm/include/llvm/Support/VersionTuple.h +++ b/llvm/include/llvm/Support/VersionTuple.h @@ -87,6 +87,13 @@ public: return Build; } + /// Return a version tuple that contains only the first 3 version components. + VersionTuple withoutBuild() const { + if (HasBuild) + return VersionTuple(Major, Minor, Subminor); + return *this; + } + /// Determine if two version numbers are equivalent. If not /// provided, minor and subminor version numbers are considered to be zero. friend bool operator==(const VersionTuple &X, const VersionTuple &Y) { diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index c844d9d194f0..e45e6e756786 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -126,7 +126,7 @@ public: /// Only information available on most platforms is included. class directory_entry { std::string Path; - llvm::sys::fs::file_type Type; + llvm::sys::fs::file_type Type = llvm::sys::fs::file_type::type_unknown; public: directory_entry() = default; @@ -293,7 +293,7 @@ public: /// \param Path A path that is modified to be an absolute path. /// \returns success if \a path has been made absolute, otherwise a /// platform-specific error_code. - std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const; + virtual std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const; }; /// Gets an \p vfs::FileSystem for the 'real' file system, as seen by @@ -532,7 +532,7 @@ class RedirectingFileSystemParser; /// \endverbatim /// /// All configuration options are optional. -/// 'case-sensitive': <boolean, default=true> +/// 'case-sensitive': <boolean, default=(true for Posix, false for Windows)> /// 'use-external-names': <boolean, default=true> /// 'overlay-relative': <boolean, default=false> /// 'fallthrough': <boolean, default=true> @@ -651,6 +651,17 @@ private: return ExternalFSValidWD && IsFallthrough; } + // In a RedirectingFileSystem, keys can be specified in Posix or Windows + // style (or even a mixture of both), so this comparison helper allows + // slashes (representing a root) to match backslashes (and vice versa). Note + // that, other than the root, patch components should not contain slashes or + // backslashes. + bool pathComponentMatches(llvm::StringRef lhs, llvm::StringRef rhs) const { + if ((CaseSensitive ? lhs.equals(rhs) : lhs.equals_lower(rhs))) + return true; + return (lhs == "/" && rhs == "\\") || (lhs == "\\" && rhs == "/"); + } + /// The root(s) of the virtual file system. std::vector<std::unique_ptr<Entry>> Roots; @@ -674,7 +685,12 @@ private: /// Whether to perform case-sensitive comparisons. /// /// Currently, case-insensitive matching only works correctly with ASCII. - bool CaseSensitive = true; + bool CaseSensitive = +#ifdef _WIN32 + false; +#else + true; +#endif /// IsRelativeOverlay marks whether a ExternalContentsPrefixDir path must /// be prefixed in every 'external-contents' when reading from YAML files. @@ -733,6 +749,8 @@ public: std::error_code isLocal(const Twine &Path, bool &Result) override; + std::error_code makeAbsolute(SmallVectorImpl<char> &Path) const override; + directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override; void setExternalContentsPrefixDir(StringRef PrefixDir); diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h index a3bfa7dc4678..8642069ad540 100644 --- a/llvm/include/llvm/Support/YAMLTraits.h +++ b/llvm/include/llvm/Support/YAMLTraits.h @@ -1531,7 +1531,7 @@ private: document_iterator DocIterator; std::vector<bool> BitValuesUsed; HNode *CurrentNode = nullptr; - bool ScalarMatchFound; + bool ScalarMatchFound = false; }; /// @@ -2035,4 +2035,9 @@ template <typename T> struct StdMapStringCustomMappingTraitsImpl { } \ } +LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::Hex64) +LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::Hex32) +LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::Hex16) +LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::Hex8) + #endif // LLVM_SUPPORT_YAMLTRAITS_H diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h index 0debc5da7a68..c8770c337588 100644 --- a/llvm/include/llvm/Support/raw_ostream.h +++ b/llvm/include/llvm/Support/raw_ostream.h @@ -21,6 +21,7 @@ #include <cstring> #include <string> #include <system_error> +#include <type_traits> namespace llvm { @@ -64,7 +65,7 @@ private: /// this buffer. char *OutBufStart, *OutBufEnd, *OutBufCur; - enum BufferKind { + enum class BufferKind { Unbuffered = 0, InternalBuffer, ExternalBuffer @@ -97,7 +98,8 @@ public: static const Colors RESET = Colors::RESET; explicit raw_ostream(bool unbuffered = false) - : BufferMode(unbuffered ? Unbuffered : InternalBuffer) { + : BufferMode(unbuffered ? BufferKind::Unbuffered + : BufferKind::InternalBuffer) { // Start out ready to flush. OutBufStart = OutBufEnd = OutBufCur = nullptr; } @@ -121,13 +123,13 @@ public: /// Set the stream to be buffered, using the specified buffer size. void SetBufferSize(size_t Size) { flush(); - SetBufferAndMode(new char[Size], Size, InternalBuffer); + SetBufferAndMode(new char[Size], Size, BufferKind::InternalBuffer); } size_t GetBufferSize() const { // If we're supposed to be buffered but haven't actually gotten around // to allocating the buffer yet, return the value that would be used. - if (BufferMode != Unbuffered && OutBufStart == nullptr) + if (BufferMode != BufferKind::Unbuffered && OutBufStart == nullptr) return preferred_buffer_size(); // Otherwise just return the size of the allocated buffer. @@ -139,7 +141,7 @@ public: /// when the stream is being set to unbuffered. void SetUnbuffered() { flush(); - SetBufferAndMode(nullptr, 0, Unbuffered); + SetBufferAndMode(nullptr, 0, BufferKind::Unbuffered); } size_t GetNumBytesInBuffer() const { @@ -325,7 +327,7 @@ protected: /// use only by subclasses which can arrange for the output to go directly /// into the desired output buffer, instead of being copied on each flush. void SetBuffer(char *BufferStart, size_t Size) { - SetBufferAndMode(BufferStart, Size, ExternalBuffer); + SetBufferAndMode(BufferStart, Size, BufferKind::ExternalBuffer); } /// Return an efficient buffer size for the underlying output mechanism. @@ -353,6 +355,17 @@ private: virtual void anchor(); }; +/// Call the appropriate insertion operator, given an rvalue reference to a +/// raw_ostream object and return a stream of the same type as the argument. +template <typename OStream, typename T> +typename std::enable_if<!std::is_reference<OStream>::value && + std::is_base_of<raw_ostream, OStream>::value, + OStream &&>::type +operator<<(OStream &&OS, const T &Value) { + OS << Value; + return std::move(OS); +} + /// An abstract base class for streams implementations that also support a /// pwrite operation. This is useful for code that can mostly stream out data, /// but needs to patch in a header that needs to know the output size. @@ -384,7 +397,7 @@ public: class raw_fd_ostream : public raw_pwrite_stream { int FD; bool ShouldClose; - bool SupportsSeeking; + bool SupportsSeeking = false; bool ColorEnabled = true; #ifdef _WIN32 @@ -395,7 +408,7 @@ class raw_fd_ostream : public raw_pwrite_stream { std::error_code EC; - uint64_t pos; + uint64_t pos = 0; /// See raw_ostream::write_impl. void write_impl(const char *Ptr, size_t Size) override; diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h index 73ed342a6101..a553ec99aaa4 100644 --- a/llvm/include/llvm/TableGen/Record.h +++ b/llvm/include/llvm/TableGen/Record.h @@ -751,7 +751,7 @@ public: /// class UnOpInit : public OpInit, public FoldingSetNode { public: - enum UnaryOp : uint8_t { CAST, HEAD, TAIL, SIZE, EMPTY }; + enum UnaryOp : uint8_t { CAST, HEAD, TAIL, SIZE, EMPTY, GETOP }; private: Init *LHS; @@ -802,7 +802,7 @@ class BinOpInit : public OpInit, public FoldingSetNode { public: enum BinaryOp : uint8_t { ADD, MUL, AND, OR, SHL, SRA, SRL, LISTCONCAT, LISTSPLAT, STRCONCAT, CONCAT, EQ, NE, LE, LT, GE, - GT }; + GT, SETOP }; private: Init *LHS, *RHS; @@ -1663,6 +1663,12 @@ public: /// the value is not the right type. Record *getValueAsDef(StringRef FieldName) const; + /// This method looks up the specified field and returns its value as a + /// Record, returning null if the field exists but is "uninitialized" + /// (i.e. set to `?`), and throwing an exception if the field does not + /// exist or if its value is not the right type. + Record *getValueAsOptionalDef(StringRef FieldName) const; + /// This method looks up the specified field and returns its /// value as a bit, throwing an exception if the field does not exist or if /// the value is not the right type. diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 4b49dfd4dd18..0ee9c3916c10 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -330,7 +330,7 @@ def G_SELECT : GenericInstruction { } // Generic pointer offset. -def G_GEP : GenericInstruction { +def G_PTR_ADD : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1, type1:$src2); let hasSideEffects = 0; @@ -670,7 +670,7 @@ def G_FEXP2 : GenericInstruction { let hasSideEffects = 0; } -// Floating point base-2 logarithm of a value. +// Floating point base-e logarithm of a value. def G_FLOG : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); @@ -758,6 +758,12 @@ def G_INTRINSIC_ROUND : GenericInstruction { let hasSideEffects = 0; } +def G_READCYCLECOUNTER : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins); + let hasSideEffects = 1; +} + //------------------------------------------------------------------------------ // Memory ops //------------------------------------------------------------------------------ @@ -1006,6 +1012,26 @@ def G_BRJT : GenericInstruction { let isTerminator = 1; } +def G_READ_REGISTER : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins unknown:$register); + let hasSideEffects = 1; + + // Assume convergent. It's probably not worth the effort of somehow + // modeling convergent and nonconvergent register accesses. + let isConvergent = 1; +} + +def G_WRITE_REGISTER : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins unknown:$register, type0:$value); + let hasSideEffects = 1; + + // Assume convergent. It's probably not worth the effort of somehow + // modeling convergent and nonconvergent register accesses. + let isConvergent = 1; +} + //------------------------------------------------------------------------------ // Vector ops //------------------------------------------------------------------------------ diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index dcac399fd693..35f7a41e1cbf 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -66,11 +66,21 @@ class GIDefKindWithArgs; /// is incorrect. def root : GIDefKind; +/// Declares data that is passed from the match stage to the apply stage. +class GIDefMatchData<string type> : GIDefKind { + /// A C++ type name indicating the storage type. + string Type = type; +} + +def extending_load_matchdata : GIDefMatchData<"PreferredTuple">; +def indexed_load_store_matchdata : GIDefMatchData<"IndexedLoadStoreMatchInfo">; + /// The operator at the root of a GICombineRule.Match dag. def match; /// All arguments of the match operator must be either: /// * A subclass of GIMatchKind /// * A subclass of GIMatchKindWithArgs +/// * A subclass of Instruction /// * A MIR code block (deprecated) /// The GIMatchKind and GIMatchKindWithArgs cases are described in more detail /// in their definitions below. @@ -79,6 +89,10 @@ def match; class GIMatchKind; class GIMatchKindWithArgs; +/// In lieu of having proper macro support. Trivial one-off opcode checks can be +/// performed with this. +def wip_match_opcode : GIMatchKindWithArgs; + /// The operator at the root of a GICombineRule.Apply dag. def apply; /// All arguments of the apply operator must be subclasses of GIApplyKind, or @@ -89,15 +103,38 @@ class GIApplyKindWithArgs; def copy_prop : GICombineRule< (defs root:$d), - (match [{ return Helper.matchCombineCopy(${d}); }]), - (apply [{ Helper.applyCombineCopy(${d}); }])>; + (match (COPY $d, $s):$mi, + [{ return Helper.matchCombineCopy(*${mi}); }]), + (apply [{ Helper.applyCombineCopy(*${mi}); }])>; def trivial_combines : GICombineGroup<[copy_prop]>; +def extending_loads : GICombineRule< + (defs root:$root, extending_load_matchdata:$matchinfo), + (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD):$root, + [{ return Helper.matchCombineExtendingLoads(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyCombineExtendingLoads(*${root}, ${matchinfo}); }])>; +def combines_for_extload: GICombineGroup<[extending_loads]>; + +def combine_indexed_load_store : GICombineRule< + (defs root:$root, indexed_load_store_matchdata:$matchinfo), + (match (wip_match_opcode G_LOAD, G_SEXTLOAD, G_ZEXTLOAD, G_STORE):$root, + [{ return Helper.matchCombineIndexedLoadStore(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyCombineIndexedLoadStore(*${root}, ${matchinfo}); }])>; + // FIXME: Is there a reason this wasn't in tryCombine? I've left it out of // all_combines because it wasn't there. def elide_br_by_inverting_cond : GICombineRule< - (defs root:$d), - (match [{ return Helper.matchElideBrByInvertingCond(${d}); }]), - (apply [{ Helper.applyElideBrByInvertingCond(${d}); }])>; + (defs root:$root), + (match (wip_match_opcode G_BR):$root, + [{ return Helper.matchElideBrByInvertingCond(*${root}); }]), + (apply [{ Helper.applyElideBrByInvertingCond(*${root}); }])>; + +def ptr_add_immed_matchdata : GIDefMatchData<"PtrAddChain">; +def ptr_add_immed_chain : GICombineRule< + (defs root:$d, ptr_add_immed_matchdata:$matchinfo), + (match (wip_match_opcode G_PTR_ADD):$d, + [{ return Helper.matchPtrAddImmedChain(*${d}, ${matchinfo}); }]), + (apply [{ Helper.applyPtrAddImmedChain(*${d}, ${matchinfo}); }])>; -def all_combines : GICombineGroup<[trivial_combines]>; +def all_combines : GICombineGroup<[trivial_combines, ptr_add_immed_chain, + combines_for_extload, combine_indexed_load_store]>; diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index b846d2252b8d..2129588d4aed 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -100,6 +100,7 @@ def : GINodeEquiv<G_CTTZ_ZERO_UNDEF, cttz_zero_undef>; def : GINodeEquiv<G_CTPOP, ctpop>; def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, vector_extract>; def : GINodeEquiv<G_CONCAT_VECTORS, concat_vectors>; +def : GINodeEquiv<G_BUILD_VECTOR, build_vector>; def : GINodeEquiv<G_FCEIL, fceil>; def : GINodeEquiv<G_FCOS, fcos>; def : GINodeEquiv<G_FSIN, fsin>; @@ -108,6 +109,8 @@ def : GINodeEquiv<G_FSQRT, fsqrt>; def : GINodeEquiv<G_FFLOOR, ffloor>; def : GINodeEquiv<G_FRINT, frint>; def : GINodeEquiv<G_FNEARBYINT, fnearbyint>; +def : GINodeEquiv<G_INTRINSIC_TRUNC, ftrunc>; +def : GINodeEquiv<G_INTRINSIC_ROUND, fround>; def : GINodeEquiv<G_FCOPYSIGN, fcopysign>; def : GINodeEquiv<G_SMIN, smin>; def : GINodeEquiv<G_SMAX, smax>; @@ -117,6 +120,7 @@ def : GINodeEquiv<G_FMINNUM, fminnum>; def : GINodeEquiv<G_FMAXNUM, fmaxnum>; def : GINodeEquiv<G_FMINNUM_IEEE, fminnum_ieee>; def : GINodeEquiv<G_FMAXNUM_IEEE, fmaxnum_ieee>; +def : GINodeEquiv<G_READCYCLECOUNTER, readcyclecounter>; // Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some // complications that tablegen must take care of. For example, Predicates such diff --git a/llvm/include/llvm/Target/GlobalISel/Target.td b/llvm/include/llvm/Target/GlobalISel/Target.td index 538ca65e1162..8e868c4c207b 100644 --- a/llvm/include/llvm/Target/GlobalISel/Target.td +++ b/llvm/include/llvm/Target/GlobalISel/Target.td @@ -55,6 +55,12 @@ class GIComplexOperandMatcher<LLT type, string matcherfn> { class GICustomOperandRenderer<string rendererfn> { // The function renders the operand(s) of the matched instruction to // the specified instruction. It should be of the form: - // void render(MachineInstrBuilder &MIB, const MachineInstr &MI) + // void render(MachineInstrBuilder &MIB, const MachineInstr &MI, + // int OpIdx = -1) + // + // If OpIdx is specified (i.e. not invalid/negative), this + // references the source operand MI.getOperand(OpIdx). Otherwise, + // this is the value defined by MI. This is to support the case + // where there is no corresponding instruction to match. string RendererFn = rendererfn; } diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index dd8679661b9a..b122b51bb169 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -530,6 +530,7 @@ class Instruction : InstructionEncoding { bit hasCtrlDep = 0; // Does this instruction r/w ctrl-flow chains? bit isNotDuplicable = 0; // Is it unsafe to duplicate this instruction? bit isConvergent = 0; // Is this instruction convergent? + bit isAuthenticated = 0; // Does this instruction authenticate a pointer? bit isAsCheapAsAMove = 0; // As cheap (or cheaper) than a move instruction. bit hasExtraSrcRegAllocReq = 0; // Sources have special regalloc requirement? bit hasExtraDefRegAllocReq = 0; // Defs have special regalloc requirement? @@ -1210,7 +1211,7 @@ def PATCHABLE_TAIL_CALL : StandardPseudoInstruction { } def PATCHABLE_EVENT_CALL : StandardPseudoInstruction { let OutOperandList = (outs); - let InOperandList = (ins ptr_rc:$event, i8imm:$size); + let InOperandList = (ins ptr_rc:$event, unknown:$size); let AsmString = "# XRay Custom Event Log."; let usesCustomInserter = 1; let isCall = 1; @@ -1220,7 +1221,7 @@ def PATCHABLE_EVENT_CALL : StandardPseudoInstruction { } def PATCHABLE_TYPED_EVENT_CALL : StandardPseudoInstruction { let OutOperandList = (outs); - let InOperandList = (ins i16imm:$type, ptr_rc:$event, i32imm:$size); + let InOperandList = (ins unknown:$type, ptr_rc:$event, unknown:$size); let AsmString = "# XRay Typed Event Log."; let usesCustomInserter = 1; let isCall = 1; @@ -1292,7 +1293,7 @@ class AsmParser { // ReportMultipleNearMisses - // When 0, the assembly matcher reports an error for one encoding or operand // that did not match the parsed instruction. - // When 1, the assmebly matcher returns a list of encodings that were close + // When 1, the assembly matcher returns a list of encodings that were close // to matching the parsed instruction, so to allow more detailed error // messages. bit ReportMultipleNearMisses = 0; diff --git a/llvm/include/llvm/Target/TargetCallingConv.td b/llvm/include/llvm/Target/TargetCallingConv.td index 7b1973cc3828..d5f3931c3d5d 100644 --- a/llvm/include/llvm/Target/TargetCallingConv.td +++ b/llvm/include/llvm/Target/TargetCallingConv.td @@ -51,6 +51,11 @@ class CCIfSwiftSelf<CCAction A> : CCIf<"ArgFlags.isSwiftSelf()", A> { class CCIfSwiftError<CCAction A> : CCIf<"ArgFlags.isSwiftError()", A> { } +/// CCIfCFGuardTarget - If the current argument has cfguardtarget parameter +/// attribute, apply Action A. +class CCIfCFGuardTarget<CCAction A> : CCIf<"ArgFlags.isCFGuardTarget()", A> { +} + /// CCIfConsecutiveRegs - If the current argument has InConsecutiveRegs /// parameter attribute, apply Action A. class CCIfConsecutiveRegs<CCAction A> : CCIf<"ArgFlags.isInConsecutiveRegs()", A> { diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index 285c0ec0fb90..176ae39b17a7 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -271,15 +271,6 @@ public: /// PassManagerBuilder::addExtension. virtual void adjustPassManager(PassManagerBuilder &) {} - /// These enums are meant to be passed into addPassesToEmitFile to indicate - /// what type of file to emit, and returned by it to indicate what type of - /// file could actually be made. - enum CodeGenFileType { - CGFT_AssemblyFile, - CGFT_ObjectFile, - CGFT_Null // Do not emit any output. - }; - /// Add passes to the specified pass manager to get the specified file /// emitted. Typically this will involve several steps of code generation. /// This method should return true if emission of this file type is not diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index 8cc2a6010879..84c6ee2a6387 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -107,7 +107,7 @@ namespace llvm { public: TargetOptions() : PrintMachineCode(false), UnsafeFPMath(false), NoInfsFPMath(false), - NoNaNsFPMath(false), NoTrappingFPMath(false), + NoNaNsFPMath(false), NoTrappingFPMath(true), NoSignedZerosFPMath(false), HonorSignDependentRoundingFPMathOption(false), NoZerosInBSS(false), GuaranteedTailCallOpt(false), StackSymbolOrdering(true), @@ -115,11 +115,11 @@ namespace llvm { DisableIntegratedAS(false), RelaxELFRelocations(false), FunctionSections(false), DataSections(false), UniqueSectionNames(true), TrapUnreachable(false), - NoTrapAfterNoreturn(false), EmulatedTLS(false), + NoTrapAfterNoreturn(false), TLSSize(0), EmulatedTLS(false), ExplicitEmulatedTLS(false), EnableIPRA(false), EmitStackSizeSection(false), EnableMachineOutliner(false), SupportsDefaultOutlining(false), EmitAddrsig(false), - EnableDebugEntryValues(false) {} + EnableDebugEntryValues(false), ForceDwarfFrameSection(false) {} /// PrintMachineCode - This flag is enabled when the -print-machineinstrs /// option is specified on the command line, and should enable debugging @@ -231,6 +231,9 @@ namespace llvm { /// noreturn calls, even if TrapUnreachable is true. unsigned NoTrapAfterNoreturn : 1; + /// Bit size of immediate TLS offsets (0 == use the default). + unsigned TLSSize : 8; + /// EmulatedTLS - This flag enables emulated TLS model, using emutls /// function in the runtime library.. unsigned EmulatedTLS : 1; @@ -256,6 +259,9 @@ namespace llvm { /// Emit debug info about parameter's entry values. unsigned EnableDebugEntryValues : 1; + /// Emit DWARF debug frame section. + unsigned ForceDwarfFrameSection : 1; + /// FloatABIType - This setting is set by -float-abi=xxx option is specfied /// on the command line. This setting may either be Default, Soft, or Hard. /// Default selects the target's default behavior. Soft selects the ABI for diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 441f3d7d118d..1700c6c4640d 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -124,7 +124,7 @@ def SDTIntSatNoShOp : SDTypeProfile<1, 2, [ // ssat with no shift def SDTIntBinHiLoOp : SDTypeProfile<2, 2, [ // mulhi, mullo, sdivrem, udivrem SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>,SDTCisInt<0> ]>; -def SDTIntScaledBinOp : SDTypeProfile<1, 3, [ // smulfix, umulfix +def SDTIntScaledBinOp : SDTypeProfile<1, 3, [ // smulfix, sdivfix, etc SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisInt<0>, SDTCisInt<3> ]>; @@ -224,13 +224,13 @@ def SDTIStore : SDTypeProfile<1, 3, [ // indexed store SDTCisSameAs<0, 2>, SDTCisPtrTy<0>, SDTCisPtrTy<3> ]>; -def SDTMaskedStore: SDTypeProfile<0, 3, [ // masked store - SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>, SDTCisSameNumEltsAs<0, 2> +def SDTMaskedStore: SDTypeProfile<0, 4, [ // masked store + SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisSameNumEltsAs<0, 3> ]>; -def SDTMaskedLoad: SDTypeProfile<1, 3, [ // masked load - SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>, SDTCisSameAs<0, 3>, - SDTCisSameNumEltsAs<0, 2> +def SDTMaskedLoad: SDTypeProfile<1, 4, [ // masked load + SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisSameAs<0, 4>, + SDTCisSameNumEltsAs<0, 3> ]>; def SDTVecShuffle : SDTypeProfile<1, 2, [ @@ -400,6 +400,8 @@ def smulfix : SDNode<"ISD::SMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]> def smulfixsat : SDNode<"ISD::SMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>; def umulfix : SDNode<"ISD::UMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]>; def umulfixsat : SDNode<"ISD::UMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>; +def sdivfix : SDNode<"ISD::SDIVFIX" , SDTIntScaledBinOp>; +def udivfix : SDNode<"ISD::UDIVFIX" , SDTIntScaledBinOp>; def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>; def sext_invec : SDNode<"ISD::SIGN_EXTEND_VECTOR_INREG", SDTExtInvec>; @@ -530,6 +532,12 @@ def strict_fminnum : SDNode<"ISD::STRICT_FMINNUM", def strict_fmaxnum : SDNode<"ISD::STRICT_FMAXNUM", SDTFPBinOp, [SDNPHasChain, SDNPCommutative, SDNPAssociative]>; +def strict_fminimum : SDNode<"ISD::STRICT_FMINIMUM", + SDTFPBinOp, [SDNPHasChain, + SDNPCommutative, SDNPAssociative]>; +def strict_fmaximum : SDNode<"ISD::STRICT_FMAXIMUM", + SDTFPBinOp, [SDNPHasChain, + SDNPCommutative, SDNPAssociative]>; def strict_fpround : SDNode<"ISD::STRICT_FP_ROUND", SDTFPRoundOp, [SDNPHasChain]>; def strict_fpextend : SDNode<"ISD::STRICT_FP_EXTEND", @@ -538,6 +546,10 @@ def strict_fp_to_sint : SDNode<"ISD::STRICT_FP_TO_SINT", SDTFPToIntOp, [SDNPHasChain]>; def strict_fp_to_uint : SDNode<"ISD::STRICT_FP_TO_UINT", SDTFPToIntOp, [SDNPHasChain]>; +def strict_sint_to_fp : SDNode<"ISD::STRICT_SINT_TO_FP", + SDTIntToFPOp, [SDNPHasChain]>; +def strict_uint_to_fp : SDNode<"ISD::STRICT_UINT_TO_FP", + SDTIntToFPOp, [SDNPHasChain]>; def setcc : SDNode<"ISD::SETCC" , SDTSetCC>; def select : SDNode<"ISD::SELECT" , SDTSelect>; @@ -1380,6 +1392,12 @@ def any_fmaxnum : PatFrags<(ops node:$lhs, node:$rhs), def any_fminnum : PatFrags<(ops node:$lhs, node:$rhs), [(strict_fminnum node:$lhs, node:$rhs), (fminnum node:$lhs, node:$rhs)]>; +def any_fmaximum : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fmaximum node:$lhs, node:$rhs), + (fmaximum node:$lhs, node:$rhs)]>; +def any_fminimum : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fminimum node:$lhs, node:$rhs), + (fminimum node:$lhs, node:$rhs)]>; def any_fpround : PatFrags<(ops node:$src), [(strict_fpround node:$src), (fpround node:$src)]>; @@ -1398,6 +1416,12 @@ def any_fp_to_sint : PatFrags<(ops node:$src), def any_fp_to_uint : PatFrags<(ops node:$src), [(strict_fp_to_uint node:$src), (fp_to_uint node:$src)]>; +def any_sint_to_fp : PatFrags<(ops node:$src), + [(strict_sint_to_fp node:$src), + (sint_to_fp node:$src)]>; +def any_uint_to_fp : PatFrags<(ops node:$src), + [(strict_uint_to_fp node:$src), + (uint_to_fp node:$src)]>; multiclass binary_atomic_op_ord<SDNode atomic_op> { def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val), diff --git a/llvm/include/llvm/Transforms/CFGuard.h b/llvm/include/llvm/Transforms/CFGuard.h new file mode 100644 index 000000000000..86fcbc3c13e8 --- /dev/null +++ b/llvm/include/llvm/Transforms/CFGuard.h @@ -0,0 +1,26 @@ +//===-- CFGuard.h - CFGuard Transformations ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// Windows Control Flow Guard passes (/guard:cf). +//===---------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_CFGUARD_H +#define LLVM_TRANSFORMS_CFGUARD_H + +namespace llvm { + +class FunctionPass; + +/// Insert Control FLow Guard checks on indirect function calls. +FunctionPass *createCFGuardCheckPass(); + +/// Insert Control FLow Guard dispatches on indirect function calls. +FunctionPass *createCFGuardDispatchPass(); + +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Transforms/Coroutines.h b/llvm/include/llvm/Transforms/Coroutines.h index 9df3ec0f3ef4..ef05f549fbc1 100644 --- a/llvm/include/llvm/Transforms/Coroutines.h +++ b/llvm/include/llvm/Transforms/Coroutines.h @@ -20,17 +20,17 @@ class PassManagerBuilder; void addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder); /// Lower coroutine intrinsics that are not needed by later passes. -Pass *createCoroEarlyPass(); +Pass *createCoroEarlyLegacyPass(); /// Split up coroutines into multiple functions driving their state machines. -Pass *createCoroSplitPass(); +Pass *createCoroSplitLegacyPass(); /// Analyze coroutines use sites, devirtualize resume/destroy calls and elide /// heap allocation for coroutine frame where possible. -Pass *createCoroElidePass(); +Pass *createCoroElideLegacyPass(); /// Lower all remaining coroutine intrinsics. -Pass *createCoroCleanupPass(); +Pass *createCoroCleanupLegacyPass(); } diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h index 3dbe0fcd76ea..f7430a83e8d7 100644 --- a/llvm/include/llvm/Transforms/IPO/Attributor.h +++ b/llvm/include/llvm/Transforms/IPO/Attributor.h @@ -72,7 +72,8 @@ // - Define a class (transitively) inheriting from AbstractAttribute and one // (which could be the same) that (transitively) inherits from AbstractState. // For the latter, consider the already available BooleanState and -// IntegerState if they fit your needs, e.g., you require only a bit-encoding. +// {Inc,Dec,Bit}IntegerState if they fit your needs, e.g., you require only a +// number tracking or bit-encoding. // - Implement all pure methods. Also use overloading if the attribute is not // conforming with the "default" behavior: A (set of) LLVM-IR attribute(s) for // an argument, call site argument, function return value, or function. See @@ -104,6 +105,7 @@ #include "llvm/Analysis/MustExecute.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/ConstantRange.h" #include "llvm/IR/PassManager.h" namespace llvm { @@ -114,7 +116,7 @@ struct AAIsDead; class Function; -/// Simple enum class that forces the status to be spelled out explicitly. +/// Simple enum classes that forces properties to be spelled out explicitly. /// ///{ enum class ChangeStatus { @@ -124,6 +126,11 @@ enum class ChangeStatus { ChangeStatus operator|(ChangeStatus l, ChangeStatus r); ChangeStatus operator&(ChangeStatus l, ChangeStatus r); + +enum class DepClassTy { + REQUIRED, + OPTIONAL, +}; ///} /// Helper to describe and deal with positions in the LLVM-IR. @@ -251,22 +258,14 @@ struct IRPosition { /// sufficient to determine where arguments will be manifested. This is, so /// far, only the case for call site arguments as the value is not sufficient /// to pinpoint them. Instead, we can use the call site as an anchor. - /// - ///{ - Value &getAnchorValue() { + Value &getAnchorValue() const { assert(KindOrArgNo != IRP_INVALID && "Invalid position does not have an anchor value!"); return *AnchorVal; } - const Value &getAnchorValue() const { - return const_cast<IRPosition *>(this)->getAnchorValue(); - } - ///} /// Return the associated function, if any. - /// - ///{ - Function *getAssociatedFunction() { + Function *getAssociatedFunction() const { if (auto *CB = dyn_cast<CallBase>(AnchorVal)) return CB->getCalledFunction(); assert(KindOrArgNo != IRP_INVALID && @@ -280,32 +279,12 @@ struct IRPosition { return cast<Instruction>(V).getFunction(); return nullptr; } - const Function *getAssociatedFunction() const { - return const_cast<IRPosition *>(this)->getAssociatedFunction(); - } - ///} /// Return the associated argument, if any. - /// - ///{ - Argument *getAssociatedArgument() { - if (auto *Arg = dyn_cast<Argument>(&getAnchorValue())) - return Arg; - int ArgNo = getArgNo(); - if (ArgNo < 0) - return nullptr; - Function *AssociatedFn = getAssociatedFunction(); - if (!AssociatedFn || AssociatedFn->arg_size() <= unsigned(ArgNo)) - return nullptr; - return AssociatedFn->arg_begin() + ArgNo; - } - const Argument *getAssociatedArgument() const { - return const_cast<IRPosition *>(this)->getAssociatedArgument(); - } - ///} + Argument *getAssociatedArgument() const; /// Return true if the position refers to a function interface, that is the - /// function scope, the function return, or an argumnt. + /// function scope, the function return, or an argument. bool isFnInterfaceKind() const { switch (getPositionKind()) { case IRPosition::IRP_FUNCTION: @@ -318,9 +297,7 @@ struct IRPosition { } /// Return the Function surrounding the anchor value. - /// - ///{ - Function *getAnchorScope() { + Function *getAnchorScope() const { Value &V = getAnchorValue(); if (isa<Function>(V)) return &cast<Function>(V); @@ -330,15 +307,9 @@ struct IRPosition { return cast<Instruction>(V).getFunction(); return nullptr; } - const Function *getAnchorScope() const { - return const_cast<IRPosition *>(this)->getAnchorScope(); - } - ///} /// Return the context instruction, if any. - /// - ///{ - Instruction *getCtxI() { + Instruction *getCtxI() const { Value &V = getAnchorValue(); if (auto *I = dyn_cast<Instruction>(&V)) return I; @@ -350,15 +321,9 @@ struct IRPosition { return &(F->getEntryBlock().front()); return nullptr; } - const Instruction *getCtxI() const { - return const_cast<IRPosition *>(this)->getCtxI(); - } - ///} /// Return the value this abstract attribute is associated with. - /// - ///{ - Value &getAssociatedValue() { + Value &getAssociatedValue() const { assert(KindOrArgNo != IRP_INVALID && "Invalid position does not have an associated value!"); if (getArgNo() < 0 || isa<Argument>(AnchorVal)) @@ -366,10 +331,6 @@ struct IRPosition { assert(isa<CallBase>(AnchorVal) && "Expected a call base!"); return *cast<CallBase>(AnchorVal)->getArgOperand(getArgNo()); } - const Value &getAssociatedValue() const { - return const_cast<IRPosition *>(this)->getAssociatedValue(); - } - ///} /// Return the argument number of the associated value if it is an argument or /// call site argument, otherwise a negative value. @@ -428,8 +389,12 @@ struct IRPosition { /// single attribute of any kind in \p AKs, there are "subsuming" positions /// that could have an attribute as well. This method returns all attributes /// found in \p Attrs. + /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions, + /// e.g., the function position if this is an + /// argument position, should be ignored. void getAttrs(ArrayRef<Attribute::AttrKind> AKs, - SmallVectorImpl<Attribute> &Attrs) const; + SmallVectorImpl<Attribute> &Attrs, + bool IgnoreSubsumingPositions = false) const; /// Return the attribute of kind \p AK existing in the IR at this position. Attribute getAttr(Attribute::AttrKind AK) const { @@ -448,7 +413,7 @@ struct IRPosition { } /// Remove the attribute of kind \p AKs existing in the IR at this position. - void removeAttrs(ArrayRef<Attribute::AttrKind> AKs) { + void removeAttrs(ArrayRef<Attribute::AttrKind> AKs) const { if (getPositionKind() == IRP_INVALID || getPositionKind() == IRP_FLOAT) return; @@ -501,6 +466,7 @@ private: /// Verify internal invariants. void verify(); +protected: /// The value this position is anchored at. Value *AnchorVal; @@ -545,7 +511,7 @@ template <> struct DenseMapInfo<IRPosition> { /// - the argument of the callee (IRP_ARGUMENT), if known /// - the callee (IRP_FUNCTION), if known /// - the position the call site argument is associated with if it is not -/// anchored to the call site, e.g., if it is an arugment then the argument +/// anchored to the call site, e.g., if it is an argument then the argument /// (IRP_ARGUMENT) class SubsumingPositionIterator { SmallVector<IRPosition, 4> IRPositions; @@ -642,6 +608,12 @@ struct InformationCache { return AG.getAnalysis<AAManager>(F); } + /// Return the analysis result from a pass \p AP for function \p F. + template <typename AP> + typename AP::Result *getAnalysisResultForFunction(const Function &F) { + return AG.getAnalysis<AP>(F); + } + /// Return SCC size on call graph for function \p F. unsigned getSccSize(const Function &F) { if (!SccSizeOpt.hasValue()) @@ -723,7 +695,11 @@ struct Attributor { : InfoCache(InfoCache), DepRecomputeInterval(DepRecomputeInterval), Whitelist(Whitelist) {} - ~Attributor() { DeleteContainerPointers(AllAbstractAttributes); } + ~Attributor() { + DeleteContainerPointers(AllAbstractAttributes); + for (auto &It : ArgumentReplacementMap) + DeleteContainerPointers(It.second); + } /// Run the analyses until a fixpoint is reached or enforced (timeout). /// @@ -755,8 +731,10 @@ struct Attributor { /// the `Attributor::recordDependence` method. template <typename AAType> const AAType &getAAFor(const AbstractAttribute &QueryingAA, - const IRPosition &IRP, bool TrackDependence = true) { - return getOrCreateAAFor<AAType>(IRP, &QueryingAA, TrackDependence); + const IRPosition &IRP, bool TrackDependence = true, + DepClassTy DepClass = DepClassTy::REQUIRED) { + return getOrCreateAAFor<AAType>(IRP, &QueryingAA, TrackDependence, + DepClass); } /// Explicitly record a dependence from \p FromAA to \p ToAA, that is if @@ -766,10 +744,12 @@ struct Attributor { /// with the TrackDependence flag passed to the method set to false. This can /// be beneficial to avoid false dependences but it requires the users of /// `getAAFor` to explicitly record true dependences through this method. + /// The \p DepClass flag indicates if the dependence is striclty necessary. + /// That means for required dependences, if \p FromAA changes to an invalid + /// state, \p ToAA can be moved to a pessimistic fixpoint because it required + /// information from \p FromAA but none are available anymore. void recordDependence(const AbstractAttribute &FromAA, - const AbstractAttribute &ToAA) { - QueryMap[&FromAA].insert(const_cast<AbstractAttribute *>(&ToAA)); - } + const AbstractAttribute &ToAA, DepClassTy DepClass); /// Introduce a new abstract attribute into the fixpoint analysis. /// @@ -784,7 +764,7 @@ struct Attributor { "'AbstractAttribute'!"); // Put the attribute in the lookup map structure and the container we use to // keep track of all attributes. - IRPosition &IRP = AA.getIRPosition(); + const IRPosition &IRP = AA.getIRPosition(); auto &KindToAbstractAttributeMap = AAMap[IRP]; assert(!KindToAbstractAttributeMap.count(&AAType::ID) && "Attribute already in map!"); @@ -825,10 +805,80 @@ struct Attributor { identifyDefaultAbstractAttributes(const_cast<Function &>(F)); } - /// Record that \p I is deleted after information was manifested. + /// Record that \p U is to be replaces with \p NV after information was + /// manifested. This also triggers deletion of trivially dead istructions. + bool changeUseAfterManifest(Use &U, Value &NV) { + Value *&V = ToBeChangedUses[&U]; + if (V && (V->stripPointerCasts() == NV.stripPointerCasts() || + isa_and_nonnull<UndefValue>(V))) + return false; + assert((!V || V == &NV || isa<UndefValue>(NV)) && + "Use was registered twice for replacement with different values!"); + V = &NV; + return true; + } + + /// Helper function to replace all uses of \p V with \p NV. Return true if + /// there is any change. + bool changeValueAfterManifest(Value &V, Value &NV) { + bool Changed = false; + for (auto &U : V.uses()) + Changed |= changeUseAfterManifest(U, NV); + + return Changed; + } + + /// Get pointer operand of memory accessing instruction. If \p I is + /// not a memory accessing instruction, return nullptr. If \p AllowVolatile, + /// is set to false and the instruction is volatile, return nullptr. + static const Value *getPointerOperand(const Instruction *I, + bool AllowVolatile) { + if (auto *LI = dyn_cast<LoadInst>(I)) { + if (!AllowVolatile && LI->isVolatile()) + return nullptr; + return LI->getPointerOperand(); + } + + if (auto *SI = dyn_cast<StoreInst>(I)) { + if (!AllowVolatile && SI->isVolatile()) + return nullptr; + return SI->getPointerOperand(); + } + + if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I)) { + if (!AllowVolatile && CXI->isVolatile()) + return nullptr; + return CXI->getPointerOperand(); + } + + if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) { + if (!AllowVolatile && RMWI->isVolatile()) + return nullptr; + return RMWI->getPointerOperand(); + } + + return nullptr; + } + + /// Record that \p I is to be replaced with `unreachable` after information + /// was manifested. + void changeToUnreachableAfterManifest(Instruction *I) { + ToBeChangedToUnreachableInsts.insert(I); + } + + /// Record that \p II has at least one dead successor block. This information + /// is used, e.g., to replace \p II with a call, after information was + /// manifested. + void registerInvokeWithDeadSuccessor(InvokeInst &II) { + InvokeWithDeadSuccessor.push_back(&II); + } + + /// Record that \p I is deleted after information was manifested. This also + /// triggers deletion of trivially dead istructions. void deleteAfterManifest(Instruction &I) { ToBeDeletedInsts.insert(&I); } - /// Record that \p BB is deleted after information was manifested. + /// Record that \p BB is deleted after information was manifested. This also + /// triggers deletion of trivially dead istructions. void deleteAfterManifest(BasicBlock &BB) { ToBeDeletedBlocks.insert(&BB); } /// Record that \p F is deleted after information was manifested. @@ -839,6 +889,104 @@ struct Attributor { /// If \p LivenessAA is not provided it is queried. bool isAssumedDead(const AbstractAttribute &AA, const AAIsDead *LivenessAA); + /// Check \p Pred on all (transitive) uses of \p V. + /// + /// This method will evaluate \p Pred on all (transitive) uses of the + /// associated value and return true if \p Pred holds every time. + bool checkForAllUses(const function_ref<bool(const Use &, bool &)> &Pred, + const AbstractAttribute &QueryingAA, const Value &V); + + /// Helper struct used in the communication between an abstract attribute (AA) + /// that wants to change the signature of a function and the Attributor which + /// applies the changes. The struct is partially initialized with the + /// information from the AA (see the constructor). All other members are + /// provided by the Attributor prior to invoking any callbacks. + struct ArgumentReplacementInfo { + /// Callee repair callback type + /// + /// The function repair callback is invoked once to rewire the replacement + /// arguments in the body of the new function. The argument replacement info + /// is passed, as build from the registerFunctionSignatureRewrite call, as + /// well as the replacement function and an iteratore to the first + /// replacement argument. + using CalleeRepairCBTy = std::function<void( + const ArgumentReplacementInfo &, Function &, Function::arg_iterator)>; + + /// Abstract call site (ACS) repair callback type + /// + /// The abstract call site repair callback is invoked once on every abstract + /// call site of the replaced function (\see ReplacedFn). The callback needs + /// to provide the operands for the call to the new replacement function. + /// The number and type of the operands appended to the provided vector + /// (second argument) is defined by the number and types determined through + /// the replacement type vector (\see ReplacementTypes). The first argument + /// is the ArgumentReplacementInfo object registered with the Attributor + /// through the registerFunctionSignatureRewrite call. + using ACSRepairCBTy = + std::function<void(const ArgumentReplacementInfo &, AbstractCallSite, + SmallVectorImpl<Value *> &)>; + + /// Simple getters, see the corresponding members for details. + ///{ + + Attributor &getAttributor() const { return A; } + const Function &getReplacedFn() const { return ReplacedFn; } + const Argument &getReplacedArg() const { return ReplacedArg; } + unsigned getNumReplacementArgs() const { return ReplacementTypes.size(); } + const SmallVectorImpl<Type *> &getReplacementTypes() const { + return ReplacementTypes; + } + + ///} + + private: + /// Constructor that takes the argument to be replaced, the types of + /// the replacement arguments, as well as callbacks to repair the call sites + /// and new function after the replacement happened. + ArgumentReplacementInfo(Attributor &A, Argument &Arg, + ArrayRef<Type *> ReplacementTypes, + CalleeRepairCBTy &&CalleeRepairCB, + ACSRepairCBTy &&ACSRepairCB) + : A(A), ReplacedFn(*Arg.getParent()), ReplacedArg(Arg), + ReplacementTypes(ReplacementTypes.begin(), ReplacementTypes.end()), + CalleeRepairCB(std::move(CalleeRepairCB)), + ACSRepairCB(std::move(ACSRepairCB)) {} + + /// Reference to the attributor to allow access from the callbacks. + Attributor &A; + + /// The "old" function replaced by ReplacementFn. + const Function &ReplacedFn; + + /// The "old" argument replaced by new ones defined via ReplacementTypes. + const Argument &ReplacedArg; + + /// The types of the arguments replacing ReplacedArg. + const SmallVector<Type *, 8> ReplacementTypes; + + /// Callee repair callback, see CalleeRepairCBTy. + const CalleeRepairCBTy CalleeRepairCB; + + /// Abstract call site (ACS) repair callback, see ACSRepairCBTy. + const ACSRepairCBTy ACSRepairCB; + + /// Allow access to the private members from the Attributor. + friend struct Attributor; + }; + + /// Register a rewrite for a function signature. + /// + /// The argument \p Arg is replaced with new ones defined by the number, + /// order, and types in \p ReplacementTypes. The rewiring at the call sites is + /// done through \p ACSRepairCB and at the callee site through + /// \p CalleeRepairCB. + /// + /// \returns True, if the replacement was registered, false otherwise. + bool registerFunctionSignatureRewrite( + Argument &Arg, ArrayRef<Type *> ReplacementTypes, + ArgumentReplacementInfo::CalleeRepairCBTy &&CalleeRepairCB, + ArgumentReplacementInfo::ACSRepairCBTy &&ACSRepairCB); + /// Check \p Pred on all function call sites. /// /// This method will evaluate \p Pred on call sites and return @@ -913,7 +1061,8 @@ private: template <typename AAType> const AAType &getOrCreateAAFor(const IRPosition &IRP, const AbstractAttribute *QueryingAA = nullptr, - bool TrackDependence = false) { + bool TrackDependence = false, + DepClassTy DepClass = DepClassTy::OPTIONAL) { if (const AAType *AAPtr = lookupAAFor<AAType>(IRP, QueryingAA, TrackDependence)) return *AAPtr; @@ -941,7 +1090,8 @@ private: AA.update(*this); if (TrackDependence && AA.getState().isValidState()) - QueryMap[&AA].insert(const_cast<AbstractAttribute *>(QueryingAA)); + recordDependence(AA, const_cast<AbstractAttribute &>(*QueryingAA), + DepClass); return AA; } @@ -949,7 +1099,8 @@ private: template <typename AAType> const AAType *lookupAAFor(const IRPosition &IRP, const AbstractAttribute *QueryingAA = nullptr, - bool TrackDependence = false) { + bool TrackDependence = false, + DepClassTy DepClass = DepClassTy::OPTIONAL) { static_assert(std::is_base_of<AbstractAttribute, AAType>::value, "Cannot query an attribute with a type not derived from " "'AbstractAttribute'!"); @@ -963,12 +1114,18 @@ private: KindToAbstractAttributeMap.lookup(&AAType::ID))) { // Do not register a dependence on an attribute with an invalid state. if (TrackDependence && AA->getState().isValidState()) - QueryMap[AA].insert(const_cast<AbstractAttribute *>(QueryingAA)); + recordDependence(*AA, const_cast<AbstractAttribute &>(*QueryingAA), + DepClass); return AA; } return nullptr; } + /// Apply all requested function signature rewrites + /// (\see registerFunctionSignatureRewrite) and return Changed if the module + /// was altered. + ChangeStatus rewriteFunctionSignatures(); + /// The set of all abstract attributes. ///{ using AAVector = SmallVector<AbstractAttribute *, 64>; @@ -987,14 +1144,29 @@ private: /// A map from abstract attributes to the ones that queried them through calls /// to the getAAFor<...>(...) method. ///{ - using QueryMapTy = - MapVector<const AbstractAttribute *, SetVector<AbstractAttribute *>>; + struct QueryMapValueTy { + /// Set of abstract attributes which were used but not necessarily required + /// for a potential optimistic state. + SetVector<AbstractAttribute *> OptionalAAs; + + /// Set of abstract attributes which were used and which were necessarily + /// required for any potential optimistic state. + SetVector<AbstractAttribute *> RequiredAAs; + }; + using QueryMapTy = MapVector<const AbstractAttribute *, QueryMapValueTy>; QueryMapTy QueryMap; ///} + /// Map to remember all requested signature changes (= argument replacements). + DenseMap<Function *, SmallVector<ArgumentReplacementInfo *, 8>> + ArgumentReplacementMap; + /// The information cache that holds pre-processed (LLVM-IR) information. InformationCache &InfoCache; + /// Set if the attribute currently updated did query a non-fix attribute. + bool QueriedNonFixAA; + /// Number of iterations until the dependences between abstract attributes are /// recomputed. const unsigned DepRecomputeInterval; @@ -1005,6 +1177,16 @@ private: /// A set to remember the functions we already assume to be live and visited. DenseSet<const Function *> VisitedFunctions; + /// Uses we replace with a new value after manifest is done. We will remove + /// then trivially dead instructions as well. + DenseMap<Use *, Value *> ToBeChangedUses; + + /// Instructions we replace with `unreachable` insts after manifest is done. + SmallDenseSet<WeakVH, 16> ToBeChangedToUnreachableInsts; + + /// Invoke instructions with at least a single dead successor block. + SmallVector<WeakVH, 16> InvokeWithDeadSuccessor; + /// Functions, blocks, and instructions we delete after manifest is done. /// ///{ @@ -1027,9 +1209,10 @@ private: /// /// All methods need to be implemented by the subclass. For the common use case, /// a single boolean state or a bit-encoded state, the BooleanState and -/// IntegerState classes are already provided. An abstract attribute can inherit -/// from them to get the abstract state interface and additional methods to -/// directly modify the state based if needed. See the class comments for help. +/// {Inc,Dec,Bit}IntegerState classes are already provided. An abstract +/// attribute can inherit from them to get the abstract state interface and +/// additional methods to directly modify the state based if needed. See the +/// class comments for help. struct AbstractState { virtual ~AbstractState() {} @@ -1068,15 +1251,15 @@ struct AbstractState { /// force/inidicate a fixpoint. If an optimistic one is indicated, the known /// state will catch up with the assumed one, for a pessimistic fixpoint it is /// the other way around. -struct IntegerState : public AbstractState { - /// Underlying integer type, we assume 32 bits to be enough. - using base_t = uint32_t; +template <typename base_ty, base_ty BestState, base_ty WorstState> +struct IntegerStateBase : public AbstractState { + using base_t = base_ty; - /// Initialize the (best) state. - IntegerState(base_t BestState = ~0) : Assumed(BestState) {} + /// Return the best possible representable state. + static constexpr base_t getBestState() { return BestState; } /// Return the worst possible representable state. - static constexpr base_t getWorstState() { return 0; } + static constexpr base_t getWorstState() { return WorstState; } /// See AbstractState::isValidState() /// NOTE: For now we simply pretend that the worst possible state is invalid. @@ -1103,117 +1286,338 @@ struct IntegerState : public AbstractState { /// Return the assumed state encoding. base_t getAssumed() const { return Assumed; } + /// Equality for IntegerStateBase. + bool + operator==(const IntegerStateBase<base_t, BestState, WorstState> &R) const { + return this->getAssumed() == R.getAssumed() && + this->getKnown() == R.getKnown(); + } + + /// Inequality for IntegerStateBase. + bool + operator!=(const IntegerStateBase<base_t, BestState, WorstState> &R) const { + return !(*this == R); + } + + /// "Clamp" this state with \p R. The result is subtype dependent but it is + /// intended that only information assumed in both states will be assumed in + /// this one afterwards. + void operator^=(const IntegerStateBase<base_t, BestState, WorstState> &R) { + handleNewAssumedValue(R.getAssumed()); + } + + void operator|=(const IntegerStateBase<base_t, BestState, WorstState> &R) { + joinOR(R.getAssumed(), R.getKnown()); + } + + void operator&=(const IntegerStateBase<base_t, BestState, WorstState> &R) { + joinAND(R.getAssumed(), R.getKnown()); + } + +protected: + /// Handle a new assumed value \p Value. Subtype dependent. + virtual void handleNewAssumedValue(base_t Value) = 0; + + /// Handle a new known value \p Value. Subtype dependent. + virtual void handleNewKnownValue(base_t Value) = 0; + + /// Handle a value \p Value. Subtype dependent. + virtual void joinOR(base_t AssumedValue, base_t KnownValue) = 0; + + /// Handle a new assumed value \p Value. Subtype dependent. + virtual void joinAND(base_t AssumedValue, base_t KnownValue) = 0; + + /// The known state encoding in an integer of type base_t. + base_t Known = getWorstState(); + + /// The assumed state encoding in an integer of type base_t. + base_t Assumed = getBestState(); +}; + +/// Specialization of the integer state for a bit-wise encoding. +template <typename base_ty = uint32_t, base_ty BestState = ~base_ty(0), + base_ty WorstState = 0> +struct BitIntegerState + : public IntegerStateBase<base_ty, BestState, WorstState> { + using base_t = base_ty; + /// Return true if the bits set in \p BitsEncoding are "known bits". bool isKnown(base_t BitsEncoding) const { - return (Known & BitsEncoding) == BitsEncoding; + return (this->Known & BitsEncoding) == BitsEncoding; } /// Return true if the bits set in \p BitsEncoding are "assumed bits". bool isAssumed(base_t BitsEncoding) const { - return (Assumed & BitsEncoding) == BitsEncoding; + return (this->Assumed & BitsEncoding) == BitsEncoding; } /// Add the bits in \p BitsEncoding to the "known bits". - IntegerState &addKnownBits(base_t Bits) { + BitIntegerState &addKnownBits(base_t Bits) { // Make sure we never miss any "known bits". - Assumed |= Bits; - Known |= Bits; + this->Assumed |= Bits; + this->Known |= Bits; return *this; } /// Remove the bits in \p BitsEncoding from the "assumed bits" if not known. - IntegerState &removeAssumedBits(base_t BitsEncoding) { - // Make sure we never loose any "known bits". - Assumed = (Assumed & ~BitsEncoding) | Known; - return *this; + BitIntegerState &removeAssumedBits(base_t BitsEncoding) { + return intersectAssumedBits(~BitsEncoding); } /// Remove the bits in \p BitsEncoding from the "known bits". - IntegerState &removeKnownBits(base_t BitsEncoding) { - Known = (Known & ~BitsEncoding); + BitIntegerState &removeKnownBits(base_t BitsEncoding) { + this->Known = (this->Known & ~BitsEncoding); return *this; } /// Keep only "assumed bits" also set in \p BitsEncoding but all known ones. - IntegerState &intersectAssumedBits(base_t BitsEncoding) { + BitIntegerState &intersectAssumedBits(base_t BitsEncoding) { // Make sure we never loose any "known bits". - Assumed = (Assumed & BitsEncoding) | Known; + this->Assumed = (this->Assumed & BitsEncoding) | this->Known; return *this; } +private: + void handleNewAssumedValue(base_t Value) override { + intersectAssumedBits(Value); + } + void handleNewKnownValue(base_t Value) override { addKnownBits(Value); } + void joinOR(base_t AssumedValue, base_t KnownValue) override { + this->Known |= KnownValue; + this->Assumed |= AssumedValue; + } + void joinAND(base_t AssumedValue, base_t KnownValue) override { + this->Known &= KnownValue; + this->Assumed &= AssumedValue; + } +}; + +/// Specialization of the integer state for an increasing value, hence ~0u is +/// the best state and 0 the worst. +template <typename base_ty = uint32_t, base_ty BestState = ~base_ty(0), + base_ty WorstState = 0> +struct IncIntegerState + : public IntegerStateBase<base_ty, BestState, WorstState> { + using base_t = base_ty; + /// Take minimum of assumed and \p Value. - IntegerState &takeAssumedMinimum(base_t Value) { + IncIntegerState &takeAssumedMinimum(base_t Value) { // Make sure we never loose "known value". - Assumed = std::max(std::min(Assumed, Value), Known); + this->Assumed = std::max(std::min(this->Assumed, Value), this->Known); return *this; } /// Take maximum of known and \p Value. - IntegerState &takeKnownMaximum(base_t Value) { + IncIntegerState &takeKnownMaximum(base_t Value) { // Make sure we never loose "known value". - Assumed = std::max(Value, Assumed); - Known = std::max(Value, Known); + this->Assumed = std::max(Value, this->Assumed); + this->Known = std::max(Value, this->Known); return *this; } - /// Equality for IntegerState. - bool operator==(const IntegerState &R) const { - return this->getAssumed() == R.getAssumed() && - this->getKnown() == R.getKnown(); +private: + void handleNewAssumedValue(base_t Value) override { + takeAssumedMinimum(Value); + } + void handleNewKnownValue(base_t Value) override { takeKnownMaximum(Value); } + void joinOR(base_t AssumedValue, base_t KnownValue) override { + this->Known = std::max(this->Known, KnownValue); + this->Assumed = std::max(this->Assumed, AssumedValue); + } + void joinAND(base_t AssumedValue, base_t KnownValue) override { + this->Known = std::min(this->Known, KnownValue); + this->Assumed = std::min(this->Assumed, AssumedValue); } +}; - /// Inequality for IntegerState. - bool operator!=(const IntegerState &R) const { return !(*this == R); } +/// Specialization of the integer state for a decreasing value, hence 0 is the +/// best state and ~0u the worst. +template <typename base_ty = uint32_t> +struct DecIntegerState : public IntegerStateBase<base_ty, 0, ~base_ty(0)> { + using base_t = base_ty; - /// "Clamp" this state with \p R. The result is the minimum of the assumed - /// information but not less than what was known before. - /// - /// TODO: Consider replacing the operator with a call or using it only when - /// we can also take the maximum of the known information, thus when - /// \p R is not dependent on additional assumed state. - IntegerState operator^=(const IntegerState &R) { - takeAssumedMinimum(R.Assumed); + /// Take maximum of assumed and \p Value. + DecIntegerState &takeAssumedMaximum(base_t Value) { + // Make sure we never loose "known value". + this->Assumed = std::min(std::max(this->Assumed, Value), this->Known); return *this; } - /// "Clamp" this state with \p R. The result is the maximum of the known - /// information but not more than what was assumed before. - IntegerState operator+=(const IntegerState &R) { - takeKnownMaximum(R.Known); + /// Take minimum of known and \p Value. + DecIntegerState &takeKnownMinimum(base_t Value) { + // Make sure we never loose "known value". + this->Assumed = std::min(Value, this->Assumed); + this->Known = std::min(Value, this->Known); return *this; } - /// Make this the minimum, known and assumed, of this state and \p R. - IntegerState operator&=(const IntegerState &R) { - Known = std::min(Known, R.Known); - Assumed = std::min(Assumed, R.Assumed); - return *this; +private: + void handleNewAssumedValue(base_t Value) override { + takeAssumedMaximum(Value); + } + void handleNewKnownValue(base_t Value) override { takeKnownMinimum(Value); } + void joinOR(base_t AssumedValue, base_t KnownValue) override { + this->Assumed = std::min(this->Assumed, KnownValue); + this->Assumed = std::min(this->Assumed, AssumedValue); + } + void joinAND(base_t AssumedValue, base_t KnownValue) override { + this->Assumed = std::max(this->Assumed, KnownValue); + this->Assumed = std::max(this->Assumed, AssumedValue); } +}; - /// Make this the maximum, known and assumed, of this state and \p R. - IntegerState operator|=(const IntegerState &R) { - Known = std::max(Known, R.Known); - Assumed = std::max(Assumed, R.Assumed); - return *this; +/// Simple wrapper for a single bit (boolean) state. +struct BooleanState : public IntegerStateBase<bool, 1, 0> { + using base_t = IntegerStateBase::base_t; + + /// Set the assumed value to \p Value but never below the known one. + void setAssumed(bool Value) { Assumed &= (Known | Value); } + + /// Set the known and asssumed value to \p Value. + void setKnown(bool Value) { + Known |= Value; + Assumed |= Value; } -private: - /// The known state encoding in an integer of type base_t. - base_t Known = getWorstState(); + /// Return true if the state is assumed to hold. + bool isAssumed() const { return getAssumed(); } - /// The assumed state encoding in an integer of type base_t. - base_t Assumed; -}; + /// Return true if the state is known to hold. + bool isKnown() const { return getKnown(); } -/// Simple wrapper for a single bit (boolean) state. -struct BooleanState : public IntegerState { - BooleanState() : IntegerState(1){}; +private: + void handleNewAssumedValue(base_t Value) override { + if (!Value) + Assumed = Known; + } + void handleNewKnownValue(base_t Value) override { + if (Value) + Known = (Assumed = Value); + } + void joinOR(base_t AssumedValue, base_t KnownValue) override { + Known |= KnownValue; + Assumed |= AssumedValue; + } + void joinAND(base_t AssumedValue, base_t KnownValue) override { + Known &= KnownValue; + Assumed &= AssumedValue; + } }; +/// State for an integer range. +struct IntegerRangeState : public AbstractState { + + /// Bitwidth of the associated value. + uint32_t BitWidth; + + /// State representing assumed range, initially set to empty. + ConstantRange Assumed; + + /// State representing known range, initially set to [-inf, inf]. + ConstantRange Known; + + IntegerRangeState(uint32_t BitWidth) + : BitWidth(BitWidth), Assumed(ConstantRange::getEmpty(BitWidth)), + Known(ConstantRange::getFull(BitWidth)) {} + + /// Return the worst possible representable state. + static ConstantRange getWorstState(uint32_t BitWidth) { + return ConstantRange::getFull(BitWidth); + } + + /// Return the best possible representable state. + static ConstantRange getBestState(uint32_t BitWidth) { + return ConstantRange::getEmpty(BitWidth); + } + + /// Return associated values' bit width. + uint32_t getBitWidth() const { return BitWidth; } + + /// See AbstractState::isValidState() + bool isValidState() const override { + return BitWidth > 0 && !Assumed.isFullSet(); + } + + /// See AbstractState::isAtFixpoint() + bool isAtFixpoint() const override { return Assumed == Known; } + + /// See AbstractState::indicateOptimisticFixpoint(...) + ChangeStatus indicateOptimisticFixpoint() override { + Known = Assumed; + return ChangeStatus::CHANGED; + } + + /// See AbstractState::indicatePessimisticFixpoint(...) + ChangeStatus indicatePessimisticFixpoint() override { + Assumed = Known; + return ChangeStatus::CHANGED; + } + + /// Return the known state encoding + ConstantRange getKnown() const { return Known; } + + /// Return the assumed state encoding. + ConstantRange getAssumed() const { return Assumed; } + + /// Unite assumed range with the passed state. + void unionAssumed(const ConstantRange &R) { + // Don't loose a known range. + Assumed = Assumed.unionWith(R).intersectWith(Known); + } + + /// See IntegerRangeState::unionAssumed(..). + void unionAssumed(const IntegerRangeState &R) { + unionAssumed(R.getAssumed()); + } + + /// Unite known range with the passed state. + void unionKnown(const ConstantRange &R) { + // Don't loose a known range. + Known = Known.unionWith(R); + Assumed = Assumed.unionWith(Known); + } + + /// See IntegerRangeState::unionKnown(..). + void unionKnown(const IntegerRangeState &R) { unionKnown(R.getKnown()); } + + /// Intersect known range with the passed state. + void intersectKnown(const ConstantRange &R) { + Assumed = Assumed.intersectWith(R); + Known = Known.intersectWith(R); + } + + /// See IntegerRangeState::intersectKnown(..). + void intersectKnown(const IntegerRangeState &R) { + intersectKnown(R.getKnown()); + } + + /// Equality for IntegerRangeState. + bool operator==(const IntegerRangeState &R) const { + return getAssumed() == R.getAssumed() && getKnown() == R.getKnown(); + } + + /// "Clamp" this state with \p R. The result is subtype dependent but it is + /// intended that only information assumed in both states will be assumed in + /// this one afterwards. + IntegerRangeState operator^=(const IntegerRangeState &R) { + // NOTE: `^=` operator seems like `intersect` but in this case, we need to + // take `union`. + unionAssumed(R); + return *this; + } + + IntegerRangeState operator&=(const IntegerRangeState &R) { + // NOTE: `&=` operator seems like `intersect` but in this case, we need to + // take `union`. + unionKnown(R); + unionAssumed(R); + return *this; + } +}; /// Helper struct necessary as the modular build fails if the virtual method /// IRAttribute::manifest is defined in the Attributor.cpp. struct IRAttributeManifest { - static ChangeStatus manifestAttrs(Attributor &A, IRPosition &IRP, + static ChangeStatus manifestAttrs(Attributor &A, const IRPosition &IRP, const ArrayRef<Attribute> &DeducedAttrs); }; @@ -1238,12 +1642,12 @@ struct IRAttribute : public IRPosition, public Base { /// See AbstractAttribute::initialize(...). virtual void initialize(Attributor &A) override { - if (hasAttr(getAttrKind())) { + const IRPosition &IRP = this->getIRPosition(); + if (isa<UndefValue>(IRP.getAssociatedValue()) || hasAttr(getAttrKind())) { this->getState().indicateOptimisticFixpoint(); return; } - const IRPosition &IRP = this->getIRPosition(); bool IsFnInterface = IRP.isFnInterfaceKind(); const Function *FnScope = IRP.getAnchorScope(); // TODO: Not all attributes require an exact definition. Find a way to @@ -1259,6 +1663,8 @@ struct IRAttribute : public IRPosition, public Base { /// See AbstractAttribute::manifest(...). ChangeStatus manifest(Attributor &A) override { + if (isa<UndefValue>(getIRPosition().getAssociatedValue())) + return ChangeStatus::UNCHANGED; SmallVector<Attribute, 4> DeducedAttrs; getDeducedAttributes(getAnchorValue().getContext(), DeducedAttrs); return IRAttributeManifest::manifestAttrs(A, getIRPosition(), DeducedAttrs); @@ -1274,11 +1680,7 @@ struct IRAttribute : public IRPosition, public Base { } /// Return an IR position, see struct IRPosition. - /// - ///{ - IRPosition &getIRPosition() override { return *this; } const IRPosition &getIRPosition() const override { return *this; } - ///} }; /// Base struct for all "concrete attribute" deductions. @@ -1383,9 +1785,6 @@ protected: /// add statistics for them. virtual void trackStatistics() const = 0; - /// Return an IR position, see struct IRPosition. - virtual IRPosition &getIRPosition() = 0; - /// The actual update/transfer function which has to be implemented by the /// derived classes. /// @@ -1404,7 +1803,11 @@ raw_ostream &operator<<(raw_ostream &OS, ChangeStatus S); raw_ostream &operator<<(raw_ostream &OS, IRPosition::Kind); raw_ostream &operator<<(raw_ostream &OS, const IRPosition &); raw_ostream &operator<<(raw_ostream &OS, const AbstractState &State); -raw_ostream &operator<<(raw_ostream &OS, const IntegerState &S); +template <typename base_ty, base_ty BestState, base_ty WorstState> +raw_ostream & +operator<<(raw_ostream &OS, + const IntegerStateBase<base_ty, BestState, WorstState> &State); +raw_ostream &operator<<(raw_ostream &OS, const IntegerRangeState &State); ///} struct AttributorPass : public PassInfoMixin<AttributorPass> { @@ -1550,6 +1953,66 @@ struct AAWillReturn static const char ID; }; +/// An abstract attribute for undefined behavior. +struct AAUndefinedBehavior + : public StateWrapper<BooleanState, AbstractAttribute>, + public IRPosition { + AAUndefinedBehavior(const IRPosition &IRP) : IRPosition(IRP) {} + + /// Return true if "undefined behavior" is assumed. + bool isAssumedToCauseUB() const { return getAssumed(); } + + /// Return true if "undefined behavior" is assumed for a specific instruction. + virtual bool isAssumedToCauseUB(Instruction *I) const = 0; + + /// Return true if "undefined behavior" is known. + bool isKnownToCauseUB() const { return getKnown(); } + + /// Return true if "undefined behavior" is known for a specific instruction. + virtual bool isKnownToCauseUB(Instruction *I) const = 0; + + /// Return an IR position, see struct IRPosition. + const IRPosition &getIRPosition() const override { return *this; } + + /// Create an abstract attribute view for the position \p IRP. + static AAUndefinedBehavior &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + +/// An abstract interface to determine reachability of point A to B. +struct AAReachability : public StateWrapper<BooleanState, AbstractAttribute>, + public IRPosition { + AAReachability(const IRPosition &IRP) : IRPosition(IRP) {} + + /// Returns true if 'From' instruction is assumed to reach, 'To' instruction. + /// Users should provide two positions they are interested in, and the class + /// determines (and caches) reachability. + bool isAssumedReachable(const Instruction *From, + const Instruction *To) const { + return true; + } + + /// Returns true if 'From' instruction is known to reach, 'To' instruction. + /// Users should provide two positions they are interested in, and the class + /// determines (and caches) reachability. + bool isKnownReachable(const Instruction *From, const Instruction *To) const { + return true; + } + + /// Return an IR position, see struct IRPosition. + const IRPosition &getIRPosition() const override { return *this; } + + /// Create an abstract attribute view for the position \p IRP. + static AAReachability &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// Unique ID (due to the unique address) + static const char ID; +}; + /// An abstract interface for all noalias attributes. struct AANoAlias : public IRAttribute<Attribute::NoAlias, @@ -1612,6 +2075,9 @@ struct AAIsDead : public StateWrapper<BooleanState, AbstractAttribute>, public IRPosition { AAIsDead(const IRPosition &IRP) : IRPosition(IRP) {} + /// Returns true if the underlying value is assumed dead. + virtual bool isAssumedDead() const = 0; + /// Returns true if \p BB is assumed dead. virtual bool isAssumedDead(const BasicBlock *BB) const = 0; @@ -1639,11 +2105,7 @@ struct AAIsDead : public StateWrapper<BooleanState, AbstractAttribute>, } /// Return an IR position, see struct IRPosition. - /// - ///{ - IRPosition &getIRPosition() override { return *this; } const IRPosition &getIRPosition() const override { return *this; } - ///} /// Create an abstract attribute view for the position \p IRP. static AAIsDead &createForPosition(const IRPosition &IRP, Attributor &A); @@ -1656,7 +2118,45 @@ struct AAIsDead : public StateWrapper<BooleanState, AbstractAttribute>, struct DerefState : AbstractState { /// State representing for dereferenceable bytes. - IntegerState DerefBytesState; + IncIntegerState<> DerefBytesState; + + /// Map representing for accessed memory offsets and sizes. + /// A key is Offset and a value is size. + /// If there is a load/store instruction something like, + /// p[offset] = v; + /// (offset, sizeof(v)) will be inserted to this map. + /// std::map is used because we want to iterate keys in ascending order. + std::map<int64_t, uint64_t> AccessedBytesMap; + + /// Helper function to calculate dereferenceable bytes from current known + /// bytes and accessed bytes. + /// + /// int f(int *A){ + /// *A = 0; + /// *(A+2) = 2; + /// *(A+1) = 1; + /// *(A+10) = 10; + /// } + /// ``` + /// In that case, AccessedBytesMap is `{0:4, 4:4, 8:4, 40:4}`. + /// AccessedBytesMap is std::map so it is iterated in accending order on + /// key(Offset). So KnownBytes will be updated like this: + /// + /// |Access | KnownBytes + /// |(0, 4)| 0 -> 4 + /// |(4, 4)| 4 -> 8 + /// |(8, 4)| 8 -> 12 + /// |(40, 4) | 12 (break) + void computeKnownDerefBytesFromAccessedMap() { + int64_t KnownBytes = DerefBytesState.getKnown(); + for (auto &Access : AccessedBytesMap) { + if (KnownBytes < Access.first) + break; + KnownBytes = std::max(KnownBytes, Access.first + (int64_t)Access.second); + } + + DerefBytesState.takeKnownMaximum(KnownBytes); + } /// State representing that whether the value is globaly dereferenceable. BooleanState GlobalState; @@ -1687,6 +2187,9 @@ struct DerefState : AbstractState { /// Update known dereferenceable bytes. void takeKnownDerefBytesMaximum(uint64_t Bytes) { DerefBytesState.takeKnownMaximum(Bytes); + + // Known bytes might increase. + computeKnownDerefBytesFromAccessedMap(); } /// Update assumed dereferenceable bytes. @@ -1694,37 +2197,38 @@ struct DerefState : AbstractState { DerefBytesState.takeAssumedMinimum(Bytes); } + /// Add accessed bytes to the map. + void addAccessedBytes(int64_t Offset, uint64_t Size) { + AccessedBytesMap[Offset] = std::max(AccessedBytesMap[Offset], Size); + + // Known bytes might increase. + computeKnownDerefBytesFromAccessedMap(); + } + /// Equality for DerefState. bool operator==(const DerefState &R) { return this->DerefBytesState == R.DerefBytesState && this->GlobalState == R.GlobalState; } - /// Inequality for IntegerState. + /// Inequality for DerefState. bool operator!=(const DerefState &R) { return !(*this == R); } - /// See IntegerState::operator^= + /// See IntegerStateBase::operator^= DerefState operator^=(const DerefState &R) { DerefBytesState ^= R.DerefBytesState; GlobalState ^= R.GlobalState; return *this; } - /// See IntegerState::operator+= - DerefState operator+=(const DerefState &R) { - DerefBytesState += R.DerefBytesState; - GlobalState += R.GlobalState; - return *this; - } - - /// See IntegerState::operator&= + /// See IntegerStateBase::operator&= DerefState operator&=(const DerefState &R) { DerefBytesState &= R.DerefBytesState; GlobalState &= R.GlobalState; return *this; } - /// See IntegerState::operator|= + /// See IntegerStateBase::operator|= DerefState operator|=(const DerefState &R) { DerefBytesState |= R.DerefBytesState; GlobalState |= R.GlobalState; @@ -1777,16 +2281,18 @@ struct AADereferenceable static const char ID; }; +using AAAlignmentStateType = + IncIntegerState<uint32_t, /* maximal alignment */ 1U << 29, 0>; /// An abstract interface for all align attributes. -struct AAAlign - : public IRAttribute<Attribute::Alignment, - StateWrapper<IntegerState, AbstractAttribute>> { +struct AAAlign : public IRAttribute< + Attribute::Alignment, + StateWrapper<AAAlignmentStateType, AbstractAttribute>> { AAAlign(const IRPosition &IRP) : IRAttribute(IRP) {} /// Return assumed alignment. unsigned getAssumedAlign() const { return getAssumed(); } - /// Return known alignemnt. + /// Return known alignment. unsigned getKnownAlign() const { return getKnown(); } /// Create an abstract attribute view for the position \p IRP. @@ -1798,8 +2304,9 @@ struct AAAlign /// An abstract interface for all nocapture attributes. struct AANoCapture - : public IRAttribute<Attribute::NoCapture, - StateWrapper<IntegerState, AbstractAttribute>> { + : public IRAttribute< + Attribute::NoCapture, + StateWrapper<BitIntegerState<uint16_t, 7, 0>, AbstractAttribute>> { AANoCapture(const IRPosition &IRP) : IRAttribute(IRP) {} /// State encoding bits. A set bit in the state means the property holds. @@ -1852,11 +2359,7 @@ struct AAValueSimplify : public StateWrapper<BooleanState, AbstractAttribute>, AAValueSimplify(const IRPosition &IRP) : IRPosition(IRP) {} /// Return an IR position, see struct IRPosition. - /// - ///{ - IRPosition &getIRPosition() { return *this; } const IRPosition &getIRPosition() const { return *this; } - ///} /// Return an assumed simplified value if a single candidate is found. If /// there cannot be one, return original value. If it is not clear yet, return @@ -1882,11 +2385,7 @@ struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute>, bool isKnownHeapToStack() const { return getKnown(); } /// Return an IR position, see struct IRPosition. - /// - ///{ - IRPosition &getIRPosition() { return *this; } const IRPosition &getIRPosition() const { return *this; } - ///} /// Create an abstract attribute view for the position \p IRP. static AAHeapToStack &createForPosition(const IRPosition &IRP, Attributor &A); @@ -1897,8 +2396,9 @@ struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute>, /// An abstract interface for all memory related attributes. struct AAMemoryBehavior - : public IRAttribute<Attribute::ReadNone, - StateWrapper<IntegerState, AbstractAttribute>> { + : public IRAttribute< + Attribute::ReadNone, + StateWrapper<BitIntegerState<uint8_t, 3>, AbstractAttribute>> { AAMemoryBehavior(const IRPosition &IRP) : IRAttribute(IRP) {} /// State encoding bits. A set bit in the state means the property holds. @@ -1943,6 +2443,55 @@ struct AAMemoryBehavior static const char ID; }; +/// An abstract interface for range value analysis. +struct AAValueConstantRange : public IntegerRangeState, + public AbstractAttribute, + public IRPosition { + AAValueConstantRange(const IRPosition &IRP) + : IntegerRangeState( + IRP.getAssociatedValue().getType()->getIntegerBitWidth()), + IRPosition(IRP) {} + + /// Return an IR position, see struct IRPosition. + const IRPosition &getIRPosition() const override { return *this; } + + /// See AbstractAttribute::getState(...). + IntegerRangeState &getState() override { return *this; } + const AbstractState &getState() const override { return *this; } + + /// Create an abstract attribute view for the position \p IRP. + static AAValueConstantRange &createForPosition(const IRPosition &IRP, + Attributor &A); + + /// Return an assumed range for the assocaited value a program point \p CtxI. + /// If \p I is nullptr, simply return an assumed range. + virtual ConstantRange + getAssumedConstantRange(Attributor &A, + const Instruction *CtxI = nullptr) const = 0; + + /// Return a known range for the assocaited value at a program point \p CtxI. + /// If \p I is nullptr, simply return a known range. + virtual ConstantRange + getKnownConstantRange(Attributor &A, + const Instruction *CtxI = nullptr) const = 0; + + /// Return an assumed constant for the assocaited value a program point \p + /// CtxI. + Optional<ConstantInt *> + getAssumedConstantInt(Attributor &A, const Instruction *CtxI = nullptr) const { + ConstantRange RangeV = getAssumedConstantRange(A, CtxI); + if (auto *C = RangeV.getSingleElement()) + return cast<ConstantInt>( + ConstantInt::get(getAssociatedValue().getType(), *C)); + if (RangeV.isEmptySet()) + return llvm::None; + return nullptr; + } + + /// Unique ID (due to the unique address) + static const char ID; +}; + } // end namespace llvm #endif // LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H diff --git a/llvm/include/llvm/Transforms/IPO/FunctionImport.h b/llvm/include/llvm/Transforms/IPO/FunctionImport.h index bbf270c400af..b4dde7b199ff 100644 --- a/llvm/include/llvm/Transforms/IPO/FunctionImport.h +++ b/llvm/include/llvm/Transforms/IPO/FunctionImport.h @@ -98,7 +98,7 @@ public: using ImportMapTy = StringMap<FunctionsToImportTy>; /// The set contains an entry for every global value the module exports. - using ExportSetTy = std::unordered_set<GlobalValue::GUID>; + using ExportSetTy = DenseSet<ValueInfo>; /// A function of this type is used to load modules referenced by the index. using ModuleLoaderTy = diff --git a/llvm/include/llvm/Transforms/IPO/MergeFunctions.h b/llvm/include/llvm/Transforms/IPO/MergeFunctions.h new file mode 100644 index 000000000000..822f0fd99188 --- /dev/null +++ b/llvm/include/llvm/Transforms/IPO/MergeFunctions.h @@ -0,0 +1,32 @@ +//===- MergeFunctions.h - Merge Identical Functions -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass transforms simple global variables that never have their address +// taken. If obviously true, it marks read/write globals as constant, deletes +// variables only stored to, etc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_MERGEFUNCTIONS_H +#define LLVM_TRANSFORMS_IPO_MERGEFUNCTIONS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Module; + +/// Merge identical functions. +class MergeFunctionsPass : public PassInfoMixin<MergeFunctionsPass> { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_IPO_MERGEFUNCTIONS_H diff --git a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h index 22435e4ed1e5..8af2af7f352f 100644 --- a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h +++ b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h @@ -251,7 +251,7 @@ void runWholeProgramDevirtOnIndex( /// devirt target names for any locals that were exported. void updateIndexWPDForExports( ModuleSummaryIndex &Summary, - function_ref<bool(StringRef, GlobalValue::GUID)> isExported, + function_ref<bool(StringRef, ValueInfo)> isExported, std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap); } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h index 8894d96e591f..d7a6662510d3 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h @@ -24,13 +24,14 @@ namespace llvm { class InstCombinePass : public PassInfoMixin<InstCombinePass> { InstCombineWorklist Worklist; - bool ExpensiveCombines; + const bool ExpensiveCombines; + const unsigned MaxIterations; public: static StringRef name() { return "InstCombinePass"; } - explicit InstCombinePass(bool ExpensiveCombines = true) - : ExpensiveCombines(ExpensiveCombines) {} + explicit InstCombinePass(bool ExpensiveCombines = true); + explicit InstCombinePass(bool ExpensiveCombines, unsigned MaxIterations); PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; @@ -42,14 +43,14 @@ public: class InstructionCombiningPass : public FunctionPass { InstCombineWorklist Worklist; const bool ExpensiveCombines; + const unsigned MaxIterations; public: static char ID; // Pass identification, replacement for typeid - InstructionCombiningPass(bool ExpensiveCombines = true) - : FunctionPass(ID), ExpensiveCombines(ExpensiveCombines) { - initializeInstructionCombiningPassPass(*PassRegistry::getPassRegistry()); - } + explicit InstructionCombiningPass(bool ExpensiveCombines = true); + explicit InstructionCombiningPass(bool ExpensiveCombines, + unsigned MaxIterations); void getAnalysisUsage(AnalysisUsage &AU) const override; bool runOnFunction(Function &F) override; @@ -68,6 +69,8 @@ public: // %Z = add int 2, %X // FunctionPass *createInstructionCombiningPass(bool ExpensiveCombines = true); +FunctionPass *createInstructionCombiningPass(bool ExpensiveCombines, + unsigned MaxIterations); } #endif diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombineWorklist.h b/llvm/include/llvm/Transforms/InstCombine/InstCombineWorklist.h index 6c33bdbafbd2..3ed0a820db10 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombineWorklist.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombineWorklist.h @@ -24,8 +24,8 @@ namespace llvm { /// InstCombineWorklist - This is the worklist management logic for /// InstCombine. class InstCombineWorklist { - SmallVector<Instruction*, 256> Worklist; - DenseMap<Instruction*, unsigned> WorklistMap; + SmallVector<Instruction *, 256> Worklist; + DenseMap<Instruction *, unsigned> WorklistMap; public: InstCombineWorklist() = default; @@ -38,6 +38,9 @@ public: /// Add - Add the specified instruction to the worklist if it isn't already /// in it. void Add(Instruction *I) { + assert(I); + assert(I->getParent() && "Instruction not inserted yet?"); + if (WorklistMap.insert(std::make_pair(I, Worklist.size())).second) { LLVM_DEBUG(dbgs() << "IC: ADD: " << *I << '\n'); Worklist.push_back(I); diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h index f06230b6f366..1f2842836303 100644 --- a/llvm/include/llvm/Transforms/Scalar.h +++ b/llvm/include/llvm/Transforms/Scalar.h @@ -18,7 +18,6 @@ namespace llvm { -class BasicBlockPass; class Function; class FunctionPass; class ModulePass; @@ -50,13 +49,19 @@ FunctionPass *createSCCPPass(); //===----------------------------------------------------------------------===// // // DeadInstElimination - This pass quickly removes trivially dead instructions -// without modifying the CFG of the function. It is a BasicBlockPass, so it -// runs efficiently when queued next to other BasicBlockPass's. +// without modifying the CFG of the function. It is a FunctionPass. // Pass *createDeadInstEliminationPass(); //===----------------------------------------------------------------------===// // +// RedundantDbgInstElimination - This pass removes redundant dbg intrinsics +// without modifying the CFG of the function. It is a FunctionPass. +// +Pass *createRedundantDbgInstEliminationPass(); + +//===----------------------------------------------------------------------===// +// // DeadCodeElimination - This pass is more powerful than DeadInstElimination, // because it is worklist driven that can potentially revisit instructions when // their other instructions become dead, to eliminate chains of dead @@ -363,6 +368,12 @@ Pass *createLowerGuardIntrinsicPass(); //===----------------------------------------------------------------------===// // +// LowerMatrixIntrinsics - Lower matrix intrinsics to vector operations. +// +Pass *createLowerMatrixIntrinsicsPass(); + +//===----------------------------------------------------------------------===// +// // LowerWidenableCondition - Lower widenable condition to i1 true. // Pass *createLowerWidenableConditionPass(); diff --git a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h index 39039b093241..26d4a2476a86 100644 --- a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h +++ b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h @@ -14,7 +14,7 @@ // cost. If the constant can be folded into the instruction (the cost is // TCC_Free) or the cost is just a simple operation (TCC_BASIC), then we don't // consider it expensive and leave it alone. This is the default behavior and -// the default implementation of getIntImmCost will always return TCC_Free. +// the default implementation of getIntImmCostInst will always return TCC_Free. // // If the cost is more than TCC_BASIC, then the integer constant can't be folded // into the instruction and it might be beneficial to hoist the constant. diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h index 8a64768af6b5..5a3d30de16a3 100644 --- a/llvm/include/llvm/Transforms/Scalar/GVN.h +++ b/llvm/include/llvm/Transforms/Scalar/GVN.h @@ -94,7 +94,7 @@ public: // value number to the index of Expression in Expressions. We use it // instead of a DenseMap because filling such mapping is faster than // filling a DenseMap and the compile time is a little better. - uint32_t nextExprNumber; + uint32_t nextExprNumber = 0; std::vector<Expression> Expressions; std::vector<uint32_t> ExprIdx; @@ -107,9 +107,9 @@ public: DenseMap<std::pair<uint32_t, const BasicBlock *>, uint32_t>; PhiTranslateMap PhiTranslateTable; - AliasAnalysis *AA; - MemoryDependenceResults *MD; - DominatorTree *DT; + AliasAnalysis *AA = nullptr; + MemoryDependenceResults *MD = nullptr; + DominatorTree *DT = nullptr; uint32_t nextValueNumber = 1; @@ -130,6 +130,7 @@ public: ValueTable(const ValueTable &Arg); ValueTable(ValueTable &&Arg); ~ValueTable(); + ValueTable &operator=(const ValueTable &Arg); uint32_t lookupOrAdd(Value *V); uint32_t lookup(Value *V, bool Verify = true) const; @@ -154,14 +155,14 @@ private: friend class gvn::GVNLegacyPass; friend struct DenseMapInfo<Expression>; - MemoryDependenceResults *MD; - DominatorTree *DT; - const TargetLibraryInfo *TLI; - AssumptionCache *AC; + MemoryDependenceResults *MD = nullptr; + DominatorTree *DT = nullptr; + const TargetLibraryInfo *TLI = nullptr; + AssumptionCache *AC = nullptr; SetVector<BasicBlock *> DeadBlocks; - OptimizationRemarkEmitter *ORE; - ImplicitControlFlowTracking *ICF; - LoopInfo *LI; + OptimizationRemarkEmitter *ORE = nullptr; + ImplicitControlFlowTracking *ICF = nullptr; + LoopInfo *LI = nullptr; ValueTable VN; @@ -293,8 +294,8 @@ private: }; /// Create a legacy GVN pass. This also allows parameterizing whether or not -/// loads are eliminated by the pass. -FunctionPass *createGVNPass(bool NoLoads = false); +/// MemDep is enabled. +FunctionPass *createGVNPass(bool NoMemDepAnalysis = false); /// A simple and fast domtree-based GVN pass to hoist common expressions /// from sibling branches. diff --git a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h index 0464d40c45e6..a8beb8ca6e05 100644 --- a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h +++ b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h @@ -109,7 +109,17 @@ public: void FindLoopHeaders(Function &F); bool ProcessBlock(BasicBlock *BB); - bool ThreadEdge(BasicBlock *BB, const SmallVectorImpl<BasicBlock *> &PredBBs, + bool MaybeMergeBasicBlockIntoOnlyPred(BasicBlock *BB); + void UpdateSSA(BasicBlock *BB, BasicBlock *NewBB, + DenseMap<Instruction *, Value *> &ValueMapping); + DenseMap<Instruction *, Value *> CloneInstructions(BasicBlock::iterator BI, + BasicBlock::iterator BE, + BasicBlock *NewBB, + BasicBlock *PredBB); + bool TryThreadEdge(BasicBlock *BB, + const SmallVectorImpl<BasicBlock *> &PredBBs, + BasicBlock *SuccBB); + void ThreadEdge(BasicBlock *BB, const SmallVectorImpl<BasicBlock *> &PredBBs, BasicBlock *SuccBB); bool DuplicateCondBranchOnPHIIntoPred( BasicBlock *BB, const SmallVectorImpl<BasicBlock *> &PredBBs); diff --git a/llvm/include/llvm/Transforms/Scalar/LICM.h b/llvm/include/llvm/Transforms/Scalar/LICM.h index f0ea928abd49..a8f1c5348617 100644 --- a/llvm/include/llvm/Transforms/Scalar/LICM.h +++ b/llvm/include/llvm/Transforms/Scalar/LICM.h @@ -34,6 +34,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { diff --git a/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h b/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h index 7920269b0fb2..233963528595 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h @@ -15,9 +15,7 @@ namespace llvm { -class Loop; -struct LoopStandardAnalysisResults; -class LPMUpdater; +class Function; /// A simple loop rotation transformation. class LoopUnrollAndJamPass : public PassInfoMixin<LoopUnrollAndJamPass> { @@ -25,8 +23,7 @@ class LoopUnrollAndJamPass : public PassInfoMixin<LoopUnrollAndJamPass> { public: explicit LoopUnrollAndJamPass(int OptLevel = 2) : OptLevel(OptLevel) {} - PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, - LoopStandardAnalysisResults &AR, LPMUpdater &U); + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h b/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h index afeb1f1da029..7b049bdc8ad1 100644 --- a/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h +++ b/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h @@ -12,6 +12,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/CommandLine.h" namespace llvm { diff --git a/llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h b/llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h new file mode 100644 index 000000000000..2f75cd5017aa --- /dev/null +++ b/llvm/include/llvm/Transforms/Scalar/LowerMatrixIntrinsics.h @@ -0,0 +1,24 @@ +//===- LowerMatrixIntrinsics.h - Lower matrix intrinsics. -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass lowers matrix intrinsics down to vector operations. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_LOWERMATRIXINTRINSICSPASS_H +#define LLVM_TRANSFORMS_SCALAR_LOWERMATRIXINTRINSICSPASS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { +struct LowerMatrixIntrinsicsPass : PassInfoMixin<LowerMatrixIntrinsicsPass> { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; +} // namespace llvm + +#endif diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h index 6e03453babf1..bb31646ce462 100644 --- a/llvm/include/llvm/Transforms/Utils.h +++ b/llvm/include/llvm/Transforms/Utils.h @@ -119,6 +119,13 @@ ModulePass *createStripNonLineTableDebugInfoPass(); // number of conditional branches in the hot paths based on profiles. // FunctionPass *createControlHeightReductionLegacyPass(); + +//===----------------------------------------------------------------------===// +// +// InjectTLIMappingsLegacy - populates the VFABI attribute with the +// scalar-to-vector mappings from the TargetLibraryInfo. +// +FunctionPass *createInjectTLIMappingsLegacyPass(); } #endif diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index 698e57fd0394..dec8447c9f52 100644 --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -94,6 +94,10 @@ bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU = nullptr, MemoryDependenceResults *MemDep = nullptr, bool PredecessorWithTwoSuccessors = false); +/// Try to remove redundant dbg.value instructions from given basic block. +/// Returns true if at least one instruction was removed. +bool RemoveRedundantDbgInstrs(BasicBlock *BB); + /// Replace all uses of an instruction (specified by BI) with a value, then /// remove and delete the original instruction. void ReplaceInstWithValue(BasicBlock::InstListType &BIL, diff --git a/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h b/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h new file mode 100644 index 000000000000..32eb7cc2ab04 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/CodeMoverUtils.h @@ -0,0 +1,56 @@ +//===- Transform/Utils/CodeMoverUtils.h - CodeMover Utils -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This family of functions determine movements are safe on basic blocks, and +// instructions contained within a function. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_CODEMOVERUTILS_H +#define LLVM_TRANSFORMS_UTILS_CODEMOVERUTILS_H + +namespace llvm { + +class BasicBlock; +class DependenceInfo; +class DominatorTree; +class Instruction; +class PostDominatorTree; + +/// Return true if \p I0 and \p I1 are control flow equivalent. +/// Two instructions are control flow equivalent if when one executes, +/// the other is guaranteed to execute. This is determined using dominators +/// and post-dominators: if A dominates B and B post-dominates A then A and B +/// are control-flow equivalent. +bool isControlFlowEquivalent(const Instruction &I0, const Instruction &I1, + const DominatorTree &DT, + const PostDominatorTree &PDT); + +/// Return true if \p BB0 and \p BB1 are control flow equivalent. +/// Two basic blocks are control flow equivalent if when one executes, the other +/// is guaranteed to execute. This is determined using dominators and +/// post-dominators: if A dominates B and B post-dominates A then A and B are +/// control-flow equivalent. +bool isControlFlowEquivalent(const BasicBlock &BB0, const BasicBlock &BB1, + const DominatorTree &DT, + const PostDominatorTree &PDT); + +/// Return true if \p I can be safely moved before \p InsertPoint. +bool isSafeToMoveBefore(Instruction &I, Instruction &InsertPoint, + const DominatorTree &DT, const PostDominatorTree &PDT, + DependenceInfo &DI); + +/// Move instructions from \p FromBB bottom up to the beginning of \p ToBB +/// when proven safe. +void moveInstsBottomUp(BasicBlock &FromBB, BasicBlock &ToBB, + const DominatorTree &DT, const PostDominatorTree &PDT, + DependenceInfo &DI); + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_CODEMOVERUTILS_H diff --git a/llvm/include/llvm/Transforms/Utils/Debugify.h b/llvm/include/llvm/Transforms/Utils/Debugify.h new file mode 100644 index 000000000000..0b5ec738750d --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/Debugify.h @@ -0,0 +1,70 @@ +//===- Debugify.h - Attach synthetic debug info to everything -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file Interface to the `debugify` synthetic debug info testing utility. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORM_UTILS_DEBUGIFY_H +#define LLVM_TRANSFORM_UTILS_DEBUGIFY_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/IR/PassManager.h" + +llvm::ModulePass *createDebugifyModulePass(); +llvm::FunctionPass *createDebugifyFunctionPass(); + +struct NewPMDebugifyPass : public llvm::PassInfoMixin<NewPMDebugifyPass> { + llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &AM); +}; + +/// Track how much `debugify` information has been lost. +struct DebugifyStatistics { + /// Number of missing dbg.values. + unsigned NumDbgValuesMissing = 0; + + /// Number of dbg.values expected. + unsigned NumDbgValuesExpected = 0; + + /// Number of instructions with empty debug locations. + unsigned NumDbgLocsMissing = 0; + + /// Number of instructions expected to have debug locations. + unsigned NumDbgLocsExpected = 0; + + /// Get the ratio of missing/expected dbg.values. + float getMissingValueRatio() const { + return float(NumDbgValuesMissing) / float(NumDbgLocsExpected); + } + + /// Get the ratio of missing/expected instructions with locations. + float getEmptyLocationRatio() const { + return float(NumDbgLocsMissing) / float(NumDbgLocsExpected); + } +}; + +/// Map pass names to a per-pass DebugifyStatistics instance. +using DebugifyStatsMap = llvm::MapVector<llvm::StringRef, DebugifyStatistics>; + +llvm::ModulePass * +createCheckDebugifyModulePass(bool Strip = false, + llvm::StringRef NameOfWrappedPass = "", + DebugifyStatsMap *StatsMap = nullptr); + +llvm::FunctionPass * +createCheckDebugifyFunctionPass(bool Strip = false, + llvm::StringRef NameOfWrappedPass = "", + DebugifyStatsMap *StatsMap = nullptr); + +struct NewPMCheckDebugifyPass + : public llvm::PassInfoMixin<NewPMCheckDebugifyPass> { + llvm::PreservedAnalyses run(llvm::Module &M, llvm::ModuleAnalysisManager &AM); +}; + +#endif // LLVM_TRANSFORM_UTILS_DEBUGIFY_H diff --git a/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h b/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h index 9c2a9ea531ea..2c6c3adc8dad 100644 --- a/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h +++ b/llvm/include/llvm/Transforms/Utils/FunctionImportUtils.h @@ -49,7 +49,7 @@ class FunctionImportGlobalProcessing { DenseMap<const Comdat *, Comdat *> RenamedComdats; /// Check if we should promote the given local value to global scope. - bool shouldPromoteLocalToGlobal(const GlobalValue *SGV); + bool shouldPromoteLocalToGlobal(const GlobalValue *SGV, ValueInfo VI); #ifndef NDEBUG /// Check if the given value is a local that can't be renamed (promoted). @@ -67,11 +67,9 @@ class FunctionImportGlobalProcessing { /// import SGV as a definition, otherwise import as a declaration. bool doImportAsDefinition(const GlobalValue *SGV); - /// Get the name for SGV that should be used in the linked destination - /// module. Specifically, this handles the case where we need to rename - /// a local that is being promoted to global scope, which it will always - /// do when \p DoPromote is true (or when importing a local). - std::string getName(const GlobalValue *SGV, bool DoPromote); + /// Get the name for a local SGV that should be promoted and renamed to global + /// scope in the linked destination module. + std::string getPromotedName(const GlobalValue *SGV); /// Process globals so that they can be used in ThinLTO. This includes /// promoting local variables so that they can be reference externally by @@ -107,9 +105,6 @@ public: } bool run(); - - static bool doImportAsDefinition(const GlobalValue *SGV, - SetVector<GlobalValue *> *GlobalsToImport); }; /// Perform in-place global value handling on the given Module for diff --git a/llvm/include/llvm/Transforms/Utils/GuardUtils.h b/llvm/include/llvm/Transforms/Utils/GuardUtils.h index 3b365c56a5c0..7ab5d9ef4f23 100644 --- a/llvm/include/llvm/Transforms/Utils/GuardUtils.h +++ b/llvm/include/llvm/Transforms/Utils/GuardUtils.h @@ -14,15 +14,30 @@ namespace llvm { +class BranchInst; class CallInst; class Function; +class Value; /// Splits control flow at point of \p Guard, replacing it with explicit branch /// by the condition of guard's first argument. The taken branch then goes to /// the block that contains \p Guard's successors, and the non-taken branch /// goes to a newly-created deopt block that contains a sole call of the -/// deoptimize function \p DeoptIntrinsic. -void makeGuardControlFlowExplicit(Function *DeoptIntrinsic, CallInst *Guard); +/// deoptimize function \p DeoptIntrinsic. If 'UseWC' is set, preserve the +/// widenable nature of the guard by lowering to equivelent form. If not set, +/// lower to a form without widenable semantics. +void makeGuardControlFlowExplicit(Function *DeoptIntrinsic, CallInst *Guard, + bool UseWC); + +/// Given a branch we know is widenable (defined per Analysis/GuardUtils.h), +/// widen it such that condition 'NewCond' is also known to hold on the taken +/// path. Branch remains widenable after transform. +void widenWidenableBranch(BranchInst *WidenableBR, Value *NewCond); + +/// Given a branch we know is widenable (defined per Analysis/GuardUtils.h), +/// *set* it's condition such that (only) 'Cond' is known to hold on the taken +/// path and that the branch remains widenable after transform. +void setWidenableBranchCond(BranchInst *WidenableBR, Value *Cond); } // llvm diff --git a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h new file mode 100644 index 000000000000..84e4fee51c26 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h @@ -0,0 +1,37 @@ +//===- InjectTLIMAppings.h - TLI to VFABI attribute injection ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Populates the VFABI attribute with the scalar-to-vector mappings +// from the TargetLibraryInfo. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H +#define LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H + +#include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" + +namespace llvm { +class InjectTLIMappings : public PassInfoMixin<InjectTLIMappings> { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +// Legacy pass +class InjectTLIMappingsLegacy : public FunctionPass { +public: + static char ID; + InjectTLIMappingsLegacy() : FunctionPass(ID) { + initializeInjectTLIMappingsLegacyPass(*PassRegistry::getPassRegistry()); + } + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnFunction(Function &F) override; +}; + +} // End namespace llvm +#endif // LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index 9fcb2f64d79b..d1dc0b3e46b9 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -354,6 +354,10 @@ AllocaInst *findAllocaForValue(Value *V, /// Returns true if any debug users were updated. bool salvageDebugInfo(Instruction &I); +/// Salvage all debug users of the instruction \p I or mark it as undef if it +/// cannot be salvaged. +void salvageDebugInfoOrMarkUndef(Instruction &I); + /// Implementation of salvageDebugInfo, applying only to instructions in /// \p Insns, rather than all debug users of \p I. bool salvageDebugInfoForDbgValues(Instruction &I, diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index d32f08717e9b..9ed96809ed99 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -73,7 +73,8 @@ bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, /// /// Returns true if any modifications are made. bool formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist, - DominatorTree &DT, LoopInfo &LI); + DominatorTree &DT, LoopInfo &LI, + ScalarEvolution *SE); /// Put loop into LCSSA form. /// @@ -132,7 +133,7 @@ bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, /// ORE. It returns changed status. bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, TargetLibraryInfo *, Loop *, AliasSetTracker *, - MemorySSAUpdater *, ICFLoopSafetyInfo *, + MemorySSAUpdater *, ScalarEvolution *, ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *); /// This function deletes dead loops. The caller of this function needs to diff --git a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h index c69af5588741..c2da86406e71 100644 --- a/llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -13,6 +13,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_MODULEUTILS_H #define LLVM_TRANSFORMS_UTILS_MODULEUTILS_H +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringRef.h" #include <utility> // for std::pair @@ -108,6 +109,13 @@ void filterDeadComdatFunctions( /// unique identifier for this module, so we return the empty string. std::string getUniqueModuleId(Module *M); +class CallInst; +namespace VFABI { +/// Overwrite the Vector Function ABI variants attribute with the names provide +/// in \p VariantMappings. +void setVectorVariantNames(CallInst *CI, + const SmallVector<std::string, 8> &VariantMappings); +} // End VFABI namespace } // End llvm namespace #endif // LLVM_TRANSFORMS_UTILS_MODULEUTILS_H diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h index 88c2ef787ad8..610668adcfa5 100644 --- a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h +++ b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h @@ -175,6 +175,7 @@ private: Value *optimizeMemCmp(CallInst *CI, IRBuilder<> &B); Value *optimizeBCmp(CallInst *CI, IRBuilder<> &B); Value *optimizeMemCmpBCmpCommon(CallInst *CI, IRBuilder<> &B); + Value *optimizeMemCCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeMemPCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeMemCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeMemMove(CallInst *CI, IRBuilder<> &B); diff --git a/llvm/include/llvm/Transforms/Utils/SizeOpts.h b/llvm/include/llvm/Transforms/Utils/SizeOpts.h index 1a052c694e6d..ba0f86c45263 100644 --- a/llvm/include/llvm/Transforms/Utils/SizeOpts.h +++ b/llvm/include/llvm/Transforms/Utils/SizeOpts.h @@ -13,6 +13,20 @@ #ifndef LLVM_TRANSFORMS_UTILS_SIZEOPTS_H #define LLVM_TRANSFORMS_UTILS_SIZEOPTS_H +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/Support/CommandLine.h" + +using namespace llvm; + +extern cl::opt<bool> EnablePGSO; +extern cl::opt<bool> PGSOLargeWorkingSetSizeOnly; +extern cl::opt<bool> PGSOIRPassOrTestOnly; +extern cl::opt<bool> PGSOColdCodeOnly; +extern cl::opt<bool> ForcePGSO; +extern cl::opt<int> PgsoCutoffInstrProf; +extern cl::opt<int> PgsoCutoffSampleProf; + namespace llvm { class BasicBlock; @@ -20,14 +34,73 @@ class BlockFrequencyInfo; class Function; class ProfileSummaryInfo; -/// Returns true if function \p F is suggested to be size-optimized base on the +enum class PGSOQueryType { + IRPass, // A query call from an IR-level transform pass. + Test, // A query call from a unit test. + Other, // Others. +}; + +template<typename AdapterT, typename FuncT, typename BFIT> +bool shouldFuncOptimizeForSizeImpl(const FuncT *F, ProfileSummaryInfo *PSI, + BFIT *BFI, PGSOQueryType QueryType) { + assert(F); + if (!PSI || !BFI || !PSI->hasProfileSummary()) + return false; + if (ForcePGSO) + return true; + if (!EnablePGSO) + return false; + // Temporarily enable size optimizations only for the IR pass or test query + // sites for gradual commit/rollout. This is to be removed later. + if (PGSOIRPassOrTestOnly && !(QueryType == PGSOQueryType::IRPass || + QueryType == PGSOQueryType::Test)) + return false; + if (PGSOColdCodeOnly || + (PGSOLargeWorkingSetSizeOnly && !PSI->hasLargeWorkingSetSize())) { + // Even if the working set size isn't large, size-optimize cold code. + return AdapterT::isFunctionColdInCallGraph(F, PSI, *BFI); + } + return !AdapterT::isFunctionHotInCallGraphNthPercentile( + PSI->hasSampleProfile() ? PgsoCutoffSampleProf : PgsoCutoffInstrProf, + F, PSI, *BFI); +} + +template<typename AdapterT, typename BlockT, typename BFIT> +bool shouldOptimizeForSizeImpl(const BlockT *BB, ProfileSummaryInfo *PSI, + BFIT *BFI, PGSOQueryType QueryType) { + assert(BB); + if (!PSI || !BFI || !PSI->hasProfileSummary()) + return false; + if (ForcePGSO) + return true; + if (!EnablePGSO) + return false; + // Temporarily enable size optimizations only for the IR pass or test query + // sites for gradual commit/rollout. This is to be removed later. + if (PGSOIRPassOrTestOnly && !(QueryType == PGSOQueryType::IRPass || + QueryType == PGSOQueryType::Test)) + return false; + if (PGSOColdCodeOnly || + (PGSOLargeWorkingSetSizeOnly && !PSI->hasLargeWorkingSetSize())) { + // Even if the working set size isn't large, size-optimize cold code. + return AdapterT::isColdBlock(BB, PSI, BFI); + } + return !AdapterT::isHotBlockNthPercentile( + PSI->hasSampleProfile() ? PgsoCutoffSampleProf : PgsoCutoffInstrProf, + BB, PSI, BFI); +} + +/// Returns true if function \p F is suggested to be size-optimized based on the /// profile. -bool shouldOptimizeForSize(Function *F, ProfileSummaryInfo *PSI, - BlockFrequencyInfo *BFI); -/// Returns true if basic block \p BB is suggested to be size-optimized base -/// on the profile. -bool shouldOptimizeForSize(BasicBlock *BB, ProfileSummaryInfo *PSI, - BlockFrequencyInfo *BFI); +bool shouldOptimizeForSize(const Function *F, ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI, + PGSOQueryType QueryType = PGSOQueryType::Other); + +/// Returns true if basic block \p BB is suggested to be size-optimized based on +/// the profile. +bool shouldOptimizeForSize(const BasicBlock *BB, ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI, + PGSOQueryType QueryType = PGSOQueryType::Other); } // end namespace llvm diff --git a/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h b/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h index f68534ecd2eb..b8a4fe72ea25 100644 --- a/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h +++ b/llvm/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h @@ -29,9 +29,7 @@ struct UnifyFunctionExitNodes : public FunctionPass { public: static char ID; // Pass identification, replacement for typeid - UnifyFunctionExitNodes() : FunctionPass(ID) { - initializeUnifyFunctionExitNodesPass(*PassRegistry::getPassRegistry()); - } + UnifyFunctionExitNodes(); // We can preserve non-critical-edgeness when we unify function exit nodes void getAnalysisUsage(AnalysisUsage &AU) const override; diff --git a/llvm/include/llvm/Transforms/Vectorize.h b/llvm/include/llvm/Transforms/Vectorize.h index 88a0e49d0fae..bca78d073003 100644 --- a/llvm/include/llvm/Transforms/Vectorize.h +++ b/llvm/include/llvm/Transforms/Vectorize.h @@ -16,7 +16,6 @@ namespace llvm { class BasicBlock; -class BasicBlockPass; class Pass; //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h index d824e2903ef3..ac6efc7c695f 100644 --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorize.h @@ -58,6 +58,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/CommandLine.h" #include <functional> namespace llvm { diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h index 32ccc8a46380..237781dfe22e 100644 --- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -24,6 +24,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/CommandLine.h" namespace llvm { @@ -138,7 +139,7 @@ private: bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R); bool vectorizeStoreChain(ArrayRef<Value *> Chain, slpvectorizer::BoUpSLP &R, - unsigned VecRegSize); + unsigned Idx); bool vectorizeStores(ArrayRef<StoreInst *> Stores, slpvectorizer::BoUpSLP &R); diff --git a/llvm/include/llvm/XRay/FDRRecords.h b/llvm/include/llvm/XRay/FDRRecords.h index e3e16f71e2fe..d53e6aa812f2 100644 --- a/llvm/include/llvm/XRay/FDRRecords.h +++ b/llvm/include/llvm/XRay/FDRRecords.h @@ -289,7 +289,7 @@ public: }; class CallArgRecord : public MetadataRecord { - uint64_t Arg; + uint64_t Arg = 0; friend class RecordInitializer; public: @@ -371,8 +371,8 @@ public: class FunctionRecord : public Record { RecordTypes Kind; - int32_t FuncId; - uint32_t Delta; + int32_t FuncId = 0; + uint32_t Delta = 0; friend class RecordInitializer; static constexpr unsigned kFunctionRecordSize = 8; diff --git a/llvm/include/llvm/module.modulemap b/llvm/include/llvm/module.modulemap index ecb3b37004fd..05de40698e35 100644 --- a/llvm/include/llvm/module.modulemap +++ b/llvm/include/llvm/module.modulemap @@ -152,6 +152,13 @@ module LLVM_DebugInfo_CodeView { textual header "DebugInfo/CodeView/CodeViewSymbols.def" } +module LLVM_DWARFLinker { + requires cplusplus + + umbrella "DWARFLinker" + module * { export * } +} + module LLVM_ExecutionEngine { requires cplusplus @@ -169,12 +176,31 @@ module LLVM_ExecutionEngine { // FIXME: These exclude directives were added as a workaround for // <rdar://problem/29247092> and should be removed once it is fixed. - exclude header "ExecutionEngine/Orc/RawByteChannel.h" - exclude header "ExecutionEngine/Orc/RPCUtils.h" exclude header "ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h" exclude header "ExecutionEngine/Orc/OrcRemoteTargetClient.h" exclude header "ExecutionEngine/Orc/OrcRemoteTargetServer.h" exclude header "ExecutionEngine/Orc/RemoteObjectLayer.h" + + // Exclude headers from LLVM_OrcSupport. + exclude header "ExecutionEngine/Orc/OrcError.h" + exclude header "ExecutionEngine/Orc/RPC/RPCUtils.h" + exclude header "ExecutionEngine/Orc/RPC/RPCSerialization.h" + exclude header "ExecutionEngine/Orc/RPC/RawByteChannel.h" + +} + +// Orc utilities that don't depend only on Support (not ExecutionEngine or +// IR). This is a workaround for ExecutionEngine's broken layering, and will +// be removed in the future. +module LLVM_OrcSupport { + requires cplusplus + + header "ExecutionEngine/Orc/OrcError.h" + header "ExecutionEngine/Orc/RPC/RPCUtils.h" + header "ExecutionEngine/Orc/RPC/RPCSerialization.h" + header "ExecutionEngine/Orc/RPC/RawByteChannel.h" + + export * } module LLVM_Pass { @@ -250,6 +276,7 @@ module LLVM_IR { module * { export * } // These are intended for (repeated) textual inclusion. + textual header "IR/ConstrainedOps.def" textual header "IR/DebugInfoFlags.def" textual header "IR/Instruction.def" textual header "IR/Metadata.def" |